From phk at projects.linpro.no Sat Jul 1 05:44:04 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 1 Jul 2006 07:44:04 +0200 (CEST) Subject: r277 - trunk/varnish-cache/include Message-ID: <20060701054404.C0CD41EC386@projects.linpro.no> Author: phk Date: 2006-07-01 07:44:04 +0200 (Sat, 01 Jul 2006) New Revision: 277 Modified: trunk/varnish-cache/include/http_headers.h Log: Mark more headers as not end-to-end Modified: trunk/varnish-cache/include/http_headers.h =================================================================== --- trunk/varnish-cache/include/http_headers.h 2006-06-30 20:22:58 UTC (rev 276) +++ trunk/varnish-cache/include/http_headers.h 2006-07-01 05:44:04 UTC (rev 277) @@ -9,6 +9,8 @@ * f unused * g unused * + * see [RFC2616 13.5.1 End-to-end and Hop-by-hop Headers] + * * a b c d e f g *-------------------------------------------------------------------- */ @@ -47,16 +49,16 @@ HTTPH("Location", H_Location, 2, 0, 0, 0, 0) /* RFC2616 14.30 */ HTTPH("Max-Forwards", H_Max_Forwards, 1, 0, 0, 0, 0) /* RFC2616 14.31 */ HTTPH("Pragma", H_Pragma, 1, 0, 0, 0, 0) /* RFC2616 14.32 */ -HTTPH("Proxy-Authenticate", H_Proxy_Authenticate, 2, 0, 0, 0, 0) /* RFC2616 14.33 */ -HTTPH("Proxy-Authorization", H_Proxy_Authorization, 1, 0, 0, 0, 0) /* RFC2616 14.34 */ +HTTPH("Proxy-Authenticate", H_Proxy_Authenticate, 2, 3, 0, 0, 0) /* RFC2616 14.33 */ +HTTPH("Proxy-Authorization", H_Proxy_Authorization, 1, 3, 0, 0, 0) /* RFC2616 14.34 */ HTTPH("Range", H_Range, 1, 0, 0, 0, 0) /* RFC2616 14.35 */ HTTPH("Referer", H_Referer, 1, 0, 0, 0, 0) /* RFC2616 14.36 */ HTTPH("Retry-After", H_Retry_After, 2, 0, 0, 0, 0) /* RFC2616 14.37 */ HTTPH("Server", H_Server, 2, 0, 0, 0, 0) /* RFC2616 14.38 */ -HTTPH("TE", H_TE, 1, 0, 0, 0, 0) /* RFC2616 14.39 */ -HTTPH("Trailer", H_Trailer, 1, 0, 0, 0, 0) /* RFC2616 14.40 */ +HTTPH("TE", H_TE, 1, 3, 0, 0, 0) /* RFC2616 14.39 */ +HTTPH("Trailer", H_Trailer, 1, 3, 0, 0, 0) /* RFC2616 14.40 */ HTTPH("Transfer-Encoding", H_Transfer_Encoding, 2, 3, 0, 0, 0) /* RFC2616 14.41 */ -HTTPH("Upgrade", H_Upgrade, 2, 0, 0, 0, 0) /* RFC2616 14.42 */ +HTTPH("Upgrade", H_Upgrade, 2, 3, 0, 0, 0) /* RFC2616 14.42 */ HTTPH("User-Agent", H_User_Agent, 1, 0, 0, 0, 0) /* RFC2616 14.43 */ HTTPH("Vary", H_Vary, 2, 0, 0, 0, 0) /* RFC2616 14.44 */ HTTPH("Via", H_Via, 2, 0, 0, 0, 0) /* RFC2616 14.45 */ From des at projects.linpro.no Mon Jul 3 08:30:33 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Mon, 3 Jul 2006 10:30:33 +0200 (CEST) Subject: r278 - trunk/varnish-cache Message-ID: <20060703083033.6692B1EC362@projects.linpro.no> Author: des Date: 2006-07-03 10:30:33 +0200 (Mon, 03 Jul 2006) New Revision: 278 Modified: trunk/varnish-cache/configure.ac Log: Replace --enable-wall and --enable-pedantic with --enable-developer-warnings, which is roughly equivalent to FreeBSD's WARNS level 5 or 6. Modified: trunk/varnish-cache/configure.ac =================================================================== --- trunk/varnish-cache/configure.ac 2006-07-01 05:44:04 UTC (rev 277) +++ trunk/varnish-cache/configure.ac 2006-07-03 08:30:33 UTC (rev 278) @@ -15,12 +15,10 @@ # Compiler flags (assume GCC). # This section *must* come before AC_PROG_CC / AC_PROG_CPP. CFLAGS="${CFLAGS:--O2}" -AC_ARG_ENABLE(wall, - AS_HELP_STRING([--enable-wall],[use -Wall (default is NO)]), - CFLAGS="${CFLAGS} -Wall") -AC_ARG_ENABLE(pedantic, - AS_HELP_STRING([--enable-pedantic],[enable pedantic warnings (default is NO)]), - CFLAGS="${CFLAGS} -pedantic") +DEVELOPER_CFLAGS="-Wall -Wno-unused-parameter -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith -Wreturn-type -Wcast-qual -Wwrite-strings -Wswitch -Wshadow -Wcast-align -Wunused-parameter -Wchar-subscripts -Winline -Wnested-externs -Wredundant-decls" +AC_ARG_ENABLE(developer-warnings, + AS_HELP_STRING([--enable-developer-warnings],[enable strict warnings (default is NO)]), + CFLAGS="${CFLAGS} ${DEVELOPER_CFLAGS}") AC_ARG_ENABLE(werror, AS_HELP_STRING([--enable-werror],[use -Werror (default is NO)]), CFLAGS="${CFLAGS} -Werror") From phk at projects.linpro.no Mon Jul 3 12:41:14 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 3 Jul 2006 14:41:14 +0200 (CEST) Subject: r279 - trunk/varnish-cache/bin/varnishd Message-ID: <20060703124114.6B5EC1EC325@projects.linpro.no> Author: phk Date: 2006-07-03 14:41:14 +0200 (Mon, 03 Jul 2006) New Revision: 279 Added: trunk/varnish-cache/bin/varnishd/hash_classic.c trunk/varnish-cache/bin/varnishd/hash_slinger.h Modified: trunk/varnish-cache/bin/varnishd/Makefile.am trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/hash_simple_list.c trunk/varnish-cache/bin/varnishd/heritage.h trunk/varnish-cache/bin/varnishd/mgt.h trunk/varnish-cache/bin/varnishd/rfc2616.c trunk/varnish-cache/bin/varnishd/varnishd.c Log: Add another hash-method with better real-world survival chances: A classic bucketed hash table of lists. Hash is MD5. Number of buckets and number of mutexes can be configured at command line. Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-03 08:30:33 UTC (rev 278) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-03 12:41:14 UTC (rev 279) @@ -21,6 +21,7 @@ cache_vrt.c \ cli_event.c \ hash_simple_list.c \ + hash_classic.c \ mgt_child.c \ rfc2616.c \ storage_file.c \ Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-03 08:30:33 UTC (rev 278) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-03 12:41:14 UTC (rev 279) @@ -27,21 +27,8 @@ struct worker; #endif -/* Hashing -----------------------------------------------------------*/ +#include "hash_slinger.h" -typedef void hash_init_f(void); -typedef struct objhead *hash_lookup_f(const char *key, struct objhead *nobj); -typedef int hash_deref_f(struct objhead *obj); - -struct hash_slinger { - const char *name; - hash_init_f *init; - hash_lookup_f *lookup; - hash_deref_f *deref; -}; - -extern struct hash_slinger hsl_slinger; - /* Storage -----------------------------------------------------------*/ struct storage { Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-03 08:30:33 UTC (rev 278) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-03 12:41:14 UTC (rev 279) @@ -245,7 +245,7 @@ sp->obj->xid = sp->xid; fd = VBE_GetFd(sp->backend, &fd_token, sp->xid); - assert(fd != -1); + assert(fd != -1); /* XXX: handle this */ VSL(SLT_Backend, sp->fd, "%d %s", fd, sp->backend->vcl_name); hp = http_New(); Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-03 08:30:33 UTC (rev 278) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-03 12:41:14 UTC (rev 279) @@ -14,6 +14,7 @@ #include "libvarnish.h" #include "shmlog.h" +#include "heritage.h" #include "cache.h" static struct hash_slinger *hash; @@ -129,7 +130,7 @@ HSH_Init(void) { - hash = &hsl_slinger; - if (hash->init != NULL) - hash->init(); + hash = heritage.hash; + if (hash->start != NULL) + hash->start(); } Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-03 08:30:33 UTC (rev 278) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-03 12:41:14 UTC (rev 279) @@ -1,5 +1,7 @@ /* * $Id$ + * + * XXX: automatic thread-pool size adaptation. */ #include @@ -14,6 +16,7 @@ #include #include "libvarnish.h" +#include "heritage.h" #include "shmlog.h" #include "vcl.h" #include "cache.h" @@ -154,9 +157,10 @@ AZ(pthread_cond_init(&shdcnd, NULL)); - for (i = 0; i < 5; i++) + for (i = 0; i < heritage.wthread_min; i++) { AZ(pthread_create(&tp, NULL, CacheWorker, NULL)); - AZ(pthread_detach(tp)); + AZ(pthread_detach(tp)); + } srandomdev(); xids = random(); } Added: trunk/varnish-cache/bin/varnishd/hash_classic.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-03 08:30:33 UTC (rev 278) +++ trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-03 12:41:14 UTC (rev 279) @@ -0,0 +1,188 @@ +/* + * $Id$ + * + * A classic bucketed hash + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/*--------------------------------------------------------------------*/ + +struct hcl_entry { + TAILQ_ENTRY(hcl_entry) list; + char *key; + struct objhead *obj; + unsigned refcnt; + unsigned hash; + unsigned mtx; +}; + +TAILQ_HEAD(hcl_head, hcl_entry); + +static struct hcl_head *hcl_head; +static unsigned hcl_nhash = 256; +static unsigned hcl_nmtx = 16; +static pthread_mutex_t *hcl_mutex; + +/*-------------------------------------------------------------------- + * The ->init method allows the management process to pass arguments + */ + +static int +hcl_init(const char *p) +{ + int i; + unsigned u1, u2; + + i = sscanf(p, "%u,%u", &u1, &u2); + if (i == 0) + return (0); + if (u1 == 0 || (i == 2 && (u2 == 0 || u2 > u1))) { + fprintf(stderr, "Invallid parameters to hash \"classic\":\n"); + fprintf(stderr, + "\t-h classic,[,]\n"); + return (1); + } + hcl_nhash = u1; + if (i == 1) { + hcl_nmtx = hcl_nhash / 16; + if (hcl_nmtx < 1) + hcl_nmtx = 1; + return(0); + } else { + hcl_nmtx = hcl_nhash / u2; + if (hcl_nmtx < 1) + hcl_nmtx = 1; + } + fprintf(stderr, "Classic hash: %u buckets %u mutexes\n", + hcl_nhash, hcl_nmtx); + return (0); +} + +/*-------------------------------------------------------------------- + * The ->start method is called during cache process start and allows + * initialization to happen before the first lookup. + */ + +static void +hcl_start(void) +{ + unsigned u; + + hcl_head = calloc(sizeof *hcl_head, hcl_nhash); + assert(hcl_head != NULL); + hcl_mutex = calloc(sizeof *hcl_mutex, hcl_nmtx); + assert(hcl_mutex != NULL); + + + for (u = 0; u < hcl_nhash; u++) + TAILQ_INIT(&hcl_head[u]); + + for (u = 0; u < hcl_nmtx; u++) + AZ(pthread_mutex_init(&hcl_mutex[u], NULL)); +} + +/*-------------------------------------------------------------------- + * Lookup and possibly insert element. + * If nobj != NULL and the lookup does not find key, nobj is inserted. + * If nobj == NULL and the lookup does not find key, NULL is returned. + * A reference to the returned object is held. + */ + +static struct objhead * +hcl_lookup(const char *key, struct objhead *nobj) +{ + struct hcl_entry *he, *he2; + MD5_CTX c; + unsigned char md5[MD5_DIGEST_LENGTH]; + unsigned u1, u2; + int i; + + MD5Init(&c); + MD5Update(&c, key, strlen(key)); + MD5Final(md5, &c); + memcpy(&u1, md5, sizeof u1); + u1 %= hcl_nhash; + memcpy(&u2, md5 + sizeof u1, sizeof u2); + u2 %= hcl_nmtx; + + AZ(pthread_mutex_lock(&hcl_mutex[u2])); + TAILQ_FOREACH(he, &hcl_head[u1], list) { + i = strcmp(key, he->key); + if (i < 0) + continue; + if (i == 0) { + he->refcnt++; + nobj = he->obj; + nobj->hashpriv = he; + AZ(pthread_mutex_unlock(&hcl_mutex[u2])); + return (nobj); + } + if (nobj == NULL) { + AZ(pthread_mutex_unlock(&hcl_mutex[u2])); + return (NULL); + } + break; + } + he2 = calloc(sizeof *he2, 1); + assert(he2 != NULL); + he2->obj = nobj; + he2->refcnt = 1; + he2->hash = u1; + he2->mtx = u2; + he2->key = strdup(key); + assert(he2->key != NULL); + nobj->hashpriv = he2; + if (he != NULL) + TAILQ_INSERT_BEFORE(he, he2, list); + else + TAILQ_INSERT_TAIL(&hcl_head[u1], he2, list); + AZ(pthread_mutex_unlock(&hcl_mutex[u2])); + return (nobj); +} + +/*-------------------------------------------------------------------- + * Dereference and if no references are left, free. + */ + +static int +hcl_deref(struct objhead *obj) +{ + struct hcl_entry *he; + int ret; + unsigned mtx; + + assert(obj->hashpriv != NULL); + he = obj->hashpriv; + mtx = he->mtx; + AZ(pthread_mutex_lock(&hcl_mutex[mtx])); + if (--he->refcnt == 0) { + free(he->key); + TAILQ_REMOVE(&hcl_head[he->hash], he, list); + free(he); + ret = 0; + } else + ret = 1; + AZ(pthread_mutex_unlock(&hcl_mutex[mtx])); + return (ret); +} + +/*--------------------------------------------------------------------*/ + +struct hash_slinger hcl_slinger = { + "classic", + hcl_init, + hcl_start, + hcl_lookup, + hcl_deref, +}; Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-07-03 08:30:33 UTC (rev 278) +++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-07-03 12:41:14 UTC (rev 279) @@ -112,6 +112,7 @@ struct hash_slinger hsl_slinger = { "simple_list", + NULL, hsl_init, hsl_lookup, hsl_deref, Added: trunk/varnish-cache/bin/varnishd/hash_slinger.h =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2006-07-03 08:30:33 UTC (rev 278) +++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2006-07-03 12:41:14 UTC (rev 279) @@ -0,0 +1,16 @@ +/* + * $Id$ + */ + +typedef int hash_init_f(const char *); +typedef void hash_start_f(void); +typedef struct objhead *hash_lookup_f(const char *key, struct objhead *nobj); +typedef int hash_deref_f(struct objhead *obj); + +struct hash_slinger { + const char *name; + hash_init_f *init; + hash_start_f *start; + hash_lookup_f *lookup; + hash_deref_f *deref; +}; Modified: trunk/varnish-cache/bin/varnishd/heritage.h =================================================================== --- trunk/varnish-cache/bin/varnishd/heritage.h 2006-07-03 08:30:33 UTC (rev 278) +++ trunk/varnish-cache/bin/varnishd/heritage.h 2006-07-03 12:41:14 UTC (rev 279) @@ -31,7 +31,13 @@ /* Storage method */ struct stevedore *stevedore; + /* Hash method */ + struct hash_slinger *hash; + unsigned default_ttl; + + /* Worker threads */ + unsigned wthread_min, wthread_max; }; extern struct heritage heritage; Modified: trunk/varnish-cache/bin/varnishd/mgt.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt.h 2006-07-03 08:30:33 UTC (rev 278) +++ trunk/varnish-cache/bin/varnishd/mgt.h 2006-07-03 12:41:14 UTC (rev 279) @@ -19,5 +19,10 @@ extern struct stevedore sma_stevedore; extern struct stevedore smf_stevedore; +#include "hash_slinger.h" + +extern struct hash_slinger hsl_slinger; +extern struct hash_slinger hcl_slinger; + void VSL_MgtInit(const char *fn, unsigned size); extern struct varnish_stats *VSL_stats; Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-03 08:30:33 UTC (rev 278) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-03 12:41:14 UTC (rev 279) @@ -9,6 +9,7 @@ #include "cache.h" #include "libvarnish.h" #include "heritage.h" + /*-------------------------------------------------------------------- * From RFC2616, 13.2.3 Age Calculations * @@ -106,6 +107,7 @@ switch (http_GetStatus(hp)) { case 200: /* OK */ sp->obj->valid = 1; + /* FALLTHROUGH */ case 203: /* Non-Authoritative Information */ case 300: /* Multiple Choices */ case 301: /* Moved Permanently */ Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-03 08:30:33 UTC (rev 278) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-03 12:41:14 UTC (rev 279) @@ -297,26 +297,47 @@ /*--------------------------------------------------------------------*/ +static int +cmp_hash(struct hash_slinger *s, const char *p, const char *q) +{ + if (strlen(s->name) != q - p) + return (1); + if (strncmp(s->name, p, q - p)) + return (1); + return (0); +} + static void -usage(void) +setup_hash(const char *sflag) { - fprintf(stderr, "usage: varnishd [options]\n"); - fprintf(stderr, " %-28s # %s\n", "-b", "backend_IP_number"); - fprintf(stderr, " %-28s # %s\n", "-d", "debug"); - fprintf(stderr, " %-28s # %s\n", "-f", "VCL_file"); - fprintf(stderr, " %-28s # %s\n", "-p number", "TCP listen port"); - fprintf(stderr, " %-28s # %s\n", - "-s kind[,storageoptions]", "Backend storage specification"); - fprintf(stderr, " %-28s # %s\n", "-t", "Default TTL"); -#if 0 - -c clusterid at cluster_controller - -m memory_limit - -s kind[,storage-options] - -l logfile,logsize - -u uid - -a CLI_port -#endif - exit(1); + const char *p, *q; + struct hash_slinger *hp; + + p = strchr(sflag, ','); + if (p == NULL) + q = p = strchr(sflag, '\0'); + else + q = p + 1; + assert(p != NULL); + assert(q != NULL); + if (!cmp_hash(&hcl_slinger, sflag, p)) { + hp = &hcl_slinger; + } else if (!cmp_hash(&hsl_slinger, sflag, p)) { + hp = &hsl_slinger; + } else { + fprintf(stderr, "Unknown hash method \"%*.*s\"\n", + p - sflag, p - sflag, sflag); + exit (2); + } + heritage.hash = hp; + if (hp->init != NULL) { + if (hp->init(q)) + exit (1); + } else if (*q) { + fprintf(stderr, "Hash method \"%s\" takes no arguments\n", + hp->name); + exit (1); + } } /*--------------------------------------------------------------------*/ @@ -361,6 +382,36 @@ /*--------------------------------------------------------------------*/ +static void +usage(void) +{ + fprintf(stderr, "usage: varnishd [options]\n"); + fprintf(stderr, " %-28s # %s\n", "-b backend", + "backend IP or hostname"); + fprintf(stderr, " %-28s # %s\n", "-d", "debug"); + fprintf(stderr, " %-28s # %s\n", "-f file", "VCL_file"); + fprintf(stderr, " %-28s # %s\n", + "-h kind[,hashoptions]", "Hash specification"); + fprintf(stderr, " %-28s # %s\n", "-p number", "TCP listen port"); + fprintf(stderr, " %-28s # %s\n", + "-s kind[,storageoptions]", "Backend storage specification"); + fprintf(stderr, " %-28s # %s\n", "-t", "Default TTL"); + fprintf(stderr, " %-28s # %s\n", "-w int[,int]", + "Number of worker threads (fixed/{min,max})"); +#if 0 + -c clusterid at cluster_controller + -m memory_limit + -s kind[,storage-options] + -l logfile,logsize + -u uid + -a CLI_port +#endif + exit(1); +} + + +/*--------------------------------------------------------------------*/ + /* for development purposes */ #include #include @@ -368,20 +419,24 @@ int main(int argc, char *argv[]) { - int o; + int o, i; + unsigned ua, ub; const char *portnumber = "8080"; unsigned dflag = 1; /* XXX: debug=on for now */ const char *bflag = NULL; const char *fflag = NULL; const char *sflag = "file"; + const char *hflag = "classic"; register_printf_render_std((const unsigned char *)"HVQ"); VCC_InitCompile(); heritage.default_ttl = 120; + heritage.wthread_min = 5; + heritage.wthread_max = 5; - while ((o = getopt(argc, argv, "b:df:p:s:t:")) != -1) + while ((o = getopt(argc, argv, "b:df:h:p:s:t:w:")) != -1) switch (o) { case 'b': bflag = optarg; @@ -392,6 +447,9 @@ case 'f': fflag = optarg; break; + case 'h': + hflag = optarg; + break; case 'p': portnumber = optarg; break; @@ -401,6 +459,15 @@ case 't': heritage.default_ttl = strtoul(optarg, NULL, 0); break; + case 'w': + i = sscanf(optarg, "%u,%u", &ua, &ub); + if (i == 0) + usage(); + heritage.wthread_min = ua; + heritage.wthread_max = ua; + if (i == 2) + heritage.wthread_max = ub; + break; default: usage(); } @@ -430,6 +497,7 @@ exit (1); setup_storage(sflag); + setup_hash(hflag); /* * XXX: Lacking the suspend/resume facility (due to the socket API From phk at projects.linpro.no Mon Jul 3 14:36:03 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 3 Jul 2006 16:36:03 +0200 (CEST) Subject: r280 - trunk/varnish-cache/bin/varnishd Message-ID: <20060703143603.945E01EC383@projects.linpro.no> Author: phk Date: 2006-07-03 16:36:03 +0200 (Mon, 03 Jul 2006) New Revision: 280 Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c Log: s/init/start/ Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-07-03 12:41:14 UTC (rev 279) +++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-07-03 14:36:03 UTC (rev 280) @@ -32,7 +32,7 @@ */ static void -hsl_init(void) +hsl_start(void) { AZ(pthread_mutex_init(&hsl_mutex, NULL)); @@ -113,7 +113,7 @@ struct hash_slinger hsl_slinger = { "simple_list", NULL, - hsl_init, + hsl_start, hsl_lookup, hsl_deref, }; From phk at projects.linpro.no Mon Jul 3 14:37:51 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 3 Jul 2006 16:37:51 +0200 (CEST) Subject: r281 - trunk/varnish-cache/bin/varnishd Message-ID: <20060703143751.930A41EC362@projects.linpro.no> Author: phk Date: 2006-07-03 16:37:51 +0200 (Mon, 03 Jul 2006) New Revision: 281 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c trunk/varnish-cache/bin/varnishd/storage_malloc.c Log: s/unsigned/size_t/ Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-03 14:36:03 UTC (rev 280) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-03 14:37:51 UTC (rev 281) @@ -465,7 +465,7 @@ /*--------------------------------------------------------------------*/ static struct storage * -smf_alloc(struct stevedore *st, unsigned size) +smf_alloc(struct stevedore *st, size_t size) { struct smf *smf; struct smf_sc *sc = st->priv; Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-07-03 14:36:03 UTC (rev 280) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-07-03 14:37:51 UTC (rev 281) @@ -16,7 +16,7 @@ }; static struct storage * -sma_alloc(struct stevedore *st, unsigned size) +sma_alloc(struct stevedore *st, size_t size) { struct sma *sma; From phk at projects.linpro.no Mon Jul 3 14:39:02 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 3 Jul 2006 16:39:02 +0200 (CEST) Subject: r282 - trunk/varnish-cache/bin/varnishd Message-ID: <20060703143902.5B0111EC362@projects.linpro.no> Author: phk Date: 2006-07-03 16:39:02 +0200 (Mon, 03 Jul 2006) New Revision: 282 Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c Log: Include pthread.h Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2006-07-03 14:37:51 UTC (rev 281) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2006-07-03 14:39:02 UTC (rev 282) @@ -8,6 +8,7 @@ #include #include #include +#include #include "shmlog.h" #include "cli_priv.h" From phk at projects.linpro.no Mon Jul 3 14:39:40 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 3 Jul 2006 16:39:40 +0200 (CEST) Subject: r283 - trunk/varnish-cache/bin/varnishd Message-ID: <20060703143940.4561D1EC383@projects.linpro.no> Author: phk Date: 2006-07-03 16:39:40 +0200 (Mon, 03 Jul 2006) New Revision: 283 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c Log: avoid const poison Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-03 14:39:02 UTC (rev 282) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-03 14:39:40 UTC (rev 283) @@ -131,7 +131,7 @@ /* cache_acceptor.c */ void vca_write(struct sess *sp, void *ptr, size_t len); -void vca_write_obj(struct sess *sp, const char *b, unsigned l); +void vca_write_obj(struct sess *sp, char *b, unsigned l); void vca_flush(struct sess *sp); void vca_return_session(struct sess *sp); void vca_close_session(struct sess *sp, const char *why); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-03 14:39:02 UTC (rev 282) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-03 14:39:40 UTC (rev 283) @@ -85,7 +85,7 @@ } void -vca_write_obj(struct sess *sp, const char *b, unsigned l) +vca_write_obj(struct sess *sp, char *b, unsigned l) { struct storage *st; unsigned u = 0; From phk at projects.linpro.no Mon Jul 3 15:01:14 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 3 Jul 2006 17:01:14 +0200 (CEST) Subject: r284 - trunk/varnish-cache/lib/libvarnish Message-ID: <20060703150114.A34F91EC362@projects.linpro.no> Author: phk Date: 2006-07-03 17:01:14 +0200 (Mon, 03 Jul 2006) New Revision: 284 Modified: trunk/varnish-cache/lib/libvarnish/binary_heap.c Log: more asserts Modified: trunk/varnish-cache/lib/libvarnish/binary_heap.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/binary_heap.c 2006-07-03 14:39:40 UTC (rev 283) +++ trunk/varnish-cache/lib/libvarnish/binary_heap.c 2006-07-03 15:01:14 UTC (rev 284) @@ -136,6 +136,7 @@ { unsigned u; + assert(bh != NULL); assert(bh->magic == BINHEAP_MAGIC); assert(bh->length >= bh->next); if (bh->length == bh->next) { @@ -158,6 +159,7 @@ binheap_root(struct binheap *bh) { + assert(bh != NULL); assert(bh->magic == BINHEAP_MAGIC); if(bh->next == 0) return (NULL); @@ -168,6 +170,7 @@ binheap_delete(struct binheap *bh, unsigned idx) { + assert(bh != NULL); assert(bh->magic == BINHEAP_MAGIC); assert(bh->next > 0); assert(idx < bh->next); From phk at projects.linpro.no Mon Jul 3 17:59:13 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 3 Jul 2006 19:59:13 +0200 (CEST) Subject: r285 - trunk/varnish-cache/bin/varnishd Message-ID: <20060703175913.57EC21EC22A@projects.linpro.no> Author: phk Date: 2006-07-03 19:59:13 +0200 (Mon, 03 Jul 2006) New Revision: 285 Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c Log: Better arg checking Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-03 15:01:14 UTC (rev 284) +++ trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-03 17:59:13 UTC (rev 285) @@ -45,7 +45,7 @@ unsigned u1, u2; i = sscanf(p, "%u,%u", &u1, &u2); - if (i == 0) + if (i <= 0) return (0); if (u1 == 0 || (i == 2 && (u2 == 0 || u2 > u1))) { fprintf(stderr, "Invallid parameters to hash \"classic\":\n"); From phk at projects.linpro.no Mon Jul 3 18:03:44 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 3 Jul 2006 20:03:44 +0200 (CEST) Subject: r286 - trunk/varnish-cache/bin/varnishd Message-ID: <20060703180344.327D41EC22A@projects.linpro.no> Author: phk Date: 2006-07-03 20:03:44 +0200 (Mon, 03 Jul 2006) New Revision: 286 Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c Log: Close race condition Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-03 17:59:13 UTC (rev 285) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-03 18:03:44 UTC (rev 286) @@ -139,8 +139,8 @@ { AZ(pthread_mutex_init(&exp_mtx, NULL)); + exp_heap = binheap_new(NULL, object_cmp, object_update); + assert(exp_heap != NULL); AZ(pthread_create(&exp_thread, NULL, exp_prefetch, NULL)); AZ(pthread_create(&exp_thread, NULL, exp_hangman, NULL)); - exp_heap = binheap_new(NULL, object_cmp, object_update); - assert(exp_heap != NULL); } From phk at projects.linpro.no Mon Jul 3 18:35:39 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 3 Jul 2006 20:35:39 +0200 (CEST) Subject: r287 - trunk/varnish-cache/bin/varnishd Message-ID: <20060703183539.D2A7F1EC22A@projects.linpro.no> Author: phk Date: 2006-07-03 20:35:39 +0200 (Mon, 03 Jul 2006) New Revision: 287 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c Log: Handle IPv6 address -> string conversion too Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-03 18:03:44 UTC (rev 286) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-03 18:35:39 UTC (rev 287) @@ -6,7 +6,7 @@ #include "vcl_returns.h" -#define VCA_ADDRBUFSIZE 32 /* Sizeof ascii network address */ +#define VCA_ADDRBUFSIZE 64 /* Sizeof ascii network address */ struct event_base; struct cli; Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-03 18:03:44 UTC (rev 286) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-03 18:35:39 UTC (rev 287) @@ -122,9 +122,9 @@ { socklen_t l; struct sessmem *sm; - struct sockaddr addr; + struct sockaddr addr[2]; struct sess *sp; - char port[10]; + char port[NI_MAXSERV]; int i; VSL_stats->client_conn++; @@ -141,17 +141,21 @@ sp->mem = sm; l = sizeof addr; - sp->fd = accept(fd, &addr, &l); + sp->fd = accept(fd, addr, &l); if (sp->fd < 0) { free(sp); return; } i = 1; AZ(setsockopt(sp->fd, SOL_SOCKET, SO_NOSIGPIPE, &i, sizeof i)); - AZ(getnameinfo(&addr, l, + i = getnameinfo(addr, l, sp->addr, VCA_ADDRBUFSIZE, - port, sizeof port, NI_NUMERICHOST | NI_NUMERICSERV)); - strlcat(sp->addr, ":", VCA_ADDRBUFSIZE); + port, sizeof port, NI_NUMERICHOST | NI_NUMERICSERV); + if (i) { + printf("getnameinfo = %d %s\n", i, + gai_strerror(i)); + } + strlcat(sp->addr, " ", VCA_ADDRBUFSIZE); strlcat(sp->addr, port, VCA_ADDRBUFSIZE); VSL(SLT_SessionOpen, sp->fd, "%s", sp->addr); sp->http = http_New(); From phk at projects.linpro.no Mon Jul 3 19:35:06 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 3 Jul 2006 21:35:06 +0200 (CEST) Subject: r288 - trunk/varnish-cache/contrib/libevent Message-ID: <20060703193506.19A4B1EC325@projects.linpro.no> Author: phk Date: 2006-07-03 21:35:05 +0200 (Mon, 03 Jul 2006) New Revision: 288 Modified: trunk/varnish-cache/contrib/libevent/event.c Log: I have grumbled about the evilness of "current_base" before, and this just proves the point: If two threads call event_init() at the same time, they will both stomp on the same memory via current_base, and in all likelyhood, neither of them will manage to get the job done properly. Instead work on a local variable and don't assign to current_base until the setup is complete. This should be submitted to Niels Provos Modified: trunk/varnish-cache/contrib/libevent/event.c =================================================================== --- trunk/varnish-cache/contrib/libevent/event.c 2006-07-03 18:35:39 UTC (rev 287) +++ trunk/varnish-cache/contrib/libevent/event.c 2006-07-03 19:35:05 UTC (rev 288) @@ -164,36 +164,38 @@ event_init(void) { int i; + struct event_base *mybase; - if ((current_base = calloc(1, sizeof(struct event_base))) == NULL) + if ((mybase = calloc(1, sizeof(struct event_base))) == NULL) event_err(1, "%s: calloc"); event_sigcb = NULL; event_gotsig = 0; - gettime(¤t_base->event_tv); + gettime(&mybase->event_tv); - RB_INIT(¤t_base->timetree); - TAILQ_INIT(¤t_base->eventqueue); + RB_INIT(&mybase->timetree); + TAILQ_INIT(&mybase->eventqueue); TAILQ_INIT(&signalqueue); - current_base->evbase = NULL; - for (i = 0; eventops[i] && !current_base->evbase; i++) { - current_base->evsel = eventops[i]; + mybase->evbase = NULL; + for (i = 0; eventops[i] && NULL == mybase->evbase; i++) { + mybase->evsel = eventops[i]; - current_base->evbase = current_base->evsel->init(); + mybase->evbase = mybase->evsel->init(); } - if (current_base->evbase == NULL) + if (mybase->evbase == NULL) event_errx(1, "%s: no event mechanism available", __func__); if (getenv("EVENT_SHOW_METHOD")) event_msgx("libevent using: %s\n", - current_base->evsel->name); + mybase->evsel->name); /* allocate a single active event queue */ - event_base_priority_init(current_base, 1); + event_base_priority_init(mybase, 1); - return (current_base); + current_base = mybase; + return (mybase); } void From phk at projects.linpro.no Mon Jul 3 19:45:12 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 3 Jul 2006 21:45:12 +0200 (CEST) Subject: r289 - trunk/varnish-cache/bin/varnishd Message-ID: <20060703194512.B6A921EC325@projects.linpro.no> Author: phk Date: 2006-07-03 21:45:12 +0200 (Mon, 03 Jul 2006) New Revision: 289 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: My workaround for the missing OFF_T_MAX definition was not safe, use a hardcoded "1<<30" which is. Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-03 19:35:05 UTC (rev 288) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-03 19:45:12 UTC (rev 289) @@ -419,10 +419,12 @@ void *p; off_t h; - if (*fail < sc->pagesize * MINPAGES) + assert(sz != 0); + + if (*fail < (uintmax_t)sc->pagesize * MINPAGES) return; - if (sz < *fail && sz < SIZE_T_MAX) { + if (sz > 0 && sz < *fail && sz < SIZE_T_MAX) { p = mmap(NULL, sz, PROT_READ|PROT_WRITE, MAP_NOCORE | MAP_NOSYNC | MAP_SHARED, sc->fd, off); if (p != MAP_FAILED) { @@ -448,7 +450,7 @@ smf_open(struct stevedore *st) { struct smf_sc *sc; - off_t fail = SIZE_T_MAX; + off_t fail = 1 << 30; /* XXX: where is OFF_T_MAX ? */ off_t sum = 0; sc = st->priv; From phk at projects.linpro.no Tue Jul 4 09:21:34 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 4 Jul 2006 11:21:34 +0200 (CEST) Subject: r290 - trunk/varnish-cache/lib/libvcl Message-ID: <20060704092134.20D3D1EC30B@projects.linpro.no> Author: phk Date: 2006-07-04 11:21:33 +0200 (Tue, 04 Jul 2006) New Revision: 290 Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c Log: Add missing '%' Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-07-03 19:45:12 UTC (rev 289) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-07-04 09:21:33 UTC (rev 290) @@ -1676,7 +1676,7 @@ pos++; } - Fc(tl, 0, " [%3u] = { %4u, %3u, 0, \"T\" },\n", + Fc(tl, 0, " [%3u] = { %4u, %3u, 0, \"%T\" },\n", t->cnt, lin, pos + 1, t); } Fc(tl, 0, "};\n"); From phk at projects.linpro.no Tue Jul 4 09:28:21 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 4 Jul 2006 11:28:21 +0200 (CEST) Subject: r291 - trunk/varnish-cache/bin/varnishd Message-ID: <20060704092821.266011EC30B@projects.linpro.no> Author: phk Date: 2006-07-04 11:28:21 +0200 (Tue, 04 Jul 2006) New Revision: 291 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Close non HTTP/1.1 request connections Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-04 09:21:33 UTC (rev 290) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-04 09:28:21 UTC (rev 291) @@ -171,6 +171,7 @@ int http_GetHdr(struct http *hp, const char *hdr, char **ptr); int http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr); int http_GetReq(struct http *hp, char **b); +int http_GetProto(struct http *hp, char **b); int http_GetStatus(struct http *hp); int http_HdrIs(struct http *hp, const char *hdr, const char *val); int http_GetTail(struct http *hp, unsigned len, char **b, char **e); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-04 09:21:33 UTC (rev 290) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-04 09:28:21 UTC (rev 291) @@ -165,6 +165,15 @@ } int +http_GetProto(struct http *hp, char **b) +{ + if (hp->proto == NULL) + return (0); + *b = hp->proto; + return (1); +} + +int http_GetTail(struct http *hp, unsigned len, char **b, char **e) { Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-04 09:21:33 UTC (rev 290) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-04 09:28:21 UTC (rev 291) @@ -116,6 +116,9 @@ if (http_GetHdr(sp->http, "Connection", &b) && !strcmp(b, "close")) { vca_close_session(sp, "Connection header"); + } else if (http_GetProto(sp->http, &b) && + strcmp(b, "HTTP/1.1")) { + vca_close_session(sp, "not HTTP/1.1"); } AZ(pthread_mutex_lock(&sessmtx)); From des at projects.linpro.no Tue Jul 4 12:01:44 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Tue, 4 Jul 2006 14:01:44 +0200 (CEST) Subject: r292 - in trunk/varnish-tools: . fetcher recursor Message-ID: <20060704120144.997291EC22A@projects.linpro.no> Author: des Date: 2006-07-04 14:01:44 +0200 (Tue, 04 Jul 2006) New Revision: 292 Added: trunk/varnish-tools/fetcher/ trunk/varnish-tools/fetcher/Makefile trunk/varnish-tools/fetcher/fetcher.c trunk/varnish-tools/recursor/ trunk/varnish-tools/recursor/recursor.pl Log: Stuff we use for testing. Added: trunk/varnish-tools/fetcher/Makefile =================================================================== --- trunk/varnish-tools/fetcher/Makefile 2006-07-04 09:28:21 UTC (rev 291) +++ trunk/varnish-tools/fetcher/Makefile 2006-07-04 12:01:44 UTC (rev 292) @@ -0,0 +1,9 @@ +# +# $Id$ +# + +PROG = fetcher +WARNS ?= 6 +MAN = + +.include Property changes on: trunk/varnish-tools/fetcher/Makefile ___________________________________________________________________ Name: svn:keywords + Id Added: trunk/varnish-tools/fetcher/fetcher.c =================================================================== --- trunk/varnish-tools/fetcher/fetcher.c 2006-07-04 09:28:21 UTC (rev 291) +++ trunk/varnish-tools/fetcher/fetcher.c 2006-07-04 12:01:44 UTC (rev 292) @@ -0,0 +1,99 @@ +/* + * $Id$ + */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +static const char *req_pattern = +"GET http://varnish-test-1.linpro.no/cgi-bin/recursor.pl?foo=%d HTTP/1.1\r\n" +"Host: varnish-test-1.linpro.no\r\n" +"Connection: keep\r\n" +"\r\n"; + +static const char * +read_line(FILE *f) +{ + static char *buf; + static size_t bufsz; + const char *line; + size_t len; + + if ((line = fgetln(f, &len)) == NULL) + return (NULL); + while (len && (line[len - 1] == '\r' || line[len - 1] == '\n')) + --len; + if (bufsz < len + 1) { + bufsz = len * 2; + if ((buf = realloc(buf, bufsz)) == NULL) + err(1, "realloc()"); + } + memcpy(buf, line, len); + buf[len] = '\0'; +#ifdef DEBUG + fprintf(stderr, "<<< [%s]\n", buf); +#endif + return (buf); +} + +int +main(void) +{ + struct addrinfo hints, *res; + int clen, code, ctr, error, sd; + const char *line; + FILE *f; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + if ((error = getaddrinfo("varnish-test-2.linpro.no", "http", &hints, &res)) != 0) + errx(1, "%s", gai_strerror(error)); + if ((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) < 0) + err(1, "socket()"); + if (connect(sd, res->ai_addr, res->ai_addrlen) < 0) + err(1, "connect()"); + if ((f = fdopen(sd, "w+")) == NULL) + err(1, "fdopen()"); + for (ctr = 0; ctr < 5000; ++ctr) { + fprintf(stderr, "\r%d ", ctr); +#ifdef DEBUG + fprintf(stderr, req_pattern, ctr); +#endif + fprintf(f, req_pattern, ctr); + + /* get response header */ + if ((line = read_line(f)) == NULL) + errx(1, "protocol error"); + if (sscanf(line, "HTTP/%*d.%*d %d %*s", &code) != 1) + errx(1, "protocol error"); + if (code != 200) + errx(1, "code %d", code); + + /* get content-length */ + clen = -1; + for (;;) { + if ((line = read_line(f)) == NULL) + errx(1, "protocol error"); + if (line[0] == '\0') + break; + sscanf(line, "Content-Length: %d\n", &clen); + } + if (clen == -1) + errx(1, "no content length"); + + /* eat contents */ + while (clen--) + if (getc(f) == EOF) + errx(1, "connection prematurely closed"); + } + fclose(f); + exit(0); +} Property changes on: trunk/varnish-tools/fetcher/fetcher.c ___________________________________________________________________ Name: svn:keywords + Id Added: trunk/varnish-tools/recursor/recursor.pl =================================================================== --- trunk/varnish-tools/recursor/recursor.pl 2006-07-04 09:28:21 UTC (rev 291) +++ trunk/varnish-tools/recursor/recursor.pl 2006-07-04 12:01:44 UTC (rev 292) @@ -0,0 +1,19 @@ +#!/usr/local/bin/perl -w +# +# $Id$ +# + +use strict; +use CGI; + +my $q = new CGI; + +my $foo = int($q->param('foo')); +my $i = ($foo * 2) % 5000; +my $j = ($foo * 2 + 1) % 5000; + +print $q->header(-expires=>'+60m'); + +print "

Page $foo

\n"; +print "

Link $i

\n"; +print "

Link $j

\n"; Property changes on: trunk/varnish-tools/recursor/recursor.pl ___________________________________________________________________ Name: svn:executable + * Name: svn:keywords + Id From des at projects.linpro.no Tue Jul 4 12:03:13 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Tue, 4 Jul 2006 14:03:13 +0200 (CEST) Subject: r293 - trunk/varnish-tools/fetcher Message-ID: <20060704120313.CE9691EC35E@projects.linpro.no> Author: des Date: 2006-07-04 14:03:13 +0200 (Tue, 04 Jul 2006) New Revision: 293 Modified: trunk/varnish-tools/fetcher/fetcher.c Log: Add some comments. Modified: trunk/varnish-tools/fetcher/fetcher.c =================================================================== --- trunk/varnish-tools/fetcher/fetcher.c 2006-07-04 12:01:44 UTC (rev 292) +++ trunk/varnish-tools/fetcher/fetcher.c 2006-07-04 12:03:13 UTC (rev 293) @@ -51,6 +51,7 @@ const char *line; FILE *f; + /* connect to accelerator */ memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_INET; hints.ai_socktype = SOCK_STREAM; @@ -62,12 +63,15 @@ err(1, "connect()"); if ((f = fdopen(sd, "w+")) == NULL) err(1, "fdopen()"); + for (ctr = 0; ctr < 5000; ++ctr) { fprintf(stderr, "\r%d ", ctr); + + /* send request */ + fprintf(f, req_pattern, ctr); #ifdef DEBUG fprintf(stderr, req_pattern, ctr); #endif - fprintf(f, req_pattern, ctr); /* get response header */ if ((line = read_line(f)) == NULL) @@ -95,5 +99,6 @@ errx(1, "connection prematurely closed"); } fclose(f); + exit(0); } From des at projects.linpro.no Tue Jul 4 13:11:39 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Tue, 4 Jul 2006 15:11:39 +0200 (CEST) Subject: r294 - trunk/varnish-tools/fetcher Message-ID: <20060704131139.7FF151EC30B@projects.linpro.no> Author: des Date: 2006-07-04 15:11:39 +0200 (Tue, 04 Jul 2006) New Revision: 294 Modified: trunk/varnish-tools/fetcher/fetcher.c Log: Correct the Connection header, and add ability to perform HEAD requests. Modified: trunk/varnish-tools/fetcher/fetcher.c =================================================================== --- trunk/varnish-tools/fetcher/fetcher.c 2006-07-04 12:03:13 UTC (rev 293) +++ trunk/varnish-tools/fetcher/fetcher.c 2006-07-04 13:11:39 UTC (rev 294) @@ -11,11 +11,15 @@ #include #include #include +#include +static const char HEAD[] = "HEAD"; +static const char GET[] = "GET"; +static const char *method = GET; static const char *req_pattern = -"GET http://varnish-test-1.linpro.no/cgi-bin/recursor.pl?foo=%d HTTP/1.1\r\n" +"%s /cgi-bin/recursor.pl?foo=%d HTTP/1.1\r\n" "Host: varnish-test-1.linpro.no\r\n" -"Connection: keep\r\n" +"Connection: Keep-Alive\r\n" "\r\n"; static const char * @@ -43,14 +47,36 @@ return (buf); } +static void +usage(void) +{ + fprintf(stderr, "usage: fetcher [-h]\n"); + exit(1); +} + int -main(void) +main(int argc, char *argv[]) { struct addrinfo hints, *res; - int clen, code, ctr, error, sd; + int clen, code, ctr, error, opt, sd; const char *line; FILE *f; + while ((opt = getopt(argc, argv, "h")) != -1) + switch (opt) { + case 'h': + method = HEAD; + break; + default: + usage(); + } + + argc -= optind; + argv += optind; + + if (argc != 0) + usage(); + /* connect to accelerator */ memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_INET; @@ -65,12 +91,13 @@ err(1, "fdopen()"); for (ctr = 0; ctr < 5000; ++ctr) { - fprintf(stderr, "\r%d ", ctr); /* send request */ - fprintf(f, req_pattern, ctr); + fprintf(f, req_pattern, method, ctr); #ifdef DEBUG - fprintf(stderr, req_pattern, ctr); + fprintf(stderr, req_pattern, method, ctr); +#else + fprintf(stderr, "\r%d ", ctr); #endif /* get response header */ @@ -94,9 +121,10 @@ errx(1, "no content length"); /* eat contents */ - while (clen--) - if (getc(f) == EOF) - errx(1, "connection prematurely closed"); + if (method != HEAD) + while (clen--) + if (getc(f) == EOF) + errx(1, "connection prematurely closed"); } fclose(f); From phk at projects.linpro.no Tue Jul 4 13:44:07 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 4 Jul 2006 15:44:07 +0200 (CEST) Subject: r295 - trunk/varnish-cache/bin/varnishd Message-ID: <20060704134407.9A9121EC35E@projects.linpro.no> Author: phk Date: 2006-07-04 15:44:07 +0200 (Tue, 04 Jul 2006) New Revision: 295 Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c Log: Fix off by one error during wraparound. Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-04 13:11:39 UTC (rev 294) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-04 13:44:07 UTC (rev 295) @@ -28,11 +28,21 @@ * This variant copies a byte-range directly to the log, without * taking the detour over sprintf() */ + +static void +vsl_wrap(void) +{ + + *logstart = SLT_ENDMARKER; + logstart[loghead->ptr] = SLT_WRAPMARKER; + loghead->ptr = 0; +} void VSLR(enum shmlogtag tag, unsigned id, const char *b, const char *e) { - unsigned char *p, *q; + unsigned char *p; + unsigned l; assert(b != NULL); if (e == NULL) @@ -40,31 +50,28 @@ assert(e != NULL); /* Truncate */ - if (e - b > 255) - e = b + 255; + l = e - b; + if (l > 255) { + l = 255; + e = b + l; + } AZ(pthread_mutex_lock(&vsl_mutex)); - q = NULL; - p = logstart + loghead->ptr; - assert(p < logend); + assert(loghead->ptr < loghead->size); /* Wrap if necessary */ - if (p + 4 + (e - b) > logend) { - q = p; - p = logstart; - *p = SLT_ENDMARKER; - } - p[1] = e - b; + if (loghead->ptr + 5 + l > loghead->size) + vsl_wrap(); + p = logstart + loghead->ptr; + p[1] = l; p[2] = id >> 8; p[3] = id & 0xff; - memcpy(p + 4, b, e - b); + memcpy(p + 4, b, l); + p[4 + l] = SLT_ENDMARKER; p[0] = tag; - if (q != NULL) - *q = SLT_WRAPMARKER; - - loghead->ptr = (p + 4 + (e - b)) - logstart; - + loghead->ptr += 4 + l; + assert(loghead->ptr < loghead->size); AZ(pthread_mutex_unlock(&vsl_mutex)); } @@ -73,29 +80,27 @@ VSL(enum shmlogtag tag, unsigned id, const char *fmt, ...) { va_list ap; - unsigned char *p, *q; + unsigned char *p; unsigned m, n; va_start(ap, fmt); AZ(pthread_mutex_lock(&vsl_mutex)); - q = NULL; - p = logstart + loghead->ptr; - assert(p < logend); + assert(loghead->ptr < loghead->size); /* * Wrap early if we approach the end - * 32 is arbitraryly larger than minimum of 4. + * 32 is arbitraryly larger than minimum of 5. */ - if (p + 32 > logend) { - q = p; - p = logstart; - *p = SLT_ENDMARKER; - } + if (loghead->ptr + 32 > loghead->size) + vsl_wrap(); + + p = logstart + loghead->ptr; n = 0; if (fmt != NULL) { while (1) { - m = logend - (p + 4); + /* We need 4 four header + 1 for ENDMARKER */ + m = loghead->size - (loghead->ptr + 5); if (m > 256) m = 256; n = vsnprintf((char *)p + 4, m, fmt, ap); @@ -103,23 +108,18 @@ n = 255; /* we truncate long fields */ if (n < m) break; - /* wraparound */ - assert(q == NULL); - q = p; - p = logstart; - *p = SLT_ENDMARKER; + vsl_wrap(); continue; /* Try again */ } } p[1] = n; p[2] = id >> 8; p[3] = id & 0xff; + p[4 + n] = SLT_ENDMARKER; p[0] = tag; - if (q != NULL) - *q = SLT_WRAPMARKER; - - loghead->ptr = (p + 4 + n) - logstart; + loghead->ptr += 4 + n; + assert(loghead->ptr < loghead->size); AZ(pthread_mutex_unlock(&vsl_mutex)); @@ -151,6 +151,7 @@ struct shmloghead slh; int i; + unlink(fn); heritage.vsl_fd = open(fn, O_RDWR | O_CREAT, 0600); if (heritage.vsl_fd < 0) { fprintf(stderr, "Could not open %s: %s\n", @@ -169,10 +170,8 @@ i = write(heritage.vsl_fd, &slh, sizeof slh); assert(i == sizeof slh); AZ(ftruncate(heritage.vsl_fd, sizeof slh + size)); - heritage.vsl_size = slh.size + slh.start; - } else { - heritage.vsl_size = slh.size + slh.start; } + heritage.vsl_size = slh.size + slh.start; /* * Call VSL_Init so that we get a VSL_stats pointer in the From phk at projects.linpro.no Tue Jul 4 13:45:16 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 4 Jul 2006 15:45:16 +0200 (CEST) Subject: r296 - trunk/varnish-cache/bin/varnishlog Message-ID: <20060704134516.A08F71EC35E@projects.linpro.no> Author: phk Date: 2006-07-04 15:45:16 +0200 (Tue, 04 Jul 2006) New Revision: 296 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c Log: Shorter sleeps: 1s -> 50msec Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-04 13:44:07 UTC (rev 295) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-04 13:45:16 UTC (rev 296) @@ -135,7 +135,7 @@ p = VSL_NextLog(loghead, &q); if (p == NULL) { fflush(stdout); - sleep(1); + usleep(50000); continue; } if (o_flag) { From phk at projects.linpro.no Tue Jul 4 14:18:02 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 4 Jul 2006 16:18:02 +0200 (CEST) Subject: r297 - trunk/varnish-cache/bin/varnishd Message-ID: <20060704141802.200211EC35E@projects.linpro.no> Author: phk Date: 2006-07-04 16:18:02 +0200 (Tue, 04 Jul 2006) New Revision: 297 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: Lock with a mutex Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-04 13:45:16 UTC (rev 296) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-04 14:18:02 UTC (rev 297) @@ -53,6 +53,7 @@ struct smfhead order; struct smfhead free; struct smfhead used; + pthread_mutex_t mtx; }; /*--------------------------------------------------------------------*/ @@ -462,6 +463,7 @@ /* XXX */ if (sum < MINPAGES * getpagesize()) exit (2); + AZ(pthread_mutex_init(&sc->mtx, NULL)); } /*--------------------------------------------------------------------*/ @@ -474,7 +476,9 @@ size += (sc->pagesize - 1); size &= ~(sc->pagesize - 1); + AZ(pthread_mutex_lock(&sc->mtx)); smf = alloc_smf(sc, size); + AZ(pthread_mutex_unlock(&sc->mtx)); assert(smf != NULL); smf->s.space = size; smf->s.priv = smf; @@ -493,14 +497,16 @@ struct smf_sc *sc; assert(size <= s->space); - assert(size > 0); + assert(size > 0); /* XXX: seen */ smf = (struct smf *)(s->priv); assert(size <= smf->size); sc = smf->sc; size += (sc->pagesize - 1); size &= ~(sc->pagesize - 1); if (smf->size > size) { + AZ(pthread_mutex_lock(&sc->mtx)); trim_smf(smf, size); + AZ(pthread_mutex_unlock(&sc->mtx)); smf->s.space = size; } } @@ -511,9 +517,13 @@ smf_free(struct storage *s) { struct smf *smf; + struct smf_sc *sc; smf = (struct smf *)(s->priv); + sc = smf->sc; + AZ(pthread_mutex_lock(&sc->mtx)); free_smf(smf); + AZ(pthread_mutex_unlock(&sc->mtx)); } /*--------------------------------------------------------------------*/ From phk at projects.linpro.no Tue Jul 4 14:18:35 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 4 Jul 2006 16:18:35 +0200 (CEST) Subject: r298 - trunk/varnish-cache/bin/varnishd Message-ID: <20060704141835.0A81B1EC35E@projects.linpro.no> Author: phk Date: 2006-07-04 16:18:34 +0200 (Tue, 04 Jul 2006) New Revision: 298 Modified: trunk/varnish-cache/bin/varnishd/cache_main.c Log: load VCL earlier Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-04 14:18:02 UTC (rev 297) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-04 14:18:34 UTC (rev 298) @@ -92,7 +92,9 @@ { NULL } }; -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * XXX: Think more about which order we start things + */ void child_main(void) @@ -108,6 +110,7 @@ setbuf(stderr, NULL); printf("Child starts\n"); + CVCL_Load(heritage.vcl_file, "boot"); AZ(pthread_mutex_init(&sessmtx, NULL)); VBE_Init(); VSL_Init(); @@ -125,7 +128,6 @@ if (stevedore->open != NULL) stevedore->open(stevedore); - CVCL_Load(heritage.vcl_file, "boot"); cli = cli_setup(eb, heritage.fds[2], heritage.fds[1], 0, cli_proto); evtimer_set(&ev_keepalive, timer_keepalive, NULL); From phk at projects.linpro.no Tue Jul 4 14:19:12 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 4 Jul 2006 16:19:12 +0200 (CEST) Subject: r299 - trunk/varnish-cache/bin/varnishd Message-ID: <20060704141912.85B131EC30B@projects.linpro.no> Author: phk Date: 2006-07-04 16:19:12 +0200 (Tue, 04 Jul 2006) New Revision: 299 Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c Log: retry backend open, log diagnostics Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-04 14:18:34 UTC (rev 298) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-04 14:19:12 UTC (rev 299) @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -101,11 +102,14 @@ do { s = socket(res0->ai_family, res0->ai_socktype, res0->ai_protocol); - if (s < 0) + if (s < 0) { + VSL(SLT_Debug, 0, "Socket errno=%d", errno); continue; + } error = connect(s, res0->ai_addr, res0->ai_addrlen); if (!error) break; + VSL(SLT_Debug, 0, "Connect errno=%d", errno); close(s); s = -1; } while ((res0 = res0->ai_next) != NULL); From phk at projects.linpro.no Tue Jul 4 14:45:01 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 4 Jul 2006 16:45:01 +0200 (CEST) Subject: r300 - trunk/varnish-cache/bin/varnishd Message-ID: <20060704144501.C661E1EC35E@projects.linpro.no> Author: phk Date: 2006-07-04 16:45:01 +0200 (Tue, 04 Jul 2006) New Revision: 300 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Make room for protective terminating NUL Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-04 14:19:12 UTC (rev 299) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-04 14:45:01 UTC (rev 300) @@ -305,11 +305,13 @@ { struct http *hp = arg; char *p; + unsigned l; int i; - assert(hp->v < hp->e); + l = hp->e - hp->v; + assert(l > 1); errno = 0; - i = read(fd, hp->v, hp->e - hp->v); + i = read(fd, hp->v, l - 1); if (i <= 0) { if (hp->v != hp->s) VSL(SLT_HttpError, fd, From des at projects.linpro.no Tue Jul 4 14:52:54 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Tue, 4 Jul 2006 16:52:54 +0200 (CEST) Subject: r301 - trunk/varnish-tools/fetcher Message-ID: <20060704145254.BC5421EC30B@projects.linpro.no> Author: des Date: 2006-07-04 16:52:54 +0200 (Tue, 04 Jul 2006) New Revision: 301 Modified: trunk/varnish-tools/fetcher/fetcher.c Log: More iterations, and use fread() instead of getc(). Modified: trunk/varnish-tools/fetcher/fetcher.c =================================================================== --- trunk/varnish-tools/fetcher/fetcher.c 2006-07-04 14:45:01 UTC (rev 300) +++ trunk/varnish-tools/fetcher/fetcher.c 2006-07-04 14:52:54 UTC (rev 301) @@ -13,6 +13,9 @@ #include #include +#define MAX_CTR 5000 + +static char data[8192]; static const char HEAD[] = "HEAD"; static const char GET[] = "GET"; static const char *method = GET; @@ -58,7 +61,8 @@ main(int argc, char *argv[]) { struct addrinfo hints, *res; - int clen, code, ctr, error, opt, sd; + int code, ctr, error, opt, sd; + size_t clen; const char *line; FILE *f; @@ -90,14 +94,15 @@ if ((f = fdopen(sd, "w+")) == NULL) err(1, "fdopen()"); - for (ctr = 0; ctr < 5000; ++ctr) { + for (ctr = 0; ctr < 500000; ++ctr) { /* send request */ - fprintf(f, req_pattern, method, ctr); + fprintf(f, req_pattern, method, ctr % MAX_CTR); #ifdef DEBUG - fprintf(stderr, req_pattern, method, ctr); + fprintf(stderr, req_pattern, method, ctr % MAX_CTR); #else - fprintf(stderr, "\r%d ", ctr); + if (ctr % 163 == 0) + fprintf(stderr, "\r%d ", ctr); #endif /* get response header */ @@ -109,22 +114,22 @@ errx(1, "code %d", code); /* get content-length */ - clen = -1; + clen = 0; for (;;) { if ((line = read_line(f)) == NULL) errx(1, "protocol error"); if (line[0] == '\0') break; - sscanf(line, "Content-Length: %d\n", &clen); + sscanf(line, "Content-Length: %zu\n", &clen); } - if (clen == -1) - errx(1, "no content length"); /* eat contents */ - if (method != HEAD) - while (clen--) - if (getc(f) == EOF) - errx(1, "connection prematurely closed"); + if (method == HEAD) + continue; + while (clen > 0) { + size_t rlen = clen > sizeof(data) ? sizeof(data) : clen; + clen -= fread(data, 1, rlen, f); + } } fclose(f); From phk at projects.linpro.no Tue Jul 4 19:36:00 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 4 Jul 2006 21:36:00 +0200 (CEST) Subject: r302 - trunk/varnish-cache/bin/varnishd Message-ID: <20060704193600.A4A151EC22A@projects.linpro.no> Author: phk Date: 2006-07-04 21:36:00 +0200 (Tue, 04 Jul 2006) New Revision: 302 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Fix pipelining. A braino in http_Dissect() resulted in an off-by-one error (protected with assert now) Move any remaning bytes in buffer to front and check for a complete header before arming the eventloop on the session. Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-04 14:52:54 UTC (rev 301) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-04 19:36:00 UTC (rev 302) @@ -291,11 +291,34 @@ VSLR(SLT_LostHeader, fd, p, q); } } - if (*++p == '\r') + assert(hp->t == r); +} + +/*--------------------------------------------------------------------*/ + +static int +http_header_complete(struct http *hp) +{ + char *p; + + p = hp->s; + while (1) { + /* XXX: we could save location of all linebreaks for later */ + p = strchr(p, '\n'); + if (p == NULL) + return (0); p++; + if (*p == '\r') + p++; + if (*p != '\n') + continue; + break; + } hp->t = ++p; + return (1); } + /*--------------------------------------------------------------------*/ #include @@ -330,22 +353,9 @@ hp->v += i; *hp->v = '\0'; + if (!http_header_complete(hp)) + return; - p = hp->s; - while (1) { - /* XXX: we could save location of all linebreaks for later */ - p = strchr(p, '\n'); - if (p == NULL) - return; - p++; - if (*p == '\r') - p++; - if (*p != '\n') - continue; - break; - } - hp->t = ++p; - event_del(&hp->ev); if (hp->callback != NULL) hp->callback(hp->arg, 1); @@ -356,9 +366,18 @@ void http_RecvHead(struct http *hp, int fd, struct event_base *eb, http_callback_f *func, void *arg) { + unsigned l; assert(hp != NULL); - assert(hp->t == hp->s || hp->t == hp->v); /* XXX pipelining */ + if (hp->t > hp->s && hp->t < hp->v) { + l = hp->v - hp->t; + memmove(hp->s, hp->t, l); + hp->v = hp->s + l; + if (http_header_complete(hp)) { + func(arg, 1); + return; + } + } hp->callback = func; hp->arg = arg; hp->v = hp->s; From phk at projects.linpro.no Tue Jul 4 20:00:27 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 4 Jul 2006 22:00:27 +0200 (CEST) Subject: r303 - trunk/varnish-cache/bin/varnishd Message-ID: <20060704200027.991F91EC30B@projects.linpro.no> Author: phk Date: 2006-07-04 22:00:27 +0200 (Tue, 04 Jul 2006) New Revision: 303 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_shmlog.c Log: Fix HEAD requests: Make modes to http_BuildSbuf descriptive enums. Send GET to backend also for HEAD requests. Don't return body for HEAD requests. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-04 19:36:00 UTC (rev 302) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-04 20:00:27 UTC (rev 303) @@ -178,7 +178,13 @@ int http_GetURL(struct http *hp, char **b); void http_RecvHead(struct http *hp, int fd, struct event_base *eb, http_callback_f *func, void *arg); void http_Dissect(struct http *sp, int fd, int rr); -void http_BuildSbuf(int fd, int resp, struct sbuf *sb, struct http *hp); +enum http_build { + Build_Pipe, + Build_Pass, + Build_Fetch, + Build_Reply, +}; +void http_BuildSbuf(int fd, enum http_build mode, struct sbuf *sb, struct http *hp); /* cache_main.c */ extern pthread_mutex_t sessmtx; Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-04 19:36:00 UTC (rev 302) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-04 20:00:27 UTC (rev 303) @@ -89,18 +89,22 @@ { struct storage *st; unsigned u = 0; + char *r; if (l == 0) l = strlen(b); vca_write(sp, b, l); - TAILQ_FOREACH(st, &sp->obj->store, list) { - u += st->len; - if (st->stevedore->send != NULL) - st->stevedore->send(st, sp); - else - vca_write(sp, st->ptr, st->len); + assert(http_GetReq(sp->http, &r)); + if (!strcmp(r, "GET")) { + TAILQ_FOREACH(st, &sp->obj->store, list) { + u += st->len; + if (st->stevedore->send != NULL) + st->stevedore->send(st, sp); + else + vca_write(sp, st->ptr, st->len); + } + assert(u == sp->obj->len); } - assert(u == sp->obj->len); vca_flush(sp); } Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-04 19:36:00 UTC (rev 302) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-04 20:00:27 UTC (rev 303) @@ -249,7 +249,7 @@ VSL(SLT_Backend, sp->fd, "%d %s", fd, sp->backend->vcl_name); hp = http_New(); - http_BuildSbuf(fd, 1, w->sb, sp->http); + http_BuildSbuf(fd, Build_Fetch, w->sb, sp->http); i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); time(&sp->t_req); @@ -272,7 +272,7 @@ if (sp->obj->cacheable) EXP_Insert(sp->obj); - http_BuildSbuf(sp->fd, 3, w->sb, hp); + http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); if (body) { if (http_GetHdr(hp, "Content-Length", &b)) cls = fetch_straight(w, sp, fd, hp, b); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-04 19:36:00 UTC (rev 302) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-04 20:00:27 UTC (rev 303) @@ -415,39 +415,52 @@ /*--------------------------------------------------------------------*/ void -http_BuildSbuf(int fd, int resp, struct sbuf *sb, struct http *hp) +http_BuildSbuf(int fd, enum http_build mode, struct sbuf *sb, struct http *hp) { - unsigned u; + unsigned u, sup; sbuf_clear(sb); assert(sb != NULL); - if (resp == 2 || resp == 3) { + switch (mode) { + case Build_Reply: sbuf_cat(sb, hp->proto); sbuf_cat(sb, " "); sbuf_cat(sb, hp->status); sbuf_cat(sb, " "); sbuf_cat(sb, hp->response); - } else if (resp == 1) { + sup = 2; + break; + case Build_Pipe: + case Build_Pass: sbuf_cat(sb, hp->req); sbuf_cat(sb, " "); sbuf_cat(sb, hp->url); sbuf_cat(sb, " "); sbuf_cat(sb, hp->proto); - } else { - printf("resp = %d\n", resp); - assert(resp == 1 || resp == 2); + sup = 2; + break; + case Build_Fetch: + sbuf_cat(sb, "GET "); + sbuf_cat(sb, hp->url); + sbuf_cat(sb, " "); + sbuf_cat(sb, hp->proto); + sup = 1; + break; + default: + printf("mode = %d\n", mode); + assert(mode == 1 || mode == 2); } sbuf_cat(sb, "\r\n"); for (u = 0; u < hp->nhdr; u++) { - if (http_supress(hp->hdr[u], resp)) + if (http_supress(hp->hdr[u], sup)) continue; if (1) VSL(SLT_BldHdr, fd, "%s", hp->hdr[u]); sbuf_cat(sb, hp->hdr[u]); sbuf_cat(sb, "\r\n"); } - if (resp != 3) { + if (mode != Build_Reply) { sbuf_cat(sb, "\r\n"); sbuf_finish(sb); } Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-04 19:36:00 UTC (rev 302) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-04 20:00:27 UTC (rev 303) @@ -160,7 +160,7 @@ fd = VBE_GetFd(sp->backend, &fd_token, sp->xid); assert(fd != -1); - http_BuildSbuf(fd, 1, w->sb, sp->http); + http_BuildSbuf(fd, Build_Pass, w->sb, sp->http); i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-04 19:36:00 UTC (rev 302) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-04 20:00:27 UTC (rev 303) @@ -51,7 +51,7 @@ fd = VBE_GetFd(sp->backend, &fd_token, sp->xid); assert(fd != -1); - http_BuildSbuf(fd, 0, w->sb, sp->http); /* XXX: 0 ?? */ + http_BuildSbuf(fd, Build_Pipe, w->sb, sp->http); /* XXX: 0 ?? */ i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); assert(__LINE__ == 0); Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-04 19:36:00 UTC (rev 302) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-04 20:00:27 UTC (rev 303) @@ -160,6 +160,7 @@ AZ(pthread_cond_init(&shdcnd, NULL)); + VSL(SLT_Debug, 0, "Starting %u worker threads", heritage.wthread_min); for (i = 0; i < heritage.wthread_min; i++) { AZ(pthread_create(&tp, NULL, CacheWorker, NULL)); AZ(pthread_detach(tp)); Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-04 19:36:00 UTC (rev 302) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-04 20:00:27 UTC (rev 303) @@ -151,7 +151,6 @@ struct shmloghead slh; int i; - unlink(fn); heritage.vsl_fd = open(fn, O_RDWR | O_CREAT, 0600); if (heritage.vsl_fd < 0) { fprintf(stderr, "Could not open %s: %s\n", From phk at projects.linpro.no Tue Jul 4 21:34:39 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 4 Jul 2006 23:34:39 +0200 (CEST) Subject: r304 - trunk/varnish-cache/bin/varnishd Message-ID: <20060704213439.250EE1EC35E@projects.linpro.no> Author: phk Date: 2006-07-04 23:34:39 +0200 (Tue, 04 Jul 2006) New Revision: 304 Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c Log: Long comment describing how TTL calculation will be done. Review encouraged. Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-04 20:00:27 UTC (rev 303) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-04 21:34:39 UTC (rev 304) @@ -10,7 +10,67 @@ #include "libvarnish.h" #include "heritage.h" + /*-------------------------------------------------------------------- + * TTL and Age calculation in Varnish + * + * RFC2616 has a lot to say about how caches should calculate the TTL + * and expiry times of objects, but it sort of misses the case that + * applies to Varnish: the server-side cache. + * + * A normal cache, shared or single-client, has no symbiotic relationship + * with the server, and therefore must take a very defensive attitude + * if the Data/Expiry/Age/max-age data does not make sense. Overall + * the policy described in section 13 of RFC 2616 results in no caching + * happening on the first little sign of trouble. + * + * Varnish on the other hand tries to offload as many transactions from + * the backend as possible, and therefore just passing through everything + * if there is a clock-skew between backend and Varnish is not a workable + * choice. + * + * Varnish implements a policy which is RFC2616 compliant when there + * is no clockskew, and falls back to a new "clockless cache" mode otherwise. + * Our "clockless cache" model is syntehsized from the bits of RFC2616 + * that talks about how a cache should react to a clockless origin server, + * and more or uses the inverse logic for the opposite relationship. + * + * /* Marker for no retirement age determined */ + * retirement_age = INT_MAX + * + * /* If we have a max-age directive, respect it */ + * if (max-age) + * retirement_age = max(0,min(retirement_age, max-age - Age:)) + * + * /* If we have no Date: and Expires: looks sensible, use it */ + * if (!date && expires > our_clock) { + * ttd = min(our_clock + retirement_age, Expires:) + * + * /* If Date: is in the past, and Expires: looks sensible, use it */ + * } else if (date < our_clock && expires > our_clock) { + * ttd = min(date + retirement_age, Expires:) + * + * /* Otherwise we have clock-skew */ + * } else { + * /* If we have both date and expires, infer max-age */ + * if (date && expires) + * retirement_age = + * max(0, min(retirement_age, Expires: - Date:) + * + * /* Apply default_ttl if nothing better found */ + * if (retirement_age == INT_MAX) + * retirement_age = default_ttl + * + * /* Apply the max-age we can up with */ + * ttd = our_clock + retirement_age + * } + * + * /* Apply hard limits */ + * ttd = max(ttd, our_clock + hard_lower_ttl) + * ttd = min(ttd, our_clock + hard_upper_ttl) + */ + +/*-------------------------------------------------------------------- * From RFC2616, 13.2.3 Age Calculations * * age_value From phk at projects.linpro.no Tue Jul 4 22:08:33 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 00:08:33 +0200 (CEST) Subject: r305 - trunk/varnish-cache/bin/varnishd Message-ID: <20060704220833.C81DF1EC30B@projects.linpro.no> Author: phk Date: 2006-07-05 00:08:33 +0200 (Wed, 05 Jul 2006) New Revision: 305 Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c Log: Implement our TTL calculation. A first quick check against the weird timestamps from the VG frontend squids indicates sensible behaviour. Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-04 21:34:39 UTC (rev 304) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-04 22:08:33 UTC (rev 305) @@ -5,7 +5,9 @@ #include #include #include +#include +#include "shmlog.h" #include "cache.h" #include "libvarnish.h" #include "heritage.h" @@ -35,124 +37,88 @@ * that talks about how a cache should react to a clockless origin server, * and more or uses the inverse logic for the opposite relationship. * - * /* Marker for no retirement age determined */ - * retirement_age = INT_MAX - * - * /* If we have a max-age directive, respect it */ - * if (max-age) - * retirement_age = max(0,min(retirement_age, max-age - Age:)) - * - * /* If we have no Date: and Expires: looks sensible, use it */ - * if (!date && expires > our_clock) { - * ttd = min(our_clock + retirement_age, Expires:) - * - * /* If Date: is in the past, and Expires: looks sensible, use it */ - * } else if (date < our_clock && expires > our_clock) { - * ttd = min(date + retirement_age, Expires:) - * - * /* Otherwise we have clock-skew */ - * } else { - * /* If we have both date and expires, infer max-age */ - * if (date && expires) - * retirement_age = - * max(0, min(retirement_age, Expires: - Date:) - * - * /* Apply default_ttl if nothing better found */ - * if (retirement_age == INT_MAX) - * retirement_age = default_ttl - * - * /* Apply the max-age we can up with */ - * ttd = our_clock + retirement_age - * } - * - * /* Apply hard limits */ - * ttd = max(ttd, our_clock + hard_lower_ttl) - * ttd = min(ttd, our_clock + hard_upper_ttl) */ -/*-------------------------------------------------------------------- - * From RFC2616, 13.2.3 Age Calculations - * - * age_value - * is the value of Age: header received by the cache with - * this response. - * date_value - * is the value of the origin server's Date: header - * request_time - * is the (local) time when the cache made the request - * that resulted in this cached response - * response_time - * is the (local) time when the cache received the response - * now - * is the current (local) time - * - * apparent_age = max(0, response_time - date_value); - * corrected_received_age = max(apparent_age, age_value); - * response_delay = response_time - request_time; - * corrected_initial_age = corrected_received_age + response_delay; - * resident_time = now - response_time; - * current_age = corrected_initial_age + resident_time; - * - */ +#if PSEUDO_CODE + /* Marker for no retirement age determined */ + retirement_age = INT_MAX + /* If we have a max-age directive, respect it */ + if (max-age) + retirement_age = max(0,min(retirement_age, max-age - Age:)) + + /* If Date: is not in future and Expires: looks sensible, use it */ + if ((!date || date < our_clock) && expires > our_clock) { + ttd = min(our_clock + retirement_age, Expires:) + + /* Otherwise we have clock-skew */ + } else { + /* If we have both date and expires, infer max-age */ + if (date && expires) + retirement_age = + max(0, min(retirement_age, Expires: - Date:) + + /* Apply default_ttl if nothing better found */ + if (retirement_age == INT_MAX) + retirement_age = default_ttl + + /* Apply the max-age we can up with */ + ttd = our_clock + retirement_age + } + + /* Apply hard limits */ + ttd = max(ttd, our_clock + hard_lower_ttl) + ttd = min(ttd, our_clock + hard_upper_ttl) +#endif + static time_t RFC2616_Ttl(struct http *hp, time_t t_req, time_t t_resp) { - time_t h_date = 0, h_expires = 0, h_age = 0; - time_t apparent_age = 0, corrected_received_age; - time_t response_delay, corrected_initial_age; - time_t max_age = -1, ttl; - time_t fudge; + int retirement_age; + unsigned u1, u2; + time_t h_date, h_expires, ttd; char *p; + + retirement_age = INT_MAX; - if (http_GetHdrField(hp, "Cache-Control", "max-age", &p)) - max_age = strtoul(p, NULL, 0); + u1 = u2 = 0; + if (http_GetHdrField(hp, "Cache-Control", "max-age", &p)) { + u1 = strtoul(p, NULL, 0); + u2 = 0; + if (http_GetHdr(hp, "Age", &p)) + u2 = strtoul(p, NULL, 0); + if (u2 <= u1) + retirement_age = u1 - u2; + } + h_date = 0; if (http_GetHdr(hp, "Date", &p)) h_date = TIM_parse(p); - if (h_date + 3600 < t_resp) { - fudge = t_resp - h_date; - h_date += fudge; - } else - fudge = 0; + h_expires = 0; + if (http_GetHdr(hp, "Expires", &p)) + h_expires = TIM_parse(p); - if (h_date < t_resp) - apparent_age = t_resp - h_date; + if (h_date < t_req && h_expires > t_req) { + ttd = h_expires; + if (retirement_age != INT_MAX && t_req + retirement_age < ttd) + ttd = t_req + retirement_age; + } else { + if (h_date != 0 && h_expires != 0) { + if (h_date < h_expires && + h_expires - h_date < retirement_age) + retirement_age = h_expires - h_date; + } + if (retirement_age == INT_MAX) + retirement_age = heritage.default_ttl; - if (http_GetHdr(hp, "Age", &p)) - h_age = strtoul(p, NULL, 0); - - if (h_age > apparent_age) - corrected_received_age = h_age; - else - corrected_received_age = apparent_age; - - response_delay = t_resp - t_req; - corrected_initial_age = corrected_received_age + response_delay; - - if (http_GetHdr(hp, "Expires", &p)) { - h_expires = TIM_parse(p) + fudge; + ttd = t_req + retirement_age; } - - printf("Date: %d\n", h_date); - printf("Recv: %d\n", t_resp); - printf("Expires: %d\n", h_expires); - printf("Age: %d\n", h_age); - printf("CIAge: %d\n", corrected_initial_age); - printf("Max-Age: %d\n", max_age); - ttl = 0; - if (max_age >= 0) - ttl = t_resp + max_age - corrected_initial_age; - if (h_expires && h_expires < ttl) - ttl = h_expires; - if (ttl == 0) - ttl = t_resp + heritage.default_ttl; - printf("TTL: %d (%+d)\n", ttl, ttl - t_resp); - if (ttl < t_resp) - return (0); - - return (ttl); + VSL(SLT_Debug, 0, + "TTD: max-age %u Age: %u Date: %d (%d) Expires %d (%d) our_clock %d" + " -> ttd %d (%d)", + u1, u2, h_date, h_date - t_req, h_expires, h_expires - t_req, t_req, ttd, ttd - t_req); + return (ttd); } int From phk at phk.freebsd.dk Tue Jul 4 22:10:56 2006 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Tue, 04 Jul 2006 22:10:56 +0000 Subject: r305 - trunk/varnish-cache/bin/varnishd In-Reply-To: Your message of "Wed, 05 Jul 2006 00:08:33 +0200." <20060704220833.C81DF1EC30B@projects.linpro.no> Message-ID: <91860.1152051056@critter.freebsd.dk> In message <20060704220833.C81DF1EC30B at projects.linpro.no>, phk at projects.linpro .no writes: >Author: phk >Date: 2006-07-05 00:08:33 +0200 (Wed, 05 Jul 2006) >New Revision: 305 > >Modified: > trunk/varnish-cache/bin/varnishd/rfc2616.c >Log: >Implement our TTL calculation. > >A first quick check against the weird timestamps from the VG frontend >squids indicates sensible behaviour. The only instance which made me wonder is this one: 11 27 23 URL 12 8 24 Protocol 10 3 24 Status <200> 0f 2 24 Response 13 35 24 Header 13 28 24 Header 13 23 24 Header 13 28 24 Header 13 8 24 Header 13 27 24 Header 13 17 24 Header We get an Age: header, but no expires and no max-age, so we apply the default_ttl (120 seconds): 01 114 0 Debug ttd 1152050795 (120)> -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at projects.linpro.no Wed Jul 5 07:45:32 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 09:45:32 +0200 (CEST) Subject: r306 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705074532.C79291EC1F9@projects.linpro.no> Author: phk Date: 2006-07-05 09:45:32 +0200 (Wed, 05 Jul 2006) New Revision: 306 Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c Log: Be more conservative about wraparound and take them up front in all cases. Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-04 22:08:33 UTC (rev 305) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-05 07:45:32 UTC (rev 306) @@ -60,7 +60,7 @@ assert(loghead->ptr < loghead->size); /* Wrap if necessary */ - if (loghead->ptr + 5 + l > loghead->size) + if (loghead->ptr + 4 + l + 1 > loghead->size) vsl_wrap(); p = logstart + loghead->ptr; p[1] = l; @@ -88,29 +88,16 @@ AZ(pthread_mutex_lock(&vsl_mutex)); assert(loghead->ptr < loghead->size); - /* - * Wrap early if we approach the end - * 32 is arbitraryly larger than minimum of 5. - */ - if (loghead->ptr + 32 > loghead->size) + /* Wrap if we cannot fit a full size record */ + if (loghead->ptr + 4 + 255 + 1 > loghead->size) vsl_wrap(); p = logstart + loghead->ptr; n = 0; if (fmt != NULL) { - while (1) { - /* We need 4 four header + 1 for ENDMARKER */ - m = loghead->size - (loghead->ptr + 5); - if (m > 256) - m = 256; - n = vsnprintf((char *)p + 4, m, fmt, ap); - if (n >= 255) - n = 255; /* we truncate long fields */ - if (n < m) - break; - vsl_wrap(); - continue; /* Try again */ - } + n = vsnprintf((char *)(p + 4), 256, fmt, ap); + if (n > 255) + n = 255; /* we truncate long fields */ } p[1] = n; p[2] = id >> 8; @@ -151,7 +138,7 @@ struct shmloghead slh; int i; - heritage.vsl_fd = open(fn, O_RDWR | O_CREAT, 0600); + heritage.vsl_fd = open(fn, O_RDWR | O_CREAT, 0644); if (heritage.vsl_fd < 0) { fprintf(stderr, "Could not open %s: %s\n", fn, strerror(errno)); From des at projects.linpro.no Wed Jul 5 07:49:56 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Wed, 5 Jul 2006 09:49:56 +0200 (CEST) Subject: r307 - trunk/varnish-tools/fetcher Message-ID: <20060705074956.1673F1EC325@projects.linpro.no> Author: des Date: 2006-07-05 09:49:55 +0200 (Wed, 05 Jul 2006) New Revision: 307 Modified: trunk/varnish-tools/fetcher/fetcher.c Log: Refactoring. Modified: trunk/varnish-tools/fetcher/fetcher.c =================================================================== --- trunk/varnish-tools/fetcher/fetcher.c 2006-07-05 07:45:32 UTC (rev 306) +++ trunk/varnish-tools/fetcher/fetcher.c 2006-07-05 07:49:55 UTC (rev 307) @@ -13,17 +13,9 @@ #include #include -#define MAX_CTR 5000 +static int verbose; static char data[8192]; -static const char HEAD[] = "HEAD"; -static const char GET[] = "GET"; -static const char *method = GET; -static const char *req_pattern = -"%s /cgi-bin/recursor.pl?foo=%d HTTP/1.1\r\n" -"Host: varnish-test-1.linpro.no\r\n" -"Connection: Keep-Alive\r\n" -"\r\n"; static const char * read_line(FILE *f) @@ -44,33 +36,115 @@ } memcpy(buf, line, len); buf[len] = '\0'; -#ifdef DEBUG - fprintf(stderr, "<<< [%s]\n", buf); -#endif + if (verbose) + fprintf(stderr, "<<< [%s]\n", buf); return (buf); } +static int +open_socket(const char *host, const char *port) +{ + struct addrinfo hints, *res; + int error, sd; + + /* connect to accelerator */ + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + if ((error = getaddrinfo(host, port, &hints, &res)) != 0) + errx(1, "%s", gai_strerror(error)); + if ((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) < 0) + err(1, "socket()"); + if (connect(sd, res->ai_addr, res->ai_addrlen) < 0) + err(1, "connect()"); + return (sd); +} + +static const char HEAD[] = "HEAD"; +static const char GET[] = "GET"; + static void +send_request(FILE *f, const char *method, const char *host, const char *url) +{ + static const char *req_pattern = + "%s %s HTTP/1.1\r\n" + "Host: %s\r\n" + "Connection: Keep-Alive\r\n" + "\r\n"; + + /* send request */ + if (fprintf(f, req_pattern, method, url, host) < 0) + errx(1, "fprintf()"); + if (verbose) + fprintf(stderr, req_pattern, method, url, host); +} + +static void +receive_response(FILE *f, const char *method) +{ + const char *line; + size_t clen, rlen; + int code; + + /* get response header */ + if ((line = read_line(f)) == NULL) + errx(1, "protocol error"); + if (sscanf(line, "HTTP/%*d.%*d %d %*s", &code) != 1) + errx(1, "protocol error"); + if (code != 200) + errx(1, "code %d", code); + + /* get content-length */ + clen = 0; + for (;;) { + if ((line = read_line(f)) == NULL) + errx(1, "protocol error"); + if (line[0] == '\0') + break; + sscanf(line, "Content-Length: %zu\n", &clen); + } + + /* eat contents */ + if (method == HEAD) + return; + while (clen > 0) { + rlen = clen > sizeof(data) ? sizeof(data) : clen; + clen -= fread(data, 1, rlen, f); + } +} + +static void usage(void) { fprintf(stderr, "usage: fetcher [-h]\n"); exit(1); } +#define MAX_CTR 5000 + int main(int argc, char *argv[]) { - struct addrinfo hints, *res; - int code, ctr, error, opt, sd; - size_t clen; - const char *line; + char url[PATH_MAX]; + int i, opt, sd; FILE *f; - while ((opt = getopt(argc, argv, "h")) != -1) + const char *method = GET; + const char *host = "varnish-test-1.linpro.no"; + const char *url_pattern = "/cgi-bin/recursor.pl?foo=%d"; + int ctr = 500000; + + while ((opt = getopt(argc, argv, "c:hv")) != -1) switch (opt) { + case 'c': + ctr = atoi(optarg); + break; case 'h': method = HEAD; break; + case 'v': + verbose++; + break; default: usage(); } @@ -81,55 +155,15 @@ if (argc != 0) usage(); - /* connect to accelerator */ - memset(&hints, 0, sizeof(hints)); - hints.ai_family = AF_INET; - hints.ai_socktype = SOCK_STREAM; - if ((error = getaddrinfo("varnish-test-2.linpro.no", "http", &hints, &res)) != 0) - errx(1, "%s", gai_strerror(error)); - if ((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) < 0) - err(1, "socket()"); - if (connect(sd, res->ai_addr, res->ai_addrlen) < 0) - err(1, "connect()"); + sd = open_socket("varnish-test-2.linpro.no", "8080"); if ((f = fdopen(sd, "w+")) == NULL) err(1, "fdopen()"); - for (ctr = 0; ctr < 500000; ++ctr) { - - /* send request */ - fprintf(f, req_pattern, method, ctr % MAX_CTR); -#ifdef DEBUG - fprintf(stderr, req_pattern, method, ctr % MAX_CTR); -#else - if (ctr % 163 == 0) - fprintf(stderr, "\r%d ", ctr); -#endif - - /* get response header */ - if ((line = read_line(f)) == NULL) - errx(1, "protocol error"); - if (sscanf(line, "HTTP/%*d.%*d %d %*s", &code) != 1) - errx(1, "protocol error"); - if (code != 200) - errx(1, "code %d", code); - - /* get content-length */ - clen = 0; - for (;;) { - if ((line = read_line(f)) == NULL) - errx(1, "protocol error"); - if (line[0] == '\0') - break; - sscanf(line, "Content-Length: %zu\n", &clen); - } - - /* eat contents */ - if (method == HEAD) - continue; - while (clen > 0) { - size_t rlen = clen > sizeof(data) ? sizeof(data) : clen; - clen -= fread(data, 1, rlen, f); - } + for (i = 0; i < ctr; ++i) { + fprintf(stderr, "\r%d ", i); + snprintf(url, sizeof url, url_pattern, ctr % MAX_CTR); + send_request(f, method, host, url); + receive_response(f, method); } fclose(f); From des at projects.linpro.no Wed Jul 5 07:53:22 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Wed, 5 Jul 2006 09:53:22 +0200 (CEST) Subject: r308 - trunk/varnish-tools/fetcher Message-ID: <20060705075322.C4E691EC1F9@projects.linpro.no> Author: des Date: 2006-07-05 09:53:22 +0200 (Wed, 05 Jul 2006) New Revision: 308 Modified: trunk/varnish-tools/fetcher/fetcher.c Log: Use the correct variable when generating the URL. Modified: trunk/varnish-tools/fetcher/fetcher.c =================================================================== --- trunk/varnish-tools/fetcher/fetcher.c 2006-07-05 07:49:55 UTC (rev 307) +++ trunk/varnish-tools/fetcher/fetcher.c 2006-07-05 07:53:22 UTC (rev 308) @@ -160,8 +160,9 @@ err(1, "fdopen()"); for (i = 0; i < ctr; ++i) { - fprintf(stderr, "\r%d ", i); - snprintf(url, sizeof url, url_pattern, ctr % MAX_CTR); + if (i % 163 == 0) + fprintf(stderr, "\r%d ", i); + snprintf(url, sizeof url, url_pattern, i % MAX_CTR); send_request(f, method, host, url); receive_response(f, method); } From phk at projects.linpro.no Wed Jul 5 08:08:47 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 10:08:47 +0200 (CEST) Subject: r309 - trunk/varnish-cache/bin/varnishstat Message-ID: <20060705080847.096F71EC22A@projects.linpro.no> Author: phk Date: 2006-07-05 10:08:46 +0200 (Wed, 05 Jul 2006) New Revision: 309 Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c Log: Show also rate in curses display Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-07-05 07:53:22 UTC (rev 308) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-07-05 08:08:46 UTC (rev 309) @@ -10,6 +10,7 @@ #include #include #include +#include #include "shmlog.h" #include "varnishapi.h" @@ -19,8 +20,11 @@ { int c; struct shmloghead *lh; - struct varnish_stats *VSL_stats; + struct varnish_stats *VSL_stats, copy; int c_flag = 0; + uintmax_t ju; + struct timespec ts; + double tt, lt; lh = VSL_OpenLog(); @@ -38,15 +42,22 @@ } if (c_flag) { + memset(©, 0, sizeof copy); initscr(); erase(); while (1) { move(0,0); + clock_gettime(CLOCK_MONOTONIC, &ts); + tt = ts.tv_nsec * 1e-9 + ts.tv_sec; + lt = tt - lt; #define MAC_STAT(n,t,f,d) \ - printw("%12ju " d "\n", (VSL_stats->n)); + ju = VSL_stats->n; \ + printw("%12ju %10.2f " d "\n", ju, (ju - copy.n)/lt); \ + copy.n = ju; #include "stat_field.h" #undef MAC_STAT + lt = tt; refresh(); sleep(1); } From des at projects.linpro.no Wed Jul 5 08:13:01 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Wed, 5 Jul 2006 10:13:01 +0200 (CEST) Subject: r310 - trunk/varnish-tools/fetcher Message-ID: <20060705081301.7A6BE1EC22A@projects.linpro.no> Author: des Date: 2006-07-05 10:13:01 +0200 (Wed, 05 Jul 2006) New Revision: 310 Modified: trunk/varnish-tools/fetcher/fetcher.c Log: Add signal handling and statistics. Modified: trunk/varnish-tools/fetcher/fetcher.c =================================================================== --- trunk/varnish-tools/fetcher/fetcher.c 2006-07-05 08:08:46 UTC (rev 309) +++ trunk/varnish-tools/fetcher/fetcher.c 2006-07-05 08:13:01 UTC (rev 310) @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -113,7 +114,15 @@ } } +static volatile sig_atomic_t got_sig; + static void +handler(int sig) +{ + got_sig = sig; +} + +static void usage(void) { fprintf(stderr, "usage: fetcher [-h]\n"); @@ -125,6 +134,8 @@ int main(int argc, char *argv[]) { + struct timeval start, stop; + double elapsed; char url[PATH_MAX]; int i, opt, sd; FILE *f; @@ -159,14 +170,22 @@ if ((f = fdopen(sd, "w+")) == NULL) err(1, "fdopen()"); - for (i = 0; i < ctr; ++i) { - if (i % 163 == 0) - fprintf(stderr, "\r%d ", i); + got_sig = 0; + signal(SIGINT, handler); + signal(SIGTERM, handler); + gettimeofday(&start, NULL); + for (i = 0; i < ctr && !got_sig; ++i) { snprintf(url, sizeof url, url_pattern, i % MAX_CTR); send_request(f, method, host, url); receive_response(f, method); } + gettimeofday(&stop, NULL); fclose(f); + elapsed = (stop.tv_sec * 1000000.0 + stop.tv_usec) - + (start.tv_sec * 1000000.0 + start.tv_usec); + fprintf(stderr, "%d requests in %.3f seconds (%d rps)\n", + i, elapsed / 1000000, (int)(i / (elapsed / 1000000))); + exit(0); } From phk at projects.linpro.no Wed Jul 5 09:08:37 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 11:08:37 +0200 (CEST) Subject: r311 - trunk/varnish-cache/include Message-ID: <20060705090837.53DE31EC325@projects.linpro.no> Author: phk Date: 2006-07-05 11:08:37 +0200 (Wed, 05 Jul 2006) New Revision: 311 Modified: trunk/varnish-cache/include/shmlog_tags.h Log: Add Length tag Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-07-05 08:13:01 UTC (rev 310) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-07-05 09:08:37 UTC (rev 311) @@ -21,6 +21,7 @@ SLTM(Backend) SLTM(Request) SLTM(Response) +SLTM(Length) SLTM(Status) SLTM(URL) SLTM(Protocol) From phk at projects.linpro.no Wed Jul 5 09:10:30 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 11:10:30 +0200 (CEST) Subject: r312 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705091030.6DD921EC325@projects.linpro.no> Author: phk Date: 2006-07-05 11:10:30 +0200 (Wed, 05 Jul 2006) New Revision: 312 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/rfc2616.c Log: Log reponse code and object length Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-05 09:08:37 UTC (rev 311) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-05 09:10:30 UTC (rev 312) @@ -60,6 +60,8 @@ unsigned heap_idx; unsigned ban_seq; + unsigned response; + unsigned valid; unsigned cacheable; Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-05 09:08:37 UTC (rev 311) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-05 09:10:30 UTC (rev 312) @@ -63,7 +63,7 @@ while (cl != 0) { i = read(fd, p, cl); - assert(i > 0); + assert(i > 0); /* XXX seen */ p += i; cl -= i; } @@ -245,6 +245,8 @@ sp->obj->xid = sp->xid; fd = VBE_GetFd(sp->backend, &fd_token, sp->xid); + if (fd == -1) + fd = VBE_GetFd(sp->backend, &fd_token, sp->xid); assert(fd != -1); /* XXX: handle this */ VSL(SLT_Backend, sp->fd, "%d %s", fd, sp->backend->vcl_name); @@ -287,6 +289,9 @@ sbuf_finish(w->sb); sp->obj->header = strdup(sbuf_data(w->sb)); + VSL(SLT_Response, sp->fd, "%u", sp->obj->response); + VSL(SLT_Length, sp->fd, "%u", sp->obj->len); + vca_write_obj(sp, sp->obj->header, 0); if (http_GetHdr(hp, "Connection", &b) && !strcasecmp(b, "close")) Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-05 09:08:37 UTC (rev 311) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-05 09:10:30 UTC (rev 312) @@ -130,7 +130,8 @@ * Initial cacheability determination per [RFC2616, 13.4] * We do not support ranges yet, so 206 is out. */ - switch (http_GetStatus(hp)) { + sp->obj->response = http_GetStatus(hp); + switch (sp->obj->response) { case 200: /* OK */ sp->obj->valid = 1; /* FALLTHROUGH */ From phk at projects.linpro.no Wed Jul 5 09:11:02 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 11:11:02 +0200 (CEST) Subject: r313 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705091102.04A721EC325@projects.linpro.no> Author: phk Date: 2006-07-05 11:11:01 +0200 (Wed, 05 Jul 2006) New Revision: 313 Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c Log: log responsecode and length Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-05 09:10:30 UTC (rev 312) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-05 09:11:01 UTC (rev 313) @@ -50,6 +50,8 @@ { + VSL(SLT_Response, sp->fd, "%u", sp->obj->response); + VSL(SLT_Length, sp->fd, "%u", sp->obj->len); vca_write_obj(sp, sp->obj->header, 0); HSH_Deref(sp->obj); sp->obj = NULL; From phk at projects.linpro.no Wed Jul 5 09:32:13 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 11:32:13 +0200 (CEST) Subject: r314 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705093213.110A11EC22A@projects.linpro.no> Author: phk Date: 2006-07-05 11:32:12 +0200 (Wed, 05 Jul 2006) New Revision: 314 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Unify logging in the response handling Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-05 09:11:01 UTC (rev 313) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-05 09:32:12 UTC (rev 314) @@ -91,6 +91,9 @@ unsigned u = 0; char *r; + VSL(SLT_Response, sp->fd, "%u", sp->obj->response); + VSL(SLT_Length, sp->fd, "%u", sp->obj->len); + if (l == 0) l = strlen(b); vca_write(sp, b, l); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-05 09:11:01 UTC (rev 313) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-05 09:32:12 UTC (rev 314) @@ -289,9 +289,6 @@ sbuf_finish(w->sb); sp->obj->header = strdup(sbuf_data(w->sb)); - VSL(SLT_Response, sp->fd, "%u", sp->obj->response); - VSL(SLT_Length, sp->fd, "%u", sp->obj->len); - vca_write_obj(sp, sp->obj->header, 0); if (http_GetHdr(hp, "Connection", &b) && !strcasecmp(b, "close")) Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-05 09:11:01 UTC (rev 313) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-05 09:32:12 UTC (rev 314) @@ -50,8 +50,6 @@ { - VSL(SLT_Response, sp->fd, "%u", sp->obj->response); - VSL(SLT_Length, sp->fd, "%u", sp->obj->len); vca_write_obj(sp, sp->obj->header, 0); HSH_Deref(sp->obj); sp->obj = NULL; From phk at projects.linpro.no Wed Jul 5 09:44:53 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 11:44:53 +0200 (CEST) Subject: r315 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705094453.96A251EC325@projects.linpro.no> Author: phk Date: 2006-07-05 11:44:53 +0200 (Wed, 05 Jul 2006) New Revision: 315 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/rfc2616.c Log: Add Age and Via header to responses. Change arguments to vca_write_obj() (It should really be "send_repsonse()" ?) Store received age and time entered into cache in object. Generate Age: and Via: headers as part of response. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-05 09:32:12 UTC (rev 314) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-05 09:44:53 UTC (rev 315) @@ -67,6 +67,9 @@ unsigned busy; unsigned len; + + time_t age; + time_t entered; time_t ttl; char *header; @@ -133,7 +136,7 @@ /* cache_acceptor.c */ void vca_write(struct sess *sp, void *ptr, size_t len); -void vca_write_obj(struct sess *sp, char *b, unsigned l); +void vca_write_obj(struct worker *w, struct sess *sp); void vca_flush(struct sess *sp); void vca_return_session(struct sess *sp); void vca_close_session(struct sess *sp, const char *why); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-05 09:32:12 UTC (rev 314) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-05 09:44:53 UTC (rev 315) @@ -85,18 +85,25 @@ } void -vca_write_obj(struct sess *sp, char *b, unsigned l) +vca_write_obj(struct worker *w, struct sess *sp) { struct storage *st; unsigned u = 0; char *r; + VSL(SLT_Response, sp->fd, "%u", sp->obj->response); VSL(SLT_Length, sp->fd, "%u", sp->obj->len); - if (l == 0) - l = strlen(b); - vca_write(sp, b, l); + vca_write(sp, sp->obj->header, strlen(sp->obj->header)); + + sbuf_clear(w->sb); + sbuf_printf(w->sb, "Age: %u\r\n", + sp->obj->age + sp->t_req - sp->obj->entered); + sbuf_printf(w->sb, "Via: 1.1 varnish\r\n"); + sbuf_printf(w->sb, "\r\n"); + sbuf_finish(w->sb); + vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); assert(http_GetReq(sp->http, &r)); if (!strcmp(r, "GET")) { TAILQ_FOREACH(st, &sp->obj->store, list) { Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-05 09:32:12 UTC (rev 314) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-05 09:44:53 UTC (rev 315) @@ -285,11 +285,10 @@ sbuf_printf(w->sb, "Content-Length: %u\r\n", sp->obj->len); } else cls = 0; - sbuf_cat(w->sb, "\r\n"); sbuf_finish(w->sb); sp->obj->header = strdup(sbuf_data(w->sb)); - vca_write_obj(sp, sp->obj->header, 0); + vca_write_obj(w, sp); if (http_GetHdr(hp, "Connection", &b) && !strcasecmp(b, "close")) cls = 1; Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-05 09:32:12 UTC (rev 314) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-05 09:44:53 UTC (rev 315) @@ -50,7 +50,7 @@ { - vca_write_obj(sp, sp->obj->header, 0); + vca_write_obj(w, sp); HSH_Deref(sp->obj); sp->obj = NULL; return (1); @@ -139,6 +139,8 @@ return; } + time(&sp->t_req); + /* * No locking necessary, we're serialized in the acceptor thread */ Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-05 09:32:12 UTC (rev 314) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-05 09:44:53 UTC (rev 315) @@ -72,7 +72,7 @@ #endif static time_t -RFC2616_Ttl(struct http *hp, time_t t_req, time_t t_resp) +RFC2616_Ttl(struct http *hp, time_t t_req, time_t t_resp, struct object *obj) { int retirement_age; unsigned u1, u2; @@ -85,8 +85,10 @@ if (http_GetHdrField(hp, "Cache-Control", "max-age", &p)) { u1 = strtoul(p, NULL, 0); u2 = 0; - if (http_GetHdr(hp, "Age", &p)) + if (http_GetHdr(hp, "Age", &p)) { u2 = strtoul(p, NULL, 0); + obj->age = u2; + } if (u2 <= u1) retirement_age = u1 - u2; } @@ -148,7 +150,8 @@ break; } - sp->obj->ttl = RFC2616_Ttl(hp, sp->t_req, sp->t_resp); + sp->obj->ttl = RFC2616_Ttl(hp, sp->t_req, sp->t_resp, sp->obj); + sp->obj->entered = sp->t_req; if (sp->obj->ttl == 0) { sp->obj->cacheable = 0; } From phk at projects.linpro.no Wed Jul 5 09:56:35 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 11:56:35 +0200 (CEST) Subject: r316 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705095635.8546A1EC325@projects.linpro.no> Author: phk Date: 2006-07-05 11:56:35 +0200 (Wed, 05 Jul 2006) New Revision: 316 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/stevedore.h trunk/varnish-cache/bin/varnishd/storage_file.c Log: Send headers with sendfile Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-05 09:44:53 UTC (rev 315) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-05 09:56:35 UTC (rev 316) @@ -36,7 +36,7 @@ static pthread_t vca_thread; -#define SESS_IOVS 5 +#define SESS_IOVS 10 static struct event accept_e[2 * HERITAGE_NSOCKS]; @@ -108,10 +108,14 @@ if (!strcmp(r, "GET")) { TAILQ_FOREACH(st, &sp->obj->store, list) { u += st->len; - if (st->stevedore->send != NULL) - st->stevedore->send(st, sp); - else + if (st->stevedore->send == NULL) { vca_write(sp, st->ptr, st->len); + continue; + } + st->stevedore->send(st, sp, + sp->mem->iov, sp->mem->niov, sp->mem->liov); + sp->mem->niov = 0; + sp->mem->liov = 0; } assert(u == sp->obj->len); } Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2006-07-05 09:44:53 UTC (rev 315) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2006-07-05 09:56:35 UTC (rev 316) @@ -4,13 +4,14 @@ struct stevedore; struct sess; +struct iovec; typedef void storage_init_f(struct stevedore *, const char *spec); typedef void storage_open_f(struct stevedore *); typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); typedef void storage_trim_f(struct storage *, size_t size); typedef void storage_free_f(struct storage *); -typedef void storage_send_f(struct storage *, struct sess *); +typedef void storage_send_f(struct storage *, struct sess *, struct iovec *, int niovec, size_t liovec); struct stevedore { const char *name; Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-05 09:44:53 UTC (rev 315) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-05 09:56:35 UTC (rev 316) @@ -529,23 +529,26 @@ /*--------------------------------------------------------------------*/ static void -smf_send(struct storage *st, struct sess *sp) +smf_send(struct storage *st, struct sess *sp, struct iovec *iov, int niov, size_t liov) { struct smf *smf; int i; off_t sent; + struct sf_hdtr sfh; smf = st->priv; - vca_flush(sp); + memset(&sfh, 0, sizeof sfh); + sfh.headers = iov; + sfh.hdr_cnt = niov; i = sendfile(smf->sc->fd, sp->fd, smf->offset, - st->len, NULL, &sent, 0); - if (sent == st->len) + st->len, &sfh, &sent, 0); + if (sent == st->len + liov) return; - printf("sent i=%d sent=%ju size=%ju errno=%d\n", - i, (uintmax_t)sent, (uintmax_t)st->len, errno); + printf("sent i=%d sent=%ju size=%ju liov=%ju errno=%d\n", + i, (uintmax_t)sent, (uintmax_t)st->len, liov, errno); vca_close_session(sp, "remote closed"); } From phk at projects.linpro.no Wed Jul 5 09:59:33 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 11:59:33 +0200 (CEST) Subject: r317 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705095933.DF8841EC325@projects.linpro.no> Author: phk Date: 2006-07-05 11:59:33 +0200 (Wed, 05 Jul 2006) New Revision: 317 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c Log: Create an X-Varnish header and put the XID there. Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-05 09:56:35 UTC (rev 316) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-05 09:59:33 UTC (rev 317) @@ -101,6 +101,7 @@ sbuf_printf(w->sb, "Age: %u\r\n", sp->obj->age + sp->t_req - sp->obj->entered); sbuf_printf(w->sb, "Via: 1.1 varnish\r\n"); + sbuf_printf(w->sb, "X-Varnish: xid %u\r\n", sp->obj->xid); sbuf_printf(w->sb, "\r\n"); sbuf_finish(w->sb); vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); From phk at projects.linpro.no Wed Jul 5 10:01:04 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 12:01:04 +0200 (CEST) Subject: r318 - trunk/varnish-cache/include Message-ID: <20060705100104.BB9C01EC1F9@projects.linpro.no> Author: phk Date: 2006-07-05 12:01:04 +0200 (Wed, 05 Jul 2006) New Revision: 318 Modified: trunk/varnish-cache/include/http_headers.h Log: Suppress Accept-Ranges for now. Modified: trunk/varnish-cache/include/http_headers.h =================================================================== --- trunk/varnish-cache/include/http_headers.h 2006-07-05 09:59:33 UTC (rev 317) +++ trunk/varnish-cache/include/http_headers.h 2006-07-05 10:01:04 UTC (rev 318) @@ -21,7 +21,7 @@ HTTPH("Accept-Charset", H_Accept_Charset, 1, 0, 0, 0, 0) /* RFC2616 14.2 */ HTTPH("Accept-Encoding", H_Accept_Encoding, 1, 0, 0, 0, 0) /* RFC2616 14.3 */ HTTPH("Accept-Language", H_Accept_Language, 1, 0, 0, 0, 0) /* RFC2616 14.4 */ -HTTPH("Accept-Ranges", H_Accept_Ranges, 2, 0, 0, 0, 0) /* RFC2616 14.5 */ +HTTPH("Accept-Ranges", H_Accept_Ranges, 2, 3, 0, 0, 0) /* RFC2616 14.5 */ HTTPH("Age", H_Age, 2, 0, 0, 0, 0) /* RFC2616 14.6 */ HTTPH("Allow", H_Allow, 2, 0, 0, 0, 0) /* RFC2616 14.7 */ HTTPH("Authorization", H_Authorization, 1, 0, 0, 0, 0) /* RFC2616 14.8 */ From des at projects.linpro.no Wed Jul 5 10:39:09 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Wed, 5 Jul 2006 12:39:09 +0200 (CEST) Subject: r319 - trunk/varnish-tools/recursor Message-ID: <20060705103909.8B0801EC381@projects.linpro.no> Author: des Date: 2006-07-05 12:39:09 +0200 (Wed, 05 Jul 2006) New Revision: 319 Modified: trunk/varnish-tools/recursor/recursor.pl Log: Increase the period to 100000 documents Send CRLF instead of just LF Vary expiry from 300 to 1800 seconds Tack on 8 kB of blank lines Modified: trunk/varnish-tools/recursor/recursor.pl =================================================================== --- trunk/varnish-tools/recursor/recursor.pl 2006-07-05 10:01:04 UTC (rev 318) +++ trunk/varnish-tools/recursor/recursor.pl 2006-07-05 10:39:09 UTC (rev 319) @@ -9,11 +9,13 @@ my $q = new CGI; my $foo = int($q->param('foo')); -my $i = ($foo * 2) % 5000; -my $j = ($foo * 2 + 1) % 5000; +my $i = ($foo * 2) % 100000; +my $j = ($foo * 2 + 1) % 100000; -print $q->header(-expires=>'+60m'); +my $exp = ($foo % 1500) + 300; # 300 to 1800 seconds +print $q->header(-expires=>"+$exp"); -print "

Page $foo

\n"; -print "

Link $i

\n"; -print "

Link $j

\n"; +print "

Page $foo

\r\n"; +print "

Link $i

\r\n"; +print "

Link $j

\r\n"; +print "\r\n"x4096 From phk at projects.linpro.no Wed Jul 5 10:55:46 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 12:55:46 +0200 (CEST) Subject: r320 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705105546.7AA3B1EC381@projects.linpro.no> Author: phk Date: 2006-07-05 12:55:46 +0200 (Wed, 05 Jul 2006) New Revision: 320 Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c Log: Don't leak objects headers Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-05 10:39:09 UTC (rev 319) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-05 10:55:46 UTC (rev 320) @@ -123,6 +123,7 @@ return; assert(TAILQ_EMPTY(&oh->objects)); AZ(pthread_mutex_destroy(&oh->mtx)); + free(oh->header); free(oh); } From phk at projects.linpro.no Wed Jul 5 10:59:31 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 12:59:31 +0200 (CEST) Subject: r321 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705105931.5655A1EC381@projects.linpro.no> Author: phk Date: 2006-07-05 12:59:31 +0200 (Wed, 05 Jul 2006) New Revision: 321 Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c Log: Free the right header. Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-05 10:55:46 UTC (rev 320) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-05 10:59:31 UTC (rev 321) @@ -110,6 +110,7 @@ if (o == NULL) return; + free(o->header); AZ(pthread_cond_destroy(&o->cv)); TAILQ_FOREACH_SAFE(st, &o->store, list, stn) { @@ -123,7 +124,6 @@ return; assert(TAILQ_EMPTY(&oh->objects)); AZ(pthread_mutex_destroy(&oh->mtx)); - free(oh->header); free(oh); } From phk at projects.linpro.no Wed Jul 5 11:09:11 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 13:09:11 +0200 (CEST) Subject: r322 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705110911.AA8D11EC325@projects.linpro.no> Author: phk Date: 2006-07-05 13:09:11 +0200 (Wed, 05 Jul 2006) New Revision: 322 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c Log: Free the session memory correctly Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-05 10:59:31 UTC (rev 321) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-05 11:09:11 UTC (rev 322) @@ -242,7 +242,7 @@ } else { if (sp->http != NULL) http_Delete(sp->http); - free(sp); + free(sp->mem); } } From des at projects.linpro.no Wed Jul 5 11:40:24 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Wed, 5 Jul 2006 13:40:24 +0200 (CEST) Subject: r323 - trunk/varnish-tools/fetcher Message-ID: <20060705114024.446021EC325@projects.linpro.no> Author: des Date: 2006-07-05 13:40:24 +0200 (Wed, 05 Jul 2006) New Revision: 323 Modified: trunk/varnish-tools/fetcher/fetcher.c Log: Don't return zero if we caught a signal. Modified: trunk/varnish-tools/fetcher/fetcher.c =================================================================== --- trunk/varnish-tools/fetcher/fetcher.c 2006-07-05 11:09:11 UTC (rev 322) +++ trunk/varnish-tools/fetcher/fetcher.c 2006-07-05 11:40:24 UTC (rev 323) @@ -187,5 +187,5 @@ fprintf(stderr, "%d requests in %.3f seconds (%d rps)\n", i, elapsed / 1000000, (int)(i / (elapsed / 1000000))); - exit(0); + exit(got_sig); } From phk at projects.linpro.no Wed Jul 5 11:42:28 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 13:42:28 +0200 (CEST) Subject: r324 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705114228.332911EC381@projects.linpro.no> Author: phk Date: 2006-07-05 13:42:28 +0200 (Wed, 05 Jul 2006) New Revision: 324 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c Log: Don't lead http header Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-05 11:40:24 UTC (rev 323) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-05 11:42:28 UTC (rev 324) @@ -302,5 +302,7 @@ if (!sp->obj->cacheable) HSH_Deref(sp->obj); + http_Delete(hp); + return (1); } From des at projects.linpro.no Wed Jul 5 11:44:45 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Wed, 5 Jul 2006 13:44:45 +0200 (CEST) Subject: r325 - trunk/varnish-tools/fetcher Message-ID: <20060705114445.795A01EC325@projects.linpro.no> Author: des Date: 2006-07-05 13:44:45 +0200 (Wed, 05 Jul 2006) New Revision: 325 Modified: trunk/varnish-tools/fetcher/fetcher.c Log: Increase MAX_CTR to match recursor.pl, and set default count to 10 x MAX_CTR. Modified: trunk/varnish-tools/fetcher/fetcher.c =================================================================== --- trunk/varnish-tools/fetcher/fetcher.c 2006-07-05 11:42:28 UTC (rev 324) +++ trunk/varnish-tools/fetcher/fetcher.c 2006-07-05 11:44:45 UTC (rev 325) @@ -129,7 +129,7 @@ exit(1); } -#define MAX_CTR 5000 +#define MAX_CTR 100000 int main(int argc, char *argv[]) @@ -143,7 +143,7 @@ const char *method = GET; const char *host = "varnish-test-1.linpro.no"; const char *url_pattern = "/cgi-bin/recursor.pl?foo=%d"; - int ctr = 500000; + int ctr = MAX_CTR * 10; while ((opt = getopt(argc, argv, "c:hv")) != -1) switch (opt) { From phk at projects.linpro.no Wed Jul 5 12:17:49 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 14:17:49 +0200 (CEST) Subject: r326 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705121749.0B9F21EC381@projects.linpro.no> Author: phk Date: 2006-07-05 14:17:48 +0200 (Wed, 05 Jul 2006) New Revision: 326 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Another pipeline fix: don't clobber a pipelined partial header Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-05 11:44:45 UTC (rev 325) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-05 12:17:48 UTC (rev 326) @@ -377,11 +377,12 @@ func(arg, 1); return; } + } else { + hp->v = hp->s; + hp->t = hp->s; } hp->callback = func; hp->arg = arg; - hp->v = hp->s; - hp->t = hp->s; event_set(&hp->ev, fd, EV_READ | EV_PERSIST, http_read_f, hp); event_base_set(eb, &hp->ev); event_add(&hp->ev, NULL); /* XXX: timeout */ From phk at projects.linpro.no Wed Jul 5 13:09:01 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 15:09:01 +0200 (CEST) Subject: r327 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705130901.8FB6F1EC381@projects.linpro.no> Author: phk Date: 2006-07-05 15:09:01 +0200 (Wed, 05 Jul 2006) New Revision: 327 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: Some asserts to guard against trouble. Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-05 12:17:48 UTC (rev 326) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-05 13:09:01 UTC (rev 327) @@ -268,10 +268,9 @@ { struct smf *sp, *sp2; - TAILQ_FOREACH(sp, &sc->free, status) { + TAILQ_FOREACH(sp, &sc->free, status) if (sp->size >= bytes) break; - } if (sp == NULL) return (sp); @@ -478,6 +477,7 @@ size &= ~(sc->pagesize - 1); AZ(pthread_mutex_lock(&sc->mtx)); smf = alloc_smf(sc, size); + assert(smf->size == size); AZ(pthread_mutex_unlock(&sc->mtx)); assert(smf != NULL); smf->s.space = size; @@ -506,6 +506,7 @@ if (smf->size > size) { AZ(pthread_mutex_lock(&sc->mtx)); trim_smf(smf, size); + assert(smf->size == size); AZ(pthread_mutex_unlock(&sc->mtx)); smf->s.space = size; } From phk at projects.linpro.no Wed Jul 5 13:13:56 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 15:13:56 +0200 (CEST) Subject: r328 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060705131356.3819D1EC381@projects.linpro.no> Author: phk Date: 2006-07-05 15:13:56 +0200 (Wed, 05 Jul 2006) New Revision: 328 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/storage_file.c trunk/varnish-cache/include/stat_field.h Log: Add more stats Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-05 13:09:01 UTC (rev 327) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-05 13:13:56 UTC (rev 328) @@ -154,6 +154,7 @@ * XXX: this is probably one we should handle * XXX: accept, emit error NNN and close */ + VSL_stats->n_sess++; sp = &sm->s; sp->rd_e = &sm->e; @@ -242,6 +243,7 @@ } else { if (sp->http != NULL) http_Delete(sp->http); + VSL_stats->n_sess--; free(sp->mem); } } Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-05 13:09:01 UTC (rev 327) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-05 13:13:56 UTC (rev 328) @@ -90,10 +90,13 @@ memset(&hint, 0, sizeof hint); hint.ai_family = PF_UNSPEC; hint.ai_socktype = SOCK_STREAM; + res = NULL; error = getaddrinfo(bp->hostname, bp->portname == NULL ? "http" : bp->portname, &hint, &res); if (error) { + if (res != NULL) + freeaddrinfo(res); fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(error)); return; @@ -138,6 +141,7 @@ if (vc->fd < 0) { vc->vbe->nconn--; free(vc); + VSL_stats->n_vbe_conn--; } else { vc->inuse = 0; event_add(&vc->ev, NULL); @@ -172,6 +176,7 @@ event_del(&vc->ev); close(vc->fd); free(vc); + VSL_stats->n_vbe_conn--; } /* Backend monitoring thread -----------------------------------------*/ @@ -220,6 +225,7 @@ if (vp == NULL) { vp = calloc(sizeof *vp, 1); assert(vp != NULL); + VSL_stats->n_vbe++; TAILQ_INIT(&vp->fconn); TAILQ_INIT(&vp->bconn); vp->ip = bp->ip; @@ -235,6 +241,7 @@ AZ(pthread_mutex_unlock(&vbemtx)); } else { vc = calloc(sizeof *vc, 1); + VSL_stats->n_vbe_conn++; assert(vc != NULL); vc->vbe = vp; vc->fd = -1; Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-05 13:09:01 UTC (rev 327) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-05 13:13:56 UTC (rev 328) @@ -287,6 +287,7 @@ cls = 0; sbuf_finish(w->sb); sp->obj->header = strdup(sbuf_data(w->sb)); + VSL_stats->n_header++; vca_write_obj(w, sp); Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-05 13:09:01 UTC (rev 327) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-05 13:13:56 UTC (rev 328) @@ -33,6 +33,7 @@ assert(w->nobjhead != NULL); TAILQ_INIT(&w->nobjhead->objects); AZ(pthread_mutex_init(&w->nobjhead->mtx, NULL)); + VSL_stats->n_objecthead++; } if (w->nobj == NULL) { w->nobj = calloc(sizeof *w->nobj, 1); @@ -40,6 +41,7 @@ w->nobj->busy = 1; TAILQ_INIT(&w->nobj->store); AZ(pthread_cond_init(&w->nobj->cv, NULL)); + VSL_stats->n_object++; } assert(http_GetURL(h, &b)); @@ -110,7 +112,10 @@ if (o == NULL) return; - free(o->header); + if (o->header != NULL) { + free(o->header); + VSL_stats->n_header--; + } AZ(pthread_cond_destroy(&o->cv)); TAILQ_FOREACH_SAFE(st, &o->store, list, stn) { @@ -118,12 +123,14 @@ st->stevedore->free(st); } free(o); + VSL_stats->n_object--; /* Drop our ref on the objhead */ if (hash->deref(oh)) return; assert(TAILQ_EMPTY(&oh->objects)); AZ(pthread_mutex_destroy(&oh->mtx)); + VSL_stats->n_objecthead--; free(oh); } Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-05 13:09:01 UTC (rev 327) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-05 13:13:56 UTC (rev 328) @@ -53,6 +53,7 @@ hp = calloc(sizeof *hp, 1); assert(hp != NULL); + VSL_stats->n_http++; hp->s = malloc(http_bufsize); assert(hp->s != NULL); @@ -74,6 +75,7 @@ free(hp->hdr); free(hp->s); free(hp); + VSL_stats->n_http--; } /*--------------------------------------------------------------------*/ @@ -369,10 +371,12 @@ unsigned l; assert(hp != NULL); + VSL(SLT_Debug, fd, "Recv t %u v %u", hp->t - hp->s, hp->v - hp->s); if (hp->t > hp->s && hp->t < hp->v) { l = hp->v - hp->t; memmove(hp->s, hp->t, l); hp->v = hp->s + l; + hp->t = hp->s; if (http_header_complete(hp)) { func(arg, 1); return; Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-05 13:09:01 UTC (rev 327) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-05 13:13:56 UTC (rev 328) @@ -22,6 +22,7 @@ #include #include "libvarnish.h" +#include "shmlog.h" #include "cache.h" #define MINPAGES 128 @@ -284,6 +285,7 @@ /* Split from front */ sp2 = malloc(sizeof *sp2); assert(sp2 != NULL); + VSL_stats->n_smf++; *sp2 = *sp; sp->offset += bytes; @@ -320,6 +322,7 @@ TAILQ_REMOVE(&sc->order, sp2, order); TAILQ_REMOVE(&sc->free, sp2, status); free(sp2); + VSL_stats->n_smf--; } sp2 = TAILQ_PREV(sp, smfhead, order); @@ -331,6 +334,7 @@ sp2->age = sp->age; TAILQ_REMOVE(&sc->order, sp, order); free(sp); + VSL_stats->n_smf--; TAILQ_REMOVE(&sc->free, sp2, status); sp = sp2; } @@ -359,6 +363,7 @@ assert(bytes > 0); sp2 = malloc(sizeof *sp2); assert(sp2 != NULL); + VSL_stats->n_smf++; *sp2 = *sp; sp2->size -= bytes; @@ -381,6 +386,7 @@ sp = calloc(sizeof *sp, 1); assert(sp != NULL); + VSL_stats->n_smf++; sp->sc = sc; Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2006-07-05 13:09:01 UTC (rev 327) +++ trunk/varnish-cache/include/stat_field.h 2006-07-05 13:13:56 UTC (rev 328) @@ -9,3 +9,13 @@ MAC_STAT(backend_conn, uint64_t, "u", "Backend connections initiated") MAC_STAT(backend_recycle, uint64_t, "u", "Backend connections recyles") +MAC_STAT(n_sess, uint64_t, "u", "N struct sess"); +MAC_STAT(n_object, uint64_t, "u", "N struct object"); +MAC_STAT(n_objecthead, uint64_t, "u", "N struct objecthead"); +MAC_STAT(n_header, uint64_t, "u", "N struct header"); +MAC_STAT(n_smf, uint64_t, "u", "N struct smf"); +MAC_STAT(n_http, uint64_t, "u", "N struct http"); +MAC_STAT(n_vbe, uint64_t, "u", "N struct vbe"); +MAC_STAT(n_vbe_conn, uint64_t, "u", "N struct vbe_conn"); + + From phk at projects.linpro.no Wed Jul 5 13:19:14 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 15:19:14 +0200 (CEST) Subject: r329 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705131914.B76631EC381@projects.linpro.no> Author: phk Date: 2006-07-05 15:19:14 +0200 (Wed, 05 Jul 2006) New Revision: 329 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c Log: Send "Connection: close" if not HTTP/1.1 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-05 13:13:56 UTC (rev 328) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-05 13:19:14 UTC (rev 329) @@ -102,6 +102,8 @@ sp->obj->age + sp->t_req - sp->obj->entered); sbuf_printf(w->sb, "Via: 1.1 varnish\r\n"); sbuf_printf(w->sb, "X-Varnish: xid %u\r\n", sp->obj->xid); + if (http_GetProto(sp->http, &r) && strcmp(r, "HTTP/1.1")) + sbuf_printf(w->sb, "Connection: close\r\n"); sbuf_printf(w->sb, "\r\n"); sbuf_finish(w->sb); vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); From phk at projects.linpro.no Wed Jul 5 13:44:20 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 15:44:20 +0200 (CEST) Subject: r330 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705134420.5B6471EC1F9@projects.linpro.no> Author: phk Date: 2006-07-05 15:44:20 +0200 (Wed, 05 Jul 2006) New Revision: 330 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: Assert non-null first Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-05 13:19:14 UTC (rev 329) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-05 13:44:20 UTC (rev 330) @@ -483,9 +483,9 @@ size &= ~(sc->pagesize - 1); AZ(pthread_mutex_lock(&sc->mtx)); smf = alloc_smf(sc, size); - assert(smf->size == size); AZ(pthread_mutex_unlock(&sc->mtx)); assert(smf != NULL); + assert(smf->size == size); smf->s.space = size; smf->s.priv = smf; smf->s.ptr = smf->ptr; From phk at projects.linpro.no Wed Jul 5 13:54:41 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 15:54:41 +0200 (CEST) Subject: r331 - trunk/varnish-cache/bin/varnishstat Message-ID: <20060705135441.4CED21EC325@projects.linpro.no> Author: phk Date: 2006-07-05 15:54:41 +0200 (Wed, 05 Jul 2006) New Revision: 331 Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c Log: Calculate rate as signed. Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-07-05 13:44:20 UTC (rev 330) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-07-05 13:54:41 UTC (rev 331) @@ -22,7 +22,7 @@ struct shmloghead *lh; struct varnish_stats *VSL_stats, copy; int c_flag = 0; - uintmax_t ju; + intmax_t ju; struct timespec ts; double tt, lt; @@ -53,7 +53,7 @@ lt = tt - lt; #define MAC_STAT(n,t,f,d) \ ju = VSL_stats->n; \ - printw("%12ju %10.2f " d "\n", ju, (ju - copy.n)/lt); \ + printw("%12ju %10.2f " d "\n", ju, (ju - (intmax_t)copy.n)/lt); \ copy.n = ju; #include "stat_field.h" #undef MAC_STAT From des at projects.linpro.no Wed Jul 5 14:39:02 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Wed, 5 Jul 2006 16:39:02 +0200 (CEST) Subject: r332 - trunk/varnish-tools/fetcher Message-ID: <20060705143902.C7D611EC381@projects.linpro.no> Author: des Date: 2006-07-05 16:39:02 +0200 (Wed, 05 Jul 2006) New Revision: 332 Modified: trunk/varnish-tools/fetcher/fetcher.c Log: Add randomization and pipelining. Modified: trunk/varnish-tools/fetcher/fetcher.c =================================================================== --- trunk/varnish-tools/fetcher/fetcher.c 2006-07-05 13:54:41 UTC (rev 331) +++ trunk/varnish-tools/fetcher/fetcher.c 2006-07-05 14:39:02 UTC (rev 332) @@ -14,6 +14,7 @@ #include #include +static int random_order; static int verbose; static char data[8192]; @@ -64,6 +65,8 @@ static const char HEAD[] = "HEAD"; static const char GET[] = "GET"; +static int reqcount; + static void send_request(FILE *f, const char *method, const char *host, const char *url) { @@ -73,6 +76,8 @@ "Connection: Keep-Alive\r\n" "\r\n"; + reqcount++; + /* send request */ if (fprintf(f, req_pattern, method, url, host) < 0) errx(1, "fprintf()"); @@ -80,6 +85,8 @@ fprintf(stderr, req_pattern, method, url, host); } +static int respcount; + static void receive_response(FILE *f, const char *method) { @@ -87,6 +94,8 @@ size_t clen, rlen; int code; + respcount++; + /* get response header */ if ((line = read_line(f)) == NULL) errx(1, "protocol error"); @@ -110,7 +119,12 @@ return; while (clen > 0) { rlen = clen > sizeof(data) ? sizeof(data) : clen; - clen -= fread(data, 1, rlen, f); + rlen = fread(data, 1, rlen, f); + if (rlen == 0) + err(1, "fread()"); + if (verbose) + fprintf(stderr, "read %zu bytes\n", rlen); + clen -= rlen; } } @@ -137,22 +151,29 @@ struct timeval start, stop; double elapsed; char url[PATH_MAX]; - int i, opt, sd; + int opt, sd; FILE *f; const char *method = GET; const char *host = "varnish-test-1.linpro.no"; const char *url_pattern = "/cgi-bin/recursor.pl?foo=%d"; int ctr = MAX_CTR * 10; + int depth = 1; - while ((opt = getopt(argc, argv, "c:hv")) != -1) + while ((opt = getopt(argc, argv, "c:d:hrv")) != -1) switch (opt) { case 'c': ctr = atoi(optarg); break; + case 'd': + depth = atoi(optarg); + break; case 'h': method = HEAD; break; + case 'r': + random_order++; + break; case 'v': verbose++; break; @@ -166,6 +187,9 @@ if (argc != 0) usage(); + if (random_order) + srandomdev(); + sd = open_socket("varnish-test-2.linpro.no", "8080"); if ((f = fdopen(sd, "w+")) == NULL) err(1, "fdopen()"); @@ -174,9 +198,14 @@ signal(SIGINT, handler); signal(SIGTERM, handler); gettimeofday(&start, NULL); - for (i = 0; i < ctr && !got_sig; ++i) { - snprintf(url, sizeof url, url_pattern, i % MAX_CTR); - send_request(f, method, host, url); + while (respcount < ctr && !got_sig) { + while (reqcount < ctr && reqcount - respcount < depth && !got_sig) { + int serial = (random_order ? random() : reqcount) % MAX_CTR; + if (!verbose && (random_order || (reqcount % 29) == 0)) + fprintf(stderr, "\r%d ", serial); + snprintf(url, sizeof url, url_pattern, serial); + send_request(f, method, host, url); + } receive_response(f, method); } gettimeofday(&stop, NULL); @@ -185,7 +214,7 @@ elapsed = (stop.tv_sec * 1000000.0 + stop.tv_usec) - (start.tv_sec * 1000000.0 + start.tv_usec); fprintf(stderr, "%d requests in %.3f seconds (%d rps)\n", - i, elapsed / 1000000, (int)(i / (elapsed / 1000000))); + reqcount, elapsed / 1000000, (int)(reqcount / (elapsed / 1000000))); exit(got_sig); } From phk at projects.linpro.no Wed Jul 5 14:40:03 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 16:40:03 +0200 (CEST) Subject: r333 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705144003.69ACE1EC381@projects.linpro.no> Author: phk Date: 2006-07-05 16:40:03 +0200 (Wed, 05 Jul 2006) New Revision: 333 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: printf format Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-05 14:39:02 UTC (rev 332) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-05 14:40:03 UTC (rev 333) @@ -555,7 +555,7 @@ if (sent == st->len + liov) return; printf("sent i=%d sent=%ju size=%ju liov=%ju errno=%d\n", - i, (uintmax_t)sent, (uintmax_t)st->len, liov, errno); + i, (uintmax_t)sent, (uintmax_t)st->len, (uintmax_t)liov, errno); vca_close_session(sp, "remote closed"); } From phk at projects.linpro.no Wed Jul 5 14:40:14 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 5 Jul 2006 16:40:14 +0200 (CEST) Subject: r334 - trunk/varnish-cache/bin/varnishd Message-ID: <20060705144014.415061EC325@projects.linpro.no> Author: phk Date: 2006-07-05 16:40:14 +0200 (Wed, 05 Jul 2006) New Revision: 334 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Fix off by one Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-05 14:40:03 UTC (rev 333) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-05 14:40:14 UTC (rev 334) @@ -192,6 +192,7 @@ *b = hp->t; *e = hp->t + len; hp->t += len; + assert(hp->t <= hp->v); return (1); } @@ -217,6 +218,7 @@ ; VSLR(SLT_Request, fd, hp->req, p); *p++ = '\0'; + assert(!strcmp(hp->req, "GET")); /* Next find the URI */ while (isspace(*p)) @@ -293,6 +295,7 @@ VSLR(SLT_LostHeader, fd, p, q); } } + assert(hp->t <= hp->v); assert(hp->t == r); } @@ -316,7 +319,10 @@ continue; break; } - hp->t = ++p; + if (++p > hp->v) + return (0); + hp->t = p; + assert(hp->t <= hp->v); return (1); } @@ -371,6 +377,8 @@ unsigned l; assert(hp != NULL); + assert(hp->v <= hp->e); + assert(hp->t <= hp->v); VSL(SLT_Debug, fd, "Recv t %u v %u", hp->t - hp->s, hp->v - hp->s); if (hp->t > hp->s && hp->t < hp->v) { l = hp->v - hp->t; From des at projects.linpro.no Thu Jul 6 08:07:09 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Thu, 6 Jul 2006 10:07:09 +0200 (CEST) Subject: r335 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706080709.914F31EC35E@projects.linpro.no> Author: des Date: 2006-07-06 10:07:09 +0200 (Thu, 06 Jul 2006) New Revision: 335 Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c Log: Use instead of to get both the prototype for ioctl(2) and the definition of FIONREAD. Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-05 14:40:14 UTC (rev 334) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-06 08:07:09 UTC (rev 335) @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include From phk at projects.linpro.no Thu Jul 6 08:32:38 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 10:32:38 +0200 (CEST) Subject: r336 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706083238.A87A21EC382@projects.linpro.no> Author: phk Date: 2006-07-06 10:32:38 +0200 (Thu, 06 Jul 2006) New Revision: 336 Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c Log: 404 handling Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-06 08:07:09 UTC (rev 335) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-06 08:32:38 UTC (rev 336) @@ -135,13 +135,13 @@ sp->obj->response = http_GetStatus(hp); switch (sp->obj->response) { case 200: /* OK */ - sp->obj->valid = 1; - /* FALLTHROUGH */ case 203: /* Non-Authoritative Information */ case 300: /* Multiple Choices */ case 301: /* Moved Permanently */ case 410: /* Gone */ + case 404: /* Not Found */ sp->obj->cacheable = 1; + sp->obj->valid = 1; body = 1; break; default: @@ -157,6 +157,5 @@ } return (body); - } From des at projects.linpro.no Thu Jul 6 08:41:26 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Thu, 6 Jul 2006 10:41:26 +0200 (CEST) Subject: r337 - in trunk/varnish-cache: bin/varnishd lib/libvarnish Message-ID: <20060706084126.E86B41EC35E@projects.linpro.no> Author: des Date: 2006-07-06 10:41:26 +0200 (Thu, 06 Jul 2006) New Revision: 337 Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/mgt_child.c trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/lib/libvarnish/argv.c Log: Portability: cast unused parameters to void instead of marking them __unused. Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-06 08:32:38 UTC (rev 336) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-06 08:41:26 UTC (rev 337) @@ -129,11 +129,14 @@ */ static void -vbe_rdp(int fd, short event __unused, void *arg __unused) +vbe_rdp(int fd, short event, void *arg) { struct vbe_conn *vc; int i; + (void)event; + (void)arg; + i = read(fd, &vc, sizeof vc); assert(i == sizeof vc); AZ(pthread_mutex_lock(&vbemtx)); @@ -157,11 +160,13 @@ */ static void -vbe_rdf(int fd, short event __unused, void *arg) +vbe_rdf(int fd, short event, void *arg) { struct vbe_conn *vc; int j; + (void)event; + vc = arg; AZ(pthread_mutex_lock(&vbemtx)); if (vc->inuse) { @@ -182,10 +187,12 @@ /* Backend monitoring thread -----------------------------------------*/ static void * -vbe_main(void *priv __unused) +vbe_main(void *priv) { struct event pev; + (void)priv; + vbe_evb = event_init(); assert(vbe_evb != NULL); Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_child.c 2006-07-06 08:32:38 UTC (rev 336) +++ trunk/varnish-cache/bin/varnishd/mgt_child.c 2006-07-06 08:41:26 UTC (rev 337) @@ -156,9 +156,11 @@ } static void -cli_wrcb(struct bufferevent *bev __unused, void *arg __unused) +cli_wrcb(struct bufferevent *bev, void *arg) { + (void)bev; + (void)arg; } static void Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-06 08:32:38 UTC (rev 336) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-06 08:41:26 UTC (rev 337) @@ -50,9 +50,11 @@ } static void -m_cli_func_passthrough(struct cli *cli, char **av __unused, void *priv) +m_cli_func_passthrough(struct cli *cli, char **av, void *priv) { + (void)av; + cli_suspend(cli); mgt_child_request(cli_passthrough_cb, cli, &av[2], av[1]); } @@ -129,11 +131,13 @@ } static void -m_cli_func_config_inline(struct cli *cli, char **av, void *priv __unused) +m_cli_func_config_inline(struct cli *cli, char **av, void *priv) { char *vf; struct sbuf *sb; + (void)priv; + sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(sb != NULL); vf = VCC_Compile(sb, av[3], NULL); @@ -151,11 +155,13 @@ /* XXX: m prefix to avoid name clash */ static void -m_cli_func_config_load(struct cli *cli, char **av, void *priv __unused) +m_cli_func_config_load(struct cli *cli, char **av, void *priv) { char *vf; struct sbuf *sb; + (void)priv; + sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(sb != NULL); vf = VCC_CompileFile(sb, av[3]); @@ -194,36 +200,49 @@ /*--------------------------------------------------------------------*/ static void -m_cli_func_server_start(struct cli *cli, char **av __unused, void *priv __unused) +m_cli_func_server_start(struct cli *cli, char **av, void *priv) { + (void)cli; + (void)av; + (void)priv; + mgt_child_start(); } /*--------------------------------------------------------------------*/ static void -m_cli_func_server_stop(struct cli *cli, char **av __unused, void *priv __unused) +m_cli_func_server_stop(struct cli *cli, char **av, void *priv) { + (void)cli; + (void)av; + (void)priv; + mgt_child_stop(); } /*--------------------------------------------------------------------*/ static void -m_cli_func_verbose(struct cli *cli, char **av __unused, void *priv) +m_cli_func_verbose(struct cli *cli, char **av, void *priv) { + (void)av; + (void)priv; + cli->verbose = !cli->verbose; } static void -m_cli_func_ping(struct cli *cli, char **av, void *priv __unused) +m_cli_func_ping(struct cli *cli, char **av, void *priv) { time_t t; + (void)priv; + if (av[2] != NULL) { cli_out(cli, "Got your %s\n", av[2]); } @@ -234,9 +253,12 @@ /*--------------------------------------------------------------------*/ static void -m_cli_func_stats(struct cli *cli, char **av, void *priv __unused) +m_cli_func_stats(struct cli *cli, char **av, void *priv) { + (void)av; + (void)priv; + assert (VSL_stats != NULL); #define MAC_STAT(n,t,f,d) \ cli_out(cli, "%12ju " d "\n", (VSL_stats->n)); Modified: trunk/varnish-cache/lib/libvarnish/argv.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/argv.c 2006-07-06 08:32:38 UTC (rev 336) +++ trunk/varnish-cache/lib/libvarnish/argv.c 2006-07-06 08:41:26 UTC (rev 337) @@ -200,10 +200,13 @@ } int -main(int argc __unused, char **argv __unused) +main(int argc, char **argv) { char buf[BUFSIZ]; + (void)argc; + (void)argv; + register_printf_render_std("V"); while (fgets(buf, sizeof buf, stdin)) From des at projects.linpro.no Thu Jul 6 08:43:02 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Thu, 6 Jul 2006 10:43:02 +0200 (CEST) Subject: r338 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706084302.59B2E1EC35E@projects.linpro.no> Author: des Date: 2006-07-06 10:43:02 +0200 (Thu, 06 Jul 2006) New Revision: 338 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c Log: Portability: Linux does not have SO_NOSIGPIPE. Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-06 08:41:26 UTC (rev 337) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-06 08:43:02 UTC (rev 338) @@ -168,8 +168,10 @@ free(sp); return; } +#ifdef SO_NOSIGPIPE /* XXX Linux */ i = 1; AZ(setsockopt(sp->fd, SOL_SOCKET, SO_NOSIGPIPE, &i, sizeof i)); +#endif i = getnameinfo(addr, l, sp->addr, VCA_ADDRBUFSIZE, port, sizeof port, NI_NUMERICHOST | NI_NUMERICSERV); From des at projects.linpro.no Thu Jul 6 08:43:53 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Thu, 6 Jul 2006 10:43:53 +0200 (CEST) Subject: r339 - in trunk/varnish-cache: bin/varnishd lib/libvarnishapi Message-ID: <20060706084353.7E5521EC382@projects.linpro.no> Author: des Date: 2006-07-06 10:43:53 +0200 (Thu, 06 Jul 2006) New Revision: 339 Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c trunk/varnish-cache/bin/varnishd/storage_file.c trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Portability: don't use non-portable mmap(2) flags. Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-06 08:43:02 UTC (rev 338) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-06 08:43:53 UTC (rev 339) @@ -18,6 +18,14 @@ #include "heritage.h" +#ifndef MAP_HASSEMAPHORE +#define MAP_HASSEMAPHORE 0 /* XXX Linux */ +#endif + +#ifndef MAP_NOSYNC +#define MAP_NOSYNC 0 /* XXX Linux */ +#endif + struct varnish_stats *VSL_stats; static struct shmloghead *loghead; Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-06 08:43:02 UTC (rev 338) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-06 08:43:53 UTC (rev 339) @@ -25,6 +25,14 @@ #include "shmlog.h" #include "cache.h" +#ifndef MAP_NOCORE +#define MAP_NOCORE 0 /* XXX Linux */ +#endif + +#ifndef MAP_NOSYNC +#define MAP_NOSYNC 0 /* XXX Linux */ +#endif + #define MINPAGES 128 /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-06 08:43:02 UTC (rev 338) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-06 08:43:53 UTC (rev 339) @@ -13,6 +13,10 @@ #include "shmlog.h" #include "varnishapi.h" +#ifndef MAP_HASSEMAPHORE +#define MAP_HASSEMAPHORE 0 /* XXX Linux */ +#endif + static unsigned char *logstart, *logend; struct shmloghead * From phk at projects.linpro.no Thu Jul 6 08:45:23 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 10:45:23 +0200 (CEST) Subject: r340 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706084523.1E58E1EC382@projects.linpro.no> Author: phk Date: 2006-07-06 10:45:23 +0200 (Thu, 06 Jul 2006) New Revision: 340 Modified: trunk/varnish-cache/bin/varnishd/tcp.c Log: Add accept filters Modified: trunk/varnish-cache/bin/varnishd/tcp.c =================================================================== --- trunk/varnish-cache/bin/varnishd/tcp.c 2006-07-06 08:43:53 UTC (rev 339) +++ trunk/varnish-cache/bin/varnishd/tcp.c 2006-07-06 08:45:23 UTC (rev 340) @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -13,6 +14,21 @@ #include "libvarnish.h" static void +accept_filter(int fd) +{ + struct accept_filter_arg afa; + int i; + + bzero(&afa, sizeof(afa)); + strcpy(afa.af_name, "httpready"); + errno = 0; + i = setsockopt(fd, SOL_SOCKET, SO_ACCEPTFILTER, + &afa, sizeof(afa)); + printf("Acceptfilter(%d, httpready): %d %s\n", + fd, i, strerror(errno)); +} + +static void create_listen_socket(const char *addr, const char *port, int *sp, int nsp) { struct addrinfo ai, *r0, *r1; @@ -64,10 +80,14 @@ &heritage.sock_remote[0], HERITAGE_NSOCKS); for (u = 0; u < HERITAGE_NSOCKS; u++) { - if (heritage.sock_local[u] >= 0) + if (heritage.sock_local[u] >= 0) { AZ(listen(heritage.sock_local[u], 16)); - if (heritage.sock_remote[u] >= 0) + accept_filter(heritage.sock_local[u]); + } + if (heritage.sock_remote[u] >= 0) { AZ(listen(heritage.sock_remote[u], 16)); + accept_filter(heritage.sock_remote[u]); + } } return (0); } From phk at projects.linpro.no Thu Jul 6 08:45:38 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 10:45:38 +0200 (CEST) Subject: r341 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706084538.97B201EC1F9@projects.linpro.no> Author: phk Date: 2006-07-06 10:45:38 +0200 (Thu, 06 Jul 2006) New Revision: 341 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Increas shmem size to 8M Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-06 08:45:23 UTC (rev 340) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-06 08:45:38 UTC (rev 341) @@ -530,7 +530,7 @@ */ open_tcp(portnumber); - VSL_MgtInit(SHMLOG_FILENAME, 1024*1024); + VSL_MgtInit(SHMLOG_FILENAME, 8*1024*1024); testme(); From phk at projects.linpro.no Thu Jul 6 09:00:28 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 11:00:28 +0200 (CEST) Subject: r342 - trunk/varnish-cache/bin/varnishlog Message-ID: <20060706090028.1330F1EC382@projects.linpro.no> Author: phk Date: 2006-07-06 11:00:27 +0200 (Thu, 06 Jul 2006) New Revision: 342 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c Log: Add -r file and -w file options. These read/write from/to a binary file. Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-06 08:45:38 UTC (rev 341) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-06 09:00:27 UTC (rev 342) @@ -102,6 +102,12 @@ /*--------------------------------------------------------------------*/ +static void +Usage(void) +{ + fprintf(stderr, "Usage: varnishlog [-o] [-w file] [-r file]\n"); + exit(2); +} int main(int argc, char **argv) @@ -110,6 +116,11 @@ unsigned u; unsigned char *p, *q; int o_flag = 0; + char *w_opt = NULL; + FILE *wfile = NULL; + char *r_opt = NULL; + FILE *rfile = NULL; + unsigned char rbuf[255+4]; struct shmloghead *loghead; loghead = VSL_OpenLog(); @@ -117,25 +128,73 @@ for (i = 0; stagnames[i].tag != SLT_ENDMARKER; i++) tagnames[stagnames[i].tag] = stagnames[i].name; - while ((c = getopt(argc, argv, "o")) != -1) { + while ((c = getopt(argc, argv, "or:w:")) != -1) { switch (c) { case 'o': o_flag = 1; break; + case 'r': + r_opt = optarg; + break; + case 'w': + w_opt = optarg; + break; default: - fprintf(stderr, "Usage: varnishlog [-o]\n"); - exit (2); + Usage(); } } + if (r_opt != NULL && w_opt != NULL) + Usage(); + if (o_flag && w_opt != NULL) + Usage(); + + if (r_opt != NULL) { + rfile = fopen(r_opt, "r"); + if (rfile == NULL) + perror(r_opt); + u = 0; + } + if (w_opt != NULL) { + wfile = fopen(w_opt, "w"); + if (wfile == NULL) + perror(w_opt); + u = 0; + } + q = NULL; - while (VSL_NextLog(loghead, &q) != NULL) - ; + if (r_opt == NULL) { + while (VSL_NextLog(loghead, &q) != NULL) + ; + } while (1) { - p = VSL_NextLog(loghead, &q); - if (p == NULL) { - fflush(stdout); - usleep(50000); + if (r_opt == NULL) { + p = VSL_NextLog(loghead, &q); + if (p == NULL) { + if (w_opt == NULL) + fflush(stdout); + usleep(50000); + continue; + } + } else { + i = fread(rbuf, 4, 1, rfile); + if (i != 1) + break; + if (rbuf[1] > 0) + i = fread(rbuf + 4, rbuf[1], 1, rfile); + if (i != 1) + break; + p = rbuf; + } + if (wfile != NULL) { + i = fwrite(p, 4 + p[1], 1, wfile); + if (i != 1) + perror(w_opt); + u++; + if (!(u % 1000)) { + printf("%u\r", u); + fflush(stdout); + } continue; } if (o_flag) { From phk at projects.linpro.no Thu Jul 6 09:06:37 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 11:06:37 +0200 (CEST) Subject: r343 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706090637.086791EC382@projects.linpro.no> Author: phk Date: 2006-07-06 11:06:36 +0200 (Thu, 06 Jul 2006) New Revision: 343 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Remove debugging code. Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-06 09:00:27 UTC (rev 342) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-06 09:06:36 UTC (rev 343) @@ -218,7 +218,6 @@ ; VSLR(SLT_Request, fd, hp->req, p); *p++ = '\0'; - assert(!strcmp(hp->req, "GET")); /* Next find the URI */ while (isspace(*p)) From phk at projects.linpro.no Thu Jul 6 09:08:49 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 11:08:49 +0200 (CEST) Subject: r344 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706090849.E031D1EC382@projects.linpro.no> Author: phk Date: 2006-07-06 11:08:49 +0200 (Thu, 06 Jul 2006) New Revision: 344 Modified: trunk/varnish-cache/bin/varnishd/tcp.c Log: Only complain if accept_filters fail Modified: trunk/varnish-cache/bin/varnishd/tcp.c =================================================================== --- trunk/varnish-cache/bin/varnishd/tcp.c 2006-07-06 09:06:36 UTC (rev 343) +++ trunk/varnish-cache/bin/varnishd/tcp.c 2006-07-06 09:08:49 UTC (rev 344) @@ -24,8 +24,9 @@ errno = 0; i = setsockopt(fd, SOL_SOCKET, SO_ACCEPTFILTER, &afa, sizeof(afa)); - printf("Acceptfilter(%d, httpready): %d %s\n", - fd, i, strerror(errno)); + if (i) + printf("Acceptfilter(%d, httpready): %d %s\n", + fd, i, strerror(errno)); } static void From des at projects.linpro.no Thu Jul 6 09:13:54 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Thu, 6 Jul 2006 11:13:54 +0200 (CEST) Subject: r345 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706091354.F244B1EC35E@projects.linpro.no> Author: des Date: 2006-07-06 11:13:54 +0200 (Thu, 06 Jul 2006) New Revision: 345 Added: trunk/varnish-cache/bin/varnishd/varnishd.1 Modified: trunk/varnish-cache/bin/varnishd/Makefile.am Log: Add a rudimentary man page. Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-06 09:08:49 UTC (rev 344) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-06 09:13:54 UTC (rev 345) @@ -4,6 +4,8 @@ bin_PROGRAMS = varnishd +man_MANS = varnishd.1 + varnishd_SOURCES = \ cache_acceptor.c \ cache_backend.c \ Added: trunk/varnish-cache/bin/varnishd/varnishd.1 =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.1 2006-07-06 09:08:49 UTC (rev 344) +++ trunk/varnish-cache/bin/varnishd/varnishd.1 2006-07-06 09:13:54 UTC (rev 345) @@ -0,0 +1,72 @@ +.\" +.\" $Id$ +.\" +.Dd July 6, 2006 +.Dt VARNISHD 1 +.Os +.Sh NAME +.Nm varnishd +.Nd HTTP accelerator daemon +.Sh SYNOPSIS +.Nm +.Op Fl b Ar host Ns Op : Ns Ar port +.Op Fl d +.Op Fl f Ar config +.Op Fl h Ar type Ns Op , Ns Ar options +.Op Fl p Ar port +.Op Fl s Ar type Ns Op , Ns Ar options +.Op Fl t Ar ttl +.Op Fl w Ar min Ns Op , Ns Ar max +.Sh DESCRIPTION +The +.Nm +daemon accepts HTTP requests from clients, passes them on to a backend +server and caches the returned documents to better satisfy future +requests for the same document. +.Pp +The following options are available: +.Bl -tag -width Fl +.It Fl b Ar host Ns Op : Ns Ar port +Use the specified +.Ar host +as backend server. +If +.Ar port +is not specified, the default is 8080. +.It Fl d +Enables debugging mode. +.It Fl f Ar config +Specifies a VCL configuration file to use instead of the hardcoded +default. +.It Fl h Ar type Ns Op , Ns Ar options +Specifies the hash algorithm. +.It Fl p Ar port +Specifies which port to listen on for incoming client connections. +.It Fl s Ar type Ns Op , Ns Ar options +Specifies the storage backend. +.It Fl t Ar ttl +Specifies a hard minimum time to live for cached documents. +.It Fl w Ar min Ns Op , Ns Ar max +Specifies the number of worker threads to start. +If only +.Ar min +is specified, +.Nm +will start +.Ar min +threads. +If both +.Ar min +and +.Ar max +are specified, +.Nm +the number of worker threads will vary within the specified range +according to system load. +.El +.Sh HISTORY +The +.Nm +daemon was developed by +.An Poul-Henning Kamp Aq phk at freebsd.dk +in cooperation with Verdens Gang AS and Linpro AS. From des at projects.linpro.no Thu Jul 6 09:31:37 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Thu, 6 Jul 2006 11:31:37 +0200 (CEST) Subject: r346 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706093137.84E131EC382@projects.linpro.no> Author: des Date: 2006-07-06 11:31:37 +0200 (Thu, 06 Jul 2006) New Revision: 346 Modified: trunk/varnish-cache/bin/varnishd/varnishd.1 Log: Expand keywords Property changes on: trunk/varnish-cache/bin/varnishd/varnishd.1 ___________________________________________________________________ Name: svn:keywords + Id From des at projects.linpro.no Thu Jul 6 09:31:45 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Thu, 6 Jul 2006 11:31:45 +0200 (CEST) Subject: r347 - trunk/varnish-cache/bin/varnishlog Message-ID: <20060706093145.549C01EC382@projects.linpro.no> Author: des Date: 2006-07-06 11:31:45 +0200 (Thu, 06 Jul 2006) New Revision: 347 Added: trunk/varnish-cache/bin/varnishlog/varnishlog.1 Modified: trunk/varnish-cache/bin/varnishlog/Makefile.am Log: Add a rudimentary man page. Modified: trunk/varnish-cache/bin/varnishlog/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishlog/Makefile.am 2006-07-06 09:31:37 UTC (rev 346) +++ trunk/varnish-cache/bin/varnishlog/Makefile.am 2006-07-06 09:31:45 UTC (rev 347) @@ -4,6 +4,8 @@ bin_PROGRAMS = varnishlog +man_MANS = varnishlog.1 + varnishlog_SOURCES = varnishlog.c varnishlog_LDADD = \ Added: trunk/varnish-cache/bin/varnishlog/varnishlog.1 =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.1 2006-07-06 09:31:37 UTC (rev 346) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.1 2006-07-06 09:31:45 UTC (rev 347) @@ -0,0 +1,40 @@ +.\" +.\" $Id$ +.\" +.Dd July 6, 2006 +.Dt VARNISHLOG 1 +.Os +.Sh NAME +.Nm varnishlog +.Nd HTTP accelerator log watcher +.Sh SYNOPSIS +.Nm +.Op Fl o +.Op Fl r Ar file +.Op Fl w Ar file +.Sh DESCRIPTION +The +.Nm +utility reads and presents +.Xr varnishd 1 +shared memory logs. +.Pp +The following options are available: +.Bl -tag -width Fl +.It Fl o +Group log entries by request ID. +.It Fl r Ar file +Read log entries from +.Ar file +instead of shared memory. +.It Fl w Ar file +Write log entries to +.Ar file +instead of displaying them. +.El +.Sh HISTORY +The +.Nm +utility was developed by +.An Poul-Henning Kamp Aq phk at freebsd.dk +in cooperation with Verdens Gang AS and Linpro AS. Property changes on: trunk/varnish-cache/bin/varnishlog/varnishlog.1 ___________________________________________________________________ Name: svn:keywords + Id From phk at projects.linpro.no Thu Jul 6 10:18:33 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 12:18:33 +0200 (CEST) Subject: r348 - trunk/varnish-cache/bin/varnishstat Message-ID: <20060706101833.CC4281EC1F9@projects.linpro.no> Author: phk Date: 2006-07-06 12:18:33 +0200 (Thu, 06 Jul 2006) New Revision: 348 Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c Log: Add exponential hitrate displays (with -c) Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-07-06 09:31:45 UTC (rev 347) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-07-06 10:18:33 UTC (rev 348) @@ -15,6 +15,15 @@ #include "shmlog.h" #include "varnishapi.h" +static void +myexp(double *acc, double val, unsigned *n, unsigned nmax) +{ + + if (*n < nmax) + (*n)++; + (*acc) += (val - *acc) / (double)*n; +} + int main(int argc, char **argv) { @@ -24,8 +33,13 @@ int c_flag = 0; intmax_t ju; struct timespec ts; - double tt, lt; + double tt, lt, hit, miss, ratio; + double a1, a2, a3; + unsigned n1, n2, n3; + a1 = a2 = a3 = 0; + n1 = n2 = n3 = 0; + lh = VSL_OpenLog(); VSL_stats = &lh->stats; @@ -47,10 +61,26 @@ erase(); while (1) { - move(0,0); clock_gettime(CLOCK_MONOTONIC, &ts); tt = ts.tv_nsec * 1e-9 + ts.tv_sec; lt = tt - lt; + move(0,0); + hit = (intmax_t)VSL_stats->cache_hit - + (intmax_t)copy.cache_hit; + miss = (intmax_t)VSL_stats->cache_miss - + (intmax_t)copy.cache_miss; + hit /= lt; + miss /= lt; + if (hit + miss != 0) { + ratio = hit / (hit + miss); + myexp(&a1, ratio, &n1, 10); + myexp(&a2, ratio, &n2, 100); + myexp(&a3, ratio, &n3, 1000); + } + printw("Hitrate ratio: %8u %8u %8u\n", n1, n2, n3); + printw("Hitrate avg: %8.4f %8.4f %8.4f\n", a1, a2, a3); + printw("\n"); + #define MAC_STAT(n,t,f,d) \ ju = VSL_stats->n; \ printw("%12ju %10.2f " d "\n", ju, (ju - (intmax_t)copy.n)/lt); \ From phk at projects.linpro.no Thu Jul 6 11:18:34 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 13:18:34 +0200 (CEST) Subject: r349 - trunk/varnish-cache Message-ID: <20060706111834.34DD01EC35F@projects.linpro.no> Author: phk Date: 2006-07-06 13:18:34 +0200 (Thu, 06 Jul 2006) New Revision: 349 Added: trunk/varnish-cache/autogen.phk Log: My version. Added: trunk/varnish-cache/autogen.phk =================================================================== --- trunk/varnish-cache/autogen.phk 2006-07-06 10:18:33 UTC (rev 348) +++ trunk/varnish-cache/autogen.phk 2006-07-06 11:18:34 UTC (rev 349) @@ -0,0 +1,35 @@ +#!/bin/sh +# +# $Id: autogen.sh 118 2006-04-05 09:40:22Z phk $ +# + +set -ex + +if [ -d /usr/local/gnu-autotools/bin ] ; then + PATH=${PATH}:/usr/local/gnu-autotools/bin + export PATH +fi + +base=$(cd $(dirname $0) && pwd) +for dir in $base $base/contrib/libevent ; do + ( + echo $dir + cd $dir + aclocal + libtoolize --copy --force + aclocal + autoheader + automake --add-missing --copy --foreign + autoconf + ) +done + +sh configure \ + --enable-pedantic \ + --enable-wall \ + --enable-werror \ + --enable-dependency-tracking \ + "CFLAGS=-std=c99 -Wno-format" + +# This is a safety-measure during development +( cd lib/libvcl && ./*.tcl ) Property changes on: trunk/varnish-cache/autogen.phk ___________________________________________________________________ Name: svn:executable + * From des at projects.linpro.no Thu Jul 6 13:02:28 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Thu, 6 Jul 2006 15:02:28 +0200 (CEST) Subject: r350 - trunk/varnish-cache/bin/varnishstat Message-ID: <20060706130228.3F03E1EC362@projects.linpro.no> Author: des Date: 2006-07-06 15:02:28 +0200 (Thu, 06 Jul 2006) New Revision: 350 Added: trunk/varnish-cache/bin/varnishstat/varnishstat.1 Modified: trunk/varnish-cache/bin/varnishstat/Makefile.am Log: Add a rudimentary man page. Modified: trunk/varnish-cache/bin/varnishstat/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishstat/Makefile.am 2006-07-06 11:18:34 UTC (rev 349) +++ trunk/varnish-cache/bin/varnishstat/Makefile.am 2006-07-06 13:02:28 UTC (rev 350) @@ -4,6 +4,8 @@ bin_PROGRAMS = varnishstat +man_MANS = varnishstat.1 + varnishstat_SOURCES = varnishstat.c varnishstat_LDADD = \ Added: trunk/varnish-cache/bin/varnishstat/varnishstat.1 =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.1 2006-07-06 11:18:34 UTC (rev 349) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.1 2006-07-06 13:02:28 UTC (rev 350) @@ -0,0 +1,31 @@ +.\" +.\" $Id$ +.\" +.Dd July 6, 2006 +.Dt VARNISHSTAT 1 +.Os +.Sh NAME +.Nm varnishstat +.Nd HTTP accelerator statistics +.Sh SYNOPSIS +.Nm +.Op Fl c +.Sh DESCRIPTION +The +.Nm +utility displays statistics from a running +.Xr varnishd 1 +instance. +.Pp +The following options are available: +.Bl -tag -width Fl +.It Fl c +Present a continuously updating display using +.Xr curses 3 . +.El +.Sh HISTORY +The +.Nm +utility was developed by +.An Poul-Henning Kamp Aq phk at freebsd.dk +in cooperation with Verdens Gang AS and Linpro AS. Property changes on: trunk/varnish-cache/bin/varnishstat/varnishstat.1 ___________________________________________________________________ Name: svn:keywords + Id From phk at projects.linpro.no Thu Jul 6 13:27:22 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 15:27:22 +0200 (CEST) Subject: r351 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706132722.0BC6F1EC362@projects.linpro.no> Author: phk Date: 2006-07-06 15:27:21 +0200 (Thu, 06 Jul 2006) New Revision: 351 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Skip space before request/response Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-06 13:02:28 UTC (rev 350) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-06 13:27:21 UTC (rev 351) @@ -210,11 +210,13 @@ { char *p, *q, *r; + for (p = hp->s ; p < hp->v && isspace(*p); p++) + continue; assert(hp->t != NULL); if (rr == 1) { /* First, isolate and possibly identify request type */ - hp->req = hp->s; - for (p = hp->s; isalpha(*p); p++) + hp->req = p; + for (; isalpha(*p); p++) ; VSLR(SLT_Request, fd, hp->req, p); *p++ = '\0'; @@ -244,8 +246,7 @@ p++; } else { /* First, protocol */ - hp->proto = hp->s; - p = hp->s; + hp->proto = p; while (!isspace(*p)) p++; VSLR(SLT_Protocol, fd, hp->proto, p); @@ -295,6 +296,8 @@ } } assert(hp->t <= hp->v); + if (hp->t != r) + printf("hp->t %p r %p\n", hp->t, r); assert(hp->t == r); } @@ -305,7 +308,10 @@ { char *p; - p = hp->s; + for (p = hp->s ; p < hp->v && isspace(*p); p++) + continue; + if (p >= hp->v) + return (0); while (1) { /* XXX: we could save location of all linebreaks for later */ p = strchr(p, '\n'); From phk at projects.linpro.no Thu Jul 6 13:33:34 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 15:33:34 +0200 (CEST) Subject: r352 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706133334.75F3A1EC362@projects.linpro.no> Author: phk Date: 2006-07-06 15:33:34 +0200 (Thu, 06 Jul 2006) New Revision: 352 Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c Log: Fix braino in Pass handling Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-06 13:27:21 UTC (rev 351) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-06 13:33:34 UTC (rev 352) @@ -175,7 +175,7 @@ event_base_loop(w->eb, 0); http_Dissect(hp, fd, 2); - http_BuildSbuf(sp->fd, 2, w->sb, hp); + http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); if (http_GetHdr(hp, "Content-Length", &b)) From phk at projects.linpro.no Thu Jul 6 13:40:33 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 15:40:33 +0200 (CEST) Subject: r353 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706134033.590B81EC362@projects.linpro.no> Author: phk Date: 2006-07-06 15:40:33 +0200 (Thu, 06 Jul 2006) New Revision: 353 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: Silence typical broken client connection messages and move the interesting ones to shmem Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-06 13:33:34 UTC (rev 352) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-06 13:40:33 UTC (rev 353) @@ -562,9 +562,12 @@ st->len, &sfh, &sent, 0); if (sent == st->len + liov) return; - printf("sent i=%d sent=%ju size=%ju liov=%ju errno=%d\n", + vca_close_session(sp, "remote closed"); + if (errno == EPIPE || errno == ENOTCONN) + return; + VSL(SLT_Debug, sp->fd, + "sent i=%d sent=%ju size=%ju liov=%ju errno=%d\n", i, (uintmax_t)sent, (uintmax_t)st->len, (uintmax_t)liov, errno); - vca_close_session(sp, "remote closed"); } /*--------------------------------------------------------------------*/ From phk at projects.linpro.no Thu Jul 6 20:23:48 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 22:23:48 +0200 (CEST) Subject: r354 - trunk/varnish-cache/bin/varnishlog Message-ID: <20060706202348.3BA7E1EC362@projects.linpro.no> Author: phk Date: 2006-07-06 22:23:48 +0200 (Thu, 06 Jul 2006) New Revision: 354 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c Log: Improvements to flush things at timeout and at the end etc. Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-06 13:40:33 UTC (rev 353) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-06 20:23:48 UTC (rev 354) @@ -36,6 +36,20 @@ static struct sbuf *ob[65536]; +static void +clean_order() +{ + unsigned u; + + for (u = 0; u < 65536; u++) { + if (ob[u] == NULL) + continue; + sbuf_finish(ob[u]); + if (sbuf_len(ob[u])) + printf("%s\n", sbuf_data(ob[u])); + } +} + static void order(unsigned char *p) { @@ -113,7 +127,7 @@ main(int argc, char **argv) { int i, c; - unsigned u; + unsigned u, v; unsigned char *p, *q; int o_flag = 0; char *w_opt = NULL; @@ -150,7 +164,10 @@ Usage(); if (r_opt != NULL) { - rfile = fopen(r_opt, "r"); + if (!strcmp(r_opt, "-")) + rfile = stdin; + else + rfile = fopen(r_opt, "r"); if (rfile == NULL) perror(r_opt); u = 0; @@ -173,6 +190,10 @@ if (p == NULL) { if (w_opt == NULL) fflush(stdout); + else if (++v == 100) { + fflush(wfile); + printf("\nFlushed\n"); + } usleep(50000); continue; } @@ -187,6 +208,7 @@ p = rbuf; } if (wfile != NULL) { + v = 0; i = fwrite(p, 4 + p[1], 1, wfile); if (i != 1) perror(w_opt); @@ -208,4 +230,6 @@ fwrite(p + 4, p[1], 1, stdout); printf(">\n"); } + if (o_flag) + clean_order(); } From phk at projects.linpro.no Thu Jul 6 20:27:00 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 22:27:00 +0200 (CEST) Subject: r355 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706202700.1636C1EC362@projects.linpro.no> Author: phk Date: 2006-07-06 22:26:59 +0200 (Thu, 06 Jul 2006) New Revision: 355 Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c Log: The grim reaper needs to wait for objects refcount to drop to zero. Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-06 20:23:48 UTC (rev 354) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-06 20:26:59 UTC (rev 355) @@ -53,8 +53,13 @@ while (1) { time (&t); AZ(pthread_mutex_lock(&exp_mtx)); - o = TAILQ_FIRST(&exp_deathrow); - if (o == NULL || o->ttl >= t) { /* XXX: > or >= ? */ + TAILQ_FOREACH(o, &exp_deathrow, deathrow) { + if (o->ttl >= t) + break; + if (o->refcnt == 0) + break; + } + if (o == NULL || o->ttl >= t || o->refcnt > 0) { AZ(pthread_mutex_unlock(&exp_mtx)); sleep(1); continue; From phk at projects.linpro.no Thu Jul 6 20:29:26 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 22:29:26 +0200 (CEST) Subject: r356 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706202926.F234F1EC362@projects.linpro.no> Author: phk Date: 2006-07-06 22:29:26 +0200 (Thu, 06 Jul 2006) New Revision: 356 Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c Log: Make sure the grim reaper doesn't touch busy objects either. Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-06 20:26:59 UTC (rev 355) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-06 20:29:26 UTC (rev 356) @@ -56,6 +56,11 @@ TAILQ_FOREACH(o, &exp_deathrow, deathrow) { if (o->ttl >= t) break; + if (o->busy) { + VSL(SLT_Debug, 0, + "Grim Reaper: Busy object xid %u", o->xid); + continue; + } if (o->refcnt == 0) break; } From phk at projects.linpro.no Thu Jul 6 21:00:38 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 23:00:38 +0200 (CEST) Subject: r357 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706210038.64C821EC35F@projects.linpro.no> Author: phk Date: 2006-07-06 23:00:38 +0200 (Thu, 06 Jul 2006) New Revision: 357 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Strengthen HTTP parsing Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-06 20:29:26 UTC (rev 356) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-06 21:00:38 UTC (rev 357) @@ -210,9 +210,11 @@ { char *p, *q, *r; - for (p = hp->s ; p < hp->v && isspace(*p); p++) + assert(hp->t != NULL); + assert(hp->s < hp->t); + assert(hp->t <= hp->v); + for (p = hp->s ; isspace(*p); p++) continue; - assert(hp->t != NULL); if (rr == 1) { /* First, isolate and possibly identify request type */ hp->req = p; @@ -228,22 +230,25 @@ while (!isspace(*p)) p++; VSLR(SLT_URL, fd, hp->url, p); - *p++ = '\0'; + if (*p != '\n') { + *p++ = '\0'; - /* Finally, look for protocol, if any */ - while (isspace(*p) && *p != '\n') - p++; - hp->proto = p; - if (*p != '\n') { - while (!isspace(*p)) + /* Finally, look for protocol, if any */ + while (isspace(*p) && *p != '\n') p++; + if (*p != '\n') { + hp->proto = p; + while (!isspace(*p)) + p++; + if (*p != '\n') + *p++ = '\0'; + while (isspace(*p) && *p != '\n') + p++; + } } - VSLR(SLT_Protocol, fd, hp->proto, p); *p++ = '\0'; - - while (isspace(*p) && *p != '\n') - p++; - p++; + if (hp->proto != NULL) + VSLR(SLT_Protocol, fd, hp->proto, p); } else { /* First, protocol */ hp->proto = p; @@ -369,6 +374,7 @@ if (!http_header_complete(hp)) return; + assert(hp->t != NULL); event_del(&hp->ev); if (hp->callback != NULL) hp->callback(hp->arg, 1); @@ -391,6 +397,7 @@ hp->v = hp->s + l; hp->t = hp->s; if (http_header_complete(hp)) { + assert(func != NULL); func(arg, 1); return; } @@ -453,15 +460,21 @@ sbuf_cat(sb, hp->req); sbuf_cat(sb, " "); sbuf_cat(sb, hp->url); - sbuf_cat(sb, " "); - sbuf_cat(sb, hp->proto); + if (hp->proto != NULL) { + sbuf_cat(sb, " "); + sbuf_cat(sb, hp->proto); + } sup = 2; break; case Build_Fetch: sbuf_cat(sb, "GET "); sbuf_cat(sb, hp->url); - sbuf_cat(sb, " "); - sbuf_cat(sb, hp->proto); + if (hp->proto != NULL) { + sbuf_cat(sb, " "); + sbuf_cat(sb, hp->proto); + } else { + sbuf_cat(sb, " HTTP/1.1"); + } sup = 1; break; default: From phk at projects.linpro.no Thu Jul 6 21:03:43 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 23:03:43 +0200 (CEST) Subject: r358 - trunk/varnish-cache/bin/varnishlog Message-ID: <20060706210343.04A261EC35F@projects.linpro.no> Author: phk Date: 2006-07-06 23:03:42 +0200 (Thu, 06 Jul 2006) New Revision: 358 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c Log: Also flush ordered after the long timeout. Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-06 21:00:38 UTC (rev 357) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-06 21:03:42 UTC (rev 358) @@ -47,6 +47,7 @@ sbuf_finish(ob[u]); if (sbuf_len(ob[u])) printf("%s\n", sbuf_data(ob[u])); + sbuf_clear(ob[u]); } } @@ -188,9 +189,11 @@ if (r_opt == NULL) { p = VSL_NextLog(loghead, &q); if (p == NULL) { - if (w_opt == NULL) + if (w_opt == NULL) { + if (o_flag && ++v == 100) + clean_order(); fflush(stdout); - else if (++v == 100) { + } else if (++v == 100) { fflush(wfile); printf("\nFlushed\n"); } @@ -207,8 +210,8 @@ break; p = rbuf; } + v = 0; if (wfile != NULL) { - v = 0; i = fwrite(p, 4 + p[1], 1, wfile); if (i != 1) perror(w_opt); From phk at projects.linpro.no Thu Jul 6 21:04:14 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 23:04:14 +0200 (CEST) Subject: r359 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706210414.557781EC362@projects.linpro.no> Author: phk Date: 2006-07-06 23:04:14 +0200 (Thu, 06 Jul 2006) New Revision: 359 Added: trunk/varnish-cache/bin/varnishd/cache_response.c Modified: trunk/varnish-cache/bin/varnishd/Makefile.am Log: Add cache_response for yelling at clients Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-06 21:03:42 UTC (rev 358) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-06 21:04:14 UTC (rev 359) @@ -18,6 +18,7 @@ cache_pool.c \ cache_pass.c \ cache_pipe.c \ + cache_response.c \ cache_shmlog.c \ cache_vcl.c \ cache_vrt.c \ Added: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-06 21:03:42 UTC (rev 358) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-06 21:04:14 UTC (rev 359) @@ -0,0 +1,3 @@ +/* + * $Id$ + */ From phk at projects.linpro.no Thu Jul 6 21:47:51 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 23:47:51 +0200 (CEST) Subject: r360 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706214751.518861EC1F9@projects.linpro.no> Author: phk Date: 2006-07-06 23:47:51 +0200 (Thu, 06 Jul 2006) New Revision: 360 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_response.c Log: Refuse all requests without a protocol field with a 400 Implement a function to say "400" with. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-06 21:04:14 UTC (rev 359) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-06 21:47:51 UTC (rev 360) @@ -182,7 +182,7 @@ int http_GetTail(struct http *hp, unsigned len, char **b, char **e); int http_GetURL(struct http *hp, char **b); void http_RecvHead(struct http *hp, int fd, struct event_base *eb, http_callback_f *func, void *arg); -void http_Dissect(struct http *sp, int fd, int rr); +int http_Dissect(struct http *sp, int fd, int rr); enum http_build { Build_Pipe, Build_Pass, @@ -217,6 +217,9 @@ #endif extern struct varnish_stats *VSL_stats; +/* cache_response.c */ +void RES_Error(struct worker *w, struct sess *sp, int error, const char *msg); + /* cache_vcl.c */ void RelVCL(struct VCL_conf *vc); struct VCL_conf *GetVCL(void); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-06 21:04:14 UTC (rev 359) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-06 21:47:51 UTC (rev 360) @@ -265,7 +265,7 @@ http_RecvHead(hp, fd, w->eb, NULL, NULL); event_base_loop(w->eb, 0); time(&sp->t_resp); - http_Dissect(hp, fd, 2); + assert(http_Dissect(hp, fd, 2) == 0); body = RFC2616_cache_policy(sp, hp); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-06 21:04:14 UTC (rev 359) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-06 21:47:51 UTC (rev 360) @@ -205,7 +205,7 @@ /*--------------------------------------------------------------------*/ -void +int http_Dissect(struct http *hp, int fd, int rr) { char *p, *q, *r; @@ -216,7 +216,7 @@ for (p = hp->s ; isspace(*p); p++) continue; if (rr == 1) { - /* First, isolate and possibly identify request type */ + /* First, the request type (GET/HEAD etc) */ hp->req = p; for (; isalpha(*p); p++) ; @@ -224,31 +224,34 @@ *p++ = '\0'; /* Next find the URI */ - while (isspace(*p)) + while (isspace(*p) && *p != '\n') p++; + if (*p == '\n') + return (400); hp->url = p; while (!isspace(*p)) p++; VSLR(SLT_URL, fd, hp->url, p); - if (*p != '\n') { - *p++ = '\0'; + if (*p == '\n') + return (400); + *p++ = '\0'; - /* Finally, look for protocol, if any */ - while (isspace(*p) && *p != '\n') - p++; - if (*p != '\n') { - hp->proto = p; - while (!isspace(*p)) - p++; - if (*p != '\n') - *p++ = '\0'; - while (isspace(*p) && *p != '\n') - p++; - } - } + /* Finally, look for protocol */ + while (isspace(*p) && *p != '\n') + p++; + if (*p == '\n') + return (400); + hp->proto = p; + while (!isspace(*p)) + p++; + VSLR(SLT_Protocol, fd, hp->proto, p); + if (*p != '\n') + *p++ = '\0'; + while (isspace(*p) && *p != '\n') + p++; + if (*p != '\n') + return (400); *p++ = '\0'; - if (hp->proto != NULL) - VSLR(SLT_Protocol, fd, hp->proto, p); } else { /* First, protocol */ hp->proto = p; @@ -304,6 +307,7 @@ if (hp->t != r) printf("hp->t %p r %p\n", hp->t, r); assert(hp->t == r); + return (0); } /*--------------------------------------------------------------------*/ @@ -390,6 +394,7 @@ assert(hp != NULL); assert(hp->v <= hp->e); assert(hp->t <= hp->v); + if (0) VSL(SLT_Debug, fd, "Recv t %u v %u", hp->t - hp->s, hp->v - hp->s); if (hp->t > hp->s && hp->t < hp->v) { l = hp->v - hp->t; @@ -460,21 +465,15 @@ sbuf_cat(sb, hp->req); sbuf_cat(sb, " "); sbuf_cat(sb, hp->url); - if (hp->proto != NULL) { - sbuf_cat(sb, " "); - sbuf_cat(sb, hp->proto); - } + sbuf_cat(sb, " "); + sbuf_cat(sb, hp->proto); sup = 2; break; case Build_Fetch: sbuf_cat(sb, "GET "); sbuf_cat(sb, hp->url); - if (hp->proto != NULL) { - sbuf_cat(sb, " "); - sbuf_cat(sb, hp->proto); - } else { - sbuf_cat(sb, " HTTP/1.1"); - } + sbuf_cat(sb, " "); + sbuf_cat(sb, hp->proto); sup = 1; break; default: Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-06 21:04:14 UTC (rev 359) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-06 21:47:51 UTC (rev 360) @@ -84,7 +84,11 @@ sp->vcl = GetVCL(); AZ(pthread_mutex_unlock(&sessmtx)); - http_Dissect(sp->http, sp->fd, 1); + done = http_Dissect(sp->http, sp->fd, 1); + if (done != 0) { + RES_Error(&w, sp, done, NULL); + goto out; + } sp->backend = sp->vcl->backend[0]; @@ -121,6 +125,7 @@ vca_close_session(sp, "not HTTP/1.1"); } +out: AZ(pthread_mutex_lock(&sessmtx)); RelVCL(sp->vcl); sp->vcl = NULL; Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-06 21:04:14 UTC (rev 359) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-06 21:47:51 UTC (rev 360) @@ -1,3 +1,57 @@ /* * $Id$ */ + +#include /* XXX: for NULL ?? */ +#include +#include +#include + +#include "libvarnish.h" +#include "sbuf.h" +#include "cache.h" + + +/*--------------------------------------------------------------------*/ + +void +RES_Error(struct worker *w, struct sess *sp, int error, const char *msg) +{ + char buf[40]; + + if (msg == NULL) { + switch (error) { + case 400: msg = "Bad Request"; break; + default: msg = "HTTP request error"; break; + } + } + + sbuf_clear(w->sb); + sbuf_printf(w->sb, "HTTP/1.1 %03d %s\r\n", error, msg); + TIM_format(sp->t_req, buf); + sbuf_printf(w->sb, "Date: %s\r\n", buf); + sbuf_cat(w->sb, + "Server: Varnish\r\n" + "Connection: close\r\n" + "content-Type: text/html; charset=iso-8859-1\r\n" + "\r\n" + "\r\n" + "\r\n" + " \r\n"); + sbuf_printf(w->sb, " %03d %s\r\n", error, msg); + sbuf_cat(w->sb, + " \r\n" + " \r\n"); + sbuf_printf(w->sb, "

Error %03d %s

\r\n", error, msg); + sbuf_cat(w->sb, + " Your HTTP protocol request did not make sense.\r\n" + "

\r\n" + " \r\n" + " Varnish\r\n" + " \r\n" + "\r\n"); + sbuf_finish(w->sb); + vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); + vca_flush(sp); + vca_close_session(sp, msg); +} From phk at projects.linpro.no Thu Jul 6 21:56:43 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 23:56:43 +0200 (CEST) Subject: r361 - trunk/varnish-cache/include Message-ID: <20060706215643.9CC0F1EC35F@projects.linpro.no> Author: phk Date: 2006-07-06 23:56:43 +0200 (Thu, 06 Jul 2006) New Revision: 361 Modified: trunk/varnish-cache/include/shmlog_tags.h Log: Add shmem tag for Hits Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-07-06 21:47:51 UTC (rev 360) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-07-06 21:56:43 UTC (rev 361) @@ -32,6 +32,7 @@ SLTM(VCL_trace) SLTM(VCL_return) SLTM(XID) +SLTM(Hit) SLTM(ExpBan) SLTM(ExpPick) SLTM(ExpKill) From phk at projects.linpro.no Thu Jul 6 21:57:11 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 23:57:11 +0200 (CEST) Subject: r362 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706215711.0FE001EC35F@projects.linpro.no> Author: phk Date: 2006-07-06 23:57:10 +0200 (Thu, 06 Jul 2006) New Revision: 362 Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Log the found objects XID when we have a hash-hit. Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-06 21:56:43 UTC (rev 361) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-06 21:57:10 UTC (rev 362) @@ -40,6 +40,7 @@ VCL_miss_method(sp); } else { VSL_stats->cache_hit++; + VSL(SLT_Hit, sp->fd, "%u", o->xid); VCL_hit_method(sp); } return (0); From phk at projects.linpro.no Thu Jul 6 21:57:41 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 6 Jul 2006 23:57:41 +0200 (CEST) Subject: r363 - trunk/varnish-cache/bin/varnishd Message-ID: <20060706215741.208F01EC362@projects.linpro.no> Author: phk Date: 2006-07-06 23:57:41 +0200 (Thu, 06 Jul 2006) New Revision: 363 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c Log: Always log the numeric code as "Status" Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-06 21:57:10 UTC (rev 362) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-06 21:57:41 UTC (rev 363) @@ -92,7 +92,7 @@ char *r; - VSL(SLT_Response, sp->fd, "%u", sp->obj->response); + VSL(SLT_Status, sp->fd, "%u", sp->obj->response); VSL(SLT_Length, sp->fd, "%u", sp->obj->len); vca_write(sp, sp->obj->header, strlen(sp->obj->header)); From phk at projects.linpro.no Thu Jul 6 22:33:07 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 7 Jul 2006 00:33:07 +0200 (CEST) Subject: r364 - trunk/varnish-cache/include Message-ID: <20060706223307.97B9C1EC35F@projects.linpro.no> Author: phk Date: 2006-07-07 00:33:07 +0200 (Fri, 07 Jul 2006) New Revision: 364 Modified: trunk/varnish-cache/include/http_headers.h Log: Filter out Content-Range headers. Modified: trunk/varnish-cache/include/http_headers.h =================================================================== --- trunk/varnish-cache/include/http_headers.h 2006-07-06 21:57:41 UTC (rev 363) +++ trunk/varnish-cache/include/http_headers.h 2006-07-06 22:33:07 UTC (rev 364) @@ -32,7 +32,7 @@ HTTPH("Content-Length", H_Content_Length, 2, 2, 0, 0, 0) /* RFC2616 14.13 */ HTTPH("Content-Location", H_Content_Location, 2, 0, 0, 0, 0) /* RFC2616 14.14 */ HTTPH("Content-MD5", H_Content_MD5, 2, 0, 0, 0, 0) /* RFC2616 14.15 */ -HTTPH("Content-Range", H_Content_Range, 2, 0, 0, 0, 0) /* RFC2616 14.16 */ +HTTPH("Content-Range", H_Content_Range, 2, 3, 0, 0, 0) /* RFC2616 14.16 */ HTTPH("Content-Type", H_Content_Type, 2, 0, 0, 0, 0) /* RFC2616 14.17 */ HTTPH("Date", H_Date, 2, 0, 0, 0, 0) /* RFC2616 14.18 */ HTTPH("ETag", H_ETag, 2, 0, 0, 0, 0) /* RFC2616 14.19 */ From phk at projects.linpro.no Fri Jul 7 06:27:32 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 7 Jul 2006 08:27:32 +0200 (CEST) Subject: r365 - trunk/varnish-cache/bin/varnishlog Message-ID: <20060707062732.78D401EC385@projects.linpro.no> Author: phk Date: 2006-07-07 08:27:32 +0200 (Fri, 07 Jul 2006) New Revision: 365 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c Log: Strvis(3) debug data. Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-06 22:33:07 UTC (rev 364) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-07 06:27:32 UTC (rev 365) @@ -11,6 +11,7 @@ #include #include #include +#include #include "shmlog.h" #include "varnishapi.h" @@ -32,6 +33,18 @@ static const char *tagnames[256]; +static char * +vis_it(unsigned char *p) +{ + static char visbuf[255*4 + 3 + 1]; + + strcpy(visbuf, " ["); + strvisx(visbuf + 2, p + 4, p[1], + VIS_OCTAL | VIS_TAB | VIS_NL); + strcat(visbuf, "]"); + return (visbuf); +} + /* Ordering-----------------------------------------------------------*/ static struct sbuf *ob[65536]; @@ -83,6 +96,13 @@ sbuf_cat(ob[u], ">\n"); } break; + case SLT_Debug: + sbuf_printf(ob[u], "%02x %3d %4d %-12s", + p[0], p[1], u, tagnames[p[0]]); + if (p[1] > 0) + sbuf_cat(ob[u], vis_it(p)); + sbuf_cat(ob[u], "\n"); + break; default: sbuf_printf(ob[u], "%02x %3d %4d %-12s", p[0], p[1], u, tagnames[p[0]]); @@ -227,11 +247,20 @@ continue; } u = (p[2] << 8) | p[3]; - printf("%02x %3d %4d %-12s <", + printf("%02x %3d %4d %-12s", p[0], p[1], u, tagnames[p[0]]); - if (p[1] > 0) - fwrite(p + 4, p[1], 1, stdout); - printf(">\n"); + + if (p[1] > 0) { + if (p[0] != SLT_Debug) { + printf(" <"); + fwrite(p + 4, p[1], 1, stdout); + printf(">"); + } else { + fputs(vis_it(p), stdout); + } + + } + printf("\n"); } if (o_flag) clean_order(); From phk at projects.linpro.no Fri Jul 7 06:29:23 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 7 Jul 2006 08:29:23 +0200 (CEST) Subject: r366 - trunk/varnish-cache/bin/varnishd Message-ID: <20060707062923.44AC51EC385@projects.linpro.no> Author: phk Date: 2006-07-07 08:29:23 +0200 (Fri, 07 Jul 2006) New Revision: 366 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Dump our buffer as Debug if we have HTTP header trouble. Issue 400 for HTTP header buffer overflow. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-07 06:27:32 UTC (rev 365) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-07 06:29:23 UTC (rev 366) @@ -169,7 +169,7 @@ void HSH_Init(void); /* cache_http.c */ -typedef void http_callback_f(void *, int good); +typedef void http_callback_f(void *, int bad); struct http; struct http *http_New(void); void http_Delete(struct http *hp); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-07 06:27:32 UTC (rev 365) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-07 06:29:23 UTC (rev 366) @@ -226,21 +226,27 @@ /* Next find the URI */ while (isspace(*p) && *p != '\n') p++; - if (*p == '\n') + if (*p == '\n') { + VSLR(SLT_Debug, fd, hp->s, hp->v); return (400); + } hp->url = p; while (!isspace(*p)) p++; VSLR(SLT_URL, fd, hp->url, p); - if (*p == '\n') + if (*p == '\n') { + VSLR(SLT_Debug, fd, hp->s, hp->v); return (400); + } *p++ = '\0'; /* Finally, look for protocol */ while (isspace(*p) && *p != '\n') p++; - if (*p == '\n') + if (*p == '\n') { + VSLR(SLT_Debug, fd, hp->s, hp->v); return (400); + } hp->proto = p; while (!isspace(*p)) p++; @@ -249,8 +255,10 @@ *p++ = '\0'; while (isspace(*p) && *p != '\n') p++; - if (*p != '\n') + if (*p != '\n') { + VSLR(SLT_Debug, fd, hp->s, hp->v); return (400); + } *p++ = '\0'; } else { /* First, protocol */ @@ -354,6 +362,14 @@ int i; l = hp->e - hp->v; + if (l <= 1) { + VSLR(SLT_Debug, fd, hp->s, hp->v); + hp->t = NULL; + event_del(&hp->ev); + if (hp->callback != NULL) + hp->callback(hp->arg, 1); + return; + } assert(l > 1); errno = 0; i = read(fd, hp->v, l - 1); @@ -366,10 +382,11 @@ VSL(SLT_HttpError, fd, "Received nothing"); else VSL(SLT_HttpError, fd, "Received errno %d", errno); + VSLR(SLT_Debug, fd, hp->s, hp->v); hp->t = NULL; event_del(&hp->ev); if (hp->callback != NULL) - hp->callback(hp->arg, 0); + hp->callback(hp->arg, 2); return; } @@ -381,7 +398,7 @@ assert(hp->t != NULL); event_del(&hp->ev); if (hp->callback != NULL) - hp->callback(hp->arg, 1); + hp->callback(hp->arg, 0); } /*--------------------------------------------------------------------*/ @@ -403,7 +420,7 @@ hp->t = hp->s; if (http_header_complete(hp)) { assert(func != NULL); - func(arg, 1); + func(arg, 0); return; } } else { Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-07 06:27:32 UTC (rev 365) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-07 06:29:23 UTC (rev 366) @@ -135,12 +135,15 @@ } void -DealWithSession(void *arg, int good) +DealWithSession(void *arg, int bad) { struct sess *sp = arg; - if (!good) { - vca_close_session(sp, "no request"); + if (bad) { + if (bad == 1) + vca_close_session(sp, "overflow"); + else + vca_close_session(sp, "no request"); vca_return_session(sp); return; } From phk at projects.linpro.no Fri Jul 7 07:15:33 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 7 Jul 2006 09:15:33 +0200 (CEST) Subject: r367 - trunk/varnish-cache/bin/varnishd Message-ID: <20060707071533.D7A561EC1F6@projects.linpro.no> Author: phk Date: 2006-07-07 09:15:33 +0200 (Fri, 07 Jul 2006) New Revision: 367 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Time idle TCP connections out after 30 seconds Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-07 06:29:23 UTC (rev 366) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-07 07:15:33 UTC (rev 367) @@ -202,7 +202,7 @@ /* cache_pool.c */ void CacheInitPool(void); -void DealWithSession(void *arg, int good); +void DealWithSession(void *arg); /* cache_shmlog.c */ void VSL_Init(void); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-07 06:29:23 UTC (rev 366) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-07 07:15:33 UTC (rev 367) @@ -34,11 +34,15 @@ static struct event pipe_e; static int pipes[2]; +static struct event tick_e; +static struct timeval tick_rate; + static pthread_t vca_thread; #define SESS_IOVS 10 static struct event accept_e[2 * HERITAGE_NSOCKS]; +static TAILQ_HEAD(,sess) sesshead = TAILQ_HEAD_INITIALIZER(sesshead); struct sessmem { struct sess s; @@ -128,6 +132,41 @@ /*--------------------------------------------------------------------*/ static void +vca_tick(int a, short b, void *c) +{ + struct sess *sp, *sp2; + time_t t; + + printf("vca_tick\n"); + evtimer_add(&tick_e, &tick_rate); + time(&t); + TAILQ_FOREACH_SAFE(sp, &sesshead, list, sp2) { + if (sp->t_resp + 30 < t) { + TAILQ_REMOVE(&sesshead, sp, list); + vca_close_session(sp, "timeout"); + vca_return_session(sp); + } + } +} + +static void +vca_callback(void *arg, int bad) +{ + struct sess *sp = arg; + + TAILQ_REMOVE(&sesshead, sp, list); + if (bad) { + if (bad == 1) + vca_close_session(sp, "overflow"); + else + vca_close_session(sp, "no request"); + vca_return_session(sp); + return; + } + DealWithSession(sp); +} + +static void pipe_f(int fd, short event, void *arg) { struct sess *sp; @@ -135,7 +174,9 @@ i = read(fd, &sp, sizeof sp); assert(i == sizeof sp); - http_RecvHead(sp->http, sp->fd, evb, DealWithSession, sp); + time(&sp->t_resp); + TAILQ_INSERT_TAIL(&sesshead, sp, list); + http_RecvHead(sp->http, sp->fd, evb, vca_callback, sp); } static void @@ -182,8 +223,10 @@ strlcat(sp->addr, " ", VCA_ADDRBUFSIZE); strlcat(sp->addr, port, VCA_ADDRBUFSIZE); VSL(SLT_SessionOpen, sp->fd, "%s", sp->addr); + time(&sp->t_resp); + TAILQ_INSERT_TAIL(&sesshead, sp, list); sp->http = http_New(); - http_RecvHead(sp->http, sp->fd, evb, DealWithSession, sp); + http_RecvHead(sp->http, sp->fd, evb, vca_callback, sp); } static void * @@ -199,6 +242,11 @@ event_base_set(evb, &pipe_e); event_add(&pipe_e, NULL); + evtimer_set(&tick_e, vca_tick, NULL); + event_base_set(evb, &tick_e); + + evtimer_add(&tick_e, &tick_rate); + ep = accept_e; for (u = 0; u < HERITAGE_NSOCKS; u++) { if (heritage.sock_local[u] >= 0) { @@ -258,5 +306,7 @@ VCA_Init(void) { + tick_rate.tv_sec = 1; + tick_rate.tv_usec = 0; AZ(pthread_create(&vca_thread, NULL, vca_main, NULL)); } Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-07 06:29:23 UTC (rev 366) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-07 07:15:33 UTC (rev 367) @@ -135,19 +135,10 @@ } void -DealWithSession(void *arg, int bad) +DealWithSession(void *arg) { struct sess *sp = arg; - if (bad) { - if (bad == 1) - vca_close_session(sp, "overflow"); - else - vca_close_session(sp, "no request"); - vca_return_session(sp); - return; - } - time(&sp->t_req); /* From phk at projects.linpro.no Fri Jul 7 07:22:01 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 7 Jul 2006 09:22:01 +0200 (CEST) Subject: r368 - trunk/varnish-cache/bin/varnishd Message-ID: <20060707072201.C31591EC389@projects.linpro.no> Author: phk Date: 2006-07-07 09:22:01 +0200 (Fri, 07 Jul 2006) New Revision: 368 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c Log: Remove debugging printf Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-07 07:15:33 UTC (rev 367) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-07 07:22:01 UTC (rev 368) @@ -137,7 +137,6 @@ struct sess *sp, *sp2; time_t t; - printf("vca_tick\n"); evtimer_add(&tick_e, &tick_rate); time(&t); TAILQ_FOREACH_SAFE(sp, &sesshead, list, sp2) { From phk at projects.linpro.no Fri Jul 7 07:22:15 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 7 Jul 2006 09:22:15 +0200 (CEST) Subject: r369 - trunk/varnish-cache/bin/varnishd Message-ID: <20060707072215.DC8CE1EC1F6@projects.linpro.no> Author: phk Date: 2006-07-07 09:22:15 +0200 (Fri, 07 Jul 2006) New Revision: 369 Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c Log: Fix pipe mode Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-07 07:22:01 UTC (rev 368) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-07 07:22:15 UTC (rev 369) @@ -47,24 +47,18 @@ int fd, i; void *fd_token; struct edir e1, e2; + char *b, *e; fd = VBE_GetFd(sp->backend, &fd_token, sp->xid); assert(fd != -1); - http_BuildSbuf(fd, Build_Pipe, w->sb, sp->http); /* XXX: 0 ?? */ + http_BuildSbuf(fd, Build_Pipe, w->sb, sp->http); i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); - assert(__LINE__ == 0); -#if 0 - { int j; - j = 0; - i = sp->rcv_len - sp->rcv_ptr; - if (i > 0) { - j = write(sp->fd, sp->rcv + sp->rcv_ptr, i); - assert(j == i); + if (http_GetTail(sp->http, 99999999, &b, &e) && b != e) { /* XXX */ + i = write(fd, b, e - b); + assert(i == e - b); } - } -#endif e1.fd = fd; e2.fd = sp->fd; From phk at projects.linpro.no Fri Jul 7 07:22:35 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 7 Jul 2006 09:22:35 +0200 (CEST) Subject: r370 - trunk/varnish-cache/bin/varnishd Message-ID: <20060707072235.19B0E1EC389@projects.linpro.no> Author: phk Date: 2006-07-07 09:22:34 +0200 (Fri, 07 Jul 2006) New Revision: 370 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Don't filter headers in Pipe mode Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-07 07:22:15 UTC (rev 369) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-07 07:22:34 UTC (rev 370) @@ -478,6 +478,13 @@ sup = 2; break; case Build_Pipe: + sbuf_cat(sb, hp->req); + sbuf_cat(sb, " "); + sbuf_cat(sb, hp->url); + sbuf_cat(sb, " "); + sbuf_cat(sb, hp->proto); + sup = 0; + break; case Build_Pass: sbuf_cat(sb, hp->req); sbuf_cat(sb, " "); From phk at projects.linpro.no Fri Jul 7 07:22:56 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 7 Jul 2006 09:22:56 +0200 (CEST) Subject: r371 - trunk/varnish-cache/bin/varnishd Message-ID: <20060707072256.09C4D1EC1F9@projects.linpro.no> Author: phk Date: 2006-07-07 09:22:55 +0200 (Fri, 07 Jul 2006) New Revision: 371 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Pipe anything different from HEAD & GET Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-07 07:22:34 UTC (rev 370) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-07 07:22:55 UTC (rev 371) @@ -82,7 +82,7 @@ "}\n" "sub vcl_recv {\n" " if (req.request != \"GET\" && req.request != \"HEAD\") {\n" - " pass;\n" + " pipe;\n" " }\n" " if (req.http.Authenticate || req.http.Cookie) {\n" " pass;\n" From phk at projects.linpro.no Fri Jul 7 07:25:38 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 7 Jul 2006 09:25:38 +0200 (CEST) Subject: r372 - trunk/varnish-cache/bin/varnishd Message-ID: <20060707072538.BA3B61EC389@projects.linpro.no> Author: phk Date: 2006-07-07 09:25:38 +0200 (Fri, 07 Jul 2006) New Revision: 372 Modified: trunk/varnish-cache/bin/varnishd/tcp.c Log: Tell why bind(2) fails Modified: trunk/varnish-cache/bin/varnishd/tcp.c =================================================================== --- trunk/varnish-cache/bin/varnishd/tcp.c 2006-07-07 07:22:55 UTC (rev 371) +++ trunk/varnish-cache/bin/varnishd/tcp.c 2006-07-07 07:25:38 UTC (rev 372) @@ -55,6 +55,10 @@ assert(i == 0); i = bind(s, r1->ai_addr, r1->ai_addrlen); + if (i != 0) { + perror("bind"); + continue; + } assert(i == 0); *sp = s; sp++; From phk at projects.linpro.no Fri Jul 7 07:48:49 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 7 Jul 2006 09:48:49 +0200 (CEST) Subject: r373 - trunk/varnish-cache/bin/varnishd Message-ID: <20060707074849.B2BDF1EC389@projects.linpro.no> Author: phk Date: 2006-07-07 09:48:49 +0200 (Fri, 07 Jul 2006) New Revision: 373 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c Log: Set SO_LINGER to zero Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-07 07:25:38 UTC (rev 372) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-07 07:48:49 UTC (rev 373) @@ -187,6 +187,7 @@ struct sess *sp; char port[NI_MAXSERV]; int i; + struct linger linger; VSL_stats->client_conn++; @@ -212,6 +213,12 @@ i = 1; AZ(setsockopt(sp->fd, SOL_SOCKET, SO_NOSIGPIPE, &i, sizeof i)); #endif +#ifdef SO_LINGER /* XXX Linux*/ + linger.l_onoff = 0; + linger.l_linger = 0; + AZ(setsockopt(sp->fd, SOL_SOCKET, SO_LINGER, &linger, sizeof linger)); +#endif + i = getnameinfo(addr, l, sp->addr, VCA_ADDRBUFSIZE, port, sizeof port, NI_NUMERICHOST | NI_NUMERICSERV); From phk at projects.linpro.no Fri Jul 7 08:40:03 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 7 Jul 2006 10:40:03 +0200 (CEST) Subject: r374 - trunk/varnish-cache/bin/varnishd Message-ID: <20060707084003.9027B1EC1F9@projects.linpro.no> Author: phk Date: 2006-07-07 10:40:03 +0200 (Fri, 07 Jul 2006) New Revision: 374 Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c Log: Close gracefully in pipe mode Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-07 07:48:49 UTC (rev 373) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-07 08:40:03 UTC (rev 374) @@ -37,7 +37,11 @@ event_del(&ep->ev); } else { j = write(ep->fd, buf, i); - assert(i == j); + if (i != j) { + shutdown(fd, SHUT_WR); + shutdown(ep->fd, SHUT_RD); + event_del(&ep->ev); + } } } @@ -57,7 +61,11 @@ assert(i == sbuf_len(w->sb)); if (http_GetTail(sp->http, 99999999, &b, &e) && b != e) { /* XXX */ i = write(fd, b, e - b); - assert(i == e - b); + if (i != e - b) { + close (fd); + vca_close_session(sp, "pipe"); + VBE_ClosedFd(fd_token); + } } e1.fd = fd; From des at projects.linpro.no Sat Jul 8 08:52:56 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Sat, 8 Jul 2006 10:52:56 +0200 (CEST) Subject: r375 - tags Message-ID: <20060708085256.22FB31EC223@projects.linpro.no> Author: des Date: 2006-07-08 10:52:56 +0200 (Sat, 08 Jul 2006) New Revision: 375 Added: tags/live-test-1/ Log: The state of the tree at the end of live test 1 - this is the code that ran for 100 minutes on Friday. Copied: tags/live-test-1 (from rev 374, trunk) From des at projects.linpro.no Sat Jul 8 19:25:08 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Sat, 8 Jul 2006 21:25:08 +0200 (CEST) Subject: r376 - trunk/varnish-cache Message-ID: <20060708192508.E6B111EC381@projects.linpro.no> Author: des Date: 2006-07-08 21:25:08 +0200 (Sat, 08 Jul 2006) New Revision: 376 Modified: trunk/varnish-cache/autogen.phk Log: Expand keywords. Modified: trunk/varnish-cache/autogen.phk =================================================================== --- trunk/varnish-cache/autogen.phk 2006-07-08 08:52:56 UTC (rev 375) +++ trunk/varnish-cache/autogen.phk 2006-07-08 19:25:08 UTC (rev 376) @@ -1,6 +1,6 @@ #!/bin/sh # -# $Id: autogen.sh 118 2006-04-05 09:40:22Z phk $ +# $Id$ # set -ex Property changes on: trunk/varnish-cache/autogen.phk ___________________________________________________________________ Name: svn:keywords + Id From des at projects.linpro.no Sat Jul 8 19:25:16 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Sat, 8 Jul 2006 21:25:16 +0200 (CEST) Subject: r377 - trunk/varnish-cache Message-ID: <20060708192516.21D551EC38E@projects.linpro.no> Author: des Date: 2006-07-08 21:25:16 +0200 (Sat, 08 Jul 2006) New Revision: 377 Added: trunk/varnish-cache/autogen.des Log: My version. Added: trunk/varnish-cache/autogen.des =================================================================== --- trunk/varnish-cache/autogen.des 2006-07-08 19:25:08 UTC (rev 376) +++ trunk/varnish-cache/autogen.des 2006-07-08 19:25:16 UTC (rev 377) @@ -0,0 +1,14 @@ +#!/bin/sh +# +# $Id$ +# + +set -ex + +./autogen.sh + +./configure \ + --enable-developer-warnings \ + --enable-debugging-symbols \ + --enable-dependency-tracking \ + --prefix=/opt/varnish Property changes on: trunk/varnish-cache/autogen.des ___________________________________________________________________ Name: svn:executable + * Name: svn:keywords + Id From des at projects.linpro.no Sat Jul 8 19:27:53 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Sat, 8 Jul 2006 21:27:53 +0200 (CEST) Subject: r378 - trunk/varnish-cache/lib/libvcl Message-ID: <20060708192753.8BFFF1EC38E@projects.linpro.no> Author: des Date: 2006-07-08 21:27:53 +0200 (Sat, 08 Jul 2006) New Revision: 378 Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c Log: Fix "set backend.port". Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-07-08 19:25:16 UTC (rev 377) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-07-08 19:27:53 UTC (rev 378) @@ -1294,7 +1294,7 @@ return; } if (t_port != NULL) { - port = EncString(tl->t); + port = EncString(t_port); ep = CheckHostPort(host, port); if (ep != NULL) { sbuf_printf(tl->sb, "Backend '%T': %s\n", t_be, ep); From des at projects.linpro.no Sat Jul 8 19:46:11 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Sat, 8 Jul 2006 21:46:11 +0200 (CEST) Subject: r379 - trunk/varnish-cache Message-ID: <20060708194611.9CD8E1EC35E@projects.linpro.no> Author: des Date: 2006-07-08 21:46:11 +0200 (Sat, 08 Jul 2006) New Revision: 379 Modified: trunk/varnish-cache/configure.ac Log: Add --enable-debugging-symbols which enables debugging sysmbols and disables inlining and builtins. Modified: trunk/varnish-cache/configure.ac =================================================================== --- trunk/varnish-cache/configure.ac 2006-07-08 19:27:53 UTC (rev 378) +++ trunk/varnish-cache/configure.ac 2006-07-08 19:46:11 UTC (rev 379) @@ -19,6 +19,9 @@ AC_ARG_ENABLE(developer-warnings, AS_HELP_STRING([--enable-developer-warnings],[enable strict warnings (default is NO)]), CFLAGS="${CFLAGS} ${DEVELOPER_CFLAGS}") +AC_ARG_ENABLE(debugging-symbols, + AS_HELP_STRING([--enable-debugging-symbols],[enable debugging symbols (default is NO)]), + CFLAGS="${CFLAGS} -g -fno-inline -fno-builtin") AC_ARG_ENABLE(werror, AS_HELP_STRING([--enable-werror],[use -Werror (default is NO)]), CFLAGS="${CFLAGS} -Werror") From phk at projects.linpro.no Sat Jul 8 19:54:10 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 8 Jul 2006 21:54:10 +0200 (CEST) Subject: r380 - trunk/varnish-cache/bin/varnishd Message-ID: <20060708195410.8C1061EC38E@projects.linpro.no> Author: phk Date: 2006-07-08 21:54:10 +0200 (Sat, 08 Jul 2006) New Revision: 380 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_http.c Log: Move struct http into cache.h The original reasoning for having it private to cache_http.c was to avoid pollution with event.h related structures but since that pollution is happening other ways anyway, the cost is too high. Include pthread.h, sys/time.h, and event.h from cache.h Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-08 19:46:11 UTC (rev 379) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-08 19:54:10 UTC (rev 380) @@ -2,7 +2,10 @@ * $Id$ */ +#include +#include #include +#include #include "vcl_returns.h" @@ -15,7 +18,36 @@ struct object; struct objhead; -#ifdef EV_TIMEOUT +/*-------------------------------------------------------------------- + * HTTP Request/Response/Header handling structure. + * RSN: struct worker and struct session will have one of these embedded. + */ + +typedef void http_callback_f(void *, int bad); + +struct http { + struct event ev; + http_callback_f *callback; + void *arg; + + char *s; /* start of buffer */ + char *e; /* end of buffer */ + char *v; /* valid bytes */ + char *t; /* start of trailing data */ + + + char *req; + char *url; + char *proto; + char *status; + char *response; + + char **hdr; + unsigned nhdr; +}; + +/*--------------------------------------------------------------------*/ + struct worker { struct event_base *eb; struct event e1, e2; @@ -23,9 +55,6 @@ struct objhead *nobjhead; struct object *nobj; }; -#else -struct worker; -#endif #include "hash_slinger.h" @@ -169,8 +198,6 @@ void HSH_Init(void); /* cache_http.c */ -typedef void http_callback_f(void *, int bad); -struct http; struct http *http_New(void); void http_Delete(struct http *hp); int http_GetHdr(struct http *hp, const char *hdr, char **ptr); @@ -187,7 +214,7 @@ Build_Pipe, Build_Pass, Build_Fetch, - Build_Reply, + Build_Reply }; void http_BuildSbuf(int fd, enum http_build mode, struct sbuf *sb, struct http *hp); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-08 19:46:11 UTC (rev 379) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-08 19:54:10 UTC (rev 380) @@ -23,29 +23,6 @@ /*--------------------------------------------------------------------*/ -struct http { - struct event ev; - http_callback_f *callback; - void *arg; - - char *s; /* start of buffer */ - char *e; /* end of buffer */ - char *v; /* valid bytes */ - char *t; /* start of trailing data */ - - - char *req; - char *url; - char *proto; - char *status; - char *response; - - char **hdr; - unsigned nhdr; -}; - -/*--------------------------------------------------------------------*/ - struct http * http_New(void) { From phk at projects.linpro.no Sat Jul 8 20:18:42 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 8 Jul 2006 22:18:42 +0200 (CEST) Subject: r381 - trunk/varnish-cache/bin/varnishd Message-ID: <20060708201842.919D91EC38F@projects.linpro.no> Author: phk Date: 2006-07-08 22:18:42 +0200 (Sat, 08 Jul 2006) New Revision: 381 Modified: trunk/varnish-cache/bin/varnishd/flint.lnt Log: Silence a bogus warning Modified: trunk/varnish-cache/bin/varnishd/flint.lnt =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.lnt 2006-07-08 19:54:10 UTC (rev 380) +++ trunk/varnish-cache/bin/varnishd/flint.lnt 2006-07-08 20:18:42 UTC (rev 381) @@ -43,3 +43,5 @@ -e641 // Converting enum 'cli_status_e' to int -e786 // String concatenation within initializer + +-emacro(19, MAC_STAT) // Useless Declaration From phk at projects.linpro.no Sat Jul 8 20:19:17 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 8 Jul 2006 22:19:17 +0200 (CEST) Subject: r382 - trunk/varnish-cache/bin/varnishd Message-ID: <20060708201917.99F251EC38F@projects.linpro.no> Author: phk Date: 2006-07-08 22:19:17 +0200 (Sat, 08 Jul 2006) New Revision: 382 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_main.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_shmlog.c trunk/varnish-cache/bin/varnishd/cache_vcl.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishd/hash_classic.c trunk/varnish-cache/bin/varnishd/hash_simple_list.c trunk/varnish-cache/bin/varnishd/rfc2616.c trunk/varnish-cache/bin/varnishd/storage_file.c trunk/varnish-cache/bin/varnishd/storage_malloc.c Log: Sanitize #includes a bit in the cache process by moving fundamental #includes to cache.h Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-08 20:19:17 UTC (rev 382) @@ -2,6 +2,7 @@ * $Id$ */ +#include #include #include #include Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-08 20:19:17 UTC (rev 382) @@ -8,20 +8,16 @@ #include #include -#include #include #include -#include #include #include #include -#include #include #include -#include #include "config.h" #include "compat.h" Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-08 20:19:17 UTC (rev 382) @@ -25,10 +25,7 @@ #include #include -#include #include -#include -#include #include #include #include @@ -38,8 +35,6 @@ #include #include -#include - #include "libvarnish.h" #include "shmlog.h" #include "cache.h" Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2006-07-08 20:19:17 UTC (rev 382) @@ -4,11 +4,9 @@ * Ban processing */ -#include #include #include #include -#include #include "shmlog.h" #include "cli_priv.h" Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-08 20:19:17 UTC (rev 382) @@ -4,9 +4,7 @@ * Expiry of cached objects and execution of prefetcher */ -#include #include -#include #include #include "libvarnish.h" Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-08 20:19:17 UTC (rev 382) @@ -2,7 +2,6 @@ * $Id$ */ -#include #include #include #include @@ -10,13 +9,10 @@ #include #include #include -#include -#include #include #include #include #include -#include #include "libvarnish.h" #include "shmlog.h" Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-08 20:19:17 UTC (rev 382) @@ -4,13 +4,9 @@ #include #include -#include #include #include #include -#include -#include -#include #include "libvarnish.h" #include "shmlog.h" Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-08 20:19:17 UTC (rev 382) @@ -5,13 +5,10 @@ */ #include -#include #include #include #include -#include #include -#include #include #include "libvarnish.h" Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-08 20:19:17 UTC (rev 382) @@ -4,15 +4,9 @@ #include #include -#include #include #include -#include -#include - -#include - #include #include Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-08 20:19:17 UTC (rev 382) @@ -2,20 +2,16 @@ * $Id$ */ -#include #include #include #include #include #include #include -#include -#include #include #include #include #include -#include #include "libvarnish.h" #include "shmlog.h" Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-08 20:19:17 UTC (rev 382) @@ -2,16 +2,12 @@ * $Id$ */ -#include #include #include #include #include -#include -#include #include #include -#include #include "libvarnish.h" #include "shmlog.h" Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-08 20:19:17 UTC (rev 382) @@ -7,13 +7,8 @@ #include #include #include -#include -#include #include -#include -#include #include -#include #include "libvarnish.h" #include "heritage.h" Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-08 20:19:17 UTC (rev 382) @@ -7,14 +7,13 @@ #include #include #include -#include #include #include #include -#include #include "libvarnish.h" #include "shmlog.h" +#include "cache.h" #include "heritage.h" Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-07-08 20:19:17 UTC (rev 382) @@ -5,11 +5,8 @@ #include #include #include -#include #include -#include #include -#include #include "cli.h" #include "cli_priv.h" Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-08 20:19:17 UTC (rev 382) @@ -7,11 +7,8 @@ #include #include -#include #include -#include #include -#include #include "cli.h" #include "cli_priv.h" Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-08 20:19:17 UTC (rev 382) @@ -4,12 +4,9 @@ * A classic bucketed hash */ -#include #include #include #include -#include -#include #include #include Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-07-08 20:19:17 UTC (rev 382) @@ -4,12 +4,9 @@ * This is the reference hash(/lookup) implementation */ -#include #include #include #include -#include -#include #include #include Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-08 20:19:17 UTC (rev 382) @@ -4,7 +4,6 @@ #include #include -#include #include #include "shmlog.h" Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-08 20:19:17 UTC (rev 382) @@ -4,7 +4,6 @@ * Storage method based on mmap'ed file */ -#include #include #include #include @@ -13,9 +12,7 @@ #include #include #include -#include #include -#include #include #include #include Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-07-08 20:18:42 UTC (rev 381) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-07-08 20:19:17 UTC (rev 382) @@ -4,10 +4,7 @@ * Storage method based on malloc(3) */ -#include #include -#include -#include #include "cache.h" From phk at projects.linpro.no Sat Jul 8 20:29:31 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 8 Jul 2006 22:29:31 +0200 (CEST) Subject: r383 - trunk/varnish-cache/bin/varnishd Message-ID: <20060708202931.905571EC35E@projects.linpro.no> Author: phk Date: 2006-07-08 22:29:31 +0200 (Sat, 08 Jul 2006) New Revision: 383 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_shmlog.c Log: A few edits for FlexeLint Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-08 20:19:17 UTC (rev 382) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-08 20:29:31 UTC (rev 383) @@ -51,7 +51,6 @@ struct worker { struct event_base *eb; - struct event e1, e2; struct sbuf *sb; struct objhead *nobjhead; struct object *nobj; Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-08 20:19:17 UTC (rev 382) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-08 20:29:31 UTC (rev 383) @@ -268,6 +268,8 @@ p++; hp->nhdr = 0; + r = NULL; /* For FlexeLint */ + assert(p < hp->v); /* http_header_complete() guarantees this */ for (; p < hp->v; p = r) { q = strchr(p, '\n'); assert(q != NULL); @@ -331,7 +333,6 @@ http_read_f(int fd, short event, void *arg) { struct http *hp = arg; - char *p; unsigned l; int i; @@ -475,6 +476,7 @@ sup = 1; break; default: + sup = 0; /* for flexelint */ printf("mode = %d\n", mode); assert(mode == 1 || mode == 2); } Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-08 20:19:17 UTC (rev 382) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-08 20:29:31 UTC (rev 383) @@ -28,7 +28,7 @@ struct varnish_stats *VSL_stats; static struct shmloghead *loghead; -static unsigned char *logstart, *logend; +static unsigned char *logstart; static pthread_mutex_t vsl_mutex; /* @@ -88,7 +88,7 @@ { va_list ap; unsigned char *p; - unsigned m, n; + unsigned n; va_start(ap, fmt); @@ -132,7 +132,6 @@ /* XXX check sanity of loghead */ logstart = (unsigned char *)loghead + loghead->start; - logend = logstart + loghead->size; VSL_stats = &loghead->stats; AZ(pthread_mutex_init(&vsl_mutex, NULL)); } From des at projects.linpro.no Sun Jul 9 06:35:48 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Sun, 9 Jul 2006 08:35:48 +0200 (CEST) Subject: r384 - trunk/varnish-cache Message-ID: <20060709063548.CAAA81EC390@projects.linpro.no> Author: des Date: 2006-07-09 08:35:48 +0200 (Sun, 09 Jul 2006) New Revision: 384 Modified: trunk/varnish-cache/configure.ac Log: Use the modern version of AM_INIT_AUTOMAKE, allowing automake to deduce the correct distribution name. Modified: trunk/varnish-cache/configure.ac =================================================================== --- trunk/varnish-cache/configure.ac 2006-07-08 20:29:31 UTC (rev 383) +++ trunk/varnish-cache/configure.ac 2006-07-09 06:35:48 UTC (rev 384) @@ -10,7 +10,7 @@ AC_CANONICAL_SYSTEM AC_LANG(C) -AM_INIT_AUTOMAKE(AC_PACKAGE_NAME, AC_PACKAGE_VERSION) +AM_INIT_AUTOMAKE # Compiler flags (assume GCC). # This section *must* come before AC_PROG_CC / AC_PROG_CPP. From des at projects.linpro.no Sun Jul 9 07:13:17 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Sun, 9 Jul 2006 09:13:17 +0200 (CEST) Subject: r385 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060709071317.0E1E81EC38E@projects.linpro.no> Author: des Date: 2006-07-09 09:13:16 +0200 (Sun, 09 Jul 2006) New Revision: 385 Modified: trunk/varnish-cache/bin/varnishd/Makefile.am trunk/varnish-cache/include/Makefile.am trunk/varnish-cache/lib/libvcl/Makefile.am Log: List header files so they are included in the distribution tarball. Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-09 06:35:48 UTC (rev 384) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-09 07:13:16 UTC (rev 385) @@ -7,6 +7,13 @@ man_MANS = varnishd.1 varnishd_SOURCES = \ + cache.h \ + cli_event.h \ + hash_slinger.h \ + heritage.h \ + mgt.h \ + stevedore.h \ + \ cache_acceptor.c \ cache_backend.c \ cache_ban.c \ Modified: trunk/varnish-cache/include/Makefile.am =================================================================== --- trunk/varnish-cache/include/Makefile.am 2006-07-09 06:35:48 UTC (rev 384) +++ trunk/varnish-cache/include/Makefile.am 2006-07-09 07:13:16 UTC (rev 385) @@ -1,12 +1,23 @@ # $Id$ -include_HEADERS = \ +noinst_HEADERS = \ binary_heap.h \ + cli.h \ + cli_priv.h \ compat.h \ hash.h \ + http_headers.h \ libvarnish.h \ + libvcl.h \ queue.h \ sbuf.h \ + shmlog.h \ + shmlog_tags.h \ + stat_field.h \ + stats.h \ tree.h \ varnish/assert.h \ - varnishapi.h + varnishapi.h \ + vcl.h \ + vcl_returns.h \ + vrt.h Modified: trunk/varnish-cache/lib/libvcl/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvcl/Makefile.am 2006-07-09 06:35:48 UTC (rev 384) +++ trunk/varnish-cache/lib/libvcl/Makefile.am 2006-07-09 07:13:16 UTC (rev 385) @@ -5,5 +5,8 @@ lib_LTLIBRARIES = libvcl.la libvcl_la_SOURCES = \ + vcl_priv.h \ + vcl_token_defs.h \ + \ vcl_compile.c \ vcl_fixed_token.c From des at projects.linpro.no Sun Jul 9 09:16:25 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Sun, 9 Jul 2006 11:16:25 +0200 (CEST) Subject: r386 - trunk/varnish-cache Message-ID: <20060709091625.52CB51EC391@projects.linpro.no> Author: des Date: 2006-07-09 11:16:25 +0200 (Sun, 09 Jul 2006) New Revision: 386 Modified: trunk/varnish-cache/configure.ac Log: Don't use braces where they aren't needed. Let automake know about config.h. Bump version number to mark that we have passed the first live test. Modified: trunk/varnish-cache/configure.ac =================================================================== --- trunk/varnish-cache/configure.ac 2006-07-09 07:13:16 UTC (rev 385) +++ trunk/varnish-cache/configure.ac 2006-07-09 09:16:25 UTC (rev 386) @@ -3,9 +3,9 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Linpro AS / Verdens Gang AS]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [0.1], [varnish-dev at projects.linpro.no]) -AC_CONFIG_SRCDIR([include/varnishapi.h]) -AC_CONFIG_HEADER([config.h]) +AC_INIT([Varnish], [0.2], [varnish-dev at projects.linpro.no]) +AC_CONFIG_SRCDIR(include/varnishapi.h) +AM_CONFIG_HEADER(config.h) AC_CANONICAL_SYSTEM AC_LANG(C) From phk at projects.linpro.no Sun Jul 9 21:01:55 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 9 Jul 2006 23:01:55 +0200 (CEST) Subject: r387 - trunk/varnish-cache/bin/varnishlog Message-ID: <20060709210155.3F78B1EC38D@projects.linpro.no> Author: phk Date: 2006-07-09 23:01:55 +0200 (Sun, 09 Jul 2006) New Revision: 387 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c Log: Add a -h opt which modifies -o to remove "trivial" entries. Trivial are {GET,HEAD} which gets a hit and returns 200 {GET,HEAD} which gets a miss, fetches, inserts and returns 200 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-09 09:16:25 UTC (rev 386) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-09 21:01:55 UTC (rev 387) @@ -48,6 +48,8 @@ /* Ordering-----------------------------------------------------------*/ static struct sbuf *ob[65536]; +static int hc[65536]; +static int xrf[65536]; static void clean_order() @@ -65,15 +67,16 @@ } static void -order(unsigned char *p) +order(unsigned char *p, int h_opt) { - unsigned u; + unsigned u, v; u = (p[2] << 8) | p[3]; if (ob[u] == NULL) { ob[u] = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(ob[u] != NULL); } + v = 0; switch (p[0]) { case SLT_VCL_call: sbuf_printf(ob[u], "%02x %3d %4d %-12s", @@ -82,6 +85,8 @@ sbuf_cat(ob[u], " <"); sbuf_bcat(ob[u], p + 4, p[1]); } + if (h_opt && p[1] == 3 && !memcmp(p + 4, "hit", 3)) + hc[u]++; break; case SLT_VCL_trace: if (p[1] > 0) { @@ -95,15 +100,69 @@ sbuf_bcat(ob[u], p + 4, p[1]); sbuf_cat(ob[u], ">\n"); } + if (h_opt && p[1] == 7 && !memcmp(p + 4, "deliver", 7)) + hc[u]++; + if (h_opt && p[1] == 6 && !memcmp(p + 4, "insert", 6)) { + if (hc[xrf[u]] == 1) { + hc[u] += 2; + hc[xrf[u]] = 4; + } + } break; case SLT_Debug: + if (p[1] == 0) + break; + if (!h_opt) + ; + else if (p[1] > 4 && !memcmp(p + 4, "TTD:", 4)) + break; sbuf_printf(ob[u], "%02x %3d %4d %-12s", p[0], p[1], u, tagnames[p[0]]); if (p[1] > 0) sbuf_cat(ob[u], vis_it(p)); sbuf_cat(ob[u], "\n"); break; + case SLT_HttpError: + if (!h_opt) + v = 1; + else if (p[1] == 16 && !memcmp(p + 4, "Received nothing", 16)) + ; + else if (p[1] == 17 && !memcmp(p + 4, "Received errno 54", 17)) + ; + else + v = 1; + break; + case SLT_SessionClose: + if (!h_opt) + v = 1; + else if (p[1] == 10 && !memcmp(p + 4, "no request", 10)) + ; + else if (p[1] == 7 && !memcmp(p + 4, "timeout", 7)) + ; + else + v = 1; + break; + case SLT_Request: + if (h_opt && p[1] == 3 && !memcmp(p + 4, "GET", 3)) + hc[u]++; + if (h_opt && p[1] == 4 && !memcmp(p + 4, "HEAD", 3)) + hc[u]++; + v = 1; + break; + case SLT_Backend: + xrf[u] = atoi(p + 4); + v = 1; + break; + case SLT_Status: + if (h_opt && p[1] == 3 && !memcmp(p + 4, "200", 3)) + hc[u]++; + v = 1; + break; default: + v = 1; + break; + } + if (v) { sbuf_printf(ob[u], "%02x %3d %4d %-12s", p[0], p[1], u, tagnames[p[0]]); if (p[1] > 0) { @@ -112,7 +171,6 @@ sbuf_cat(ob[u], ">"); } sbuf_cat(ob[u], "\n"); - break; } if (u == 0) { sbuf_finish(ob[u]); @@ -125,8 +183,11 @@ case SLT_SessionReuse: case SLT_BackendClose: sbuf_finish(ob[u]); - printf("%s\n", sbuf_data(ob[u])); + if ((hc[u] != 4 || h_opt == 0) && sbuf_len(ob[u]) > 1) + printf("%s\n", sbuf_data(ob[u])); sbuf_clear(ob[u]); + hc[u] = 0; + xrf[u] = 0; break; default: break; @@ -155,6 +216,7 @@ FILE *wfile = NULL; char *r_opt = NULL; FILE *rfile = NULL; + int h_opt = 0; unsigned char rbuf[255+4]; struct shmloghead *loghead; @@ -163,8 +225,11 @@ for (i = 0; stagnames[i].tag != SLT_ENDMARKER; i++) tagnames[stagnames[i].tag] = stagnames[i].name; - while ((c = getopt(argc, argv, "or:w:")) != -1) { + while ((c = getopt(argc, argv, "hor:w:")) != -1) { switch (c) { + case 'h': + h_opt = 1; + break; case 'o': o_flag = 1; break; @@ -243,7 +308,7 @@ continue; } if (o_flag) { - order(p); + order(p, h_opt); continue; } u = (p[2] << 8) | p[3]; From phk at projects.linpro.no Sun Jul 9 21:21:47 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 9 Jul 2006 23:21:47 +0200 (CEST) Subject: r388 - trunk/varnish-cache/include Message-ID: <20060709212147.3A62F1EC390@projects.linpro.no> Author: phk Date: 2006-07-09 23:21:47 +0200 (Sun, 09 Jul 2006) New Revision: 388 Modified: trunk/varnish-cache/include/http_headers.h Log: Don't pass cache-control through. Modified: trunk/varnish-cache/include/http_headers.h =================================================================== --- trunk/varnish-cache/include/http_headers.h 2006-07-09 21:01:55 UTC (rev 387) +++ trunk/varnish-cache/include/http_headers.h 2006-07-09 21:21:47 UTC (rev 388) @@ -25,7 +25,7 @@ HTTPH("Age", H_Age, 2, 0, 0, 0, 0) /* RFC2616 14.6 */ HTTPH("Allow", H_Allow, 2, 0, 0, 0, 0) /* RFC2616 14.7 */ HTTPH("Authorization", H_Authorization, 1, 0, 0, 0, 0) /* RFC2616 14.8 */ -HTTPH("Cache-Control", H_Cache_Control, 3, 0, 0, 0, 0) /* RFC2616 14.9 */ +HTTPH("Cache-Control", H_Cache_Control, 3, 3, 0, 0, 0) /* RFC2616 14.9 */ HTTPH("Connection", H_Connection, 3, 3, 0, 0, 0) /* RFC2616 14.10 */ HTTPH("Content-Encoding", H_Content_Encoding, 2, 0, 0, 0, 0) /* RFC2616 14.11 */ HTTPH("Content-Langugae", H_Content_Language, 2, 0, 0, 0, 0) /* RFC2616 14.12 */ From phk at projects.linpro.no Mon Jul 10 07:04:37 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 09:04:37 +0200 (CEST) Subject: r389 - trunk/varnish-cache/bin/varnishd Message-ID: <20060710070437.0FBA41EC382@projects.linpro.no> Author: phk Date: 2006-07-10 09:04:36 +0200 (Mon, 10 Jul 2006) New Revision: 389 Modified: trunk/varnish-cache/bin/varnishd/cache.h Log: More sensible order of pointers Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-09 21:21:47 UTC (rev 388) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 07:04:36 UTC (rev 389) @@ -32,11 +32,10 @@ void *arg; char *s; /* start of buffer */ + char *t; /* start of trailing data */ char *e; /* end of buffer */ char *v; /* valid bytes */ - char *t; /* start of trailing data */ - char *req; char *url; char *proto; From phk at projects.linpro.no Mon Jul 10 07:07:13 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 09:07:13 +0200 (CEST) Subject: r390 - trunk/varnish-cache/bin/varnishd Message-ID: <20060710070713.A17441EC393@projects.linpro.no> Author: phk Date: 2006-07-10 09:07:13 +0200 (Mon, 10 Jul 2006) New Revision: 390 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c Log: Turn a comment into english Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-10 07:04:36 UTC (rev 389) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-10 07:07:13 UTC (rev 390) @@ -19,10 +19,10 @@ #include "cache.h" /* - * Chunked encoding is a hack. We prefer to have a single or a few - * large storage objects, and a terribly long list of small ones. + * Chunked encoding is a hack. We prefer to have a single chunk or a + * few large chunks, and not a terribly long list of small ones. * If our stevedore can trim, we alloc big chunks and trim the last one - * at the end know the result. + * at the end when we know the result. * * Good testcase: http://www.washingtonpost.com/ */ From phk at projects.linpro.no Mon Jul 10 07:54:05 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 09:54:05 +0200 (CEST) Subject: r391 - trunk/varnish-cache/bin/varnishd Message-ID: <20060710075405.379831EC382@projects.linpro.no> Author: phk Date: 2006-07-10 09:54:05 +0200 (Mon, 10 Jul 2006) New Revision: 391 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pipe.c Log: Take the vbe_conn (backend connection) structure out of the closet. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 07:07:13 UTC (rev 390) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 07:54:05 UTC (rev 391) @@ -57,6 +57,16 @@ #include "hash_slinger.h" +/* Backend Connection ------------------------------------------------*/ + +struct vbe_conn { + TAILQ_ENTRY(vbe_conn) list; + struct vbe *vbe; + int fd; + struct event ev; + int inuse; +}; + /* Storage -----------------------------------------------------------*/ struct storage { @@ -172,9 +182,9 @@ /* cache_backend.c */ void VBE_Init(void); -int VBE_GetFd(struct backend *bp, void **ptr, unsigned xid); -void VBE_ClosedFd(void *ptr); -void VBE_RecycleFd(void *ptr); +struct vbe_conn *VBE_GetFd(struct backend *bp, unsigned xid); +void VBE_ClosedFd(struct vbe_conn *vc); +void VBE_RecycleFd(struct vbe_conn *vc); /* cache_ban.c */ void BAN_Init(void); Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-10 07:07:13 UTC (rev 390) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-10 07:54:05 UTC (rev 391) @@ -39,16 +39,6 @@ #include "shmlog.h" #include "cache.h" -/* A backend connection */ - -struct vbe_conn { - TAILQ_ENTRY(vbe_conn) list; - struct vbe *vbe; - int fd; - struct event ev; - int inuse; -}; - /* A backend IP */ struct vbe { @@ -211,8 +201,8 @@ * new connection. */ -int -VBE_GetFd(struct backend *bp, void **ptr, unsigned xid) +struct vbe_conn * +VBE_GetFd(struct backend *bp, unsigned xid) { struct vbe *vp; struct vbe_conn *vc; @@ -252,24 +242,31 @@ vp->nconn++; AZ(pthread_mutex_unlock(&vbemtx)); connect_to_backend(vc, bp); - VSL_stats->backend_conn++; - event_set(&vc->ev, vc->fd, EV_READ | EV_PERSIST, vbe_rdf, vc); - event_base_set(vbe_evb, &vc->ev); + if (vc->fd < 0) { + AZ(pthread_mutex_lock(&vbemtx)); + TAILQ_REMOVE(&vc->vbe->bconn, vc, list); + vp->nconn--; + AZ(pthread_mutex_unlock(&vbemtx)); + free(vc); + vc = NULL; + } else { + VSL_stats->backend_conn++; + event_set(&vc->ev, vc->fd, EV_READ | EV_PERSIST, vbe_rdf, vc); + event_base_set(vbe_evb, &vc->ev); + } } - *ptr = vc; - VSL(SLT_BackendXID, vc->fd, "%u", xid); - return (vc->fd); + if (vc != NULL) + VSL(SLT_BackendXID, vc->fd, "%u", xid); + return (vc); } /* Close a connection ------------------------------------------------*/ void -VBE_ClosedFd(void *ptr) +VBE_ClosedFd(struct vbe_conn *vc) { - struct vbe_conn *vc; int i; - vc = ptr; VSL(SLT_BackendClose, vc->fd, ""); close(vc->fd); vc->fd = -1; @@ -280,13 +277,11 @@ /* Recycle a connection ----------------------------------------------*/ void -VBE_RecycleFd(void *ptr) +VBE_RecycleFd(struct vbe_conn *vc) { - struct vbe_conn *vc; int i; VSL_stats->backend_recycle++; - vc = ptr; VSL(SLT_BackendReuse, vc->fd, ""); i = write(vbe_pipe[1], &vc, sizeof vc); assert(i == sizeof vc); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-10 07:07:13 UTC (rev 390) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-10 07:54:05 UTC (rev 391) @@ -232,23 +232,23 @@ int FetchSession(struct worker *w, struct sess *sp) { - int fd, i, cls; - void *fd_token; + int i, cls; + struct vbe_conn *vc; struct http *hp; char *b; int body; sp->obj->xid = sp->xid; - fd = VBE_GetFd(sp->backend, &fd_token, sp->xid); - if (fd == -1) - fd = VBE_GetFd(sp->backend, &fd_token, sp->xid); - assert(fd != -1); /* XXX: handle this */ - VSL(SLT_Backend, sp->fd, "%d %s", fd, sp->backend->vcl_name); + vc = VBE_GetFd(sp->backend, sp->xid); + if (vc == NULL) + vc = VBE_GetFd(sp->backend, sp->xid); + assert(vc != NULL); /* XXX: handle this */ + VSL(SLT_Backend, sp->fd, "%d %s", vc->fd, sp->backend->vcl_name); hp = http_New(); - http_BuildSbuf(fd, Build_Fetch, w->sb, sp->http); - i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); + http_BuildSbuf(vc->fd, Build_Fetch, w->sb, sp->http); + i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); time(&sp->t_req); @@ -258,10 +258,10 @@ * XXX: It might be cheaper to avoid the event_engine and simply * XXX: read(2) the header */ - http_RecvHead(hp, fd, w->eb, NULL, NULL); + http_RecvHead(hp, vc->fd, w->eb, NULL, NULL); event_base_loop(w->eb, 0); time(&sp->t_resp); - assert(http_Dissect(hp, fd, 2) == 0); + assert(http_Dissect(hp, vc->fd, 2) == 0); body = RFC2616_cache_policy(sp, hp); @@ -273,11 +273,11 @@ http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); if (body) { if (http_GetHdr(hp, "Content-Length", &b)) - cls = fetch_straight(w, sp, fd, hp, b); + cls = fetch_straight(w, sp, vc->fd, hp, b); else if (http_HdrIs(hp, "Transfer-Encoding", "chunked")) - cls = fetch_chunked(w, sp, fd, hp); + cls = fetch_chunked(w, sp, vc->fd, hp); else - cls = fetch_eof(w, sp, fd, hp); + cls = fetch_eof(w, sp, vc->fd, hp); sbuf_printf(w->sb, "Content-Length: %u\r\n", sp->obj->len); } else cls = 0; @@ -291,9 +291,9 @@ cls = 1; if (cls) - VBE_ClosedFd(fd_token); + VBE_ClosedFd(vc); else - VBE_RecycleFd(fd_token); + VBE_RecycleFd(vc); HSH_Unbusy(sp->obj); if (!sp->obj->cacheable) Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-10 07:07:13 UTC (rev 390) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-10 07:54:05 UTC (rev 391) @@ -147,17 +147,17 @@ void PassSession(struct worker *w, struct sess *sp) { - int fd, i; - void *fd_token; + int i; + struct vbe_conn *vc; struct http *hp; char *b; int cls; - fd = VBE_GetFd(sp->backend, &fd_token, sp->xid); - assert(fd != -1); + vc = VBE_GetFd(sp->backend, sp->xid); + assert(vc != NULL); - http_BuildSbuf(fd, Build_Pass, w->sb, sp->http); - i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); + http_BuildSbuf(vc->fd, Build_Pass, w->sb, sp->http); + i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); /* XXX: copy any contents */ @@ -167,19 +167,19 @@ * XXX: read(2) the header */ hp = http_New(); - http_RecvHead(hp, fd, w->eb, NULL, NULL); + http_RecvHead(hp, vc->fd, w->eb, NULL, NULL); event_base_loop(w->eb, 0); - http_Dissect(hp, fd, 2); + http_Dissect(hp, vc->fd, 2); http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); if (http_GetHdr(hp, "Content-Length", &b)) - cls = pass_straight(w, sp, fd, hp, b); + cls = pass_straight(w, sp, vc->fd, hp, b); else if (http_HdrIs(hp, "Connection", "close")) - cls = pass_straight(w, sp, fd, hp, NULL); + cls = pass_straight(w, sp, vc->fd, hp, NULL); else if (http_HdrIs(hp, "Transfer-Encoding", "chunked")) - cls = pass_chunked(w, sp, fd, hp); + cls = pass_chunked(w, sp, vc->fd, hp); else { INCOMPL(); cls = 1; @@ -190,7 +190,7 @@ cls = 1; if (cls) - VBE_ClosedFd(fd_token); + VBE_ClosedFd(vc); else - VBE_RecycleFd(fd_token); + VBE_RecycleFd(vc); } Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-10 07:07:13 UTC (rev 390) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-10 07:54:05 UTC (rev 391) @@ -44,36 +44,35 @@ void PipeSession(struct worker *w, struct sess *sp) { - int fd, i; - void *fd_token; + int i; + struct vbe_conn *vc; struct edir e1, e2; char *b, *e; - fd = VBE_GetFd(sp->backend, &fd_token, sp->xid); - assert(fd != -1); + vc = VBE_GetFd(sp->backend, sp->xid); + assert(vc != NULL); - http_BuildSbuf(fd, Build_Pipe, w->sb, sp->http); - i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); + http_BuildSbuf(vc->fd, Build_Pipe, w->sb, sp->http); + i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); if (http_GetTail(sp->http, 99999999, &b, &e) && b != e) { /* XXX */ - i = write(fd, b, e - b); + i = write(vc->fd, b, e - b); if (i != e - b) { - close (fd); + close (vc->fd); vca_close_session(sp, "pipe"); - VBE_ClosedFd(fd_token); + VBE_ClosedFd(vc); } } - e1.fd = fd; + e1.fd = vc->fd; e2.fd = sp->fd; event_set(&e1.ev, sp->fd, EV_READ | EV_PERSIST, rdf, &e1); event_base_set(w->eb, &e1.ev); - event_set(&e2.ev, fd, EV_READ | EV_PERSIST, rdf, &e2); + event_set(&e2.ev, vc->fd, EV_READ | EV_PERSIST, rdf, &e2); event_base_set(w->eb, &e2.ev); event_add(&e1.ev, NULL); event_add(&e2.ev, NULL); event_base_loop(w->eb, 0); - close (fd); vca_close_session(sp, "pipe"); - VBE_ClosedFd(fd_token); + VBE_ClosedFd(vc); } From phk at phk.freebsd.dk Mon Jul 10 08:01:30 2006 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Mon, 10 Jul 2006 08:01:30 +0000 Subject: r391 - trunk/varnish-cache/bin/varnishd In-Reply-To: Your message of "Mon, 10 Jul 2006 09:54:05 +0200." <20060710075405.379831EC382@projects.linpro.no> Message-ID: <72568.1152518490@critter.freebsd.dk> In message <20060710075405.379831EC382 at projects.linpro.no>, phk at projects.linpro .no writes: >Take the vbe_conn (backend connection) structure out of the closet. This is the beginning of a bit of remodelling. The basic idea is that we have N(sessions) > N(worker) >> N(backend connections) And therefore it makes sense to shuffle memory allocations as far right in that picture as possible. The other part is that I want to reduce the number of memory allocations a little bit further by allocating compound objects wherever possible. This means that struct vbe_conn will grow a struct http and a receive buffer for talking to the backend. Along the same lines, the uio array and space for generating headers will move (a bit further) into the worker thread. It will look pretty drastic in the commits, but it is pretty simple under all the noise. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at projects.linpro.no Mon Jul 10 08:10:30 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 10:10:30 +0200 (CEST) Subject: r392 - trunk/varnish-cache/bin/varnishd Message-ID: <20060710081030.2C9941EC393@projects.linpro.no> Author: phk Date: 2006-07-10 10:10:30 +0200 (Mon, 10 Jul 2006) New Revision: 392 Modified: trunk/varnish-cache/bin/varnishd/heritage.h trunk/varnish-cache/bin/varnishd/varnishd.c Log: Put three memory allocation hints into heritage: mem_http_1_line (512) Maximum length of the reqeust/response line of a HTTP message There is no point in filling the entire buffer with junk if we get a preposterously long first line. mem_http_header (4096) Maximum length of entire HTTP header. If we overflow this we return 400. mem_workspace (currently 0) In the future this will be the space we use for constructing headers to send and edits done from VCL. Modified: trunk/varnish-cache/bin/varnishd/heritage.h =================================================================== --- trunk/varnish-cache/bin/varnishd/heritage.h 2006-07-10 07:54:05 UTC (rev 391) +++ trunk/varnish-cache/bin/varnishd/heritage.h 2006-07-10 08:10:30 UTC (rev 392) @@ -38,6 +38,11 @@ /* Worker threads */ unsigned wthread_min, wthread_max; + + /* Memory allocation hints */ + unsigned mem_http_1_line; + unsigned mem_http_header; + unsigned mem_workspace; }; extern struct heritage heritage; Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-10 07:54:05 UTC (rev 391) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-10 08:10:30 UTC (rev 392) @@ -457,6 +457,9 @@ heritage.default_ttl = 120; heritage.wthread_min = 5; heritage.wthread_max = 5; + heritage.mem_http_1_line= 512; + heritage.mem_http_header= 4096; + heritage.mem_workspace = 0; while ((o = getopt(argc, argv, "b:df:h:p:s:t:w:")) != -1) switch (o) { From phk at projects.linpro.no Mon Jul 10 08:41:27 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 10:41:27 +0200 (CEST) Subject: r393 - trunk/varnish-cache/bin/varnishd Message-ID: <20060710084127.2BC2B1EC396@projects.linpro.no> Author: phk Date: 2006-07-10 10:41:26 +0200 (Mon, 10 Jul 2006) New Revision: 393 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/heritage.h trunk/varnish-cache/bin/varnishd/varnishd.c Log: Add heritage.mem_http_headers which is the maximum number of headers we recognize. Add http_Init() which initializes struct http given sufficient space. Respect heritage mem_* values in http_New() (while we still have it) Allocate backend connections (vbe_conn) with super allocation with space for http and workspace. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 08:10:30 UTC (rev 392) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 08:41:26 UTC (rev 393) @@ -42,8 +42,8 @@ char *status; char *response; + unsigned nhdr; char **hdr; - unsigned nhdr; }; /*--------------------------------------------------------------------*/ @@ -61,10 +61,12 @@ struct vbe_conn { TAILQ_ENTRY(vbe_conn) list; + struct vbc_mem *vbcm; struct vbe *vbe; int fd; struct event ev; int inuse; + struct http *http; }; /* Storage -----------------------------------------------------------*/ @@ -207,6 +209,7 @@ void HSH_Init(void); /* cache_http.c */ +void http_Init(struct http *ht, void *space); struct http *http_New(void); void http_Delete(struct http *hp); int http_GetHdr(struct http *hp, const char *hdr, char **ptr); Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-10 08:10:30 UTC (rev 392) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-10 08:41:26 UTC (rev 393) @@ -36,9 +36,16 @@ #include #include "libvarnish.h" +#include "heritage.h" #include "shmlog.h" #include "cache.h" +struct vbc_mem { + struct vbe_conn vbe; + struct http http; + char *http_hdr; +}; + /* A backend IP */ struct vbe { @@ -57,6 +64,36 @@ static struct event_base *vbe_evb; static int vbe_pipe[2]; +/*--------------------------------------------------------------------*/ + +static struct vbe_conn * +vbe_new_conn(void) +{ + struct vbc_mem *vbcm; + + vbcm = calloc( + sizeof *vbcm + + heritage.mem_http_headers * sizeof vbcm->http_hdr + + heritage.mem_http_headerspace + + heritage.mem_workspace, + 1); + if (vbcm == NULL) + return (NULL); + VSL_stats->n_vbe_conn++; + vbcm->vbe.vbcm = vbcm; + vbcm->vbe.http = &vbcm->http; + http_Init(&vbcm->http, (void *)(vbcm + 1)); + return (&vbcm->vbe); +} + +static void +vbe_delete_conn(struct vbe_conn *vb) +{ + + VSL_stats->n_vbe_conn--; + free(vb->vbcm); +} + /*-------------------------------------------------------------------- * XXX: we should not call getaddrinfo() every time, we should cache * and apply round-robin with blacklisting of entries that do not respond @@ -128,8 +165,7 @@ TAILQ_REMOVE(&vc->vbe->bconn, vc, list); if (vc->fd < 0) { vc->vbe->nconn--; - free(vc); - VSL_stats->n_vbe_conn--; + vbe_delete_conn(vc); } else { vc->inuse = 0; event_add(&vc->ev, NULL); @@ -165,8 +201,7 @@ AZ(pthread_mutex_unlock(&vbemtx)); event_del(&vc->ev); close(vc->fd); - free(vc); - VSL_stats->n_vbe_conn--; + vbe_delete_conn(vc); } /* Backend monitoring thread -----------------------------------------*/ @@ -217,7 +252,6 @@ if (vp == NULL) { vp = calloc(sizeof *vp, 1); assert(vp != NULL); - VSL_stats->n_vbe++; TAILQ_INIT(&vp->fconn); TAILQ_INIT(&vp->bconn); vp->ip = bp->ip; @@ -232,14 +266,16 @@ TAILQ_INSERT_TAIL(&vp->bconn, vc, list); AZ(pthread_mutex_unlock(&vbemtx)); } else { - vc = calloc(sizeof *vc, 1); - VSL_stats->n_vbe_conn++; - assert(vc != NULL); + vc = vbe_new_conn(); + if (vc == NULL) { + AZ(pthread_mutex_unlock(&vbemtx)); + return (NULL); + } + vp->nconn++; vc->vbe = vp; vc->fd = -1; vc->inuse = 1; TAILQ_INSERT_TAIL(&vp->bconn, vc, list); - vp->nconn++; AZ(pthread_mutex_unlock(&vbemtx)); connect_to_backend(vc, bp); if (vc->fd < 0) { @@ -247,16 +283,15 @@ TAILQ_REMOVE(&vc->vbe->bconn, vc, list); vp->nconn--; AZ(pthread_mutex_unlock(&vbemtx)); - free(vc); - vc = NULL; - } else { - VSL_stats->backend_conn++; - event_set(&vc->ev, vc->fd, EV_READ | EV_PERSIST, vbe_rdf, vc); - event_base_set(vbe_evb, &vc->ev); + vbe_delete_conn(vc); + return (NULL); } + VSL_stats->backend_conn++; + event_set(&vc->ev, vc->fd, + EV_READ | EV_PERSIST, vbe_rdf, vc); + event_base_set(vbe_evb, &vc->ev); } - if (vc != NULL) - VSL(SLT_BackendXID, vc->fd, "%u", xid); + VSL(SLT_BackendXID, vc->fd, "%u", xid); return (vc); } Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-10 08:10:30 UTC (rev 392) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-10 08:41:26 UTC (rev 393) @@ -13,11 +13,23 @@ #include "libvarnish.h" #include "shmlog.h" +#include "heritage.h" #include "cache.h" -static unsigned http_bufsize = 4096; -static unsigned http_nhdr = 128; +/*--------------------------------------------------------------------*/ +void +http_Init(struct http *hp, void *space) +{ + char *sp = space; + + memset(hp, 0, sizeof *hp); + hp->hdr = (void *)sp; + sp += heritage.mem_http_headers * sizeof hp->hdr; + hp->s = sp; + hp->e = hp->s + heritage.mem_http_headerspace; +} + /*--------------------------------------------------------------------*/ struct http * @@ -29,14 +41,14 @@ assert(hp != NULL); VSL_stats->n_http++; - hp->s = malloc(http_bufsize); + hp->s = malloc(heritage.mem_http_headerspace); assert(hp->s != NULL); - hp->e = hp->s + http_bufsize; + hp->e = hp->s + heritage.mem_http_headerspace; hp->v = hp->s; hp->t = hp->s; - hp->hdr = malloc(sizeof *hp->hdr * http_nhdr); + hp->hdr = malloc(sizeof *hp->hdr * heritage.mem_http_headers); assert(hp->hdr != NULL); return (hp); @@ -280,7 +292,7 @@ if (p == q) break; - if (hp->nhdr < http_nhdr) { + if (hp->nhdr < heritage.mem_http_headers) { hp->hdr[hp->nhdr++] = p; VSLR(SLT_Header, fd, p, q); } else { Modified: trunk/varnish-cache/bin/varnishd/heritage.h =================================================================== --- trunk/varnish-cache/bin/varnishd/heritage.h 2006-07-10 08:10:30 UTC (rev 392) +++ trunk/varnish-cache/bin/varnishd/heritage.h 2006-07-10 08:41:26 UTC (rev 393) @@ -41,7 +41,8 @@ /* Memory allocation hints */ unsigned mem_http_1_line; - unsigned mem_http_header; + unsigned mem_http_headerspace; + unsigned mem_http_headers; unsigned mem_workspace; }; Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-10 08:10:30 UTC (rev 392) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-10 08:41:26 UTC (rev 393) @@ -458,7 +458,8 @@ heritage.wthread_min = 5; heritage.wthread_max = 5; heritage.mem_http_1_line= 512; - heritage.mem_http_header= 4096; + heritage.mem_http_headerspace= 4096; + heritage.mem_http_headers= 32; heritage.mem_workspace = 0; while ((o = getopt(argc, argv, "b:df:h:p:s:t:w:")) != -1) From phk at projects.linpro.no Mon Jul 10 09:07:29 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 11:07:29 +0200 (CEST) Subject: r394 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060710090729.B41EC1EC382@projects.linpro.no> Author: phk Date: 2006-07-10 11:07:29 +0200 (Mon, 10 Jul 2006) New Revision: 394 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/include/stat_field.h Log: Allocate struct http as part of the session allocation. Remove http_New() and http_Delete() Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 08:41:26 UTC (rev 393) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 09:07:29 UTC (rev 394) @@ -149,7 +149,6 @@ struct VCL_conf *vcl; /* Various internal stuff */ - struct event *rd_e; struct sessmem *mem; time_t t0; }; @@ -210,8 +209,6 @@ /* cache_http.c */ void http_Init(struct http *ht, void *space); -struct http *http_New(void); -void http_Delete(struct http *hp); int http_GetHdr(struct http *hp, const char *hdr, char **ptr); int http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr); int http_GetReq(struct http *hp, char **b); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-10 08:41:26 UTC (rev 393) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-10 09:07:29 UTC (rev 394) @@ -41,13 +41,45 @@ static TAILQ_HEAD(,sess) sesshead = TAILQ_HEAD_INITIALIZER(sesshead); struct sessmem { - struct sess s; - struct event e; + struct sess sess; struct iovec iov[SESS_IOVS]; int niov; size_t liov; + struct http http; + char *http_hdr; }; +/*--------------------------------------------------------------------*/ + +static struct sess * +vca_new_sess(void) +{ + struct sessmem *sm; + + sm = calloc( + sizeof *sm + + heritage.mem_http_headers * sizeof sm->http_hdr + + heritage.mem_http_headerspace + + heritage.mem_workspace, + 1); + if (sm == NULL) + return (NULL); + VSL_stats->n_sess++; + sm->sess.mem = sm; + sm->sess.http = &sm->http; + http_Init(&sm->http, (void *)(sm + 1)); + return (&sm->sess); +} + +static void +vca_delete_sess(struct sess *sp) +{ + + VSL_stats->n_sess--; + free(sp->mem); +} + + /*-------------------------------------------------------------------- * Write data to client * We try to use writev() if possible in order to minimize number of @@ -178,7 +210,6 @@ accept_f(int fd, short event, void *arg) { socklen_t l; - struct sessmem *sm; struct sockaddr addr[2]; struct sess *sp; char port[NI_MAXSERV]; @@ -187,22 +218,14 @@ VSL_stats->client_conn++; - (void)arg; - sm = calloc(sizeof *sm, 1); - assert(sm != NULL); /* - * XXX: this is probably one we should handle - * XXX: accept, emit error NNN and close - */ - VSL_stats->n_sess++; + sp = vca_new_sess(); + assert(sp != NULL); /* XXX handle */ - sp = &sm->s; - sp->rd_e = &sm->e; - sp->mem = sm; l = sizeof addr; sp->fd = accept(fd, addr, &l); if (sp->fd < 0) { - free(sp); + vca_delete_sess(sp); return; } #ifdef SO_NOSIGPIPE /* XXX Linux */ @@ -227,7 +250,6 @@ VSL(SLT_SessionOpen, sp->fd, "%s", sp->addr); time(&sp->t_resp); TAILQ_INSERT_TAIL(&sesshead, sp, list); - sp->http = http_New(); http_RecvHead(sp->http, sp->fd, evb, vca_callback, sp); } @@ -295,10 +317,7 @@ VSL(SLT_SessionReuse, sp->fd, "%s", sp->addr); write(pipes[1], &sp, sizeof sp); } else { - if (sp->http != NULL) - http_Delete(sp->http); - VSL_stats->n_sess--; - free(sp->mem); + vca_delete_sess(sp); } } Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-10 08:41:26 UTC (rev 393) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-10 09:07:29 UTC (rev 394) @@ -246,7 +246,7 @@ assert(vc != NULL); /* XXX: handle this */ VSL(SLT_Backend, sp->fd, "%d %s", vc->fd, sp->backend->vcl_name); - hp = http_New(); + hp = vc->http; http_BuildSbuf(vc->fd, Build_Fetch, w->sb, sp->http); i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); @@ -299,7 +299,5 @@ if (!sp->obj->cacheable) HSH_Deref(sp->obj); - http_Delete(hp); - return (1); } Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-10 08:41:26 UTC (rev 393) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-10 09:07:29 UTC (rev 394) @@ -32,40 +32,6 @@ /*--------------------------------------------------------------------*/ -struct http * -http_New(void) -{ - struct http *hp; - - hp = calloc(sizeof *hp, 1); - assert(hp != NULL); - VSL_stats->n_http++; - - hp->s = malloc(heritage.mem_http_headerspace); - assert(hp->s != NULL); - - hp->e = hp->s + heritage.mem_http_headerspace; - hp->v = hp->s; - hp->t = hp->s; - - hp->hdr = malloc(sizeof *hp->hdr * heritage.mem_http_headers); - assert(hp->hdr != NULL); - - return (hp); -} - -void -http_Delete(struct http *hp) -{ - - free(hp->hdr); - free(hp->s); - free(hp); - VSL_stats->n_http--; -} - -/*--------------------------------------------------------------------*/ - int http_GetHdr(struct http *hp, const char *hdr, char **ptr) { Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-10 08:41:26 UTC (rev 393) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-10 09:07:29 UTC (rev 394) @@ -166,7 +166,7 @@ * XXX: It might be cheaper to avoid the event_engine and simply * XXX: read(2) the header */ - hp = http_New(); + hp = vc->http; http_RecvHead(hp, vc->fd, w->eb, NULL, NULL); event_base_loop(w->eb, 0); http_Dissect(hp, vc->fd, 2); Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2006-07-10 08:41:26 UTC (rev 393) +++ trunk/varnish-cache/include/stat_field.h 2006-07-10 09:07:29 UTC (rev 394) @@ -14,7 +14,6 @@ MAC_STAT(n_objecthead, uint64_t, "u", "N struct objecthead"); MAC_STAT(n_header, uint64_t, "u", "N struct header"); MAC_STAT(n_smf, uint64_t, "u", "N struct smf"); -MAC_STAT(n_http, uint64_t, "u", "N struct http"); MAC_STAT(n_vbe, uint64_t, "u", "N struct vbe"); MAC_STAT(n_vbe_conn, uint64_t, "u", "N struct vbe_conn"); From phk at phk.freebsd.dk Mon Jul 10 09:09:14 2006 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Mon, 10 Jul 2006 09:09:14 +0000 Subject: r394 - in trunk/varnish-cache: bin/varnishd include In-Reply-To: Your message of "Mon, 10 Jul 2006 11:07:29 +0200." <20060710090729.B41EC1EC382@projects.linpro.no> Message-ID: <77782.1152522554@critter.freebsd.dk> In message <20060710090729.B41EC1EC382 at projects.linpro.no>, phk at projects.linpro .no writes: >New Revision: 394 >Log: >Allocate struct http as part of the session allocation. > >Remove http_New() and http_Delete() This eliminates three mallocs for each session and three for each backend transaction. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at projects.linpro.no Mon Jul 10 09:28:26 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 11:28:26 +0200 (CEST) Subject: r395 - trunk/varnish-cache/bin/varnishlog Message-ID: <20060710092826.28EFF1EC38D@projects.linpro.no> Author: phk Date: 2006-07-10 11:28:26 +0200 (Mon, 10 Jul 2006) New Revision: 395 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c Log: Silence some warnings Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-10 09:07:29 UTC (rev 394) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-10 09:28:26 UTC (rev 395) @@ -52,7 +52,7 @@ static int xrf[65536]; static void -clean_order() +clean_order(void) { unsigned u; @@ -256,14 +256,14 @@ rfile = fopen(r_opt, "r"); if (rfile == NULL) perror(r_opt); - u = 0; } if (w_opt != NULL) { wfile = fopen(w_opt, "w"); if (wfile == NULL) perror(w_opt); - u = 0; } + u = 0; + v = 0; q = NULL; if (r_opt == NULL) { @@ -329,4 +329,5 @@ } if (o_flag) clean_order(); + return (0); } From phk at projects.linpro.no Mon Jul 10 09:47:57 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 11:47:57 +0200 (CEST) Subject: r396 - trunk/varnish-cache/bin/varnishd Message-ID: <20060710094757.6C3AC1EC395@projects.linpro.no> Author: phk Date: 2006-07-10 11:47:56 +0200 (Mon, 10 Jul 2006) New Revision: 396 Added: trunk/varnish-cache/bin/varnishd/cache_shmlog.h Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_main.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_shmlog.c trunk/varnish-cache/bin/varnishd/cache_vcl.c trunk/varnish-cache/bin/varnishd/cli_event.c trunk/varnish-cache/bin/varnishd/mgt.h trunk/varnish-cache/bin/varnishd/mgt_child.c trunk/varnish-cache/bin/varnishd/tcp.c trunk/varnish-cache/bin/varnishd/varnishd.c Log: Fix a bunch warnings, all cosmetic. I'm using __unused for now, if we need to use something different we can do a find/replace. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 09:47:56 UTC (rev 396) @@ -241,6 +241,8 @@ void DealWithSession(void *arg); /* cache_shmlog.c */ +#include "cache_shmlog.h" + void VSL_Init(void); #ifdef SHMLOGHEAD_MAGIC void VSLR(enum shmlogtag tag, unsigned id, const char *b, const char *e); @@ -251,7 +253,6 @@ assert(__LINE__ == 0); \ } while (0) #endif -extern struct varnish_stats *VSL_stats; /* cache_response.c */ void RES_Error(struct worker *w, struct sess *sp, int error, const char *msg); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-10 09:47:56 UTC (rev 396) @@ -160,7 +160,7 @@ /*--------------------------------------------------------------------*/ static void -vca_tick(int a, short b, void *c) +vca_tick(int a __unused, short b __unused, void *c __unused) { struct sess *sp, *sp2; time_t t; @@ -194,7 +194,7 @@ } static void -pipe_f(int fd, short event, void *arg) +pipe_f(int fd, short event __unused, void *arg __unused) { struct sess *sp; int i; @@ -207,7 +207,7 @@ } static void -accept_f(int fd, short event, void *arg) +accept_f(int fd, short event __unused, void *arg __unused) { socklen_t l; struct sockaddr addr[2]; @@ -254,7 +254,7 @@ } static void * -vca_main(void *arg) +vca_main(void *arg __unused) { unsigned u; struct event *ep; @@ -292,8 +292,8 @@ } event_base_loop(evb, 0); - - return ("FOOBAR"); + assert(0 == 1); + return (NULL); } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-10 09:47:56 UTC (rev 396) @@ -151,14 +151,11 @@ */ static void -vbe_rdp(int fd, short event, void *arg) +vbe_rdp(int fd, short event __unused, void *arg __unused) { struct vbe_conn *vc; int i; - (void)event; - (void)arg; - i = read(fd, &vc, sizeof vc); assert(i == sizeof vc); AZ(pthread_mutex_lock(&vbemtx)); @@ -181,13 +178,11 @@ */ static void -vbe_rdf(int fd, short event, void *arg) +vbe_rdf(int fd __unused, short event __unused, void *arg) { struct vbe_conn *vc; int j; - (void)event; - vc = arg; AZ(pthread_mutex_lock(&vbemtx)); if (vc->inuse) { @@ -207,12 +202,10 @@ /* Backend monitoring thread -----------------------------------------*/ static void * -vbe_main(void *priv) +vbe_main(void *priv __unused) { struct event pev; - (void)priv; - vbe_evb = event_init(); assert(vbe_evb != NULL); Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2006-07-10 09:47:56 UTC (rev 396) @@ -71,7 +71,7 @@ } void -cli_func_url_purge(struct cli *cli, char **av, void *priv) +cli_func_url_purge(struct cli *cli, char **av, void *priv __unused) { AddBan(av[2]); Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-10 09:47:56 UTC (rev 396) @@ -43,7 +43,7 @@ */ static void * -exp_hangman(void *arg) +exp_hangman(void *arg __unused) { struct object *o; time_t t; @@ -84,7 +84,7 @@ */ static void * -exp_prefetch(void *arg) +exp_prefetch(void *arg __unused) { struct object *o; time_t t; @@ -116,14 +116,12 @@ } assert(sp.handling == VCL_RET_DISCARD); } - - return ("FOOBAR"); } /*--------------------------------------------------------------------*/ static int -object_cmp(void *priv, void *a, void *b) +object_cmp(void *priv __unused, void *a, void *b) { struct object *aa, *bb; @@ -133,7 +131,7 @@ } static void -object_update(void *priv, void *p, unsigned u) +object_update(void *priv __unused, void *p, unsigned u) { struct object *o = p; Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-10 09:47:56 UTC (rev 396) @@ -30,7 +30,7 @@ /*--------------------------------------------------------------------*/ static int -fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char *b) +fetch_straight(struct sess *sp, int fd, struct http *hp, char *b) { int i; char *e; @@ -70,7 +70,7 @@ /* XXX: Cleanup. It must be possible somehow :-( */ static int -fetch_chunked(struct worker *w, struct sess *sp, int fd, struct http *hp) +fetch_chunked(struct sess *sp, int fd, struct http *hp) { int i; char *b, *q, *e; @@ -180,7 +180,7 @@ #include static int -fetch_eof(struct worker *w, struct sess *sp, int fd, struct http *hp) +fetch_eof(struct sess *sp, int fd, struct http *hp) { int i; char *b, *e; @@ -273,11 +273,11 @@ http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); if (body) { if (http_GetHdr(hp, "Content-Length", &b)) - cls = fetch_straight(w, sp, vc->fd, hp, b); + cls = fetch_straight(sp, vc->fd, hp, b); else if (http_HdrIs(hp, "Transfer-Encoding", "chunked")) - cls = fetch_chunked(w, sp, vc->fd, hp); + cls = fetch_chunked(sp, vc->fd, hp); else - cls = fetch_eof(w, sp, vc->fd, hp); + cls = fetch_eof(sp, vc->fd, hp); sbuf_printf(w->sb, "Content-Length: %u\r\n", sp->obj->len); } else cls = 0; Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-10 09:47:56 UTC (rev 396) @@ -308,7 +308,7 @@ #include static void -http_read_f(int fd, short event, void *arg) +http_read_f(int fd, short event __unused, void *arg) { struct http *hp = arg; unsigned l; Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-10 09:47:56 UTC (rev 396) @@ -21,6 +21,7 @@ struct stevedore *stevedore; pthread_mutex_t sessmtx; +struct varnish_stats *VSL_stats; /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-10 09:47:56 UTC (rev 396) @@ -21,7 +21,7 @@ /*--------------------------------------------------------------------*/ static int -pass_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char *bi) +pass_straight(struct sess *sp, int fd, struct http *hp, char *bi) { int i; char *b, *e; @@ -63,7 +63,7 @@ /*--------------------------------------------------------------------*/ static int -pass_chunked(struct worker *w, struct sess *sp, int fd, struct http *hp) +pass_chunked(struct sess *sp, int fd, struct http *hp) { int i, j; char *b, *q, *e; @@ -175,11 +175,11 @@ vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); if (http_GetHdr(hp, "Content-Length", &b)) - cls = pass_straight(w, sp, vc->fd, hp, b); + cls = pass_straight(sp, vc->fd, hp, b); else if (http_HdrIs(hp, "Connection", "close")) - cls = pass_straight(w, sp, vc->fd, hp, NULL); + cls = pass_straight(sp, vc->fd, hp, NULL); else if (http_HdrIs(hp, "Transfer-Encoding", "chunked")) - cls = pass_chunked(w, sp, vc->fd, hp); + cls = pass_chunked(sp, vc->fd, hp); else { INCOMPL(); cls = 1; Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-10 09:47:56 UTC (rev 396) @@ -19,7 +19,7 @@ }; static void -rdf(int fd, short event, void *arg) +rdf(int fd, short event __unused, void *arg) { int i, j; struct edir *ep; Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-10 09:47:56 UTC (rev 396) @@ -26,7 +26,6 @@ #endif struct varnish_stats *VSL_stats; - static struct shmloghead *loghead; static unsigned char *logstart; static pthread_mutex_t vsl_mutex; @@ -132,8 +131,8 @@ /* XXX check sanity of loghead */ logstart = (unsigned char *)loghead + loghead->start; - VSL_stats = &loghead->stats; AZ(pthread_mutex_init(&vsl_mutex, NULL)); + VSL_stats = &loghead->stats; } /*--------------------------------------------------------------------*/ Added: trunk/varnish-cache/bin/varnishd/cache_shmlog.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.h 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.h 2006-07-10 09:47:56 UTC (rev 396) @@ -0,0 +1,6 @@ +/* + * $Id$ + */ + +void VSL_MgtInit(const char *fn, unsigned size); +extern struct varnish_stats *VSL_stats; Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-07-10 09:47:56 UTC (rev 396) @@ -99,7 +99,7 @@ } void -cli_func_config_list(struct cli *cli, char **av, void *priv) +cli_func_config_list(struct cli *cli, char **av __unused, void *priv __unused) { struct vcls *vcl; @@ -123,7 +123,7 @@ } void -cli_func_config_load(struct cli *cli, char **av, void *priv) +cli_func_config_load(struct cli *cli, char **av, void *priv __unused) { struct vcls *vcl; @@ -166,13 +166,13 @@ } void -cli_func_config_unload(struct cli *cli, char **av, void *priv) +cli_func_config_unload(struct cli *cli, char **av __unused, void *priv __unused) { cli_result(cli, CLIS_UNIMPL); } void -cli_func_config_use(struct cli *cli, char **av, void *priv) +cli_func_config_use(struct cli *cli, char **av, void *priv __unused) { struct vcls *vcl; Modified: trunk/varnish-cache/bin/varnishd/cli_event.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cli_event.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/cli_event.c 2006-07-10 09:47:56 UTC (rev 396) @@ -110,7 +110,7 @@ } static void -wrcb(struct bufferevent *bev, void *arg) +wrcb(struct bufferevent *bev __unused, void *arg) { struct cli *cli = arg; Modified: trunk/varnish-cache/bin/varnishd/mgt.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt.h 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/mgt.h 2006-07-10 09:47:56 UTC (rev 396) @@ -24,5 +24,4 @@ extern struct hash_slinger hsl_slinger; extern struct hash_slinger hcl_slinger; -void VSL_MgtInit(const char *fn, unsigned size); -extern struct varnish_stats *VSL_stats; +#include "cache_shmlog.h" Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_child.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/mgt_child.c 2006-07-10 09:47:56 UTC (rev 396) @@ -58,7 +58,7 @@ */ static void -std_rdcb(struct bufferevent *bev, void *arg) +std_rdcb(struct bufferevent *bev, void *arg __unused) { const char *p; @@ -132,7 +132,7 @@ } static void -cli_rdcb(struct bufferevent *bev, void *arg) +cli_rdcb(struct bufferevent *bev, void *arg __unused) { const char *p; char **av; @@ -174,14 +174,14 @@ /*--------------------------------------------------------------------*/ static void -child_pingpong_ccb(unsigned u, const char *r, void *priv) +child_pingpong_ccb(unsigned u __unused, const char *r __unused, void *priv __unused) { /* XXX: reset keepalive timer */ } static void -child_pingpong(int a, short b, void *c) +child_pingpong(int a __unused, short b __unused, void *c __unused) { time_t t; struct timeval tv; Modified: trunk/varnish-cache/bin/varnishd/tcp.c =================================================================== --- trunk/varnish-cache/bin/varnishd/tcp.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/tcp.c 2006-07-10 09:47:56 UTC (rev 396) @@ -11,6 +11,7 @@ #include #include "heritage.h" +#include "mgt.h" #include "libvarnish.h" static void Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-10 09:28:26 UTC (rev 395) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-10 09:47:56 UTC (rev 396) @@ -50,7 +50,7 @@ } static void -m_cli_func_passthrough(struct cli *cli, char **av, void *priv) +m_cli_func_passthrough(struct cli *cli, char **av, void *priv __unused) { (void)av; From phk at projects.linpro.no Mon Jul 10 09:48:26 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 11:48:26 +0200 (CEST) Subject: r397 - in trunk/varnish-cache/lib: libvarnish libvcl Message-ID: <20060710094826.9DC9C1EC38D@projects.linpro.no> Author: phk Date: 2006-07-10 11:48:26 +0200 (Mon, 10 Jul 2006) New Revision: 397 Modified: trunk/varnish-cache/lib/libvarnish/time.c trunk/varnish-cache/lib/libvcl/vcl_compile.c Log: Fix cosmetic warnings Modified: trunk/varnish-cache/lib/libvarnish/time.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/time.c 2006-07-10 09:47:56 UTC (rev 396) +++ trunk/varnish-cache/lib/libvarnish/time.c 2006-07-10 09:48:26 UTC (rev 397) @@ -23,6 +23,8 @@ #include #include +#include "libvarnish.h" + void TIM_format(time_t t, char *p) { Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-07-10 09:47:56 UTC (rev 396) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-07-10 09:48:26 UTC (rev 397) @@ -647,8 +647,9 @@ } /*--------------------------------------------------------------------*/ + static struct var * -HeaderVar(struct tokenlist *tl, struct token *t, struct var *vh) +HeaderVar(struct tokenlist *tl __unused, struct token *t, struct var *vh) { char *p; struct var *v; @@ -1847,18 +1848,18 @@ /*--------------------------------------------------------------------*/ -int -VCC_T_render(FILE *f, const struct printf_info *info, const void *const *args) +static int +VCC_T_render(FILE *f, const struct printf_info *info __unused, const void *const *args) { const struct token *t; - t = *((const struct token **) (args[0])); + t = *((const struct token * const*) (args[0])); return (fprintf(f, "%*.*s", t->e - t->b, t->e - t->b, t->b)); } -int -VCC_T_arginfo(const struct printf_info *info, size_t n, int *argtypes) +static int +VCC_T_arginfo(const struct printf_info *info __unused, size_t n, int *argtypes) { if (n > 0) From phk at projects.linpro.no Mon Jul 10 09:51:01 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 11:51:01 +0200 (CEST) Subject: r398 - trunk/varnish-cache/bin/varnishstat Message-ID: <20060710095101.3D1701EC382@projects.linpro.no> Author: phk Date: 2006-07-10 11:51:01 +0200 (Mon, 10 Jul 2006) New Revision: 398 Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c Log: Silence cosmetic warning. Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-07-10 09:48:26 UTC (rev 397) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-07-10 09:51:01 UTC (rev 398) @@ -40,6 +40,8 @@ a1 = a2 = a3 = 0; n1 = n2 = n3 = 0; + lt = 0; + lh = VSL_OpenLog(); VSL_stats = &lh->stats; From phk at projects.linpro.no Mon Jul 10 09:52:06 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 11:52:06 +0200 (CEST) Subject: r399 - trunk/varnish-cache/lib/libvarnishapi Message-ID: <20060710095206.BDAF11EC38D@projects.linpro.no> Author: phk Date: 2006-07-10 11:52:06 +0200 (Mon, 10 Jul 2006) New Revision: 399 Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Fix cosmetic warning Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-10 09:51:01 UTC (rev 398) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-10 09:52:06 UTC (rev 399) @@ -59,7 +59,7 @@ } unsigned char * -VSL_NextLog(struct shmloghead *lh, unsigned char **pp) +VSL_NextLog(struct shmloghead *lh __unused, unsigned char **pp) { unsigned char *p; From phk at projects.linpro.no Mon Jul 10 10:05:48 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 12:05:48 +0200 (CEST) Subject: r400 - trunk/varnish-cache/bin/varnishd Message-ID: <20060710100548.95D2D1EC35F@projects.linpro.no> Author: phk Date: 2006-07-10 12:05:48 +0200 (Mon, 10 Jul 2006) New Revision: 400 Modified: trunk/varnish-cache/bin/varnishd/heritage.h trunk/varnish-cache/bin/varnishd/varnishd.c Log: Drop the max length of first line, it's too expensive to enforce. Modified: trunk/varnish-cache/bin/varnishd/heritage.h =================================================================== --- trunk/varnish-cache/bin/varnishd/heritage.h 2006-07-10 09:52:06 UTC (rev 399) +++ trunk/varnish-cache/bin/varnishd/heritage.h 2006-07-10 10:05:48 UTC (rev 400) @@ -40,7 +40,6 @@ unsigned wthread_min, wthread_max; /* Memory allocation hints */ - unsigned mem_http_1_line; unsigned mem_http_headerspace; unsigned mem_http_headers; unsigned mem_workspace; Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-10 09:52:06 UTC (rev 399) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-10 10:05:48 UTC (rev 400) @@ -457,7 +457,6 @@ heritage.default_ttl = 120; heritage.wthread_min = 5; heritage.wthread_max = 5; - heritage.mem_http_1_line= 512; heritage.mem_http_headerspace= 4096; heritage.mem_http_headers= 32; heritage.mem_workspace = 0; From phk at projects.linpro.no Mon Jul 10 10:06:21 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 12:06:21 +0200 (CEST) Subject: r401 - trunk/varnish-cache/bin/varnishd Message-ID: <20060710100621.B55821EC382@projects.linpro.no> Author: phk Date: 2006-07-10 12:06:21 +0200 (Mon, 10 Jul 2006) New Revision: 401 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Polish HTTP reception a little bit Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-10 10:05:48 UTC (rev 400) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-10 10:06:21 UTC (rev 401) @@ -279,10 +279,15 @@ { char *p; + assert(hp->v <= hp->e); + assert(*hp->v == '\0'); + /* Skip any leading white space */ for (p = hp->s ; p < hp->v && isspace(*p); p++) continue; - if (p >= hp->v) + if (p >= hp->v) { + hp->v = hp->s; return (0); + } while (1) { /* XXX: we could save location of all linebreaks for later */ p = strchr(p, '\n'); @@ -312,46 +317,42 @@ { struct http *hp = arg; unsigned l; - int i; + int i, ret = 0; l = hp->e - hp->v; if (l <= 1) { + VSL(SLT_HttpError, fd, "Received too much"); VSLR(SLT_Debug, fd, hp->s, hp->v); hp->t = NULL; - event_del(&hp->ev); - if (hp->callback != NULL) - hp->callback(hp->arg, 1); - return; + ret = 1; + } else { + errno = 0; + i = read(fd, hp->v, l - 1); + if (i > 0) { + hp->v += i; + *hp->v = '\0'; + if (!http_header_complete(hp)) + return; + } else { + if (hp->v != hp->s) { + VSL(SLT_HttpError, fd, + "Received (only) %d bytes, errno %d", + hp->v - hp->s, errno); + VSLR(SLT_Debug, fd, hp->s, hp->v); + } else if (errno == 0) + VSL(SLT_HttpError, fd, "Received nothing"); + else + VSL(SLT_HttpError, fd, + "Received errno %d", errno); + hp->t = NULL; + ret = 2; + } } - assert(l > 1); - errno = 0; - i = read(fd, hp->v, l - 1); - if (i <= 0) { - if (hp->v != hp->s) - VSL(SLT_HttpError, fd, - "Received (only) %d bytes, errno %d", - hp->v - hp->s, errno); - else if (errno == 0) - VSL(SLT_HttpError, fd, "Received nothing"); - else - VSL(SLT_HttpError, fd, "Received errno %d", errno); - VSLR(SLT_Debug, fd, hp->s, hp->v); - hp->t = NULL; - event_del(&hp->ev); - if (hp->callback != NULL) - hp->callback(hp->arg, 2); - return; - } - hp->v += i; - *hp->v = '\0'; - if (!http_header_complete(hp)) - return; - - assert(hp->t != NULL); + assert(hp->t != NULL || ret != 0); event_del(&hp->ev); if (hp->callback != NULL) - hp->callback(hp->arg, 0); + hp->callback(hp->arg, ret); } /*--------------------------------------------------------------------*/ From phk at projects.linpro.no Mon Jul 10 10:31:49 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 12:31:49 +0200 (CEST) Subject: r402 - trunk/varnish-cache/bin/varnishd Message-ID: <20060710103149.861B01EC383@projects.linpro.no> Author: phk Date: 2006-07-10 12:31:49 +0200 (Mon, 10 Jul 2006) New Revision: 402 Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pipe.c Log: put backend session linkage in shmemlog for pipe and pass Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-10 10:06:21 UTC (rev 401) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-10 10:31:49 UTC (rev 402) @@ -155,6 +155,7 @@ vc = VBE_GetFd(sp->backend, sp->xid); assert(vc != NULL); + VSL(SLT_Backend, sp->fd, "%d %s", vc->fd, sp->backend->vcl_name); http_BuildSbuf(vc->fd, Build_Pass, w->sb, sp->http); i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-10 10:06:21 UTC (rev 401) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-10 10:31:49 UTC (rev 402) @@ -51,6 +51,7 @@ vc = VBE_GetFd(sp->backend, sp->xid); assert(vc != NULL); + VSL(SLT_Backend, sp->fd, "%d %s", vc->fd, sp->backend->vcl_name); http_BuildSbuf(vc->fd, Build_Pipe, w->sb, sp->http); i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); From phk at projects.linpro.no Mon Jul 10 10:56:12 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 12:56:12 +0200 (CEST) Subject: r403 - trunk/varnish-cache/bin/varnishd Message-ID: <20060710105612.AB5051EC383@projects.linpro.no> Author: phk Date: 2006-07-10 12:56:12 +0200 (Mon, 10 Jul 2006) New Revision: 403 Added: trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/shmlog.c Removed: trunk/varnish-cache/bin/varnishd/cache_shmlog.c trunk/varnish-cache/bin/varnishd/cache_shmlog.h Modified: trunk/varnish-cache/bin/varnishd/Makefile.am trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/mgt.h Log: Be more consistent. cache_shmlog.c contains stuff for both cache and mgt, so remove the cache_ prefix. Rename cache_shmlog.h to common.h and put joint stuff there. Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-10 10:31:49 UTC (rev 402) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-10 10:56:12 UTC (rev 403) @@ -26,7 +26,6 @@ cache_pass.c \ cache_pipe.c \ cache_response.c \ - cache_shmlog.c \ cache_vcl.c \ cache_vrt.c \ cli_event.c \ @@ -34,6 +33,7 @@ hash_classic.c \ mgt_child.c \ rfc2616.c \ + shmlog.c \ storage_file.c \ storage_malloc.c \ tcp.c \ Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 10:31:49 UTC (rev 402) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 10:56:12 UTC (rev 403) @@ -9,6 +9,7 @@ #include #include "vcl_returns.h" +#include "common.h" #define VCA_ADDRBUFSIZE 64 /* Sizeof ascii network address */ @@ -241,7 +242,6 @@ void DealWithSession(void *arg); /* cache_shmlog.c */ -#include "cache_shmlog.h" void VSL_Init(void); #ifdef SHMLOGHEAD_MAGIC Deleted: trunk/varnish-cache/bin/varnishd/cache_shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-10 10:31:49 UTC (rev 402) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-07-10 10:56:12 UTC (rev 403) @@ -1,174 +0,0 @@ -/* - * $Id$ - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "libvarnish.h" -#include "shmlog.h" -#include "cache.h" - -#include "heritage.h" - -#ifndef MAP_HASSEMAPHORE -#define MAP_HASSEMAPHORE 0 /* XXX Linux */ -#endif - -#ifndef MAP_NOSYNC -#define MAP_NOSYNC 0 /* XXX Linux */ -#endif - -struct varnish_stats *VSL_stats; -static struct shmloghead *loghead; -static unsigned char *logstart; -static pthread_mutex_t vsl_mutex; - -/* - * This variant copies a byte-range directly to the log, without - * taking the detour over sprintf() - */ - -static void -vsl_wrap(void) -{ - - *logstart = SLT_ENDMARKER; - logstart[loghead->ptr] = SLT_WRAPMARKER; - loghead->ptr = 0; -} - -void -VSLR(enum shmlogtag tag, unsigned id, const char *b, const char *e) -{ - unsigned char *p; - unsigned l; - - assert(b != NULL); - if (e == NULL) - e = strchr(b, '\0'); - assert(e != NULL); - - /* Truncate */ - l = e - b; - if (l > 255) { - l = 255; - e = b + l; - } - - AZ(pthread_mutex_lock(&vsl_mutex)); - assert(loghead->ptr < loghead->size); - - /* Wrap if necessary */ - if (loghead->ptr + 4 + l + 1 > loghead->size) - vsl_wrap(); - p = logstart + loghead->ptr; - p[1] = l; - p[2] = id >> 8; - p[3] = id & 0xff; - memcpy(p + 4, b, l); - p[4 + l] = SLT_ENDMARKER; - p[0] = tag; - - loghead->ptr += 4 + l; - assert(loghead->ptr < loghead->size); - AZ(pthread_mutex_unlock(&vsl_mutex)); -} - - -void -VSL(enum shmlogtag tag, unsigned id, const char *fmt, ...) -{ - va_list ap; - unsigned char *p; - unsigned n; - - va_start(ap, fmt); - - AZ(pthread_mutex_lock(&vsl_mutex)); - assert(loghead->ptr < loghead->size); - - /* Wrap if we cannot fit a full size record */ - if (loghead->ptr + 4 + 255 + 1 > loghead->size) - vsl_wrap(); - - p = logstart + loghead->ptr; - n = 0; - if (fmt != NULL) { - n = vsnprintf((char *)(p + 4), 256, fmt, ap); - if (n > 255) - n = 255; /* we truncate long fields */ - } - p[1] = n; - p[2] = id >> 8; - p[3] = id & 0xff; - p[4 + n] = SLT_ENDMARKER; - p[0] = tag; - - loghead->ptr += 4 + n; - assert(loghead->ptr < loghead->size); - - AZ(pthread_mutex_unlock(&vsl_mutex)); - - va_end(ap); -} - -void -VSL_Init(void) -{ - - loghead = mmap(NULL, heritage.vsl_size, - PROT_READ|PROT_WRITE, - MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED, - heritage.vsl_fd, 0); - assert(loghead != MAP_FAILED); - - /* XXX check sanity of loghead */ - logstart = (unsigned char *)loghead + loghead->start; - AZ(pthread_mutex_init(&vsl_mutex, NULL)); - VSL_stats = &loghead->stats; -} - -/*--------------------------------------------------------------------*/ - -void -VSL_MgtInit(const char *fn, unsigned size) -{ - struct shmloghead slh; - int i; - - heritage.vsl_fd = open(fn, O_RDWR | O_CREAT, 0644); - if (heritage.vsl_fd < 0) { - fprintf(stderr, "Could not open %s: %s\n", - fn, strerror(errno)); - exit (1); - } - i = read(heritage.vsl_fd, &slh, sizeof slh); - if (i != sizeof slh || slh.magic != SHMLOGHEAD_MAGIC) { - /* XXX more checks */ - - slh.magic = SHMLOGHEAD_MAGIC; - slh.size = size; - slh.ptr = 0; - slh.start = sizeof slh; - AZ(lseek(heritage.vsl_fd, 0, SEEK_SET)); - i = write(heritage.vsl_fd, &slh, sizeof slh); - assert(i == sizeof slh); - AZ(ftruncate(heritage.vsl_fd, sizeof slh + size)); - } - heritage.vsl_size = slh.size + slh.start; - - /* - * Call VSL_Init so that we get a VSL_stats pointer in the - * management process as well. - */ - VSL_Init(); - memset(VSL_stats, 0, sizeof *VSL_stats); -} - Deleted: trunk/varnish-cache/bin/varnishd/cache_shmlog.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.h 2006-07-10 10:31:49 UTC (rev 402) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.h 2006-07-10 10:56:12 UTC (rev 403) @@ -1,6 +0,0 @@ -/* - * $Id$ - */ - -void VSL_MgtInit(const char *fn, unsigned size); -extern struct varnish_stats *VSL_stats; Copied: trunk/varnish-cache/bin/varnishd/common.h (from rev 396, trunk/varnish-cache/bin/varnishd/cache_shmlog.h) Modified: trunk/varnish-cache/bin/varnishd/mgt.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt.h 2006-07-10 10:31:49 UTC (rev 402) +++ trunk/varnish-cache/bin/varnishd/mgt.h 2006-07-10 10:56:12 UTC (rev 403) @@ -2,6 +2,8 @@ * $Id$ */ +#include "common.h" + extern struct event_base *mgt_eb; void mgt_child_start(void); @@ -24,4 +26,3 @@ extern struct hash_slinger hsl_slinger; extern struct hash_slinger hcl_slinger; -#include "cache_shmlog.h" Copied: trunk/varnish-cache/bin/varnishd/shmlog.c (from rev 396, trunk/varnish-cache/bin/varnishd/cache_shmlog.c) From phk at projects.linpro.no Mon Jul 10 11:24:37 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 13:24:37 +0200 (CEST) Subject: r404 - trunk/varnish-cache/bin/varnishd Message-ID: <20060710112437.BE0641EC383@projects.linpro.no> Author: phk Date: 2006-07-10 13:24:37 +0200 (Mon, 10 Jul 2006) New Revision: 404 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/tcp.c Log: Move sockaddr->ascii conversion to tcp.c shmlog both ends of backend connections. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 10:56:12 UTC (rev 403) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 11:24:37 UTC (rev 404) @@ -11,8 +11,6 @@ #include "vcl_returns.h" #include "common.h" -#define VCA_ADDRBUFSIZE 64 /* Sizeof ascii network address */ - struct event_base; struct cli; struct sbuf; @@ -133,7 +131,7 @@ unsigned xid; /* formatted ascii client address */ - char addr[VCA_ADDRBUFSIZE]; + char addr[TCP_ADDRBUFFSIZE]; /* HTTP request */ struct http *http; Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-10 10:56:12 UTC (rev 403) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-10 11:24:37 UTC (rev 404) @@ -210,9 +210,8 @@ accept_f(int fd, short event __unused, void *arg __unused) { socklen_t l; - struct sockaddr addr[2]; + struct sockaddr addr[2]; /* XXX: IPv6 hack */ struct sess *sp; - char port[NI_MAXSERV]; int i; struct linger linger; @@ -238,15 +237,7 @@ AZ(setsockopt(sp->fd, SOL_SOCKET, SO_LINGER, &linger, sizeof linger)); #endif - i = getnameinfo(addr, l, - sp->addr, VCA_ADDRBUFSIZE, - port, sizeof port, NI_NUMERICHOST | NI_NUMERICSERV); - if (i) { - printf("getnameinfo = %d %s\n", i, - gai_strerror(i)); - } - strlcat(sp->addr, " ", VCA_ADDRBUFSIZE); - strlcat(sp->addr, port, VCA_ADDRBUFSIZE); + TCP_name(addr, l, sp->addr); VSL(SLT_SessionOpen, sp->fd, "%s", sp->addr); time(&sp->t_resp); TAILQ_INSERT_TAIL(&sesshead, sp, list); Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-10 10:56:12 UTC (rev 403) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-10 11:24:37 UTC (rev 404) @@ -106,6 +106,7 @@ { struct addrinfo *res, *res0, hint; int error, s; + char buf[TCP_ADDRBUFFSIZE * 2 + 1], *p; assert(bp != NULL); assert(bp->hostname != NULL); @@ -132,15 +133,22 @@ continue; } error = connect(s, res0->ai_addr, res0->ai_addrlen); - if (!error) + if (!error) break; VSL(SLT_Debug, 0, "Connect errno=%d", errno); close(s); s = -1; } while ((res0 = res0->ai_next) != NULL); + vc->fd = s; + if (s >= 0) { + TCP_myname(s, buf); + p = strchr(buf, '\0'); + assert(p != NULL); + *p++ = ' '; + TCP_name(res0->ai_addr, res0->ai_addrlen, p); + VSL(SLT_BackendOpen, vc->fd, buf); + } freeaddrinfo(res); - vc->fd = s; - VSL(SLT_BackendOpen, vc->fd, ""); return; } Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2006-07-10 10:56:12 UTC (rev 403) +++ trunk/varnish-cache/bin/varnishd/common.h 2006-07-10 11:24:37 UTC (rev 404) @@ -2,5 +2,16 @@ * $Id$ */ +struct sockaddr; + +/* shmlog.c */ void VSL_MgtInit(const char *fn, unsigned size); extern struct varnish_stats *VSL_stats; + +/* tcp.c */ +#define TCP_ADDRBUFFSIZE 64 /* Sizeof ascii representation */ + +void TCP_name(struct sockaddr *addr, unsigned l, char *buf); +void TCP_myname(int sock, char *buf); + + Modified: trunk/varnish-cache/bin/varnishd/tcp.c =================================================================== --- trunk/varnish-cache/bin/varnishd/tcp.c 2006-07-10 10:56:12 UTC (rev 403) +++ trunk/varnish-cache/bin/varnishd/tcp.c 2006-07-10 11:24:37 UTC (rev 404) @@ -14,6 +14,40 @@ #include "mgt.h" #include "libvarnish.h" +/*--------------------------------------------------------------------*/ + +void +TCP_name(struct sockaddr *addr, unsigned l, char *buf) +{ + int i; + char port[NI_MAXSERV]; + + i = getnameinfo(addr, l, buf, TCP_ADDRBUFFSIZE, + port, sizeof port, NI_NUMERICHOST | NI_NUMERICSERV); + if (i) { + printf("getnameinfo = %d %s\n", i, gai_strerror(i)); + strcpy(buf, "Conversion:Failed"); + return; + } + strlcat(buf, " ", TCP_ADDRBUFFSIZE); + strlcat(buf, port, TCP_ADDRBUFFSIZE); +} + +/*--------------------------------------------------------------------*/ + +void +TCP_myname(int sock, char *buf) +{ + struct sockaddr addr[2]; /* XXX: IPv6 hack */ + socklen_t l; + + l = sizeof addr; + AZ(getsockname(sock, addr, &l)); + TCP_name(addr, l, buf); +} + +/*--------------------------------------------------------------------*/ + static void accept_filter(int fd) { From phk at projects.linpro.no Mon Jul 10 12:00:20 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 14:00:20 +0200 (CEST) Subject: r405 - trunk/varnish-cache/bin/varnishd Message-ID: <20060710120020.79CD91EC228@projects.linpro.no> Author: phk Date: 2006-07-10 14:00:20 +0200 (Mon, 10 Jul 2006) New Revision: 405 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_backend.c Log: Rework the "connect to backend logic". Avoid calling getaddrinfo() for every connect by catching the result in the backend structure. Minimize number of socket/connect calls by caching the last good address in the backend structure. If all addresses in the cached getaddrinfo() result fails, call getaddrinfo() again (to catch DNS changes) and try the list again. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 11:24:37 UTC (rev 404) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 12:00:20 UTC (rev 405) @@ -157,8 +157,10 @@ const char *hostname; const char *portname; unsigned ip; + + struct addrinfo *addr; + struct addrinfo *last_addr; #if 0 - struct addrinfo *addr; double responsetime; double timeout; double bandwidth; Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-10 11:24:37 UTC (rev 404) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-10 12:00:20 UTC (rev 405) @@ -94,22 +94,17 @@ free(vb->vbcm); } -/*-------------------------------------------------------------------- - * XXX: we should not call getaddrinfo() every time, we should cache - * and apply round-robin with blacklisting of entries that do not respond - * etc. Periodic re-lookups to capture changed DNS records would also - * be a good thing in that case. - */ +/*--------------------------------------------------------------------*/ static void -connect_to_backend(struct vbe_conn *vc, struct backend *bp) +vbe_lookup(struct backend *bp) { - struct addrinfo *res, *res0, hint; - int error, s; - char buf[TCP_ADDRBUFFSIZE * 2 + 1], *p; + struct addrinfo *res, hint; + int error; - assert(bp != NULL); - assert(bp->hostname != NULL); + if (bp->addr != NULL) + freeaddrinfo(bp->addr); + memset(&hint, 0, sizeof hint); hint.ai_family = PF_UNSPEC; hint.ai_socktype = SOCK_STREAM; @@ -120,38 +115,97 @@ if (error) { if (res != NULL) freeaddrinfo(res); - fprintf(stderr, "getaddrinfo: %s\n", - gai_strerror(error)); + printf("getaddrinfo: %s\n", gai_strerror(error)); /* XXX */ + bp->addr = NULL; return; } - res0 = res; - do { - s = socket(res0->ai_family, res0->ai_socktype, - res0->ai_protocol); - if (s < 0) { - VSL(SLT_Debug, 0, "Socket errno=%d", errno); - continue; - } - error = connect(s, res0->ai_addr, res0->ai_addrlen); - if (!error) - break; - VSL(SLT_Debug, 0, "Connect errno=%d", errno); + bp->addr = res; +} + +/*--------------------------------------------------------------------*/ + +static int +vbe_sock_conn(struct addrinfo *ai) +{ + int s; + + s = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (s < 0) + return (s); + else if (connect(s, ai->ai_addr, ai->ai_addrlen)) { close(s); s = -1; - } while ((res0 = res0->ai_next) != NULL); - vc->fd = s; - if (s >= 0) { - TCP_myname(s, buf); - p = strchr(buf, '\0'); - assert(p != NULL); - *p++ = ' '; - TCP_name(res0->ai_addr, res0->ai_addrlen, p); - VSL(SLT_BackendOpen, vc->fd, buf); + } + return (s); +} + +/*--------------------------------------------------------------------*/ + +static int +vbe_conn_try(struct backend *bp, struct addrinfo **pai) +{ + struct addrinfo *ai; + int s; + + /* First try the cached good address, and any following it */ + for (ai = bp->last_addr; ai != NULL; ai = ai->ai_next) { + s = vbe_sock_conn(ai); + if (s >= 0) { + bp->last_addr = ai; + *pai = ai; + return (s); + } } - freeaddrinfo(res); - return; + + /* Then try the list until the cached last good address */ + for (ai = bp->addr; ai != bp->last_addr; ai = ai->ai_next) { + s = vbe_sock_conn(ai); + if (s >= 0) { + bp->last_addr = ai; + *pai = ai; + return (s); + } + } + + /* Then do another lookup to catch DNS changes */ + vbe_lookup(bp); + + /* And try the entire list */ + for (ai = bp->addr; ai != NULL; ai = ai->ai_next) { + s = vbe_sock_conn(ai); + if (s >= 0) { + bp->last_addr = ai; + *pai = ai; + return (s); + } + } + + return (-1); } +static int +vbe_connect(struct backend *bp) +{ + int s; + char buf[TCP_ADDRBUFFSIZE * 2 + 1], *p; + struct addrinfo *ai; + + assert(bp != NULL); + assert(bp->hostname != NULL); + + s = vbe_conn_try(bp, &ai); + if (s < 0) + return (s); + + TCP_myname(s, buf); + p = strchr(buf, '\0'); + assert(p != NULL); + *p++ = ' '; + TCP_name(ai->ai_addr, ai->ai_addrlen, p); + VSL(SLT_BackendOpen, s, buf); + return (s); +} + /*-------------------------------------------------------------------- * When backend connections have been used, they are passed to us through * the vbe_pipe. If fd == -1 it has been closed and will be reclaimed, @@ -278,7 +332,7 @@ vc->inuse = 1; TAILQ_INSERT_TAIL(&vp->bconn, vc, list); AZ(pthread_mutex_unlock(&vbemtx)); - connect_to_backend(vc, bp); + vc->fd = vbe_connect(bp); if (vc->fd < 0) { AZ(pthread_mutex_lock(&vbemtx)); TAILQ_REMOVE(&vc->vbe->bconn, vc, list); From phk at projects.linpro.no Mon Jul 10 13:48:01 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 15:48:01 +0200 (CEST) Subject: r406 - trunk/varnish-cache/bin/varnishd Message-ID: <20060710134801.626B21EC228@projects.linpro.no> Author: phk Date: 2006-07-10 15:48:01 +0200 (Mon, 10 Jul 2006) New Revision: 406 Modified: trunk/varnish-cache/bin/varnishd/shmlog.c Log: Dump errno and strerror in assert Modified: trunk/varnish-cache/bin/varnishd/shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-10 12:00:20 UTC (rev 405) +++ trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-10 13:48:01 UTC (rev 406) @@ -172,3 +172,16 @@ memset(VSL_stats, 0, sizeof *VSL_stats); } +/*--------------------------------------------------------------------*/ + +void +__assert(const char *func, const char *file, int line, const char *failedexpr) +{ + (void)fprintf(stderr, + "\r\nAssertion failed: (%s)\n" + " function %s, file %s, line %d.\n" + " errno %d = \"%s\"\n", + failedexpr, func, file, line, errno, strerror(errno)); + abort(); + /* NOTREACHED */ +} From phk at projects.linpro.no Mon Jul 10 13:58:24 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 15:58:24 +0200 (CEST) Subject: r407 - trunk/varnish-cache/contrib/libevent Message-ID: <20060710135824.ED7871EC35F@projects.linpro.no> Author: phk Date: 2006-07-10 15:58:24 +0200 (Mon, 10 Jul 2006) New Revision: 407 Modified: trunk/varnish-cache/contrib/libevent/event.c Log: Don't ever set current_base in our version of libevent in order to flush out any bugs it might cause. Modified: trunk/varnish-cache/contrib/libevent/event.c =================================================================== --- trunk/varnish-cache/contrib/libevent/event.c 2006-07-10 13:48:01 UTC (rev 406) +++ trunk/varnish-cache/contrib/libevent/event.c 2006-07-10 13:58:24 UTC (rev 407) @@ -194,7 +194,9 @@ /* allocate a single active event queue */ event_base_priority_init(mybase, 1); +#if 0 current_base = mybase; +#endif return (mybase); } @@ -504,7 +506,8 @@ ev->ev_pncalls = NULL; /* by default, we put new events into the middle priority */ - ev->ev_pri = current_base->nactivequeues/2; + if (current_base != NULL) + ev->ev_pri = current_base->nactivequeues/2; } int From phk at projects.linpro.no Mon Jul 10 13:59:13 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 15:59:13 +0200 (CEST) Subject: r408 - trunk/varnish-cache/bin/varnishd Message-ID: <20060710135913.EA66D1EC382@projects.linpro.no> Author: phk Date: 2006-07-10 15:59:13 +0200 (Mon, 10 Jul 2006) New Revision: 408 Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c trunk/varnish-cache/bin/varnishd/varnishd.c Log: use explicit eventbase. Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_child.c 2006-07-10 13:58:24 UTC (rev 407) +++ trunk/varnish-cache/bin/varnishd/mgt_child.c 2006-07-10 13:59:13 UTC (rev 408) @@ -237,14 +237,19 @@ child_std = bufferevent_new(child_fds[0], std_rdcb, std_wrcb, std_excb, NULL); assert(child_std != NULL); + bufferevent_base_set(mgt_eb, child_std); bufferevent_enable(child_std, EV_READ); + child_cli0 = bufferevent_new(heritage.fds[0], cli_rdcb, cli_wrcb, cli_excb, NULL); assert(child_cli0 != NULL); + bufferevent_base_set(mgt_eb, child_cli0); bufferevent_enable(child_cli0, EV_READ); + child_cli1 = bufferevent_new(heritage.fds[3], cli_rdcb, cli_wrcb, cli_excb, NULL); assert(child_cli1 != NULL); + bufferevent_base_set(mgt_eb, child_cli1); evtimer_set(&ev_child_pingpong, child_pingpong, NULL); event_base_set(mgt_eb, &ev_child_pingpong); Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-10 13:58:24 UTC (rev 407) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-10 13:59:13 UTC (rev 408) @@ -309,6 +309,7 @@ cli = cli_setup(mgt_eb, 0, 1, 1, cli_proto); signal_set(&e_sigchld, SIGCHLD, mgt_sigchld, NULL); + event_base_set(mgt_eb, &e_sigchld); signal_add(&e_sigchld, NULL); i = event_base_loop(mgt_eb, 0); From phk at projects.linpro.no Mon Jul 10 14:52:05 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 16:52:05 +0200 (CEST) Subject: r409 - in trunk/varnish-cache: bin/varnishd bin/varnishlog include Message-ID: <20060710145205.594391EC39F@projects.linpro.no> Author: phk Date: 2006-07-10 16:52:04 +0200 (Mon, 10 Jul 2006) New Revision: 409 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_main.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/heritage.h trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/bin/varnishlog/varnishlog.c trunk/varnish-cache/include/shmlog_tags.h trunk/varnish-cache/include/stat_field.h Log: Rewrite the worker thread pool code. Assign prefix WRK to the worker pool. Introduce a struct workreq since the prefetcher (when it happens) will not have a session to pass in. The worker threads get a cond_var each and are hung from a list in most recently used order. When a request is queued and the worker thread list is not empty, tickle the cond_var of the first one. If no threads were availble the max number of threads is not reached, try to start another worker thread. If the max was reached or the start filed (likely due to out of memory) indicate overflow and let the existing pool deal with it. Create only the minimum requested number of threads initially. Allow specification of the timeout before a dynamic worker thread commits suicide to be specified with -w. Default parameters are -w1,UINT_MAX,10 {min, max, timeout} Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 13:59:13 UTC (rev 408) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-10 14:52:04 UTC (rev 409) @@ -52,8 +52,17 @@ struct sbuf *sb; struct objhead *nobjhead; struct object *nobj; + + unsigned nbr; + pthread_cond_t cv; + TAILQ_ENTRY(worker) list; }; +struct workreq { + TAILQ_ENTRY(workreq) list; + struct sess *sess; +}; + #include "hash_slinger.h" /* Backend Connection ------------------------------------------------*/ @@ -150,6 +159,8 @@ /* Various internal stuff */ struct sessmem *mem; time_t t0; + + struct workreq workreq; }; struct backend { @@ -238,8 +249,8 @@ void PipeSession(struct worker *w, struct sess *sp); /* cache_pool.c */ -void CacheInitPool(void); -void DealWithSession(void *arg); +void WRK_Init(void); +void WRK_QueueSession(struct sess *sp); /* cache_shmlog.c */ Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-10 13:59:13 UTC (rev 408) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-10 14:52:04 UTC (rev 409) @@ -190,7 +190,7 @@ vca_return_session(sp); return; } - DealWithSession(sp); + WRK_QueueSession(sp); } static void Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-10 13:59:13 UTC (rev 408) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-10 14:52:04 UTC (rev 409) @@ -109,7 +109,7 @@ AZ(pthread_mutex_init(&sessmtx, NULL)); VBE_Init(); VSL_Init(); - CacheInitPool(); + WRK_Init(); VCA_Init(); EXP_Init(); Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-10 13:59:13 UTC (rev 408) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-10 14:52:04 UTC (rev 409) @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -16,11 +17,15 @@ #include "vcl.h" #include "cache.h" -static TAILQ_HEAD(, sess) shd = TAILQ_HEAD_INITIALIZER(shd); - -static pthread_cond_t shdcnd; +static pthread_mutex_t wrk_mtx; static unsigned xids; +/* Number of work requests queued in excess of worker threads available */ +static unsigned wrk_overflow; + +static TAILQ_HEAD(, worker) wrk_head = TAILQ_HEAD_INITIALIZER(wrk_head); +static TAILQ_HEAD(, workreq) wrk_reqhead = TAILQ_HEAD_INITIALIZER(wrk_reqhead); + /*--------------------------------------------------------------------*/ static int @@ -52,114 +57,202 @@ return (1); } -static void * -CacheWorker(void *priv) +static void +wrk_WorkSession(struct worker *w, struct sess *sp) { - struct sess *sp; - struct worker w; int done; char *b; - memset(&w, 0, sizeof w); - w.eb = event_init(); - assert(w.eb != NULL); - w.sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); - assert(w.sb != NULL); - - (void)priv; + time(&sp->t0); AZ(pthread_mutex_lock(&sessmtx)); - while (1) { - while (1) { - sp = TAILQ_FIRST(&shd); - if (sp != NULL) - break; - AZ(pthread_cond_wait(&shdcnd, &sessmtx)); - } - TAILQ_REMOVE(&shd, sp, list); - time(&sp->t0); - sp->vcl = GetVCL(); - AZ(pthread_mutex_unlock(&sessmtx)); + sp->vcl = GetVCL(); + AZ(pthread_mutex_unlock(&sessmtx)); - done = http_Dissect(sp->http, sp->fd, 1); - if (done != 0) { - RES_Error(&w, sp, done, NULL); - goto out; + done = http_Dissect(sp->http, sp->fd, 1); + if (done != 0) { + RES_Error(w, sp, done, NULL); + goto out; + } + + sp->backend = sp->vcl->backend[0]; + + VCL_recv_method(sp); + + for (done = 0; !done; ) { + switch(sp->handling) { + case VCL_RET_LOOKUP: + done = LookupSession(w, sp); + break; + case VCL_RET_FETCH: + done = FetchSession(w, sp); + break; + case VCL_RET_DELIVER: + done = DeliverSession(w, sp); + break; + case VCL_RET_PIPE: + PipeSession(w, sp); + done = 1; + break; + case VCL_RET_PASS: + PassSession(w, sp); + done = 1; + break; + default: + INCOMPL(); } + } + if (http_GetHdr(sp->http, "Connection", &b) && + !strcmp(b, "close")) { + vca_close_session(sp, "Connection header"); + } else if (http_GetProto(sp->http, &b) && + strcmp(b, "HTTP/1.1")) { + vca_close_session(sp, "not HTTP/1.1"); + } - sp->backend = sp->vcl->backend[0]; +out: + AZ(pthread_mutex_lock(&sessmtx)); + RelVCL(sp->vcl); + AZ(pthread_mutex_unlock(&sessmtx)); + sp->vcl = NULL; + vca_return_session(sp); +} - VCL_recv_method(sp); +/*--------------------------------------------------------------------*/ - for (done = 0; !done; ) { - switch(sp->handling) { - case VCL_RET_LOOKUP: - done = LookupSession(&w, sp); - break; - case VCL_RET_FETCH: - done = FetchSession(&w, sp); - break; - case VCL_RET_DELIVER: - done = DeliverSession(&w, sp); - break; - case VCL_RET_PIPE: - PipeSession(&w, sp); - done = 1; - break; - case VCL_RET_PASS: - PassSession(&w, sp); - done = 1; - break; - default: - INCOMPL(); - } +static void * +wrk_thread(void *priv) +{ + struct worker *w, ww; + struct workreq *wrq; + struct timespec ts; + + w = &ww; + memset(w, 0, sizeof w); + + AZ(pthread_cond_init(&w->cv, NULL)); + + w->eb = event_init(); + assert(w->eb != NULL); + + w->sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); + assert(w->sb != NULL); + + AZ(pthread_mutex_lock(&wrk_mtx)); + VSL_stats->n_wrk++; + w->nbr = VSL_stats->n_wrk; + if (priv == NULL) + VSL(SLT_WorkThread, 0, "%u born dynamic", w->nbr); + else + VSL(SLT_WorkThread, 0, "%u born permanent", w->nbr); + TAILQ_INSERT_HEAD(&wrk_head, w, list); + while (1) { + wrq = TAILQ_FIRST(&wrk_reqhead); + if (wrq != NULL) { + VSL_stats->n_wrkbusy++; + TAILQ_REMOVE(&wrk_head, w, list); + TAILQ_REMOVE(&wrk_reqhead, wrq, list); + AZ(pthread_mutex_unlock(&wrk_mtx)); + assert(wrq->sess != NULL); + wrk_WorkSession(w, wrq->sess); + AZ(pthread_mutex_lock(&wrk_mtx)); + VSL_stats->n_wrkbusy--; + TAILQ_INSERT_HEAD(&wrk_head, w, list); } - if (http_GetHdr(sp->http, "Connection", &b) && - !strcmp(b, "close")) { - vca_close_session(sp, "Connection header"); - } else if (http_GetProto(sp->http, &b) && - strcmp(b, "HTTP/1.1")) { - vca_close_session(sp, "not HTTP/1.1"); + if (wrk_overflow > 0) { + wrk_overflow--; + continue; } -out: - AZ(pthread_mutex_lock(&sessmtx)); - RelVCL(sp->vcl); - sp->vcl = NULL; - vca_return_session(sp); + /* If we are a reserved thread we don't die */ + if (priv != NULL) { + AZ(pthread_cond_wait(&w->cv, &wrk_mtx)); + continue; + } + + /* If we are a dynamic thread, time out and die */ + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += heritage.wthread_timeout; + if (pthread_cond_timedwait(&w->cv, &wrk_mtx, &ts)) { + VSL_stats->n_wrk--; + TAILQ_REMOVE(&wrk_head, w, list); + AZ(pthread_mutex_unlock(&wrk_mtx)); + VSL(SLT_WorkThread, 0, "%u suicide", w->nbr); + sbuf_delete(w->sb); + event_base_free(w->eb); + AZ(pthread_cond_destroy(&w->cv)); + return (NULL); + } } } +/*--------------------------------------------------------------------*/ + void -DealWithSession(void *arg) +WRK_QueueSession(struct sess *sp) { - struct sess *sp = arg; + struct worker *w; + pthread_t tp; time(&sp->t_req); /* * No locking necessary, we're serialized in the acceptor thread + * XXX: still ? */ sp->xid = xids++; VSL(SLT_XID, sp->fd, "%u", sp->xid); + sp->workreq.sess = sp; VSL_stats->client_req++; - AZ(pthread_mutex_lock(&sessmtx)); - TAILQ_INSERT_TAIL(&shd, sp, list); - AZ(pthread_mutex_unlock(&sessmtx)); - AZ(pthread_cond_signal(&shdcnd)); + + AZ(pthread_mutex_lock(&wrk_mtx)); + TAILQ_INSERT_TAIL(&wrk_reqhead, &sp->workreq, list); + + /* If there are idle threads, we tickle the first one into action */ + w = TAILQ_FIRST(&wrk_head); + if (w != NULL) { + AZ(pthread_cond_signal(&w->cv)); + AZ(pthread_mutex_unlock(&wrk_mtx)); + return; + } + + /* Register overflow if max threads reached */ + if (VSL_stats->n_wrk >= heritage.wthread_max) { + wrk_overflow++; + AZ(pthread_mutex_unlock(&wrk_mtx)); + return; + } + + /* Try to create a thread */ + AZ(pthread_mutex_unlock(&wrk_mtx)); + if (!pthread_create(&tp, NULL, wrk_thread, NULL)) { + AZ(pthread_detach(tp)); + return; + } + + VSL(SLT_Debug, 0, "Create worker thread failed %d %s", + errno, strerror(errno)); + + /* Register overflow */ + AZ(pthread_mutex_lock(&wrk_mtx)); + wrk_overflow++; + AZ(pthread_mutex_unlock(&wrk_mtx)); } + +/*--------------------------------------------------------------------*/ + void -CacheInitPool(void) +WRK_Init(void) { pthread_t tp; int i; - AZ(pthread_cond_init(&shdcnd, NULL)); + AZ(pthread_mutex_init(&wrk_mtx, NULL)); VSL(SLT_Debug, 0, "Starting %u worker threads", heritage.wthread_min); for (i = 0; i < heritage.wthread_min; i++) { - AZ(pthread_create(&tp, NULL, CacheWorker, NULL)); + AZ(pthread_create(&tp, NULL, wrk_thread, &i)); AZ(pthread_detach(tp)); } srandomdev(); Modified: trunk/varnish-cache/bin/varnishd/heritage.h =================================================================== --- trunk/varnish-cache/bin/varnishd/heritage.h 2006-07-10 13:59:13 UTC (rev 408) +++ trunk/varnish-cache/bin/varnishd/heritage.h 2006-07-10 14:52:04 UTC (rev 409) @@ -37,7 +37,9 @@ unsigned default_ttl; /* Worker threads */ - unsigned wthread_min, wthread_max; + unsigned wthread_min; + unsigned wthread_max; + unsigned wthread_timeout; /* Memory allocation hints */ unsigned mem_http_headerspace; Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-10 13:59:13 UTC (rev 408) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-10 14:52:04 UTC (rev 409) @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -419,8 +420,8 @@ fprintf(stderr, " %-28s # %s\n", "-s kind[,storageoptions]", "Backend storage specification"); fprintf(stderr, " %-28s # %s\n", "-t", "Default TTL"); - fprintf(stderr, " %-28s # %s\n", "-w int[,int]", - "Number of worker threads (fixed/{min,max})"); + fprintf(stderr, " %-28s # %s\n", "-w int[,int[,int]]", + "Number of worker threads (fixed/{min,max}/{min/max/timeout})"); #if 0 -c clusterid at cluster_controller -m memory_limit @@ -435,6 +436,28 @@ /*--------------------------------------------------------------------*/ +static void +tackle_warg(const char *argv) +{ + int i; + unsigned ua, ub, uc; + + i = sscanf(argv, "%u,%u,%u", &ua, &ub, &uc); + if (i == 0) + usage(); + if (ua < 1) + usage(); + heritage.wthread_min = ua; + heritage.wthread_max = ua; + heritage.wthread_timeout = 10; + if (i >= 2) + heritage.wthread_max = ub; + if (i >= 3) + heritage.wthread_timeout = uc; +} + +/*--------------------------------------------------------------------*/ + /* for development purposes */ #include #include @@ -442,8 +465,7 @@ int main(int argc, char *argv[]) { - int o, i; - unsigned ua, ub; + int o; const char *portnumber = "8080"; unsigned dflag = 1; /* XXX: debug=on for now */ const char *bflag = NULL; @@ -456,8 +478,9 @@ VCC_InitCompile(); heritage.default_ttl = 120; - heritage.wthread_min = 5; - heritage.wthread_max = 5; + heritage.wthread_min = 1; + heritage.wthread_max = UINT_MAX; + heritage.wthread_timeout = 10; heritage.mem_http_headerspace= 4096; heritage.mem_http_headers= 32; heritage.mem_workspace = 0; @@ -486,13 +509,7 @@ heritage.default_ttl = strtoul(optarg, NULL, 0); break; case 'w': - i = sscanf(optarg, "%u,%u", &ua, &ub); - if (i == 0) - usage(); - heritage.wthread_min = ua; - heritage.wthread_max = ua; - if (i == 2) - heritage.wthread_max = ub; + tackle_warg(optarg); break; default: usage(); Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-10 13:59:13 UTC (rev 408) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-10 14:52:04 UTC (rev 409) @@ -182,6 +182,7 @@ case SLT_SessionClose: case SLT_SessionReuse: case SLT_BackendClose: + case SLT_BackendReuse: sbuf_finish(ob[u]); if ((hc[u] != 4 || h_opt == 0) && sbuf_len(ob[u]) > 1) printf("%s\n", sbuf_data(ob[u])); Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-07-10 13:59:13 UTC (rev 408) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-07-10 14:52:04 UTC (rev 409) @@ -36,3 +36,4 @@ SLTM(ExpBan) SLTM(ExpPick) SLTM(ExpKill) +SLTM(WorkThread) Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2006-07-10 13:59:13 UTC (rev 408) +++ trunk/varnish-cache/include/stat_field.h 2006-07-10 14:52:04 UTC (rev 409) @@ -16,5 +16,7 @@ MAC_STAT(n_smf, uint64_t, "u", "N struct smf"); MAC_STAT(n_vbe, uint64_t, "u", "N struct vbe"); MAC_STAT(n_vbe_conn, uint64_t, "u", "N struct vbe_conn"); +MAC_STAT(n_wrk, uint64_t, "u", "N worker threads"); +MAC_STAT(n_wrkbusy, uint64_t, "u", "N busy worker threads"); From phk at projects.linpro.no Mon Jul 10 15:02:06 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 17:02:06 +0200 (CEST) Subject: r410 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060710150206.231641EC382@projects.linpro.no> Author: phk Date: 2006-07-10 17:02:06 +0200 (Mon, 10 Jul 2006) New Revision: 410 Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/include/stat_field.h Log: More statistics about worker threads. Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-10 14:52:04 UTC (rev 409) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-10 15:02:06 UTC (rev 410) @@ -138,24 +138,25 @@ assert(w->sb != NULL); AZ(pthread_mutex_lock(&wrk_mtx)); - VSL_stats->n_wrk++; w->nbr = VSL_stats->n_wrk; - if (priv == NULL) + if (priv == NULL) { + VSL_stats->n_wrk_create++; VSL(SLT_WorkThread, 0, "%u born dynamic", w->nbr); - else + } else { VSL(SLT_WorkThread, 0, "%u born permanent", w->nbr); + } TAILQ_INSERT_HEAD(&wrk_head, w, list); while (1) { wrq = TAILQ_FIRST(&wrk_reqhead); if (wrq != NULL) { - VSL_stats->n_wrkbusy++; + VSL_stats->n_wrk_busy++; TAILQ_REMOVE(&wrk_head, w, list); TAILQ_REMOVE(&wrk_reqhead, wrq, list); AZ(pthread_mutex_unlock(&wrk_mtx)); assert(wrq->sess != NULL); wrk_WorkSession(w, wrq->sess); AZ(pthread_mutex_lock(&wrk_mtx)); - VSL_stats->n_wrkbusy--; + VSL_stats->n_wrk_busy--; TAILQ_INSERT_HEAD(&wrk_head, w, list); } if (wrk_overflow > 0) { @@ -219,11 +220,13 @@ /* Register overflow if max threads reached */ if (VSL_stats->n_wrk >= heritage.wthread_max) { wrk_overflow++; + VSL_stats->n_wrk_short++; AZ(pthread_mutex_unlock(&wrk_mtx)); return; } /* Try to create a thread */ + VSL_stats->n_wrk++; AZ(pthread_mutex_unlock(&wrk_mtx)); if (!pthread_create(&tp, NULL, wrk_thread, NULL)) { AZ(pthread_detach(tp)); @@ -235,7 +238,10 @@ /* Register overflow */ AZ(pthread_mutex_lock(&wrk_mtx)); + VSL_stats->n_wrk--; wrk_overflow++; + VSL_stats->n_wrk_failed++; + VSL_stats->n_wrk_short++; AZ(pthread_mutex_unlock(&wrk_mtx)); } @@ -252,6 +258,7 @@ VSL(SLT_Debug, 0, "Starting %u worker threads", heritage.wthread_min); for (i = 0; i < heritage.wthread_min; i++) { + VSL_stats->n_wrk++; AZ(pthread_create(&tp, NULL, wrk_thread, &i)); AZ(pthread_detach(tp)); } Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2006-07-10 14:52:04 UTC (rev 409) +++ trunk/varnish-cache/include/stat_field.h 2006-07-10 15:02:06 UTC (rev 410) @@ -17,6 +17,7 @@ MAC_STAT(n_vbe, uint64_t, "u", "N struct vbe"); MAC_STAT(n_vbe_conn, uint64_t, "u", "N struct vbe_conn"); MAC_STAT(n_wrk, uint64_t, "u", "N worker threads"); -MAC_STAT(n_wrkbusy, uint64_t, "u", "N busy worker threads"); - - +MAC_STAT(n_wrk_create, uint64_t, "u", "N worker threads created"); +MAC_STAT(n_wrk_failed, uint64_t, "u", "N worker threads not created"); +MAC_STAT(n_wrk_short, uint64_t, "u", "N worker threads shortages"); +MAC_STAT(n_wrk_busy, uint64_t, "u", "N busy worker threads"); From phk at projects.linpro.no Mon Jul 10 19:54:33 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 21:54:33 +0200 (CEST) Subject: r411 - in trunk/varnish-cache: bin/varnishlog bin/varnishstat include lib/libvarnishapi Message-ID: <20060710195433.EF8801EC399@projects.linpro.no> Author: phk Date: 2006-07-10 21:54:33 +0200 (Mon, 10 Jul 2006) New Revision: 411 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c trunk/varnish-cache/bin/varnishstat/varnishstat.c trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Improve libvarnishapi interface to shared memory: Add function VSL_OpenStats() which directly returns a pointer to the varnish_stats structure. Add opaque VSL_data structure as handle to the log-tailer functions. Add generic argument parsing function for all log-tailers. Add support for generic "-r " option. Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-10 15:02:06 UTC (rev 410) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-10 19:54:33 UTC (rev 411) @@ -16,23 +16,12 @@ #include "shmlog.h" #include "varnishapi.h" -/* - * It would be simpler to use sparse array initialization and put it - * directly in tagnames, but -pedantic gets in the way - */ - -static struct tagnames { - enum shmlogtag tag; - const char *name; -} stagnames[] = { -#define SLTM(foo) { SLT_##foo, #foo }, +static const char *tagnames[] = { +#define SLTM(foo) [SLT_##foo] = #foo, #include "shmlog_tags.h" #undef SLTM - { SLT_ENDMARKER, NULL} }; -static const char *tagnames[256]; - static char * vis_it(unsigned char *p) { @@ -211,22 +200,18 @@ { int i, c; unsigned u, v; - unsigned char *p, *q; + unsigned char *p; int o_flag = 0; char *w_opt = NULL; FILE *wfile = NULL; - char *r_opt = NULL; - FILE *rfile = NULL; int h_opt = 0; - unsigned char rbuf[255+4]; - struct shmloghead *loghead; + struct VSL_data *vd; - loghead = VSL_OpenLog(); + vd = VSL_New(); - for (i = 0; stagnames[i].tag != SLT_ENDMARKER; i++) - tagnames[stagnames[i].tag] = stagnames[i].name; - - while ((c = getopt(argc, argv, "hor:w:")) != -1) { + while ((c = getopt(argc, argv, VSL_ARGS "how:")) != -1) { + if (VSL_Arg(vd, c, optarg)) + continue; switch (c) { case 'h': h_opt = 1; @@ -234,9 +219,6 @@ case 'o': o_flag = 1; break; - case 'r': - r_opt = optarg; - break; case 'w': w_opt = optarg; break; @@ -245,19 +227,12 @@ } } - if (r_opt != NULL && w_opt != NULL) - Usage(); + if (VSL_OpenLog(vd)) + exit (1); + if (o_flag && w_opt != NULL) Usage(); - if (r_opt != NULL) { - if (!strcmp(r_opt, "-")) - rfile = stdin; - else - rfile = fopen(r_opt, "r"); - if (rfile == NULL) - perror(r_opt); - } if (w_opt != NULL) { wfile = fopen(w_opt, "w"); if (wfile == NULL) @@ -266,35 +241,19 @@ u = 0; v = 0; - q = NULL; - if (r_opt == NULL) { - while (VSL_NextLog(loghead, &q) != NULL) - ; - } while (1) { - if (r_opt == NULL) { - p = VSL_NextLog(loghead, &q); - if (p == NULL) { - if (w_opt == NULL) { - if (o_flag && ++v == 100) - clean_order(); - fflush(stdout); - } else if (++v == 100) { - fflush(wfile); - printf("\nFlushed\n"); - } - usleep(50000); - continue; + p = VSL_NextLog(vd); + if (p == NULL) { + if (w_opt == NULL) { + if (o_flag && ++v == 100) + clean_order(); + fflush(stdout); + } else if (++v == 100) { + fflush(wfile); + printf("\nFlushed\n"); } - } else { - i = fread(rbuf, 4, 1, rfile); - if (i != 1) - break; - if (rbuf[1] > 0) - i = fread(rbuf + 4, rbuf[1], 1, rfile); - if (i != 1) - break; - p = rbuf; + usleep(50000); + continue; } v = 0; if (wfile != NULL) { Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-07-10 15:02:06 UTC (rev 410) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-07-10 19:54:33 UTC (rev 411) @@ -28,7 +28,6 @@ main(int argc, char **argv) { int c; - struct shmloghead *lh; struct varnish_stats *VSL_stats, copy; int c_flag = 0; intmax_t ju; @@ -42,10 +41,8 @@ lt = 0; - lh = VSL_OpenLog(); + VSL_stats = VSL_OpenStats(); - VSL_stats = &lh->stats; - while ((c = getopt(argc, argv, "c")) != -1) { switch (c) { case 'c': Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2006-07-10 15:02:06 UTC (rev 410) +++ trunk/varnish-cache/include/varnishapi.h 2006-07-10 19:54:33 UTC (rev 411) @@ -8,8 +8,13 @@ #define V_DEAD __attribute__ ((noreturn)) /* shmlog.c */ -struct shmloghead *VSL_OpenLog(void); -unsigned char *VSL_NextLog(struct shmloghead *lh, unsigned char **pp); +#define VSL_ARGS "r:" +struct VSL_data; +struct VSL_data *VSL_New(void); +int VSL_OpenLog(struct VSL_data *vd); +unsigned char *VSL_NextLog(struct VSL_data *lh); +int VSL_Arg(struct VSL_data *vd, int arg, const char *opt); +struct varnish_stats *VSL_OpenStats(void); /* varnish_debug.c */ Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-10 15:02:06 UTC (rev 410) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-10 19:54:33 UTC (rev 411) @@ -13,69 +13,149 @@ #include "shmlog.h" #include "varnishapi.h" +struct VSL_data { + struct shmloghead *head; + unsigned char *logstart; + unsigned char *logend; + unsigned char *ptr; + char *r_arg; + FILE *fi; + unsigned char rbuf[4 + 255 + 1]; +}; + #ifndef MAP_HASSEMAPHORE #define MAP_HASSEMAPHORE 0 /* XXX Linux */ #endif -static unsigned char *logstart, *logend; +static int vsl_fd; +static struct shmloghead *vsl_lh; -struct shmloghead * -VSL_OpenLog(void) +/*--------------------------------------------------------------------*/ + +static int +vsl_shmem_map(void) { - int fd; int i; - struct shmloghead slh, *lh; + struct shmloghead slh; - fd = open(SHMLOG_FILENAME, O_RDONLY); - if (fd < 0) { + if (vsl_lh != NULL) + return (0); + + vsl_fd = open(SHMLOG_FILENAME, O_RDONLY); + if (vsl_fd < 0) { fprintf(stderr, "Cannot open %s: %s\n", SHMLOG_FILENAME, strerror(errno)); - exit (1); + return (1); } - i = read(fd, &slh, sizeof slh); + i = read(vsl_fd, &slh, sizeof slh); if (i != sizeof slh) { fprintf(stderr, "Cannot read %s: %s\n", SHMLOG_FILENAME, strerror(errno)); - exit (1); + return (1); } if (slh.magic != SHMLOGHEAD_MAGIC) { fprintf(stderr, "Wrong magic number in file %s\n", SHMLOG_FILENAME); - exit (1); + return (1); } - lh = mmap(NULL, slh.size + sizeof slh, - PROT_READ, MAP_HASSEMAPHORE, fd, 0); - if (lh == MAP_FAILED) { + vsl_lh = mmap(NULL, slh.size + sizeof slh, + PROT_READ, MAP_HASSEMAPHORE, vsl_fd, 0); + if (vsl_lh == MAP_FAILED) { fprintf(stderr, "Cannot mmap %s: %s\n", SHMLOG_FILENAME, strerror(errno)); - exit (1); + return (1); } + return (0); +} - logstart = (unsigned char *)lh + lh->start; - logend = logstart + lh->size; +/*--------------------------------------------------------------------*/ - return (lh); +struct VSL_data * +VSL_New(void) +{ + struct VSL_data *vd; + + vd = calloc(sizeof *vd, 1); + return (vd); } +int +VSL_OpenLog(struct VSL_data *vd) +{ + + if (vd->r_arg != NULL) { + if (!strcmp(vd->r_arg, "-")) + vd->fi = stdin; + else + vd->fi = fopen(vd->r_arg, "r"); + if (vd->fi != NULL) + return (0); + perror(vd->r_arg); + return (1); + } + if (vsl_shmem_map()) + return (1); + + vd->logstart = (unsigned char *)vsl_lh + vsl_lh->start; + vd->logend = vd->logstart + vsl_lh->size; + vd->head = vsl_lh; + return (0); +} + unsigned char * -VSL_NextLog(struct shmloghead *lh __unused, unsigned char **pp) +VSL_NextLog(struct VSL_data *vd) { unsigned char *p; + int i; - p = *pp; + if (vd->fi != NULL) { + i = fread(vd->rbuf, 4, 1, vd->fi); + if (i != 1) + return (NULL); + if (vd->rbuf[1] > 0) { + i = fread(vd->rbuf + 4, vd->rbuf[1], 1, vd->fi); + if (i != 1) + return (NULL); + } + return (vd->rbuf); + } + + p = vd->ptr; if (p == NULL) - p = logstart; + p = vd->logstart; while (1) { if (*p == SLT_WRAPMARKER) { - p = logstart; + p = vd->logstart; continue; } if (*p == SLT_ENDMARKER) { - *pp = p; + vd->ptr = p; return (NULL); } - *pp = p + p[1] + 4; + vd->ptr = p + p[1] + 4; return (p); } } + +int +VSL_Arg(struct VSL_data *vd, int arg, const char *opt) +{ + switch (arg) { + case 'r': + vd->r_arg = strdup(opt); + return (1); + default: + return (0); + } +} + +struct varnish_stats * +VSL_OpenStats(void) +{ + + if (vsl_shmem_map()) + return (NULL); + return (&vsl_lh->stats); +} + From phk at projects.linpro.no Mon Jul 10 20:27:52 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 22:27:52 +0200 (CEST) Subject: r412 - in trunk/varnish-cache: bin/varnishlog include lib/libvarnishapi Message-ID: <20060710202752.643001EC396@projects.linpro.no> Author: phk Date: 2006-07-10 22:27:52 +0200 (Mon, 10 Jul 2006) New Revision: 412 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Add tag names array to libvarnishapi, everybody is going to need it. Implement -i tag[,tag ...] and -x tag[,tag ...] generic arguments. Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-10 19:54:33 UTC (rev 411) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-10 20:27:52 UTC (rev 412) @@ -16,11 +16,6 @@ #include "shmlog.h" #include "varnishapi.h" -static const char *tagnames[] = { -#define SLTM(foo) [SLT_##foo] = #foo, -#include "shmlog_tags.h" -#undef SLTM -}; static char * vis_it(unsigned char *p) @@ -69,7 +64,7 @@ switch (p[0]) { case SLT_VCL_call: sbuf_printf(ob[u], "%02x %3d %4d %-12s", - p[0], p[1], u, tagnames[p[0]]); + p[0], p[1], u, VSL_tags[p[0]]); if (p[1] > 0) { sbuf_cat(ob[u], " <"); sbuf_bcat(ob[u], p + 4, p[1]); @@ -106,7 +101,7 @@ else if (p[1] > 4 && !memcmp(p + 4, "TTD:", 4)) break; sbuf_printf(ob[u], "%02x %3d %4d %-12s", - p[0], p[1], u, tagnames[p[0]]); + p[0], p[1], u, VSL_tags[p[0]]); if (p[1] > 0) sbuf_cat(ob[u], vis_it(p)); sbuf_cat(ob[u], "\n"); @@ -153,7 +148,7 @@ } if (v) { sbuf_printf(ob[u], "%02x %3d %4d %-12s", - p[0], p[1], u, tagnames[p[0]]); + p[0], p[1], u, VSL_tags[p[0]]); if (p[1] > 0) { sbuf_cat(ob[u], " <"); sbuf_bcat(ob[u], p + 4, p[1]); @@ -210,7 +205,10 @@ vd = VSL_New(); while ((c = getopt(argc, argv, VSL_ARGS "how:")) != -1) { - if (VSL_Arg(vd, c, optarg)) + i = VSL_Arg(vd, c, optarg); + if (i < 0) + exit (1); + if (i > 0) continue; switch (c) { case 'h': @@ -273,7 +271,7 @@ } u = (p[2] << 8) | p[3]; printf("%02x %3d %4d %-12s", - p[0], p[1], u, tagnames[p[0]]); + p[0], p[1], u, VSL_tags[p[0]]); if (p[1] > 0) { if (p[0] != SLT_Debug) { Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2006-07-10 19:54:33 UTC (rev 411) +++ trunk/varnish-cache/include/varnishapi.h 2006-07-10 20:27:52 UTC (rev 412) @@ -8,13 +8,14 @@ #define V_DEAD __attribute__ ((noreturn)) /* shmlog.c */ -#define VSL_ARGS "r:" +#define VSL_ARGS "r:i:x:" struct VSL_data; struct VSL_data *VSL_New(void); int VSL_OpenLog(struct VSL_data *vd); unsigned char *VSL_NextLog(struct VSL_data *lh); int VSL_Arg(struct VSL_data *vd, int arg, const char *opt); struct varnish_stats *VSL_OpenStats(void); +const char *VSL_tags[256]; /* varnish_debug.c */ Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-10 19:54:33 UTC (rev 411) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-10 20:27:52 UTC (rev 412) @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -18,9 +19,12 @@ unsigned char *logstart; unsigned char *logend; unsigned char *ptr; - char *r_arg; + FILE *fi; unsigned char rbuf[4 + 255 + 1]; + + int ix_opt; + unsigned char supr[256]; }; #ifndef MAP_HASSEMAPHORE @@ -32,6 +36,14 @@ /*--------------------------------------------------------------------*/ +const char *VSL_tags[256] = { +#define SLTM(foo) [SLT_##foo] = #foo, +#include "shmlog_tags.h" +#undef SLTM +}; + +/*--------------------------------------------------------------------*/ + static int vsl_shmem_map(void) { @@ -84,22 +96,16 @@ VSL_OpenLog(struct VSL_data *vd) { - if (vd->r_arg != NULL) { - if (!strcmp(vd->r_arg, "-")) - vd->fi = stdin; - else - vd->fi = fopen(vd->r_arg, "r"); - if (vd->fi != NULL) - return (0); - perror(vd->r_arg); - return (1); - } + if (vd->fi != NULL) + return (0); + if (vsl_shmem_map()) return (1); + vd->head = vsl_lh; vd->logstart = (unsigned char *)vsl_lh + vsl_lh->start; vd->logend = vd->logstart + vsl_lh->size; - vd->head = vsl_lh; + vd->ptr = vd->logstart; return (0); } @@ -110,21 +116,22 @@ int i; if (vd->fi != NULL) { - i = fread(vd->rbuf, 4, 1, vd->fi); - if (i != 1) - return (NULL); - if (vd->rbuf[1] > 0) { - i = fread(vd->rbuf + 4, vd->rbuf[1], 1, vd->fi); + while (1) { + i = fread(vd->rbuf, 4, 1, vd->fi); if (i != 1) return (NULL); + if (vd->rbuf[1] > 0) { + i = fread(vd->rbuf + 4, vd->rbuf[1], 1, vd->fi); + if (i != 1) + return (NULL); + } + if (!vd->supr[vd->rbuf[0]]) + return (vd->rbuf); } - return (vd->rbuf); } p = vd->ptr; - if (p == NULL) - p = vd->logstart; - while (1) { + for (p = vd->ptr; ; p = vd->ptr) { if (*p == SLT_WRAPMARKER) { p = vd->logstart; continue; @@ -134,17 +141,89 @@ return (NULL); } vd->ptr = p + p[1] + 4; - return (p); + if (!vd->supr[p[0]]) + return (p); } } +/*--------------------------------------------------------------------*/ + +static int +vsl_r_arg(struct VSL_data *vd, const char *opt) +{ + + if (!strcmp(opt, "-")) + vd->fi = stdin; + else + vd->fi = fopen(opt, "r"); + if (vd->fi != NULL) + return (1); + perror(opt); + return (-1); +} + +/*--------------------------------------------------------------------*/ + +static int +vsl_ix_arg(struct VSL_data *vd, const char *opt, int arg) +{ + int i, j, l; + const char *b, *e, *p, *q; + + /* If first option is 'i', set all bits for supression */ + if (arg == 'i' && vd->ix_opt == 0) + for (i = 0; i < 256; i++) + vd->supr[i] = 1; + vd->ix_opt = 1; + + for (b = opt; *b; b = e) { + while (isspace(*b)) + b++; + e = strchr(b, ','); + if (e == NULL) + e = strchr(b, '\0'); + l = e - b; + if (*e == ',') + e++; + while (isspace(b[l - 1])) + l--; + for (i = 0; i < 256; i++) { + if (VSL_tags[i] == NULL) + continue; + p = VSL_tags[i]; + q = b; + for (j = 0; j < l; j++) + if (tolower(*q++) != tolower(*p++)) + break; + if (j != l) + continue; + + if (arg == 'x') + vd->supr[i] = 1; + else + vd->supr[i] = 0; + break; + } + if (i == 256) { + fprintf(stderr, + "Could not match \"%*.*s\" to any tag\n", l, l, b); + return (-1); + } + } + return (1); +} + +/*--------------------------------------------------------------------*/ + int VSL_Arg(struct VSL_data *vd, int arg, const char *opt) { switch (arg) { + case 'i': + case 'x': + return (vsl_ix_arg(vd, opt, arg)); case 'r': - vd->r_arg = strdup(opt); - return (1); + return (vsl_r_arg(vd, opt)); default: return (0); } From phk at projects.linpro.no Mon Jul 10 20:49:07 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 22:49:07 +0200 (CEST) Subject: r413 - in trunk/varnish-cache: include lib/libvarnishapi Message-ID: <20060710204907.48B011EC399@projects.linpro.no> Author: phk Date: 2006-07-10 22:49:07 +0200 (Mon, 10 Jul 2006) New Revision: 413 Modified: trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Implement -C, -I and -X generic options Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2006-07-10 20:27:52 UTC (rev 412) +++ trunk/varnish-cache/include/varnishapi.h 2006-07-10 20:49:07 UTC (rev 413) @@ -8,7 +8,7 @@ #define V_DEAD __attribute__ ((noreturn)) /* shmlog.c */ -#define VSL_ARGS "r:i:x:" +#define VSL_ARGS "r:i:x:CI:X:" struct VSL_data; struct VSL_data *VSL_New(void); int VSL_OpenLog(struct VSL_data *vd); Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-10 20:27:52 UTC (rev 412) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-10 20:49:07 UTC (rev 413) @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "shmlog.h" @@ -25,6 +26,10 @@ int ix_opt; unsigned char supr[256]; + + int regflags; + regex_t *regincl; + regex_t *regexcl; }; #ifndef MAP_HASSEMAPHORE @@ -89,6 +94,7 @@ struct VSL_data *vd; vd = calloc(sizeof *vd, 1); + vd->regflags = REG_EXTENDED | REG_NOSUB; return (vd); } @@ -109,29 +115,28 @@ return (0); } -unsigned char * -VSL_NextLog(struct VSL_data *vd) +/*--------------------------------------------------------------------*/ + +static unsigned char * +vsl_nextlog(struct VSL_data *vd) { unsigned char *p; int i; if (vd->fi != NULL) { - while (1) { - i = fread(vd->rbuf, 4, 1, vd->fi); + i = fread(vd->rbuf, 4, 1, vd->fi); + if (i != 1) + return (NULL); + if (vd->rbuf[1] > 0) { + i = fread(vd->rbuf + 4, vd->rbuf[1], 1, vd->fi); if (i != 1) return (NULL); - if (vd->rbuf[1] > 0) { - i = fread(vd->rbuf + 4, vd->rbuf[1], 1, vd->fi); - if (i != 1) - return (NULL); - } - if (!vd->supr[vd->rbuf[0]]) - return (vd->rbuf); } + return (vd->rbuf); } p = vd->ptr; - for (p = vd->ptr; ; p = vd->ptr) { + while (1) { if (*p == SLT_WRAPMARKER) { p = vd->logstart; continue; @@ -141,8 +146,36 @@ return (NULL); } vd->ptr = p + p[1] + 4; - if (!vd->supr[p[0]]) + return (p); + } +} + +unsigned char * +VSL_NextLog(struct VSL_data *vd) +{ + unsigned char *p; + regmatch_t rm; + int i; + + while (1) { + p = vsl_nextlog(vd); + if (p == NULL) return (p); + if (vd->supr[p[0]]) + continue; + rm.rm_so = 0; + rm.rm_eo = p[1]; + if (vd->regincl != NULL) { + i = regexec(vd->regincl, p + 4, 1, &rm, REG_STARTEND); + if (i == REG_NOMATCH) + continue; + } + if (vd->regexcl != NULL) { + i = regexec(vd->regexcl, p + 4, 1, &rm, REG_STARTEND); + if (i != REG_NOMATCH) + continue; + } + return (p); } } @@ -165,6 +198,37 @@ /*--------------------------------------------------------------------*/ static int +vsl_IX_arg(struct VSL_data *vd, const char *opt, int arg) +{ + int i; + regex_t **rp; + char buf[BUFSIZ]; + + if (arg == 'I') + rp = &vd->regincl; + else + rp = &vd->regexcl; + if (*rp != NULL) { + fprintf(stderr, "Option %c can only be given once", arg); + return (-1); + } + *rp = calloc(sizeof(regex_t), 1); + if (*rp == NULL) { + perror("malloc"); + return (-1); + } + i = regcomp(*rp, opt, vd->regflags); + if (i) { + regerror(i, *rp, buf, sizeof buf); + fprintf(stderr, "%s", buf); + return (-1); + } + return (1); +} + +/*--------------------------------------------------------------------*/ + +static int vsl_ix_arg(struct VSL_data *vd, const char *opt, int arg) { int i, j, l; @@ -219,11 +283,10 @@ VSL_Arg(struct VSL_data *vd, int arg, const char *opt) { switch (arg) { - case 'i': - case 'x': - return (vsl_ix_arg(vd, opt, arg)); - case 'r': - return (vsl_r_arg(vd, opt)); + case 'i': case 'x': return (vsl_ix_arg(vd, opt, arg)); + case 'r': return (vsl_r_arg(vd, opt)); + case 'I': case 'X': return (vsl_IX_arg(vd, opt, arg)); + case 'C': vd->regflags = REG_ICASE; return (1); default: return (0); } From phk at projects.linpro.no Mon Jul 10 21:49:56 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 23:49:56 +0200 (CEST) Subject: r414 - trunk/varnish-cache/lib/libvarnishapi Message-ID: <20060710214956.118AE1EC35F@projects.linpro.no> Author: phk Date: 2006-07-10 23:49:55 +0200 (Mon, 10 Jul 2006) New Revision: 414 Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Fix so that both -I and -X can be specified Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-10 20:49:07 UTC (rev 413) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-10 21:49:55 UTC (rev 414) @@ -163,14 +163,16 @@ return (p); if (vd->supr[p[0]]) continue; - rm.rm_so = 0; - rm.rm_eo = p[1]; if (vd->regincl != NULL) { + rm.rm_so = 0; + rm.rm_eo = p[1]; i = regexec(vd->regincl, p + 4, 1, &rm, REG_STARTEND); if (i == REG_NOMATCH) continue; } if (vd->regexcl != NULL) { + rm.rm_so = 0; + rm.rm_eo = p[1]; i = regexec(vd->regexcl, p + 4, 1, &rm, REG_STARTEND); if (i != REG_NOMATCH) continue; From phk at projects.linpro.no Mon Jul 10 21:54:11 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 10 Jul 2006 23:54:11 +0200 (CEST) Subject: r415 - in trunk/varnish-cache: . bin bin/varnishtop Message-ID: <20060710215411.9B1BA1EC382@projects.linpro.no> Author: phk Date: 2006-07-10 23:54:11 +0200 (Mon, 10 Jul 2006) New Revision: 415 Added: trunk/varnish-cache/bin/varnishtop/ trunk/varnish-cache/bin/varnishtop/Makefile.am trunk/varnish-cache/bin/varnishtop/varnishtop.c Modified: trunk/varnish-cache/bin/Makefile.am trunk/varnish-cache/configure.ac Log: Add varnishtop log-tailer. Try these: varnishtop -i url varhishtop -i header -C -I '^user-agent:' varhishtop -i header -C -I '^user-agent:' -X MSIE varhishtop -i header -C -I '^user-agent:.*MSIE' varhishtop -i header -C -I '^user-agent:.*java' You can also run them on the logfiles from the live test: zcat _vlog21.gz | varnishtop -r - -i header ... Modified: trunk/varnish-cache/bin/Makefile.am =================================================================== --- trunk/varnish-cache/bin/Makefile.am 2006-07-10 21:49:55 UTC (rev 414) +++ trunk/varnish-cache/bin/Makefile.am 2006-07-10 21:54:11 UTC (rev 415) @@ -1,3 +1,3 @@ # $Id$ -SUBDIRS = varnishd varnishlog varnishstat +SUBDIRS = varnishd varnishlog varnishstat varnishtop Added: trunk/varnish-cache/bin/varnishtop/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishtop/Makefile.am 2006-07-10 21:49:55 UTC (rev 414) +++ trunk/varnish-cache/bin/varnishtop/Makefile.am 2006-07-10 21:54:11 UTC (rev 415) @@ -0,0 +1,14 @@ +# $Id: Makefile.am 347 2006-07-06 09:31:45Z des $ + +INCLUDES = -I$(top_srcdir)/include + +bin_PROGRAMS = varnishtop + +# man_MANS = varnishtop.1 + +varnishtop_SOURCES = varnishtop.c + +varnishtop_LDADD = \ + $(top_builddir)/lib/libvarnishapi/libvarnishapi.la \ + $(top_builddir)/lib/libsbuf/libsbuf.a \ + -lcurses Added: trunk/varnish-cache/bin/varnishtop/varnishtop.c =================================================================== --- trunk/varnish-cache/bin/varnishtop/varnishtop.c 2006-07-10 21:49:55 UTC (rev 414) +++ trunk/varnish-cache/bin/varnishtop/varnishtop.c 2006-07-10 21:54:11 UTC (rev 415) @@ -0,0 +1,150 @@ +/* + * $Id: varnishlog.c 412 2006-07-10 20:27:52Z phk $ + * + * Log tailer for Varnish + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shmlog.h" +#include "queue.h" +#include "varnishapi.h" + +struct top { + unsigned char *rec; + unsigned hash; + TAILQ_ENTRY(top) list; + double count; +}; + +static TAILQ_HEAD(tophead, top) top_head = TAILQ_HEAD_INITIALIZER(top_head); + +static unsigned ntop; + +/*--------------------------------------------------------------------*/ + +static void +Usage(void) +{ + fprintf(stderr, "Usage: varnishtop\n"); + exit(2); +} + +static void +upd(void) +{ + struct top *tp, *tp2; + int l; + double t = 0; + unsigned u = 0; + + erase(); + l = 0; + mvprintw(0, 0, "list length %u\n", ntop); + TAILQ_FOREACH_SAFE(tp, &top_head, list, tp2) { + if (++l < LINES) { + printw("%10.2f %*.*s\n", + tp->count, tp->rec[1], tp->rec[1], tp->rec + 4); + t = tp->count; + } + tp->count *= .999; + if (tp->count * 10 < t || l > LINES * 10) { + TAILQ_REMOVE(&top_head, tp, list); + free(tp); + ntop--; + u++; + } + } + mvprintw(0, 40, "cleaned %u\n", u); + refresh(); +} + +int +main(int argc, char **argv) +{ + int i, c; + unsigned char *p; + struct VSL_data *vd; + unsigned u, v; + struct top *tp, *tp2; + + vd = VSL_New(); + + while ((c = getopt(argc, argv, VSL_ARGS "how:")) != -1) { + i = VSL_Arg(vd, c, optarg); + if (i < 0) + exit (1); + if (i > 0) + continue; + switch (c) { + default: + Usage(); + } + } + + if (VSL_OpenLog(vd)) + exit (1); + + initscr(); + v = 0; + while (1) { + p = VSL_NextLog(vd); + if (p == NULL) { + upd(); + usleep(50000); + continue; + } + if (++v > 100) { + upd(); + v = 0; + } + u = 0; + for (i = 4; i < 4 + p[1]; i++) + u += p[i]; + TAILQ_FOREACH(tp, &top_head, list) { + if (tp->hash != u) + continue; + if (tp->rec[0] != p[0]) + continue; + if (tp->rec[1] != p[1]) + continue; + if (memcmp(p + 4, tp->rec + 4, p[1])) + continue; + tp->count += 1.0; + break; + } + if (tp == NULL) { + ntop++; + tp = calloc(sizeof *tp + 4 + p[1], 1); + assert(tp != NULL); + tp->hash = u; + tp->rec = (void *)(tp + 1); + memcpy(tp->rec, p, 4 + p[1]); + tp->count = 1.0; + TAILQ_INSERT_TAIL(&top_head, tp, list); + } + while (1) { + tp2 = TAILQ_PREV(tp, tophead, list); + if (tp2 == NULL || tp2->count >= tp->count) + break; + TAILQ_REMOVE(&top_head, tp2, list); + TAILQ_INSERT_AFTER(&top_head, tp, tp2, list); + } + while (1) { + tp2 = TAILQ_NEXT(tp, list); + if (tp2 == NULL || tp2->count <= tp->count) + break; + TAILQ_REMOVE(&top_head, tp2, list); + TAILQ_INSERT_BEFORE(tp, tp2, list); + } + } + return (0); +} Modified: trunk/varnish-cache/configure.ac =================================================================== --- trunk/varnish-cache/configure.ac 2006-07-10 21:49:55 UTC (rev 414) +++ trunk/varnish-cache/configure.ac 2006-07-10 21:54:11 UTC (rev 415) @@ -70,6 +70,7 @@ bin/varnishd/Makefile bin/varnishlog/Makefile bin/varnishstat/Makefile + bin/varnishtop/Makefile contrib/Makefile include/Makefile lib/Makefile From des at linpro.no Tue Jul 11 06:07:11 2006 From: des at linpro.no (Dag-Erling =?iso-8859-1?Q?Sm=F8rgrav?=) Date: Tue, 11 Jul 2006 08:07:11 +0200 Subject: r412 - in trunk/varnish-cache: bin/varnishlog include lib/libvarnishapi References: <20060710202752.643001EC396@projects.linpro.no> Message-ID: phk at projects.linpro.no writes: > Log: > Add tag names array to libvarnishapi, everybody is going to need it. > > Implement -i tag[,tag ...] and -x tag[,tag ...] generic arguments. "include" and "exclude" I presume? DES -- Dag-Erling Sm?rgrav Senior Software Developer Linpro AS - www.linpro.no From des at projects.linpro.no Tue Jul 11 06:26:19 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Tue, 11 Jul 2006 08:26:19 +0200 (CEST) Subject: r416 - trunk/varnish-cache/bin/varnishlog Message-ID: <20060711062619.26F3D1EC382@projects.linpro.no> Author: des Date: 2006-07-11 08:26:19 +0200 (Tue, 11 Jul 2006) New Revision: 416 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.1 Log: Document the -C, -I, -X, -i, -x options. Add a list of log entry tags. Add cross-references to varnishd(1) and varnishstat(1). Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.1 =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.1 2006-07-10 21:54:11 UTC (rev 415) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.1 2006-07-11 06:26:19 UTC (rev 416) @@ -9,9 +9,14 @@ .Nd HTTP accelerator log watcher .Sh SYNOPSIS .Nm +.Op Fl C .Op Fl o +.Op Fl I Ar regex +.Op Fl X Ar regex +.Op Fl i Ar tag .Op Fl r Ar file .Op Fl w Ar file +.Op Fl x Ar tag .Sh DESCRIPTION The .Nm @@ -21,8 +26,16 @@ .Pp The following options are available: .Bl -tag -width Fl +.It Fl C +Ignore case when matching regular expressions. .It Fl o Group log entries by request ID. +.It Fl I Ar regex +Include log entries which match the specified regular expression. +.It Fl X Ar regex +Exclude log entries which match the specified regular expression. +.It Fl i Ar tag +Include log entries with the specified tag. .It Fl r Ar file Read log entries from .Ar file @@ -31,7 +44,49 @@ Write log entries to .Ar file instead of displaying them. +.It Fl x Ar tag +Exclude log entries with the specified tag. .El +.Sh TAGS +The following log entry tags are currently defined: +.\" keep in sync with include/shmlog_tags.h +.\" XXX add descriptions +.Bl -tag -width 16 +.It Dv Backend +.It Dv BackendClose +.It Dv BackendOpen +.It Dv BackendReuse +.It Dv BackendXID +.It Dv BldHdr +.It Dv CLI +.It Dv ClientAddr +.It Dv Debug +.It Dv Error +.It Dv ExpBan +.It Dv ExpKill +.It Dv ExpPick +.It Dv Header +.It Dv Hit +.It Dv HttpError +.It Dv Length +.It Dv LostHeader +.It Dv Protocol +.It Dv Request +.It Dv Response +.It Dv SessionClose +.It Dv SessionOpen +.It Dv SessionReuse +.It Dv Status +.It Dv URL +.It Dv VCL_call +.It Dv VCL_return +.It Dv VCL_trace +.It Dv WorkThread +.It Dv XID +.El +.Sh SEE ALSO +.Xr varnishd 1 , +.Xr varnishstat 1 .Sh HISTORY The .Nm From des at projects.linpro.no Tue Jul 11 06:27:55 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Tue, 11 Jul 2006 08:27:55 +0200 (CEST) Subject: r417 - trunk/varnish-cache/bin/varnishstat Message-ID: <20060711062755.0CE8C1EC35F@projects.linpro.no> Author: des Date: 2006-07-11 08:27:54 +0200 (Tue, 11 Jul 2006) New Revision: 417 Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.1 Log: Add cross-references. Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.1 =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.1 2006-07-11 06:26:19 UTC (rev 416) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.1 2006-07-11 06:27:54 UTC (rev 417) @@ -23,6 +23,10 @@ Present a continuously updating display using .Xr curses 3 . .El +.Sh SEE ALSO +.Xr varnishd 1 , +.Xr varnishlog 1 , +.Xr curses 3 .Sh HISTORY The .Nm From des at projects.linpro.no Tue Jul 11 06:28:18 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Tue, 11 Jul 2006 08:28:18 +0200 (CEST) Subject: r418 - trunk/varnish-cache/include Message-ID: <20060711062818.89C8A1EC382@projects.linpro.no> Author: des Date: 2006-07-11 08:28:18 +0200 (Tue, 11 Jul 2006) New Revision: 418 Modified: trunk/varnish-cache/include/shmlog_tags.h Log: Add a note to update varnishlog(1) whenever this list changes. Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-07-11 06:27:54 UTC (rev 417) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-07-11 06:28:18 UTC (rev 418) @@ -4,6 +4,8 @@ * Define the tags in the shared memory in a reusable format. * Whoever includes this get to define what the SLTM macro does. * + * REMEMBER to update the documentation (especially the varnishlog(1) man + * page) whenever this list changes. */ SLTM(Debug) From des at projects.linpro.no Tue Jul 11 06:30:38 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Tue, 11 Jul 2006 08:30:38 +0200 (CEST) Subject: r419 - trunk/varnish-cache/bin/varnishd Message-ID: <20060711063038.4B4B51EC35F@projects.linpro.no> Author: des Date: 2006-07-11 08:30:38 +0200 (Tue, 11 Jul 2006) New Revision: 419 Modified: trunk/varnish-cache/bin/varnishd/varnishd.1 Log: Add cross-references and a commented-out STANDARDS section. Modified: trunk/varnish-cache/bin/varnishd/varnishd.1 =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.1 2006-07-11 06:28:18 UTC (rev 418) +++ trunk/varnish-cache/bin/varnishd/varnishd.1 2006-07-11 06:30:38 UTC (rev 419) @@ -64,6 +64,21 @@ the number of worker threads will vary within the specified range according to system load. .El +.Sh SEE ALSO +.Xr varnishlog 1 , +.Xr varnishstat 1 +.\" .Sh STANDARDS +.\" .Rs +.\" .%A R. Fielding +.\" .%A J. Gettys +.\" .%A J. Mogul +.\" .%A H. Frystyk +.\" .%A L. Masinter +.\" .%A P. Leach +.\" .%A T. Berners-Lee +.\" .%D January 1999 +.\" .%B Hypertext Transfer Protocol -- HTTP/1.1 +.\" .%O RFC2616 .Sh HISTORY The .Nm From phk at phk.freebsd.dk Tue Jul 11 06:44:56 2006 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Tue, 11 Jul 2006 06:44:56 +0000 Subject: r416 - trunk/varnish-cache/bin/varnishlog In-Reply-To: Your message of "Tue, 11 Jul 2006 08:26:19 +0200." <20060711062619.26F3D1EC382@projects.linpro.no> Message-ID: <26601.1152600296@critter.freebsd.dk> In message <20060711062619.26F3D1EC382 at projects.linpro.no>, des at projects.linpro .no writes: >Author: des >Date: 2006-07-11 08:26:19 +0200 (Tue, 11 Jul 2006) >New Revision: 416 > >Add a list of log entry tags. I think it would be a better idea to put this list in it's own (section 5 or 7 ?) manual page -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From des at linpro.no Tue Jul 11 06:55:43 2006 From: des at linpro.no (Dag-Erling =?iso-8859-1?Q?Sm=F8rgrav?=) Date: Tue, 11 Jul 2006 08:55:43 +0200 Subject: r416 - trunk/varnish-cache/bin/varnishlog References: <26601.1152600296@critter.freebsd.dk> Message-ID: "Poul-Henning Kamp" writes: > Dag-Erling Sm?rgrav writes: > > Add a list of log entry tags. > I think it would be a better idea to put this list in it's own > (section 5 or 7 ?) manual page This is what we have for now. My intention right now is to provide sufficient documentation for testers to be able to run Varnish. The spec also calls for a full set of internal documentation, which would include man pages for the shmlog API, but we're not quite there yet. DES -- Dag-Erling Sm?rgrav Senior Software Developer Linpro AS - www.linpro.no From phk at projects.linpro.no Tue Jul 11 07:30:53 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 09:30:53 +0200 (CEST) Subject: r420 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060711073053.7FA371EC381@projects.linpro.no> Author: phk Date: 2006-07-11 09:30:53 +0200 (Tue, 11 Jul 2006) New Revision: 420 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/include/stat_field.h Log: Split http_Dissect() into http_DissectRequest() and http_DissectResponse() Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-11 06:30:38 UTC (rev 419) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-11 07:30:53 UTC (rev 420) @@ -230,7 +230,8 @@ int http_GetTail(struct http *hp, unsigned len, char **b, char **e); int http_GetURL(struct http *hp, char **b); void http_RecvHead(struct http *hp, int fd, struct event_base *eb, http_callback_f *func, void *arg); -int http_Dissect(struct http *sp, int fd, int rr); +int http_DissectRequest(struct http *sp, int fd); +int http_DissectResponse(struct http *sp, int fd); enum http_build { Build_Pipe, Build_Pass, Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-11 06:30:38 UTC (rev 419) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-11 07:30:53 UTC (rev 420) @@ -261,7 +261,7 @@ http_RecvHead(hp, vc->fd, w->eb, NULL, NULL); event_base_loop(w->eb, 0); time(&sp->t_resp); - assert(http_Dissect(hp, vc->fd, 2) == 0); + assert(http_DissectResponse(hp, vc->fd) == 0); body = RFC2616_cache_policy(sp, hp); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-11 06:30:38 UTC (rev 419) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-11 07:30:53 UTC (rev 420) @@ -157,91 +157,11 @@ /*--------------------------------------------------------------------*/ -int -http_Dissect(struct http *hp, int fd, int rr) +static int +http_dissect_hdrs(struct http *hp, int fd, char *p) { - char *p, *q, *r; + char *q, *r; - assert(hp->t != NULL); - assert(hp->s < hp->t); - assert(hp->t <= hp->v); - for (p = hp->s ; isspace(*p); p++) - continue; - if (rr == 1) { - /* First, the request type (GET/HEAD etc) */ - hp->req = p; - for (; isalpha(*p); p++) - ; - VSLR(SLT_Request, fd, hp->req, p); - *p++ = '\0'; - - /* Next find the URI */ - while (isspace(*p) && *p != '\n') - p++; - if (*p == '\n') { - VSLR(SLT_Debug, fd, hp->s, hp->v); - return (400); - } - hp->url = p; - while (!isspace(*p)) - p++; - VSLR(SLT_URL, fd, hp->url, p); - if (*p == '\n') { - VSLR(SLT_Debug, fd, hp->s, hp->v); - return (400); - } - *p++ = '\0'; - - /* Finally, look for protocol */ - while (isspace(*p) && *p != '\n') - p++; - if (*p == '\n') { - VSLR(SLT_Debug, fd, hp->s, hp->v); - return (400); - } - hp->proto = p; - while (!isspace(*p)) - p++; - VSLR(SLT_Protocol, fd, hp->proto, p); - if (*p != '\n') - *p++ = '\0'; - while (isspace(*p) && *p != '\n') - p++; - if (*p != '\n') { - VSLR(SLT_Debug, fd, hp->s, hp->v); - return (400); - } - *p++ = '\0'; - } else { - /* First, protocol */ - hp->proto = p; - while (!isspace(*p)) - p++; - VSLR(SLT_Protocol, fd, hp->proto, p); - *p++ = '\0'; - - /* Next find the status */ - while (isspace(*p)) - p++; - hp->status = p; - while (!isspace(*p)) - p++; - VSLR(SLT_Status, fd, hp->status, p); - *p++ = '\0'; - - /* Next find the response */ - while (isspace(*p)) - p++; - hp->response = p; - while (*p != '\n') - p++; - for (q = p; q > hp->response && isspace(q[-1]); q--) - continue; - *q = '\0'; - VSLR(SLT_Response, fd, hp->response, q); - p++; - } - if (*p == '\r') p++; @@ -262,6 +182,7 @@ hp->hdr[hp->nhdr++] = p; VSLR(SLT_Header, fd, p, q); } else { + VSL_stats->losthdr++; VSLR(SLT_LostHeader, fd, p, q); } } @@ -274,6 +195,111 @@ /*--------------------------------------------------------------------*/ +int +http_DissectRequest(struct http *hp, int fd) +{ + char *p; + + assert(hp->t != NULL); + assert(hp->s < hp->t); + assert(hp->t <= hp->v); + for (p = hp->s ; isspace(*p); p++) + continue; + + /* First, the request type (GET/HEAD etc) */ + hp->req = p; + for (; isalpha(*p); p++) + ; + VSLR(SLT_Request, fd, hp->req, p); + *p++ = '\0'; + + /* Next find the URI */ + while (isspace(*p) && *p != '\n') + p++; + if (*p == '\n') { + VSLR(SLT_Debug, fd, hp->s, hp->v); + return (400); + } + hp->url = p; + while (!isspace(*p)) + p++; + VSLR(SLT_URL, fd, hp->url, p); + if (*p == '\n') { + VSLR(SLT_Debug, fd, hp->s, hp->v); + return (400); + } + *p++ = '\0'; + + /* Finally, look for protocol */ + while (isspace(*p) && *p != '\n') + p++; + if (*p == '\n') { + VSLR(SLT_Debug, fd, hp->s, hp->v); + return (400); + } + hp->proto = p; + while (!isspace(*p)) + p++; + VSLR(SLT_Protocol, fd, hp->proto, p); + if (*p != '\n') + *p++ = '\0'; + while (isspace(*p) && *p != '\n') + p++; + if (*p != '\n') { + VSLR(SLT_Debug, fd, hp->s, hp->v); + return (400); + } + *p++ = '\0'; + + return (http_dissect_hdrs(hp, fd, p)); +} + +/*--------------------------------------------------------------------*/ + +int +http_DissectResponse(struct http *hp, int fd) +{ + char *p, *q; + + assert(hp->t != NULL); + assert(hp->s < hp->t); + assert(hp->t <= hp->v); + for (p = hp->s ; isspace(*p); p++) + continue; + + /* First, protocol */ + hp->proto = p; + while (!isspace(*p)) + p++; + VSLR(SLT_Protocol, fd, hp->proto, p); + *p++ = '\0'; + + /* Next find the status */ + while (isspace(*p)) + p++; + hp->status = p; + while (!isspace(*p)) + p++; + VSLR(SLT_Status, fd, hp->status, p); + *p++ = '\0'; + + /* Next find the response */ + while (isspace(*p)) + p++; + hp->response = p; + while (*p != '\n') + p++; + for (q = p; q > hp->response && isspace(q[-1]); q--) + continue; + *q = '\0'; + VSLR(SLT_Response, fd, hp->response, q); + p++; + + return (http_dissect_hdrs(hp, fd, p)); +} + +/*--------------------------------------------------------------------*/ + static int http_header_complete(struct http *hp) { Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-11 06:30:38 UTC (rev 419) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-11 07:30:53 UTC (rev 420) @@ -170,7 +170,7 @@ hp = vc->http; http_RecvHead(hp, vc->fd, w->eb, NULL, NULL); event_base_loop(w->eb, 0); - http_Dissect(hp, vc->fd, 2); + http_DissectResponse(hp, vc->fd); http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-11 06:30:38 UTC (rev 419) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-11 07:30:53 UTC (rev 420) @@ -68,7 +68,7 @@ sp->vcl = GetVCL(); AZ(pthread_mutex_unlock(&sessmtx)); - done = http_Dissect(sp->http, sp->fd, 1); + done = http_DissectRequest(sp->http, sp->fd); if (done != 0) { RES_Error(w, sp, done, NULL); goto out; Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2006-07-11 06:30:38 UTC (rev 419) +++ trunk/varnish-cache/include/stat_field.h 2006-07-11 07:30:53 UTC (rev 420) @@ -21,3 +21,5 @@ MAC_STAT(n_wrk_failed, uint64_t, "u", "N worker threads not created"); MAC_STAT(n_wrk_short, uint64_t, "u", "N worker threads shortages"); MAC_STAT(n_wrk_busy, uint64_t, "u", "N busy worker threads"); + +MAC_STAT(losthdr, uint64_t, "u", "HTTP header overflows"); From phk at projects.linpro.no Tue Jul 11 07:38:42 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 09:38:42 +0200 (CEST) Subject: r421 - trunk/varnish-cache/bin/varnishd Message-ID: <20060711073842.B27761EC38D@projects.linpro.no> Author: phk Date: 2006-07-11 09:38:42 +0200 (Tue, 11 Jul 2006) New Revision: 421 Modified: trunk/varnish-cache/bin/varnishd/cache_response.c Log: Add 500 messages. Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-11 07:30:53 UTC (rev 420) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-11 07:38:42 UTC (rev 421) @@ -22,6 +22,7 @@ if (msg == NULL) { switch (error) { case 400: msg = "Bad Request"; break; + case 500: msg = "Internal Error"; break; default: msg = "HTTP request error"; break; } } @@ -43,8 +44,18 @@ " \r\n" " \r\n"); sbuf_printf(w->sb, "

Error %03d %s

\r\n", error, msg); + switch(error) { + case 400: + sbuf_cat(w->sb, + " Your HTTP protocol request did not make sense.\r\n"); + break; + case 500: + default: + sbuf_cat(w->sb, + " Something unexpected happened.\r\n"); + break; + } sbuf_cat(w->sb, - " Your HTTP protocol request did not make sense.\r\n" "

\r\n" " \r\n" " Varnish\r\n" From phk at projects.linpro.no Tue Jul 11 11:36:14 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 13:36:14 +0200 (CEST) Subject: r422 - in trunk/varnish-cache: include lib/libvarnishapi Message-ID: <20060711113614.02D4E1EC381@projects.linpro.no> Author: phk Date: 2006-07-11 13:36:14 +0200 (Tue, 11 Jul 2006) New Revision: 422 Modified: trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Add -b[ackend] and -c[lient] generic options to logtailers Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2006-07-11 07:38:42 UTC (rev 421) +++ trunk/varnish-cache/include/varnishapi.h 2006-07-11 11:36:14 UTC (rev 422) @@ -8,7 +8,7 @@ #define V_DEAD __attribute__ ((noreturn)) /* shmlog.c */ -#define VSL_ARGS "r:i:x:CI:X:" +#define VSL_ARGS "bcr:i:x:CI:X:" struct VSL_data; struct VSL_data *VSL_New(void); int VSL_OpenLog(struct VSL_data *vd); Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-11 07:38:42 UTC (rev 421) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-11 11:36:14 UTC (rev 422) @@ -24,9 +24,14 @@ FILE *fi; unsigned char rbuf[4 + 255 + 1]; + int b_opt; + int c_opt; + int ix_opt; unsigned char supr[256]; + unsigned char dir[65536]; + int regflags; regex_t *regincl; regex_t *regexcl; @@ -39,6 +44,7 @@ static int vsl_fd; static struct shmloghead *vsl_lh; + /*--------------------------------------------------------------------*/ const char *VSL_tags[256] = { @@ -102,6 +108,9 @@ VSL_OpenLog(struct VSL_data *vd) { + if (!vd->b_opt && !vd->c_opt) + vd->b_opt = vd->c_opt = 1; + if (vd->fi != NULL) return (0); @@ -155,14 +164,30 @@ { unsigned char *p; regmatch_t rm; + unsigned u; int i; while (1) { p = vsl_nextlog(vd); if (p == NULL) return (p); + u = (p[2] << 8) | p[3]; + switch(p[0]) { + case SLT_SessionOpen: + vd->dir[u] = 1; + break; + case SLT_BackendOpen: + vd->dir[u] = 2; + break; + default: + break; + } if (vd->supr[p[0]]) continue; + if (vd->b_opt && vd->dir[u] == 1) + continue; + if (vd->c_opt && vd->dir[u] == 2) + continue; if (vd->regincl != NULL) { rm.rm_so = 0; rm.rm_eo = p[1]; @@ -285,6 +310,8 @@ VSL_Arg(struct VSL_data *vd, int arg, const char *opt) { switch (arg) { + case 'b': vd->b_opt = !vd->b_opt; return (1); + case 'c': vd->c_opt = !vd->c_opt; return (1); case 'i': case 'x': return (vsl_ix_arg(vd, opt, arg)); case 'r': return (vsl_r_arg(vd, opt)); case 'I': case 'X': return (vsl_IX_arg(vd, opt, arg)); From des at projects.linpro.no Tue Jul 11 11:41:31 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Tue, 11 Jul 2006 13:41:31 +0200 (CEST) Subject: r423 - trunk/varnish-cache/bin/varnishlog Message-ID: <20060711114131.F2E701EC39F@projects.linpro.no> Author: des Date: 2006-07-11 13:41:31 +0200 (Tue, 11 Jul 2006) New Revision: 423 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.1 Log: Document -b and -c, and bump date. Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.1 =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.1 2006-07-11 11:36:14 UTC (rev 422) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.1 2006-07-11 11:41:31 UTC (rev 423) @@ -1,7 +1,7 @@ .\" .\" $Id$ .\" -.Dd July 6, 2006 +.Dd July 11, 2006 .Dt VARNISHLOG 1 .Os .Sh NAME @@ -10,6 +10,8 @@ .Sh SYNOPSIS .Nm .Op Fl C +.Op Fl b +.Op Fl c .Op Fl o .Op Fl I Ar regex .Op Fl X Ar regex @@ -28,14 +30,43 @@ .Bl -tag -width Fl .It Fl C Ignore case when matching regular expressions. +.It Fl b +Include log entries which result from communication with a backend +server. +If neither +.Fl b +nor +.Fl c +is specified, +.Nm +acts as if they both were. +.It Fl c +Include log entries which result from communication with a client. +If neither +.Fl b +nor +.Fl c +is specified, +.Nm +acts as if they both were. .It Fl o Group log entries by request ID. .It Fl I Ar regex Include log entries which match the specified regular expression. +If neither +.Fl I +nor +.Fl i +is specified, all log entries are included. .It Fl X Ar regex Exclude log entries which match the specified regular expression. .It Fl i Ar tag Include log entries with the specified tag. +If neither +.Fl I +nor +.Fl i +is specified, all log entries are included. .It Fl r Ar file Read log entries from .Ar file From phk at projects.linpro.no Tue Jul 11 12:00:22 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 14:00:22 +0200 (CEST) Subject: r424 - trunk/varnish-cache/bin/varnishd Message-ID: <20060711120022.6E2C51EC395@projects.linpro.no> Author: phk Date: 2006-07-11 14:00:22 +0200 (Tue, 11 Jul 2006) New Revision: 424 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Pipe requests which come with an Expect header. XXX: document that error 417 "expectation failed" might be a more sensible policy. Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-11 11:41:31 UTC (rev 423) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-11 12:00:22 UTC (rev 424) @@ -85,6 +85,9 @@ " if (req.request != \"GET\" && req.request != \"HEAD\") {\n" " pipe;\n" " }\n" + " if (req.http.Expect) {\n" + " pipe;\n" + " }\n" " if (req.http.Authenticate || req.http.Cookie) {\n" " pass;\n" " }\n" From phk at projects.linpro.no Tue Jul 11 12:30:23 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 14:30:23 +0200 (CEST) Subject: r425 - trunk/varnish-cache/lib/libvarnishapi Message-ID: <20060711123023.BE9821EC39C@projects.linpro.no> Author: phk Date: 2006-07-11 14:30:23 +0200 (Tue, 11 Jul 2006) New Revision: 425 Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Make -b and -c less nonsensical when not specified Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-11 12:00:22 UTC (rev 424) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-11 12:30:23 UTC (rev 425) @@ -108,8 +108,6 @@ VSL_OpenLog(struct VSL_data *vd) { - if (!vd->b_opt && !vd->c_opt) - vd->b_opt = vd->c_opt = 1; if (vd->fi != NULL) return (0); From phk at projects.linpro.no Tue Jul 11 12:31:45 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 14:31:45 +0200 (CEST) Subject: r426 - trunk/varnish-cache/bin/varnishd Message-ID: <20060711123145.056751EC395@projects.linpro.no> Author: phk Date: 2006-07-11 14:31:44 +0200 (Tue, 11 Jul 2006) New Revision: 426 Added: trunk/varnish-cache/bin/varnishd/cache_center.c Modified: trunk/varnish-cache/bin/varnishd/Makefile.am trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Start centralizing the flow of requests through varnish so we get one source file with the highest level of policy. Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-11 12:30:23 UTC (rev 425) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-11 12:31:44 UTC (rev 426) @@ -17,6 +17,7 @@ cache_acceptor.c \ cache_backend.c \ cache_ban.c \ + cache_center.c \ cache_expire.c \ cache_fetch.c \ cache_hash.c \ Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-11 12:30:23 UTC (rev 425) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-11 12:31:44 UTC (rev 426) @@ -205,6 +205,9 @@ void BAN_NewObj(struct object *o); int BAN_CheckObject(struct object *o, const char *url); +/* cache_center.c [CNT] */ +void CNT_Session(struct worker *w, struct sess *sp); + /* cache_expiry.c */ void EXP_Insert(struct object *o); void EXP_Init(void); Added: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-11 12:30:23 UTC (rev 425) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-11 12:31:44 UTC (rev 426) @@ -0,0 +1,108 @@ +/* + * $Id$ + * + */ + +#include +#include +#include +#include +#include +#include + +#include "libvarnish.h" +#include "heritage.h" +#include "shmlog.h" +#include "vcl.h" +#include "cache.h" + +/*--------------------------------------------------------------------*/ + +static int +LookupSession(struct worker *w, struct sess *sp) +{ + struct object *o; + + o = HSH_Lookup(w, sp->http); + sp->obj = o; + if (o->busy) { + VSL_stats->cache_miss++; + VCL_miss_method(sp); + } else { + VSL_stats->cache_hit++; + VSL(SLT_Hit, sp->fd, "%u", o->xid); + VCL_hit_method(sp); + } + return (0); +} + +static int +DeliverSession(struct worker *w, struct sess *sp) +{ + + + vca_write_obj(w, sp); + HSH_Deref(sp->obj); + sp->obj = NULL; + return (1); +} + +void +CNT_Session(struct worker *w, struct sess *sp) +{ + int done; + char *b; + + time(&sp->t0); + AZ(pthread_mutex_lock(&sessmtx)); + sp->vcl = GetVCL(); + AZ(pthread_mutex_unlock(&sessmtx)); + + done = http_DissectRequest(sp->http, sp->fd); + if (done != 0) { + RES_Error(w, sp, done, NULL); + goto out; + } + + sp->backend = sp->vcl->backend[0]; + + VCL_recv_method(sp); + + for (done = 0; !done; ) { + switch(sp->handling) { + case VCL_RET_LOOKUP: + done = LookupSession(w, sp); + break; + case VCL_RET_FETCH: + done = FetchSession(w, sp); + break; + case VCL_RET_DELIVER: + done = DeliverSession(w, sp); + break; + case VCL_RET_PIPE: + PipeSession(w, sp); + done = 1; + break; + case VCL_RET_PASS: + PassSession(w, sp); + done = 1; + break; + default: + INCOMPL(); + } + } + if (http_GetHdr(sp->http, "Connection", &b) && + !strcmp(b, "close")) { + vca_close_session(sp, "Connection header"); + } else if (http_GetProto(sp->http, &b) && + strcmp(b, "HTTP/1.1")) { + vca_close_session(sp, "not HTTP/1.1"); + } + +out: + AZ(pthread_mutex_lock(&sessmtx)); + RelVCL(sp->vcl); + AZ(pthread_mutex_unlock(&sessmtx)); + sp->vcl = NULL; + vca_return_session(sp); +} Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-11 12:30:23 UTC (rev 425) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-11 12:31:44 UTC (rev 426) @@ -28,97 +28,6 @@ /*--------------------------------------------------------------------*/ -static int -LookupSession(struct worker *w, struct sess *sp) -{ - struct object *o; - - o = HSH_Lookup(w, sp->http); - sp->obj = o; - if (o->busy) { - VSL_stats->cache_miss++; - VCL_miss_method(sp); - } else { - VSL_stats->cache_hit++; - VSL(SLT_Hit, sp->fd, "%u", o->xid); - VCL_hit_method(sp); - } - return (0); -} - -static int -DeliverSession(struct worker *w, struct sess *sp) -{ - - - vca_write_obj(w, sp); - HSH_Deref(sp->obj); - sp->obj = NULL; - return (1); -} - -static void -wrk_WorkSession(struct worker *w, struct sess *sp) -{ - int done; - char *b; - - time(&sp->t0); - AZ(pthread_mutex_lock(&sessmtx)); - sp->vcl = GetVCL(); - AZ(pthread_mutex_unlock(&sessmtx)); - - done = http_DissectRequest(sp->http, sp->fd); - if (done != 0) { - RES_Error(w, sp, done, NULL); - goto out; - } - - sp->backend = sp->vcl->backend[0]; - - VCL_recv_method(sp); - - for (done = 0; !done; ) { - switch(sp->handling) { - case VCL_RET_LOOKUP: - done = LookupSession(w, sp); - break; - case VCL_RET_FETCH: - done = FetchSession(w, sp); - break; - case VCL_RET_DELIVER: - done = DeliverSession(w, sp); - break; - case VCL_RET_PIPE: - PipeSession(w, sp); - done = 1; - break; - case VCL_RET_PASS: - PassSession(w, sp); - done = 1; - break; - default: - INCOMPL(); - } - } - if (http_GetHdr(sp->http, "Connection", &b) && - !strcmp(b, "close")) { - vca_close_session(sp, "Connection header"); - } else if (http_GetProto(sp->http, &b) && - strcmp(b, "HTTP/1.1")) { - vca_close_session(sp, "not HTTP/1.1"); - } - -out: - AZ(pthread_mutex_lock(&sessmtx)); - RelVCL(sp->vcl); - AZ(pthread_mutex_unlock(&sessmtx)); - sp->vcl = NULL; - vca_return_session(sp); -} - -/*--------------------------------------------------------------------*/ - static void * wrk_thread(void *priv) { @@ -154,7 +63,7 @@ TAILQ_REMOVE(&wrk_reqhead, wrq, list); AZ(pthread_mutex_unlock(&wrk_mtx)); assert(wrq->sess != NULL); - wrk_WorkSession(w, wrq->sess); + CNT_Session(w, wrq->sess); AZ(pthread_mutex_lock(&wrk_mtx)); VSL_stats->n_wrk_busy--; TAILQ_INSERT_HEAD(&wrk_head, w, list); From phk at projects.linpro.no Tue Jul 11 13:31:14 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 15:31:14 +0200 (CEST) Subject: r427 - trunk/varnish-cache/bin/varnishd Message-ID: <20060711133114.66E091EC39C@projects.linpro.no> Author: phk Date: 2006-07-11 15:31:14 +0200 (Tue, 11 Jul 2006) New Revision: 427 Added: trunk/varnish-cache/bin/varnishd/steps.h Modified: trunk/varnish-cache/bin/varnishd/cache.h Log: Add enum for major procesing steps Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-11 12:31:44 UTC (rev 426) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-11 13:31:14 UTC (rev 427) @@ -18,6 +18,14 @@ struct object; struct objhead; +/*--------------------------------------------------------------------*/ + +enum step { +#define STEP(l, u) STP_##u, +#include "steps.h" +#undef STEP +}; + /*-------------------------------------------------------------------- * HTTP Request/Response/Header handling structure. * RSN: struct worker and struct session will have one of these embedded. @@ -148,6 +156,7 @@ time_t t_req; time_t t_resp; + enum step step; unsigned handling; TAILQ_ENTRY(sess) list; Added: trunk/varnish-cache/bin/varnishd/steps.h =================================================================== --- trunk/varnish-cache/bin/varnishd/steps.h 2006-07-11 12:31:44 UTC (rev 426) +++ trunk/varnish-cache/bin/varnishd/steps.h 2006-07-11 13:31:14 UTC (rev 427) @@ -0,0 +1,13 @@ +/* $Id$ */ + +STEP(recv, RECV) +STEP(pipe, PIPE) +STEP(pass, PASS) +STEP(passbody, PASSBODY) +STEP(lookup, LOOKUP) +STEP(miss, MISS) +STEP(hit, HIT) +STEP(fetch, FETCH) +STEP(deliver, DELIVER) +STEP(error, ERROR) +STEP(done, DONE) From phk at projects.linpro.no Tue Jul 11 15:18:54 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 17:18:54 +0200 (CEST) Subject: r428 - trunk/varnish-cache/bin/varnishd Message-ID: <20060711151854.B96E61EC382@projects.linpro.no> Author: phk Date: 2006-07-11 17:18:54 +0200 (Tue, 11 Jul 2006) New Revision: 428 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: This commit breaks warnish temporarily: Insert the new master state engine. A dot(1) graph is embedded in the source code and can be extracted with: sed -n '/^DOT/s///p' cache_center.c | dot -Tps > /tmp/_.ps Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-11 13:31:14 UTC (rev 427) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-11 15:18:54 UTC (rev 428) @@ -1,8 +1,33 @@ /* * $Id$ * + * This file contains the central state machine for pushing requests. + * + * We cannot just use direct calls because it is possible to kick a + * request back to the lookup stage (usually after a rewrite). The + * state engine also allows us to break the processing up into some + * logical chunks which improves readability a little bit. + * + * Since the states are rather nasty in detail, I have decided to embedd + * a dot(1) graph in the source code comments. So to see the big picture, + * extract the DOT lines and run though dot(1), for instance with the + * command: + * sed -n '/^DOT/s///p' cache_center.c | dot -Tps > /tmp/_.ps + * */ +/* +DOT digraph vcl_center { +DOT page="8.2,11.7" +DOT size="6.3,9.7" +DOT margin="1.0" +DOT start [ +DOT shape=hexagon +DOT label="Start" +DOT ] +DOT start -> RECV + */ + #include #include #include @@ -47,62 +72,333 @@ return (1); } -void -CNT_Session(struct worker *w, struct sess *sp) +/*-------------------------------------------------------------------- +DOT subgraph cluster_deliver { +DOT deliver [ +DOT shape=ellipse +DOT label="Build & send header" +DOT ] +DOT DELIVER -> deliver [style=bold] +DOT deliver2 [ +DOT shape=ellipse +DOT label="Send object" +DOT ] +DOT deliver -> deliver2 [style=bold] +DOT } +DOT deliver2 -> DONE [style=bold] + */ + +static void cnt_deliver(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } + +/*-------------------------------------------------------------------- +DOT DONE [ +DOT shape=hexagon +DOT label="Request completed" +DOT ] + */ + +#if 0 + if (http_GetHdr(sp->http, "Connection", &b) && + !strcmp(b, "close")) { + vca_close_session(sp, "Connection header"); + } else if (http_GetProto(sp->http, &b) && + strcmp(b, "HTTP/1.1")) { + vca_close_session(sp, "not HTTP/1.1"); +#endif + +static void cnt_done(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } + +/*-------------------------------------------------------------------- +DOT subgraph cluster_error { +DOT error [ +DOT shape=ellipse +DOT label="Issue HTTP error" +DOT ] +DOT ERROR -> error +DOT } +DOT error -> DONE + */ + +static void cnt_error(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } + +/*-------------------------------------------------------------------- +DOT subgraph cluster_fetch { +DOT fetch [ +DOT shape=ellipse +DOT label="find obj.ttl\nobj.cacheable" +DOT ] +DOT FETCH -> fetch [style=bold] +DOT vcl_fetch [ +DOT shape=box +DOT label="vcl_fetch()" +DOT ] +DOT fetch -> vcl_fetch [style=bold] +DOT fetch_lookup [ +DOT shape=ellipse +DOT label="obj.cacheable=false\nunbusy obj\ndiscard body\n" +DOT ] +DOT vcl_fetch -> fetch_lookup [label="lookup", style=dotted, weight=0] +DOT fetch_pass [ +DOT shape=ellipse +DOT label="obj.cacheable=false\nunbusy obj" +DOT ] +DOT vcl_fetch -> fetch_pass [label="pass"] +DOT fetch_ipass [ +DOT shape=ellipse +DOT label="obj.cacheable=true\nobj.pass=true\nunbusy obj" +DOT ] +DOT vcl_fetch -> fetch_ipass [label="insert_pass"] +DOT fetch_insert [ +DOT shape=ellipse +DOT label="rcv body\nobj.cacheable=true\nunbusy" +DOT ] +DOT vcl_fetch -> fetch_insert [label="insert", style=bold] +DOT fetch_error [ +DOT shape=ellipse +DOT label="disc body\nobj.cacheable=false\nunbusy" +DOT ] +DOT vcl_fetch -> fetch_error [label="error"] +DOT } +DOT fetch_lookup -> LOOKUP [style=dotted, weight=0] +DOT fetch_pass -> PASSBODY +DOT fetch_ipass -> PASSBODY +DOT fetch_insert -> DELIVER [style=bold] +DOT fetch_error -> ERROR + */ + +static void cnt_fetch(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } + +/*-------------------------------------------------------------------- +DOT subgraph cluster_hit { +DOT hit [ +DOT shape=box +DOT label="vcl_hit()" +DOT ] +DOT HIT -> hit [style=bold] +DOT hit2 [ +DOT shape=diamond +DOT label="obj.pass ?" +DOT ] +DOT hit -> hit2 [label=deliver, style=bold] +DOT hit_lookup [ +DOT shape=ellipse +DOT label="unbusy" +DOT ] +DOT hit -> hit_lookup [label="lookup", style=dotted, weight=0] +DOT hit_error [ +DOT shape=ellipse +DOT label="unbusy" +DOT ] +DOT hit -> hit_error [label="error", weight=0] +DOT hit_pass [ +DOT shape=ellipse +DOT label="unbusy" +DOT ] +DOT hit -> hit_pass [label=pass] +DOT hit2 -> hit_pass +DOT } +DOT hit_error -> ERROR +DOT hit_pass -> PASS +DOT hit_lookup -> LOOKUP [style=dotted, weight=0] +DOT hit2 -> DELIVER [style=bold] + */ + +static void cnt_hit(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } + +/*-------------------------------------------------------------------- +DOT subgraph cluster_lookup { +DOT lookup [ +DOT shape=ellipse +DOT label="find obj in cache" +DOT ] +DOT LOOKUP -> lookup [style=bold] +DOT lookup2 [ +DOT shape=ellipse +DOT label="Insert new busy object" +DOT ] +DOT lookup -> lookup2 [style=bold] +DOT } +DOT lookup -> HIT [label="hit", style=bold] +DOT lookup2 -> MISS [label="miss", style=bold] + */ + +static void cnt_lookup(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } + +/*-------------------------------------------------------------------- +DOT subgraph cluster_miss { +DOT miss [ +DOT shape=box +DOT label="vcl_miss()" +DOT ] +DOT MISS -> miss [style=bold] +DOT miss_error [ +DOT shape=ellipse +DOT label="obj.cacheable=false\nunbusy" +DOT ] +DOT miss -> miss_error [label="error"] +DOT miss_pass [ +DOT shape=ellipse +DOT label="obj.cacheable=false\nunbusy" +DOT ] +DOT miss -> miss_pass [label="pass"] +DOT miss_lookup [ +DOT shape=ellipse +DOT label="obj.cacheable=false\nunbusy" +DOT ] +DOT miss -> miss_lookup [label="lookup", style=dotted, weight=0] +DOT miss_fetch [ +DOT shape=ellipse +DOT label="fetch obj headers\nfrom backend" +DOT ] +DOT miss -> miss_fetch [label="fetch", style=bold] +DOT } +DOT miss_error -> ERROR +DOT miss_pass -> PASS +DOT miss_fetch -> FETCH [style=bold] +DOT miss_lookup -> LOOKUP [style=dotted, weight=0] +DOT + */ + +static void cnt_miss(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } + +/*-------------------------------------------------------------------- +DOT subgraph cluster_pass { +DOT pass [ +DOT shape=ellipse +DOT label="send to bke\nrx bkehdr" +DOT ] +DOT PASS -> pass +DOT } +DOT pass -> PASSBODY + */ + +static void cnt_pass(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } + +/*-------------------------------------------------------------------- +DOT subgraph cluster_passbody { +DOT passbody [ +DOT shape=ellipse +DOT label="send hdrs\npass body\n" +DOT ] +DOT PASSBODY -> passbody +DOT } +DOT passbody -> DONE + */ + +static void cnt_passbody(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } + +/*-------------------------------------------------------------------- +DOT subgraph cluster_pipe { +DOT pipe [ +DOT shape=ellipse +DOT label="build&send hdr\npipe until close" +DOT ] +DOT PIPE -> pipe +DOT } +DOT pipe -> DONE + */ + +static void cnt_pipe(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } + +/*-------------------------------------------------------------------- +DOT subgraph cluster_recv { +DOT recv [ +DOT shape=box +DOT label="vcl_recv()" +DOT ] +DOT RECV -> recv +DOT recv_lookup [ +DOT shape=ellipse +DOT label="discard any body" +DOT ] +DOT recv -> recv_lookup [label="lookup"] +DOT recv_error [ +DOT shape=ellipse +DOT label="discard any body" +DOT ] +DOT recv -> recv_error [label="error"] +DOT } +DOT recv -> PIPE [label="pipe"] +DOT recv -> PASS [label="pass"] +DOT recv_lookup -> LOOKUP +DOT recv_error -> ERROR + */ + +static void +cnt_recv(struct worker *w, struct sess *sp) { int done; - char *b; - time(&sp->t0); - AZ(pthread_mutex_lock(&sessmtx)); - sp->vcl = GetVCL(); - AZ(pthread_mutex_unlock(&sessmtx)); - done = http_DissectRequest(sp->http, sp->fd); if (done != 0) { RES_Error(w, sp, done, NULL); - goto out; + sp->step = STP_DONE; + return; } sp->backend = sp->vcl->backend[0]; + /* XXX: Handle TRACE & OPTIONS of Max-Forwards = 0 */ + + /* XXX: determine if request comes with body */ + VCL_recv_method(sp); - for (done = 0; !done; ) { - switch(sp->handling) { - case VCL_RET_LOOKUP: - done = LookupSession(w, sp); - break; - case VCL_RET_FETCH: - done = FetchSession(w, sp); - break; - case VCL_RET_DELIVER: - done = DeliverSession(w, sp); - break; - case VCL_RET_PIPE: - PipeSession(w, sp); - done = 1; - break; - case VCL_RET_PASS: - PassSession(w, sp); - done = 1; - break; - default: - INCOMPL(); + switch(sp->handling) { + case VCL_RET_LOOKUP: + /* XXX: discard req body, if any */ + sp->step = STP_LOOKUP; + return; + case VCL_RET_PIPE: + sp->step = STP_PIPE; + return; + case VCL_RET_PASS: + sp->step = STP_PASS; + return; + case VCL_RET_ERROR: + /* XXX: discard req body, if any */ + sp->step = STP_ERROR; + return; + default: + INCOMPL(); + } +} + + +/*-------------------------------------------------------------------- + * Central state engine dispatcher. + * + * We grab a VCL reference, and keeps kicking the session around until + * it has had enough. + * + */ + +void +CNT_Session(struct worker *w, struct sess *sp) +{ + + time(&sp->t0); + AZ(pthread_mutex_lock(&sessmtx)); + sp->vcl = GetVCL(); + AZ(pthread_mutex_unlock(&sessmtx)); + + for (sp->step = STP_RECV; sp->step != STP_DONE; ) { + switch (sp->step) { +#define STEP(l,u) case STP_##u: cnt_##l(w, sp); break; +#include "steps.h" +#undef STEP + default: INCOMPL(); } } - if (http_GetHdr(sp->http, "Connection", &b) && - !strcmp(b, "close")) { - vca_close_session(sp, "Connection header"); - } else if (http_GetProto(sp->http, &b) && - strcmp(b, "HTTP/1.1")) { - vca_close_session(sp, "not HTTP/1.1"); - } -out: AZ(pthread_mutex_lock(&sessmtx)); RelVCL(sp->vcl); AZ(pthread_mutex_unlock(&sessmtx)); sp->vcl = NULL; + vca_return_session(sp); } + +/* +DOT } +*/ From phk at projects.linpro.no Tue Jul 11 15:54:05 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 17:54:05 +0200 (CEST) Subject: r429 - trunk/varnish-cache/bin/varnishd Message-ID: <20060711155405.4AAA31EC381@projects.linpro.no> Author: phk Date: 2006-07-11 17:54:05 +0200 (Tue, 11 Jul 2006) New Revision: 429 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: Make things work again by stuffing the old functions into the new state engine. Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-11 15:18:54 UTC (rev 428) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-11 15:54:05 UTC (rev 429) @@ -41,37 +41,6 @@ #include "vcl.h" #include "cache.h" -/*--------------------------------------------------------------------*/ - -static int -LookupSession(struct worker *w, struct sess *sp) -{ - struct object *o; - - o = HSH_Lookup(w, sp->http); - sp->obj = o; - if (o->busy) { - VSL_stats->cache_miss++; - VCL_miss_method(sp); - } else { - VSL_stats->cache_hit++; - VSL(SLT_Hit, sp->fd, "%u", o->xid); - VCL_hit_method(sp); - } - return (0); -} - -static int -DeliverSession(struct worker *w, struct sess *sp) -{ - - - vca_write_obj(w, sp); - HSH_Deref(sp->obj); - sp->obj = NULL; - return (1); -} - /*-------------------------------------------------------------------- DOT subgraph cluster_deliver { DOT deliver [ @@ -97,17 +66,21 @@ DOT ] */ -#if 0 + +static void +cnt_done(struct worker *w, struct sess *sp) +{ + char *b; + if (http_GetHdr(sp->http, "Connection", &b) && !strcmp(b, "close")) { vca_close_session(sp, "Connection header"); } else if (http_GetProto(sp->http, &b) && strcmp(b, "HTTP/1.1")) { vca_close_session(sp, "not HTTP/1.1"); -#endif + } +} -static void cnt_done(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } - /*-------------------------------------------------------------------- DOT subgraph cluster_error { DOT error [ @@ -203,8 +176,18 @@ DOT hit2 -> DELIVER [style=bold] */ -static void cnt_hit(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } +static void +cnt_hit(struct worker *w, struct sess *sp) +{ + VCL_hit_method(sp); + + vca_write_obj(w, sp); + HSH_Deref(sp->obj); + sp->obj = NULL; + sp->step = STP_DONE; +} + /*-------------------------------------------------------------------- DOT subgraph cluster_lookup { DOT lookup [ @@ -222,8 +205,23 @@ DOT lookup2 -> MISS [label="miss", style=bold] */ -static void cnt_lookup(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } +static void +cnt_lookup(struct worker *w, struct sess *sp) +{ + struct object *o; + o = HSH_Lookup(w, sp->http); + sp->obj = o; + if (o->busy) { + VSL_stats->cache_miss++; + sp->step = STP_MISS; + } else { + VSL_stats->cache_hit++; + VSL(SLT_Hit, sp->fd, "%u", o->xid); + sp->step = STP_HIT; + } +} + /*-------------------------------------------------------------------- DOT subgraph cluster_miss { DOT miss [ @@ -259,8 +257,26 @@ DOT */ -static void cnt_miss(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } +static void +cnt_miss(struct worker *w, struct sess *sp) +{ + VCL_miss_method(sp); + if (sp->handling == VCL_RET_ERROR) + INCOMPL(); + if (sp->handling == VCL_RET_PASS) + INCOMPL(); + if (sp->handling == VCL_RET_LOOKUP) + INCOMPL(); + if (sp->handling == VCL_RET_FETCH) { + /* XXX */ + FetchSession(w, sp); + sp->step = STP_DONE; + return; + } + INCOMPL(); +} + /*-------------------------------------------------------------------- DOT subgraph cluster_pass { DOT pass [ @@ -391,6 +407,8 @@ } } + cnt_done(w, sp); /* The loop doesn't do this */ + AZ(pthread_mutex_lock(&sessmtx)); RelVCL(sp->vcl); AZ(pthread_mutex_unlock(&sessmtx)); From phk at projects.linpro.no Tue Jul 11 16:03:25 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 18:03:25 +0200 (CEST) Subject: r430 - in trunk/varnish-cache: include lib/libvcl Message-ID: <20060711160325.4991D1EC382@projects.linpro.no> Author: phk Date: 2006-07-11 18:03:25 +0200 (Tue, 11 Jul 2006) New Revision: 430 Modified: trunk/varnish-cache/include/vcl_returns.h trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl trunk/varnish-cache/lib/libvcl/vcl_token_defs.h Log: Add "insert_pass" action in VCL Modified: trunk/varnish-cache/include/vcl_returns.h =================================================================== --- trunk/varnish-cache/include/vcl_returns.h 2006-07-11 15:54:05 UTC (rev 429) +++ trunk/varnish-cache/include/vcl_returns.h 2006-07-11 16:03:25 UTC (rev 430) @@ -13,26 +13,28 @@ VCL_RET_MAC(lookup, LOOKUP, (1 << 1)) VCL_RET_MAC(pipe, PIPE, (1 << 2)) VCL_RET_MAC(pass, PASS, (1 << 3)) -VCL_RET_MAC(fetch, FETCH, (1 << 4)) -VCL_RET_MAC(insert, INSERT, (1 << 5)) -VCL_RET_MAC(deliver, DELIVER, (1 << 6)) -VCL_RET_MAC(discard, DISCARD, (1 << 7)) +VCL_RET_MAC(insert_pass, INSERT_PASS, (1 << 4)) +VCL_RET_MAC(fetch, FETCH, (1 << 5)) +VCL_RET_MAC(insert, INSERT, (1 << 6)) +VCL_RET_MAC(deliver, DELIVER, (1 << 7)) +VCL_RET_MAC(discard, DISCARD, (1 << 8)) #else #define VCL_RET_ERROR (1 << 0) #define VCL_RET_LOOKUP (1 << 1) #define VCL_RET_PIPE (1 << 2) #define VCL_RET_PASS (1 << 3) -#define VCL_RET_FETCH (1 << 4) -#define VCL_RET_INSERT (1 << 5) -#define VCL_RET_DELIVER (1 << 6) -#define VCL_RET_DISCARD (1 << 7) -#define VCL_RET_MAX 8 +#define VCL_RET_INSERT_PASS (1 << 4) +#define VCL_RET_FETCH (1 << 5) +#define VCL_RET_INSERT (1 << 6) +#define VCL_RET_DELIVER (1 << 7) +#define VCL_RET_DISCARD (1 << 8) +#define VCL_RET_MAX 9 #endif #ifdef VCL_MET_MAC VCL_MET_MAC(recv,RECV,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_LOOKUP)) VCL_MET_MAC(miss,MISS,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_FETCH)) VCL_MET_MAC(hit,HIT,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_DELIVER)) -VCL_MET_MAC(fetch,FETCH,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_INSERT)) +VCL_MET_MAC(fetch,FETCH,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_INSERT|VCL_RET_INSERT_PASS)) VCL_MET_MAC(timeout,TIMEOUT,(VCL_RET_FETCH|VCL_RET_DISCARD)) #endif Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-07-11 15:54:05 UTC (rev 429) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-07-11 16:03:25 UTC (rev 430) @@ -230,6 +230,13 @@ return (0); case 'i': if (p[0] == 'i' && p[1] == 'n' && p[2] == 's' && + p[3] == 'e' && p[4] == 'r' && p[5] == 't' && + p[6] == '_' && p[7] == 'p' && p[8] == 'a' && + p[9] == 's' && p[10] == 's' && !isvar(p[11])) { + *q = p + 11; + return (T_INSERT_PASS); + } + if (p[0] == 'i' && p[1] == 'n' && p[2] == 's' && p[3] == 'e' && p[4] == 'r' && p[5] == 't' && !isvar(p[6])) { *q = p + 6; @@ -393,6 +400,7 @@ vcl_tnames[T_INC] = "++"; vcl_tnames[T_INCR] = "+="; vcl_tnames[T_INSERT] = "insert"; + vcl_tnames[T_INSERT_PASS] = "insert_pass"; vcl_tnames[T_LEQ] = "<="; vcl_tnames[T_LOOKUP] = "lookup"; vcl_tnames[T_MUL] = "*="; @@ -418,10 +426,11 @@ fputs("#define VCL_RET_LOOKUP (1 << 1)\n", f); fputs("#define VCL_RET_PIPE (1 << 2)\n", f); fputs("#define VCL_RET_PASS (1 << 3)\n", f); - fputs("#define VCL_RET_FETCH (1 << 4)\n", f); - fputs("#define VCL_RET_INSERT (1 << 5)\n", f); - fputs("#define VCL_RET_DELIVER (1 << 6)\n", f); - fputs("#define VCL_RET_DISCARD (1 << 7)\n", f); + fputs("#define VCL_RET_INSERT_PASS (1 << 4)\n", f); + fputs("#define VCL_RET_FETCH (1 << 5)\n", f); + fputs("#define VCL_RET_INSERT (1 << 6)\n", f); + fputs("#define VCL_RET_DELIVER (1 << 7)\n", f); + fputs("#define VCL_RET_DISCARD (1 << 8)\n", f); fputs("/*\n", f); fputs(" * $Id$\n", f); fputs(" *\n", f); Modified: trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-07-11 15:54:05 UTC (rev 429) +++ trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-07-11 16:03:25 UTC (rev 430) @@ -10,7 +10,7 @@ {recv {error pass pipe lookup}} {miss {error pass pipe fetch}} {hit {error pass pipe deliver}} - {fetch {error pass pipe insert}} + {fetch {error pass pipe insert insert_pass}} {timeout {fetch discard}} } @@ -21,6 +21,7 @@ lookup pipe pass + insert_pass fetch insert deliver Modified: trunk/varnish-cache/lib/libvcl/vcl_token_defs.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-07-11 15:54:05 UTC (rev 429) +++ trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-07-11 16:03:25 UTC (rev 430) @@ -26,27 +26,28 @@ #define T_LOOKUP 144 #define T_PIPE 145 #define T_PASS 146 -#define T_FETCH 147 -#define T_INSERT 148 -#define T_DELIVER 149 -#define T_DISCARD 150 -#define T_INC 151 -#define T_DEC 152 -#define T_CAND 153 -#define T_COR 154 -#define T_LEQ 155 -#define T_EQ 156 -#define T_NEQ 157 -#define T_GEQ 158 -#define T_SHR 159 -#define T_SHL 160 -#define T_INCR 161 -#define T_DECR 162 -#define T_MUL 163 -#define T_DIV 164 -#define ID 165 -#define VAR 166 -#define CNUM 167 -#define CSTR 168 -#define EOI 169 -#define METHOD 170 +#define T_INSERT_PASS 147 +#define T_FETCH 148 +#define T_INSERT 149 +#define T_DELIVER 150 +#define T_DISCARD 151 +#define T_INC 152 +#define T_DEC 153 +#define T_CAND 154 +#define T_COR 155 +#define T_LEQ 156 +#define T_EQ 157 +#define T_NEQ 158 +#define T_GEQ 159 +#define T_SHR 160 +#define T_SHL 161 +#define T_INCR 162 +#define T_DECR 163 +#define T_MUL 164 +#define T_DIV 165 +#define ID 166 +#define VAR 167 +#define CNUM 168 +#define CSTR 169 +#define EOI 170 +#define METHOD 171 From phk at projects.linpro.no Tue Jul 11 16:10:10 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 18:10:10 +0200 (CEST) Subject: r431 - trunk/varnish-cache/bin/varnishd Message-ID: <20060711161010.BC2911EC395@projects.linpro.no> Author: phk Date: 2006-07-11 18:10:10 +0200 (Tue, 11 Jul 2006) New Revision: 431 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_fetch.c Log: Distribute code from FetchSession almost correctly Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-11 16:03:25 UTC (rev 430) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-11 16:10:10 UTC (rev 431) @@ -161,6 +161,8 @@ TAILQ_ENTRY(sess) list; + struct vbe_conn *vbc; + struct http *bkd_http; struct backend *backend; struct object *obj; struct VCL_conf *vcl; @@ -223,7 +225,8 @@ void EXP_TTLchange(struct object *o); /* cache_fetch.c */ -int FetchSession(struct worker *w, struct sess *sp); +int FetchBody(struct worker *w, struct sess *sp); +int FetchHeaders(struct worker *w, struct sess *sp); /* cache_hash.c */ struct object *HSH_Lookup(struct worker *w, struct http *h); Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-11 16:03:25 UTC (rev 430) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-11 16:10:10 UTC (rev 431) @@ -57,8 +57,14 @@ DOT deliver2 -> DONE [style=bold] */ -static void cnt_deliver(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } +static void +cnt_deliver(struct worker *w, struct sess *sp) +{ + vca_write_obj(w, sp); + sp->step = STP_DONE; +} + /*-------------------------------------------------------------------- DOT DONE [ DOT shape=hexagon @@ -139,8 +145,30 @@ DOT fetch_error -> ERROR */ -static void cnt_fetch(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } +static void +cnt_fetch(struct worker *w, struct sess *sp) +{ + RFC2616_cache_policy(sp, sp->bkd_http); + + VCL_fetch_method(sp); + + if (sp->handling == VCL_RET_LOOKUP) + INCOMPL(); + if (sp->handling == VCL_RET_PASS) + INCOMPL(); + if (sp->handling == VCL_RET_INSERT_PASS) + INCOMPL(); + if (sp->handling == VCL_RET_INSERT) { + FetchBody(w, sp); + sp->step = STP_DELIVER; + return; + } + if (sp->handling == VCL_RET_ERROR) + INCOMPL(); + INCOMPL(); +} + /*-------------------------------------------------------------------- DOT subgraph cluster_hit { DOT hit [ @@ -208,16 +236,14 @@ static void cnt_lookup(struct worker *w, struct sess *sp) { - struct object *o; - o = HSH_Lookup(w, sp->http); - sp->obj = o; - if (o->busy) { + sp->obj = HSH_Lookup(w, sp->http); + if (sp->obj->busy) { VSL_stats->cache_miss++; sp->step = STP_MISS; } else { VSL_stats->cache_hit++; - VSL(SLT_Hit, sp->fd, "%u", o->xid); + VSL(SLT_Hit, sp->fd, "%u", sp->obj->xid); sp->step = STP_HIT; } } @@ -269,9 +295,8 @@ if (sp->handling == VCL_RET_LOOKUP) INCOMPL(); if (sp->handling == VCL_RET_FETCH) { - /* XXX */ - FetchSession(w, sp); - sp->step = STP_DONE; + FetchHeaders(w, sp); + sp->step = STP_FETCH; return; } INCOMPL(); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-11 16:03:25 UTC (rev 430) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-11 16:10:10 UTC (rev 431) @@ -229,8 +229,9 @@ } /*--------------------------------------------------------------------*/ + int -FetchSession(struct worker *w, struct sess *sp) +FetchBody(struct worker *w, struct sess *sp) { int i, cls; struct vbe_conn *vc; @@ -238,38 +239,9 @@ char *b; int body; - sp->obj->xid = sp->xid; + vc = sp->vbc; + hp = sp->bkd_http; - vc = VBE_GetFd(sp->backend, sp->xid); - if (vc == NULL) - vc = VBE_GetFd(sp->backend, sp->xid); - assert(vc != NULL); /* XXX: handle this */ - VSL(SLT_Backend, sp->fd, "%d %s", vc->fd, sp->backend->vcl_name); - - hp = vc->http; - http_BuildSbuf(vc->fd, Build_Fetch, w->sb, sp->http); - i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); - assert(i == sbuf_len(w->sb)); - time(&sp->t_req); - - /* XXX: copy any contents */ - - /* - * XXX: It might be cheaper to avoid the event_engine and simply - * XXX: read(2) the header - */ - http_RecvHead(hp, vc->fd, w->eb, NULL, NULL); - event_base_loop(w->eb, 0); - time(&sp->t_resp); - assert(http_DissectResponse(hp, vc->fd) == 0); - - body = RFC2616_cache_policy(sp, hp); - - VCL_fetch_method(sp); - - if (sp->obj->cacheable) - EXP_Insert(sp->obj); - http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); if (body) { if (http_GetHdr(hp, "Content-Length", &b)) @@ -285,8 +257,6 @@ sp->obj->header = strdup(sbuf_data(w->sb)); VSL_stats->n_header++; - vca_write_obj(w, sp); - if (http_GetHdr(hp, "Connection", &b) && !strcasecmp(b, "close")) cls = 1; @@ -295,9 +265,50 @@ else VBE_RecycleFd(vc); + if (sp->obj->cacheable) + EXP_Insert(sp->obj); HSH_Unbusy(sp->obj); if (!sp->obj->cacheable) HSH_Deref(sp->obj); + return (0); +} - return (1); +/*--------------------------------------------------------------------*/ + +int +FetchHeaders(struct worker *w, struct sess *sp) +{ + int i, cls; + struct vbe_conn *vc; + struct http *hp; + char *b; + int body; + + sp->obj->xid = sp->xid; + + vc = VBE_GetFd(sp->backend, sp->xid); + if (vc == NULL) + vc = VBE_GetFd(sp->backend, sp->xid); + assert(vc != NULL); /* XXX: handle this */ + VSL(SLT_Backend, sp->fd, "%d %s", vc->fd, sp->backend->vcl_name); + + hp = vc->http; + http_BuildSbuf(vc->fd, Build_Fetch, w->sb, sp->http); + i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); + assert(i == sbuf_len(w->sb)); + time(&sp->t_req); + + /* XXX: copy any body ?? */ + + /* + * XXX: It might be cheaper to avoid the event_engine and simply + * XXX: read(2) the header + */ + http_RecvHead(hp, vc->fd, w->eb, NULL, NULL); + event_base_loop(w->eb, 0); + time(&sp->t_resp); + assert(http_DissectResponse(hp, vc->fd) == 0); + sp->vbc = vc; + sp->bkd_http = hp; + return (0); } From phk at projects.linpro.no Tue Jul 11 16:17:13 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 18:17:13 +0200 (CEST) Subject: r432 - trunk/varnish-cache/bin/varnishd Message-ID: <20060711161713.316DD1EC395@projects.linpro.no> Author: phk Date: 2006-07-11 18:17:13 +0200 (Tue, 11 Jul 2006) New Revision: 432 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: add short descriptive comments to each state Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-11 16:10:10 UTC (rev 431) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-11 16:17:13 UTC (rev 432) @@ -41,7 +41,10 @@ #include "vcl.h" #include "cache.h" + /*-------------------------------------------------------------------- + * We have a refcounted object on the session, now deliver it. + * DOT subgraph cluster_deliver { DOT deliver [ DOT shape=ellipse @@ -65,14 +68,17 @@ sp->step = STP_DONE; } + /*-------------------------------------------------------------------- + * This is the final state, figure out if we should close or recycle + * the client connection + * DOT DONE [ DOT shape=hexagon DOT label="Request completed" DOT ] */ - static void cnt_done(struct worker *w, struct sess *sp) { @@ -87,7 +93,10 @@ } } + /*-------------------------------------------------------------------- + * Emit an error + * DOT subgraph cluster_error { DOT error [ DOT shape=ellipse @@ -100,7 +109,11 @@ static void cnt_error(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } + /*-------------------------------------------------------------------- + * We have fetched the headers from the backend, ask the VCL code what + * to do next, then head off in that direction. + * DOT subgraph cluster_fetch { DOT fetch [ DOT shape=ellipse @@ -169,7 +182,10 @@ INCOMPL(); } + /*-------------------------------------------------------------------- + * We had a cache hit. Ask VCL, then march off as instructed. + * DOT subgraph cluster_hit { DOT hit [ DOT shape=box @@ -216,7 +232,10 @@ sp->step = STP_DONE; } + /*-------------------------------------------------------------------- + * Look up request in hash table + * DOT subgraph cluster_lookup { DOT lookup [ DOT shape=ellipse @@ -248,7 +267,10 @@ } } + /*-------------------------------------------------------------------- + * We had a miss, ask VCL, proceed as instructed + * DOT subgraph cluster_miss { DOT miss [ DOT shape=box @@ -302,7 +324,11 @@ INCOMPL(); } + /*-------------------------------------------------------------------- + * Start pass processing by getting headers from backend, then + * continue in passbody. + * DOT subgraph cluster_pass { DOT pass [ DOT shape=ellipse @@ -315,7 +341,11 @@ static void cnt_pass(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } + /*-------------------------------------------------------------------- + * We get here when we have the backends headers, send them to client + * and pass any body the backend may have on as well. + * DOT subgraph cluster_passbody { DOT passbody [ DOT shape=ellipse @@ -328,7 +358,11 @@ static void cnt_passbody(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } + /*-------------------------------------------------------------------- + * Ship the request header to the backend unchanged, then pipe + * until one of the ends close the connection. + * DOT subgraph cluster_pipe { DOT pipe [ DOT shape=ellipse @@ -341,7 +375,10 @@ static void cnt_pipe(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } + /*-------------------------------------------------------------------- + * Dispatch the request as instructed by VCL + * DOT subgraph cluster_recv { DOT recv [ DOT shape=box From phk at projects.linpro.no Tue Jul 11 16:25:50 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 18:25:50 +0200 (CEST) Subject: r433 - trunk/varnish-cache/bin/varnishd Message-ID: <20060711162550.152811EC395@projects.linpro.no> Author: phk Date: 2006-07-11 18:25:49 +0200 (Tue, 11 Jul 2006) New Revision: 433 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Allow backend to be specified as "host:port" to -b Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-11 16:17:13 UTC (rev 432) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-11 16:25:49 UTC (rev 433) @@ -66,6 +66,7 @@ vcl_default(const char *bflag) { char *buf, *vf; + const char *p, *q; struct sbuf *sb; /* @@ -76,10 +77,19 @@ * XXX: a bug for a backend to not reply at that time, so then * XXX: again: we should check it here in the "trivial" case. */ + p = strchr(bflag, ':'); + if (p != NULL) { + q = p + 1; + } else { + p = strchr(bflag, '\0'); + q = "http"; + } + buf = NULL; asprintf(&buf, "backend default {\n" - " set backend.host = \"%s\";\n" + " set backend.host = \"%*.*s\";\n" + " set backend.port = \"%s\";\n" "}\n" "sub vcl_recv {\n" " if (req.request != \"GET\" && req.request != \"HEAD\") {\n" @@ -117,7 +127,7 @@ "sub vcl_timeout {\n" " discard;\n" "}\n" - "", bflag); + "", p - bflag, p - bflag, bflag, q); assert(buf != NULL); sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(sb != NULL); From phk at projects.linpro.no Tue Jul 11 16:31:05 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 18:31:05 +0200 (CEST) Subject: r434 - trunk/varnish-cache/bin/varnishd Message-ID: <20060711163105.71BC31EC395@projects.linpro.no> Author: phk Date: 2006-07-11 18:31:05 +0200 (Tue, 11 Jul 2006) New Revision: 434 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Start the cache process automatically, I've gotten tired of typing "start" :-) Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-11 16:25:49 UTC (rev 433) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-11 16:31:05 UTC (rev 434) @@ -326,6 +326,8 @@ event_base_set(mgt_eb, &e_sigchld); signal_add(&e_sigchld, NULL); + mgt_child_start(); + i = event_base_loop(mgt_eb, 0); if (i != 0) printf("event_dispatch() = %d\n", i); From phk at projects.linpro.no Tue Jul 11 17:23:41 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 19:23:41 +0200 (CEST) Subject: r435 - in trunk/varnish-cache: . bin bin/varnishtester Message-ID: <20060711172341.546DA1EC38E@projects.linpro.no> Author: phk Date: 2006-07-11 19:23:41 +0200 (Tue, 11 Jul 2006) New Revision: 435 Added: trunk/varnish-cache/bin/varnishtester/ trunk/varnish-cache/bin/varnishtester/Makefile.am trunk/varnish-cache/bin/varnishtester/varnishtester.c Modified: trunk/varnish-cache/bin/Makefile.am trunk/varnish-cache/configure.ac Log: Add stub varnishtester program Modified: trunk/varnish-cache/bin/Makefile.am =================================================================== --- trunk/varnish-cache/bin/Makefile.am 2006-07-11 16:31:05 UTC (rev 434) +++ trunk/varnish-cache/bin/Makefile.am 2006-07-11 17:23:41 UTC (rev 435) @@ -1,3 +1,3 @@ # $Id$ -SUBDIRS = varnishd varnishlog varnishstat varnishtop +SUBDIRS = varnishd varnishlog varnishstat varnishtester varnishtop Added: trunk/varnish-cache/bin/varnishtester/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishtester/Makefile.am 2006-07-11 16:31:05 UTC (rev 434) +++ trunk/varnish-cache/bin/varnishtester/Makefile.am 2006-07-11 17:23:41 UTC (rev 435) @@ -0,0 +1,16 @@ +# $Id: Makefile.am 426 2006-07-11 12:31:44Z phk $ + +INCLUDES = -I$(top_srcdir)/include -I$(top_srcdir)/contrib/libevent + +bin_PROGRAMS = varnishtester + +# man_MANS = varnishd.1 + +varnishtester_SOURCES = \ + varnishtester.c + +# varnishd_LDFLAGS = -export-dynamic + +varnishtester_LDADD = \ + $(top_builddir)/lib/libvarnish/libvarnish.la \ + $(top_builddir)/contrib/libevent/libevent.la Added: trunk/varnish-cache/bin/varnishtester/varnishtester.c =================================================================== --- trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 16:31:05 UTC (rev 434) +++ trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 17:23:41 UTC (rev 435) @@ -0,0 +1,29 @@ + +#include + +#include "libvarnish.h" + + +/*-------------------------------------------------------------------- + * stubs to survice linkage + */ + +#include "cli_priv.h" + +struct cli; + +void cli_out(struct cli *cli, const char *fmt, ...) { (void)cli; (void)fmt; abort(); } +void cli_param(struct cli *cli) { (void)cli; abort(); } +void cli_result(struct cli *cli, unsigned res) { (void)cli; (void)res; abort(); } + +/*--------------------------------------------------------------------*/ + +int +main(int argc, char **argv) +{ + + (void)argc; + (void)argv; + + return (0); +} Modified: trunk/varnish-cache/configure.ac =================================================================== --- trunk/varnish-cache/configure.ac 2006-07-11 16:31:05 UTC (rev 434) +++ trunk/varnish-cache/configure.ac 2006-07-11 17:23:41 UTC (rev 435) @@ -70,6 +70,7 @@ bin/varnishd/Makefile bin/varnishlog/Makefile bin/varnishstat/Makefile + bin/varnishtester/Makefile bin/varnishtop/Makefile contrib/Makefile include/Makefile From phk at projects.linpro.no Tue Jul 11 17:53:44 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 19:53:44 +0200 (CEST) Subject: r436 - trunk/varnish-cache/bin/varnishd Message-ID: <20060711175344.3BCC61EC381@projects.linpro.no> Author: phk Date: 2006-07-11 19:53:44 +0200 (Tue, 11 Jul 2006) New Revision: 436 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Unbuffer stdout/stderr Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-11 17:23:41 UTC (rev 435) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-11 17:53:44 UTC (rev 436) @@ -489,6 +489,9 @@ const char *hflag = "classic"; register_printf_render_std((const unsigned char *)"HVQ"); + + setbuf(stdout, NULL); + setbuf(stderr, NULL); VCC_InitCompile(); From phk at projects.linpro.no Tue Jul 11 18:16:14 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 20:16:14 +0200 (CEST) Subject: r437 - trunk/varnish-cache/bin/varnishd Message-ID: <20060711181614.0E3F91EC382@projects.linpro.no> Author: phk Date: 2006-07-11 20:16:13 +0200 (Tue, 11 Jul 2006) New Revision: 437 Modified: trunk/varnish-cache/bin/varnishd/mgt.h trunk/varnish-cache/bin/varnishd/mgt_child.c trunk/varnish-cache/bin/varnishd/varnishd.c Log: Give 'exit' CLI command some bite. we may want to be more careful later on. Modified: trunk/varnish-cache/bin/varnishd/mgt.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt.h 2006-07-11 17:53:44 UTC (rev 436) +++ trunk/varnish-cache/bin/varnishd/mgt.h 2006-07-11 18:16:13 UTC (rev 437) @@ -8,6 +8,7 @@ void mgt_child_start(void); void mgt_child_stop(void); +void mgt_child_kill(void); void mgt_sigchld(int, short, void *); typedef void mgt_ccb_f(unsigned, const char *, void *); Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_child.c 2006-07-11 17:53:44 UTC (rev 436) +++ trunk/varnish-cache/bin/varnishd/mgt_child.c 2006-07-11 18:16:13 UTC (rev 437) @@ -283,6 +283,16 @@ /*--------------------------------------------------------------------*/ void +mgt_child_kill(void) +{ + + desired = H_STOP; + kill(child_pid, 9); +} + +/*--------------------------------------------------------------------*/ + +void mgt_sigchld(int a, short b, void *c) { pid_t p; Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-11 17:53:44 UTC (rev 436) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-11 18:16:13 UTC (rev 437) @@ -240,6 +240,16 @@ /*--------------------------------------------------------------------*/ static void +m_cli_func_exit(struct cli *cli, char **av, void *priv) +{ + + mgt_child_kill(); + exit (0); +} + +/*--------------------------------------------------------------------*/ + +static void m_cli_func_verbose(struct cli *cli, char **av, void *priv) { @@ -304,7 +314,7 @@ { CLI_ZERO }, { CLI_HELP, cli_func_help, cli_proto }, { CLI_VERBOSE, m_cli_func_verbose, NULL }, - { CLI_EXIT }, + { CLI_EXIT, m_cli_func_exit, NULL}, { CLI_QUIT }, { CLI_BYE }, { NULL } From phk at projects.linpro.no Tue Jul 11 18:21:06 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 20:21:06 +0200 (CEST) Subject: r438 - trunk/varnish-cache/bin/varnishtester Message-ID: <20060711182106.E7A151EC38E@projects.linpro.no> Author: phk Date: 2006-07-11 20:21:06 +0200 (Tue, 11 Jul 2006) New Revision: 438 Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c Log: Minimal ability to start and stop a varnishd Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c =================================================================== --- trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 18:16:13 UTC (rev 437) +++ trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 18:21:06 UTC (rev 438) @@ -1,7 +1,15 @@ +#include #include +#include +#include +#include +#include +#include +#include #include "libvarnish.h" +#include "event.h" /*-------------------------------------------------------------------- @@ -18,12 +26,148 @@ /*--------------------------------------------------------------------*/ +static struct event_base *eb; +static struct bufferevent *e_cmd; +static struct bufferevent *e_pipe2; +static int pipe1[2]; +static int pipe2[2]; +static pid_t child; + +/*--------------------------------------------------------------------*/ + +static void +rd_pipe2(struct bufferevent *bev, void *arg) +{ + char *p; + + (void)arg; + p = evbuffer_readline(bev->input); + if (p == NULL) + return; + printf("V: <<%s>>\n", p); +} + +static void +ex_pipe2(struct bufferevent *bev, short what, void *arg) +{ + int i, status; + + (void)arg; + printf("%s(%p, 0x%x, %p)\n", __func__, bev, what, arg); + bufferevent_disable(e_pipe2, EV_READ); + + i = wait4(child, &status, 0, NULL); + printf("stopped i %d status 0x%x\n", i, status); + child = 0; + close(pipe1[1]); + close(pipe2[0]); +} + +/*--------------------------------------------------------------------*/ + +static void +cmd_start(char **av) +{ + + printf("%s()\n", __func__); + (void)av; + assert(pipe(pipe1) == 0); + assert(pipe(pipe2) == 0); + + e_pipe2 = bufferevent_new(pipe2[0], rd_pipe2, NULL, ex_pipe2, NULL); + assert(e_pipe2 != NULL); + bufferevent_base_set(eb, e_pipe2); + bufferevent_enable(e_pipe2, EV_READ); + + child = fork(); + if (!child) { + dup2(pipe1[0], 0); + dup2(pipe2[1], 1); + dup2(pipe2[1], 2); + close(pipe1[0]); + close(pipe1[1]); + close(pipe2[0]); + close(pipe2[1]); + write(2, "Forked\n", 7); + assert(chdir("../varnishd") == 0); + execl( + "./varnishd", + "varnishd", + "-blocalhost:8081", + "-sfile,/tmp/,10m", + NULL); + perror("execl"); + exit (2); + } + close(pipe1[0]); + close(pipe2[1]); +} + + +/*--------------------------------------------------------------------*/ + +static void +cmd_stop(char **av) +{ + + (void)av; + if (child == 0) { + fprintf(stderr, "No child running\n"); + exit (2); + } + write(pipe1[1], "exit\r", 5); + /* XXX: arm timeout */ +} + +/*--------------------------------------------------------------------*/ + +static void +rd_cmd(struct bufferevent *bev, void *arg) +{ + char *p; + char **av; + + (void)bev; + (void)arg; + p = evbuffer_readline(bev->input); + if (p == NULL) + return; + av = ParseArgv(p, 0); + if (av[0] != NULL) { + fprintf(stderr, "%s\n", av[0]); + exit (1); + } + if (av[1] == NULL) + return; + if (!strcmp(av[1], "start")) + cmd_start(av + 2); + else if (!strcmp(av[1], "stop")) + cmd_stop(av + 2); + else { + fprintf(stderr, "Unknown command \"%s\"\n", av[1]); + exit (2); + } + FreeArgv(av); +} + +/*--------------------------------------------------------------------*/ + int main(int argc, char **argv) { + setbuf(stdout, NULL); (void)argc; (void)argv; - return (0); + eb = event_init(); + assert(eb != NULL); + + e_cmd = bufferevent_new(0, rd_cmd, NULL, NULL, NULL); + assert(e_cmd != NULL); + bufferevent_base_set(eb, e_cmd); + bufferevent_enable(e_cmd, EV_READ); + + event_base_loop(eb, 0); + return (2); } From phk at projects.linpro.no Tue Jul 11 19:10:05 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 21:10:05 +0200 (CEST) Subject: r439 - trunk/varnish-cache/bin/varnishtester Message-ID: <20060711191005.2B7501EC38E@projects.linpro.no> Author: phk Date: 2006-07-11 21:10:05 +0200 (Tue, 11 Jul 2006) New Revision: 439 Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c Log: Add a server facility to act as backend for varnish Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c =================================================================== --- trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 18:21:06 UTC (rev 438) +++ trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 19:10:05 UTC (rev 439) @@ -5,11 +5,14 @@ #include #include #include +#include #include #include +#include #include "libvarnish.h" #include "event.h" +#include "queue.h" /*-------------------------------------------------------------------- @@ -27,7 +30,154 @@ /*--------------------------------------------------------------------*/ static struct event_base *eb; -static struct bufferevent *e_cmd; + +/*--------------------------------------------------------------------*/ + +static int serv_sock = -1; +static struct event e_acc; +static int sock_acc = -1; +static struct bufferevent *e_racc; + +struct serv { + TAILQ_ENTRY(serv) list; + char *data; +}; + +static TAILQ_HEAD(,serv) serv_head = TAILQ_HEAD_INITIALIZER(serv_head); + +static void +rd_acc(struct bufferevent *bev, void *arg) +{ + char *p; + struct serv *sp; + int *ip; + + ip = arg; + while (1) { + p = evbuffer_readline(bev->input); + if (p == NULL) + return; + printf("A: <<%s>>\n", p); + if (*p == '\0') { + sp = TAILQ_FIRST(&serv_head); + assert(sp != NULL); + write(*ip, sp->data, strlen(sp->data)); + if (TAILQ_NEXT(sp, list) != NULL) { + TAILQ_REMOVE(&serv_head, sp, list); + free(sp->data); + free(sp); + } + } + } +} + +static void +ex_acc(struct bufferevent *bev, short what, void *arg) +{ + int *ip; + + ip = arg; + printf("%s(%p, 0x%x, %p)\n", __func__, bev, what, arg); + bufferevent_disable(bev, EV_READ); + bufferevent_free(bev); + close(*ip); + free(ip); +} + +static void +acc_sock(int fd, short event, void *arg) +{ + struct sockaddr addr[2]; /* XXX: IPv6 hack */ + socklen_t l; + struct linger linger; + int *ip; + + ip = calloc(sizeof *ip, 1); + (void)event; + (void)arg; + l = sizeof addr; + fd = accept(fd, addr, &l); + if (fd < 0) { + perror("accept"); + exit (2); + } +#ifdef SO_LINGER /* XXX Linux ? */ + linger.l_onoff = 0; + linger.l_linger = 0; + assert(setsockopt(fd, SOL_SOCKET, SO_LINGER, + &linger, sizeof linger) == 0); +#endif + *ip = fd; + e_racc = bufferevent_new(fd, rd_acc, NULL, ex_acc, ip); + assert(e_racc != NULL); + bufferevent_base_set(eb, e_racc); + bufferevent_enable(e_racc, EV_READ); +} + +static void +open_serv_sock(void) +{ + struct addrinfo ai, *r0, *r1; + int i, j, s = -1; + + memset(&ai, 0, sizeof ai); + ai.ai_family = PF_UNSPEC; + ai.ai_socktype = SOCK_STREAM; + ai.ai_flags = AI_PASSIVE; + i = getaddrinfo("localhost", "8081", &ai, &r0); + + if (i) { + fprintf(stderr, "getaddrinfo failed: %s\n", gai_strerror(i)); + return; + } + + for (r1 = r0; r1 != NULL; r1 = r1->ai_next) { + s = socket(r1->ai_family, r1->ai_socktype, r1->ai_protocol); + if (s < 0) + continue; + j = 1; + i = setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &j, sizeof j); + assert(i == 0); + + i = bind(s, r1->ai_addr, r1->ai_addrlen); + if (i != 0) { + perror("bind"); + continue; + } + assert(i == 0); + serv_sock = s; + break; + } + freeaddrinfo(r0); + if (s < 0) { + perror("bind"); + exit (2); + } + + listen(s, 16); + + event_set(&e_acc, s, EV_READ | EV_PERSIST, acc_sock, NULL); + event_base_set(eb, &e_acc); + event_add(&e_acc, NULL); +} + +static void +cmd_serve(char **av) +{ + struct serv *sp; + int i; + + for (i = 0; av[i] != NULL; i++) { + sp = calloc(sizeof *sp, 1); + assert(sp != NULL); + sp->data = strdup(av[i]); + assert(sp->data != NULL); + TAILQ_INSERT_TAIL(&serv_head, sp, list); + } +} + +/*--------------------------------------------------------------------*/ + static struct bufferevent *e_pipe2; static int pipe1[2]; static int pipe2[2]; @@ -41,10 +191,12 @@ char *p; (void)arg; - p = evbuffer_readline(bev->input); - if (p == NULL) - return; - printf("V: <<%s>>\n", p); + while (1) { + p = evbuffer_readline(bev->input); + if (p == NULL) + return; + printf("V: <<%s>>\n", p); + } } static void @@ -143,6 +295,8 @@ cmd_start(av + 2); else if (!strcmp(av[1], "stop")) cmd_stop(av + 2); + else if (!strcmp(av[1], "serve")) + cmd_serve(av + 2); else { fprintf(stderr, "Unknown command \"%s\"\n", av[1]); exit (2); @@ -152,6 +306,8 @@ /*--------------------------------------------------------------------*/ +static struct bufferevent *e_cmd; + int main(int argc, char **argv) { @@ -163,6 +319,8 @@ eb = event_init(); assert(eb != NULL); + open_serv_sock(); + e_cmd = bufferevent_new(0, rd_cmd, NULL, NULL, NULL); assert(e_cmd != NULL); bufferevent_base_set(eb, e_cmd); From phk at projects.linpro.no Tue Jul 11 19:16:42 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 21:16:42 +0200 (CEST) Subject: r440 - trunk/varnish-cache/bin/varnishtester Message-ID: <20060711191642.EEE5B1EC38E@projects.linpro.no> Author: phk Date: 2006-07-11 21:16:42 +0200 (Tue, 11 Jul 2006) New Revision: 440 Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c Log: If the first char of the serve string is '!', close connection after sending string. Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c =================================================================== --- trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 19:10:05 UTC (rev 439) +++ trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 19:16:42 UTC (rev 440) @@ -41,6 +41,7 @@ struct serv { TAILQ_ENTRY(serv) list; char *data; + int close; }; static TAILQ_HEAD(,serv) serv_head = TAILQ_HEAD_INITIALIZER(serv_head); @@ -57,11 +58,13 @@ p = evbuffer_readline(bev->input); if (p == NULL) return; - printf("A: <<%s>>\n", p); + printf("B: <<%s>>\n", p); if (*p == '\0') { sp = TAILQ_FIRST(&serv_head); assert(sp != NULL); write(*ip, sp->data, strlen(sp->data)); + if (sp->close) + shutdown(*ip, SHUT_WR); if (TAILQ_NEXT(sp, list) != NULL) { TAILQ_REMOVE(&serv_head, sp, list); free(sp->data); @@ -76,8 +79,8 @@ { int *ip; + (void)what; ip = arg; - printf("%s(%p, 0x%x, %p)\n", __func__, bev, what, arg); bufferevent_disable(bev, EV_READ); bufferevent_free(bev); close(*ip); @@ -170,7 +173,12 @@ for (i = 0; av[i] != NULL; i++) { sp = calloc(sizeof *sp, 1); assert(sp != NULL); - sp->data = strdup(av[i]); + if (av[i][0] == '!') { + sp->close = 1; + sp->data = strdup(av[i] + 1); + } else { + sp->data = strdup(av[i]); + } assert(sp->data != NULL); TAILQ_INSERT_TAIL(&serv_head, sp, list); } From phk at projects.linpro.no Tue Jul 11 19:29:16 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 21:29:16 +0200 (CEST) Subject: r441 - trunk/varnish-cache/bin/varnishtester Message-ID: <20060711192916.36A371EC38E@projects.linpro.no> Author: phk Date: 2006-07-11 21:29:16 +0200 (Tue, 11 Jul 2006) New Revision: 441 Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c Log: Add "cli" to tell varnishd things Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c =================================================================== --- trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 19:16:42 UTC (rev 440) +++ trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 19:29:16 UTC (rev 441) @@ -282,6 +282,20 @@ /*--------------------------------------------------------------------*/ static void +cmd_cli(char **av) +{ + + if (child == 0) { + fprintf(stderr, "No child running\n"); + exit (2); + } + write(pipe1[1], av[0], strlen(av[0])); + write(pipe1[1], "\n", 1); +} + +/*--------------------------------------------------------------------*/ + +static void rd_cmd(struct bufferevent *bev, void *arg) { char *p; @@ -305,6 +319,8 @@ cmd_stop(av + 2); else if (!strcmp(av[1], "serve")) cmd_serve(av + 2); + else if (!strcmp(av[1], "cli")) + cmd_cli(av + 2); else { fprintf(stderr, "Unknown command \"%s\"\n", av[1]); exit (2); From phk at projects.linpro.no Tue Jul 11 20:37:43 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 22:37:43 +0200 (CEST) Subject: r442 - trunk/varnish-cache/bin/varnishtester Message-ID: <20060711203743.E04E21EC395@projects.linpro.no> Author: phk Date: 2006-07-11 22:37:43 +0200 (Tue, 11 Jul 2006) New Revision: 442 Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c Log: A vcl keyword for loading a new config Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c =================================================================== --- trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 19:29:16 UTC (rev 441) +++ trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 20:37:43 UTC (rev 442) @@ -194,6 +194,15 @@ /*--------------------------------------------------------------------*/ static void +cli_write(const char *s) +{ + + write(pipe1[1], s, strlen(s)); +} + +/*--------------------------------------------------------------------*/ + +static void rd_pipe2(struct bufferevent *bev, void *arg) { char *p; @@ -275,7 +284,7 @@ fprintf(stderr, "No child running\n"); exit (2); } - write(pipe1[1], "exit\r", 5); + cli_write("exit\n"); /* XXX: arm timeout */ } @@ -289,13 +298,42 @@ fprintf(stderr, "No child running\n"); exit (2); } - write(pipe1[1], av[0], strlen(av[0])); - write(pipe1[1], "\n", 1); + cli_write(av[0]); + cli_write("\n"); } /*--------------------------------------------------------------------*/ static void +cmd_vcl(char **av) +{ + char *p, buf[5]; + + if (child == 0) { + fprintf(stderr, "No child running\n"); + exit (2); + } + if (av[0] == NULL || av[1] == NULL) { + fprintf(stderr, "usage: vcl $name $vcl\n"); + exit (2); + } + cli_write("config.inline "); + cli_write(av[0]); + cli_write(" \""); + for (p = av[1]; *p; p++) { + if (*p < ' ' || *p == '"' || *p == '\\' || *p > '~') { + sprintf(buf, "\\%03o", *p); + cli_write(buf); + } else { + write(pipe1[1], p, 1); + } + } + cli_write("\"\n"); +} + +/*--------------------------------------------------------------------*/ + +static void rd_cmd(struct bufferevent *bev, void *arg) { char *p; @@ -321,6 +359,8 @@ cmd_serve(av + 2); else if (!strcmp(av[1], "cli")) cmd_cli(av + 2); + else if (!strcmp(av[1], "vcl")) + cmd_vcl(av + 2); else { fprintf(stderr, "Unknown command \"%s\"\n", av[1]); exit (2); From phk at projects.linpro.no Tue Jul 11 20:49:18 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 22:49:18 +0200 (CEST) Subject: r443 - trunk/varnish-cache/bin/varnishtester Message-ID: <20060711204918.DAE371EC38E@projects.linpro.no> Author: phk Date: 2006-07-11 22:49:18 +0200 (Tue, 11 Jul 2006) New Revision: 443 Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c Log: open and close commands Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c =================================================================== --- trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 20:37:43 UTC (rev 442) +++ trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 20:49:18 UTC (rev 443) @@ -35,7 +35,6 @@ static int serv_sock = -1; static struct event e_acc; -static int sock_acc = -1; static struct bufferevent *e_racc; struct serv { @@ -333,7 +332,64 @@ /*--------------------------------------------------------------------*/ +static int req_sock = -1; + static void +cmd_open(char **av) +{ + struct addrinfo ai, *r0, *r1; + int i, j, s = -1; + + (void)av; + memset(&ai, 0, sizeof ai); + ai.ai_family = PF_UNSPEC; + ai.ai_socktype = SOCK_STREAM; + ai.ai_flags = AI_PASSIVE; + i = getaddrinfo("localhost", "8080", &ai, &r0); + + if (i) { + fprintf(stderr, "getaddrinfo failed: %s\n", gai_strerror(i)); + return; + } + + for (r1 = r0; r1 != NULL; r1 = r1->ai_next) { + s = socket(r1->ai_family, r1->ai_socktype, r1->ai_protocol); + if (s < 0) + continue; + j = 1; + i = setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &j, sizeof j); + assert(i == 0); + + i = connect(s, r1->ai_addr, r1->ai_addrlen); + if (i) { + perror("connect"); + close(s); + s = -1; + continue; + } + assert(i == 0); + req_sock = s; + break; + } + freeaddrinfo(r0); + if (s < 0) { + perror("connect"); + exit (2); + } +} + +static void +cmd_close(char **av) +{ + + (void)av; + close(req_sock); + req_sock = -1; +} + +/*--------------------------------------------------------------------*/ + +static void rd_cmd(struct bufferevent *bev, void *arg) { char *p; @@ -361,6 +417,10 @@ cmd_cli(av + 2); else if (!strcmp(av[1], "vcl")) cmd_vcl(av + 2); + else if (!strcmp(av[1], "open")) + cmd_open(av + 2); + else if (!strcmp(av[1], "close")) + cmd_close(av + 2); else { fprintf(stderr, "Unknown command \"%s\"\n", av[1]); exit (2); From phk at projects.linpro.no Tue Jul 11 21:01:07 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 23:01:07 +0200 (CEST) Subject: r444 - trunk/varnish-cache/bin/varnishd Message-ID: <20060711210107.594CD1EC382@projects.linpro.no> Author: phk Date: 2006-07-11 23:01:07 +0200 (Tue, 11 Jul 2006) New Revision: 444 Modified: trunk/varnish-cache/bin/varnishd/cache_main.c Log: Add a printf when cached is ready for the benefit of varnishtester Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-11 20:49:18 UTC (rev 443) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-11 21:01:07 UTC (rev 444) @@ -129,6 +129,7 @@ event_base_set(eb, &ev_keepalive); arm_keepalive(); + printf("Ready\n"); i = event_base_loop(eb, 0); if (i != 0) printf("event_dispatch() = %d\n", i); From phk at projects.linpro.no Tue Jul 11 21:04:45 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 23:04:45 +0200 (CEST) Subject: r445 - trunk/varnish-cache/bin/varnishtester Message-ID: <20060711210445.F0FEC1EC381@projects.linpro.no> Author: phk Date: 2006-07-11 23:04:45 +0200 (Tue, 11 Jul 2006) New Revision: 445 Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c Log: Add Pause() and Resume() to pace script execution Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c =================================================================== --- trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 21:01:07 UTC (rev 444) +++ trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 21:04:45 UTC (rev 445) @@ -31,6 +31,10 @@ static struct event_base *eb; + +static void Pause(void); +static void Resume(void); + /*--------------------------------------------------------------------*/ static int serv_sock = -1; @@ -212,6 +216,8 @@ if (p == NULL) return; printf("V: <<%s>>\n", p); + if (!strcmp(p, "Child said ")) + Resume(); } } @@ -237,7 +243,6 @@ cmd_start(char **av) { - printf("%s()\n", __func__); (void)av; assert(pipe(pipe1) == 0); assert(pipe(pipe2) == 0); @@ -269,6 +274,7 @@ } close(pipe1[0]); close(pipe2[1]); + Pause(); } @@ -389,49 +395,69 @@ /*--------------------------------------------------------------------*/ +static struct bufferevent *e_cmd; + static void rd_cmd(struct bufferevent *bev, void *arg) { char *p; char **av; + int run = 1; (void)bev; (void)arg; - p = evbuffer_readline(bev->input); - if (p == NULL) - return; - av = ParseArgv(p, 0); - if (av[0] != NULL) { - fprintf(stderr, "%s\n", av[0]); - exit (1); + while (run) { + p = evbuffer_readline(bev->input); + if (p == NULL) + return; + printf("]: <<%s>>\n", p); + av = ParseArgv(p, 0); + if (av[0] != NULL) { + fprintf(stderr, "%s\n", av[0]); + exit (1); + } + if (av[1] == NULL) + return; + if (!strcmp(av[1], "start")) { + cmd_start(av + 2); + run = 0; + } else if (!strcmp(av[1], "stop")) + cmd_stop(av + 2); + else if (!strcmp(av[1], "serve")) + cmd_serve(av + 2); + else if (!strcmp(av[1], "cli")) + cmd_cli(av + 2); + else if (!strcmp(av[1], "vcl")) + cmd_vcl(av + 2); + else if (!strcmp(av[1], "open")) + cmd_open(av + 2); + else if (!strcmp(av[1], "close")) + cmd_close(av + 2); + else { + fprintf(stderr, "Unknown command \"%s\"\n", av[1]); + exit (2); + } + FreeArgv(av); } - if (av[1] == NULL) - return; - if (!strcmp(av[1], "start")) - cmd_start(av + 2); - else if (!strcmp(av[1], "stop")) - cmd_stop(av + 2); - else if (!strcmp(av[1], "serve")) - cmd_serve(av + 2); - else if (!strcmp(av[1], "cli")) - cmd_cli(av + 2); - else if (!strcmp(av[1], "vcl")) - cmd_vcl(av + 2); - else if (!strcmp(av[1], "open")) - cmd_open(av + 2); - else if (!strcmp(av[1], "close")) - cmd_close(av + 2); - else { - fprintf(stderr, "Unknown command \"%s\"\n", av[1]); - exit (2); - } - FreeArgv(av); } +static void +Pause() +{ + printf("X: Pause\n"); + bufferevent_disable(e_cmd, EV_READ); +} + +static void +Resume() +{ + printf("X: Resume\n"); + bufferevent_enable(e_cmd, EV_READ); + rd_cmd(e_cmd, NULL); +} + /*--------------------------------------------------------------------*/ -static struct bufferevent *e_cmd; - int main(int argc, char **argv) { From phk at projects.linpro.no Tue Jul 11 21:30:38 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 23:30:38 +0200 (CEST) Subject: r446 - trunk/varnish-cache/bin/varnishtester Message-ID: <20060711213038.935091EC38E@projects.linpro.no> Author: phk Date: 2006-07-11 23:30:38 +0200 (Tue, 11 Jul 2006) New Revision: 446 Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c Log: add a req command, various adjustments Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c =================================================================== --- trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 21:04:45 UTC (rev 445) +++ trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 21:30:38 UTC (rev 446) @@ -339,8 +339,47 @@ /*--------------------------------------------------------------------*/ static int req_sock = -1; +static struct bufferevent *e_req; static void +req_write(const char *s) +{ + + write(req_sock, s, strlen(s)); +} + +/*--------------------------------------------------------------------*/ +static void +rd_req(struct bufferevent *bev, void *arg) +{ + char *p; + + (void)arg; + while (1) { + p = evbuffer_readline(bev->input); + if (p == NULL) + return; + printf("R: <<%s>>\n", p); + } +} + +static void +ex_req(struct bufferevent *bev, short what, void *arg) +{ + + (void)arg; + printf("%s(%p, 0x%x, %p)\n", __func__, bev, what, arg); + bufferevent_disable(e_req, EV_READ); + bufferevent_free(e_req); + e_req = NULL; + close(req_sock); + req_sock = -1; + Resume(); +} + + +/*--------------------------------------------------------------------*/ +static void cmd_open(char **av) { struct addrinfo ai, *r0, *r1; @@ -382,27 +421,54 @@ perror("connect"); exit (2); } + e_req = bufferevent_new(s, rd_req, NULL, ex_req, NULL); + assert(e_req != NULL); + bufferevent_base_set(eb, e_req); + bufferevent_enable(e_req, EV_READ); } static void cmd_close(char **av) { + if (req_sock == -1) + return; (void)av; + bufferevent_disable(e_req, EV_READ); + bufferevent_free(e_req); + e_req = NULL; close(req_sock); req_sock = -1; } /*--------------------------------------------------------------------*/ +static void +cmd_req(char **av) +{ + char *p = av[0]; + + if (req_sock == -1) + cmd_open(av); + if (*p == '!') { + req_write(p + 1); + shutdown(req_sock, SHUT_WR); + } else { + req_write(p); + } + Pause(); +} + +/*--------------------------------------------------------------------*/ + static struct bufferevent *e_cmd; +static int run = 1; static void rd_cmd(struct bufferevent *bev, void *arg) { char *p; char **av; - int run = 1; (void)bev; (void)arg; @@ -418,10 +484,9 @@ } if (av[1] == NULL) return; - if (!strcmp(av[1], "start")) { + if (!strcmp(av[1], "start")) cmd_start(av + 2); - run = 0; - } else if (!strcmp(av[1], "stop")) + else if (!strcmp(av[1], "stop")) cmd_stop(av + 2); else if (!strcmp(av[1], "serve")) cmd_serve(av + 2); @@ -433,6 +498,8 @@ cmd_open(av + 2); else if (!strcmp(av[1], "close")) cmd_close(av + 2); + else if (!strcmp(av[1], "req")) + cmd_req(av + 2); else { fprintf(stderr, "Unknown command \"%s\"\n", av[1]); exit (2); @@ -442,16 +509,34 @@ } static void +ex_cmd(struct bufferevent *bev, short what, void *arg) +{ + + (void)arg; + printf("%s(%p, 0x%x, %p)\n", __func__, bev, what, arg); + bufferevent_disable(e_cmd, EV_READ); + bufferevent_free(e_cmd); + e_cmd = NULL; + cmd_close(NULL); + cmd_stop(NULL); + exit(0); +} + +static void Pause() { + assert(run == 1); printf("X: Pause\n"); + run = 0; bufferevent_disable(e_cmd, EV_READ); } static void Resume() { + assert(run == 0); printf("X: Resume\n"); + run = 1; bufferevent_enable(e_cmd, EV_READ); rd_cmd(e_cmd, NULL); } @@ -471,7 +556,7 @@ open_serv_sock(); - e_cmd = bufferevent_new(0, rd_cmd, NULL, NULL, NULL); + e_cmd = bufferevent_new(0, rd_cmd, NULL, ex_cmd, NULL); assert(e_cmd != NULL); bufferevent_base_set(eb, e_cmd); bufferevent_enable(e_cmd, EV_READ); From phk at projects.linpro.no Tue Jul 11 21:35:31 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 11 Jul 2006 23:35:31 +0200 (CEST) Subject: r447 - trunk/varnish-cache/bin/varnishtester Message-ID: <20060711213531.27B441EC38E@projects.linpro.no> Author: phk Date: 2006-07-11 23:35:31 +0200 (Tue, 11 Jul 2006) New Revision: 447 Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c Log: EOF detection in libevent is buggy ?? Add exit cmd. Pause after cli until we see "OK" Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c =================================================================== --- trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 21:30:38 UTC (rev 446) +++ trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 21:35:31 UTC (rev 447) @@ -218,6 +218,8 @@ printf("V: <<%s>>\n", p); if (!strcmp(p, "Child said ")) Resume(); + else if (!strcmp(p, "OK")) + Resume(); } } @@ -305,6 +307,7 @@ } cli_write(av[0]); cli_write("\n"); + Pause(); } /*--------------------------------------------------------------------*/ @@ -461,6 +464,18 @@ /*--------------------------------------------------------------------*/ +static void +cmd_exit(char **av) +{ + + (void)av; + cmd_close(NULL); + cmd_stop(NULL); + exit (0); +} + +/*--------------------------------------------------------------------*/ + static struct bufferevent *e_cmd; static int run = 1; @@ -500,6 +515,8 @@ cmd_close(av + 2); else if (!strcmp(av[1], "req")) cmd_req(av + 2); + else if (!strcmp(av[1], "exit")) + cmd_exit(av + 2); else { fprintf(stderr, "Unknown command \"%s\"\n", av[1]); exit (2); From des at linpro.no Tue Jul 11 21:42:33 2006 From: des at linpro.no (Dag-Erling =?iso-8859-1?Q?Sm=F8rgrav?=) Date: Tue, 11 Jul 2006 23:42:33 +0200 Subject: r428 - trunk/varnish-cache/bin/varnishd References: <20060711151854.B96E61EC382@projects.linpro.no> Message-ID: phk at projects.linpro.no writes: > A dot(1) graph is embedded in the source code and can be extracted > with: > > sed -n '/^DOT/s///p' cache_center.c | dot -Tps > /tmp/_.ps I think I just went blind... :) DES -- Dag-Erling Sm?rgrav Senior Software Developer Linpro AS - www.linpro.no From phk at projects.linpro.no Wed Jul 12 07:45:59 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 12 Jul 2006 09:45:59 +0200 (CEST) Subject: r448 - trunk/varnish-cache/bin/varnishtester Message-ID: <20060712074559.772181EC3A0@projects.linpro.no> Author: phk Date: 2006-07-12 09:45:59 +0200 (Wed, 12 Jul 2006) New Revision: 448 Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c Log: Pause after 'vcl' command Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c =================================================================== --- trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-11 21:35:31 UTC (rev 447) +++ trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-12 07:45:59 UTC (rev 448) @@ -337,6 +337,7 @@ } } cli_write("\"\n"); + Pause(); } /*--------------------------------------------------------------------*/ From phk at projects.linpro.no Wed Jul 12 08:34:48 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 12 Jul 2006 10:34:48 +0200 (CEST) Subject: r449 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060712083448.B45E51EC380@projects.linpro.no> Author: phk Date: 2006-07-12 10:34:48 +0200 (Wed, 12 Jul 2006) New Revision: 449 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/include/libvcl.h trunk/varnish-cache/lib/libvcl/vcl_compile.c Log: Teach the VCL compiler about default functions, so that users will not have to copy&paste the default methods if they have no special requirements for a particular method. No such facility exists for backends, so a backend description is now the minumum VCL program. When we initialize the VCL compiler we hand it a piece of source code with the "default code", this must include definitions of all methods named with a "default_" prefix (ie: "default_vcl_recv" etc). During compilation we always compile this piece of source code in (after the user supplied VCL source). If the user did not provide a particular method, the default method is used instead. The user can also call the default method directly, for instance by: sub vcl_recv { if (req.http.Expect) { error; } call default_vcl_recv; } Later on, this could be expanded to allow other subroutines to be included in the default VCL for the users calling convenience. Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-12 07:45:59 UTC (rev 448) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-12 08:34:48 UTC (rev 449) @@ -33,6 +33,46 @@ /*--------------------------------------------------------------------*/ +static const char *default_vcl = + "sub default_vcl_recv {\n" + " if (req.request != \"GET\" && req.request != \"HEAD\") {\n" + " pipe;\n" + " }\n" + " if (req.http.Expect) {\n" + " pipe;\n" + " }\n" + " if (req.http.Authenticate || req.http.Cookie) {\n" + " pass;\n" + " }\n" + " lookup;\n" + "}\n" + "\n" + "sub default_vcl_hit {\n" + " if (!obj.cacheable) {\n" + " pass;\n" + " }\n" + " deliver;\n" + "}\n" + "\n" + "sub default_vcl_miss {\n" + " fetch;\n" + "}\n" + "\n" + "sub default_vcl_fetch {\n" + " if (!obj.valid) {\n" + " error;\n" + " }\n" + " if (!obj.cacheable) {\n" + " pass;\n" + " }\n" + " insert;\n" + "}\n" + "sub default_vcl_timeout {\n" + " discard;\n" + "}\n"; + +/*--------------------------------------------------------------------*/ + struct heritage heritage; struct event_base *mgt_eb; @@ -90,44 +130,7 @@ "backend default {\n" " set backend.host = \"%*.*s\";\n" " set backend.port = \"%s\";\n" - "}\n" - "sub vcl_recv {\n" - " if (req.request != \"GET\" && req.request != \"HEAD\") {\n" - " pipe;\n" - " }\n" - " if (req.http.Expect) {\n" - " pipe;\n" - " }\n" - " if (req.http.Authenticate || req.http.Cookie) {\n" - " pass;\n" - " }\n" - " lookup;\n" - "}\n" - "\n" - "sub vcl_hit {\n" - " if (!obj.cacheable) {\n" - " pass;\n" - " }\n" - " deliver;\n" - "}\n" - "\n" - "sub vcl_miss {\n" - " fetch;\n" - "}\n" - "\n" - "sub vcl_fetch {\n" - " if (!obj.valid) {\n" - " error;\n" - " }\n" - " if (!obj.cacheable) {\n" - " pass;\n" - " }\n" - " insert;\n" - "}\n" - "sub vcl_timeout {\n" - " discard;\n" - "}\n" - "", p - bflag, p - bflag, bflag, q); + "}\n", p - bflag, p - bflag, bflag, q); assert(buf != NULL); sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(sb != NULL); @@ -243,6 +246,9 @@ m_cli_func_exit(struct cli *cli, char **av, void *priv) { + (void)cli; + (void)av; + (void)priv; mgt_child_kill(); exit (0); } @@ -503,7 +509,7 @@ setbuf(stdout, NULL); setbuf(stderr, NULL); - VCC_InitCompile(); + VCC_InitCompile(default_vcl); heritage.default_ttl = 120; heritage.wthread_min = 1; Modified: trunk/varnish-cache/include/libvcl.h =================================================================== --- trunk/varnish-cache/include/libvcl.h 2006-07-12 07:45:59 UTC (rev 448) +++ trunk/varnish-cache/include/libvcl.h 2006-07-12 08:34:48 UTC (rev 449) @@ -4,6 +4,6 @@ char *VCC_Compile(struct sbuf *sb, const char *b, const char *e); char *VCC_CompileFile(struct sbuf *sb, const char *fn); -void VCC_InitCompile(void); +void VCC_InitCompile(const char *default_vcl); Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-07-12 07:45:59 UTC (rev 448) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-07-12 08:34:48 UTC (rev 449) @@ -128,10 +128,11 @@ static struct method { const char *name; + const char *defname; unsigned returns; } method_tab[] = { #define VCL_RET_MAC(a,b,c) -#define VCL_MET_MAC(a,b,c) { "vcl_"#a, c }, +#define VCL_MET_MAC(a,b,c) { "vcl_"#a, "default_vcl_"#a, c }, #include "vcl_returns.h" #undef VCL_MET_MAC #undef VCL_RET_MAC @@ -195,6 +196,7 @@ static void Cond_0(struct tokenlist *tl); static struct proc *AddProc(struct tokenlist *tl, struct token *t, int def); static void AddCall(struct tokenlist *tl, struct token *t); +static const char *vcc_default_vcl_b, *vcc_default_vcl_e; /*--------------------------------------------------------------------*/ @@ -223,13 +225,22 @@ ErrWhere(struct tokenlist *tl, struct token *t) { unsigned lin, pos, x, y; - const char *p, *l; + const char *p, *l, *f, *b, *e; lin = 1; pos = 0; if (t->tok == METHOD) return; - for (l = p = tl->b; p < t->b; p++) { + if (t->b >= vcc_default_vcl_b && t->b < vcc_default_vcl_e) { + f = "Default VCL code (compiled in)"; + b = vcc_default_vcl_b; + e = vcc_default_vcl_e; + } else { + f = "VCL code"; + b = tl->b; + e = tl->e; + } + for (l = p = b; p < t->b; p++) { if (*p == '\n') { lin++; pos = 0; @@ -240,9 +251,9 @@ } else pos++; } - sbuf_printf(tl->sb, "Line %d Pos %d\n", lin, pos); + sbuf_printf(tl->sb, "In %s Line %d Pos %d\n", f, lin, pos); x = y = 0; - for (p = l; p < tl->e && *p != '\n'; p++) { + for (p = l; p < e && *p != '\n'; p++) { if (*p == '\t') { y &= ~7; y += 8; @@ -258,7 +269,7 @@ } sbuf_cat(tl->sb, "\n"); x = y = 0; - for (p = l; p < tl->e && *p != '\n'; p++) { + for (p = l; p < e && *p != '\n'; p++) { if (p >= t->b && p < t->e) { sbuf_bcat(tl->sb, "#", 1); x++; @@ -452,6 +463,20 @@ return (r); } +static int +FindRefStr(struct tokenlist *tl, const char *s, enum ref_type type) +{ + struct ref *r; + + TAILQ_FOREACH(r, &tl->refs, list) { + if (r->type != type) + continue; + if (IdIs(r->name, s)) + return (1); + } + return (0); +} + static void AddRef(struct tokenlist *tl, struct token *t, enum ref_type type) { @@ -1381,8 +1406,13 @@ t->e = e; TAILQ_INSERT_TAIL(&tl->tokens, t, list); tl->t = t; - if (0) - fprintf(stderr, "+ %s\n", vcl_tnames[tok]); + if (0) { + fprintf(stderr, "[%s %*.*s] ", + vcl_tnames[tok], + e - b, e - b, b); + if (tok == EOI) + fprintf(stderr, "\n"); + } } /*-------------------------------------------------------------------- @@ -1478,8 +1508,6 @@ ErrWhere(tl, tl->t); return; } - /* Add End Of Input token */ - AddToken(tl, EOI, p, p); } /*-------------------------------------------------------------------- @@ -1575,6 +1603,8 @@ TAILQ_FOREACH(p, &tl->procs, list) { for(m = method_tab; m->name != NULL; m++) { + if (IdIs(p->name, m->defname)) + p->called = 1; if (IdIs(p->name, m->name)) break; } @@ -1722,8 +1752,13 @@ Fc(tl, 0, "\t.nref = VGC_NREFS,\n"); #define VCL_RET_MAC(l,u,b) #define VCL_MET_MAC(l,u,b) \ - Fc(tl, 0, "\t." #l "_func = VGC_function_vcl_" #l ",\n"); \ - AddRefStr(tl, "vcl_" #l, R_FUNC); + if (FindRefStr(tl, "vcl_" #l, R_FUNC)) { \ + Fc(tl, 0, "\t." #l "_func = VGC_function_vcl_" #l ",\n"); \ + AddRefStr(tl, "vcl_" #l, R_FUNC); \ + } else { \ + Fc(tl, 0, "\t." #l "_func = VGC_function_default_vcl_" #l ",\n"); \ + } \ + AddRefStr(tl, "default_vcl_" #l, R_FUNC); #include "vcl_returns.h" #undef VCL_MET_MAC #undef VCL_RET_MAC @@ -1762,6 +1797,8 @@ assert(e != NULL); tokens.e = e; Lexer(&tokens, b, e); + Lexer(&tokens, vcc_default_vcl_b, vcc_default_vcl_e); + AddToken(&tokens, EOI, e, e); if (tokens.err) goto done; tokens.t = TAILQ_FIRST(&tokens.tokens); @@ -1870,10 +1907,14 @@ /*--------------------------------------------------------------------*/ void -VCC_InitCompile(void) +VCC_InitCompile(const char *default_vcl) { struct var *v; + vcc_default_vcl_b = default_vcl; + vcc_default_vcl_e = strchr(default_vcl, '\0'); + assert(vcc_default_vcl_e != NULL); + register_printf_function ('T', VCC_T_render, VCC_T_arginfo); vcl_init_tnames(); for (v = vars; v->name != NULL; v++) From phk at projects.linpro.no Wed Jul 12 08:44:57 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 12 Jul 2006 10:44:57 +0200 (CEST) Subject: r450 - trunk/varnish-cache/bin/varnishtester Message-ID: <20060712084457.11AD01EC380@projects.linpro.no> Author: phk Date: 2006-07-12 10:44:56 +0200 (Wed, 12 Jul 2006) New Revision: 450 Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c Log: Always insert a backend when vcl is compiled. Respect '#' comments in script file. Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c =================================================================== --- trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-12 08:34:48 UTC (rev 449) +++ trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-12 08:44:56 UTC (rev 450) @@ -328,6 +328,15 @@ cli_write("config.inline "); cli_write(av[0]); cli_write(" \""); + + /* Always insert our magic backend first */ + + cli_write( + "backend default {\\n" + " set backend.host = \\\"localhost\\\";\\n" + " set backend.port = \\\"8081\\\";\\n" + "}\\n"); + for (p = av[1]; *p; p++) { if (*p < ' ' || *p == '"' || *p == '\\' || *p > '~') { sprintf(buf, "\\%03o", *p); @@ -493,13 +502,13 @@ if (p == NULL) return; printf("]: <<%s>>\n", p); - av = ParseArgv(p, 0); + av = ParseArgv(p, 1); if (av[0] != NULL) { fprintf(stderr, "%s\n", av[0]); exit (1); } if (av[1] == NULL) - return; + continue; if (!strcmp(av[1], "start")) cmd_start(av + 2); else if (!strcmp(av[1], "stop")) From phk at projects.linpro.no Wed Jul 12 08:56:09 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 12 Jul 2006 10:56:09 +0200 (CEST) Subject: r451 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712085609.CDACB1EC382@projects.linpro.no> Author: phk Date: 2006-07-12 10:56:09 +0200 (Wed, 12 Jul 2006) New Revision: 451 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_main.c trunk/varnish-cache/bin/varnishd/cache_vcl.c Log: Fix CLI "config.load" Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-12 08:44:56 UTC (rev 450) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-12 08:56:09 UTC (rev 451) @@ -287,7 +287,7 @@ /* cache_vcl.c */ void RelVCL(struct VCL_conf *vc); struct VCL_conf *GetVCL(void); -int CVCL_Load(const char *fn, const char *name); +int CVCL_Load(const char *fn, const char *name, struct cli *cli); #define VCL_RET_MAC(l,u,b) #define VCL_MET_MAC(l,u,b) void VCL_##l##_method(struct sess *); Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-12 08:44:56 UTC (rev 450) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-12 08:56:09 UTC (rev 451) @@ -105,7 +105,7 @@ setbuf(stderr, NULL); printf("Child starts\n"); - CVCL_Load(heritage.vcl_file, "boot"); + CVCL_Load(heritage.vcl_file, "boot", NULL); AZ(pthread_mutex_init(&sessmtx, NULL)); VBE_Init(); VSL_Init(); Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-07-12 08:44:56 UTC (rev 450) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-07-12 08:56:09 UTC (rev 451) @@ -58,30 +58,59 @@ /*--------------------------------------------------------------------*/ +static struct vcls * +find_vcls(const char *name) +{ + struct vcls *vcl; + + TAILQ_FOREACH(vcl, &vcl_head, list) + if (!strcmp(vcl->name, name)) + return (vcl); + return (NULL); +} + int -CVCL_Load(const char *fn, const char *name) +CVCL_Load(const char *fn, const char *name, struct cli *cli) { struct vcls *vcl; + vcl = find_vcls(name); + if (vcl != NULL) { + if (cli == NULL) + fprintf(stderr, "Config '%s' already loaded", name); + else + cli_out(cli, "Config '%s' already loaded", name); + return (1); + } + vcl = calloc(sizeof *vcl, 1); assert(vcl != NULL); vcl->dlh = dlopen(fn, RTLD_NOW | RTLD_LOCAL); unlink(fn); if (vcl->dlh == NULL) { - fprintf(stderr, "dlopen(%s): %s\n", fn, dlerror()); + if (cli == NULL) + fprintf(stderr, "dlopen(%s): %s\n", fn, dlerror()); + else + cli_out(cli, "dlopen(%s): %s\n", fn, dlerror()); free(vcl); return (1); } vcl->conf = dlsym(vcl->dlh, "VCL_conf"); if (vcl->conf == NULL) { - fprintf(stderr, "No VCL_conf symbol\n"); + if (cli == NULL) + fprintf(stderr, "No VCL_conf symbol\n"); + else + cli_out(cli, "No VCL_conf symbol\n"); dlclose(vcl->dlh); free(vcl); return (1); } if (vcl->conf->magic != VCL_CONF_MAGIC) { - fprintf(stderr, "Wrong VCL_CONF_MAGIC\n"); + if (cli == NULL) + fprintf(stderr, "Wrong VCL_CONF_MAGIC\n"); + else + cli_out(cli, "Wrong VCL_CONF_MAGIC\n"); dlclose(vcl->dlh); free(vcl); return (1); @@ -93,7 +122,10 @@ if (active_vcl == NULL) active_vcl = vcl; AZ(pthread_mutex_unlock(&sessmtx)); - fprintf(stderr, "Loaded \"%s\" as \"%s\"\n", fn , name); + if (cli == NULL) + fprintf(stderr, "Loaded \"%s\" as \"%s\"\n", fn , name); + else + cli_out(cli, "Loaded \"%s\" as \"%s\"\n", fn , name); vcl->conf->init_func(); return (0); } @@ -111,57 +143,12 @@ } } -static struct vcls * -find_vcls(const char *name) -{ - struct vcls *vcl; - - TAILQ_FOREACH(vcl, &vcl_head, list) - if (!strcmp(vcl->name, name)) - return (vcl); - return (NULL); -} - void cli_func_config_load(struct cli *cli, char **av, void *priv __unused) { - struct vcls *vcl; - vcl = find_vcls(av[2]); - if (vcl != NULL) { - cli_out(cli, "Config '%s' already loaded", av[2]); + if (CVCL_Load(av[3], av[2], cli)) cli_result(cli, CLIS_PARAM); - return; - } - vcl = calloc(sizeof *vcl, 1); - assert(vcl != NULL); - - vcl->dlh = dlopen(av[3], RTLD_NOW | RTLD_LOCAL); - if (vcl->dlh == NULL) { - cli_out(cli, "dlopen(%s): %s\n", av[3], dlerror()); - cli_result(cli, CLIS_PARAM); - free(vcl); - return; - } - vcl->conf = dlsym(vcl->dlh, "VCL_conf"); - if (vcl->conf == NULL) { - cli_out(cli, "No VCL_conf symbol\n"); - cli_result(cli, CLIS_PARAM); - dlclose(vcl->dlh); - free(vcl); - return; - } - if (vcl->conf->magic != VCL_CONF_MAGIC) { - cli_out(cli, "Wrong VCL_CONF_MAGIC\n"); - cli_result(cli, CLIS_PARAM); - dlclose(vcl->dlh); - free(vcl); - return; - } - vcl->name = strdup(av[2]); - assert(vcl->name != NULL); - TAILQ_INSERT_TAIL(&vcl_head, vcl, list); - cli_out(cli, "Loaded \"%s\" from \"%s\"\n", vcl->name , av[3]); return; } From phk at projects.linpro.no Wed Jul 12 11:45:53 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 12 Jul 2006 13:45:53 +0200 (CEST) Subject: r452 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712114553.B58DE1EC399@projects.linpro.no> Author: phk Date: 2006-07-12 13:45:53 +0200 (Wed, 12 Jul 2006) New Revision: 452 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: Make pipe work again Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 08:56:09 UTC (rev 451) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 11:45:53 UTC (rev 452) @@ -373,9 +373,15 @@ DOT pipe -> DONE */ -static void cnt_pipe(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } +static void +cnt_pipe(struct worker *w, struct sess *sp) +{ + PipeSession(w, sp); + sp->step = STP_DONE; +} + /*-------------------------------------------------------------------- * Dispatch the request as instructed by VCL * From phk at projects.linpro.no Wed Jul 12 11:48:07 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 12 Jul 2006 13:48:07 +0200 (CEST) Subject: r453 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712114807.9F0041EC38E@projects.linpro.no> Author: phk Date: 2006-07-12 13:48:07 +0200 (Wed, 12 Jul 2006) New Revision: 453 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_pass.c Log: Make Pass work again Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 11:45:53 UTC (rev 452) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 11:48:07 UTC (rev 453) @@ -339,9 +339,15 @@ DOT pass -> PASSBODY */ -static void cnt_pass(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } +static void +cnt_pass(struct worker *w, struct sess *sp) +{ + PassSession(w, sp); + sp->step = STP_DONE; /* XXX */ +} + /*-------------------------------------------------------------------- * We get here when we have the backends headers, send them to client * and pass any body the backend may have on as well. Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-12 11:45:53 UTC (rev 452) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-12 11:48:07 UTC (rev 453) @@ -182,8 +182,7 @@ else if (http_HdrIs(hp, "Transfer-Encoding", "chunked")) cls = pass_chunked(sp, vc->fd, hp); else { - INCOMPL(); - cls = 1; + cls = pass_straight(sp, vc->fd, hp, NULL); } vca_flush(sp); From phk at projects.linpro.no Wed Jul 12 12:04:28 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 12 Jul 2006 14:04:28 +0200 (CEST) Subject: r454 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712120428.E2D601EC381@projects.linpro.no> Author: phk Date: 2006-07-12 14:04:28 +0200 (Wed, 12 Jul 2006) New Revision: 454 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/hash_classic.c trunk/varnish-cache/bin/varnishd/hash_simple_list.c trunk/varnish-cache/bin/varnishd/hash_slinger.h Log: Hash on both URL and Host header. If no host header, hash on URL twice. Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-12 11:48:07 UTC (rev 453) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-12 12:04:28 UTC (rev 454) @@ -233,11 +233,11 @@ int FetchBody(struct worker *w, struct sess *sp) { - int i, cls; + int cls; struct vbe_conn *vc; struct http *hp; char *b; - int body; + int body = 1; /* XXX */ vc = sp->vbc; hp = sp->bkd_http; @@ -278,11 +278,9 @@ int FetchHeaders(struct worker *w, struct sess *sp) { - int i, cls; + int i; struct vbe_conn *vc; struct http *hp; - char *b; - int body; sp->obj->xid = sp->xid; Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-12 11:48:07 UTC (rev 453) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-12 12:04:28 UTC (rev 454) @@ -20,7 +20,7 @@ { struct objhead *oh; struct object *o; - char *b; + char *b, *c; assert(hash != NULL); /* Precreate an objhead and object in case we need them */ @@ -41,7 +41,9 @@ } assert(http_GetURL(h, &b)); - oh = hash->lookup(b, w->nobjhead); + if (!http_GetHdr(h, "Host", &c)) + c = b; + oh = hash->lookup(b, c, w->nobjhead); if (oh == w->nobjhead) w->nobjhead = NULL; AZ(pthread_mutex_lock(&oh->mtx)); Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-12 11:48:07 UTC (rev 453) +++ trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-12 12:04:28 UTC (rev 454) @@ -17,7 +17,8 @@ struct hcl_entry { TAILQ_ENTRY(hcl_entry) list; - char *key; + char *key1; + char *key2; struct objhead *obj; unsigned refcnt; unsigned hash; @@ -97,7 +98,7 @@ */ static struct objhead * -hcl_lookup(const char *key, struct objhead *nobj) +hcl_lookup(const char *key1, const char *key2, struct objhead *nobj) { struct hcl_entry *he, *he2; MD5_CTX c; @@ -106,7 +107,9 @@ int i; MD5Init(&c); - MD5Update(&c, key, strlen(key)); + MD5Update(&c, key1, strlen(key1)); + MD5Update(&c, "", 1); + MD5Update(&c, key2, strlen(key2)); MD5Final(md5, &c); memcpy(&u1, md5, sizeof u1); u1 %= hcl_nhash; @@ -115,30 +118,36 @@ AZ(pthread_mutex_lock(&hcl_mutex[u2])); TAILQ_FOREACH(he, &hcl_head[u1], list) { - i = strcmp(key, he->key); + i = strcmp(key1, he->key1); if (i < 0) continue; - if (i == 0) { - he->refcnt++; - nobj = he->obj; - nobj->hashpriv = he; - AZ(pthread_mutex_unlock(&hcl_mutex[u2])); - return (nobj); - } - if (nobj == NULL) { - AZ(pthread_mutex_unlock(&hcl_mutex[u2])); - return (NULL); - } - break; + if (i > 0) + break; + i = strcmp(key2, he->key2); + if (i < 0) + continue; + if (i > 0) + break; + he->refcnt++; + nobj = he->obj; + nobj->hashpriv = he; + AZ(pthread_mutex_unlock(&hcl_mutex[u2])); + return (nobj); } + if (nobj == NULL) { + AZ(pthread_mutex_unlock(&hcl_mutex[u2])); + return (NULL); + } he2 = calloc(sizeof *he2, 1); assert(he2 != NULL); he2->obj = nobj; he2->refcnt = 1; he2->hash = u1; he2->mtx = u2; - he2->key = strdup(key); - assert(he2->key != NULL); + he2->key1 = strdup(key1); + assert(he2->key1 != NULL); + he2->key2 = strdup(key2); + assert(he2->key2 != NULL); nobj->hashpriv = he2; if (he != NULL) TAILQ_INSERT_BEFORE(he, he2, list); @@ -164,7 +173,8 @@ mtx = he->mtx; AZ(pthread_mutex_lock(&hcl_mutex[mtx])); if (--he->refcnt == 0) { - free(he->key); + free(he->key1); + free(he->key2); TAILQ_REMOVE(&hcl_head[he->hash], he, list); free(he); ret = 0; Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-07-12 11:48:07 UTC (rev 453) +++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-07-12 12:04:28 UTC (rev 454) @@ -15,7 +15,8 @@ struct hsl_entry { TAILQ_ENTRY(hsl_entry) list; - char *key; + char *key1; + char *key2; struct objhead *obj; unsigned refcnt; }; @@ -43,35 +44,41 @@ */ static struct objhead * -hsl_lookup(const char *key, struct objhead *nobj) +hsl_lookup(const char *key1, const char *key2, struct objhead *nobj) { struct hsl_entry *he, *he2; int i; AZ(pthread_mutex_lock(&hsl_mutex)); TAILQ_FOREACH(he, &hsl_head, list) { - i = strcmp(key, he->key); + i = strcmp(key1, he->key1); if (i < 0) continue; - if (i == 0) { - he->refcnt++; - nobj = he->obj; - nobj->hashpriv = he; - AZ(pthread_mutex_unlock(&hsl_mutex)); - return (nobj); - } - if (nobj == NULL) { - AZ(pthread_mutex_unlock(&hsl_mutex)); - return (NULL); - } - break; + if (i > 0) + break; + i = strcmp(key2, he->key2); + if (i < 0) + continue; + if (i > 0) + break; + he->refcnt++; + nobj = he->obj; + nobj->hashpriv = he; + AZ(pthread_mutex_unlock(&hsl_mutex)); + return (nobj); } + if (nobj == NULL) { + AZ(pthread_mutex_unlock(&hsl_mutex)); + return (NULL); + } he2 = calloc(sizeof *he2, 1); assert(he2 != NULL); he2->obj = nobj; he2->refcnt = 1; - he2->key = strdup(key); - assert(he2->key != NULL); + he2->key1 = strdup(key1); + assert(he2->key1 != NULL); + he2->key2 = strdup(key2); + assert(he2->key2 != NULL); nobj->hashpriv = he2; if (he != NULL) TAILQ_INSERT_BEFORE(he, he2, list); @@ -95,7 +102,8 @@ he = obj->hashpriv; AZ(pthread_mutex_lock(&hsl_mutex)); if (--he->refcnt == 0) { - free(he->key); + free(he->key1); + free(he->key2); TAILQ_REMOVE(&hsl_head, he, list); free(he); ret = 0; Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2006-07-12 11:48:07 UTC (rev 453) +++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2006-07-12 12:04:28 UTC (rev 454) @@ -4,7 +4,7 @@ typedef int hash_init_f(const char *); typedef void hash_start_f(void); -typedef struct objhead *hash_lookup_f(const char *key, struct objhead *nobj); +typedef struct objhead *hash_lookup_f(const char *key1, const char *key2, struct objhead *nobj); typedef int hash_deref_f(struct objhead *obj); struct hash_slinger { From phk at projects.linpro.no Wed Jul 12 13:27:42 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 12 Jul 2006 15:27:42 +0200 (CEST) Subject: r455 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712132742.D77431EC38E@projects.linpro.no> Author: phk Date: 2006-07-12 15:27:42 +0200 (Wed, 12 Jul 2006) New Revision: 455 Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c Log: Add explanation for locking, some minor polishing. Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-12 12:04:28 UTC (rev 454) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-12 13:27:42 UTC (rev 455) @@ -1,5 +1,29 @@ /* * $Id$ + * + * This is the central hash-table code, it relies on a chosen hash + * implmentation only for the actual hashing, all the housekeeping + * happens here. + * + * We have two kinds of structures, objecthead and object. An objecthead + * corresponds to a given (Host:, URL) tupple, and the objects hung from + * the objecthead may represent various variations (ie: Vary: header, + * different TTL etc) instances of that web-entity. + * + * Each objecthead has a mutex which locks both its own fields, the + * list of objects and fields in the objects. + * + * The hash implementation must supply a reference count facility on + * the objecthead, and return with a reference held after a lookup. + * + * Lookups in the hash implementation returns with a ref held and each + * object hung from the objhead holds a ref as well. + * + * Objects have refcounts which are locked by the objecthead mutex. + * + * New objects are always marked busy, and they can go from busy to + * not busy only once. + * */ #include @@ -35,6 +59,7 @@ w->nobj = calloc(sizeof *w->nobj, 1); assert(w->nobj != NULL); w->nobj->busy = 1; + w->nobj->refcnt = 1; TAILQ_INIT(&w->nobj->store); AZ(pthread_cond_init(&w->nobj->cv, NULL)); VSL_stats->n_object++; @@ -51,8 +76,11 @@ o->refcnt++; if (o->busy) AZ(pthread_cond_wait(&o->cv, &oh->mtx)); - /* XXX: check TTL */ - if (o->ttl == 0) { + /* XXX: check ttl */ + /* XXX: check Vary: */ + if (!o->cacheable) { + /* ignore */ + } else if (o->ttl == 0) { /* Object banned but not reaped yet */ } else if (BAN_CheckObject(o, b)) { o->ttl = 0; @@ -71,7 +99,6 @@ /* Insert (precreated) object in objecthead */ o = w->nobj; w->nobj = NULL; - o->refcnt = 1; o->objhead = oh; TAILQ_INSERT_TAIL(&oh->objects, o, list); /* NB: do not deref objhead the new object inherits our reference */ @@ -95,19 +122,19 @@ { struct objhead *oh; struct storage *st, *stn; + unsigned r; oh = o->objhead; /* drop ref on object */ AZ(pthread_mutex_lock(&oh->mtx)); - if (--o->refcnt == 0) + r = --o->refcnt; + if (!r) TAILQ_REMOVE(&oh->objects, o, list); - else - o = NULL; AZ(pthread_mutex_unlock(&oh->mtx)); /* If still referenced, done */ - if (o == NULL) + if (r != 0) return; if (o->header != NULL) { Property changes on: trunk/varnish-cache/bin/varnishd/cache_hash.c ___________________________________________________________________ Name: svn:keywords + Id From phk at projects.linpro.no Wed Jul 12 13:28:27 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 12 Jul 2006 15:28:27 +0200 (CEST) Subject: r456 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712132827.058D61EC38E@projects.linpro.no> Author: phk Date: 2006-07-12 15:28:26 +0200 (Wed, 12 Jul 2006) New Revision: 456 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: First stab at implementing pass in vcl_miss() Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 13:27:42 UTC (rev 455) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 13:28:26 UTC (rev 456) @@ -312,8 +312,15 @@ VCL_miss_method(sp); if (sp->handling == VCL_RET_ERROR) INCOMPL(); - if (sp->handling == VCL_RET_PASS) - INCOMPL(); + if (sp->handling == VCL_RET_PASS) { + sp->obj->cacheable = 0; + HSH_Unbusy(sp->obj); + HSH_Deref(sp->obj); + sp->obj = 0; + PassSession(w, sp); + sp->step = STP_DONE; + return; + } if (sp->handling == VCL_RET_LOOKUP) INCOMPL(); if (sp->handling == VCL_RET_FETCH) { Property changes on: trunk/varnish-cache/bin/varnishd/cache_center.c ___________________________________________________________________ Name: svn:keywords + Id From phk at projects.linpro.no Wed Jul 12 13:28:57 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 12 Jul 2006 15:28:57 +0200 (CEST) Subject: r457 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712132857.5C8B81EC381@projects.linpro.no> Author: phk Date: 2006-07-12 15:28:57 +0200 (Wed, 12 Jul 2006) New Revision: 457 Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_response.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/hash_classic.c trunk/varnish-cache/bin/varnishd/hash_slinger.h trunk/varnish-cache/bin/varnishd/rfc2616.c trunk/varnish-cache/bin/varnishd/steps.h trunk/varnish-cache/bin/varnishd/stevedore.h trunk/varnish-cache/bin/varnishd/storage_file.c Log: Enable Id keyword Property changes on: trunk/varnish-cache/bin/varnishd/cache_ban.c ___________________________________________________________________ Name: svn:keywords + Id Property changes on: trunk/varnish-cache/bin/varnishd/cache_expire.c ___________________________________________________________________ Name: svn:keywords + Id Property changes on: trunk/varnish-cache/bin/varnishd/cache_response.c ___________________________________________________________________ Name: svn:keywords + Id Property changes on: trunk/varnish-cache/bin/varnishd/cache_vrt.c ___________________________________________________________________ Name: svn:keywords + Id Property changes on: trunk/varnish-cache/bin/varnishd/common.h ___________________________________________________________________ Name: svn:keywords + Id Property changes on: trunk/varnish-cache/bin/varnishd/hash_classic.c ___________________________________________________________________ Name: svn:keywords + Id Property changes on: trunk/varnish-cache/bin/varnishd/hash_slinger.h ___________________________________________________________________ Name: svn:keywords + Id Property changes on: trunk/varnish-cache/bin/varnishd/rfc2616.c ___________________________________________________________________ Name: svn:keywords + Id Property changes on: trunk/varnish-cache/bin/varnishd/steps.h ___________________________________________________________________ Name: svn:keywords + Id Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2006-07-12 13:28:26 UTC (rev 456) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2006-07-12 13:28:57 UTC (rev 457) @@ -1,5 +1,5 @@ /* - * $Id: cache.h 164 2006-05-01 12:45:20Z phk $ + * $Id$ */ struct stevedore; Property changes on: trunk/varnish-cache/bin/varnishd/stevedore.h ___________________________________________________________________ Name: svn:keywords + Id Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-12 13:28:26 UTC (rev 456) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-12 13:28:57 UTC (rev 457) @@ -1,5 +1,5 @@ /* - * $Id: storage_malloc.c 170 2006-06-13 07:57:32Z phk $ + * $Id$ * * Storage method based on mmap'ed file */ Property changes on: trunk/varnish-cache/bin/varnishd/storage_file.c ___________________________________________________________________ Name: svn:keywords + Id From phk at projects.linpro.no Wed Jul 12 14:13:50 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 12 Jul 2006 16:13:50 +0200 (CEST) Subject: r458 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712141350.CAEEB1EC38E@projects.linpro.no> Author: phk Date: 2006-07-12 16:13:50 +0200 (Wed, 12 Jul 2006) New Revision: 458 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: Make Pass possible from vcl_hit() Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 13:28:57 UTC (rev 457) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 14:13:50 UTC (rev 458) @@ -225,11 +225,23 @@ { VCL_hit_method(sp); - - vca_write_obj(w, sp); - HSH_Deref(sp->obj); - sp->obj = NULL; - sp->step = STP_DONE; + if (sp->handling == VCL_RET_LOOKUP) + INCOMPL(); + if (sp->handling == VCL_RET_PASS) { + PassSession(w, sp); + sp->step = STP_DONE; + return; + } + if (sp->handling == VCL_RET_ERROR) + INCOMPL(); + if (sp->handling == VCL_RET_DELIVER) { + vca_write_obj(w, sp); + HSH_Deref(sp->obj); + sp->obj = NULL; + sp->step = STP_DONE; + return; + } + INCOMPL(); } From phk at projects.linpro.no Wed Jul 12 15:07:42 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 12 Jul 2006 17:07:42 +0200 (CEST) Subject: r459 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060712150742.D06ED1EC38E@projects.linpro.no> Author: phk Date: 2006-07-12 17:07:42 +0200 (Wed, 12 Jul 2006) New Revision: 459 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/include/shmlog_tags.h trunk/varnish-cache/include/stat_field.h Log: Implement "insert_pass" mode where we cache that an entity must be passed. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-12 14:13:50 UTC (rev 458) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-12 15:07:42 UTC (rev 459) @@ -116,6 +116,8 @@ unsigned heap_idx; unsigned ban_seq; + unsigned pass; + unsigned response; unsigned valid; @@ -260,6 +262,7 @@ /* cache_pass.c */ void PassSession(struct worker *w, struct sess *sp); +void PassBody(struct worker *w, struct sess *sp); /* cache_pipe.c */ void PipeSession(struct worker *w, struct sess *sp); Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 14:13:50 UTC (rev 458) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 15:07:42 UTC (rev 459) @@ -84,6 +84,7 @@ { char *b; + (void)w; if (http_GetHdr(sp->http, "Connection", &b) && !strcmp(b, "close")) { vca_close_session(sp, "Connection header"); @@ -168,11 +169,24 @@ if (sp->handling == VCL_RET_LOOKUP) INCOMPL(); - if (sp->handling == VCL_RET_PASS) - INCOMPL(); - if (sp->handling == VCL_RET_INSERT_PASS) - INCOMPL(); + if (sp->handling == VCL_RET_PASS) { + sp->obj->cacheable = 0; + HSH_Unbusy(sp->obj); + HSH_Deref(sp->obj); + sp->obj = NULL; + sp->step = STP_PASSBODY; + return; + } + if (sp->handling == VCL_RET_INSERT_PASS) { + sp->obj->pass = 1; + sp->obj->cacheable = 1; + HSH_Unbusy(sp->obj); + sp->obj = NULL; + sp->step = STP_PASSBODY; + return; + } if (sp->handling == VCL_RET_INSERT) { + sp->obj->cacheable = 1; FetchBody(w, sp); sp->step = STP_DELIVER; return; @@ -225,15 +239,10 @@ { VCL_hit_method(sp); - if (sp->handling == VCL_RET_LOOKUP) - INCOMPL(); - if (sp->handling == VCL_RET_PASS) { - PassSession(w, sp); - sp->step = STP_DONE; - return; - } - if (sp->handling == VCL_RET_ERROR) - INCOMPL(); + + if (sp->handling == VCL_RET_DELIVER && sp->obj->pass) + sp->handling = VCL_RET_PASS; + if (sp->handling == VCL_RET_DELIVER) { vca_write_obj(w, sp); HSH_Deref(sp->obj); @@ -241,6 +250,20 @@ sp->step = STP_DONE; return; } + if (sp->handling == VCL_RET_PASS) { + HSH_Deref(sp->obj); + sp->obj = NULL; + PassSession(w, sp); + sp->step = STP_PASSBODY; + return; + } + + if (sp->handling == VCL_RET_ERROR) + INCOMPL(); + + if (sp->handling == VCL_RET_LOOKUP) + INCOMPL(); + INCOMPL(); } @@ -272,11 +295,17 @@ if (sp->obj->busy) { VSL_stats->cache_miss++; sp->step = STP_MISS; - } else { - VSL_stats->cache_hit++; - VSL(SLT_Hit, sp->fd, "%u", sp->obj->xid); + return; + } + if (sp->obj->pass) { + VSL_stats->cache_hitpass++; + VSL(SLT_HitPass, sp->fd, "%u", sp->obj->xid); sp->step = STP_HIT; - } + return; + } + VSL_stats->cache_hit++; + VSL(SLT_Hit, sp->fd, "%u", sp->obj->xid); + sp->step = STP_HIT; } @@ -330,7 +359,7 @@ HSH_Deref(sp->obj); sp->obj = 0; PassSession(w, sp); - sp->step = STP_DONE; + sp->step = STP_PASSBODY; return; } if (sp->handling == VCL_RET_LOOKUP) @@ -363,7 +392,7 @@ { PassSession(w, sp); - sp->step = STP_DONE; /* XXX */ + sp->step = STP_PASSBODY; } @@ -381,7 +410,13 @@ DOT passbody -> DONE */ -static void cnt_passbody(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } +static void +cnt_passbody(struct worker *w, struct sess *sp) +{ + PassBody(w, sp); + sp->step = STP_DONE; + return; +} /*-------------------------------------------------------------------- @@ -493,7 +528,11 @@ for (sp->step = STP_RECV; sp->step != STP_DONE; ) { switch (sp->step) { -#define STEP(l,u) case STP_##u: cnt_##l(w, sp); break; +#define STEP(l,u) \ + case STP_##u: \ + VSL(SLT_Debug, sp->fd, "State " #u); \ + cnt_##l(w, sp); \ + break; #include "steps.h" #undef STEP default: INCOMPL(); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-12 14:13:50 UTC (rev 458) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-12 15:07:42 UTC (rev 459) @@ -265,9 +265,9 @@ else VBE_RecycleFd(vc); - if (sp->obj->cacheable) - EXP_Insert(sp->obj); HSH_Unbusy(sp->obj); + /* Hold on to the reference count, it's not released until + * expiry */ if (!sp->obj->cacheable) HSH_Deref(sp->obj); return (0); Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-12 14:13:50 UTC (rev 458) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-12 15:07:42 UTC (rev 459) @@ -111,6 +111,8 @@ HSH_Unbusy(struct object *o) { + if (o->cacheable) + EXP_Insert(o); AZ(pthread_mutex_lock(&o->objhead->mtx)); o->busy = 0; AZ(pthread_mutex_unlock(&o->objhead->mtx)); Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-12 14:13:50 UTC (rev 458) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-12 15:07:42 UTC (rev 459) @@ -144,34 +144,20 @@ /*--------------------------------------------------------------------*/ + void -PassSession(struct worker *w, struct sess *sp) +PassBody(struct worker *w, struct sess *sp) { - int i; struct vbe_conn *vc; struct http *hp; char *b; int cls; - vc = VBE_GetFd(sp->backend, sp->xid); + hp = sp->bkd_http; + assert(hp != NULL); + vc = sp->vbc; assert(vc != NULL); - VSL(SLT_Backend, sp->fd, "%d %s", vc->fd, sp->backend->vcl_name); - http_BuildSbuf(vc->fd, Build_Pass, w->sb, sp->http); - i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); - assert(i == sbuf_len(w->sb)); - - /* XXX: copy any contents */ - - /* - * XXX: It might be cheaper to avoid the event_engine and simply - * XXX: read(2) the header - */ - hp = vc->http; - http_RecvHead(hp, vc->fd, w->eb, NULL, NULL); - event_base_loop(w->eb, 0); - http_DissectResponse(hp, vc->fd); - http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); @@ -194,3 +180,34 @@ else VBE_RecycleFd(vc); } + +/*--------------------------------------------------------------------*/ +void +PassSession(struct worker *w, struct sess *sp) +{ + int i; + struct vbe_conn *vc; + struct http *hp; + + vc = VBE_GetFd(sp->backend, sp->xid); + assert(vc != NULL); + VSL(SLT_Backend, sp->fd, "%d %s", vc->fd, sp->backend->vcl_name); + + http_BuildSbuf(vc->fd, Build_Pass, w->sb, sp->http); + i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); + assert(i == sbuf_len(w->sb)); + + /* XXX: copy any contents */ + + /* + * XXX: It might be cheaper to avoid the event_engine and simply + * XXX: read(2) the header + */ + hp = vc->http; + http_RecvHead(hp, vc->fd, w->eb, NULL, NULL); + event_base_loop(w->eb, 0); + http_DissectResponse(hp, vc->fd); + + sp->bkd_http = hp; + sp->vbc = vc; +} Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-07-12 14:13:50 UTC (rev 458) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-07-12 15:07:42 UTC (rev 459) @@ -35,6 +35,7 @@ SLTM(VCL_return) SLTM(XID) SLTM(Hit) +SLTM(HitPass) SLTM(ExpBan) SLTM(ExpPick) SLTM(ExpKill) Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2006-07-12 14:13:50 UTC (rev 458) +++ trunk/varnish-cache/include/stat_field.h 2006-07-12 15:07:42 UTC (rev 459) @@ -4,6 +4,7 @@ MAC_STAT(client_req, uint64_t, "u", "Client requests received") MAC_STAT(cache_hit, uint64_t, "u", "Cache hits") +MAC_STAT(cache_hitpass, uint64_t, "u", "Cache hits for pass") MAC_STAT(cache_miss, uint64_t, "u", "Cache misses") MAC_STAT(backend_conn, uint64_t, "u", "Backend connections initiated") From phk at projects.linpro.no Wed Jul 12 19:28:06 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 12 Jul 2006 21:28:06 +0200 (CEST) Subject: r460 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712192806.ECAF41EC390@projects.linpro.no> Author: phk Date: 2006-07-12 21:28:06 +0200 (Wed, 12 Jul 2006) New Revision: 460 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_main.c trunk/varnish-cache/bin/varnishd/cache_vcl.c Log: Move sessmtx to cache_vcl.c and call it vcl_mtx. Clean up naming for consistency while here. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-12 15:07:42 UTC (rev 459) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-12 19:28:06 UTC (rev 460) @@ -257,9 +257,6 @@ }; void http_BuildSbuf(int fd, enum http_build mode, struct sbuf *sb, struct http *hp); -/* cache_main.c */ -extern pthread_mutex_t sessmtx; - /* cache_pass.c */ void PassSession(struct worker *w, struct sess *sp); void PassBody(struct worker *w, struct sess *sp); @@ -288,9 +285,10 @@ void RES_Error(struct worker *w, struct sess *sp, int error, const char *msg); /* cache_vcl.c */ -void RelVCL(struct VCL_conf *vc); -struct VCL_conf *GetVCL(void); -int CVCL_Load(const char *fn, const char *name, struct cli *cli); +void VCL_Init(void); +void VCL_Rel(struct VCL_conf *vc); +struct VCL_conf *VCL_Get(void); +int VCL_Load(const char *fn, const char *name, struct cli *cli); #define VCL_RET_MAC(l,u,b) #define VCL_MET_MAC(l,u,b) void VCL_##l##_method(struct sess *); Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 15:07:42 UTC (rev 459) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 19:28:06 UTC (rev 460) @@ -522,9 +522,7 @@ { time(&sp->t0); - AZ(pthread_mutex_lock(&sessmtx)); - sp->vcl = GetVCL(); - AZ(pthread_mutex_unlock(&sessmtx)); + sp->vcl = VCL_Get(); for (sp->step = STP_RECV; sp->step != STP_DONE; ) { switch (sp->step) { @@ -541,9 +539,7 @@ cnt_done(w, sp); /* The loop doesn't do this */ - AZ(pthread_mutex_lock(&sessmtx)); - RelVCL(sp->vcl); - AZ(pthread_mutex_unlock(&sessmtx)); + VCL_Rel(sp->vcl); sp->vcl = NULL; vca_return_session(sp); Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-12 15:07:42 UTC (rev 459) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-12 19:28:06 UTC (rev 460) @@ -103,10 +103,10 @@ AZ(pthread_mutex_unlock(&exp_mtx)); VSL(SLT_ExpPick, 0, "%u", o->xid); - sp.vcl = GetVCL(); + sp.vcl = VCL_Get(); sp.obj = o; VCL_timeout_method(&sp); - RelVCL(sp.vcl); + VCL_Rel(sp.vcl); if (sp.handling == VCL_RET_DISCARD) { AZ(pthread_mutex_lock(&exp_mtx)); Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-12 15:07:42 UTC (rev 459) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-12 19:28:06 UTC (rev 460) @@ -20,7 +20,6 @@ struct stevedore *stevedore; -pthread_mutex_t sessmtx; struct varnish_stats *VSL_stats; /*--------------------------------------------------------------------*/ @@ -105,8 +104,9 @@ setbuf(stderr, NULL); printf("Child starts\n"); - CVCL_Load(heritage.vcl_file, "boot", NULL); - AZ(pthread_mutex_init(&sessmtx, NULL)); + VCL_Init(); + VCL_Load(heritage.vcl_file, "boot", NULL); + VBE_Init(); VSL_Init(); WRK_Init(); Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-07-12 15:07:42 UTC (rev 459) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-07-12 19:28:06 UTC (rev 460) @@ -20,6 +20,7 @@ const char *name; void *dlh; struct VCL_conf *conf; + unsigned busy; }; /* @@ -30,36 +31,39 @@ TAILQ_HEAD_INITIALIZER(vcl_head); -static struct vcls *active_vcl; /* protected by sessmtx */ +static struct vcls *vcl_active; /* protected by vcl_mtx */ +static pthread_mutex_t vcl_mtx; /*--------------------------------------------------------------------*/ struct VCL_conf * -GetVCL(void) +VCL_Get(void) { struct VCL_conf *vc; - /* XXX: assert sessmtx (procects active_vcl && ->busy) */ - assert(active_vcl != NULL); - vc = active_vcl->conf; + AZ(pthread_mutex_lock(&vcl_mtx)); + assert(vcl_active != NULL); + vc = vcl_active->conf; assert(vc != NULL); vc->busy++; + AZ(pthread_mutex_unlock(&vcl_mtx)); return (vc); } void -RelVCL(struct VCL_conf *vc) +VCL_Rel(struct VCL_conf *vc) { - /* XXX: assert sessmtx (procects ->busy) */ + AZ(pthread_mutex_lock(&vcl_mtx)); vc->busy--; + AZ(pthread_mutex_unlock(&vcl_mtx)); } /*--------------------------------------------------------------------*/ static struct vcls * -find_vcls(const char *name) +vcl_find(const char *name) { struct vcls *vcl; @@ -70,11 +74,11 @@ } int -CVCL_Load(const char *fn, const char *name, struct cli *cli) +VCL_Load(const char *fn, const char *name, struct cli *cli) { struct vcls *vcl; - vcl = find_vcls(name); + vcl = vcl_find(name); if (vcl != NULL) { if (cli == NULL) fprintf(stderr, "Config '%s' already loaded", name); @@ -118,10 +122,10 @@ vcl->name = strdup(name); assert(vcl->name != NULL); TAILQ_INSERT_TAIL(&vcl_head, vcl, list); - AZ(pthread_mutex_lock(&sessmtx)); - if (active_vcl == NULL) - active_vcl = vcl; - AZ(pthread_mutex_unlock(&sessmtx)); + AZ(pthread_mutex_lock(&vcl_mtx)); + if (vcl_active == NULL) + vcl_active = vcl; + AZ(pthread_mutex_unlock(&vcl_mtx)); if (cli == NULL) fprintf(stderr, "Loaded \"%s\" as \"%s\"\n", fn , name); else @@ -130,6 +134,8 @@ return (0); } +/*--------------------------------------------------------------------*/ + void cli_func_config_list(struct cli *cli, char **av __unused, void *priv __unused) { @@ -137,7 +143,7 @@ TAILQ_FOREACH(vcl, &vcl_head, list) { cli_out(cli, "%s %6u %s\n", - vcl == active_vcl ? "* " : " ", + vcl == vcl_active ? "* " : " ", vcl->conf->busy, vcl->name); } @@ -147,7 +153,7 @@ cli_func_config_load(struct cli *cli, char **av, void *priv __unused) { - if (CVCL_Load(av[3], av[2], cli)) + if (VCL_Load(av[3], av[2], cli)) cli_result(cli, CLIS_PARAM); return; } @@ -163,11 +169,11 @@ { struct vcls *vcl; - vcl = find_vcls(av[2]); + vcl = vcl_find(av[2]); if (vcl != NULL) { - AZ(pthread_mutex_lock(&sessmtx)); - active_vcl = vcl; - AZ(pthread_mutex_unlock(&sessmtx)); + AZ(pthread_mutex_lock(&vcl_mtx)); + vcl_active = vcl; + AZ(pthread_mutex_unlock(&vcl_mtx)); } else { cli_out(cli, "No config named '%s' loaded", av[2]); cli_result(cli, CLIS_PARAM); @@ -224,3 +230,12 @@ #include "vcl_returns.h" #undef VCL_MET_MAC #undef VCL_RET_MAC + +/*--------------------------------------------------------------------*/ + +void +VCL_Init() +{ + + AZ(pthread_mutex_init(&vcl_mtx, NULL)); +} From phk at projects.linpro.no Wed Jul 12 20:21:47 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 12 Jul 2006 22:21:47 +0200 (CEST) Subject: r461 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712202147.342601EC38D@projects.linpro.no> Author: phk Date: 2006-07-12 22:21:47 +0200 (Wed, 12 Jul 2006) New Revision: 461 Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c Log: More polishing. Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-07-12 19:28:06 UTC (rev 460) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-07-12 20:21:47 UTC (rev 461) @@ -1,5 +1,10 @@ /* * $Id$ + * + * Interface *to* compiled VCL code: Loading, unloading, calling into etc. + * + * The interface *from* the compiled VCL code is in cache_vrt.c. + * */ #include @@ -20,7 +25,6 @@ const char *name; void *dlh; struct VCL_conf *conf; - unsigned busy; }; /* @@ -91,7 +95,13 @@ assert(vcl != NULL); vcl->dlh = dlopen(fn, RTLD_NOW | RTLD_LOCAL); - unlink(fn); + + /* + * Delete the file, either we got hold of it, or we couldn't care + * less about it anyway. + */ + (void)unlink(fn); + if (vcl->dlh == NULL) { if (cli == NULL) fprintf(stderr, "dlopen(%s): %s\n", fn, dlerror()); @@ -106,7 +116,7 @@ fprintf(stderr, "No VCL_conf symbol\n"); else cli_out(cli, "No VCL_conf symbol\n"); - dlclose(vcl->dlh); + (void)dlclose(vcl->dlh); free(vcl); return (1); } @@ -115,7 +125,7 @@ fprintf(stderr, "Wrong VCL_CONF_MAGIC\n"); else cli_out(cli, "Wrong VCL_CONF_MAGIC\n"); - dlclose(vcl->dlh); + (void)dlclose(vcl->dlh); free(vcl); return (1); } @@ -137,10 +147,12 @@ /*--------------------------------------------------------------------*/ void -cli_func_config_list(struct cli *cli, char **av __unused, void *priv __unused) +cli_func_config_list(struct cli *cli, char **av, void *priv) { struct vcls *vcl; + (void)av; + (void)priv; TAILQ_FOREACH(vcl, &vcl_head, list) { cli_out(cli, "%s %6u %s\n", vcl == vcl_active ? "* " : " ", @@ -150,25 +162,32 @@ } void -cli_func_config_load(struct cli *cli, char **av, void *priv __unused) +cli_func_config_load(struct cli *cli, char **av, void *priv) { + (void)av; + (void)priv; if (VCL_Load(av[3], av[2], cli)) cli_result(cli, CLIS_PARAM); return; } void -cli_func_config_unload(struct cli *cli, char **av __unused, void *priv __unused) +cli_func_config_unload(struct cli *cli, char **av, void *priv) { + + (void)av; + (void)priv; cli_result(cli, CLIS_UNIMPL); } void -cli_func_config_use(struct cli *cli, char **av, void *priv __unused) +cli_func_config_use(struct cli *cli, char **av, void *priv) { struct vcls *vcl; + (void)av; + (void)priv; vcl = vcl_find(av[2]); if (vcl != NULL) { AZ(pthread_mutex_lock(&vcl_mtx)); @@ -183,7 +202,7 @@ /*--------------------------------------------------------------------*/ static const char * -HandlingName(unsigned u) +vcl_handlingname(unsigned u) { switch (u) { @@ -192,41 +211,27 @@ #include "vcl_returns.h" #undef VCL_RET_MAC #undef VCL_RET_MAC_E - default: return (NULL); + default: + return (NULL); } } -static void -CheckHandling(struct sess *sp, const char *func, unsigned bitmap) -{ - unsigned u; +#define VCL_RET_MAC(l,u,b) - u = sp->handling; - if (u & (u - 1)) - VSL(SLT_Error, sp->fd, - "Illegal handling after %s function: 0x%x", func, u); - else if (!(u & bitmap)) - VSL(SLT_Error, sp->fd, - "Wrong handling after %s function: 0x%x", func, u); - else - return; - sp->handling = VCL_RET_ERROR; +#define VCL_MET_MAC(func, xxx, bitmap) \ +void \ +VCL_##func##_method(struct sess *sp) \ +{ \ + \ + sp->handling = 0; \ + VSL(SLT_VCL_call, sp->fd, "%s", #func); \ + sp->vcl->func##_func(sp); \ + VSL(SLT_VCL_return, sp->fd, "%s", \ + vcl_handlingname(sp->handling)); \ + assert(sp->handling & bitmap); \ + assert(!(sp->handling & ~bitmap)); \ } -#define VCL_method(func, bitmap) \ -void \ -VCL_##func##_method(struct sess *sp) \ -{ \ - \ - sp->handling = 0; \ - VSL(SLT_VCL_call, sp->fd, "%s", #func); \ - sp->vcl->func##_func(sp); \ - CheckHandling(sp, #func, (bitmap)); \ - VSL(SLT_VCL_return, sp->fd, "%s", HandlingName(sp->handling)); \ -} - -#define VCL_RET_MAC(l,u,b) -#define VCL_MET_MAC(l,u,b) VCL_method(l, b) #include "vcl_returns.h" #undef VCL_MET_MAC #undef VCL_RET_MAC From phk at projects.linpro.no Wed Jul 12 22:01:15 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 13 Jul 2006 00:01:15 +0200 (CEST) Subject: r462 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712220115.1D89D1EC38E@projects.linpro.no> Author: phk Date: 2006-07-13 00:01:15 +0200 (Thu, 13 Jul 2006) New Revision: 462 Modified: trunk/varnish-cache/bin/varnishd/cache.h Log: Improve the INCOMPL() macro. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-12 20:21:47 UTC (rev 461) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-12 22:01:15 UTC (rev 462) @@ -277,7 +277,8 @@ #define HERE() VSL(SLT_Debug, 0, "HERE: %s(%d)", __func__, __LINE__) #define INCOMPL() do { \ VSL(SLT_Debug, 0, "INCOMPLETE AT: %s(%d)", __func__, __LINE__); \ - assert(__LINE__ == 0); \ + fprintf(stderr,"INCOMPLETE AT: %s(%d)\n", (const char *)__func__, __LINE__); \ + abort(); \ } while (0) #endif From phk at projects.linpro.no Wed Jul 12 22:01:43 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 13 Jul 2006 00:01:43 +0200 (CEST) Subject: r463 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712220143.E65CE1EC38D@projects.linpro.no> Author: phk Date: 2006-07-13 00:01:43 +0200 (Thu, 13 Jul 2006) New Revision: 463 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/flint.lnt trunk/varnish-cache/bin/varnishd/varnishd.c Log: Give this file a flexelinting Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-12 22:01:15 UTC (rev 462) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-12 22:01:43 UTC (rev 463) @@ -20,7 +20,6 @@ #include #include "config.h" -#include "compat.h" #include "libvarnish.h" #include "heritage.h" #include "shmlog.h" @@ -72,7 +71,7 @@ } static void -vca_delete_sess(struct sess *sp) +vca_delete_sess(const struct sess *sp) { VSL_stats->n_sess--; @@ -160,13 +159,16 @@ /*--------------------------------------------------------------------*/ static void -vca_tick(int a __unused, short b __unused, void *c __unused) +vca_tick(int a, short b, void *c) { struct sess *sp, *sp2; time_t t; - evtimer_add(&tick_e, &tick_rate); - time(&t); + (void)a; + (void)b; + (void)c; + AZ(evtimer_add(&tick_e, &tick_rate)); + (void)time(&t); TAILQ_FOREACH_SAFE(sp, &sesshead, list, sp2) { if (sp->t_resp + 30 < t) { TAILQ_REMOVE(&sesshead, sp, list); @@ -194,20 +196,22 @@ } static void -pipe_f(int fd, short event __unused, void *arg __unused) +pipe_f(int fd, short event, void *arg) { struct sess *sp; int i; + (void)event; + (void)arg; i = read(fd, &sp, sizeof sp); assert(i == sizeof sp); - time(&sp->t_resp); + sp->t_resp = time(NULL); TAILQ_INSERT_TAIL(&sesshead, sp, list); http_RecvHead(sp->http, sp->fd, evb, vca_callback, sp); } static void -accept_f(int fd, short event __unused, void *arg __unused) +accept_f(int fd, short event, void *arg) { socklen_t l; struct sockaddr addr[2]; /* XXX: IPv6 hack */ @@ -215,6 +219,8 @@ int i; struct linger linger; + (void)event; + (void)arg; VSL_stats->client_conn++; sp = vca_new_sess(); @@ -239,28 +245,31 @@ TCP_name(addr, l, sp->addr); VSL(SLT_SessionOpen, sp->fd, "%s", sp->addr); - time(&sp->t_resp); + (void)time(&sp->t_resp); TAILQ_INSERT_TAIL(&sesshead, sp, list); http_RecvHead(sp->http, sp->fd, evb, vca_callback, sp); } static void * -vca_main(void *arg __unused) +vca_main(void *arg) { unsigned u; struct event *ep; + (void)arg; + AZ(pipe(pipes)); evb = event_init(); + assert(evb != NULL); event_set(&pipe_e, pipes[0], EV_READ | EV_PERSIST, pipe_f, NULL); - event_base_set(evb, &pipe_e); - event_add(&pipe_e, NULL); + AZ(event_base_set(evb, &pipe_e)); + AZ(event_add(&pipe_e, NULL)); evtimer_set(&tick_e, vca_tick, NULL); - event_base_set(evb, &tick_e); + AZ(event_base_set(evb, &tick_e)); - evtimer_add(&tick_e, &tick_rate); + AZ(evtimer_add(&tick_e, &tick_rate)); ep = accept_e; for (u = 0; u < HERITAGE_NSOCKS; u++) { @@ -268,23 +277,22 @@ event_set(ep, heritage.sock_local[u], EV_READ | EV_PERSIST, accept_f, NULL); - event_base_set(evb, ep); - event_add(ep, NULL); + AZ(event_base_set(evb, ep)); + AZ(event_add(ep, NULL)); ep++; } if (heritage.sock_remote[u] >= 0) { event_set(ep, heritage.sock_remote[u], EV_READ | EV_PERSIST, accept_f, NULL); - event_base_set(evb, ep); - event_add(ep, NULL); + AZ(event_base_set(evb, ep)); + AZ(event_add(ep, NULL)); ep++; } } - event_base_loop(evb, 0); - assert(0 == 1); - return (NULL); + AZ(event_base_loop(evb, 0)); + INCOMPL(); } /*--------------------------------------------------------------------*/ @@ -294,7 +302,8 @@ { VSL(SLT_SessionClose, sp->fd, why); - close(sp->fd); + if (sp->fd >= 0) + AZ(close(sp->fd)); sp->fd = -1; } @@ -306,7 +315,7 @@ if (sp->fd >= 0) { VSL(SLT_SessionReuse, sp->fd, "%s", sp->addr); - write(pipes[1], &sp, sizeof sp); + assert(sizeof sp == write(pipes[1], &sp, sizeof sp)); } else { vca_delete_sess(sp); } Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-12 22:01:15 UTC (rev 462) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-12 22:01:43 UTC (rev 463) @@ -199,7 +199,6 @@ TCP_myname(s, buf); p = strchr(buf, '\0'); - assert(p != NULL); *p++ = ' '; TCP_name(ai->ai_addr, ai->ai_addrlen, p); VSL(SLT_BackendOpen, s, buf); Modified: trunk/varnish-cache/bin/varnishd/flint.lnt =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.lnt 2006-07-12 22:01:15 UTC (rev 462) +++ trunk/varnish-cache/bin/varnishd/flint.lnt 2006-07-12 22:01:43 UTC (rev 463) @@ -1,47 +1,64 @@ -passes=3 -// Review all below this line +// Fix strchr() semtics, it can only return NULL if arg2 != 0 +-sem(strchr, 1p, type(1), 2n == 0 ? (@p < 1p) : (@p < 1p || @p == 0 )) --printf_code( H, void *, unsigned) --printf_code( ju, long long unsigned) --printf_code( jx, long long unsigned) - -e763 // Redundant declaration for symbol '...' previously declared +-e726 // Extraneous comma ignored +-e728 // Symbol ... not explicitly initialized +-esym(534, fprintf) // Ignoring return value of function +-esym(534, memset) // Ignoring return value of function +-esym(534, sbuf_printf) // Ignoring return value of function --e737 // Loss of sign in promotion from int to unsigned int --e715 // Symbol 'arg' (line 43) not referenced --e818 // Pointer parameter '...' could be declared as pointing to const +// cache.h +-emacro(506, INCOMPL) // Constant value Boolean --e534 // Ignoring return value of function --e767 // macro 'LIST_INIT' was defined differently +// cache_vcl.c +-efunc(525, vcl_handlingname) // Negative indentation from line +-esym(528, vcl_handlingname) // Not referenced +-e641 // Converting enum 'cli_status_e' to int --e506 // Constant value boolean --e527 // Unreachable code at token 'return' --e732 // Loss of sign (arg. no. 2) (int to unsigned int) --e774 // Boolean within 'if' always evaluates to False --e713 // Loss of precision (assignment) (unsigned long long to long long) --e574 // Signed-unsigned mix with relational +// Review all below this line --e525 // Negative indentation from line 90 --e539 // Did not expect positive indentation --e725 // Expected positive indentation from line 136 --e734 // Loss of precision (assignment) (31 bits to 8 bits) --e747 // Significant prototype coercion (arg. no. 2) long --e712 // Loss of precision (assignment) (long long to +// -printf_code( H, void *, unsigned) +// -printf_code( ju, long long unsigned) +// -printf_code( jx, long long unsigned) +// --e785 // Too few initializers for aggregate - --e766 // Header file '../../include/libvarnish.h' not used in module - --e773 // Expression-like macro 'VCL_FARGS' not parenthesized - --e788 // enum constant 'HND_Unclass' not used within defaulted switch - --e716 // while(1) ... --e641 // Converting enum 'cli_status_e' to int - --e786 // String concatenation within initializer - --emacro(19, MAC_STAT) // Useless Declaration +// +// +-e737 // Loss of sign in promotion from int to unsigned int +-e732 // Loss of sign (arg. no. 2) (int to unsigned int) +// -e715 // Symbol 'arg' (line 43) not referenced +// -e818 // Pointer parameter '...' could be declared as pointing to const +// +// -e534 // Ignoring return value of function +// -e767 // macro 'LIST_INIT' was defined differently +// +// -e506 // Constant value boolean +// -e527 // Unreachable code at token 'return' +// -e774 // Boolean within 'if' always evaluates to False +// -e713 // Loss of precision (assignment) (unsigned long long to long long) +// -e574 // Signed-unsigned mix with relational +// +// -e525 // Negative indentation from line 90 +// -e539 // Did not expect positive indentation +// -e725 // Expected positive indentation from line 136 +// -e734 // Loss of precision (assignment) (31 bits to 8 bits) +// -e747 // Significant prototype coercion (arg. no. 2) long +// -e712 // Loss of precision (assignment) (long long to +// +// +// -e785 // Too few initializers for aggregate +// +// -e766 // Header file '../../include/libvarnish.h' not used in module +// +// -e773 // Expression-like macro 'VCL_FARGS' not parenthesized +// +// -e788 // enum constant 'HND_Unclass' not used within defaulted switch +// +// -e716 // while(1) ... +// +// -e786 // String concatenation within initializer Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-12 22:01:15 UTC (rev 462) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-12 22:01:43 UTC (rev 463) @@ -122,6 +122,7 @@ q = p + 1; } else { p = strchr(bflag, '\0'); + assert(p != NULL); q = "http"; } From phk at projects.linpro.no Wed Jul 12 22:07:00 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 13 Jul 2006 00:07:00 +0200 (CEST) Subject: r464 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712220700.2E4811EC38E@projects.linpro.no> Author: phk Date: 2006-07-13 00:07:00 +0200 (Thu, 13 Jul 2006) New Revision: 464 Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c Log: Flexelint harder. Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-12 22:01:43 UTC (rev 463) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-12 22:07:00 UTC (rev 464) @@ -125,7 +125,7 @@ /*--------------------------------------------------------------------*/ static int -vbe_sock_conn(struct addrinfo *ai) +vbe_sock_conn(const struct addrinfo *ai) { int s; @@ -133,7 +133,7 @@ if (s < 0) return (s); else if (connect(s, ai->ai_addr, ai->ai_addrlen)) { - close(s); + AZ(close(s)); s = -1; } return (s); @@ -212,11 +212,13 @@ */ static void -vbe_rdp(int fd, short event __unused, void *arg __unused) +vbe_rdp(int fd, short event, void *arg) { struct vbe_conn *vc; int i; + (void)event; + (void)arg; i = read(fd, &vc, sizeof vc); assert(i == sizeof vc); AZ(pthread_mutex_lock(&vbemtx)); @@ -226,7 +228,7 @@ vbe_delete_conn(vc); } else { vc->inuse = 0; - event_add(&vc->ev, NULL); + AZ(event_add(&vc->ev, NULL)); TAILQ_INSERT_HEAD(&vc->vbe->fconn, vc, list); } AZ(pthread_mutex_unlock(&vbemtx)); @@ -239,15 +241,17 @@ */ static void -vbe_rdf(int fd __unused, short event __unused, void *arg) +vbe_rdf(int fd, short event, void *arg) { struct vbe_conn *vc; int j; + (void)fd; + (void)event; vc = arg; AZ(pthread_mutex_lock(&vbemtx)); if (vc->inuse) { - event_del(&vc->ev); + AZ(event_del(&vc->ev)); AZ(pthread_mutex_unlock(&vbemtx)); return; } @@ -255,18 +259,19 @@ VSL(SLT_BackendClose, vc->fd, "Remote (%d chars)", j); TAILQ_REMOVE(&vc->vbe->fconn, vc, list); AZ(pthread_mutex_unlock(&vbemtx)); - event_del(&vc->ev); - close(vc->fd); + AZ(event_del(&vc->ev)); + AZ(close(vc->fd)); vbe_delete_conn(vc); } /* Backend monitoring thread -----------------------------------------*/ static void * -vbe_main(void *priv __unused) +vbe_main(void *priv) { struct event pev; + (void)priv; vbe_evb = event_init(); assert(vbe_evb != NULL); @@ -274,13 +279,12 @@ memset(&pev, 0, sizeof pev); event_set(&pev, vbe_pipe[0], EV_READ | EV_PERSIST, vbe_rdp, NULL); - event_base_set(vbe_evb, &pev); - event_add(&pev, NULL); + AZ(event_base_set(vbe_evb, &pev)); + AZ(event_add(&pev, NULL)); - event_base_loop(vbe_evb, 0); + AZ(event_base_loop(vbe_evb, 0)); - assert(__LINE__ == 0); - return (NULL); + INCOMPL(); } /* Get a backend connection ------------------------------------------ @@ -343,7 +347,7 @@ VSL_stats->backend_conn++; event_set(&vc->ev, vc->fd, EV_READ | EV_PERSIST, vbe_rdf, vc); - event_base_set(vbe_evb, &vc->ev); + AZ(event_base_set(vbe_evb, &vc->ev)); } VSL(SLT_BackendXID, vc->fd, "%u", xid); return (vc); @@ -356,8 +360,9 @@ { int i; + assert(vc->fd >= 0); VSL(SLT_BackendClose, vc->fd, ""); - close(vc->fd); + AZ(close(vc->fd)); vc->fd = -1; i = write(vbe_pipe[1], &vc, sizeof vc); assert(i == sizeof vc); From phk at projects.linpro.no Wed Jul 12 22:52:08 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 13 Jul 2006 00:52:08 +0200 (CEST) Subject: r465 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712225208.9A3F91EC1F5@projects.linpro.no> Author: phk Date: 2006-07-13 00:52:08 +0200 (Thu, 13 Jul 2006) New Revision: 465 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_response.c trunk/varnish-cache/bin/varnishd/flint.lnt trunk/varnish-cache/bin/varnishd/mgt_child.c Log: More flexelinting. No bugs so far. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-12 22:07:00 UTC (rev 464) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-12 22:52:08 UTC (rev 465) @@ -5,9 +5,11 @@ #include #include #include -#include -#include +#include "queue.h" +#include "event.h" +#include "sbuf.h" + #include "vcl_returns.h" #include "common.h" Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-12 22:07:00 UTC (rev 464) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-12 22:52:08 UTC (rev 465) @@ -17,8 +17,6 @@ #include -#include - #include "config.h" #include "libvarnish.h" #include "heritage.h" Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-12 22:07:00 UTC (rev 464) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-12 22:52:08 UTC (rev 465) @@ -26,7 +26,6 @@ #include #include -#include #include #include #include Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2006-07-12 22:07:00 UTC (rev 464) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2006-07-12 22:52:08 UTC (rev 465) @@ -36,7 +36,7 @@ if (i) { char buf[512]; - regerror(i, &b->regexp, buf, sizeof buf); + (void)regerror(i, &b->regexp, buf, sizeof buf); VSL(SLT_Debug, 0, "REGEX: <%s>", buf); } b->gen = ++ban_next; @@ -71,9 +71,10 @@ } void -cli_func_url_purge(struct cli *cli, char **av, void *priv __unused) +cli_func_url_purge(struct cli *cli, char **av, void *priv) { + (void)priv; AddBan(av[2]); cli_out(cli, "PURGE %s\n", av[2]); } Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 22:07:00 UTC (rev 464) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 22:52:08 UTC (rev 465) @@ -33,10 +33,7 @@ #include #include #include -#include -#include "libvarnish.h" -#include "heritage.h" #include "shmlog.h" #include "vcl.h" #include "cache.h" @@ -521,7 +518,7 @@ CNT_Session(struct worker *w, struct sess *sp) { - time(&sp->t0); + sp->t0 = time(NULL); sp->vcl = VCL_Get(); for (sp->step = STP_RECV; sp->step != STP_DONE; ) { Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-12 22:07:00 UTC (rev 464) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-12 22:52:08 UTC (rev 465) @@ -43,13 +43,15 @@ */ static void * -exp_hangman(void *arg __unused) +exp_hangman(void *arg) { struct object *o; time_t t; + (void)arg; + while (1) { - time (&t); + t = time(NULL); AZ(pthread_mutex_lock(&exp_mtx)); TAILQ_FOREACH(o, &exp_deathrow, deathrow) { if (o->ttl >= t) @@ -64,7 +66,7 @@ } if (o == NULL || o->ttl >= t || o->refcnt > 0) { AZ(pthread_mutex_unlock(&exp_mtx)); - sleep(1); + AZ(sleep(1)); continue; } TAILQ_REMOVE(&exp_deathrow, o, deathrow); @@ -84,19 +86,21 @@ */ static void * -exp_prefetch(void *arg __unused) +exp_prefetch(void *arg) { struct object *o; time_t t; struct sess sp; + (void)arg; + while (1) { - time(&t); + t = time(NULL); AZ(pthread_mutex_lock(&exp_mtx)); o = binheap_root(exp_heap); if (o == NULL || o->ttl > t + expearly) { AZ(pthread_mutex_unlock(&exp_mtx)); - sleep(1); + AZ(sleep(1)); continue; } binheap_delete(exp_heap, 0); @@ -121,20 +125,23 @@ /*--------------------------------------------------------------------*/ static int -object_cmp(void *priv __unused, void *a, void *b) +object_cmp(void *priv, void *a, void *b) { struct object *aa, *bb; + (void)priv; + aa = a; bb = b; return (aa->ttl < bb->ttl); } static void -object_update(void *priv __unused, void *p, unsigned u) +object_update(void *priv, void *p, unsigned u) { struct object *o = p; + (void)priv; o->heap_idx = u; } Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-12 22:07:00 UTC (rev 464) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-12 22:52:08 UTC (rev 465) @@ -12,7 +12,6 @@ #include #include #include -#include #include "libvarnish.h" #include "shmlog.h" @@ -29,8 +28,9 @@ #define CHUNK_PREALLOC (128 * 1024) /*--------------------------------------------------------------------*/ + static int -fetch_straight(struct sess *sp, int fd, struct http *hp, char *b) +fetch_straight(const struct sess *sp, int fd, struct http *hp, char *b) { int i; char *e; @@ -70,7 +70,7 @@ /* XXX: Cleanup. It must be possible somehow :-( */ static int -fetch_chunked(struct sess *sp, int fd, struct http *hp) +fetch_chunked(const struct sess *sp, int fd, struct http *hp) { int i; char *b, *q, *e; @@ -180,7 +180,7 @@ #include static int -fetch_eof(struct sess *sp, int fd, struct http *hp) +fetch_eof(const struct sess *sp, int fd, struct http *hp) { int i; char *b, *e; @@ -222,7 +222,7 @@ sp->obj->len += i; } - if (st != NULL && stevedore->trim != NULL) + if (stevedore->trim != NULL) stevedore->trim(st, st->len); return (1); @@ -294,7 +294,7 @@ http_BuildSbuf(vc->fd, Build_Fetch, w->sb, sp->http); i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); - time(&sp->t_req); + sp->t_req = time(NULL); /* XXX: copy any body ?? */ @@ -303,8 +303,8 @@ * XXX: read(2) the header */ http_RecvHead(hp, vc->fd, w->eb, NULL, NULL); - event_base_loop(w->eb, 0); - time(&sp->t_resp); + (void)event_base_loop(w->eb, 0); + sp->t_resp = time(NULL); assert(http_DissectResponse(hp, vc->fd) == 0); sp->vbc = vc; sp->bkd_http = hp; Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-12 22:07:00 UTC (rev 464) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-12 22:52:08 UTC (rev 465) @@ -9,7 +9,6 @@ #include #include #include -#include #include "libvarnish.h" #include "shmlog.h" Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-12 22:07:00 UTC (rev 464) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-12 22:52:08 UTC (rev 465) @@ -11,7 +11,6 @@ #include #include #include -#include #include "libvarnish.h" #include "shmlog.h" Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-12 22:07:00 UTC (rev 464) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-12 22:52:08 UTC (rev 465) @@ -7,7 +7,6 @@ #include #include #include -#include #include "libvarnish.h" #include "shmlog.h" Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-12 22:07:00 UTC (rev 464) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-12 22:52:08 UTC (rev 465) @@ -9,7 +9,6 @@ #include #include #include -#include #include "libvarnish.h" #include "heritage.h" Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-12 22:07:00 UTC (rev 464) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-12 22:52:08 UTC (rev 465) @@ -5,10 +5,8 @@ #include /* XXX: for NULL ?? */ #include #include -#include #include "libvarnish.h" -#include "sbuf.h" #include "cache.h" Modified: trunk/varnish-cache/bin/varnishd/flint.lnt =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.lnt 2006-07-12 22:07:00 UTC (rev 464) +++ trunk/varnish-cache/bin/varnishd/flint.lnt 2006-07-12 22:52:08 UTC (rev 465) @@ -6,19 +6,33 @@ -e763 // Redundant declaration for symbol '...' previously declared -e726 // Extraneous comma ignored -e728 // Symbol ... not explicitly initialized +-e716 // while(1) ... +-e785 // Too few initializers for aggregate +-emacro(740, TAILQ_PREV) // Unusual pointer cast (incompatible indirect types) +-emacro((826), TAILQ_PREV) // Suspicious pointer-to-pointer conversion (area too small) + + +-esym(534, printf) // Ignoring return value of function -esym(534, fprintf) // Ignoring return value of function -esym(534, memset) // Ignoring return value of function +-esym(534, memcpy) // Ignoring return value of function -esym(534, sbuf_printf) // Ignoring return value of function +-esym(534, sbuf_cat) // Ignoring return value of function // cache.h -emacro(506, INCOMPL) // Constant value Boolean +// cache_center.c +-efunc(525, CNT_Session) // Negative indentation from line + // cache_vcl.c -efunc(525, vcl_handlingname) // Negative indentation from line -esym(528, vcl_handlingname) // Not referenced -e641 // Converting enum 'cli_status_e' to int +// storage_file.c + // Review all below this line // -printf_code( H, void *, unsigned) @@ -26,7 +40,13 @@ // -printf_code( jx, long long unsigned) // +-e767 // Macro redef (system queue.h vs ours ) +-e574 // Signed-unsigned mix with relational +-e712 // Loss of precision (assignment) (long long to +-e747 // Significant prototype coercion (arg. no. 2) long +-e713 // Loss of precision (assignment) (unsigned long long to long long) + // // -e737 // Loss of sign in promotion from int to unsigned int @@ -40,18 +60,13 @@ // -e506 // Constant value boolean // -e527 // Unreachable code at token 'return' // -e774 // Boolean within 'if' always evaluates to False -// -e713 // Loss of precision (assignment) (unsigned long long to long long) -// -e574 // Signed-unsigned mix with relational // // -e525 // Negative indentation from line 90 // -e539 // Did not expect positive indentation // -e725 // Expected positive indentation from line 136 // -e734 // Loss of precision (assignment) (31 bits to 8 bits) -// -e747 // Significant prototype coercion (arg. no. 2) long -// -e712 // Loss of precision (assignment) (long long to // // -// -e785 // Too few initializers for aggregate // // -e766 // Header file '../../include/libvarnish.h' not used in module // @@ -59,6 +74,5 @@ // // -e788 // enum constant 'HND_Unclass' not used within defaulted switch // -// -e716 // while(1) ... // // -e786 // String concatenation within initializer Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_child.c 2006-07-12 22:07:00 UTC (rev 464) +++ trunk/varnish-cache/bin/varnishd/mgt_child.c 2006-07-12 22:52:08 UTC (rev 465) @@ -16,7 +16,7 @@ #include #include -#include +#include "queue.h" #include #include @@ -58,10 +58,12 @@ */ static void -std_rdcb(struct bufferevent *bev, void *arg __unused) +std_rdcb(struct bufferevent *bev, void *arg) { const char *p; + (void)arg; + while (1) { p = evbuffer_readline(bev->input); if (p == NULL) @@ -74,7 +76,8 @@ std_wrcb(struct bufferevent *bev, void *arg) { - printf("%s(%p, %p)\n", __func__, (void*)bev, arg); + printf("%s(%p, %p)\n", + (const char *)__func__, (void*)bev, arg); exit (2); } @@ -82,7 +85,8 @@ std_excb(struct bufferevent *bev, short what, void *arg) { - printf("%s(%p, %d, %p)\n", __func__, (void*)bev, what, arg); + printf("%s(%p, %d, %p)\n", + (const char *)__func__, (void*)bev, what, arg); exit (2); } @@ -107,7 +111,7 @@ cli_encode_string(child_cli1->output, cr->argv[u]); } evbuffer_add_printf(child_cli1->output, "\n"); - bufferevent_enable(child_cli1, EV_WRITE); + AZ(bufferevent_enable(child_cli1, EV_WRITE)); } void @@ -132,12 +136,14 @@ } static void -cli_rdcb(struct bufferevent *bev, void *arg __unused) +cli_rdcb(struct bufferevent *bev, void *arg) { const char *p; char **av; struct creq *cr; + (void)arg; + p = evbuffer_readline(bev->input); if (p == NULL) return; @@ -167,32 +173,41 @@ cli_excb(struct bufferevent *bev, short what, void *arg) { - printf("%s(%p, %d, %p)\n", __func__, (void*)bev, what, arg); + printf("%s(%p, %d, %p)\n", + (const char *)__func__, (void*)bev, what, arg); exit (2); } /*--------------------------------------------------------------------*/ static void -child_pingpong_ccb(unsigned u __unused, const char *r __unused, void *priv __unused) +child_pingpong_ccb(unsigned u, const char *r, void *priv) { + (void)u; + (void)r; + (void)priv; + /* XXX: reset keepalive timer */ } static void -child_pingpong(int a __unused, short b __unused, void *c __unused) +child_pingpong(int a, short b, void *c) { time_t t; struct timeval tv; - time(&t); + (void)a; + (void)b; + (void)c; + + t = time(NULL); mgt_child_request(child_pingpong_ccb, NULL, NULL, "ping %ld", t); if (1) { tv.tv_sec = 10; tv.tv_usec = 0; - evtimer_del(&ev_child_pingpong); - evtimer_add(&ev_child_pingpong, &tv); + AZ(evtimer_del(&ev_child_pingpong)); + AZ(evtimer_add(&ev_child_pingpong, &tv)); } } @@ -237,22 +252,22 @@ child_std = bufferevent_new(child_fds[0], std_rdcb, std_wrcb, std_excb, NULL); assert(child_std != NULL); - bufferevent_base_set(mgt_eb, child_std); + AZ(bufferevent_base_set(mgt_eb, child_std)); bufferevent_enable(child_std, EV_READ); child_cli0 = bufferevent_new(heritage.fds[0], cli_rdcb, cli_wrcb, cli_excb, NULL); assert(child_cli0 != NULL); - bufferevent_base_set(mgt_eb, child_cli0); + AZ(bufferevent_base_set(mgt_eb, child_cli0)); bufferevent_enable(child_cli0, EV_READ); child_cli1 = bufferevent_new(heritage.fds[3], cli_rdcb, cli_wrcb, cli_excb, NULL); assert(child_cli1 != NULL); - bufferevent_base_set(mgt_eb, child_cli1); + AZ(bufferevent_base_set(mgt_eb, child_cli1)); evtimer_set(&ev_child_pingpong, child_pingpong, NULL); - event_base_set(mgt_eb, &ev_child_pingpong); + AZ(event_base_set(mgt_eb, &ev_child_pingpong)); child_pingpong(0, 0, NULL); } @@ -312,12 +327,12 @@ bufferevent_free(child_std); /* XXX: is this enough ? */ child_std = NULL; - close(heritage.fds[0]); - close(heritage.fds[1]); - close(heritage.fds[2]); - close(heritage.fds[3]); - close(child_fds[0]); - close(child_fds[1]); + AZ(close(heritage.fds[0])); + AZ(close(heritage.fds[1])); + AZ(close(heritage.fds[2])); + AZ(close(heritage.fds[3])); + AZ(close(child_fds[0])); + AZ(close(child_fds[1])); if (desired == H_START) start_child(); From phk at projects.linpro.no Wed Jul 12 23:30:49 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 13 Jul 2006 01:30:49 +0200 (CEST) Subject: r466 - trunk/varnish-cache/bin/varnishd Message-ID: <20060712233049.EA7CF1EC38D@projects.linpro.no> Author: phk Date: 2006-07-13 01:30:49 +0200 (Thu, 13 Jul 2006) New Revision: 466 Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_main.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishd/cli_event.c trunk/varnish-cache/bin/varnishd/flint.lnt trunk/varnish-cache/bin/varnishd/flint.sh trunk/varnish-cache/bin/varnishd/rfc2616.c trunk/varnish-cache/bin/varnishd/shmlog.c trunk/varnish-cache/bin/varnishd/varnishd.c Log: More Flexelinting Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-12 22:52:08 UTC (rev 465) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-12 23:30:49 UTC (rev 466) @@ -92,7 +92,7 @@ } if (o != NULL) { AZ(pthread_mutex_unlock(&oh->mtx)); - hash->deref(oh); + (void)hash->deref(oh); return (o); } Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-12 22:52:08 UTC (rev 465) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-12 23:30:49 UTC (rev 466) @@ -338,12 +338,14 @@ #include static void -http_read_f(int fd, short event __unused, void *arg) +http_read_f(int fd, short event, void *arg) { struct http *hp = arg; unsigned l; int i, ret = 0; + (void)event; + l = hp->e - hp->v; if (l <= 1) { VSL(SLT_HttpError, fd, "Received too much"); @@ -391,7 +393,8 @@ assert(hp->v <= hp->e); assert(hp->t <= hp->v); if (0) - VSL(SLT_Debug, fd, "Recv t %u v %u", hp->t - hp->s, hp->v - hp->s); + VSL(SLT_Debug, fd, "Recv t %u v %u", + hp->t - hp->s, hp->v - hp->s); if (hp->t > hp->s && hp->t < hp->v) { l = hp->v - hp->t; memmove(hp->s, hp->t, l); @@ -409,8 +412,8 @@ hp->callback = func; hp->arg = arg; event_set(&hp->ev, fd, EV_READ | EV_PERSIST, http_read_f, hp); - event_base_set(eb, &hp->ev); - event_add(&hp->ev, NULL); /* XXX: timeout */ + AZ(event_base_set(eb, &hp->ev)); + AZ(event_add(&hp->ev, NULL)); /* XXX: timeout */ } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-12 22:52:08 UTC (rev 465) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-12 23:30:49 UTC (rev 466) @@ -20,15 +20,13 @@ struct stevedore *stevedore; -struct varnish_stats *VSL_stats; - /*--------------------------------------------------------------------*/ static void timer_keepalive(int a, short b, void *c) { - printf("%s(%d, %d, %p)\n", __func__, a, (int)b, c); + printf("%s(%d, %d, %p)\n", (const char *)__func__, a, (int)b, c); printf("Heeellloooo ? Ohh bother...\n"); exit (1); } @@ -41,8 +39,8 @@ tv.tv_sec = 30; tv.tv_usec = 0; - evtimer_del(&ev_keepalive); - evtimer_add(&ev_keepalive, &tv); + AZ(evtimer_del(&ev_keepalive)); + AZ(evtimer_add(&ev_keepalive, &tv)); } /*--------------------------------------------------------------------*/ @@ -69,7 +67,7 @@ if (av[2] != NULL) { /* XXX: check clock skew is pointless here */ } - time(&t); + t = time(NULL); cli_out(cli, "PONG %ld\n", t); } @@ -126,7 +124,7 @@ cli = cli_setup(eb, heritage.fds[2], heritage.fds[1], 0, cli_proto); evtimer_set(&ev_keepalive, timer_keepalive, NULL); - event_base_set(eb, &ev_keepalive); + AZ(event_base_set(eb, &ev_keepalive)); arm_keepalive(); printf("Ready\n"); Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-12 22:52:08 UTC (rev 465) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-12 23:30:49 UTC (rev 466) @@ -12,7 +12,6 @@ #include #include -#include "libvarnish.h" #include "shmlog.h" #include "cache.h" @@ -204,7 +203,7 @@ */ hp = vc->http; http_RecvHead(hp, vc->fd, w->eb, NULL, NULL); - event_base_loop(w->eb, 0); + (void)event_base_loop(w->eb, 0); http_DissectResponse(hp, vc->fd); sp->bkd_http = hp; Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-12 22:52:08 UTC (rev 465) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-12 23:30:49 UTC (rev 466) @@ -18,24 +18,26 @@ }; static void -rdf(int fd, short event __unused, void *arg) +rdf(int fd, short event, void *arg) { int i, j; struct edir *ep; char buf[BUFSIZ]; + (void)event; + ep = arg; i = read(fd, buf, sizeof buf); if (i <= 0) { shutdown(fd, SHUT_RD); shutdown(ep->fd, SHUT_WR); - event_del(&ep->ev); + AZ(event_del(&ep->ev)); } else { j = write(ep->fd, buf, i); if (i != j) { shutdown(fd, SHUT_WR); shutdown(ep->fd, SHUT_RD); - event_del(&ep->ev); + AZ(event_del(&ep->ev)); } } } @@ -67,12 +69,12 @@ e1.fd = vc->fd; e2.fd = sp->fd; event_set(&e1.ev, sp->fd, EV_READ | EV_PERSIST, rdf, &e1); - event_base_set(w->eb, &e1.ev); + AZ(event_base_set(w->eb, &e1.ev)); event_set(&e2.ev, vc->fd, EV_READ | EV_PERSIST, rdf, &e2); - event_base_set(w->eb, &e2.ev); - event_add(&e1.ev, NULL); - event_add(&e2.ev, NULL); - event_base_loop(w->eb, 0); + AZ(event_base_set(w->eb, &e2.ev)); + AZ(event_add(&e1.ev, NULL)); + AZ(event_add(&e2.ev, NULL)); + (void)event_base_loop(w->eb, 0); vca_close_session(sp, "pipe"); VBE_ClosedFd(vc); } Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-12 22:52:08 UTC (rev 465) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-12 23:30:49 UTC (rev 466) @@ -79,7 +79,7 @@ } /* If we are a dynamic thread, time out and die */ - clock_gettime(CLOCK_REALTIME, &ts); + AZ(clock_gettime(CLOCK_REALTIME, &ts)); ts.tv_sec += heritage.wthread_timeout; if (pthread_cond_timedwait(&w->cv, &wrk_mtx, &ts)) { VSL_stats->n_wrk--; @@ -102,7 +102,7 @@ struct worker *w; pthread_t tp; - time(&sp->t_req); + sp->t_req = time(NULL); /* * No locking necessary, we're serialized in the acceptor thread Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-12 22:52:08 UTC (rev 465) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-12 23:30:49 UTC (rev 466) @@ -8,14 +8,10 @@ #include #include #include -#include -#include "cli.h" -#include "cli_priv.h" #include "shmlog.h" #include "vrt.h" #include "vcl.h" -#include "libvarnish.h" #include "cache.h" /*--------------------------------------------------------------------*/ @@ -24,6 +20,7 @@ VRT_error(struct sess *sp, unsigned err, const char *str) { + (void)sp; VSL(SLT_Debug, 0, "VCL_error(%u, %s)", err, str); } Modified: trunk/varnish-cache/bin/varnishd/cli_event.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cli_event.c 2006-07-12 22:52:08 UTC (rev 465) +++ trunk/varnish-cache/bin/varnishd/cli_event.c 2006-07-12 23:30:49 UTC (rev 466) @@ -105,16 +105,17 @@ sbuf_finish(cli->sb); /* XXX: syslog results ? */ encode_output(cli); - bufferevent_enable(cli->bev1, EV_WRITE); + AZ(bufferevent_enable(cli->bev1, EV_WRITE)); } } static void -wrcb(struct bufferevent *bev __unused, void *arg) +wrcb(struct bufferevent *bev, void *arg) { struct cli *cli = arg; - bufferevent_disable(cli->bev1, EV_WRITE); + (void)bev; + AZ(bufferevent_disable(cli->bev1, EV_WRITE)); } static void @@ -133,20 +134,20 @@ cli->bev0 = bufferevent_new(fdr, rdcb, wrcb, excb, cli); assert(cli->bev0 != NULL); - bufferevent_base_set(eb, cli->bev0); + AZ(bufferevent_base_set(eb, cli->bev0)); if (fdr == fdw) cli->bev1 = cli->bev0; else cli->bev1 = bufferevent_new(fdw, rdcb, wrcb, excb, cli); assert(cli->bev1 != NULL); - bufferevent_base_set(eb, cli->bev1); + AZ(bufferevent_base_set(eb, cli->bev1)); cli->sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(cli->sb != NULL); cli->verbose = ver; cli->cli_proto = cli_proto; - bufferevent_enable(cli->bev0, EV_READ); + AZ(bufferevent_enable(cli->bev0, EV_READ)); return (cli); } @@ -155,7 +156,7 @@ { cli->suspend = 1; - bufferevent_disable(cli->bev0, EV_READ); + AZ(bufferevent_disable(cli->bev0, EV_READ)); } void @@ -164,8 +165,8 @@ sbuf_finish(cli->sb); /* XXX: syslog results ? */ encode_output(cli); - bufferevent_enable(cli->bev1, EV_WRITE); + AZ(bufferevent_enable(cli->bev1, EV_WRITE)); cli->suspend = 0; - bufferevent_enable(cli->bev0, EV_READ); + AZ(bufferevent_enable(cli->bev0, EV_READ)); } Modified: trunk/varnish-cache/bin/varnishd/flint.lnt =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.lnt 2006-07-12 22:52:08 UTC (rev 465) +++ trunk/varnish-cache/bin/varnishd/flint.lnt 2006-07-12 23:30:49 UTC (rev 466) @@ -8,6 +8,7 @@ -e728 // Symbol ... not explicitly initialized -e716 // while(1) ... -e785 // Too few initializers for aggregate +-e786 // String concatenation within initializer -emacro(740, TAILQ_PREV) // Unusual pointer cast (incompatible indirect types) -emacro((826), TAILQ_PREV) // Suspicious pointer-to-pointer conversion (area too small) @@ -17,6 +18,8 @@ -esym(534, fprintf) // Ignoring return value of function -esym(534, memset) // Ignoring return value of function -esym(534, memcpy) // Ignoring return value of function +-esym(534, memmove) // Ignoring return value of function +-esym(534, strcpy) // Ignoring return value of function -esym(534, sbuf_printf) // Ignoring return value of function -esym(534, sbuf_cat) // Ignoring return value of function @@ -31,48 +34,19 @@ -esym(528, vcl_handlingname) // Not referenced -e641 // Converting enum 'cli_status_e' to int -// storage_file.c +// Review all below this line /////////////////////////////////////////////// -// Review all below this line - -// -printf_code( H, void *, unsigned) -// -printf_code( ju, long long unsigned) -// -printf_code( jx, long long unsigned) -// - -e767 // Macro redef (system queue.h vs ours ) -e574 // Signed-unsigned mix with relational -e712 // Loss of precision (assignment) (long long to -e747 // Significant prototype coercion (arg. no. 2) long -e713 // Loss of precision (assignment) (unsigned long long to long long) +-e506 // Constant value boolean +-e818 // Pointer parameter '...' could be declared as pointing to const +-e774 // Boolean within 'if' always evaluates to False +-e534 // Ignoring return value of function +-e557 // unrecog format +-e732 // Loss of sign (arg. no. 2) (int to unsigned +-e737 // [45] Loss of sign in promotion from int to unsigned -// -// --e737 // Loss of sign in promotion from int to unsigned int --e732 // Loss of sign (arg. no. 2) (int to unsigned int) -// -e715 // Symbol 'arg' (line 43) not referenced -// -e818 // Pointer parameter '...' could be declared as pointing to const -// -// -e534 // Ignoring return value of function -// -e767 // macro 'LIST_INIT' was defined differently -// -// -e506 // Constant value boolean -// -e527 // Unreachable code at token 'return' -// -e774 // Boolean within 'if' always evaluates to False -// -// -e525 // Negative indentation from line 90 -// -e539 // Did not expect positive indentation -// -e725 // Expected positive indentation from line 136 -// -e734 // Loss of precision (assignment) (31 bits to 8 bits) -// -// -// -// -e766 // Header file '../../include/libvarnish.h' not used in module -// -// -e773 // Expression-like macro 'VCL_FARGS' not parenthesized -// -// -e788 // enum constant 'HND_Unclass' not used within defaulted switch -// -// -// -e786 // String concatenation within initializer Modified: trunk/varnish-cache/bin/varnishd/flint.sh =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.sh 2006-07-12 22:52:08 UTC (rev 465) +++ trunk/varnish-cache/bin/varnishd/flint.sh 2006-07-12 23:30:49 UTC (rev 466) @@ -1,9 +1,30 @@ #!/bin/sh +T=/tmp/_$$ flexelint \ -I/usr/include \ -I. \ -I../../include \ -I../../contrib/libevent \ flint.lnt \ - *.c + *.c > $T 2>&1 + +for t in Error Warning Info +do + sed -n "/$t [0-9][0-9][0-9]:/s/.*\($t [0-9][0-9][0-9]\).*/\1/p" $T +done | awk ' +$2 == 830 { next } +$2 == 831 { next } + { + i=$2"_"$1 + h[i]++ + n++ + } +END { + printf "%5d %s\n", n, "Total" + for (i in h) + printf "%5d %s\n", h[i], i + } +' | sort -rn + +cat $T Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-12 22:52:08 UTC (rev 465) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-12 23:30:49 UTC (rev 466) @@ -78,6 +78,8 @@ time_t h_date, h_expires, ttd; char *p; + (void)t_resp; /* XXX */ + retirement_age = INT_MAX; u1 = u2 = 0; Modified: trunk/varnish-cache/bin/varnishd/shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-12 22:52:08 UTC (rev 465) +++ trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-12 23:30:49 UTC (rev 466) @@ -69,8 +69,8 @@ if (loghead->ptr + 4 + l + 1 > loghead->size) vsl_wrap(); p = logstart + loghead->ptr; - p[1] = l; - p[2] = id >> 8; + p[1] = l & 0xff; + p[2] = (id >> 8) & 0xff; p[3] = id & 0xff; memcpy(p + 4, b, l); p[4 + l] = SLT_ENDMARKER; Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-12 22:52:08 UTC (rev 465) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-12 23:30:49 UTC (rev 466) @@ -91,10 +91,11 @@ } static void -m_cli_func_passthrough(struct cli *cli, char **av, void *priv __unused) +m_cli_func_passthrough(struct cli *cli, char **av, void *priv) { (void)av; + (void)priv; cli_suspend(cli); mgt_child_request(cli_passthrough_cb, cli, &av[2], av[1]); @@ -277,7 +278,7 @@ if (av[2] != NULL) { cli_out(cli, "Got your %s\n", av[2]); } - time(&t); + t = time(NULL); cli_out(cli, "PONG %ld\n", t); } @@ -340,8 +341,8 @@ cli = cli_setup(mgt_eb, 0, 1, 1, cli_proto); signal_set(&e_sigchld, SIGCHLD, mgt_sigchld, NULL); - event_base_set(mgt_eb, &e_sigchld); - signal_add(&e_sigchld, NULL); + AZ(event_base_set(mgt_eb, &e_sigchld)); + AZ(signal_add(&e_sigchld, NULL)); mgt_child_start(); @@ -505,7 +506,7 @@ const char *sflag = "file"; const char *hflag = "classic"; - register_printf_render_std((const unsigned char *)"HVQ"); + (void)register_printf_render_std((const unsigned char *)"HVQ"); setbuf(stdout, NULL); setbuf(stderr, NULL); From phk at projects.linpro.no Fri Jul 14 10:16:27 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 14 Jul 2006 12:16:27 +0200 (CEST) Subject: r467 - trunk/varnish-cache/bin/varnishd Message-ID: <20060714101627.9AAF11EC380@projects.linpro.no> Author: phk Date: 2006-07-14 12:16:27 +0200 (Fri, 14 Jul 2006) New Revision: 467 Added: trunk/varnish-cache/bin/varnishd/cache_session.c Modified: trunk/varnish-cache/bin/varnishd/Makefile.am trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_main.c Log: Move session management to new file (cache_session, SES prefix) in preparation of adding client tracking. Move the iovec's from the session to the worker and give the session a pointer to the worker so we can avoid passing it around as argument. Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-12 23:30:49 UTC (rev 466) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-14 10:16:27 UTC (rev 467) @@ -27,6 +27,7 @@ cache_pass.c \ cache_pipe.c \ cache_response.c \ + cache_session.c \ cache_vcl.c \ cache_vrt.c \ cli_event.c \ Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-12 23:30:49 UTC (rev 466) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-14 10:16:27 UTC (rev 467) @@ -4,6 +4,7 @@ #include #include +#include #include #include "queue.h" @@ -13,6 +14,8 @@ #include "vcl_returns.h" #include "common.h" +#define MAX_IOVS 10 + struct event_base; struct cli; struct sbuf; @@ -66,6 +69,10 @@ unsigned nbr; pthread_cond_t cv; TAILQ_ENTRY(worker) list; + + struct iovec iov[MAX_IOVS]; + unsigned niov; + size_t liov; }; struct workreq { @@ -147,12 +154,24 @@ TAILQ_HEAD(,object) objects; }; +/* -------------------------------------------------------------------*/ + +struct client { + TAILQ_ENTRY(client) list; + unsigned nsess; + char addr[TCP_ADDRBUFFSIZE]; + uint64_t bytes; +}; + struct sess { int fd; unsigned xid; + struct worker *wrk; + /* formatted ascii client address */ char addr[TCP_ADDRBUFFSIZE]; + struct client *client; /* HTTP request */ struct http *http; @@ -270,6 +289,12 @@ void WRK_Init(void); void WRK_QueueSession(struct sess *sp); +/* cache_session.c [SES] */ +void SES_Init(void); +struct sess *SES_New(struct sockaddr *addr, unsigned len); +void SES_Delete(const struct sess *sp); + + /* cache_shmlog.c */ void VSL_Init(void); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-12 23:30:49 UTC (rev 466) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-14 10:16:27 UTC (rev 467) @@ -32,51 +32,10 @@ static pthread_t vca_thread; -#define SESS_IOVS 10 - static struct event accept_e[2 * HERITAGE_NSOCKS]; static TAILQ_HEAD(,sess) sesshead = TAILQ_HEAD_INITIALIZER(sesshead); -struct sessmem { - struct sess sess; - struct iovec iov[SESS_IOVS]; - int niov; - size_t liov; - struct http http; - char *http_hdr; -}; -/*--------------------------------------------------------------------*/ - -static struct sess * -vca_new_sess(void) -{ - struct sessmem *sm; - - sm = calloc( - sizeof *sm + - heritage.mem_http_headers * sizeof sm->http_hdr + - heritage.mem_http_headerspace + - heritage.mem_workspace, - 1); - if (sm == NULL) - return (NULL); - VSL_stats->n_sess++; - sm->sess.mem = sm; - sm->sess.http = &sm->http; - http_Init(&sm->http, (void *)(sm + 1)); - return (&sm->sess); -} - -static void -vca_delete_sess(const struct sess *sp) -{ - - VSL_stats->n_sess--; - free(sp->mem); -} - - /*-------------------------------------------------------------------- * Write data to client * We try to use writev() if possible in order to minimize number of @@ -89,13 +48,13 @@ { int i; - if (sp->fd < 0 || sp->mem->niov == 0) + if (sp->fd < 0 || sp->wrk->niov == 0) return; - i = writev(sp->fd, sp->mem->iov, sp->mem->niov); - if (i != sp->mem->liov) + i = writev(sp->fd, sp->wrk->iov, sp->wrk->niov); + if (i != sp->wrk->liov) vca_close_session(sp, "remote closed"); - sp->mem->liov = 0; - sp->mem->niov = 0; + sp->wrk->liov = 0; + sp->wrk->niov = 0; } void @@ -104,13 +63,13 @@ if (sp->fd < 0 || len == 0) return; - if (sp->mem->niov == SESS_IOVS) + if (sp->wrk->niov == MAX_IOVS) vca_flush(sp); if (sp->fd < 0) return; - sp->mem->iov[sp->mem->niov].iov_base = ptr; - sp->mem->iov[sp->mem->niov++].iov_len = len; - sp->mem->liov += len; + sp->wrk->iov[sp->wrk->niov].iov_base = ptr; + sp->wrk->iov[sp->wrk->niov++].iov_len = len; + sp->wrk->liov += len; } void @@ -145,9 +104,9 @@ continue; } st->stevedore->send(st, sp, - sp->mem->iov, sp->mem->niov, sp->mem->liov); - sp->mem->niov = 0; - sp->mem->liov = 0; + sp->wrk->iov, sp->wrk->niov, sp->wrk->liov); + sp->wrk->niov = 0; + sp->wrk->liov = 0; } assert(u == sp->obj->len); } @@ -221,16 +180,17 @@ (void)arg; VSL_stats->client_conn++; - sp = vca_new_sess(); - assert(sp != NULL); /* XXX handle */ - l = sizeof addr; - sp->fd = accept(fd, addr, &l); - if (sp->fd < 0) { - vca_delete_sess(sp); + i = accept(fd, addr, &l); + if (i < 0) { return; } + sp = SES_New(addr, l); + assert(sp != NULL); /* XXX handle */ + + sp->fd = i; + #ifdef SO_NOSIGPIPE /* XXX Linux */ i = 1; AZ(setsockopt(sp->fd, SOL_SOCKET, SO_NOSIGPIPE, &i, sizeof i)); @@ -315,7 +275,7 @@ VSL(SLT_SessionReuse, sp->fd, "%s", sp->addr); assert(sizeof sp == write(pipes[1], &sp, sizeof sp)); } else { - vca_delete_sess(sp); + SES_Delete(sp); } } Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-12 23:30:49 UTC (rev 466) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-14 10:16:27 UTC (rev 467) @@ -521,6 +521,8 @@ sp->t0 = time(NULL); sp->vcl = VCL_Get(); + sp->wrk = w; + for (sp->step = STP_RECV; sp->step != STP_DONE; ) { switch (sp->step) { #define STEP(l,u) \ Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-12 23:30:49 UTC (rev 466) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-14 10:16:27 UTC (rev 467) @@ -105,6 +105,8 @@ VCL_Init(); VCL_Load(heritage.vcl_file, "boot", NULL); + SES_Init(); + VBE_Init(); VSL_Init(); WRK_Init(); Added: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-12 23:30:49 UTC (rev 466) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-14 10:16:27 UTC (rev 467) @@ -0,0 +1,80 @@ +/* + * $Id$ + * + * Session and Client management. + * + * The client structures are kept around only as a convenience feature to + * make it possible to track down offenders and misconfigured caches. + * As such it is pure overhead and we do not want to spend too much time + * on maintaining it. + * + * We identify clients by their address only and disregard the port number, + * because the desired level of granularity is "whois is abuse@ or tech-c@ + * in the RIPE database. + */ + +#include +#include + +#include "heritage.h" +#include "cache.h" +#include "shmlog.h" + +#define CLIENT_HASH 256 + +/*--------------------------------------------------------------------*/ + +struct sessmem { + struct sess sess; + struct http http; + char *http_hdr; +}; + +/*--------------------------------------------------------------------*/ + +TAILQ_HEAD(clienthead ,client); + +static struct clienthead client_hash[CLIENT_HASH]; + +/*--------------------------------------------------------------------*/ + +struct sess * +SES_New(struct sockaddr *addr, unsigned len) +{ + struct sessmem *sm; + + (void)addr; /* XXX */ + (void)len; /* XXX */ + sm = calloc( + sizeof *sm + + heritage.mem_http_headers * sizeof sm->http_hdr + + heritage.mem_http_headerspace + + heritage.mem_workspace, + 1); + if (sm == NULL) + return (NULL); + VSL_stats->n_sess++; + sm->sess.mem = sm; + sm->sess.http = &sm->http; + http_Init(&sm->http, (void *)(sm + 1)); + return (&sm->sess); +} + +void +SES_Delete(const struct sess *sp) +{ + + VSL_stats->n_sess--; + free(sp->mem); +} + +/*--------------------------------------------------------------------*/ + +void +SES_Init() +{ + int i; + + for (i = 0; i < CLIENT_HASH; i++) + TAILQ_INIT(&client_hash[i]); +} From phk at projects.linpro.no Fri Jul 14 10:34:06 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 14 Jul 2006 12:34:06 +0200 (CEST) Subject: r468 - trunk/varnish-cache/bin/varnishd Message-ID: <20060714103406.E48A31EC381@projects.linpro.no> Author: phk Date: 2006-07-14 12:34:06 +0200 (Fri, 14 Jul 2006) New Revision: 468 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/common.h trunk/varnish-cache/bin/varnishd/tcp.c Log: Rework the way we do ascii representations of addresses Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-14 10:16:27 UTC (rev 467) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-14 10:34:06 UTC (rev 468) @@ -159,7 +159,7 @@ struct client { TAILQ_ENTRY(client) list; unsigned nsess; - char addr[TCP_ADDRBUFFSIZE]; + char addr[TCP_ADDRBUFSIZE]; uint64_t bytes; }; @@ -170,7 +170,8 @@ struct worker *wrk; /* formatted ascii client address */ - char addr[TCP_ADDRBUFFSIZE]; + char addr[TCP_ADDRBUFSIZE]; + char port[TCP_PORTBUFSIZE]; struct client *client; /* HTTP request */ Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-14 10:16:27 UTC (rev 467) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-14 10:34:06 UTC (rev 468) @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -184,6 +185,8 @@ l = sizeof addr; i = accept(fd, addr, &l); if (i < 0) { + VSL(SLT_Debug, fd, "Accept failed errno=%d", errno); + /* XXX: stats ? */ return; } sp = SES_New(addr, l); @@ -201,8 +204,8 @@ AZ(setsockopt(sp->fd, SOL_SOCKET, SO_LINGER, &linger, sizeof linger)); #endif - TCP_name(addr, l, sp->addr); - VSL(SLT_SessionOpen, sp->fd, "%s", sp->addr); + TCP_name(addr, l, sp->addr, sizeof sp->addr, sp->port, sizeof sp->port); + VSL(SLT_SessionOpen, sp->fd, "%s %s", sp->addr, sp->port); (void)time(&sp->t_resp); TAILQ_INSERT_TAIL(&sesshead, sp, list); http_RecvHead(sp->http, sp->fd, evb, vca_callback, sp); @@ -272,7 +275,7 @@ { if (sp->fd >= 0) { - VSL(SLT_SessionReuse, sp->fd, "%s", sp->addr); + VSL(SLT_SessionReuse, sp->fd, "%s %s", sp->addr, sp->port); assert(sizeof sp == write(pipes[1], &sp, sizeof sp)); } else { SES_Delete(sp); Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-14 10:16:27 UTC (rev 467) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-14 10:34:06 UTC (rev 468) @@ -186,7 +186,8 @@ vbe_connect(struct backend *bp) { int s; - char buf[TCP_ADDRBUFFSIZE * 2 + 1], *p; + char abuf1[TCP_ADDRBUFSIZE], abuf2[TCP_ADDRBUFSIZE]; + char pbuf1[TCP_PORTBUFSIZE], pbuf2[TCP_PORTBUFSIZE]; struct addrinfo *ai; assert(bp != NULL); @@ -196,11 +197,10 @@ if (s < 0) return (s); - TCP_myname(s, buf); - p = strchr(buf, '\0'); - *p++ = ' '; - TCP_name(ai->ai_addr, ai->ai_addrlen, p); - VSL(SLT_BackendOpen, s, buf); + TCP_myname(s, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); + TCP_name(ai->ai_addr, ai->ai_addrlen, + abuf2, sizeof abuf2, pbuf2, sizeof pbuf2); + VSL(SLT_BackendOpen, s, "%s %s %s %s", abuf1, pbuf1, abuf2, pbuf2); return (s); } Modified: trunk/varnish-cache/bin/varnishd/common.h =================================================================== --- trunk/varnish-cache/bin/varnishd/common.h 2006-07-14 10:16:27 UTC (rev 467) +++ trunk/varnish-cache/bin/varnishd/common.h 2006-07-14 10:34:06 UTC (rev 468) @@ -9,9 +9,9 @@ extern struct varnish_stats *VSL_stats; /* tcp.c */ -#define TCP_ADDRBUFFSIZE 64 /* Sizeof ascii representation */ +/* NI_MAXHOST and NI_MAXSERV are ridiculously long for numeric format */ +#define TCP_ADDRBUFSIZE 64 +#define TCP_PORTBUFSIZE 16 -void TCP_name(struct sockaddr *addr, unsigned l, char *buf); -void TCP_myname(int sock, char *buf); - - +void TCP_name(struct sockaddr *addr, unsigned l, char *abuf, unsigned alen, char *pbuf, unsigned plen); +void TCP_myname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen); Modified: trunk/varnish-cache/bin/varnishd/tcp.c =================================================================== --- trunk/varnish-cache/bin/varnishd/tcp.c 2006-07-14 10:16:27 UTC (rev 467) +++ trunk/varnish-cache/bin/varnishd/tcp.c 2006-07-14 10:34:06 UTC (rev 468) @@ -17,33 +17,31 @@ /*--------------------------------------------------------------------*/ void -TCP_name(struct sockaddr *addr, unsigned l, char *buf) +TCP_name(struct sockaddr *addr, unsigned l, char *abuf, unsigned alen, char *pbuf, unsigned plen) { int i; - char port[NI_MAXSERV]; - i = getnameinfo(addr, l, buf, TCP_ADDRBUFFSIZE, - port, sizeof port, NI_NUMERICHOST | NI_NUMERICSERV); + i = getnameinfo(addr, l, abuf, alen, pbuf, plen, + NI_NUMERICHOST | NI_NUMERICSERV); if (i) { printf("getnameinfo = %d %s\n", i, gai_strerror(i)); - strcpy(buf, "Conversion:Failed"); + strlcpy(abuf, "Conversion", alen); + strlcpy(pbuf, "Failed", plen); return; } - strlcat(buf, " ", TCP_ADDRBUFFSIZE); - strlcat(buf, port, TCP_ADDRBUFFSIZE); } /*--------------------------------------------------------------------*/ void -TCP_myname(int sock, char *buf) +TCP_myname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen) { struct sockaddr addr[2]; /* XXX: IPv6 hack */ socklen_t l; l = sizeof addr; AZ(getsockname(sock, addr, &l)); - TCP_name(addr, l, buf); + TCP_name(addr, l, abuf, alen, pbuf, plen); } /*--------------------------------------------------------------------*/ From phk at projects.linpro.no Fri Jul 14 11:20:10 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 14 Jul 2006 13:20:10 +0200 (CEST) Subject: r469 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060714112010.8528F1EC382@projects.linpro.no> Author: phk Date: 2006-07-14 13:20:10 +0200 (Fri, 14 Jul 2006) New Revision: 469 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_session.c trunk/varnish-cache/include/shmlog_tags.h trunk/varnish-cache/include/stat_field.h Log: Change "client" to "srcaddr", it's more descriptive. Add srcaddr management and start charging bytes to the srcaddr. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-14 10:34:06 UTC (rev 468) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-14 11:20:10 UTC (rev 469) @@ -156,11 +156,15 @@ /* -------------------------------------------------------------------*/ -struct client { - TAILQ_ENTRY(client) list; +struct srcaddr { + TAILQ_ENTRY(srcaddr) list; unsigned nsess; char addr[TCP_ADDRBUFSIZE]; + unsigned sum; + time_t first; + time_t ttl; uint64_t bytes; + struct srcaddrhead *sah; }; struct sess { @@ -172,7 +176,7 @@ /* formatted ascii client address */ char addr[TCP_ADDRBUFSIZE]; char port[TCP_PORTBUFSIZE]; - struct client *client; + struct srcaddr *srcaddr; /* HTTP request */ struct http *http; @@ -293,9 +297,11 @@ /* cache_session.c [SES] */ void SES_Init(void); struct sess *SES_New(struct sockaddr *addr, unsigned len); -void SES_Delete(const struct sess *sp); +void SES_Delete(struct sess *sp); +void SES_RefSrcAddr(struct sess *sp); +void SES_RelSrcAddr(struct sess *sp); +void SES_ChargeBytes(struct sess *sp, uint64_t bytes); - /* cache_shmlog.c */ void VSL_Init(void); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-14 10:34:06 UTC (rev 468) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-14 11:20:10 UTC (rev 469) @@ -79,6 +79,7 @@ struct storage *st; unsigned u = 0; char *r; + uint64_t bytes = 0; VSL(SLT_Status, sp->fd, "%u", sp->obj->response); @@ -96,6 +97,7 @@ sbuf_printf(w->sb, "\r\n"); sbuf_finish(w->sb); vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); + bytes += sbuf_len(w->sb); assert(http_GetReq(sp->http, &r)); if (!strcmp(r, "GET")) { TAILQ_FOREACH(st, &sp->obj->store, list) { @@ -111,6 +113,7 @@ } assert(u == sp->obj->len); } + SES_ChargeBytes(sp, bytes + u); vca_flush(sp); } Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-14 10:34:06 UTC (rev 468) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-14 11:20:10 UTC (rev 469) @@ -523,6 +523,8 @@ sp->wrk = w; + SES_RefSrcAddr(sp); + for (sp->step = STP_RECV; sp->step != STP_DONE; ) { switch (sp->step) { #define STEP(l,u) \ Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-14 10:34:06 UTC (rev 468) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-14 11:20:10 UTC (rev 469) @@ -1,5 +1,7 @@ /* * $Id$ + * + * XXX: charge bytes to srcaddr */ #include Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-14 10:34:06 UTC (rev 468) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-14 11:20:10 UTC (rev 469) @@ -1,5 +1,7 @@ /* * $Id$ + * + * XXX: charge bytes to srcaddr */ #include Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-14 10:34:06 UTC (rev 468) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-14 11:20:10 UTC (rev 469) @@ -3,24 +3,26 @@ * * Session and Client management. * - * The client structures are kept around only as a convenience feature to + * The srcaddr structures are kept around only as a convenience feature to * make it possible to track down offenders and misconfigured caches. * As such it is pure overhead and we do not want to spend too much time * on maintaining it. * - * We identify clients by their address only and disregard the port number, - * because the desired level of granularity is "whois is abuse@ or tech-c@ - * in the RIPE database. + * We identify srcaddrs instead of full addr+port because the desired level + * of granularity is "whois is abuse@ or tech-c@ in the RIPE database. */ #include +#include #include +#include "libvarnish.h" #include "heritage.h" +#include "shmlog.h" #include "cache.h" -#include "shmlog.h" #define CLIENT_HASH 256 +#define CLIENT_TTL 30 /*--------------------------------------------------------------------*/ @@ -32,10 +34,114 @@ /*--------------------------------------------------------------------*/ -TAILQ_HEAD(clienthead ,client); +TAILQ_HEAD(srcaddrhead ,srcaddr); -static struct clienthead client_hash[CLIENT_HASH]; +static struct srcaddrhead srcaddr_hash[CLIENT_HASH]; +static pthread_mutex_t ses_mtx; +/*-------------------------------------------------------------------- + * Assign a srcaddr to this session. + * + * We use a simple hash over the ascii representation of the address + * because it is nice and modular. I'm not sure how much improvement + * using the binary address would be anyway. + * + * Each hash bucket is sorted in least recently used order and if we + * need to make a new entry we recycle the first expired entry we find. + * If we find more expired entries during our search, we delete them. + */ + +void +SES_RefSrcAddr(struct sess *sp) +{ + unsigned u, v; + char *p; + struct srcaddr *c, *c2, *c3; + struct srcaddrhead *ch; + time_t now; + + for (u = 0, p = sp->addr; *p; p++) + u += u + *p; + v = u % CLIENT_HASH; + ch = &srcaddr_hash[v]; + now = time(NULL); + + AZ(pthread_mutex_lock(&ses_mtx)); + c3 = NULL; + TAILQ_FOREACH_SAFE(c, ch, list, c2) { + if (c->sum == u && !strcmp(c->addr, sp->addr)) { + c->nsess++; + c->ttl = now + CLIENT_TTL; + sp->srcaddr = c; + TAILQ_REMOVE(ch, c, list); + TAILQ_INSERT_TAIL(ch, c, list); + AZ(pthread_mutex_unlock(&ses_mtx)); + return; + } + if (c->nsess > 0 || c->ttl > now) + continue; + if (c3 == NULL) { + c3 = c; + continue; + } + TAILQ_REMOVE(ch, c2, list); + free(c2); + VSL_stats->n_srcaddr--; + } + if (c3 == NULL) { + c3 = malloc(sizeof *c3); + if (c3 != NULL) + VSL_stats->n_srcaddr++; + } else + TAILQ_REMOVE(ch, c3, list); + if (c3 != NULL) { + memset(c3, 0, sizeof *c3); + strcpy(c3->addr, sp->addr); + c3->sum = u; + c3->first = now; + c3->ttl = now + CLIENT_TTL; + c3->nsess = 1; + c3->sah = ch; + TAILQ_INSERT_TAIL(ch, c3, list); + } + sp->srcaddr = c3; + AZ(pthread_mutex_unlock(&ses_mtx)); +} + +void +SES_ChargeBytes(struct sess *sp, uint64_t bytes) +{ + struct srcaddr *sa; + time_t now; + + assert(sp->srcaddr != NULL); + sa = sp->srcaddr; + now = time(NULL); + AZ(pthread_mutex_lock(&ses_mtx)); +VSL(SLT_Debug, 0, "%ju", bytes); +VSL(SLT_Debug, 0, "%ju", sa->bytes); + sa->bytes += bytes; +VSL(SLT_Debug, 0, "%ju", sa->bytes); + sa->ttl = now + CLIENT_TTL; + TAILQ_REMOVE(sa->sah, sa, list); + TAILQ_INSERT_TAIL(sa->sah, sa, list); + bytes = sa->bytes; + AZ(pthread_mutex_unlock(&ses_mtx)); + VSL(SLT_SrcAddr, sp->fd, "%s %jd %d", + sa->addr, (intmax_t)(bytes), now - sa->first); +} + +void +SES_RelSrcAddr(struct sess *sp) +{ + + assert(sp->srcaddr != NULL); + AZ(pthread_mutex_lock(&ses_mtx)); + sp->srcaddr->nsess--; + sp->srcaddr = NULL; + AZ(pthread_mutex_unlock(&ses_mtx)); +} + /*--------------------------------------------------------------------*/ struct sess * @@ -61,10 +167,11 @@ } void -SES_Delete(const struct sess *sp) +SES_Delete(struct sess *sp) { VSL_stats->n_sess--; + SES_RelSrcAddr(sp); free(sp->mem); } @@ -76,5 +183,6 @@ int i; for (i = 0; i < CLIENT_HASH; i++) - TAILQ_INIT(&client_hash[i]); + TAILQ_INIT(&srcaddr_hash[i]); + AZ(pthread_mutex_init(&ses_mtx, NULL)); } Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-07-14 10:34:06 UTC (rev 468) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-07-14 11:20:10 UTC (rev 469) @@ -11,6 +11,7 @@ SLTM(Debug) SLTM(Error) SLTM(CLI) +SLTM(SrcAddr) SLTM(SessionOpen) SLTM(SessionReuse) SLTM(SessionClose) Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2006-07-14 10:34:06 UTC (rev 468) +++ trunk/varnish-cache/include/stat_field.h 2006-07-14 11:20:10 UTC (rev 469) @@ -10,17 +10,18 @@ MAC_STAT(backend_conn, uint64_t, "u", "Backend connections initiated") MAC_STAT(backend_recycle, uint64_t, "u", "Backend connections recyles") -MAC_STAT(n_sess, uint64_t, "u", "N struct sess"); -MAC_STAT(n_object, uint64_t, "u", "N struct object"); -MAC_STAT(n_objecthead, uint64_t, "u", "N struct objecthead"); -MAC_STAT(n_header, uint64_t, "u", "N struct header"); -MAC_STAT(n_smf, uint64_t, "u", "N struct smf"); -MAC_STAT(n_vbe, uint64_t, "u", "N struct vbe"); -MAC_STAT(n_vbe_conn, uint64_t, "u", "N struct vbe_conn"); -MAC_STAT(n_wrk, uint64_t, "u", "N worker threads"); -MAC_STAT(n_wrk_create, uint64_t, "u", "N worker threads created"); -MAC_STAT(n_wrk_failed, uint64_t, "u", "N worker threads not created"); -MAC_STAT(n_wrk_short, uint64_t, "u", "N worker threads shortages"); -MAC_STAT(n_wrk_busy, uint64_t, "u", "N busy worker threads"); +MAC_STAT(n_srcaddr, uint64_t, "u", "N struct srcaddr") +MAC_STAT(n_sess, uint64_t, "u", "N struct sess") +MAC_STAT(n_object, uint64_t, "u", "N struct object") +MAC_STAT(n_objecthead, uint64_t, "u", "N struct objecthead") +MAC_STAT(n_header, uint64_t, "u", "N struct header") +MAC_STAT(n_smf, uint64_t, "u", "N struct smf") +MAC_STAT(n_vbe, uint64_t, "u", "N struct vbe") +MAC_STAT(n_vbe_conn, uint64_t, "u", "N struct vbe_conn") +MAC_STAT(n_wrk, uint64_t, "u", "N worker threads") +MAC_STAT(n_wrk_create, uint64_t, "u", "N worker threads created") +MAC_STAT(n_wrk_failed, uint64_t, "u", "N worker threads not created") +MAC_STAT(n_wrk_short, uint64_t, "u", "N worker threads shortages") +MAC_STAT(n_wrk_busy, uint64_t, "u", "N busy worker threads") -MAC_STAT(losthdr, uint64_t, "u", "HTTP header overflows"); +MAC_STAT(losthdr, uint64_t, "u", "HTTP header overflows") From phk at projects.linpro.no Fri Jul 14 11:42:29 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 14 Jul 2006 13:42:29 +0200 (CEST) Subject: r470 - trunk/varnish-cache/bin/varnishd Message-ID: <20060714114229.A164F1EC381@projects.linpro.no> Author: phk Date: 2006-07-14 13:42:29 +0200 (Fri, 14 Jul 2006) New Revision: 470 Modified: trunk/varnish-cache/bin/varnishd/cache_session.c Log: propset Id Property changes on: trunk/varnish-cache/bin/varnishd/cache_session.c ___________________________________________________________________ Name: svn:keywords + Id From phk at projects.linpro.no Fri Jul 14 11:44:28 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 14 Jul 2006 13:44:28 +0200 (CEST) Subject: r471 - trunk/varnish-cache/bin/varnishd Message-ID: <20060714114428.D39A61EC380@projects.linpro.no> Author: phk Date: 2006-07-14 13:44:28 +0200 (Fri, 14 Jul 2006) New Revision: 471 Modified: trunk/varnish-cache/bin/varnishd/shmlog.c Log: Better and more paranoid SHMEM creation logic Modified: trunk/varnish-cache/bin/varnishd/shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-14 11:42:29 UTC (rev 470) +++ trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-14 11:44:28 UTC (rev 471) @@ -141,19 +141,29 @@ VSL_MgtInit(const char *fn, unsigned size) { struct shmloghead slh; - int i; + int i = 0; - heritage.vsl_fd = open(fn, O_RDWR | O_CREAT, 0644); - if (heritage.vsl_fd < 0) { - fprintf(stderr, "Could not open %s: %s\n", - fn, strerror(errno)); - exit (1); - } - i = read(heritage.vsl_fd, &slh, sizeof slh); - if (i != sizeof slh || slh.magic != SHMLOGHEAD_MAGIC) { + heritage.vsl_fd = open(fn, O_RDWR, 0644); + if (heritage.vsl_fd >= 0) + i = read(heritage.vsl_fd, &slh, sizeof slh); + if (heritage.vsl_fd < 0 || i != sizeof slh || + slh.magic != SHMLOGHEAD_MAGIC || + slh.hdrsize != sizeof slh) { /* XXX more checks */ + if (heritage.vsl_fd >= 0); + close(heritage.vsl_fd); + unlink(fn); + heritage.vsl_fd = open(fn, O_RDWR | O_CREAT, 0644); + if (heritage.vsl_fd < 0) { + fprintf(stderr, "Could not open %s: %s\n", + fn, strerror(errno)); + exit (1); + } + memset(&slh, 0, sizeof slh); + slh.magic = SHMLOGHEAD_MAGIC; + slh.hdrsize = sizeof slh; slh.size = size; slh.ptr = 0; slh.start = sizeof slh; From phk at projects.linpro.no Fri Jul 14 12:05:28 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 14 Jul 2006 14:05:28 +0200 (CEST) Subject: r472 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060714120528.A32F51EC395@projects.linpro.no> Author: phk Date: 2006-07-14 14:05:28 +0200 (Fri, 14 Jul 2006) New Revision: 472 Modified: trunk/varnish-cache/bin/varnishd/shmlog.c trunk/varnish-cache/include/shmlog.h Log: More SHM creation polishing Modified: trunk/varnish-cache/bin/varnishd/shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-14 11:44:28 UTC (rev 471) +++ trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-14 12:05:28 UTC (rev 472) @@ -129,6 +129,8 @@ heritage.vsl_fd, 0); assert(loghead != MAP_FAILED); + assert(loghead->magic == SHMLOGHEAD_MAGIC); + assert(loghead->hdrsize == sizeof *loghead); /* XXX check sanity of loghead */ logstart = (unsigned char *)loghead + loghead->start; AZ(pthread_mutex_init(&vsl_mutex, NULL)); @@ -151,8 +153,10 @@ slh.hdrsize != sizeof slh) { /* XXX more checks */ - if (heritage.vsl_fd >= 0); + fprintf(stderr, "Creating new SHMFILE\n"); + if (heritage.vsl_fd >= 0); { close(heritage.vsl_fd); + } unlink(fn); heritage.vsl_fd = open(fn, O_RDWR | O_CREAT, 0644); if (heritage.vsl_fd < 0) { @@ -160,8 +164,8 @@ fn, strerror(errno)); exit (1); } + memset(&slh, 0, sizeof slh); - slh.magic = SHMLOGHEAD_MAGIC; slh.hdrsize = sizeof slh; slh.size = size; Modified: trunk/varnish-cache/include/shmlog.h =================================================================== --- trunk/varnish-cache/include/shmlog.h 2006-07-14 11:44:28 UTC (rev 471) +++ trunk/varnish-cache/include/shmlog.h 2006-07-14 12:05:28 UTC (rev 472) @@ -15,6 +15,8 @@ #define SHMLOGHEAD_MAGIC 4185512498U /* From /dev/random */ unsigned magic; + unsigned hdrsize; + /* * Byte offset into the file where the fifolog starts * This allows the header to expand later. From phk at projects.linpro.no Fri Jul 14 12:12:46 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 14 Jul 2006 14:12:46 +0200 (CEST) Subject: r473 - trunk/varnish-cache/bin/varnishd Message-ID: <20060714121246.553221EC39F@projects.linpro.no> Author: phk Date: 2006-07-14 14:12:46 +0200 (Fri, 14 Jul 2006) New Revision: 473 Modified: trunk/varnish-cache/bin/varnishd/shmlog.c Log: Flexelint'ing, found a spurious ';' Modified: trunk/varnish-cache/bin/varnishd/shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-14 12:05:28 UTC (rev 472) +++ trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-14 12:12:46 UTC (rev 473) @@ -105,8 +105,8 @@ if (n > 255) n = 255; /* we truncate long fields */ } - p[1] = n; - p[2] = id >> 8; + p[1] = n & 0xff; + p[2] = (id >> 8) & 0xff; p[3] = id & 0xff; p[4 + n] = SLT_ENDMARKER; p[0] = tag; @@ -145,6 +145,7 @@ struct shmloghead slh; int i = 0; + memset(&slh, 0, sizeof slh); /* XXX: for flexelint */ heritage.vsl_fd = open(fn, O_RDWR, 0644); if (heritage.vsl_fd >= 0) i = read(heritage.vsl_fd, &slh, sizeof slh); @@ -154,10 +155,9 @@ /* XXX more checks */ fprintf(stderr, "Creating new SHMFILE\n"); - if (heritage.vsl_fd >= 0); { + if (heritage.vsl_fd >= 0) close(heritage.vsl_fd); - } - unlink(fn); + (void)unlink(fn); heritage.vsl_fd = open(fn, O_RDWR | O_CREAT, 0644); if (heritage.vsl_fd < 0) { fprintf(stderr, "Could not open %s: %s\n", @@ -174,7 +174,7 @@ AZ(lseek(heritage.vsl_fd, 0, SEEK_SET)); i = write(heritage.vsl_fd, &slh, sizeof slh); assert(i == sizeof slh); - AZ(ftruncate(heritage.vsl_fd, sizeof slh + size)); + AZ(ftruncate(heritage.vsl_fd, (off_t)sizeof slh + (off_t)size)); } heritage.vsl_size = slh.size + slh.start; From phk at projects.linpro.no Fri Jul 14 12:22:44 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 14 Jul 2006 14:22:44 +0200 (CEST) Subject: r474 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060714122244.5D4371EC38D@projects.linpro.no> Author: phk Date: 2006-07-14 14:22:44 +0200 (Fri, 14 Jul 2006) New Revision: 474 Modified: trunk/varnish-cache/bin/varnishd/shmlog.c trunk/varnish-cache/include/shmlog.h Log: Put a starttime in shmem so varnishstat can show average rates. Modified: trunk/varnish-cache/bin/varnishd/shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-14 12:12:46 UTC (rev 473) +++ trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-14 12:22:44 UTC (rev 474) @@ -134,6 +134,7 @@ /* XXX check sanity of loghead */ logstart = (unsigned char *)loghead + loghead->start; AZ(pthread_mutex_init(&vsl_mutex, NULL)); + loghead->starttime = time(NULL); VSL_stats = &loghead->stats; } Modified: trunk/varnish-cache/include/shmlog.h =================================================================== --- trunk/varnish-cache/include/shmlog.h 2006-07-14 12:12:46 UTC (rev 473) +++ trunk/varnish-cache/include/shmlog.h 2006-07-14 12:22:44 UTC (rev 474) @@ -9,6 +9,8 @@ #define SHMLOG_FILENAME "/tmp/_.vsl" +#include + #include "stats.h" struct shmloghead { @@ -17,6 +19,8 @@ unsigned hdrsize; + time_t starttime; + /* * Byte offset into the file where the fifolog starts * This allows the header to expand later. From phk at projects.linpro.no Fri Jul 14 12:30:49 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 14 Jul 2006 14:30:49 +0200 (CEST) Subject: r475 - trunk/varnish-cache/bin/varnishd Message-ID: <20060714123049.701F91EC38D@projects.linpro.no> Author: phk Date: 2006-07-14 14:30:49 +0200 (Fri, 14 Jul 2006) New Revision: 475 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: Have the states tell us if we are done yet with their return value, so that we can implement disembarking the worker thread of the object is busy. Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-14 12:22:44 UTC (rev 474) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-14 12:30:49 UTC (rev 475) @@ -57,12 +57,13 @@ DOT deliver2 -> DONE [style=bold] */ -static void +static int cnt_deliver(struct worker *w, struct sess *sp) { vca_write_obj(w, sp); sp->step = STP_DONE; + return (0); } @@ -76,7 +77,7 @@ DOT ] */ -static void +static int cnt_done(struct worker *w, struct sess *sp) { char *b; @@ -89,6 +90,7 @@ strcmp(b, "HTTP/1.1")) { vca_close_session(sp, "not HTTP/1.1"); } + return (1); } @@ -105,7 +107,7 @@ DOT error -> DONE */ -static void cnt_error(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } +static int cnt_error(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } /*-------------------------------------------------------------------- @@ -156,7 +158,7 @@ DOT fetch_error -> ERROR */ -static void +static int cnt_fetch(struct worker *w, struct sess *sp) { @@ -172,7 +174,7 @@ HSH_Deref(sp->obj); sp->obj = NULL; sp->step = STP_PASSBODY; - return; + return (0); } if (sp->handling == VCL_RET_INSERT_PASS) { sp->obj->pass = 1; @@ -180,13 +182,13 @@ HSH_Unbusy(sp->obj); sp->obj = NULL; sp->step = STP_PASSBODY; - return; + return (0); } if (sp->handling == VCL_RET_INSERT) { sp->obj->cacheable = 1; FetchBody(w, sp); sp->step = STP_DELIVER; - return; + return (0); } if (sp->handling == VCL_RET_ERROR) INCOMPL(); @@ -231,7 +233,7 @@ DOT hit2 -> DELIVER [style=bold] */ -static void +static int cnt_hit(struct worker *w, struct sess *sp) { @@ -245,14 +247,14 @@ HSH_Deref(sp->obj); sp->obj = NULL; sp->step = STP_DONE; - return; + return (0); } if (sp->handling == VCL_RET_PASS) { HSH_Deref(sp->obj); sp->obj = NULL; PassSession(w, sp); sp->step = STP_PASSBODY; - return; + return (0); } if (sp->handling == VCL_RET_ERROR) @@ -284,7 +286,7 @@ DOT lookup2 -> MISS [label="miss", style=bold] */ -static void +static int cnt_lookup(struct worker *w, struct sess *sp) { @@ -292,17 +294,18 @@ if (sp->obj->busy) { VSL_stats->cache_miss++; sp->step = STP_MISS; - return; + return (0); } if (sp->obj->pass) { VSL_stats->cache_hitpass++; VSL(SLT_HitPass, sp->fd, "%u", sp->obj->xid); sp->step = STP_HIT; - return; + return (0); } VSL_stats->cache_hit++; VSL(SLT_Hit, sp->fd, "%u", sp->obj->xid); sp->step = STP_HIT; + return (0); } @@ -343,7 +346,7 @@ DOT */ -static void +static int cnt_miss(struct worker *w, struct sess *sp) { @@ -357,14 +360,14 @@ sp->obj = 0; PassSession(w, sp); sp->step = STP_PASSBODY; - return; + return (0); } if (sp->handling == VCL_RET_LOOKUP) INCOMPL(); if (sp->handling == VCL_RET_FETCH) { FetchHeaders(w, sp); sp->step = STP_FETCH; - return; + return (0); } INCOMPL(); } @@ -384,12 +387,13 @@ DOT pass -> PASSBODY */ -static void +static int cnt_pass(struct worker *w, struct sess *sp) { PassSession(w, sp); sp->step = STP_PASSBODY; + return (0); } @@ -407,12 +411,12 @@ DOT passbody -> DONE */ -static void +static int cnt_passbody(struct worker *w, struct sess *sp) { PassBody(w, sp); sp->step = STP_DONE; - return; + return (0); } @@ -430,12 +434,13 @@ DOT pipe -> DONE */ -static void +static int cnt_pipe(struct worker *w, struct sess *sp) { PipeSession(w, sp); sp->step = STP_DONE; + return (0); } @@ -465,7 +470,7 @@ DOT recv_error -> ERROR */ -static void +static int cnt_recv(struct worker *w, struct sess *sp) { int done; @@ -474,7 +479,7 @@ if (done != 0) { RES_Error(w, sp, done, NULL); sp->step = STP_DONE; - return; + return (0); } sp->backend = sp->vcl->backend[0]; @@ -489,17 +494,17 @@ case VCL_RET_LOOKUP: /* XXX: discard req body, if any */ sp->step = STP_LOOKUP; - return; + return (0); case VCL_RET_PIPE: sp->step = STP_PIPE; - return; + return (0); case VCL_RET_PASS: sp->step = STP_PASS; - return; + return (0); case VCL_RET_ERROR: /* XXX: discard req body, if any */ sp->step = STP_ERROR; - return; + return (0); default: INCOMPL(); } @@ -517,6 +522,7 @@ void CNT_Session(struct worker *w, struct sess *sp) { + int done; sp->t0 = time(NULL); sp->vcl = VCL_Get(); @@ -525,12 +531,13 @@ SES_RefSrcAddr(sp); - for (sp->step = STP_RECV; sp->step != STP_DONE; ) { + sp->step = STP_RECV; + for (done = 0; !done; ) { switch (sp->step) { #define STEP(l,u) \ case STP_##u: \ VSL(SLT_Debug, sp->fd, "State " #u); \ - cnt_##l(w, sp); \ + done = cnt_##l(w, sp); \ break; #include "steps.h" #undef STEP @@ -538,8 +545,6 @@ } } - cnt_done(w, sp); /* The loop doesn't do this */ - VCL_Rel(sp->vcl); sp->vcl = NULL; From phk at projects.linpro.no Fri Jul 14 12:45:56 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 14 Jul 2006 14:45:56 +0200 (CEST) Subject: r476 - trunk/varnish-cache/bin/varnishd Message-ID: <20060714124556.9B7A11EC395@projects.linpro.no> Author: phk Date: 2006-07-14 14:45:56 +0200 (Fri, 14 Jul 2006) New Revision: 476 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Use the fact that we have the worker thread in struct sess now. Move initial and final processing into cnt_recv() and cnt_done() Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-14 12:30:49 UTC (rev 475) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-14 12:45:56 UTC (rev 476) @@ -245,7 +245,7 @@ int BAN_CheckObject(struct object *o, const char *url); /* cache_center.c [CNT] */ -void CNT_Session(struct worker *w, struct sess *sp); +void CNT_Session(struct sess *sp); /* cache_expiry.c */ void EXP_Insert(struct object *o); Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-14 12:30:49 UTC (rev 475) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-14 12:45:56 UTC (rev 476) @@ -58,10 +58,10 @@ */ static int -cnt_deliver(struct worker *w, struct sess *sp) +cnt_deliver(struct sess *sp) { - vca_write_obj(w, sp); + vca_write_obj(sp->wrk, sp); sp->step = STP_DONE; return (0); } @@ -78,11 +78,10 @@ */ static int -cnt_done(struct worker *w, struct sess *sp) +cnt_done(struct sess *sp) { char *b; - (void)w; if (http_GetHdr(sp->http, "Connection", &b) && !strcmp(b, "close")) { vca_close_session(sp, "Connection header"); @@ -90,6 +89,10 @@ strcmp(b, "HTTP/1.1")) { vca_close_session(sp, "not HTTP/1.1"); } + VCL_Rel(sp->vcl); + sp->vcl = NULL; + + vca_return_session(sp); return (1); } @@ -107,7 +110,7 @@ DOT error -> DONE */ -static int cnt_error(struct worker *w, struct sess *sp) { (void)w; (void)sp; INCOMPL(); } +static int cnt_error(struct sess *sp) { (void)sp; INCOMPL(); } /*-------------------------------------------------------------------- @@ -159,7 +162,7 @@ */ static int -cnt_fetch(struct worker *w, struct sess *sp) +cnt_fetch(struct sess *sp) { RFC2616_cache_policy(sp, sp->bkd_http); @@ -186,7 +189,7 @@ } if (sp->handling == VCL_RET_INSERT) { sp->obj->cacheable = 1; - FetchBody(w, sp); + FetchBody(sp->wrk, sp); sp->step = STP_DELIVER; return (0); } @@ -234,7 +237,7 @@ */ static int -cnt_hit(struct worker *w, struct sess *sp) +cnt_hit(struct sess *sp) { VCL_hit_method(sp); @@ -243,7 +246,7 @@ sp->handling = VCL_RET_PASS; if (sp->handling == VCL_RET_DELIVER) { - vca_write_obj(w, sp); + vca_write_obj(sp->wrk, sp); HSH_Deref(sp->obj); sp->obj = NULL; sp->step = STP_DONE; @@ -252,7 +255,7 @@ if (sp->handling == VCL_RET_PASS) { HSH_Deref(sp->obj); sp->obj = NULL; - PassSession(w, sp); + PassSession(sp->wrk, sp); sp->step = STP_PASSBODY; return (0); } @@ -287,10 +290,10 @@ */ static int -cnt_lookup(struct worker *w, struct sess *sp) +cnt_lookup(struct sess *sp) { - sp->obj = HSH_Lookup(w, sp->http); + sp->obj = HSH_Lookup(sp->wrk, sp->http); if (sp->obj->busy) { VSL_stats->cache_miss++; sp->step = STP_MISS; @@ -347,7 +350,7 @@ */ static int -cnt_miss(struct worker *w, struct sess *sp) +cnt_miss(struct sess *sp) { VCL_miss_method(sp); @@ -358,14 +361,14 @@ HSH_Unbusy(sp->obj); HSH_Deref(sp->obj); sp->obj = 0; - PassSession(w, sp); + PassSession(sp->wrk, sp); sp->step = STP_PASSBODY; return (0); } if (sp->handling == VCL_RET_LOOKUP) INCOMPL(); if (sp->handling == VCL_RET_FETCH) { - FetchHeaders(w, sp); + FetchHeaders(sp->wrk, sp); sp->step = STP_FETCH; return (0); } @@ -388,10 +391,10 @@ */ static int -cnt_pass(struct worker *w, struct sess *sp) +cnt_pass(struct sess *sp) { - PassSession(w, sp); + PassSession(sp->wrk, sp); sp->step = STP_PASSBODY; return (0); } @@ -412,9 +415,9 @@ */ static int -cnt_passbody(struct worker *w, struct sess *sp) +cnt_passbody(struct sess *sp) { - PassBody(w, sp); + PassBody(sp->wrk, sp); sp->step = STP_DONE; return (0); } @@ -435,10 +438,10 @@ */ static int -cnt_pipe(struct worker *w, struct sess *sp) +cnt_pipe(struct sess *sp) { - PipeSession(w, sp); + PipeSession(sp->wrk, sp); sp->step = STP_DONE; return (0); } @@ -471,13 +474,17 @@ */ static int -cnt_recv(struct worker *w, struct sess *sp) +cnt_recv(struct sess *sp) { int done; + sp->t0 = time(NULL); + sp->vcl = VCL_Get(); + SES_RefSrcAddr(sp); + done = http_DissectRequest(sp->http, sp->fd); if (done != 0) { - RES_Error(w, sp, done, NULL); + RES_Error(sp->wrk, sp, done, NULL); sp->step = STP_DONE; return (0); } @@ -520,24 +527,17 @@ */ void -CNT_Session(struct worker *w, struct sess *sp) +CNT_Session(struct sess *sp) { int done; - sp->t0 = time(NULL); - sp->vcl = VCL_Get(); - - sp->wrk = w; - - SES_RefSrcAddr(sp); - sp->step = STP_RECV; for (done = 0; !done; ) { switch (sp->step) { #define STEP(l,u) \ case STP_##u: \ VSL(SLT_Debug, sp->fd, "State " #u); \ - done = cnt_##l(w, sp); \ + done = cnt_##l(sp); \ break; #include "steps.h" #undef STEP @@ -545,10 +545,6 @@ } } - VCL_Rel(sp->vcl); - sp->vcl = NULL; - - vca_return_session(sp); } /* Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-14 12:30:49 UTC (rev 475) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-14 12:45:56 UTC (rev 476) @@ -62,7 +62,8 @@ TAILQ_REMOVE(&wrk_reqhead, wrq, list); AZ(pthread_mutex_unlock(&wrk_mtx)); assert(wrq->sess != NULL); - CNT_Session(w, wrq->sess); + wrq->sess->wrk = w; + CNT_Session(wrq->sess); AZ(pthread_mutex_lock(&wrk_mtx)); VSL_stats->n_wrk_busy--; TAILQ_INSERT_HEAD(&wrk_head, w, list); From phk at projects.linpro.no Fri Jul 14 12:47:09 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 14 Jul 2006 14:47:09 +0200 (CEST) Subject: r477 - trunk/varnish-cache/bin/varnishd Message-ID: <20060714124709.3D15B1EC381@projects.linpro.no> Author: phk Date: 2006-07-14 14:47:09 +0200 (Fri, 14 Jul 2006) New Revision: 477 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_center.c Log: Have the acceptor launch the session into STP_RECV Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-14 12:45:56 UTC (rev 476) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-14 12:47:09 UTC (rev 477) @@ -153,6 +153,7 @@ vca_return_session(sp); return; } + sp->step = STP_RECV; WRK_QueueSession(sp); } Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-14 12:45:56 UTC (rev 476) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-14 12:47:09 UTC (rev 477) @@ -531,7 +531,6 @@ { int done; - sp->step = STP_RECV; for (done = 0; !done; ) { switch (sp->step) { #define STEP(l,u) \ @@ -544,7 +543,6 @@ default: INCOMPL(); } } - } /* From phk at projects.linpro.no Fri Jul 14 13:33:26 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 14 Jul 2006 15:33:26 +0200 (CEST) Subject: r478 - trunk/varnish-cache/bin/varnishd Message-ID: <20060714133326.5903E1EC38D@projects.linpro.no> Author: phk Date: 2006-07-14 15:33:26 +0200 (Fri, 14 Jul 2006) New Revision: 478 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/steps.h Log: When during a lookup we encounter a busy object, queue the session on the objects waitinglist and disembark the worker thread so it can do something sensible in the mean time. This feature is unimportant in normal operation, but crucial to resource management if a popular URL suddenly takes a long time to reply from the backend. Without this bit if semi-nasty code, we would tie up one worker thread per client while waiting for the backend to come to it's senses. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-14 12:47:09 UTC (rev 477) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-14 13:33:26 UTC (rev 478) @@ -120,7 +120,6 @@ unsigned refcnt; unsigned xid; struct objhead *objhead; - pthread_cond_t cv; unsigned heap_idx; unsigned ban_seq; @@ -145,6 +144,8 @@ TAILQ_ENTRY(object) deathrow; TAILQ_HEAD(, storage) store; + + TAILQ_HEAD(, sess) waitinglist; }; struct objhead { @@ -257,7 +258,7 @@ int FetchHeaders(struct worker *w, struct sess *sp); /* cache_hash.c */ -struct object *HSH_Lookup(struct worker *w, struct http *h); +struct object *HSH_Lookup(struct sess *sp); void HSH_Unbusy(struct object *o); void HSH_Deref(struct object *o); void HSH_Init(void); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-14 12:47:09 UTC (rev 477) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-14 13:33:26 UTC (rev 478) @@ -32,6 +32,7 @@ static struct timeval tick_rate; static pthread_t vca_thread; +static unsigned xids; static struct event accept_e[2 * HERITAGE_NSOCKS]; static TAILQ_HEAD(,sess) sesshead = TAILQ_HEAD_INITIALIZER(sesshead); @@ -154,6 +155,9 @@ return; } sp->step = STP_RECV; + VSL_stats->client_req++; + sp->xid = xids++; + VSL(SLT_XID, sp->fd, "%u", sp->xid); WRK_QueueSession(sp); } @@ -295,4 +299,6 @@ tick_rate.tv_sec = 1; tick_rate.tv_usec = 0; AZ(pthread_create(&vca_thread, NULL, vca_main, NULL)); + srandomdev(); + xids = random(); } Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-14 12:47:09 UTC (rev 477) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-14 13:33:26 UTC (rev 478) @@ -273,17 +273,20 @@ /*-------------------------------------------------------------------- * Look up request in hash table * + * LOOKUP consists of two substates so that we can reenter if we + * encounter a busy object. + * DOT subgraph cluster_lookup { DOT lookup [ DOT shape=ellipse DOT label="find obj in cache" DOT ] DOT LOOKUP -> lookup [style=bold] -DOT lookup2 [ +DOT lookup3 [ DOT shape=ellipse DOT label="Insert new busy object" DOT ] -DOT lookup -> lookup2 [style=bold] +DOT lookup -> lookup3 [style=bold] DOT } DOT lookup -> HIT [label="hit", style=bold] DOT lookup2 -> MISS [label="miss", style=bold] @@ -292,22 +295,48 @@ static int cnt_lookup(struct sess *sp) { + sp->obj = NULL; + sp->step = STP_LOOKUP2; + return (0); +} - sp->obj = HSH_Lookup(sp->wrk, sp->http); +static int +cnt_lookup2(struct sess *sp) +{ + struct object *o; + + /* + * We don't assign to sp->obj directly because it is used + * to store state when we encounter a busy object. + */ + o = HSH_Lookup(sp); + + /* If we encountered busy-object, disembark worker thread */ + if (o == NULL) { + VSL(SLT_Debug, sp->fd, + "on waiting list on obj %u", sp->obj->xid); + return (1); + } + + sp->obj = o; + + /* If we inserted a new object it's a miss */ if (sp->obj->busy) { VSL_stats->cache_miss++; sp->step = STP_MISS; return (0); } + + /* Account separately for pass and cache objects */ if (sp->obj->pass) { VSL_stats->cache_hitpass++; VSL(SLT_HitPass, sp->fd, "%u", sp->obj->xid); - sp->step = STP_HIT; - return (0); - } - VSL_stats->cache_hit++; - VSL(SLT_Hit, sp->fd, "%u", sp->obj->xid); + } else { + VSL_stats->cache_hit++; + VSL(SLT_Hit, sp->fd, "%u", sp->obj->xid); + } sp->step = STP_HIT; +HERE(); return (0); } Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-14 12:47:09 UTC (rev 477) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-14 13:33:26 UTC (rev 478) @@ -40,13 +40,18 @@ static struct hash_slinger *hash; struct object * -HSH_Lookup(struct worker *w, struct http *h) +HSH_Lookup(struct sess *sp) { + struct worker *w; + struct http *h; struct objhead *oh; struct object *o; char *b, *c; assert(hash != NULL); + w = sp->wrk; + h = sp->http; + /* Precreate an objhead and object in case we need them */ if (w->nobjhead == NULL) { w->nobjhead = calloc(sizeof *w->nobjhead, 1); @@ -61,21 +66,32 @@ w->nobj->busy = 1; w->nobj->refcnt = 1; TAILQ_INIT(&w->nobj->store); - AZ(pthread_cond_init(&w->nobj->cv, NULL)); + TAILQ_INIT(&w->nobj->waitinglist); VSL_stats->n_object++; } assert(http_GetURL(h, &b)); if (!http_GetHdr(h, "Host", &c)) c = b; + if (sp->obj != NULL) { + o = sp->obj; + oh = o->objhead; + AZ(pthread_mutex_lock(&oh->mtx)); + goto were_back; + } oh = hash->lookup(b, c, w->nobjhead); if (oh == w->nobjhead) w->nobjhead = NULL; AZ(pthread_mutex_lock(&oh->mtx)); TAILQ_FOREACH(o, &oh->objects, list) { o->refcnt++; - if (o->busy) - AZ(pthread_cond_wait(&o->cv, &oh->mtx)); + if (o->busy) { + TAILQ_INSERT_TAIL(&o->waitinglist, sp, list); + sp->obj = o; + AZ(pthread_mutex_unlock(&oh->mtx)); + return (NULL); + } + were_back: /* XXX: check ttl */ /* XXX: check Vary: */ if (!o->cacheable) { @@ -110,13 +126,22 @@ void HSH_Unbusy(struct object *o) { + struct sess *sp; + assert(o != NULL); + assert(o->refcnt > 0); if (o->cacheable) EXP_Insert(o); AZ(pthread_mutex_lock(&o->objhead->mtx)); o->busy = 0; AZ(pthread_mutex_unlock(&o->objhead->mtx)); - AZ(pthread_cond_broadcast(&o->cv)); + while (1) { + sp = TAILQ_FIRST(&o->waitinglist); + if (sp == NULL) + break; + TAILQ_REMOVE(&o->waitinglist, sp, list); + WRK_QueueSession(sp); + } } void @@ -143,7 +168,6 @@ free(o->header); VSL_stats->n_header--; } - AZ(pthread_cond_destroy(&o->cv)); TAILQ_FOREACH_SAFE(st, &o->store, list, stn) { TAILQ_REMOVE(&o->store, st, list); Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-14 12:47:09 UTC (rev 477) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-14 13:33:26 UTC (rev 478) @@ -17,7 +17,6 @@ #include "cache.h" static pthread_mutex_t wrk_mtx; -static unsigned xids; /* Number of work requests queued in excess of worker threads available */ static unsigned wrk_overflow; @@ -105,15 +104,7 @@ sp->t_req = time(NULL); - /* - * No locking necessary, we're serialized in the acceptor thread - * XXX: still ? - */ - sp->xid = xids++; - VSL(SLT_XID, sp->fd, "%u", sp->xid); - sp->workreq.sess = sp; - VSL_stats->client_req++; AZ(pthread_mutex_lock(&wrk_mtx)); TAILQ_INSERT_TAIL(&wrk_reqhead, &sp->workreq, list); @@ -171,6 +162,4 @@ AZ(pthread_create(&tp, NULL, wrk_thread, &i)); AZ(pthread_detach(tp)); } - srandomdev(); - xids = random(); } Modified: trunk/varnish-cache/bin/varnishd/steps.h =================================================================== --- trunk/varnish-cache/bin/varnishd/steps.h 2006-07-14 12:47:09 UTC (rev 477) +++ trunk/varnish-cache/bin/varnishd/steps.h 2006-07-14 13:33:26 UTC (rev 478) @@ -5,6 +5,7 @@ STEP(pass, PASS) STEP(passbody, PASSBODY) STEP(lookup, LOOKUP) +STEP(lookup2, LOOKUP2) STEP(miss, MISS) STEP(hit, HIT) STEP(fetch, FETCH) From phk at phk.freebsd.dk Fri Jul 14 13:39:30 2006 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Fri, 14 Jul 2006 13:39:30 +0000 Subject: r478 - trunk/varnish-cache/bin/varnishd In-Reply-To: Your message of "Fri, 14 Jul 2006 15:33:26 +0200." <20060714133326.5903E1EC38D@projects.linpro.no> Message-ID: <76090.1152884370@critter.freebsd.dk> In message <20060714133326.5903E1EC38D at projects.linpro.no>, phk at projects.linpro .no writes: >Author: phk >Date: 2006-07-14 15:33:26 +0200 (Fri, 14 Jul 2006) >New Revision: 478 > >Log: >When during a lookup we encounter a busy object, queue the session on >the objects waitinglist and disembark the worker thread so it can do >something sensible in the mean time. This is the last "must have" item on my list so lets focus on the alpha release now. I'll try to figure out that 'tot' problem now. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at projects.linpro.no Fri Jul 14 13:52:12 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 14 Jul 2006 15:52:12 +0200 (CEST) Subject: r479 - trunk/varnish-cache/bin/varnishd Message-ID: <20060714135212.1599A1EC381@projects.linpro.no> Author: phk Date: 2006-07-14 15:52:11 +0200 (Fri, 14 Jul 2006) New Revision: 479 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Use space to separate host and port in backend spec. Polish usage message a bit. Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-14 13:33:26 UTC (rev 478) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-14 13:52:11 UTC (rev 479) @@ -118,7 +118,7 @@ * XXX: a bug for a backend to not reply at that time, so then * XXX: again: we should check it here in the "trivial" case. */ - p = strchr(bflag, ':'); + p = strchr(bflag, ' '); if (p != NULL) { q = p + 1; } else { @@ -444,17 +444,43 @@ { fprintf(stderr, "usage: varnishd [options]\n"); fprintf(stderr, " %-28s # %s\n", "-b backend", - "backend IP or hostname"); + "backend location"); + fprintf(stderr, " %-28s # %s\n", "", + " -b "); + fprintf(stderr, " %-28s # %s\n", "", + " -b ' '"); fprintf(stderr, " %-28s # %s\n", "-d", "debug"); fprintf(stderr, " %-28s # %s\n", "-f file", "VCL_file"); fprintf(stderr, " %-28s # %s\n", "-h kind[,hashoptions]", "Hash specification"); + fprintf(stderr, " %-28s # %s\n", "", + " -h simple_list"); + fprintf(stderr, " %-28s # %s\n", "", + " -h classic [default]"); + fprintf(stderr, " %-28s # %s\n", "", + " -h classic,"); + fprintf(stderr, " %-28s # %s\n", "", + " -h classic,,"); fprintf(stderr, " %-28s # %s\n", "-p number", "TCP listen port"); fprintf(stderr, " %-28s # %s\n", "-s kind[,storageoptions]", "Backend storage specification"); + fprintf(stderr, " %-28s # %s\n", "", + " -s malloc"); + fprintf(stderr, " %-28s # %s\n", "", + " -s file [default: use /tmp]"); + fprintf(stderr, " %-28s # %s\n", "", + " -s file,"); + fprintf(stderr, " %-28s # %s\n", "", + " -s file,,"); fprintf(stderr, " %-28s # %s\n", "-t", "Default TTL"); fprintf(stderr, " %-28s # %s\n", "-w int[,int[,int]]", - "Number of worker threads (fixed/{min,max}/{min/max/timeout})"); + "Number of worker threads"); + fprintf(stderr, " %-28s # %s\n", "", + " -w "); + fprintf(stderr, " %-28s # %s\n", "", + " -w min,max"); + fprintf(stderr, " %-28s # %s\n", "", + " -w min,max,timeout [default: -w1,INF,10]"); #if 0 -c clusterid at cluster_controller -m memory_limit From phk at projects.linpro.no Fri Jul 14 13:54:41 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 14 Jul 2006 15:54:41 +0200 (CEST) Subject: r480 - trunk/varnish-cache/bin/varnishtester Message-ID: <20060714135441.75DB01EC381@projects.linpro.no> Author: phk Date: 2006-07-14 15:54:41 +0200 (Fri, 14 Jul 2006) New Revision: 480 Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c Log: use space to separate host and port in -b Modified: trunk/varnish-cache/bin/varnishtester/varnishtester.c =================================================================== --- trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-14 13:52:11 UTC (rev 479) +++ trunk/varnish-cache/bin/varnishtester/varnishtester.c 2006-07-14 13:54:41 UTC (rev 480) @@ -268,7 +268,7 @@ execl( "./varnishd", "varnishd", - "-blocalhost:8081", + "-b", "localhost 8081", "-sfile,/tmp/,10m", NULL); perror("execl"); From phk at projects.linpro.no Tue Jul 18 08:51:42 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 10:51:42 +0200 (CEST) Subject: r481 - trunk/varnish-cache/bin/varnishd Message-ID: <20060718085142.5B96C1EC382@projects.linpro.no> Author: phk Date: 2006-07-18 10:51:42 +0200 (Tue, 18 Jul 2006) New Revision: 481 Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c Log: zero means 'all' to http_GetTail() Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-14 13:54:41 UTC (rev 480) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-18 08:51:42 UTC (rev 481) @@ -59,7 +59,7 @@ http_BuildSbuf(vc->fd, Build_Pipe, w->sb, sp->http); i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); - if (http_GetTail(sp->http, 99999999, &b, &e) && b != e) { /* XXX */ + if (http_GetTail(sp->http, 0, &b, &e) && b != e) { /* XXX */ i = write(vc->fd, b, e - b); if (i != e - b) { close (vc->fd); From phk at projects.linpro.no Tue Jul 18 08:52:07 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 10:52:07 +0200 (CEST) Subject: r482 - trunk/varnish-cache/bin/varnishd Message-ID: <20060718085207.908D51EC399@projects.linpro.no> Author: phk Date: 2006-07-18 10:52:07 +0200 (Tue, 18 Jul 2006) New Revision: 482 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: polish Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-18 08:51:42 UTC (rev 481) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-18 08:52:07 UTC (rev 482) @@ -295,7 +295,8 @@ static int cnt_lookup(struct sess *sp) { - sp->obj = NULL; + + assert(sp->obj == NULL); sp->step = STP_LOOKUP2; return (0); } @@ -307,7 +308,7 @@ /* * We don't assign to sp->obj directly because it is used - * to store state when we encounter a busy object. + * to cache state when we encounter a busy object. */ o = HSH_Lookup(sp); @@ -336,7 +337,6 @@ VSL(SLT_Hit, sp->fd, "%u", sp->obj->xid); } sp->step = STP_HIT; -HERE(); return (0); } From phk at projects.linpro.no Tue Jul 18 09:02:56 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 11:02:56 +0200 (CEST) Subject: r483 - trunk/varnish-cache/bin/varnishd Message-ID: <20060718090256.601751EC39A@projects.linpro.no> Author: phk Date: 2006-07-18 11:02:56 +0200 (Tue, 18 Jul 2006) New Revision: 483 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_vrt.c Log: Retire the http_GetReq(), http_GetURL() and http_GetProto() accessor functions now that struct http is out of the closet. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-18 08:52:07 UTC (rev 482) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-18 09:02:56 UTC (rev 483) @@ -45,8 +45,8 @@ char *s; /* start of buffer */ char *t; /* start of trailing data */ + char *v; /* end of valid bytes */ char *e; /* end of buffer */ - char *v; /* valid bytes */ char *req; char *url; @@ -267,12 +267,9 @@ void http_Init(struct http *ht, void *space); int http_GetHdr(struct http *hp, const char *hdr, char **ptr); int http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr); -int http_GetReq(struct http *hp, char **b); -int http_GetProto(struct http *hp, char **b); int http_GetStatus(struct http *hp); int http_HdrIs(struct http *hp, const char *hdr, const char *val); int http_GetTail(struct http *hp, unsigned len, char **b, char **e); -int http_GetURL(struct http *hp, char **b); void http_RecvHead(struct http *hp, int fd, struct event_base *eb, http_callback_f *func, void *arg); int http_DissectRequest(struct http *sp, int fd); int http_DissectResponse(struct http *sp, int fd); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-18 08:52:07 UTC (rev 482) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-18 09:02:56 UTC (rev 483) @@ -79,7 +79,6 @@ { struct storage *st; unsigned u = 0; - char *r; uint64_t bytes = 0; @@ -93,14 +92,14 @@ sp->obj->age + sp->t_req - sp->obj->entered); sbuf_printf(w->sb, "Via: 1.1 varnish\r\n"); sbuf_printf(w->sb, "X-Varnish: xid %u\r\n", sp->obj->xid); - if (http_GetProto(sp->http, &r) && strcmp(r, "HTTP/1.1")) + if (strcmp(sp->http->proto, "HTTP/1.1")) sbuf_printf(w->sb, "Connection: close\r\n"); sbuf_printf(w->sb, "\r\n"); sbuf_finish(w->sb); vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); bytes += sbuf_len(w->sb); - assert(http_GetReq(sp->http, &r)); - if (!strcmp(r, "GET")) { + /* XXX: conditional request handling */ + if (!strcmp(sp->http->req, "GET")) { TAILQ_FOREACH(st, &sp->obj->store, list) { u += st->len; if (st->stevedore->send == NULL) { Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-18 08:52:07 UTC (rev 482) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-18 09:02:56 UTC (rev 483) @@ -85,8 +85,7 @@ if (http_GetHdr(sp->http, "Connection", &b) && !strcmp(b, "close")) { vca_close_session(sp, "Connection header"); - } else if (http_GetProto(sp->http, &b) && - strcmp(b, "HTTP/1.1")) { + } else if (strcmp(sp->http->proto, "HTTP/1.1")) { vca_close_session(sp, "not HTTP/1.1"); } VCL_Rel(sp->vcl); Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-18 08:52:07 UTC (rev 482) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-18 09:02:56 UTC (rev 483) @@ -46,7 +46,7 @@ struct http *h; struct objhead *oh; struct object *o; - char *b, *c; + char *c; assert(hash != NULL); w = sp->wrk; @@ -70,16 +70,15 @@ VSL_stats->n_object++; } - assert(http_GetURL(h, &b)); if (!http_GetHdr(h, "Host", &c)) - c = b; + c = h->url; if (sp->obj != NULL) { o = sp->obj; oh = o->objhead; AZ(pthread_mutex_lock(&oh->mtx)); goto were_back; } - oh = hash->lookup(b, c, w->nobjhead); + oh = hash->lookup(h->url, c, w->nobjhead); if (oh == w->nobjhead) w->nobjhead = NULL; AZ(pthread_mutex_lock(&oh->mtx)); @@ -98,7 +97,7 @@ /* ignore */ } else if (o->ttl == 0) { /* Object banned but not reaped yet */ - } else if (BAN_CheckObject(o, b)) { + } else if (BAN_CheckObject(o, h->url)) { o->ttl = 0; VSL(SLT_ExpBan, 0, "%u was banned", o->xid); EXP_TTLchange(o); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-18 08:52:07 UTC (rev 482) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-18 09:02:56 UTC (rev 483) @@ -100,33 +100,6 @@ } int -http_GetReq(struct http *hp, char **b) -{ - if (hp->req == NULL) - return (0); - *b = hp->req; - return (1); -} - -int -http_GetURL(struct http *hp, char **b) -{ - if (hp->url == NULL) - return (0); - *b = hp->url; - return (1); -} - -int -http_GetProto(struct http *hp, char **b) -{ - if (hp->proto == NULL) - return (0); - *b = hp->proto; - return (1); -} - -int http_GetTail(struct http *hp, unsigned len, char **b, char **e) { Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-18 08:52:07 UTC (rev 482) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-18 09:02:56 UTC (rev 483) @@ -54,12 +54,10 @@ char * VRT_GetReq(struct sess *sp) { - char *p; assert(sp != NULL); assert(sp->http != NULL); - assert(http_GetReq(sp->http, &p)); - return (p); + return (sp->http->req); } /*--------------------------------------------------------------------*/ From phk at projects.linpro.no Tue Jul 18 10:32:40 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 12:32:40 +0200 (CEST) Subject: r484 - trunk/varnish-cache/bin/varnishd Message-ID: <20060718103240.9CCC41EC381@projects.linpro.no> Author: phk Date: 2006-07-18 12:32:40 +0200 (Tue, 18 Jul 2006) New Revision: 484 Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c Log: Use bigger buffersizes for pass mode Terminate the sbuf with the reply headers properly. Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-18 09:02:56 UTC (rev 483) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-18 10:32:40 UTC (rev 484) @@ -17,6 +17,7 @@ #include "shmlog.h" #include "cache.h" +#define PASS_BUFSIZ 8192 /*--------------------------------------------------------------------*/ @@ -27,12 +28,12 @@ char *b, *e; off_t cl; unsigned c; - char buf[BUFSIZ]; + char buf[PASS_BUFSIZ]; if (bi != NULL) cl = strtoumax(bi, NULL, 0); else - cl = (1<<30); /* XXX */ + cl = (1 << 30); i = fcntl(fd, F_GETFL); /* XXX ? */ i &= ~O_NONBLOCK; @@ -69,7 +70,7 @@ char *b, *q, *e; char *p; unsigned u; - char buf[BUFSIZ]; + char buf[PASS_BUFSIZ]; char *bp, *be; i = fcntl(fd, F_GETFL); /* XXX ? */ @@ -159,6 +160,8 @@ assert(vc != NULL); http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); + sbuf_cat(w->sb, "\r\n"); + sbuf_finish(w->sb); vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); if (http_GetHdr(hp, "Content-Length", &b)) From phk at projects.linpro.no Tue Jul 18 10:45:15 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 12:45:15 +0200 (CEST) Subject: r485 - trunk/varnish-cache/bin/varnishd Message-ID: <20060718104515.C06C21EC38D@projects.linpro.no> Author: phk Date: 2006-07-18 12:45:15 +0200 (Tue, 18 Jul 2006) New Revision: 485 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pipe.c Log: Add http_Read() which reads from a socket but soaks up any prefeched tail first and use it all the places where this logic was explicit before. Fix Refcounting on objects when we insert/deliver Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-18 10:32:40 UTC (rev 484) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-18 10:45:15 UTC (rev 485) @@ -260,6 +260,7 @@ /* cache_hash.c */ struct object *HSH_Lookup(struct sess *sp); void HSH_Unbusy(struct object *o); +void HSH_Ref(struct object *o); void HSH_Deref(struct object *o); void HSH_Init(void); @@ -270,6 +271,7 @@ int http_GetStatus(struct http *hp); int http_HdrIs(struct http *hp, const char *hdr, const char *val); int http_GetTail(struct http *hp, unsigned len, char **b, char **e); +int http_Read(struct http *hp, int fd, char *b, unsigned len); void http_RecvHead(struct http *hp, int fd, struct event_base *eb, http_callback_f *func, void *arg); int http_DissectRequest(struct http *sp, int fd); int http_DissectResponse(struct http *sp, int fd); Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-18 10:32:40 UTC (rev 484) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-18 10:45:15 UTC (rev 485) @@ -62,6 +62,8 @@ { vca_write_obj(sp->wrk, sp); + HSH_Deref(sp->obj); + sp->obj = NULL; sp->step = STP_DONE; return (0); } @@ -82,6 +84,7 @@ { char *b; + assert(sp->obj == NULL); if (http_GetHdr(sp->http, "Connection", &b) && !strcmp(b, "close")) { vca_close_session(sp, "Connection header"); @@ -182,6 +185,7 @@ sp->obj->pass = 1; sp->obj->cacheable = 1; HSH_Unbusy(sp->obj); + /* Don't HSH_Deref(sp->obj); we need the ref for storage */ sp->obj = NULL; sp->step = STP_PASSBODY; return (0); @@ -189,6 +193,8 @@ if (sp->handling == VCL_RET_INSERT) { sp->obj->cacheable = 1; FetchBody(sp->wrk, sp); + HSH_Ref(sp->obj); /* get another, STP_DELIVER will deref */ + HSH_Unbusy(sp->obj); sp->step = STP_DELIVER; return (0); } @@ -510,6 +516,8 @@ sp->vcl = VCL_Get(); SES_RefSrcAddr(sp); + assert(sp->obj == NULL); + done = http_DissectRequest(sp->http, sp->fd); if (done != 0) { RES_Error(sp->wrk, sp, done, NULL); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-18 10:32:40 UTC (rev 484) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-18 10:45:15 UTC (rev 485) @@ -33,7 +33,6 @@ fetch_straight(const struct sess *sp, int fd, struct http *hp, char *b) { int i; - char *e; unsigned char *p; off_t cl; struct storage *st; @@ -50,15 +49,8 @@ i &= ~O_NONBLOCK; i = fcntl(fd, F_SETFL, i); - if (http_GetTail(hp, cl, &b, &e)) { - i = e - b; - memcpy(p, b, i); - p += i; - cl -= i; - } - while (cl != 0) { - i = read(fd, p, cl); + i = http_Read(hp, fd, p, cl); assert(i > 0); /* XXX seen */ p += i; cl -= i; @@ -73,7 +65,7 @@ fetch_chunked(const struct sess *sp, int fd, struct http *hp) { int i; - char *b, *q, *e; + char *q; unsigned char *p; struct storage *st; unsigned u, v; @@ -88,16 +80,10 @@ bp = buf; st = NULL; while (1) { - if (http_GetTail(hp, be - bp, &b, &e)) { - memcpy(bp, b, e - b); - bp += e - b; - *bp = '\0'; - } else { - i = read(fd, bp, be - bp); - assert(i >= 0); - bp += i; - *bp = '\0'; - } + i = http_Read(hp, fd, bp, be - bp); + assert(i >= 0); + bp += i; + *bp = '\0'; u = strtoul(buf, &q, 16); if (q == NULL || q == buf) continue; @@ -151,15 +137,8 @@ break; if (v == 0) continue; - if (http_GetTail(hp, v, &b, &e)) { - memcpy(p, b, e - b); - p += e - b; - st->len += e - b; - v -= e - b; - u -= e - b; - } while (v > 0) { - i = read(fd, p, v); + i = http_Read(hp, fd, p, v); assert(i > 0); st->len += i; v -= i; @@ -183,7 +162,6 @@ fetch_eof(const struct sess *sp, int fd, struct http *hp) { int i; - char *b, *e; unsigned char *p; struct storage *st; unsigned v; @@ -204,15 +182,7 @@ } assert(p != NULL); assert(st != NULL); - if (http_GetTail(hp, v, &b, &e)) { - memcpy(p, b, e - b); - p += e - b; - v -= e - b; - st->len += e - b; - sp->obj->len += e - b; - *p = '\0'; - } - i = read(fd, p, v); + i = http_Read(hp, fd, p, v); assert(i >= 0); if (i == 0) break; Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-18 10:32:40 UTC (rev 484) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-18 10:45:15 UTC (rev 485) @@ -144,6 +144,17 @@ } void +HSH_Ref(struct object *o) +{ + struct objhead *oh; + + oh = o->objhead; + AZ(pthread_mutex_lock(&oh->mtx)); + o->refcnt++; + AZ(pthread_mutex_unlock(&oh->mtx)); +} + +void HSH_Deref(struct object *o) { struct objhead *oh; Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-18 10:32:40 UTC (rev 484) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-18 10:45:15 UTC (rev 485) @@ -120,7 +120,34 @@ return (1); } +/* Read from fd, but soak up any tail first */ + int +http_Read(struct http *hp, int fd, char *b, unsigned len) +{ + int i; + unsigned u; + + u = 0; + if (hp->t < hp->v) { + u = hp->v - hp->t; + if (u > len) + u = len; + memcpy(b, hp->t, u); + hp->t += u; + b += u; + len -= u; + } + if (len > 0) { + i = read(fd, b, len); + if (i < 0) + return (i); + u += i; + } + return (u); +} + +int http_GetStatus(struct http *hp) { Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-18 10:32:40 UTC (rev 484) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-18 10:45:15 UTC (rev 485) @@ -25,7 +25,6 @@ pass_straight(struct sess *sp, int fd, struct http *hp, char *bi) { int i; - char *b, *e; off_t cl; unsigned c; char buf[PASS_BUFSIZ]; @@ -43,13 +42,7 @@ c = cl; if (c > sizeof buf) c = sizeof buf; - if (http_GetTail(hp, c, &b, &e)) { - i = e - b; - vca_write(sp, b, i); - cl -= i; - continue; - } - i = read(fd, buf, c); + i = http_Read(hp, fd, buf, c); if (i == 0 && bi == NULL) return (1); assert(i > 0); @@ -81,15 +74,10 @@ be = buf + sizeof buf; p = buf; while (1) { - if (http_GetTail(hp, be - bp, &b, &e)) { - memcpy(bp, b, e - b); - bp += e - b; - } else { - /* XXX: must be safe from backend */ - i = read(fd, bp, be - bp); - assert(i > 0); - bp += i; - } + i = http_Read(hp, fd, bp, be - bp); + i = read(fd, bp, be - bp); + assert(i > 0); + bp += i; /* buffer valid from p to bp */ u = strtoul(p, &q, 16); @@ -159,7 +147,7 @@ vc = sp->vbc; assert(vc != NULL); - http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); + http_BuildSbuf(sp->fd, Build_Passreply, w->sb, hp); sbuf_cat(w->sb, "\r\n"); sbuf_finish(w->sb); vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-18 10:32:40 UTC (rev 484) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-18 10:45:15 UTC (rev 485) @@ -59,7 +59,7 @@ http_BuildSbuf(vc->fd, Build_Pipe, w->sb, sp->http); i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); - if (http_GetTail(sp->http, 0, &b, &e) && b != e) { /* XXX */ + if (http_GetTail(sp->http, 0, &b, &e) && b != e) { i = write(vc->fd, b, e - b); if (i != e - b) { close (vc->fd); From phk at projects.linpro.no Tue Jul 18 10:45:57 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 12:45:57 +0200 (CEST) Subject: r486 - trunk/varnish-cache/bin/varnishd Message-ID: <20060718104557.910FA1EC382@projects.linpro.no> Author: phk Date: 2006-07-18 12:45:57 +0200 (Tue, 18 Jul 2006) New Revision: 486 Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c Log: typo Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-18 10:45:15 UTC (rev 485) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-18 10:45:57 UTC (rev 486) @@ -147,7 +147,7 @@ vc = sp->vbc; assert(vc != NULL); - http_BuildSbuf(sp->fd, Build_Passreply, w->sb, hp); + http_BuildSbuf(sp->fd, Build_Pass, w->sb, hp); sbuf_cat(w->sb, "\r\n"); sbuf_finish(w->sb); vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); From phk at projects.linpro.no Tue Jul 18 10:48:21 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 12:48:21 +0200 (CEST) Subject: r487 - trunk/varnish-cache/bin/varnishd Message-ID: <20060718104821.00C8B1EC38D@projects.linpro.no> Author: phk Date: 2006-07-18 12:48:21 +0200 (Tue, 18 Jul 2006) New Revision: 487 Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c Log: Braino this time. Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-18 10:45:57 UTC (rev 486) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-18 10:48:21 UTC (rev 487) @@ -147,7 +147,7 @@ vc = sp->vbc; assert(vc != NULL); - http_BuildSbuf(sp->fd, Build_Pass, w->sb, hp); + http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); sbuf_cat(w->sb, "\r\n"); sbuf_finish(w->sb); vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); From phk at projects.linpro.no Tue Jul 18 12:27:57 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 14:27:57 +0200 (CEST) Subject: r488 - trunk/varnish-cache/bin/varnishd Message-ID: <20060718122757.F03FF1EC382@projects.linpro.no> Author: phk Date: 2006-07-18 14:27:57 +0200 (Tue, 18 Jul 2006) New Revision: 488 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_http.c Log: Use a void * for http_Read()'s buffer Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-18 10:48:21 UTC (rev 487) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-18 12:27:57 UTC (rev 488) @@ -271,7 +271,7 @@ int http_GetStatus(struct http *hp); int http_HdrIs(struct http *hp, const char *hdr, const char *val); int http_GetTail(struct http *hp, unsigned len, char **b, char **e); -int http_Read(struct http *hp, int fd, char *b, unsigned len); +int http_Read(struct http *hp, int fd, void *b, unsigned len); void http_RecvHead(struct http *hp, int fd, struct event_base *eb, http_callback_f *func, void *arg); int http_DissectRequest(struct http *sp, int fd); int http_DissectResponse(struct http *sp, int fd); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-18 10:48:21 UTC (rev 487) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-18 12:27:57 UTC (rev 488) @@ -123,10 +123,11 @@ /* Read from fd, but soak up any tail first */ int -http_Read(struct http *hp, int fd, char *b, unsigned len) +http_Read(struct http *hp, int fd, void *p, unsigned len) { int i; unsigned u; + char *b = p; u = 0; if (hp->t < hp->v) { From phk at projects.linpro.no Tue Jul 18 12:28:32 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 14:28:32 +0200 (CEST) Subject: r489 - trunk/varnish-cache/bin/varnishd Message-ID: <20060718122832.26F2A1EC38D@projects.linpro.no> Author: phk Date: 2006-07-18 14:28:32 +0200 (Tue, 18 Jul 2006) New Revision: 489 Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c Log: Assert that object is busy when we call unbusy Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-18 12:27:57 UTC (rev 488) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-18 12:28:32 UTC (rev 489) @@ -128,6 +128,7 @@ struct sess *sp; assert(o != NULL); + assert(o->busy); assert(o->refcnt > 0); if (o->cacheable) EXP_Insert(o); From phk at projects.linpro.no Tue Jul 18 12:29:11 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 14:29:11 +0200 (CEST) Subject: r490 - trunk/varnish-cache/bin/varnishd Message-ID: <20060718122911.5C08D1EC38D@projects.linpro.no> Author: phk Date: 2006-07-18 14:29:11 +0200 (Tue, 18 Jul 2006) New Revision: 490 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c Log: Remove the deref/unbusy stuff from FetchBody() it's done in central.c Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-18 12:28:32 UTC (rev 489) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-18 12:29:11 UTC (rev 490) @@ -13,7 +13,6 @@ #include #include -#include "libvarnish.h" #include "shmlog.h" #include "cache.h" @@ -235,11 +234,6 @@ else VBE_RecycleFd(vc); - HSH_Unbusy(sp->obj); - /* Hold on to the reference count, it's not released until - * expiry */ - if (!sp->obj->cacheable) - HSH_Deref(sp->obj); return (0); } From phk at projects.linpro.no Tue Jul 18 12:40:17 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 14:40:17 +0200 (CEST) Subject: r491 - trunk/varnish-cache/lib/libvarnish Message-ID: <20060718124017.A75D21EC382@projects.linpro.no> Author: phk Date: 2006-07-18 14:40:17 +0200 (Tue, 18 Jul 2006) New Revision: 491 Modified: trunk/varnish-cache/lib/libvarnish/binary_heap.c Log: Move the root index from zero to one Modified: trunk/varnish-cache/lib/libvarnish/binary_heap.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/binary_heap.c 2006-07-18 12:29:11 UTC (rev 490) +++ trunk/varnish-cache/lib/libvarnish/binary_heap.c 2006-07-18 12:40:17 UTC (rev 491) @@ -19,6 +19,8 @@ #define MIN_LENGTH 16 +#define ROOT_IDX 1 + struct binheap { unsigned magic; #define BINHEAP_MAGIC 0xf581581aU /* from /dev/random */ @@ -57,7 +59,7 @@ bh->priv = priv; bh->cmp = cmp_f; bh->update = update_f; - bh->next = 0; + bh->next = ROOT_IDX; bh->length = MIN_LENGTH; bh->array = calloc(sizeof *bh->array, bh->length); assert(bh->array != NULL); @@ -87,7 +89,7 @@ unsigned v; assert(bh->magic == BINHEAP_MAGIC); - while (u > 0) { + while (u > ROOT_IDX) { v = PARENT(u); if (bh->cmp(bh->priv, bh->array[u], bh->array[v])) { binhead_swap(bh, u, v); @@ -161,9 +163,9 @@ assert(bh != NULL); assert(bh->magic == BINHEAP_MAGIC); - if(bh->next == 0) + if(bh->next == ROOT_IDX) return (NULL); - return (bh->array[0]); + return (bh->array[ROOT_IDX]); } void @@ -172,7 +174,7 @@ assert(bh != NULL); assert(bh->magic == BINHEAP_MAGIC); - assert(bh->next > 0); + assert(bh->next > ROOT_IDX); assert(idx < bh->next); if (idx == --bh->next) return; From phk at projects.linpro.no Tue Jul 18 12:46:23 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 14:46:23 +0200 (CEST) Subject: r492 - trunk/varnish-cache/lib/libvarnish Message-ID: <20060718124623.A28F51EC38D@projects.linpro.no> Author: phk Date: 2006-07-18 14:46:23 +0200 (Tue, 18 Jul 2006) New Revision: 492 Modified: trunk/varnish-cache/lib/libvarnish/binary_heap.c Log: Set the index of deleted elements to zero Modified: trunk/varnish-cache/lib/libvarnish/binary_heap.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/binary_heap.c 2006-07-18 12:40:17 UTC (rev 491) +++ trunk/varnish-cache/lib/libvarnish/binary_heap.c 2006-07-18 12:46:23 UTC (rev 492) @@ -176,6 +176,8 @@ assert(bh->magic == BINHEAP_MAGIC); assert(bh->next > ROOT_IDX); assert(idx < bh->next); + if (bh->update != NULL) + bh->update(bh->priv, bh->array[idx], 0); if (idx == --bh->next) return; bh->array[idx] = bh->array[bh->next]; From phk at projects.linpro.no Tue Jul 18 13:18:51 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 15:18:51 +0200 (CEST) Subject: r493 - trunk/varnish-cache/lib/libvarnish Message-ID: <20060718131851.13E521EC38D@projects.linpro.no> Author: phk Date: 2006-07-18 15:18:50 +0200 (Tue, 18 Jul 2006) New Revision: 493 Modified: trunk/varnish-cache/lib/libvarnish/binary_heap.c Log: Fix the Parent calculation Modified: trunk/varnish-cache/lib/libvarnish/binary_heap.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/binary_heap.c 2006-07-18 12:46:23 UTC (rev 492) +++ trunk/varnish-cache/lib/libvarnish/binary_heap.c 2006-07-18 13:18:50 UTC (rev 493) @@ -33,7 +33,7 @@ unsigned granularity; }; -#define PARENT(u) (((u) - 1) / 2) +#define PARENT(u) ((u) / 2) #define CHILD(u,n) ((u) * 2 + (n)) /* Implementation ----------------------------------------------------*/ @@ -43,6 +43,7 @@ { assert(bh->magic == BINHEAP_MAGIC); assert(u < bh->next); + assert(bh->array[u] != NULL); if (bh->update == NULL) return; bh->update(bh->priv, bh->array[u], u); From phk at projects.linpro.no Tue Jul 18 13:19:55 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 15:19:55 +0200 (CEST) Subject: r494 - trunk/varnish-cache/bin/varnishd Message-ID: <20060718131955.9BBE31EC38D@projects.linpro.no> Author: phk Date: 2006-07-18 15:19:55 +0200 (Tue, 18 Jul 2006) New Revision: 494 Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c Log: Delete binheap root by it's index. Expect a refcount of one (the one holding the object in the hash) Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-18 13:18:50 UTC (rev 493) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-18 13:19:55 UTC (rev 494) @@ -24,6 +24,7 @@ EXP_Insert(struct object *o) { + assert(o->heap_idx == 0); AZ(pthread_mutex_lock(&exp_mtx)); binheap_insert(exp_heap, o); AZ(pthread_mutex_unlock(&exp_mtx)); @@ -32,6 +33,7 @@ void EXP_TTLchange(struct object *o) { + assert(o->heap_idx != 0); AZ(pthread_mutex_lock(&exp_mtx)); binheap_delete(exp_heap, o->heap_idx); binheap_insert(exp_heap, o); @@ -54,17 +56,19 @@ t = time(NULL); AZ(pthread_mutex_lock(&exp_mtx)); TAILQ_FOREACH(o, &exp_deathrow, deathrow) { - if (o->ttl >= t) + if (o->ttl >= t) { + o = NULL; break; + } if (o->busy) { VSL(SLT_Debug, 0, "Grim Reaper: Busy object xid %u", o->xid); continue; } - if (o->refcnt == 0) + if (o->refcnt == 1) break; } - if (o == NULL || o->ttl >= t || o->refcnt > 0) { + if (o == NULL) { AZ(pthread_mutex_unlock(&exp_mtx)); AZ(sleep(1)); continue; @@ -103,7 +107,7 @@ AZ(sleep(1)); continue; } - binheap_delete(exp_heap, 0); + binheap_delete(exp_heap, o->heap_idx); AZ(pthread_mutex_unlock(&exp_mtx)); VSL(SLT_ExpPick, 0, "%u", o->xid); From phk at projects.linpro.no Tue Jul 18 13:47:10 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 18 Jul 2006 15:47:10 +0200 (CEST) Subject: r495 - trunk/varnish-cache/bin/varnishd Message-ID: <20060718134710.C9E441EC381@projects.linpro.no> Author: phk Date: 2006-07-18 15:47:10 +0200 (Tue, 18 Jul 2006) New Revision: 495 Modified: trunk/varnish-cache/bin/varnishd/cache_session.c Log: eliminate debugging Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-18 13:19:55 UTC (rev 494) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-18 13:47:10 UTC (rev 495) @@ -118,10 +118,7 @@ sa = sp->srcaddr; now = time(NULL); AZ(pthread_mutex_lock(&ses_mtx)); -VSL(SLT_Debug, 0, "%ju", bytes); -VSL(SLT_Debug, 0, "%ju", sa->bytes); sa->bytes += bytes; -VSL(SLT_Debug, 0, "%ju", sa->bytes); sa->ttl = now + CLIENT_TTL; TAILQ_REMOVE(sa->sah, sa, list); TAILQ_INSERT_TAIL(sa->sah, sa, list); From phk at projects.linpro.no Wed Jul 19 08:33:10 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 10:33:10 +0200 (CEST) Subject: r496 - trunk/varnish-cache/bin/varnishd Message-ID: <20060719083310.CB1F21EC3A2@projects.linpro.no> Author: phk Date: 2006-07-19 10:33:10 +0200 (Wed, 19 Jul 2006) New Revision: 496 Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Properly zero the worker structure when we start a thread. Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-18 13:47:10 UTC (rev 495) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-19 08:33:10 UTC (rev 496) @@ -34,7 +34,7 @@ struct timespec ts; w = &ww; - memset(w, 0, sizeof w); + memset(w, 0, sizeof *w); AZ(pthread_cond_init(&w->cv, NULL)); From phk at projects.linpro.no Wed Jul 19 11:11:48 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 13:11:48 +0200 (CEST) Subject: r497 - trunk/varnish-cache/bin/varnishd Message-ID: <20060719111148.280DE1EC221@projects.linpro.no> Author: phk Date: 2006-07-19 13:11:48 +0200 (Wed, 19 Jul 2006) New Revision: 497 Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c Log: handle 302 for now. Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-19 08:33:10 UTC (rev 496) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-19 11:11:48 UTC (rev 497) @@ -139,6 +139,7 @@ case 203: /* Non-Authoritative Information */ case 300: /* Multiple Choices */ case 301: /* Moved Permanently */ + case 302: /* XXX */ case 410: /* Gone */ case 404: /* Not Found */ sp->obj->cacheable = 1; From phk at projects.linpro.no Wed Jul 19 11:53:06 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 13:53:06 +0200 (CEST) Subject: r498 - trunk/varnish-cache/bin/varnishd Message-ID: <20060719115306.DC1051EC3A3@projects.linpro.no> Author: phk Date: 2006-07-19 13:53:06 +0200 (Wed, 19 Jul 2006) New Revision: 498 Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c Log: Avoid the Error path for now. Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-19 11:11:48 UTC (rev 497) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-19 11:53:06 UTC (rev 498) @@ -148,6 +148,7 @@ break; default: sp->obj->cacheable = 0; + sp->obj->valid = 1; /* XXX ? */ body = 0; break; } From phk at projects.linpro.no Wed Jul 19 12:36:42 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 14:36:42 +0200 (CEST) Subject: r499 - trunk/varnish-cache/include Message-ID: <20060719123642.C73D81EC399@projects.linpro.no> Author: phk Date: 2006-07-19 14:36:42 +0200 (Wed, 19 Jul 2006) New Revision: 499 Added: trunk/varnish-cache/include/miniobj.h Log: Add miniobj.h for debugging Added: trunk/varnish-cache/include/miniobj.h =================================================================== --- trunk/varnish-cache/include/miniobj.h 2006-07-19 11:53:06 UTC (rev 498) +++ trunk/varnish-cache/include/miniobj.h 2006-07-19 12:36:42 UTC (rev 499) @@ -0,0 +1,33 @@ +/* $Id$ */ + +#define FREE_OBJ(to) \ + do { \ + (to)->magic = (0); \ + free(to); \ + } while (0) + +#define CHECK_OBJ(ptr, type_magic) \ + do { \ + assert((ptr)->magic == type_magic); \ + } while (0) + +#define CHECK_OBJ_NOTNULL(ptr, type_magic) \ + do { \ + assert((ptr) != NULL); \ + assert((ptr)->magic == type_magic); \ + } while (0) + +#define CAST_OBJ(to, from, type_magic) \ + do { \ + (to) = (from); \ + if ((to) != NULL) \ + CHECK_OBJ((to), (type_magic)); \ + } while (0); + +#define CAST_OBJ_NOTNULL(to, from, type_magic) \ + do { \ + (to) = (from); \ + assert((to) != NULL); \ + CHECK_OBJ((to), (type_magic)); \ + } while (0); + From phk at projects.linpro.no Wed Jul 19 12:37:04 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 14:37:04 +0200 (CEST) Subject: r500 - trunk/varnish-cache/bin/varnishd Message-ID: <20060719123704.506A71EC399@projects.linpro.no> Author: phk Date: 2006-07-19 14:37:03 +0200 (Wed, 19 Jul 2006) New Revision: 500 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_session.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishd/hash_classic.c Log: Use miniobj.h to catch pointer trouble Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-19 12:36:42 UTC (rev 499) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-19 12:37:03 UTC (rev 500) @@ -13,6 +13,7 @@ #include "vcl_returns.h" #include "common.h" +#include "miniobj.h" #define MAX_IOVS 10 @@ -39,6 +40,8 @@ typedef void http_callback_f(void *, int bad); struct http { + unsigned magic; +#define HTTP_MAGIC 0x6428b5c9 struct event ev; http_callback_f *callback; void *arg; @@ -61,6 +64,8 @@ /*--------------------------------------------------------------------*/ struct worker { + unsigned magic; +#define WORKER_MAGIC 0x6391adcf struct event_base *eb; struct sbuf *sb; struct objhead *nobjhead; @@ -76,6 +81,8 @@ }; struct workreq { + unsigned magic; +#define WORKREQ_MAGIC 0x5ccb4eb2 TAILQ_ENTRY(workreq) list; struct sess *sess; }; @@ -85,6 +92,8 @@ /* Backend Connection ------------------------------------------------*/ struct vbe_conn { + unsigned magic; +#define VBE_CONN_MAGIC 0x0c5e6592 TAILQ_ENTRY(vbe_conn) list; struct vbc_mem *vbcm; struct vbe *vbe; @@ -97,6 +106,8 @@ /* Storage -----------------------------------------------------------*/ struct storage { + unsigned magic; +#define STORAGE_MAGIC 0x1a4e51c0 TAILQ_ENTRY(storage) list; unsigned char *ptr; unsigned len; @@ -117,6 +128,8 @@ /* -------------------------------------------------------------------*/ struct object { + unsigned magic; +#define OBJECT_MAGIC 0x32851d42 unsigned refcnt; unsigned xid; struct objhead *objhead; @@ -149,6 +162,8 @@ }; struct objhead { + unsigned magic; +#define OBJHEAD_MAGIC 0x1b96615d void *hashpriv; pthread_mutex_t mtx; @@ -158,6 +173,8 @@ /* -------------------------------------------------------------------*/ struct srcaddr { + unsigned magic; +#define SRCADDR_MAGIC 0x375111db TAILQ_ENTRY(srcaddr) list; unsigned nsess; char addr[TCP_ADDRBUFSIZE]; @@ -169,6 +186,8 @@ }; struct sess { + unsigned magic; +#define SESS_MAGIC 0x2c2f9c5a int fd; unsigned xid; @@ -204,6 +223,8 @@ }; struct backend { + unsigned magic; +#define BACKEND_MAGIC 0x64c4c7c6 const char *vcl_name; const char *hostname; const char *portname; Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-19 12:36:42 UTC (rev 499) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-19 12:37:03 UTC (rev 500) @@ -40,6 +40,8 @@ #include "cache.h" struct vbc_mem { + unsigned magic; +#define VBC_MEM_MAGIC 0x2fd7af01 struct vbe_conn vbe; struct http http; char *http_hdr; @@ -48,6 +50,8 @@ /* A backend IP */ struct vbe { + unsigned magic; +#define VBE_MAGIC 0x079648f0 unsigned ip; TAILQ_ENTRY(vbe) list; TAILQ_HEAD(,vbe_conn) fconn; @@ -78,7 +82,9 @@ 1); if (vbcm == NULL) return (NULL); + vbcm->magic = VBC_MEM_MAGIC; VSL_stats->n_vbe_conn++; + vbcm->vbe.magic = VBE_CONN_MAGIC; vbcm->vbe.vbcm = vbcm; vbcm->vbe.http = &vbcm->http; http_Init(&vbcm->http, (void *)(vbcm + 1)); @@ -89,6 +95,8 @@ vbe_delete_conn(struct vbe_conn *vb) { + CHECK_OBJ_NOTNULL(vb, VBE_CONN_MAGIC); + CHECK_OBJ_NOTNULL(vb->vbcm, VBC_MEM_MAGIC); VSL_stats->n_vbe_conn--; free(vb->vbcm); } @@ -299,6 +307,7 @@ struct vbe *vp; struct vbe_conn *vc; + CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC); AZ(pthread_mutex_lock(&vbemtx)); vp = bp->vbe; if (vp == NULL) { Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-19 12:36:42 UTC (rev 499) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-19 12:37:03 UTC (rev 500) @@ -566,8 +566,21 @@ CNT_Session(struct sess *sp) { int done; + struct worker *w; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + w = sp->wrk; + CHECK_OBJ_NOTNULL(w, WORKER_MAGIC); + for (done = 0; !done; ) { + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + if (sp->obj != NULL) + CHECK_OBJ(sp->obj, OBJECT_MAGIC); + CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); + if (w->nobj != NULL) + CHECK_OBJ(w->nobj, OBJECT_MAGIC); + if (w->nobjhead != NULL) + CHECK_OBJ(w->nobjhead, OBJHEAD_MAGIC); switch (sp->step) { #define STEP(l,u) \ case STP_##u: \ @@ -578,6 +591,10 @@ #undef STEP default: INCOMPL(); } + if (w->nobj != NULL) + CHECK_OBJ(w->nobj, OBJECT_MAGIC); + if (w->nobjhead != NULL) + CHECK_OBJ(w->nobjhead, OBJHEAD_MAGIC); } } Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-19 12:36:42 UTC (rev 499) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-19 12:37:03 UTC (rev 500) @@ -6,6 +6,7 @@ #include #include +#include #include "libvarnish.h" #include "shmlog.h" @@ -56,6 +57,7 @@ t = time(NULL); AZ(pthread_mutex_lock(&exp_mtx)); TAILQ_FOREACH(o, &exp_deathrow, deathrow) { + CHECK_OBJ(o, OBJECT_MAGIC); if (o->ttl >= t) { o = NULL; break; @@ -94,14 +96,17 @@ { struct object *o; time_t t; - struct sess sp; + struct sess *sp; (void)arg; + sp = SES_New(NULL, 0); while (1) { t = time(NULL); AZ(pthread_mutex_lock(&exp_mtx)); o = binheap_root(exp_heap); + if (o != NULL) + CHECK_OBJ(o, OBJECT_MAGIC); if (o == NULL || o->ttl > t + expearly) { AZ(pthread_mutex_unlock(&exp_mtx)); AZ(sleep(1)); @@ -111,18 +116,18 @@ AZ(pthread_mutex_unlock(&exp_mtx)); VSL(SLT_ExpPick, 0, "%u", o->xid); - sp.vcl = VCL_Get(); - sp.obj = o; - VCL_timeout_method(&sp); - VCL_Rel(sp.vcl); + sp->vcl = VCL_Get(); + sp->obj = o; + VCL_timeout_method(sp); + VCL_Rel(sp->vcl); - if (sp.handling == VCL_RET_DISCARD) { + if (sp->handling == VCL_RET_DISCARD) { AZ(pthread_mutex_lock(&exp_mtx)); TAILQ_INSERT_TAIL(&exp_deathrow, o, deathrow); AZ(pthread_mutex_unlock(&exp_mtx)); continue; } - assert(sp.handling == VCL_RET_DISCARD); + assert(sp->handling == VCL_RET_DISCARD); } } Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-19 12:36:42 UTC (rev 499) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-19 12:37:03 UTC (rev 500) @@ -48,6 +48,9 @@ struct object *o; char *c; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC); assert(hash != NULL); w = sp->wrk; h = sp->http; @@ -56,29 +59,36 @@ if (w->nobjhead == NULL) { w->nobjhead = calloc(sizeof *w->nobjhead, 1); assert(w->nobjhead != NULL); + w->nobjhead->magic = OBJHEAD_MAGIC; TAILQ_INIT(&w->nobjhead->objects); AZ(pthread_mutex_init(&w->nobjhead->mtx, NULL)); VSL_stats->n_objecthead++; - } + } else + CHECK_OBJ_NOTNULL(w->nobjhead, OBJHEAD_MAGIC); if (w->nobj == NULL) { w->nobj = calloc(sizeof *w->nobj, 1); assert(w->nobj != NULL); + w->nobj->magic = OBJECT_MAGIC; w->nobj->busy = 1; w->nobj->refcnt = 1; TAILQ_INIT(&w->nobj->store); TAILQ_INIT(&w->nobj->waitinglist); VSL_stats->n_object++; - } + } else + CHECK_OBJ_NOTNULL(w->nobj, OBJECT_MAGIC); if (!http_GetHdr(h, "Host", &c)) c = h->url; if (sp->obj != NULL) { + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); o = sp->obj; oh = o->objhead; + CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); AZ(pthread_mutex_lock(&oh->mtx)); goto were_back; } oh = hash->lookup(h->url, c, w->nobjhead); + CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); if (oh == w->nobjhead) w->nobjhead = NULL; AZ(pthread_mutex_lock(&oh->mtx)); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-19 12:36:42 UTC (rev 499) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-19 12:37:03 UTC (rev 500) @@ -23,6 +23,7 @@ char *sp = space; memset(hp, 0, sizeof *hp); + hp->magic = HTTP_MAGIC; hp->hdr = (void *)sp; sp += heritage.mem_http_headers * sizeof hp->hdr; hp->s = sp; Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-19 12:36:42 UTC (rev 499) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-19 12:37:03 UTC (rev 500) @@ -35,6 +35,7 @@ w = &ww; memset(w, 0, sizeof *w); + w->magic = WORKER_MAGIC; AZ(pthread_cond_init(&w->cv, NULL)); @@ -54,6 +55,11 @@ } TAILQ_INSERT_HEAD(&wrk_head, w, list); while (1) { + CHECK_OBJ_NOTNULL(w, WORKER_MAGIC); + if (w->nobj != NULL) + CHECK_OBJ(w->nobj, OBJECT_MAGIC); + if (w->nobjhead != NULL) + CHECK_OBJ(w->nobjhead, OBJHEAD_MAGIC); wrq = TAILQ_FIRST(&wrk_reqhead); if (wrq != NULL) { VSL_stats->n_wrk_busy++; @@ -62,6 +68,7 @@ AZ(pthread_mutex_unlock(&wrk_mtx)); assert(wrq->sess != NULL); wrq->sess->wrk = w; + CHECK_OBJ_NOTNULL(wrq->sess, SESS_MAGIC); CNT_Session(wrq->sess); AZ(pthread_mutex_lock(&wrk_mtx)); VSL_stats->n_wrk_busy--; @@ -71,12 +78,20 @@ wrk_overflow--; continue; } + if (w->nobj != NULL) + CHECK_OBJ(w->nobj, OBJECT_MAGIC); + if (w->nobjhead != NULL) + CHECK_OBJ(w->nobjhead, OBJHEAD_MAGIC); /* If we are a reserved thread we don't die */ if (priv != NULL) { AZ(pthread_cond_wait(&w->cv, &wrk_mtx)); continue; } + if (w->nobj != NULL) + CHECK_OBJ(w->nobj, OBJECT_MAGIC); + if (w->nobjhead != NULL) + CHECK_OBJ(w->nobjhead, OBJHEAD_MAGIC); /* If we are a dynamic thread, time out and die */ AZ(clock_gettime(CLOCK_REALTIME, &ts)); @@ -91,6 +106,10 @@ AZ(pthread_cond_destroy(&w->cv)); return (NULL); } + if (w->nobj != NULL) + CHECK_OBJ(w->nobj, OBJECT_MAGIC); + if (w->nobjhead != NULL) + CHECK_OBJ(w->nobjhead, OBJHEAD_MAGIC); } } Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-19 12:36:42 UTC (rev 499) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-19 12:37:03 UTC (rev 500) @@ -27,6 +27,9 @@ /*--------------------------------------------------------------------*/ struct sessmem { + unsigned magic; +#define SESSMEM_MAGIC 0x555859c5 + struct sess sess; struct http http; char *http_hdr; @@ -156,7 +159,9 @@ 1); if (sm == NULL) return (NULL); + sm->magic = SESSMEM_MAGIC; VSL_stats->n_sess++; + sm->sess.magic = SESS_MAGIC; sm->sess.mem = sm; sm->sess.http = &sm->http; http_Init(&sm->http, (void *)(sm + 1)); @@ -167,8 +172,10 @@ SES_Delete(struct sess *sp) { + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); VSL_stats->n_sess--; SES_RelSrcAddr(sp); + CHECK_OBJ_NOTNULL(sp->mem, SESSMEM_MAGIC); free(sp->mem); } Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-19 12:36:42 UTC (rev 499) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-19 12:37:03 UTC (rev 500) @@ -20,7 +20,7 @@ VRT_error(struct sess *sp, unsigned err, const char *str) { - (void)sp; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); VSL(SLT_Debug, 0, "VCL_error(%u, %s)", err, str); } @@ -30,6 +30,7 @@ VRT_count(struct sess *sp, unsigned u) { + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); VSL(SLT_VCL_trace, sp->fd, "%u %d.%d", u, sp->vcl->ref[u].line, sp->vcl->ref[u].pos); @@ -42,6 +43,7 @@ { char *p; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); assert(sp != NULL); assert(sp->http != NULL); if (!http_GetHdr(sp->http, n, &p)) @@ -55,6 +57,7 @@ VRT_GetReq(struct sess *sp) { + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); assert(sp != NULL); assert(sp->http != NULL); return (sp->http->req); @@ -66,6 +69,7 @@ VRT_handling(struct sess *sp, unsigned hand) { + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); assert(!(hand & (hand -1))); /* must be power of two */ sp->handling = hand; } @@ -73,30 +77,35 @@ int VRT_obj_valid(struct sess *sp) { + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); return (sp->obj->valid); } int VRT_obj_cacheable(struct sess *sp) { + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); return (sp->obj->cacheable); } void VRT_set_backend_hostname(struct backend *be, const char *h) { + CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); be->hostname = h; } void VRT_set_backend_portname(struct backend *be, const char *p) { + CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); be->portname = p; } void VRT_set_backend_name(struct backend *be, const char *p) { + CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); be->vcl_name = p; } @@ -110,5 +119,6 @@ for (i = 0; i < cp->nbackend; i++) { cp->backend[i] = calloc(sizeof *cp->backend[i], 1); assert(cp->backend[i] != NULL); + cp->backend[i]->magic = BACKEND_MAGIC; } } Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-19 12:36:42 UTC (rev 499) +++ trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-19 12:37:03 UTC (rev 500) @@ -16,10 +16,12 @@ /*--------------------------------------------------------------------*/ struct hcl_entry { + unsigned magic; +#define HCL_ENTRY_MAGIC 0x0ba707bf TAILQ_ENTRY(hcl_entry) list; char *key1; char *key2; - struct objhead *obj; + struct objhead *oh; unsigned refcnt; unsigned hash; unsigned mtx; @@ -98,7 +100,7 @@ */ static struct objhead * -hcl_lookup(const char *key1, const char *key2, struct objhead *nobj) +hcl_lookup(const char *key1, const char *key2, struct objhead *noh) { struct hcl_entry *he, *he2; MD5_CTX c; @@ -106,6 +108,7 @@ unsigned u1, u2; int i; + CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC); MD5Init(&c); MD5Update(&c, key1, strlen(key1)); MD5Update(&c, "", 1); @@ -118,6 +121,7 @@ AZ(pthread_mutex_lock(&hcl_mutex[u2])); TAILQ_FOREACH(he, &hcl_head[u1], list) { + CHECK_OBJ_NOTNULL(he, HCL_ENTRY_MAGIC); i = strcmp(key1, he->key1); if (i < 0) continue; @@ -129,18 +133,19 @@ if (i > 0) break; he->refcnt++; - nobj = he->obj; - nobj->hashpriv = he; + noh = he->oh; + noh->hashpriv = he; AZ(pthread_mutex_unlock(&hcl_mutex[u2])); - return (nobj); + return (noh); } - if (nobj == NULL) { + if (noh == NULL) { AZ(pthread_mutex_unlock(&hcl_mutex[u2])); return (NULL); } he2 = calloc(sizeof *he2, 1); assert(he2 != NULL); - he2->obj = nobj; + he2->magic = HCL_ENTRY_MAGIC; + he2->oh = noh; he2->refcnt = 1; he2->hash = u1; he2->mtx = u2; @@ -148,13 +153,13 @@ assert(he2->key1 != NULL); he2->key2 = strdup(key2); assert(he2->key2 != NULL); - nobj->hashpriv = he2; + noh->hashpriv = he2; if (he != NULL) TAILQ_INSERT_BEFORE(he, he2, list); else TAILQ_INSERT_TAIL(&hcl_head[u1], he2, list); AZ(pthread_mutex_unlock(&hcl_mutex[u2])); - return (nobj); + return (noh); } /*-------------------------------------------------------------------- @@ -162,14 +167,14 @@ */ static int -hcl_deref(struct objhead *obj) +hcl_deref(struct objhead *oh) { struct hcl_entry *he; int ret; unsigned mtx; - assert(obj->hashpriv != NULL); - he = obj->hashpriv; + CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); + CAST_OBJ_NOTNULL(he, oh->hashpriv, HCL_ENTRY_MAGIC); mtx = he->mtx; AZ(pthread_mutex_lock(&hcl_mutex[mtx])); if (--he->refcnt == 0) { From phk at projects.linpro.no Wed Jul 19 19:43:39 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 21:43:39 +0200 (CEST) Subject: r501 - trunk/varnish-cache/bin/varnishd Message-ID: <20060719194339.506F11EC3AD@projects.linpro.no> Author: phk Date: 2006-07-19 21:43:39 +0200 (Wed, 19 Jul 2006) New Revision: 501 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Make sure hp->v is NUL terminated. Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-19 12:37:03 UTC (rev 500) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-19 19:43:39 UTC (rev 501) @@ -402,6 +402,7 @@ memmove(hp->s, hp->t, l); hp->v = hp->s + l; hp->t = hp->s; + *hp->v = '\0'; if (http_header_complete(hp)) { assert(func != NULL); func(arg, 0); From phk at projects.linpro.no Wed Jul 19 19:45:14 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 21:45:14 +0200 (CEST) Subject: r502 - trunk/varnish-cache/bin/varnishd Message-ID: <20060719194514.AE9BE1EC3AB@projects.linpro.no> Author: phk Date: 2006-07-19 21:45:14 +0200 (Wed, 19 Jul 2006) New Revision: 502 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: Don't bother determining if we should close if we already have done so. Also: we may not have valid headers if cache_http.c threw a 400. Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-19 19:43:39 UTC (rev 501) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-19 19:45:14 UTC (rev 502) @@ -85,7 +85,9 @@ char *b; assert(sp->obj == NULL); - if (http_GetHdr(sp->http, "Connection", &b) && + if (sp->fd < 0) { + /* Allready closed */ + } else if (http_GetHdr(sp->http, "Connection", &b) && !strcmp(b, "close")) { vca_close_session(sp, "Connection header"); } else if (strcmp(sp->http->proto, "HTTP/1.1")) { From phk at projects.linpro.no Wed Jul 19 19:45:51 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 21:45:51 +0200 (CEST) Subject: r503 - trunk/varnish-cache/bin/varnishd Message-ID: <20060719194551.012C51EC3AB@projects.linpro.no> Author: phk Date: 2006-07-19 21:45:51 +0200 (Wed, 19 Jul 2006) New Revision: 503 Modified: trunk/varnish-cache/bin/varnishd/cache_session.c Log: Delete the right list item. Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-19 19:45:14 UTC (rev 502) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-19 19:45:51 UTC (rev 503) @@ -87,8 +87,8 @@ c3 = c; continue; } - TAILQ_REMOVE(ch, c2, list); - free(c2); + TAILQ_REMOVE(ch, c, list); + free(c); VSL_stats->n_srcaddr--; } if (c3 == NULL) { From phk at projects.linpro.no Wed Jul 19 19:47:29 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 21:47:29 +0200 (CEST) Subject: r504 - trunk/varnish-cache/bin/varnishd Message-ID: <20060719194729.E11541EC399@projects.linpro.no> Author: phk Date: 2006-07-19 21:47:29 +0200 (Wed, 19 Jul 2006) New Revision: 504 Modified: trunk/varnish-cache/bin/varnishd/cache_session.c Log: Don't panic on NULL srcaddr, but revisit later when we know the details. Is it the Prefetcher ? Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-19 19:45:51 UTC (rev 503) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-19 19:47:29 UTC (rev 504) @@ -135,6 +135,11 @@ SES_RelSrcAddr(struct sess *sp) { + if (sp->srcaddr == NULL) { + /* XXX who comes this way ? */ + VSL(SLT_Debug, sp->fd, "had no srcaddr"); + return; + } assert(sp->srcaddr != NULL); AZ(pthread_mutex_lock(&ses_mtx)); sp->srcaddr->nsess--; From phk at projects.linpro.no Wed Jul 19 19:48:23 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 21:48:23 +0200 (CEST) Subject: r505 - trunk/varnish-cache/bin/varnishd Message-ID: <20060719194823.7FA591EC381@projects.linpro.no> Author: phk Date: 2006-07-19 21:48:23 +0200 (Wed, 19 Jul 2006) New Revision: 505 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Make sure width and precision arguments to printf %*.*s are ints. Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-19 19:47:29 UTC (rev 504) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-19 19:48:23 UTC (rev 505) @@ -132,7 +132,7 @@ "backend default {\n" " set backend.host = \"%*.*s\";\n" " set backend.port = \"%s\";\n" - "}\n", p - bflag, p - bflag, bflag, q); + "}\n", (int)(p - bflag), (int)(p - bflag), bflag, q); assert(buf != NULL); sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(sb != NULL); From phk at projects.linpro.no Wed Jul 19 19:49:22 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 21:49:22 +0200 (CEST) Subject: r506 - trunk/varnish-cache/bin/varnishtop Message-ID: <20060719194922.61AC71EC3AB@projects.linpro.no> Author: phk Date: 2006-07-19 21:49:22 +0200 (Wed, 19 Jul 2006) New Revision: 506 Modified: trunk/varnish-cache/bin/varnishtop/varnishtop.c Log: Update only once per second. Modified: trunk/varnish-cache/bin/varnishtop/varnishtop.c =================================================================== --- trunk/varnish-cache/bin/varnishtop/varnishtop.c 2006-07-19 19:48:23 UTC (rev 505) +++ trunk/varnish-cache/bin/varnishtop/varnishtop.c 2006-07-19 19:49:22 UTC (rev 506) @@ -45,7 +45,14 @@ int l; double t = 0; unsigned u = 0; + static time_t last; + time_t now; + now = time(NULL); + if (now == last) + return; + last = now; + erase(); l = 0; mvprintw(0, 0, "list length %u\n", ntop); From phk at projects.linpro.no Wed Jul 19 20:06:18 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 22:06:18 +0200 (CEST) Subject: r507 - trunk/varnish-cache/bin/varnishd Message-ID: <20060719200618.AB6441EC399@projects.linpro.no> Author: phk Date: 2006-07-19 22:06:18 +0200 (Wed, 19 Jul 2006) New Revision: 507 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: Don't explode on trim's to zero size. Real fix should (maybe) be to callers Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-19 19:49:22 UTC (rev 506) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-19 20:06:18 UTC (rev 507) @@ -507,6 +507,10 @@ struct smf *smf; struct smf_sc *sc; + if (size == 0) { + /* XXX: this should not happen */ + return; + } assert(size <= s->space); assert(size > 0); /* XXX: seen */ smf = (struct smf *)(s->priv); From phk at projects.linpro.no Wed Jul 19 20:07:16 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 22:07:16 +0200 (CEST) Subject: r508 - trunk/varnish-cache/bin/varnishd Message-ID: <20060719200716.801CE1EC399@projects.linpro.no> Author: phk Date: 2006-07-19 22:07:16 +0200 (Wed, 19 Jul 2006) New Revision: 508 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Use insert_pass in vcl_fetch() so we cache the uncacheability. Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-19 20:06:18 UTC (rev 507) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-19 20:07:16 UTC (rev 508) @@ -63,7 +63,7 @@ " error;\n" " }\n" " if (!obj.cacheable) {\n" - " pass;\n" + " insert_pass;\n" " }\n" " insert;\n" "}\n" From phk at projects.linpro.no Wed Jul 19 21:14:41 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 23:14:41 +0200 (CEST) Subject: r509 - in trunk/varnish-cache: bin/varnishd bin/varnishstat include Message-ID: <20060719211441.E21E81EC3AB@projects.linpro.no> Author: phk Date: 2006-07-19 23:14:41 +0200 (Wed, 19 Jul 2006) New Revision: 509 Modified: trunk/varnish-cache/bin/varnishd/cache_main.c trunk/varnish-cache/bin/varnishd/shmlog.c trunk/varnish-cache/bin/varnishstat/varnishstat.c trunk/varnish-cache/include/stats.h Log: Include a "start_time" timestamp in the stats and teach varnishstats to print it in curses mode. Some polishing and cleanup. Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-19 20:07:16 UTC (rev 508) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-19 21:14:41 UTC (rev 509) @@ -130,6 +130,7 @@ arm_keepalive(); printf("Ready\n"); + VSL_stats->start_time = time(NULL); i = event_base_loop(eb, 0); if (i != 0) printf("event_dispatch() = %d\n", i); Modified: trunk/varnish-cache/bin/varnishd/shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-19 20:07:16 UTC (rev 508) +++ trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-19 21:14:41 UTC (rev 509) @@ -136,6 +136,7 @@ AZ(pthread_mutex_init(&vsl_mutex, NULL)); loghead->starttime = time(NULL); VSL_stats = &loghead->stats; + memset(VSL_stats, 0, sizeof *VSL_stats); } /*--------------------------------------------------------------------*/ @@ -184,7 +185,6 @@ * management process as well. */ VSL_Init(); - memset(VSL_stats, 0, sizeof *VSL_stats); } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-07-19 20:07:16 UTC (rev 508) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-07-19 21:14:41 UTC (rev 509) @@ -24,23 +24,82 @@ (*acc) += (val - *acc) / (double)*n; } -int -main(int argc, char **argv) +static void +do_curses(struct varnish_stats *VSL_stats) { - int c; - struct varnish_stats *VSL_stats, copy; - int c_flag = 0; + struct varnish_stats copy; intmax_t ju; struct timespec ts; double tt, lt, hit, miss, ratio; double a1, a2, a3; unsigned n1, n2, n3; + time_t rt; + int i; - a1 = a2 = a3 = 0; + + memset(©, 0, sizeof copy); + + a1 = a2 = a3 = 0.0; n1 = n2 = n3 = 0; + initscr(); + erase(); + lt = 0; + while (1) { + clock_gettime(CLOCK_REALTIME, &ts); + tt = ts.tv_nsec * 1e-9 + ts.tv_sec; + lt = tt - lt; + rt = ts.tv_sec - VSL_stats->start_time; + + move(0,0); + i = 0; + if (rt > 86400) { + printw("%dd+", rt / 86400); + rt %= 86400; + i++; + } + printw("%02d:", rt / 3600); + rt %= 3600; + printw("%02d:", rt / 60); + rt %= 60; + printw("%02d\n", rt); + hit = (intmax_t)VSL_stats->cache_hit - + (intmax_t)copy.cache_hit; + miss = (intmax_t)VSL_stats->cache_miss - + (intmax_t)copy.cache_miss; + hit /= lt; + miss /= lt; + if (hit + miss != 0) { + ratio = hit / (hit + miss); + myexp(&a1, ratio, &n1, 10); + myexp(&a2, ratio, &n2, 100); + myexp(&a3, ratio, &n3, 1000); + } + printw("Hitrate ratio: %8u %8u %8u\n", n1, n2, n3); + printw("Hitrate avg: %8.4f %8.4f %8.4f\n", a1, a2, a3); + printw("\n"); + +#define MAC_STAT(n,t,f,d) \ + ju = VSL_stats->n; \ + printw("%12ju %10.2f " d "\n", ju, (ju - (intmax_t)copy.n)/lt); \ + copy.n = ju; +#include "stat_field.h" +#undef MAC_STAT + lt = tt; + refresh(); + sleep(1); + } +} + +int +main(int argc, char **argv) +{ + int c; + struct varnish_stats *VSL_stats; + int c_flag = 0; + VSL_stats = VSL_OpenStats(); while ((c = getopt(argc, argv, "c")) != -1) { @@ -55,41 +114,7 @@ } if (c_flag) { - memset(©, 0, sizeof copy); - initscr(); - erase(); - - while (1) { - clock_gettime(CLOCK_MONOTONIC, &ts); - tt = ts.tv_nsec * 1e-9 + ts.tv_sec; - lt = tt - lt; - move(0,0); - hit = (intmax_t)VSL_stats->cache_hit - - (intmax_t)copy.cache_hit; - miss = (intmax_t)VSL_stats->cache_miss - - (intmax_t)copy.cache_miss; - hit /= lt; - miss /= lt; - if (hit + miss != 0) { - ratio = hit / (hit + miss); - myexp(&a1, ratio, &n1, 10); - myexp(&a2, ratio, &n2, 100); - myexp(&a3, ratio, &n3, 1000); - } - printw("Hitrate ratio: %8u %8u %8u\n", n1, n2, n3); - printw("Hitrate avg: %8.4f %8.4f %8.4f\n", a1, a2, a3); - printw("\n"); - -#define MAC_STAT(n,t,f,d) \ - ju = VSL_stats->n; \ - printw("%12ju %10.2f " d "\n", ju, (ju - (intmax_t)copy.n)/lt); \ - copy.n = ju; -#include "stat_field.h" -#undef MAC_STAT - lt = tt; - refresh(); - sleep(1); - } + do_curses(VSL_stats); } else { #define MAC_STAT(n,t,f,d) \ Modified: trunk/varnish-cache/include/stats.h =================================================================== --- trunk/varnish-cache/include/stats.h 2006-07-19 20:07:16 UTC (rev 508) +++ trunk/varnish-cache/include/stats.h 2006-07-19 21:14:41 UTC (rev 509) @@ -3,6 +3,7 @@ #include struct varnish_stats { + time_t start_time; #define MAC_STAT(n,t,f,e) t n; #include "stat_field.h" #undef MAC_STAT From phk at projects.linpro.no Wed Jul 19 21:16:03 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 19 Jul 2006 23:16:03 +0200 (CEST) Subject: r510 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060719211603.EDB241EC3AB@projects.linpro.no> Author: phk Date: 2006-07-19 23:16:03 +0200 (Wed, 19 Jul 2006) New Revision: 510 Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/include/stat_field.h Log: Rework the worker thread pool logic slightly, we were leaking threads before. Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-19 21:14:41 UTC (rev 509) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-19 21:16:03 UTC (rev 510) @@ -26,11 +26,37 @@ /*--------------------------------------------------------------------*/ +static void +wrk_do_one(struct worker *w) +{ + struct workreq *wrq; + + wrq = TAILQ_FIRST(&wrk_reqhead); + assert(wrq != NULL); + VSL_stats->n_wrk_busy++; + TAILQ_REMOVE(&wrk_reqhead, wrq, list); + VSL_stats->n_wrk_queue--; + AZ(pthread_mutex_unlock(&wrk_mtx)); + assert(wrq->sess != NULL); + wrq->sess->wrk = w; + CHECK_OBJ_NOTNULL(wrq->sess, SESS_MAGIC); + if (w->nobj != NULL) + CHECK_OBJ(w->nobj, OBJECT_MAGIC); + if (w->nobjhead != NULL) + CHECK_OBJ(w->nobjhead, OBJHEAD_MAGIC); + CNT_Session(wrq->sess); + if (w->nobj != NULL) + CHECK_OBJ(w->nobj, OBJECT_MAGIC); + if (w->nobjhead != NULL) + CHECK_OBJ(w->nobjhead, OBJHEAD_MAGIC); + AZ(pthread_mutex_lock(&wrk_mtx)); + VSL_stats->n_wrk_busy--; +} + static void * wrk_thread(void *priv) { struct worker *w, ww; - struct workreq *wrq; struct timespec ts; w = &ww; @@ -50,66 +76,40 @@ if (priv == NULL) { VSL_stats->n_wrk_create++; VSL(SLT_WorkThread, 0, "%u born dynamic", w->nbr); - } else { - VSL(SLT_WorkThread, 0, "%u born permanent", w->nbr); } - TAILQ_INSERT_HEAD(&wrk_head, w, list); while (1) { CHECK_OBJ_NOTNULL(w, WORKER_MAGIC); - if (w->nobj != NULL) - CHECK_OBJ(w->nobj, OBJECT_MAGIC); - if (w->nobjhead != NULL) - CHECK_OBJ(w->nobjhead, OBJHEAD_MAGIC); - wrq = TAILQ_FIRST(&wrk_reqhead); - if (wrq != NULL) { - VSL_stats->n_wrk_busy++; - TAILQ_REMOVE(&wrk_head, w, list); - TAILQ_REMOVE(&wrk_reqhead, wrq, list); - AZ(pthread_mutex_unlock(&wrk_mtx)); - assert(wrq->sess != NULL); - wrq->sess->wrk = w; - CHECK_OBJ_NOTNULL(wrq->sess, SESS_MAGIC); - CNT_Session(wrq->sess); - AZ(pthread_mutex_lock(&wrk_mtx)); - VSL_stats->n_wrk_busy--; - TAILQ_INSERT_HEAD(&wrk_head, w, list); - } + + /* Process overflow requests, if any */ if (wrk_overflow > 0) { wrk_overflow--; + wrk_do_one(w); continue; } - if (w->nobj != NULL) - CHECK_OBJ(w->nobj, OBJECT_MAGIC); - if (w->nobjhead != NULL) - CHECK_OBJ(w->nobjhead, OBJHEAD_MAGIC); + + TAILQ_INSERT_HEAD(&wrk_head, w, list); /* If we are a reserved thread we don't die */ if (priv != NULL) { AZ(pthread_cond_wait(&w->cv, &wrk_mtx)); - continue; + } else { + /* If we are a dynamic thread, time out and die */ + AZ(clock_gettime(CLOCK_REALTIME, &ts)); + ts.tv_sec += heritage.wthread_timeout; + if (pthread_cond_timedwait(&w->cv, &wrk_mtx, &ts)) { + VSL_stats->n_wrk--; + TAILQ_REMOVE(&wrk_head, w, list); + AZ(pthread_mutex_unlock(&wrk_mtx)); + VSL(SLT_WorkThread, 0, "%u suicide", w->nbr); + sbuf_delete(w->sb); + event_base_free(w->eb); + AZ(pthread_cond_destroy(&w->cv)); + return (NULL); + } } - if (w->nobj != NULL) - CHECK_OBJ(w->nobj, OBJECT_MAGIC); - if (w->nobjhead != NULL) - CHECK_OBJ(w->nobjhead, OBJHEAD_MAGIC); - /* If we are a dynamic thread, time out and die */ - AZ(clock_gettime(CLOCK_REALTIME, &ts)); - ts.tv_sec += heritage.wthread_timeout; - if (pthread_cond_timedwait(&w->cv, &wrk_mtx, &ts)) { - VSL_stats->n_wrk--; - TAILQ_REMOVE(&wrk_head, w, list); - AZ(pthread_mutex_unlock(&wrk_mtx)); - VSL(SLT_WorkThread, 0, "%u suicide", w->nbr); - sbuf_delete(w->sb); - event_base_free(w->eb); - AZ(pthread_cond_destroy(&w->cv)); - return (NULL); - } - if (w->nobj != NULL) - CHECK_OBJ(w->nobj, OBJECT_MAGIC); - if (w->nobjhead != NULL) - CHECK_OBJ(w->nobjhead, OBJHEAD_MAGIC); + /* we are already removed from wrk_head */ + wrk_do_one(w); } } @@ -127,19 +127,22 @@ AZ(pthread_mutex_lock(&wrk_mtx)); TAILQ_INSERT_TAIL(&wrk_reqhead, &sp->workreq, list); + VSL_stats->n_wrk_queue++; /* If there are idle threads, we tickle the first one into action */ w = TAILQ_FIRST(&wrk_head); if (w != NULL) { AZ(pthread_cond_signal(&w->cv)); + TAILQ_REMOVE(&wrk_head, w, list); AZ(pthread_mutex_unlock(&wrk_mtx)); return; } - /* Register overflow if max threads reached */ + wrk_overflow++; + + /* Can we create more threads ? */ if (VSL_stats->n_wrk >= heritage.wthread_max) { - wrk_overflow++; - VSL_stats->n_wrk_short++; + VSL_stats->n_wrk_max++; AZ(pthread_mutex_unlock(&wrk_mtx)); return; } @@ -147,6 +150,7 @@ /* Try to create a thread */ VSL_stats->n_wrk++; AZ(pthread_mutex_unlock(&wrk_mtx)); + if (!pthread_create(&tp, NULL, wrk_thread, NULL)) { AZ(pthread_detach(tp)); return; @@ -158,9 +162,7 @@ /* Register overflow */ AZ(pthread_mutex_lock(&wrk_mtx)); VSL_stats->n_wrk--; - wrk_overflow++; VSL_stats->n_wrk_failed++; - VSL_stats->n_wrk_short++; AZ(pthread_mutex_unlock(&wrk_mtx)); } Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2006-07-19 21:14:41 UTC (rev 509) +++ trunk/varnish-cache/include/stat_field.h 2006-07-19 21:16:03 UTC (rev 510) @@ -21,7 +21,8 @@ MAC_STAT(n_wrk, uint64_t, "u", "N worker threads") MAC_STAT(n_wrk_create, uint64_t, "u", "N worker threads created") MAC_STAT(n_wrk_failed, uint64_t, "u", "N worker threads not created") -MAC_STAT(n_wrk_short, uint64_t, "u", "N worker threads shortages") +MAC_STAT(n_wrk_max, uint64_t, "u", "N worker threads limited") MAC_STAT(n_wrk_busy, uint64_t, "u", "N busy worker threads") +MAC_STAT(n_wrk_queue, uint64_t, "u", "N queued work requests") MAC_STAT(losthdr, uint64_t, "u", "HTTP header overflows") From des at linpro.no Thu Jul 20 07:47:35 2006 From: des at linpro.no (Dag-Erling =?iso-8859-1?Q?Sm=F8rgrav?=) Date: Thu, 20 Jul 2006 09:47:35 +0200 Subject: r500 - trunk/varnish-cache/bin/varnishd References: <20060719123704.506A71EC399@projects.linpro.no> Message-ID: phk at projects.linpro.no writes: > + if (sp->obj != NULL) > + CHECK_OBJ(sp->obj, OBJECT_MAGIC); How about integrating the NULL test in CHECK_OBJ? DES -- Dag-Erling Sm?rgrav Senior Software Developer Linpro AS - www.linpro.no From phk at projects.linpro.no Thu Jul 20 08:25:53 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 10:25:53 +0200 (CEST) Subject: r511 - trunk/varnish-cache/bin/varnishd Message-ID: <20060720082553.A31EF1EC399@projects.linpro.no> Author: phk Date: 2006-07-20 10:25:53 +0200 (Thu, 20 Jul 2006) New Revision: 511 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_response.c Log: Move the delivery functions from acceptor to response Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-19 21:16:03 UTC (rev 510) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-20 08:25:53 UTC (rev 511) @@ -247,9 +247,6 @@ /* cache_acceptor.c */ -void vca_write(struct sess *sp, void *ptr, size_t len); -void vca_write_obj(struct worker *w, struct sess *sp); -void vca_flush(struct sess *sp); void vca_return_session(struct sess *sp); void vca_close_session(struct sess *sp, const char *why); void VCA_Init(void); @@ -339,6 +336,9 @@ /* cache_response.c */ void RES_Error(struct worker *w, struct sess *sp, int error, const char *msg); +void RES_Flush(struct sess *sp); +void RES_Write(struct sess *sp, void *ptr, size_t len); +void RES_WriteObj(struct worker *w, struct sess *sp); /* cache_vcl.c */ void VCL_Init(void); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-19 21:16:03 UTC (rev 510) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-20 08:25:53 UTC (rev 511) @@ -37,88 +37,6 @@ static struct event accept_e[2 * HERITAGE_NSOCKS]; static TAILQ_HEAD(,sess) sesshead = TAILQ_HEAD_INITIALIZER(sesshead); - -/*-------------------------------------------------------------------- - * Write data to client - * We try to use writev() if possible in order to minimize number of - * syscalls made and packets sent. It also just might allow the worker - * thread to complete the request without holding stuff locked. - */ - -void -vca_flush(struct sess *sp) -{ - int i; - - if (sp->fd < 0 || sp->wrk->niov == 0) - return; - i = writev(sp->fd, sp->wrk->iov, sp->wrk->niov); - if (i != sp->wrk->liov) - vca_close_session(sp, "remote closed"); - sp->wrk->liov = 0; - sp->wrk->niov = 0; -} - -void -vca_write(struct sess *sp, void *ptr, size_t len) -{ - - if (sp->fd < 0 || len == 0) - return; - if (sp->wrk->niov == MAX_IOVS) - vca_flush(sp); - if (sp->fd < 0) - return; - sp->wrk->iov[sp->wrk->niov].iov_base = ptr; - sp->wrk->iov[sp->wrk->niov++].iov_len = len; - sp->wrk->liov += len; -} - -void -vca_write_obj(struct worker *w, struct sess *sp) -{ - struct storage *st; - unsigned u = 0; - uint64_t bytes = 0; - - - VSL(SLT_Status, sp->fd, "%u", sp->obj->response); - VSL(SLT_Length, sp->fd, "%u", sp->obj->len); - - vca_write(sp, sp->obj->header, strlen(sp->obj->header)); - - sbuf_clear(w->sb); - sbuf_printf(w->sb, "Age: %u\r\n", - sp->obj->age + sp->t_req - sp->obj->entered); - sbuf_printf(w->sb, "Via: 1.1 varnish\r\n"); - sbuf_printf(w->sb, "X-Varnish: xid %u\r\n", sp->obj->xid); - if (strcmp(sp->http->proto, "HTTP/1.1")) - sbuf_printf(w->sb, "Connection: close\r\n"); - sbuf_printf(w->sb, "\r\n"); - sbuf_finish(w->sb); - vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); - bytes += sbuf_len(w->sb); - /* XXX: conditional request handling */ - if (!strcmp(sp->http->req, "GET")) { - TAILQ_FOREACH(st, &sp->obj->store, list) { - u += st->len; - if (st->stevedore->send == NULL) { - vca_write(sp, st->ptr, st->len); - continue; - } - st->stevedore->send(st, sp, - sp->wrk->iov, sp->wrk->niov, sp->wrk->liov); - sp->wrk->niov = 0; - sp->wrk->liov = 0; - } - assert(u == sp->obj->len); - } - SES_ChargeBytes(sp, bytes + u); - vca_flush(sp); -} - -/*--------------------------------------------------------------------*/ - static void vca_tick(int a, short b, void *c) { Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-19 21:16:03 UTC (rev 510) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-20 08:25:53 UTC (rev 511) @@ -61,7 +61,7 @@ cnt_deliver(struct sess *sp) { - vca_write_obj(sp->wrk, sp); + RES_WriteObj(sp->wrk, sp); HSH_Deref(sp->obj); sp->obj = NULL; sp->step = STP_DONE; @@ -253,7 +253,7 @@ sp->handling = VCL_RET_PASS; if (sp->handling == VCL_RET_DELIVER) { - vca_write_obj(sp->wrk, sp); + RES_WriteObj(sp->wrk, sp); HSH_Deref(sp->obj); sp->obj = NULL; sp->step = STP_DONE; Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-19 21:16:03 UTC (rev 510) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-20 08:25:53 UTC (rev 511) @@ -46,8 +46,8 @@ if (i == 0 && bi == NULL) return (1); assert(i > 0); - vca_write(sp, buf, i); - vca_flush(sp); + RES_Write(sp, buf, i); + RES_Flush(sp); cl -= i; } return (0); @@ -92,7 +92,7 @@ if (u == 0) break; - vca_write(sp, p, q - p); + RES_Write(sp, p, q - p); p = q; @@ -104,28 +104,28 @@ } if (bp - p < j) j = bp - p; - vca_write(sp, p, j); + RES_Write(sp, p, j); p += j; u -= j; } while (u > 0) { if (http_GetTail(hp, u, &b, &e)) { j = e - b; - vca_write(sp, q, j); + RES_Write(sp, q, j); u -= j; } else break; } - vca_flush(sp); + RES_Flush(sp); while (u > 0) { j = u; if (j > sizeof buf) j = sizeof buf; i = read(fd, buf, j); assert(i > 0); - vca_write(sp, buf, i); + RES_Write(sp, buf, i); u -= i; - vca_flush(sp); + RES_Flush(sp); } } return (0); @@ -150,7 +150,7 @@ http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); sbuf_cat(w->sb, "\r\n"); sbuf_finish(w->sb); - vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); + RES_Write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); if (http_GetHdr(hp, "Content-Length", &b)) cls = pass_straight(sp, vc->fd, hp, b); @@ -161,7 +161,7 @@ else { cls = pass_straight(sp, vc->fd, hp, NULL); } - vca_flush(sp); + RES_Flush(sp); if (http_GetHdr(hp, "Connection", &b) && !strcasecmp(b, "close")) cls = 1; Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-19 21:16:03 UTC (rev 510) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-20 08:25:53 UTC (rev 511) @@ -3,10 +3,12 @@ */ #include /* XXX: for NULL ?? */ +#include /* XXX: for NULL ?? */ #include #include #include "libvarnish.h" +#include "shmlog.h" #include "cache.h" @@ -60,7 +62,87 @@ " \r\n" "\r\n"); sbuf_finish(w->sb); - vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); - vca_flush(sp); + RES_Write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); + RES_Flush(sp); vca_close_session(sp, msg); } + + +/*-------------------------------------------------------------------- + * Write data to client + * We try to use writev() if possible in order to minimize number of + * syscalls made and packets sent. It also just might allow the worker + * thread to complete the request without holding stuff locked. + */ + +void +RES_Flush(struct sess *sp) +{ + int i; + + if (sp->fd < 0 || sp->wrk->niov == 0) + return; + i = writev(sp->fd, sp->wrk->iov, sp->wrk->niov); + if (i != sp->wrk->liov) + vca_close_session(sp, "remote closed"); + sp->wrk->liov = 0; + sp->wrk->niov = 0; +} + +void +RES_Write(struct sess *sp, void *ptr, size_t len) +{ + + if (sp->fd < 0 || len == 0) + return; + if (sp->wrk->niov == MAX_IOVS) + RES_Flush(sp); + if (sp->fd < 0) + return; + sp->wrk->iov[sp->wrk->niov].iov_base = ptr; + sp->wrk->iov[sp->wrk->niov++].iov_len = len; + sp->wrk->liov += len; +} + +void +RES_WriteObj(struct worker *w, struct sess *sp) +{ + struct storage *st; + unsigned u = 0; + uint64_t bytes = 0; + + + VSL(SLT_Status, sp->fd, "%u", sp->obj->response); + VSL(SLT_Length, sp->fd, "%u", sp->obj->len); + + RES_Write(sp, sp->obj->header, strlen(sp->obj->header)); + + sbuf_clear(w->sb); + sbuf_printf(w->sb, "Age: %u\r\n", + sp->obj->age + sp->t_req - sp->obj->entered); + sbuf_printf(w->sb, "Via: 1.1 varnish\r\n"); + sbuf_printf(w->sb, "X-Varnish: xid %u\r\n", sp->obj->xid); + if (strcmp(sp->http->proto, "HTTP/1.1")) + sbuf_printf(w->sb, "Connection: close\r\n"); + sbuf_printf(w->sb, "\r\n"); + sbuf_finish(w->sb); + RES_Write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); + bytes += sbuf_len(w->sb); + /* XXX: conditional request handling */ + if (!strcmp(sp->http->req, "GET")) { + TAILQ_FOREACH(st, &sp->obj->store, list) { + u += st->len; + if (st->stevedore->send == NULL) { + RES_Write(sp, st->ptr, st->len); + continue; + } + st->stevedore->send(st, sp, + sp->wrk->iov, sp->wrk->niov, sp->wrk->liov); + sp->wrk->niov = 0; + sp->wrk->liov = 0; + } + assert(u == sp->obj->len); + } + SES_ChargeBytes(sp, bytes + u); + RES_Flush(sp); +} From phk at projects.linpro.no Thu Jul 20 08:29:14 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 10:29:14 +0200 (CEST) Subject: r512 - trunk/varnish-cache/bin/varnishd Message-ID: <20060720082914.CF9681EC395@projects.linpro.no> Author: phk Date: 2006-07-20 10:29:14 +0200 (Thu, 20 Jul 2006) New Revision: 512 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_response.c Log: Remove explicit worker thread arguments. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-20 08:25:53 UTC (rev 511) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-20 08:29:14 UTC (rev 512) @@ -335,10 +335,10 @@ #endif /* cache_response.c */ -void RES_Error(struct worker *w, struct sess *sp, int error, const char *msg); +void RES_Error(struct sess *sp, int error, const char *msg); void RES_Flush(struct sess *sp); void RES_Write(struct sess *sp, void *ptr, size_t len); -void RES_WriteObj(struct worker *w, struct sess *sp); +void RES_WriteObj(struct sess *sp); /* cache_vcl.c */ void VCL_Init(void); Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-20 08:25:53 UTC (rev 511) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-20 08:29:14 UTC (rev 512) @@ -61,7 +61,7 @@ cnt_deliver(struct sess *sp) { - RES_WriteObj(sp->wrk, sp); + RES_WriteObj(sp); HSH_Deref(sp->obj); sp->obj = NULL; sp->step = STP_DONE; @@ -253,7 +253,7 @@ sp->handling = VCL_RET_PASS; if (sp->handling == VCL_RET_DELIVER) { - RES_WriteObj(sp->wrk, sp); + RES_WriteObj(sp); HSH_Deref(sp->obj); sp->obj = NULL; sp->step = STP_DONE; @@ -522,7 +522,7 @@ done = http_DissectRequest(sp->http, sp->fd); if (done != 0) { - RES_Error(sp->wrk, sp, done, NULL); + RES_Error(sp, done, NULL); sp->step = STP_DONE; return (0); } Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-20 08:25:53 UTC (rev 511) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-20 08:29:14 UTC (rev 512) @@ -15,10 +15,13 @@ /*--------------------------------------------------------------------*/ void -RES_Error(struct worker *w, struct sess *sp, int error, const char *msg) +RES_Error(struct sess *sp, int error, const char *msg) { char buf[40]; + struct sbuf *sb; + sb = sp->wrk->sb; + if (msg == NULL) { switch (error) { case 400: msg = "Bad Request"; break; @@ -27,11 +30,11 @@ } } - sbuf_clear(w->sb); - sbuf_printf(w->sb, "HTTP/1.1 %03d %s\r\n", error, msg); + sbuf_clear(sb); + sbuf_printf(sb, "HTTP/1.1 %03d %s\r\n", error, msg); TIM_format(sp->t_req, buf); - sbuf_printf(w->sb, "Date: %s\r\n", buf); - sbuf_cat(w->sb, + sbuf_printf(sb, "Date: %s\r\n", buf); + sbuf_cat(sb, "Server: Varnish\r\n" "Connection: close\r\n" "content-Type: text/html; charset=iso-8859-1\r\n" @@ -39,30 +42,30 @@ "\r\n" "\r\n" " \r\n"); - sbuf_printf(w->sb, " %03d %s\r\n", error, msg); - sbuf_cat(w->sb, + sbuf_printf(sb, " %03d %s\r\n", error, msg); + sbuf_cat(sb, " \r\n" " \r\n"); - sbuf_printf(w->sb, "

Error %03d %s

\r\n", error, msg); + sbuf_printf(sb, "

Error %03d %s

\r\n", error, msg); switch(error) { case 400: - sbuf_cat(w->sb, + sbuf_cat(sb, " Your HTTP protocol request did not make sense.\r\n"); break; case 500: default: - sbuf_cat(w->sb, + sbuf_cat(sb, " Something unexpected happened.\r\n"); break; } - sbuf_cat(w->sb, + sbuf_cat(sb, "

\r\n" " \r\n" " Varnish\r\n" " \r\n" "\r\n"); - sbuf_finish(w->sb); - RES_Write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); + sbuf_finish(sb); + RES_Write(sp, sbuf_data(sb), sbuf_len(sb)); RES_Flush(sp); vca_close_session(sp, msg); } @@ -105,29 +108,32 @@ } void -RES_WriteObj(struct worker *w, struct sess *sp) +RES_WriteObj(struct sess *sp) { struct storage *st; + struct sbuf *sb; unsigned u = 0; uint64_t bytes = 0; + sb = sp->wrk->sb; + VSL(SLT_Status, sp->fd, "%u", sp->obj->response); VSL(SLT_Length, sp->fd, "%u", sp->obj->len); RES_Write(sp, sp->obj->header, strlen(sp->obj->header)); - sbuf_clear(w->sb); - sbuf_printf(w->sb, "Age: %u\r\n", + sbuf_clear(sb); + sbuf_printf(sb, "Age: %u\r\n", sp->obj->age + sp->t_req - sp->obj->entered); - sbuf_printf(w->sb, "Via: 1.1 varnish\r\n"); - sbuf_printf(w->sb, "X-Varnish: xid %u\r\n", sp->obj->xid); + sbuf_printf(sb, "Via: 1.1 varnish\r\n"); + sbuf_printf(sb, "X-Varnish: xid %u\r\n", sp->obj->xid); if (strcmp(sp->http->proto, "HTTP/1.1")) - sbuf_printf(w->sb, "Connection: close\r\n"); - sbuf_printf(w->sb, "\r\n"); - sbuf_finish(w->sb); - RES_Write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); - bytes += sbuf_len(w->sb); + sbuf_printf(sb, "Connection: close\r\n"); + sbuf_printf(sb, "\r\n"); + sbuf_finish(sb); + RES_Write(sp, sbuf_data(sb), sbuf_len(sb)); + bytes += sbuf_len(sb); /* XXX: conditional request handling */ if (!strcmp(sp->http->req, "GET")) { TAILQ_FOREACH(st, &sp->obj->store, list) { From phk at projects.linpro.no Thu Jul 20 09:29:45 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 11:29:45 +0200 (CEST) Subject: r513 - trunk/varnish-cache/bin/varnishd Message-ID: <20060720092945.D381D1EC38D@projects.linpro.no> Author: phk Date: 2006-07-20 11:29:45 +0200 (Thu, 20 Jul 2006) New Revision: 513 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_response.c Log: Implement "If-Modified-Since" conditional queries Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-20 08:29:14 UTC (rev 512) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-20 09:29:45 UTC (rev 513) @@ -56,6 +56,8 @@ char *proto; char *status; char *response; + + unsigned conds; /* If-* headers present */ unsigned nhdr; char **hdr; @@ -151,6 +153,8 @@ time_t entered; time_t ttl; + time_t last_modified; + char *header; TAILQ_ENTRY(object) list; @@ -337,7 +341,7 @@ /* cache_response.c */ void RES_Error(struct sess *sp, int error, const char *msg); void RES_Flush(struct sess *sp); -void RES_Write(struct sess *sp, void *ptr, size_t len); +void RES_Write(struct sess *sp, const void *ptr, size_t len); void RES_WriteObj(struct sess *sp); /* cache_vcl.c */ Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-20 08:29:14 UTC (rev 512) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-20 09:29:45 UTC (rev 513) @@ -12,8 +12,10 @@ #include #include #include +#include #include "shmlog.h" +#include "libvarnish.h" #include "cache.h" /* @@ -211,6 +213,8 @@ vc = sp->vbc; hp = sp->bkd_http; + if (http_GetHdr(hp, "Last-Modified", &b)) + sp->obj->last_modified = TIM_parse(b); http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); if (body) { if (http_GetHdr(hp, "Content-Length", &b)) Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-20 08:29:14 UTC (rev 512) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-20 09:29:45 UTC (rev 513) @@ -156,7 +156,10 @@ return (strtoul(hp->status, NULL /* XXX */, 10)); } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * Dissect the headers of the HTTP protocol message. + * Detect conditionals (headers which start with '^[Ii][Ff]-') + */ static int http_dissect_hdrs(struct http *hp, int fd, char *p) @@ -167,6 +170,7 @@ p++; hp->nhdr = 0; + hp->conds = 0; r = NULL; /* For FlexeLint */ assert(p < hp->v); /* http_header_complete() guarantees this */ for (; p < hp->v; p = r) { @@ -179,6 +183,11 @@ if (p == q) break; + if ((p[0] == 'i' || p[0] == 'I') && + (p[1] == 'f' || p[1] == 'F') && + p[2] == '-') + hp->conds = 1; + if (hp->nhdr < heritage.mem_http_headers) { hp->hdr[hp->nhdr++] = p; VSLR(SLT_Header, fd, p, q); Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-20 08:29:14 UTC (rev 512) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-20 09:29:45 UTC (rev 513) @@ -93,20 +93,76 @@ } void -RES_Write(struct sess *sp, void *ptr, size_t len) +RES_Write(struct sess *sp, const void *ptr, size_t len) { if (sp->fd < 0 || len == 0) return; + if (len == -1) + len = strlen(ptr); if (sp->wrk->niov == MAX_IOVS) RES_Flush(sp); if (sp->fd < 0) return; - sp->wrk->iov[sp->wrk->niov].iov_base = ptr; + sp->wrk->iov[sp->wrk->niov].iov_base = (void*)(uintptr_t)ptr; sp->wrk->iov[sp->wrk->niov++].iov_len = len; sp->wrk->liov += len; } +/*--------------------------------------------------------------------*/ + +static void +res_do_304(struct sess *sp, char *p) +{ + struct sbuf *sb; + + sb = sp->wrk->sb; + sbuf_clear(sb); + + VSL(SLT_Status, sp->fd, "%u", 304); + VSL(SLT_Length, sp->fd, "%u", 0); + RES_Write(sp, "HTTP/1.1 304 Not Modified\r\n", -1); + RES_Write(sp, "Via: 1.1 varnish\r\n", -1); + RES_Write(sp, "Last-Modified: ", -1); + RES_Write(sp, p, -1); + RES_Write(sp, "\r\n", -1); + if (strcmp(sp->http->proto, "HTTP/1.1")) + RES_Write(sp, "Connection: close\r\n", -1); + sbuf_printf(sb, "X-Varnish: xid %u\r\n", sp->obj->xid); + sbuf_printf(sb, "\r\n"); + sbuf_finish(sb); + RES_Write(sp, sbuf_data(sb), sbuf_len(sb)); + RES_Flush(sp); +} + +/*--------------------------------------------------------------------*/ + +static int +res_do_conds(struct sess *sp) +{ + char *p; + time_t ims; + + if (sp->obj->last_modified > 0 && + http_GetHdr(sp->http, "If-Modified-Since", &p)) { + ims = TIM_parse(p); + if (ims > sp->t_req) /* [RFC2616 14.25] */ + return (0); + if (ims > sp->obj->last_modified) { + VSL(SLT_Debug, sp->fd, + "Cond: %d > %d ", sp->obj->last_modified, ims); + return (0); + } + VSL(SLT_Debug, sp->fd, + "Cond: %d <= %d", sp->obj->last_modified, ims); + res_do_304(sp, p); + return (1); + } + return (0); +} + +/*--------------------------------------------------------------------*/ + void RES_WriteObj(struct sess *sp) { @@ -118,6 +174,9 @@ sb = sp->wrk->sb; + if (sp->obj->response == 200 && sp->http->conds && res_do_conds(sp)) + return; + VSL(SLT_Status, sp->fd, "%u", sp->obj->response); VSL(SLT_Length, sp->fd, "%u", sp->obj->len); From phk at projects.linpro.no Thu Jul 20 09:42:47 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 11:42:47 +0200 (CEST) Subject: r514 - trunk/varnish-cache/bin/varnishd Message-ID: <20060720094247.8710D1EC399@projects.linpro.no> Author: phk Date: 2006-07-20 11:42:47 +0200 (Thu, 20 Jul 2006) New Revision: 514 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Loop till we have everything. Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-20 09:29:45 UTC (rev 513) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-20 09:42:47 UTC (rev 514) @@ -140,11 +140,12 @@ b += u; len -= u; } - if (len > 0) { + while (len > 0) { i = read(fd, b, len); - if (i < 0) + if (i <= 0) return (i); u += i; + len -= u; } return (u); } From phk at projects.linpro.no Thu Jul 20 09:58:11 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 11:58:11 +0200 (CEST) Subject: r515 - trunk/varnish-cache/bin/varnishd Message-ID: <20060720095811.98DC61EC399@projects.linpro.no> Author: phk Date: 2006-07-20 11:58:11 +0200 (Thu, 20 Jul 2006) New Revision: 515 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Hmm, that was a bad idea. Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-20 09:42:47 UTC (rev 514) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-20 09:58:11 UTC (rev 515) @@ -127,7 +127,7 @@ http_Read(struct http *hp, int fd, void *p, unsigned len) { int i; - unsigned u; + int u; char *b = p; u = 0; @@ -140,12 +140,11 @@ b += u; len -= u; } - while (len > 0) { + if (len > 0) { i = read(fd, b, len); - if (i <= 0) + if (i < 0) return (i); u += i; - len -= u; } return (u); } From phk at projects.linpro.no Thu Jul 20 10:10:25 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 12:10:25 +0200 (CEST) Subject: r516 - trunk/varnish-cache/bin/varnishstat Message-ID: <20060720101025.085FB1EC395@projects.linpro.no> Author: phk Date: 2006-07-20 12:10:24 +0200 (Thu, 20 Jul 2006) New Revision: 516 Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c Log: Show also average since start Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-07-20 09:58:11 UTC (rev 515) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-07-20 10:10:24 UTC (rev 516) @@ -30,7 +30,7 @@ struct varnish_stats copy; intmax_t ju; struct timespec ts; - double tt, lt, hit, miss, ratio; + double tt, lt, hit, miss, ratio, up; double a1, a2, a3; unsigned n1, n2, n3; time_t rt; @@ -52,6 +52,7 @@ lt = tt - lt; rt = ts.tv_sec - VSL_stats->start_time; + up = rt; move(0,0); i = 0; @@ -83,7 +84,8 @@ #define MAC_STAT(n,t,f,d) \ ju = VSL_stats->n; \ - printw("%12ju %10.2f " d "\n", ju, (ju - (intmax_t)copy.n)/lt); \ + printw("%12ju %10.2f %10.2f " d "\n", \ + ju, (ju - (intmax_t)copy.n)/lt, ju / up); \ copy.n = ju; #include "stat_field.h" #undef MAC_STAT From phk at projects.linpro.no Thu Jul 20 10:55:19 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 12:55:19 +0200 (CEST) Subject: r517 - trunk/varnish-doc/en/varnish-architecture Message-ID: <20060720105519.0B1B81EC3A2@projects.linpro.no> Author: phk Date: 2006-07-20 12:55:18 +0200 (Thu, 20 Jul 2006) New Revision: 517 Modified: trunk/varnish-doc/en/varnish-architecture/article.xml Log: Rewrite the "components" part to match reality. Modified: trunk/varnish-doc/en/varnish-architecture/article.xml =================================================================== --- trunk/varnish-doc/en/varnish-architecture/article.xml 2006-07-20 10:10:24 UTC (rev 516) +++ trunk/varnish-doc/en/varnish-architecture/article.xml 2006-07-20 10:55:18 UTC (rev 517) @@ -13,88 +13,198 @@ Application structure

- Components + Overview - This section lists the major components in Varnish. + + The Varnish binary contains code for two co-operating + processes: the manager and the cache engine. + + + The manager process is what takes control when the binary + is executed, and after parsing command line arguments it + will compile the VCL code and fork(2) a child process which + executes the cache-engine code. + + + + A pipe connects the two processes and allows the manager + to relay and inject CLI commands to the cache process. + + +
+ +
+ Manager Process Components + + The manager process is a basic multiplexing process, of relatively + low complexity. The only major component apart from the CLI stream + multiplexer is the VCL compiler. + + +
+ Cache Process Components + + + The cache process is where all the fun happens and its components + have been constructed for maximum efficiency at the cost of some + simplicity of structure. + +
- Listener + Acceptor - The Listener monitors the listening socket and accepts - incoming client connections. Once the connection is - established, it is passed to the Accepter. + + The Acceptor monitors the listening sockets and accepts + incoming client connections. For each connection a session + is created and once enough bytes have been received to indicate + a valid HTTP request header the established, the session is + passed to the Worker Pool for processing. + - The Listener should take advantage of accept filters or - similar technologies on systems where they are - available. + + If supported by the platform, the Acceptor will use the + accept filters facility. +
- Accepter + Worker Pool - The Accepter reads an HTTP request from a client - connection. It parses the request line and header only to the - extent necessary to establish well-formedness and determine - the requested URL. + + The Worker Pool maintains a pool of worker threads which + can process requests through the State engine. Threads + are created as necessary if possible, and if they have seen + no work for a preconfigured amount of time, they will + selfdestruct to reduce resource usage. + - The Accepter then queries the Keeper about the status of - the requested document (identified by its full URL). If the - document is present and valid in the cache, the request is - passed directly to a Sender. Otherwise, it is passed to a - Retriever queue. + + Threads are used in most-recently-used order to improve + cache efficiencies and minimize working set. +
- Keeper + State Engine - The Keeper manages the document cache. XXX + + The state engine is responsible for taking each request + through the steps. This is done with a simple finite + state engine which is able to give up the worker thread + if the session is waiting for reasons where having the + worker thread is not necessary for the waiting. + + + XXX: either list the major steps from cache_central.c here + or have a major section on the flow after the components. + (phk prefers the latter.) +
- Sender + Hash and Hash methods - The Sender transfers the contents of the requested - document to the client. It examines the HTTP request header - to determine the correct way in which to do this ? Range, - If-Modified-Since, Content-Encoding and other options may - affect the type and amount of data transferred. + + The cache of objects are hashed using a pluggable algorithm. + A central hash management does the high level work while + the actual lookup is done by the pluggable method. + +
- There may be multiple concurrent Sender threads. +
+ Storage and Storage methods + + + Like hashing, storage is split into a high level layer + which calls into pluggable methods. +
- Retriever + Pass and Pipe modes - The Retriever is responsible for retrieving documents - from the content servers. It is triggered either by an - Accepter trying to satisfy a request for a document which is - not in the cache, or by the Janitor when a ?hot? document is - nearing expiry. Either way, there may be a queue of requests - waiting for the document to arrive; when it does, the - Retriever passes those requests to a Sender. + + Requests which the can not or should not be handled by + Varnish can be either passed through or piped through to + the backend. + - There may be multiple concurrent Retriever - threads. + + Passing acts on a per-request basis and tries to make the + connection to both the client and the backend reusable. + + + + Piping acts as a transparent tunnel and whatever happens + for the rest of the lifetime of the client and backend + connection is not interpreted by Varnish. +
- Janitor + Backend sessions - The Janitor keeps track of the expiry time of cached - documents and attempts to retrieve fresh copies of documents - which are soon to expire. + + Connections to the backend are managed in a pool by the + backend session module. + +
- Logger + Logging and Statistics - The Logger keeps logs of various types of events in - circular shared-memory buffers. See for details. + + Logging and statistics is done through a shared memory + data segment to which other processes can attach to subscribe + to the data. A library provides the documented interface + for this. + - It is the responsibility of each module to feed relevant - log data to the Logger. + + Logging is done in round-robin form and is therefore unaffected + by disk-I/O or other expensive log-handling. +
+ +
+ Purge/Ban procssing + + When a purge is requested via the CLI interface, the regular + expression is added to the purge list, and all requests are + checked against this list before they are served from cache. + The most recently checked purge is cached in the objects to + avoid repeated checks against the same expression. + + +
+ VCL calls and VCL runtime + + The state engine uses calls to VCL functions to determine + desired processing of each request. The compiled VCL code + is loaded as a dynamic object and executes at the speed + of compiled code. + + + The VCL and VRT code is responsible for managing the VCL + codes loaded and to provide the proper runtime environement + for them. + +
+ +
+ Expiry (and prefetch) + + + Objects in the cache are sorted in "earliest expiry" order + in a binary heap which is monitored. When an object is + a configurable number of seconds from expiring the VCL + code will be asked to determine if the object should be + discarded or prefetched. (Prefetch is not yet implemented). + +
+
From des at linpro.no Thu Jul 20 11:11:00 2006 From: des at linpro.no (Dag-Erling =?iso-8859-1?Q?Sm=F8rgrav?=) Date: Thu, 20 Jul 2006 13:11:00 +0200 Subject: r517 - trunk/varnish-doc/en/varnish-architecture References: <20060720105519.0B1B81EC3A2@projects.linpro.no> Message-ID: phk at projects.linpro.no writes: > Modified: > trunk/varnish-doc/en/varnish-architecture/article.xml > Log: > Rewrite the "components" part to match reality. Thank you very much for taking the time to write docs. If you come across any good FAQ material, BTW, please send it to me - I plan to start working on a FAQ and an installation / administration manual RSN. DES -- Dag-Erling Sm?rgrav Senior Software Developer Linpro AS - www.linpro.no From des at projects.linpro.no Thu Jul 20 11:15:16 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Thu, 20 Jul 2006 13:15:16 +0200 (CEST) Subject: r518 - trunk/varnish-doc/en/varnish-architecture Message-ID: <20060720111516.548A51EC396@projects.linpro.no> Author: des Date: 2006-07-20 13:15:16 +0200 (Thu, 20 Jul 2006) New Revision: 518 Modified: trunk/varnish-doc/en/varnish-architecture/article.xml Log: Add missing end tags + whitespace cleanup Modified: trunk/varnish-doc/en/varnish-architecture/article.xml =================================================================== --- trunk/varnish-doc/en/varnish-architecture/article.xml 2006-07-20 10:55:18 UTC (rev 517) +++ trunk/varnish-doc/en/varnish-architecture/article.xml 2006-07-20 11:15:16 UTC (rev 518) @@ -1,4 +1,5 @@ + -
@@ -41,6 +41,7 @@ low complexity. The only major component apart from the CLI stream multiplexer is the VCL compiler. +
Cache Process Components @@ -50,7 +51,7 @@ have been constructed for maximum efficiency at the cost of some simplicity of structure. - +
Acceptor @@ -95,6 +96,7 @@ if the session is waiting for reasons where having the worker thread is not necessary for the waiting. + XXX: either list the major steps from cache_central.c here or have a major section on the flow after the components. @@ -170,6 +172,7 @@
Purge/Ban procssing + When a purge is requested via the CLI interface, the regular expression is added to the purge list, and all requests are @@ -177,15 +180,18 @@ The most recently checked purge is cached in the objects to avoid repeated checks against the same expression. +
VCL calls and VCL runtime + The state engine uses calls to VCL functions to determine - desired processing of each request. The compiled VCL code + desired processing of each request. The compiled VCL code is loaded as a dynamic object and executes at the speed of compiled code. + The VCL and VRT code is responsible for managing the VCL codes loaded and to provide the proper runtime environement From des at projects.linpro.no Thu Jul 20 11:16:01 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Thu, 20 Jul 2006 13:16:01 +0200 (CEST) Subject: r519 - trunk/varnish-doc/en/varnish-architecture Message-ID: <20060720111601.E57FD1EC3A2@projects.linpro.no> Author: des Date: 2006-07-20 13:16:01 +0200 (Thu, 20 Jul 2006) New Revision: 519 Modified: trunk/varnish-doc/en/varnish-architecture/article.xml Log: Fix stylesheet path (botched in previous commit) Modified: trunk/varnish-doc/en/varnish-architecture/article.xml =================================================================== --- trunk/varnish-doc/en/varnish-architecture/article.xml 2006-07-20 11:15:16 UTC (rev 518) +++ trunk/varnish-doc/en/varnish-architecture/article.xml 2006-07-20 11:16:01 UTC (rev 519) @@ -1,5 +1,5 @@ - + Author: des Date: 2006-07-20 13:17:44 +0200 (Thu, 20 Jul 2006) New Revision: 520 Modified: trunk/varnish-doc/en/varnish-architecture/article.xml Log: Remove reference to XSL stylesheet. Modified: trunk/varnish-doc/en/varnish-architecture/article.xml =================================================================== --- trunk/varnish-doc/en/varnish-architecture/article.xml 2006-07-20 11:16:01 UTC (rev 519) +++ trunk/varnish-doc/en/varnish-architecture/article.xml 2006-07-20 11:17:44 UTC (rev 520) @@ -1,5 +1,4 @@ - Author: phk Date: 2006-07-20 14:03:10 +0200 (Thu, 20 Jul 2006) New Revision: 521 Modified: trunk/varnish-cache/bin/varnishtop/varnishtop.c Log: Add -1 option that stops comparison after first field. Useful with commands like: varnishtop -i header -1 varnishtop -i srcaddr -1 where the variable part of the entry is less relevant. Modified: trunk/varnish-cache/bin/varnishtop/varnishtop.c =================================================================== --- trunk/varnish-cache/bin/varnishtop/varnishtop.c 2006-07-20 11:17:44 UTC (rev 520) +++ trunk/varnish-cache/bin/varnishtop/varnishtop.c 2006-07-20 12:03:10 UTC (rev 521) @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -19,7 +20,8 @@ #include "varnishapi.h" struct top { - unsigned char *rec; + unsigned char rec[4 + 255]; + unsigned clen; unsigned hash; TAILQ_ENTRY(top) list; double count; @@ -78,20 +80,25 @@ main(int argc, char **argv) { int i, c; - unsigned char *p; + unsigned char *p, *q; struct VSL_data *vd; unsigned u, v; struct top *tp, *tp2; + unsigned one_flag = 0; + vd = VSL_New(); - while ((c = getopt(argc, argv, VSL_ARGS "how:")) != -1) { + while ((c = getopt(argc, argv, VSL_ARGS "1")) != -1) { i = VSL_Arg(vd, c, optarg); if (i < 0) exit (1); if (i > 0) continue; switch (c) { + case '1': + one_flag = 1; + break; default: Usage(); } @@ -114,30 +121,34 @@ v = 0; } u = 0; - for (i = 4; i < 4 + p[1]; i++) - u += p[i]; + q = p + 4; + for (i = 0; i < p[1]; i++, q++) { + if (one_flag && (*q == ':' || isspace(*q))) + break; + u += *q; + } TAILQ_FOREACH(tp, &top_head, list) { if (tp->hash != u) continue; if (tp->rec[0] != p[0]) continue; - if (tp->rec[1] != p[1]) + if (tp->clen != q - p) continue; - if (memcmp(p + 4, tp->rec + 4, p[1])) + if (memcmp(p + 4, tp->rec + 4, q - (p + 4))) continue; tp->count += 1.0; break; } if (tp == NULL) { ntop++; - tp = calloc(sizeof *tp + 4 + p[1], 1); + tp = calloc(sizeof *tp, 1); assert(tp != NULL); tp->hash = u; - tp->rec = (void *)(tp + 1); - memcpy(tp->rec, p, 4 + p[1]); tp->count = 1.0; + tp->clen = q - p; TAILQ_INSERT_TAIL(&top_head, tp, list); - } + } + memcpy(tp->rec, p, 4 + p[1]); while (1) { tp2 = TAILQ_PREV(tp, tophead, list); if (tp2 == NULL || tp2->count >= tp->count) From ssm at projects.linpro.no Thu Jul 20 12:15:49 2006 From: ssm at projects.linpro.no (ssm at projects.linpro.no) Date: Thu, 20 Jul 2006 14:15:49 +0200 (CEST) Subject: r522 - in trunk/varnish-doc: . images Message-ID: <20060720121549.438D81EC3A2@projects.linpro.no> Author: ssm Date: 2006-07-20 14:15:49 +0200 (Thu, 20 Jul 2006) New Revision: 522 Added: trunk/varnish-doc/images/ trunk/varnish-doc/images/varnish-rfc2616-and-varnish.graffle trunk/varnish-doc/images/varnish-rfc2616-and-varnish.png trunk/varnish-doc/images/varnish-rfc2616-on-http-caches.graffle trunk/varnish-doc/images/varnish-rfc2616-on-http-caches.png Log: Added varnish / RFC2616 illustrations Added: trunk/varnish-doc/images/varnish-rfc2616-and-varnish.graffle =================================================================== --- trunk/varnish-doc/images/varnish-rfc2616-and-varnish.graffle 2006-07-20 12:03:10 UTC (rev 521) +++ trunk/varnish-doc/images/varnish-rfc2616-and-varnish.graffle 2006-07-20 12:15:49 UTC (rev 522) @@ -0,0 +1,1533 @@ + + + + + CanvasColor + + w + 1 + + ColumnAlign + 1 + ColumnSpacing + 36 + CreationDate + 2006-07-17 08:35:30 +0200 + Creator + Stig Sandbeck Mathisen + GraphDocumentVersion + 4 + GraphicsList + + + Class + LineGraphic + Head + + ID + 10 + + ID + 19 + Points + + {207.425, 358.218} + {280, 332} + {456.931, 321.957} + + Style + + shadow + + Color + + a + 0.5 + b + 0 + g + 0 + r + 0 + + Draws + YES + Fuzziness + 4 + ShadowVector + {0, 2} + + stroke + + Cap + 0 + Color + + b + 0.222222 + g + 0.222222 + r + 0.222222 + + HeadArrow + 0 + Join + 0 + LineType + 1 + TailArrow + FilledArrow + Width + 2 + + + Tail + + ID + 16 + + + + Class + LineGraphic + Head + + ID + 6 + + ID + 18 + Points + + {207.425, 390.917} + {244.426, 390.683} + + Style + + shadow + + Color + + a + 0.5 + b + 0 + g + 0 + r + 0 + + Draws + YES + Fuzziness + 4 + ShadowVector + {0, 2} + + stroke + + Cap + 0 + Color + + b + 0.222222 + g + 0.222222 + r + 0.222222 + + HeadArrow + 0 + Join + 0 + LineType + 1 + TailArrow + FilledArrow + Width + 2 + + + Tail + + ID + 16 + + + + Class + Group + Graphics + + + Class + Group + Graphics + + + Class + LineGraphic + Head + + ID + 15 + + ID + 13 + Points + + {104.022, 399.22} + {126.576, 381.607} + + Style + + shadow + + Color + + a + 0.5 + b + 0 + g + 0 + r + 0 + + Draws + YES + Fuzziness + 4 + ShadowVector + {0, 2} + + stroke + + Cap + 0 + Color + + b + 0.222222 + g + 0.222222 + r + 0.222222 + + HeadArrow + 0 + Join + 0 + LineType + 1 + TailArrow + FilledArrow + Width + 2 + + + Tail + + ID + 14 + + + + Bounds + {{34.2794, 399.22}, {85.0394, 42.5197}} + Class + ShapedGraphic + ID + 14 + Shape + RoundRect + Style + + fill + + FillType + 2 + GradientColor + + a + 0.6 + archive + + YnBsaXN0MDDU + AQIDBAUGBwpZ + JGFyY2hpdmVy + WCR2ZXJzaW9u + VCR0b3BYJG9i + amVjdHNfEA9O + U0tleWVkQXJj + aGl2ZXISAAGG + oNEICVRyb290 + gAGlCwwXGyBV + JG51bGzVDQ4P + EBESExQVFl8Q + Ek5TQ3VzdG9t + Q29sb3JTcGFj + ZVxOU0NvbXBv + bmVudHNWTlND + TVlLXE5TQ29s + b3JTcGFjZVYk + Y2xhc3OAAk8Q + NjAuMjk5OTk5 + OTggMC41OTk5 + OTk5NiAwLjI5 + OTk5OTk4IDAu + MTQ5OTk5OTkg + MC42MDAwMDAw + Mk8QNzAuMjk5 + OTkyMzggMC42 + MDAwMDAwMiAw + LjI5OTk5MjM4 + IDAuMTQ5OTk2 + MTkgMC42MDAw + MDAwMgAQBYAE + 0hgRGRpUTlNJ + RBADgAPSHB0e + EFgkY2xhc3Nl + c1okY2xhc3Nu + YW1lohAfWE5T + T2JqZWN00hwd + ISKjIiMfV05T + Q29sb3JYJU5T + Q29sb3IACAAR + ABsAJAApADIA + RABJAEwAUQBT + AFkAXwBqAH8A + jACTAKAApwCp + AOIBHAEeASAB + JQEqASwBLgEz + ATwBRwFKAVMB + WAFcAWQAAAAA + AAACAQAAAAAA + AAAkAAAAAAAA + AAAAAAAAAAAB + bQ== + + b + 0.374853 + g + 0.313649 + r + 0.501686 + + + + Text + + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc\pardirnatural + +\f0\b\fs24 \cf0 HTTP origin server} + + + + Bounds + {{111.279, 339.087}, {85.0394, 42.5197}} + Class + ShapedGraphic + ID + 15 + Shape + RoundRect + Style + + fill + + FillType + 2 + GradientColor + + a + 0.6 + archive + + YnBsaXN0MDDU + AQIDBAUGBwpZ + JGFyY2hpdmVy + WCR2ZXJzaW9u + VCR0b3BYJG9i + amVjdHNfEA9O + U0tleWVkQXJj + aGl2ZXISAAGG + oNEICVRyb290 + gAGlCwwXGyBV + JG51bGzVDQ4P + EBESExQVFl8Q + Ek5TQ3VzdG9t + Q29sb3JTcGFj + ZVxOU0NvbXBv + bmVudHNWTlND + TVlLXE5TQ29s + b3JTcGFjZVYk + Y2xhc3OAAk8Q + NjAuMjk5OTk5 + OTggMC41OTk5 + OTk5NiAwLjI5 + OTk5OTk4IDAu + MTQ5OTk5OTkg + MC42MDAwMDAw + Mk8QNzAuMjk5 + OTkyMzggMC42 + MDAwMDAwMiAw + LjI5OTk5MjM4 + IDAuMTQ5OTk2 + MTkgMC42MDAw + MDAwMgAQBYAE + 0hgRGRpUTlNJ + RBADgAPSHB0e + EFgkY2xhc3Nl + c1okY2xhc3Nu + YW1lohAfWE5T + T2JqZWN00hwd + ISKjIiMfV05T + Q29sb3JYJU5T + Q29sb3IACAAR + ABsAJAApADIA + RABJAEwAUQBT + AFkAXwBqAH8A + jACTAKAApwCp + AOIBHAEeASAB + JQEqASwBLgEz + ATwBRwFKAVMB + WAFcAWQAAAAA + AAACAQAAAAAA + AAAkAAAAAAAA + AAAAAAAAAAAB + bQ== + + b + 0.374853 + g + 0.313649 + r + 0.501686 + + + + Text + + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc\pardirnatural + +\f0\b\fs24 \cf0 Varnish HTTP Cache} + + + + ID + 12 + + + Bounds + {{23.1731, 313.547}, {184.252, 155.906}} + Class + ShapedGraphic + ID + 16 + Shape + RoundedRectangle + Style + + fill + + FillType + 2 + GradientColor + + a + 0.6 + archive + + YnBsaXN0MDDU + AQIDBAUGBwpZ + JGFyY2hpdmVy + WCR2ZXJzaW9u + VCR0b3BYJG9i + amVjdHNfEA9O + U0tleWVkQXJj + aGl2ZXISAAGG + oNEICVRyb290 + gAGlCwwXGyBV + JG51bGzVDQ4P + EBESExQVFl8Q + Ek5TQ3VzdG9t + Q29sb3JTcGFj + ZVxOU0NvbXBv + bmVudHNWTlND + TVlLXE5TQ29s + b3JTcGFjZVYk + Y2xhc3OAAk8Q + NjAuMjk5OTk5 + OTggMC41OTk5 + OTk5NiAwLjI5 + OTk5OTk4IDAu + MTQ5OTk5OTkg + MC42MDAwMDAw + Mk8QNzAuMjk5 + OTkyMzggMC42 + MDAwMDAwMiAw + LjI5OTk5MjM4 + IDAuMTQ5OTk2 + MTkgMC42MDAw + MDAwMgAQBYAE + 0hgRGRpUTlNJ + RBADgAPSHB0e + EFgkY2xhc3Nl + c1okY2xhc3Nu + YW1lohAfWE5T + T2JqZWN00hwd + ISKjIiMfV05T + Q29sb3JYJU5T + Q29sb3IACAAR + ABsAJAApADIA + RABJAEwAUQBT + AFkAXwBqAH8A + jACTAKAApwCp + AOIBHAEeASAB + JQEqASwBLgEz + ATwBRwFKAVMB + WAFcAWQAAAAA + AAACAQAAAAAA + AAAkAAAAAAAA + AAAAAAAAAAAB + bQ== + + b + 0.374853 + g + 0.313649 + r + 0.501686 + + + + Text + + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc\pardirnatural + +\f0\b\fs24 \cf0 RFC 2616 Origin Server} + + TextPlacement + 2 + + + Bounds + {{8.9999, 285.201}, {212.598, 212.598}} + Class + ShapedGraphic + ID + 17 + Shape + RoundedRectangle + Style + + fill + + FillType + 2 + + + Text + + Align + 0 + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\ql\qnatural\pardirnatural + +\f0\b\fs24 \cf0 Server domain} + + TextPlacement + 0 + + + ID + 11 + + + Class + Group + Graphics + + + Class + LineGraphic + Head + + ID + 9 + + ID + 3 + Points + + {435.677, 461.28} + {456.85, 461.28} + + Style + + shadow + + Color + + a + 0.5 + b + 0 + g + 0 + r + 0 + + Draws + YES + Fuzziness + 4 + ShadowVector + {0, 2} + + stroke + + Cap + 0 + Color + + b + 0.222222 + g + 0.222222 + r + 0.222222 + + HeadArrow + 0 + Join + 0 + LineType + 1 + TailArrow + FilledArrow + Width + 2 + + + Tail + + ID + 7 + + + + Class + LineGraphic + Head + + ID + 7 + + ID + 4 + Points + + {318.497, 411.466} + {361.605, 440.228} + + Style + + shadow + + Color + + a + 0.5 + b + 0 + g + 0 + r + 0 + + Draws + YES + Fuzziness + 4 + ShadowVector + {0, 2} + + stroke + + Cap + 0 + Color + + b + 0.222222 + g + 0.222222 + r + 0.222222 + + HeadArrow + 0 + Join + 0 + LineType + 1 + TailArrow + FilledArrow + Width + 2 + + + Tail + + ID + 6 + + + + Class + LineGraphic + Head + + ID + 8 + + ID + 5 + Points + + {329.464, 390.414} + {456.85, 390.414} + + Style + + shadow + + Color + + a + 0.5 + b + 0 + g + 0 + r + 0 + + Draws + YES + Fuzziness + 4 + ShadowVector + {0, 2} + + stroke + + Cap + 0 + Color + + b + 0.222222 + g + 0.222222 + r + 0.222222 + + HeadArrow + 0 + Join + 0 + LineType + 1 + TailArrow + FilledArrow + Width + 2 + + + Tail + + ID + 6 + + + + Bounds + {{244.425, 369.154}, {85.0394, 42.5197}} + Class + ShapedGraphic + ID + 6 + Shape + RoundRect + Style + + fill + + FillType + 2 + GradientColor + + a + 0.6 + archive + + YnBsaXN0MDDU + AQIDBAUGBwpZ + JGFyY2hpdmVy + WCR2ZXJzaW9u + VCR0b3BYJG9i + amVjdHNfEA9O + U0tleWVkQXJj + aGl2ZXISAAGG + oNEICVRyb290 + gAGlCwwXGyBV + JG51bGzVDQ4P + EBESExQVFl8Q + Ek5TQ3VzdG9t + Q29sb3JTcGFj + ZVxOU0NvbXBv + bmVudHNWTlND + TVlLXE5TQ29s + b3JTcGFjZVYk + Y2xhc3OAAk8Q + NjAuNTk5OTk5 + OTYgMC4yOTk5 + OTk5OCAwLjI5 + OTk5OTk4IDAu + MTQ5OTk5OTkg + MC42MDAwMDAw + Mk8QNzAuNjAw + MDAwMDIgMC4y + OTk5OTIzOCAw + LjI5OTk5MjM4 + IDAuMTQ5OTk2 + MTkgMC42MDAw + MDAwMgAQBYAE + 0hgRGRpUTlNJ + RBADgAPSHB0e + EFgkY2xhc3Nl + c1okY2xhc3Nu + YW1lohAfWE5T + T2JqZWN00hwd + ISKjIiMfV05T + Q29sb3JYJU5T + Q29sb3IACAAR + ABsAJAApADIA + RABJAEwAUQBT + AFkAXwBqAH8A + jACTAKAApwCp + AOIBHAEeASAB + JQEqASwBLgEz + ATwBRwFKAVMB + WAFcAWQAAAAA + AAACAQAAAAAA + AAAkAAAAAAAA + AAAAAAAAAAAB + bQ== + + b + 0.451133 + g + 0.428122 + r + 0.303029 + + + + Text + + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc\pardirnatural + +\f0\b\fs24 \cf0 Shared HTTP Cache} + + + + Bounds + {{350.638, 440.02}, {85.0394, 42.5197}} + Class + ShapedGraphic + ID + 7 + Shape + RoundRect + Style + + fill + + FillType + 2 + GradientColor + + a + 0.6 + archive + + YnBsaXN0MDDU + AQIDBAUGBwpZ + JGFyY2hpdmVy + WCR2ZXJzaW9u + VCR0b3BYJG9i + amVjdHNfEA9O + U0tleWVkQXJj + aGl2ZXISAAGG + oNEICVRyb290 + gAGlCwwXGyBV + JG51bGzVDQ4P + EBESExQVFl8Q + Ek5TQ3VzdG9t + Q29sb3JTcGFj + ZVxOU0NvbXBv + bmVudHNWTlND + TVlLXE5TQ29s + b3JTcGFjZVYk + Y2xhc3OAAk8Q + NjAuNTk5OTk5 + OTYgMC4yOTk5 + OTk5OCAwLjI5 + OTk5OTk4IDAu + MTQ5OTk5OTkg + MC42MDAwMDAw + Mk8QNzAuNjAw + MDAwMDIgMC4y + OTk5OTIzOCAw + LjI5OTk5MjM4 + IDAuMTQ5OTk2 + MTkgMC42MDAw + MDAwMgAQBYAE + 0hgRGRpUTlNJ + RBADgAPSHB0e + EFgkY2xhc3Nl + c1okY2xhc3Nu + YW1lohAfWE5T + T2JqZWN00hwd + ISKjIiMfV05T + Q29sb3JYJU5T + Q29sb3IACAAR + ABsAJAApADIA + RABJAEwAUQBT + AFkAXwBqAH8A + jACTAKAApwCp + AOIBHAEeASAB + JQEqASwBLgEz + ATwBRwFKAVMB + WAFcAWQAAAAA + AAACAQAAAAAA + AAAkAAAAAAAA + AAAAAAAAAAAB + bQ== + + b + 0.451133 + g + 0.428122 + r + 0.303029 + + + + Text + + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc\pardirnatural + +\f0\b\fs24 \cf0 Private HTTP Cache} + + + + Bounds + {{456.85, 369.154}, {85.0394, 42.5197}} + Class + ShapedGraphic + ID + 8 + Shape + RoundRect + Style + + fill + + FillType + 2 + GradientColor + + a + 0.6 + archive + + YnBsaXN0MDDU + AQIDBAUGBwpZ + JGFyY2hpdmVy + WCR2ZXJzaW9u + VCR0b3BYJG9i + amVjdHNfEA9O + U0tleWVkQXJj + aGl2ZXISAAGG + oNEICVRyb290 + gAGlCwwXGyBV + JG51bGzVDQ4P + EBESExQVFl8Q + Ek5TQ3VzdG9t + Q29sb3JTcGFj + ZVxOU0NvbXBv + bmVudHNWTlND + TVlLXE5TQ29s + b3JTcGFjZVYk + Y2xhc3OAAk8Q + NjAuNTk5OTk5 + OTYgMC4yOTk5 + OTk5OCAwLjI5 + OTk5OTk4IDAu + MTQ5OTk5OTkg + MC42MDAwMDAw + Mk8QNzAuNjAw + MDAwMDIgMC4y + OTk5OTIzOCAw + LjI5OTk5MjM4 + IDAuMTQ5OTk2 + MTkgMC42MDAw + MDAwMgAQBYAE + 0hgRGRpUTlNJ + RBADgAPSHB0e + EFgkY2xhc3Nl + c1okY2xhc3Nu + YW1lohAfWE5T + T2JqZWN00hwd + ISKjIiMfV05T + Q29sb3JYJU5T + Q29sb3IACAAR + ABsAJAApADIA + RABJAEwAUQBT + AFkAXwBqAH8A + jACTAKAApwCp + AOIBHAEeASAB + JQEqASwBLgEz + ATwBRwFKAVMB + WAFcAWQAAAAA + AAACAQAAAAAA + AAAkAAAAAAAA + AAAAAAAAAAAB + bQ== + + b + 0.451133 + g + 0.428122 + r + 0.303029 + + + + Text + + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc\pardirnatural + +\f0\b\fs24 \cf0 HTTP Client} + + + + Bounds + {{456.85, 440.02}, {85.0394, 42.5197}} + Class + ShapedGraphic + ID + 9 + Shape + RoundRect + Style + + fill + + FillType + 2 + GradientColor + + a + 0.6 + archive + + YnBsaXN0MDDU + AQIDBAUGBwpZ + JGFyY2hpdmVy + WCR2ZXJzaW9u + VCR0b3BYJG9i + amVjdHNfEA9O + U0tleWVkQXJj + aGl2ZXISAAGG + oNEICVRyb290 + gAGlCwwXGyBV + JG51bGzVDQ4P + EBESExQVFl8Q + Ek5TQ3VzdG9t + Q29sb3JTcGFj + ZVxOU0NvbXBv + bmVudHNWTlND + TVlLXE5TQ29s + b3JTcGFjZVYk + Y2xhc3OAAk8Q + NjAuNTk5OTk5 + OTYgMC4yOTk5 + OTk5OCAwLjI5 + OTk5OTk4IDAu + MTQ5OTk5OTkg + MC42MDAwMDAw + Mk8QNzAuNjAw + MDAwMDIgMC4y + OTk5OTIzOCAw + LjI5OTk5MjM4 + IDAuMTQ5OTk2 + MTkgMC42MDAw + MDAwMgAQBYAE + 0hgRGRpUTlNJ + RBADgAPSHB0e + EFgkY2xhc3Nl + c1okY2xhc3Nu + YW1lohAfWE5T + T2JqZWN00hwd + ISKjIiMfV05T + Q29sb3JYJU5T + Q29sb3IACAAR + ABsAJAApADIA + RABJAEwAUQBT + AFkAXwBqAH8A + jACTAKAApwCp + AOIBHAEeASAB + JQEqASwBLgEz + ATwBRwFKAVMB + WAFcAWQAAAAA + AAACAQAAAAAA + AAAkAAAAAAAA + AAAAAAAAAAAB + bQ== + + b + 0.451133 + g + 0.428122 + r + 0.303029 + + + + Text + + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc\pardirnatural + +\f0\b\fs24 \cf0 HTTP Client} + + + + Bounds + {{456.85, 298.288}, {85.0394, 42.5197}} + Class + ShapedGraphic + ID + 10 + Shape + RoundRect + Style + + fill + + FillType + 2 + GradientColor + + a + 0.6 + archive + + YnBsaXN0MDDU + AQIDBAUGBwpZ + JGFyY2hpdmVy + WCR2ZXJzaW9u + VCR0b3BYJG9i + amVjdHNfEA9O + U0tleWVkQXJj + aGl2ZXISAAGG + oNEICVRyb290 + gAGlCwwXGyBV + JG51bGzVDQ4P + EBESExQVFl8Q + Ek5TQ3VzdG9t + Q29sb3JTcGFj + ZVxOU0NvbXBv + bmVudHNWTlND + TVlLXE5TQ29s + b3JTcGFjZVYk + Y2xhc3OAAk8Q + NjAuNTk5OTk5 + OTYgMC4yOTk5 + OTk5OCAwLjI5 + OTk5OTk4IDAu + MTQ5OTk5OTkg + MC42MDAwMDAw + Mk8QNzAuNjAw + MDAwMDIgMC4y + OTk5OTIzOCAw + LjI5OTk5MjM4 + IDAuMTQ5OTk2 + MTkgMC42MDAw + MDAwMgAQBYAE + 0hgRGRpUTlNJ + RBADgAPSHB0e + EFgkY2xhc3Nl + c1okY2xhc3Nu + YW1lohAfWE5T + T2JqZWN00hwd + ISKjIiMfV05T + Q29sb3JYJU5T + Q29sb3IACAAR + ABsAJAApADIA + RABJAEwAUQBT + AFkAXwBqAH8A + jACTAKAApwCp + AOIBHAEeASAB + JQEqASwBLgEz + ATwBRwFKAVMB + WAFcAWQAAAAA + AAACAQAAAAAA + AAAkAAAAAAAA + AAAAAAAAAAAB + bQ== + + b + 0.451133 + g + 0.428122 + r + 0.303029 + + + + Text + + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc\pardirnatural + +\f0\b\fs24 \cf0 HTTP Client} + + + + ID + 2 + + + Bounds + {{237.252, 285.201}, {311.811, 212.598}} + Class + ShapedGraphic + ID + 1 + Shape + RoundedRectangle + Style + + fill + + FillType + 2 + + + Text + + Align + 0 + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\ql\qnatural\pardirnatural + +\f0\b\fs24 \cf0 Client domain} + + TextPlacement + 0 + + + GridInfo + + GridSpacing + 14.17322826385498 + MajorGridSpacing + 10 + + GuidesLocked + NO + GuidesVisible + YES + HPages + 1 + ImageCounter + 1 + IsPalette + NO + Layers + + + Lock + NO + Name + Layer 1 + Print + YES + View + YES + + + LayoutInfo + + ChildOrdering + 0 + + LinksVisible + NO + MagnetsVisible + NO + ModificationDate + 2006-07-17 08:52:39 +0200 + Modifier + Stig Sandbeck Mathisen + Orientation + 2 + PageBreaks + YES + PageSetup + + BAt0eXBlZHN0cmVhbYED6IQBQISEhAtOU1ByaW50SW5mbwGEhAhOU09iamVjdACFkoSE + hBNOU011dGFibGVEaWN0aW9uYXJ5AISEDE5TRGljdGlvbmFyeQCUhAFpF5KEhIQITlNT + dHJpbmcBlIQBKxBOU0pvYkRpc3Bvc2l0aW9uhpKEmZkPTlNQcmludFNwb29sSm9ihpKE + mZkLTlNQYXBlclNpemWGkoSEhAdOU1ZhbHVlAJSEASqEhAx7X05TU2l6ZT1mZn2cgQJT + gQNKhpKEmZkUTlNWZXJ0aWNhbFBhZ2luYXRpb26GkoSEhAhOU051bWJlcgCdm4SXlwCG + koSZmQ9OU1BhZ2VzUGVyU2hlZXSGkoShm6KXAYaShJmZFE5TVmVydGljYWxseUNlbnRl + cmVkhpKkkoSZmQ5OU1BNUGFnZUZvcm1hdIaShISEDU5TTXV0YWJsZURhdGEAhIQGTlNE + YXRhAJSXgR2mhAdbNzU5MGNdPD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRG + LTgiPz4KPCFET0NUWVBFIHBsaXN0IFBVQkxJQyAiLS8vQXBwbGUgQ29tcHV0ZXIvL0RU + RCBQTElTVCAxLjAvL0VOIiAiaHR0cDovL3d3dy5hcHBsZS5jb20vRFREcy9Qcm9wZXJ0 + eUxpc3QtMS4wLmR0ZCI+CjxwbGlzdCB2ZXJzaW9uPSIxLjAiPgo8ZGljdD4KCTxrZXk+ + Y29tLmFwcGxlLnByaW50LlBhZ2VGb3JtYXQuUE1Ib3Jpem9udGFsUmVzPC9rZXk+Cgk8 + ZGljdD4KCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQuY3JlYXRvcjwva2V5PgoJ + CTxzdHJpbmc+Y29tLmFwcGxlLnByaW50aW5nbWFuYWdlcjwvc3RyaW5nPgoJCTxrZXk+ + Y29tLmFwcGxlLnByaW50LnRpY2tldC5pdGVtQXJyYXk8L2tleT4KCQk8YXJyYXk+CgkJ + CTxkaWN0PgoJCQkJPGtleT5jb20uYXBwbGUucHJpbnQuUGFnZUZvcm1hdC5QTUhvcml6 + b250YWxSZXM8L2tleT4KCQkJCTxyZWFsPjcyPC9yZWFsPgoJCQkJPGtleT5jb20uYXBw + bGUucHJpbnQudGlja2V0LmNsaWVudDwva2V5PgoJCQkJPHN0cmluZz5jb20uYXBwbGUu + cHJpbnRpbmdtYW5hZ2VyPC9zdHJpbmc+CgkJCQk8a2V5PmNvbS5hcHBsZS5wcmludC50 + aWNrZXQubW9kRGF0ZTwva2V5PgoJCQkJPGRhdGU+MjAwMy0wMS0yNFQxNjoyODozMVo8 + L2RhdGU+CgkJCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQuc3RhdGVGbGFnPC9r + ZXk+CgkJCQk8aW50ZWdlcj4wPC9pbnRlZ2VyPgoJCQk8L2RpY3Q+CgkJPC9hcnJheT4K + CTwvZGljdD4KCTxrZXk+Y29tLmFwcGxlLnByaW50LlBhZ2VGb3JtYXQuUE1PcmllbnRh + dGlvbjwva2V5PgoJPGRpY3Q+CgkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0LmNy + ZWF0b3I8L2tleT4KCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmludGluZ21hbmFnZXI8L3N0 + cmluZz4KCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQuaXRlbUFycmF5PC9rZXk+ + CgkJPGFycmF5PgoJCQk8ZGljdD4KCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LlBhZ2VG + b3JtYXQuUE1PcmllbnRhdGlvbjwva2V5PgoJCQkJPGludGVnZXI+MTwvaW50ZWdlcj4K + CQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5jbGllbnQ8L2tleT4KCQkJCTxz + dHJpbmc+Y29tLmFwcGxlLnByaW50aW5nbWFuYWdlcjwvc3RyaW5nPgoJCQkJPGtleT5j + b20uYXBwbGUucHJpbnQudGlja2V0Lm1vZERhdGU8L2tleT4KCQkJCTxkYXRlPjIwMDMt + MDEtMjRUMTY6Mjg6MzFaPC9kYXRlPgoJCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlj + a2V0LnN0YXRlRmxhZzwva2V5PgoJCQkJPGludGVnZXI+MDwvaW50ZWdlcj4KCQkJPC9k + aWN0PgoJCTwvYXJyYXk+Cgk8L2RpY3Q+Cgk8a2V5PmNvbS5hcHBsZS5wcmludC5QYWdl + Rm9ybWF0LlBNU2NhbGluZzwva2V5PgoJPGRpY3Q+CgkJPGtleT5jb20uYXBwbGUucHJp + bnQudGlja2V0LmNyZWF0b3I8L2tleT4KCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmludGlu + Z21hbmFnZXI8L3N0cmluZz4KCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQuaXRl + bUFycmF5PC9rZXk+CgkJPGFycmF5PgoJCQk8ZGljdD4KCQkJCTxrZXk+Y29tLmFwcGxl + LnByaW50LlBhZ2VGb3JtYXQuUE1TY2FsaW5nPC9rZXk+CgkJCQk8cmVhbD4xPC9yZWFs + PgoJCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0LmNsaWVudDwva2V5PgoJCQkJ + PHN0cmluZz5jb20uYXBwbGUucHJpbnRpbmdtYW5hZ2VyPC9zdHJpbmc+CgkJCQk8a2V5 + PmNvbS5hcHBsZS5wcmludC50aWNrZXQubW9kRGF0ZTwva2V5PgoJCQkJPGRhdGU+MjAw + My0wMS0yNFQxNjoyODozMVo8L2RhdGU+CgkJCQk8a2V5PmNvbS5hcHBsZS5wcmludC50 + aWNrZXQuc3RhdGVGbGFnPC9rZXk+CgkJCQk8aW50ZWdlcj4wPC9pbnRlZ2VyPgoJCQk8 + L2RpY3Q+CgkJPC9hcnJheT4KCTwvZGljdD4KCTxrZXk+Y29tLmFwcGxlLnByaW50LlBh + Z2VGb3JtYXQuUE1WZXJ0aWNhbFJlczwva2V5PgoJPGRpY3Q+CgkJPGtleT5jb20uYXBw + bGUucHJpbnQudGlja2V0LmNyZWF0b3I8L2tleT4KCQk8c3RyaW5nPmNvbS5hcHBsZS5w + cmludGluZ21hbmFnZXI8L3N0cmluZz4KCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNr + ZXQuaXRlbUFycmF5PC9rZXk+CgkJPGFycmF5PgoJCQk8ZGljdD4KCQkJCTxrZXk+Y29t + LmFwcGxlLnByaW50LlBhZ2VGb3JtYXQuUE1WZXJ0aWNhbFJlczwva2V5PgoJCQkJPHJl + YWw+NzI8L3JlYWw+CgkJCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQuY2xpZW50 + PC9rZXk+CgkJCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmludGluZ21hbmFnZXI8L3N0cmlu + Zz4KCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5tb2REYXRlPC9rZXk+CgkJ + CQk8ZGF0ZT4yMDAzLTAxLTI0VDE2OjI4OjMxWjwvZGF0ZT4KCQkJCTxrZXk+Y29tLmFw + cGxlLnByaW50LnRpY2tldC5zdGF0ZUZsYWc8L2tleT4KCQkJCTxpbnRlZ2VyPjA8L2lu + dGVnZXI+CgkJCTwvZGljdD4KCQk8L2FycmF5PgoJPC9kaWN0PgoJPGtleT5jb20uYXBw + bGUucHJpbnQuUGFnZUZvcm1hdC5QTVZlcnRpY2FsU2NhbGluZzwva2V5PgoJPGRpY3Q+ + CgkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0LmNyZWF0b3I8L2tleT4KCQk8c3Ry + aW5nPmNvbS5hcHBsZS5wcmludGluZ21hbmFnZXI8L3N0cmluZz4KCQk8a2V5PmNvbS5h + cHBsZS5wcmludC50aWNrZXQuaXRlbUFycmF5PC9rZXk+CgkJPGFycmF5PgoJCQk8ZGlj + dD4KCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LlBhZ2VGb3JtYXQuUE1WZXJ0aWNhbFNj + YWxpbmc8L2tleT4KCQkJCTxyZWFsPjE8L3JlYWw+CgkJCQk8a2V5PmNvbS5hcHBsZS5w + cmludC50aWNrZXQuY2xpZW50PC9rZXk+CgkJCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmlu + dGluZ21hbmFnZXI8L3N0cmluZz4KCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tl + dC5tb2REYXRlPC9rZXk+CgkJCQk8ZGF0ZT4yMDAzLTAxLTI0VDE2OjI4OjMxWjwvZGF0 + ZT4KCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5zdGF0ZUZsYWc8L2tleT4K + CQkJCTxpbnRlZ2VyPjA8L2ludGVnZXI+CgkJCTwvZGljdD4KCQk8L2FycmF5PgoJPC9k + aWN0PgoJPGtleT5jb20uYXBwbGUucHJpbnQuc3ViVGlja2V0LnBhcGVyX2luZm9fdGlj + a2V0PC9rZXk+Cgk8ZGljdD4KCQk8a2V5PmNvbS5hcHBsZS5wcmludC5QYWdlRm9ybWF0 + LlBNQWRqdXN0ZWRQYWdlUmVjdDwva2V5PgoJCTxkaWN0PgoJCQk8a2V5PmNvbS5hcHBs + ZS5wcmludC50aWNrZXQuY3JlYXRvcjwva2V5PgoJCQk8c3RyaW5nPmNvbS5hcHBsZS5w + cmludGluZ21hbmFnZXI8L3N0cmluZz4KCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlj + a2V0Lml0ZW1BcnJheTwva2V5PgoJCQk8YXJyYXk+CgkJCQk8ZGljdD4KCQkJCQk8a2V5 + PmNvbS5hcHBsZS5wcmludC5QYWdlRm9ybWF0LlBNQWRqdXN0ZWRQYWdlUmVjdDwva2V5 + PgoJCQkJCTxhcnJheT4KCQkJCQkJPHJlYWw+MC4wPC9yZWFsPgoJCQkJCQk8cmVhbD4w + LjA8L3JlYWw+CgkJCQkJCTxyZWFsPjc4MzwvcmVhbD4KCQkJCQkJPHJlYWw+NTU5PC9y + ZWFsPgoJCQkJCTwvYXJyYXk+CgkJCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0 + LmNsaWVudDwva2V5PgoJCQkJCTxzdHJpbmc+Y29tLmFwcGxlLnByaW50aW5nbWFuYWdl + cjwvc3RyaW5nPgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5tb2REYXRl + PC9rZXk+CgkJCQkJPGRhdGU+MjAwNi0wNy0xN1QwNjo1MjoyNFo8L2RhdGU+CgkJCQkJ + PGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0LnN0YXRlRmxhZzwva2V5PgoJCQkJCTxp + bnRlZ2VyPjA8L2ludGVnZXI+CgkJCQk8L2RpY3Q+CgkJCTwvYXJyYXk+CgkJPC9kaWN0 + PgoJCTxrZXk+Y29tLmFwcGxlLnByaW50LlBhZ2VGb3JtYXQuUE1BZGp1c3RlZFBhcGVy + UmVjdDwva2V5PgoJCTxkaWN0PgoJCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQu + Y3JlYXRvcjwva2V5PgoJCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmludGluZ21hbmFnZXI8 + L3N0cmluZz4KCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0Lml0ZW1BcnJheTwv + a2V5PgoJCQk8YXJyYXk+CgkJCQk8ZGljdD4KCQkJCQk8a2V5PmNvbS5hcHBsZS5wcmlu + dC5QYWdlRm9ybWF0LlBNQWRqdXN0ZWRQYXBlclJlY3Q8L2tleT4KCQkJCQk8YXJyYXk+ + CgkJCQkJCTxyZWFsPi0xODwvcmVhbD4KCQkJCQkJPHJlYWw+LTE4PC9yZWFsPgoJCQkJ + CQk8cmVhbD44MjQ8L3JlYWw+CgkJCQkJCTxyZWFsPjU3NzwvcmVhbD4KCQkJCQk8L2Fy + cmF5PgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5jbGllbnQ8L2tleT4K + CQkJCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmludGluZ21hbmFnZXI8L3N0cmluZz4KCQkJ + CQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQubW9kRGF0ZTwva2V5PgoJCQkJCTxk + YXRlPjIwMDYtMDctMTdUMDY6NTI6MjRaPC9kYXRlPgoJCQkJCTxrZXk+Y29tLmFwcGxl + LnByaW50LnRpY2tldC5zdGF0ZUZsYWc8L2tleT4KCQkJCQk8aW50ZWdlcj4wPC9pbnRl + Z2VyPgoJCQkJPC9kaWN0PgoJCQk8L2FycmF5PgoJCTwvZGljdD4KCQk8a2V5PmNvbS5h + cHBsZS5wcmludC5QYXBlckluZm8uUE1QYXBlck5hbWU8L2tleT4KCQk8ZGljdD4KCQkJ + PGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0LmNyZWF0b3I8L2tleT4KCQkJPHN0cmlu + Zz5jb20uYXBwbGUucHJpbnQucG0uUG9zdFNjcmlwdDwvc3RyaW5nPgoJCQk8a2V5PmNv + bS5hcHBsZS5wcmludC50aWNrZXQuaXRlbUFycmF5PC9rZXk+CgkJCTxhcnJheT4KCQkJ + CTxkaWN0PgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LlBhcGVySW5mby5QTVBhcGVy + TmFtZTwva2V5PgoJCQkJCTxzdHJpbmc+aXNvLWE0PC9zdHJpbmc+CgkJCQkJPGtleT5j + b20uYXBwbGUucHJpbnQudGlja2V0LmNsaWVudDwva2V5PgoJCQkJCTxzdHJpbmc+Y29t + LmFwcGxlLnByaW50LnBtLlBvc3RTY3JpcHQ8L3N0cmluZz4KCQkJCQk8a2V5PmNvbS5h + cHBsZS5wcmludC50aWNrZXQubW9kRGF0ZTwva2V5PgoJCQkJCTxkYXRlPjIwMDAtMDct + MjhUMjI6NTc6MDRaPC9kYXRlPgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tl + dC5zdGF0ZUZsYWc8L2tleT4KCQkJCQk8aW50ZWdlcj4xPC9pbnRlZ2VyPgoJCQkJPC9k + aWN0PgoJCQk8L2FycmF5PgoJCTwvZGljdD4KCQk8a2V5PmNvbS5hcHBsZS5wcmludC5Q + YXBlckluZm8uUE1VbmFkanVzdGVkUGFnZVJlY3Q8L2tleT4KCQk8ZGljdD4KCQkJPGtl + eT5jb20uYXBwbGUucHJpbnQudGlja2V0LmNyZWF0b3I8L2tleT4KCQkJPHN0cmluZz5j + b20uYXBwbGUucHJpbnQucG0uUG9zdFNjcmlwdDwvc3RyaW5nPgoJCQk8a2V5PmNvbS5h + cHBsZS5wcmludC50aWNrZXQuaXRlbUFycmF5PC9rZXk+CgkJCTxhcnJheT4KCQkJCTxk + aWN0PgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LlBhcGVySW5mby5QTVVuYWRqdXN0 + ZWRQYWdlUmVjdDwva2V5PgoJCQkJCTxhcnJheT4KCQkJCQkJPHJlYWw+MC4wPC9yZWFs + PgoJCQkJCQk8cmVhbD4wLjA8L3JlYWw+CgkJCQkJCTxyZWFsPjc4MzwvcmVhbD4KCQkJ + CQkJPHJlYWw+NTU5PC9yZWFsPgoJCQkJCTwvYXJyYXk+CgkJCQkJPGtleT5jb20uYXBw + bGUucHJpbnQudGlja2V0LmNsaWVudDwva2V5PgoJCQkJCTxzdHJpbmc+Y29tLmFwcGxl + LnByaW50aW5nbWFuYWdlcjwvc3RyaW5nPgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50 + LnRpY2tldC5tb2REYXRlPC9rZXk+CgkJCQkJPGRhdGU+MjAwMy0wMS0yNFQxNjoyODoz + MVo8L2RhdGU+CgkJCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0LnN0YXRlRmxh + Zzwva2V5PgoJCQkJCTxpbnRlZ2VyPjA8L2ludGVnZXI+CgkJCQk8L2RpY3Q+CgkJCTwv + YXJyYXk+CgkJPC9kaWN0PgoJCTxrZXk+Y29tLmFwcGxlLnByaW50LlBhcGVySW5mby5Q + TVVuYWRqdXN0ZWRQYXBlclJlY3Q8L2tleT4KCQk8ZGljdD4KCQkJPGtleT5jb20uYXBw + bGUucHJpbnQudGlja2V0LmNyZWF0b3I8L2tleT4KCQkJPHN0cmluZz5jb20uYXBwbGUu + cHJpbnQucG0uUG9zdFNjcmlwdDwvc3RyaW5nPgoJCQk8a2V5PmNvbS5hcHBsZS5wcmlu + dC50aWNrZXQuaXRlbUFycmF5PC9rZXk+CgkJCTxhcnJheT4KCQkJCTxkaWN0PgoJCQkJ + CTxrZXk+Y29tLmFwcGxlLnByaW50LlBhcGVySW5mby5QTVVuYWRqdXN0ZWRQYXBlclJl + Y3Q8L2tleT4KCQkJCQk8YXJyYXk+CgkJCQkJCTxyZWFsPi0xODwvcmVhbD4KCQkJCQkJ + PHJlYWw+LTE4PC9yZWFsPgoJCQkJCQk8cmVhbD44MjQ8L3JlYWw+CgkJCQkJCTxyZWFs + PjU3NzwvcmVhbD4KCQkJCQk8L2FycmF5PgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50 + LnRpY2tldC5jbGllbnQ8L2tleT4KCQkJCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmludGlu + Z21hbmFnZXI8L3N0cmluZz4KCQkJCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQu + bW9kRGF0ZTwva2V5PgoJCQkJCTxkYXRlPjIwMDMtMDEtMjRUMTY6Mjg6MzFaPC9kYXRl + PgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5zdGF0ZUZsYWc8L2tleT4K + CQkJCQk8aW50ZWdlcj4wPC9pbnRlZ2VyPgoJCQkJPC9kaWN0PgoJCQk8L2FycmF5PgoJ + CTwvZGljdD4KCQk8a2V5PmNvbS5hcHBsZS5wcmludC5QYXBlckluZm8ucHBkLlBNUGFw + ZXJOYW1lPC9rZXk+CgkJPGRpY3Q+CgkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tl + dC5jcmVhdG9yPC9rZXk+CgkJCTxzdHJpbmc+Y29tLmFwcGxlLnByaW50LnBtLlBvc3RT + Y3JpcHQ8L3N0cmluZz4KCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0Lml0ZW1B + cnJheTwva2V5PgoJCQk8YXJyYXk+CgkJCQk8ZGljdD4KCQkJCQk8a2V5PmNvbS5hcHBs + ZS5wcmludC5QYXBlckluZm8ucHBkLlBNUGFwZXJOYW1lPC9rZXk+CgkJCQkJPHN0cmlu + Zz5BNDwvc3RyaW5nPgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5jbGll + bnQ8L2tleT4KCQkJCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmludC5wbS5Qb3N0U2NyaXB0 + PC9zdHJpbmc+CgkJCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0Lm1vZERhdGU8 + L2tleT4KCQkJCQk8ZGF0ZT4yMDAwLTA3LTI4VDIyOjU3OjA0WjwvZGF0ZT4KCQkJCQk8 + a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQuc3RhdGVGbGFnPC9rZXk+CgkJCQkJPGlu + dGVnZXI+MTwvaW50ZWdlcj4KCQkJCTwvZGljdD4KCQkJPC9hcnJheT4KCQk8L2RpY3Q+ + CgkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0LkFQSVZlcnNpb248L2tleT4KCQk8 + c3RyaW5nPjAwLjIwPC9zdHJpbmc+CgkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0 + LnByaXZhdGVMb2NrPC9rZXk+CgkJPGZhbHNlLz4KCQk8a2V5PmNvbS5hcHBsZS5wcmlu + dC50aWNrZXQudHlwZTwva2V5PgoJCTxzdHJpbmc+Y29tLmFwcGxlLnByaW50LlBhcGVy + SW5mb1RpY2tldDwvc3RyaW5nPgoJPC9kaWN0PgoJPGtleT5jb20uYXBwbGUucHJpbnQu + dGlja2V0LkFQSVZlcnNpb248L2tleT4KCTxzdHJpbmc+MDAuMjA8L3N0cmluZz4KCTxr + ZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5wcml2YXRlTG9jazwva2V5PgoJPGZhbHNl + Lz4KCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC50eXBlPC9rZXk+Cgk8c3RyaW5n + PmNvbS5hcHBsZS5wcmludC5QYWdlRm9ybWF0VGlja2V0PC9zdHJpbmc+CjwvZGljdD4K + PC9wbGlzdD4KhpKEmZkPTlNQcmludEFsbFBhZ2VzhpKgkoSZmQhOU0NvcGllc4aSpJKE + mZkLTlNQYXBlck5hbWWGkoSZmQJBNIaShJmZD05TU2NhbGluZ0ZhY3RvcoaShKGbhIQB + ZqEBhpKEmZkJTlNQcmludGVyhpKEhIQJTlNQcmludGVyAJSShJmZCU9LSSBCNjEwMIaG + koSZmQ1OU1JpZ2h0TWFyZ2luhpKEoZuwoQCGkoSZmRZOU0hvcml6b250YWxseUNlbnRl + cmVkhpKkkoSZmRlOU1ByaW50UmV2ZXJzZU9yaWVudGF0aW9uhpKgkoSZmQ1OU0pvYkZl + YXR1cmVzhpKElpcAhpKEmZkMTlNMZWZ0TWFyZ2luhpKEoZuwoQCGkoSZmRVOU0hvcml6 + b25hbFBhZ2luYXRpb26GkqCShJmZEk5TUmV2ZXJzZVBhZ2VPcmRlcoaSoJKEmZkLTlNU + b3BNYXJnaW6GkoShm7ChAIaShJmZDk5TQm90dG9tTWFyZ2luhpKEoZuwoQCGkoSZmQpO + U0xhc3RQYWdlhpKEoZuil4J/////hpKEmZkLTlNGaXJzdFBhZ2WGkqSShJmZDU5TT3Jp + ZW50YXRpb26GkqCGhg== + + ReadOnly + NO + RowAlign + 1 + RowSpacing + 36 + SheetTitle + Canvas 1 + SmartAlignmentGuidesActive + YES + SmartDistanceGuidesActive + YES + UseEntirePage + + VPages + 1 + WindowInfo + + CurrentSheet + 0 + Frame + {{296, 0}, {643, 832}} + ShowRuler + + ShowStatusBar + + VisibleRegion + {{-34, 0}, {628, 734}} + Zoom + 1 + + + Added: trunk/varnish-doc/images/varnish-rfc2616-and-varnish.png =================================================================== (Binary files differ) Property changes on: trunk/varnish-doc/images/varnish-rfc2616-and-varnish.png ___________________________________________________________________ Name: svn:mime-type + application/octet-stream Added: trunk/varnish-doc/images/varnish-rfc2616-on-http-caches.graffle =================================================================== --- trunk/varnish-doc/images/varnish-rfc2616-on-http-caches.graffle 2006-07-20 12:03:10 UTC (rev 521) +++ trunk/varnish-doc/images/varnish-rfc2616-on-http-caches.graffle 2006-07-20 12:15:49 UTC (rev 522) @@ -0,0 +1,1225 @@ + + + + + CanvasColor + + w + 1 + + ColumnAlign + 1 + ColumnSpacing + 36 + CreationDate + 2006-07-17 08:21:36 +0200 + Creator + Stig Sandbeck Mathisen + GraphDocumentVersion + 4 + GraphicsList + + + Class + LineGraphic + Head + + ID + 66 + + ID + 70 + Points + + {154.562, 377.316} + {280, 332} + {456.931, 321.957} + + Style + + shadow + + Color + + a + 0.5 + b + 0 + g + 0 + r + 0 + + Draws + YES + Fuzziness + 4 + ShadowVector + {0, 2} + + stroke + + Cap + 0 + Color + + b + 0.222222 + g + 0.222222 + r + 0.222222 + + HeadArrow + 0 + Join + 0 + LineType + 1 + TailArrow + FilledArrow + Width + 2 + + + Tail + + ID + 68 + + + + Class + LineGraphic + Head + + ID + 62 + + ID + 69 + Points + + {157.818, 391.231} + {244.426, 390.683} + + Style + + shadow + + Color + + a + 0.5 + b + 0 + g + 0 + r + 0 + + Draws + YES + Fuzziness + 4 + ShadowVector + {0, 2} + + stroke + + Cap + 0 + Color + + b + 0.222222 + g + 0.222222 + r + 0.222222 + + HeadArrow + 0 + Join + 0 + LineType + 1 + TailArrow + FilledArrow + Width + 2 + + + Tail + + ID + 68 + + + + Bounds + {{72.7792, 370.24}, {85.0394, 42.5197}} + Class + ShapedGraphic + ID + 68 + Shape + RoundRect + Style + + fill + + FillType + 2 + GradientColor + + a + 0.6 + archive + + YnBsaXN0MDDUAQIDBAUGBwpZJGFy + Y2hpdmVyWCR2ZXJzaW9uVCR0b3BY + JG9iamVjdHNfEA9OU0tleWVkQXJj + aGl2ZXISAAGGoNEICVRyb290gAGl + CwwXGyBVJG51bGzVDQ4PEBESExQV + Fl8QEk5TQ3VzdG9tQ29sb3JTcGFj + ZVxOU0NvbXBvbmVudHNWTlNDTVlL + XE5TQ29sb3JTcGFjZVYkY2xhc3OA + Ak8QNjAuMjk5OTk5OTggMC41OTk5 + OTk5NiAwLjI5OTk5OTk4IDAuMTQ5 + OTk5OTkgMC42MDAwMDAwMk8QNzAu + Mjk5OTkyMzggMC42MDAwMDAwMiAw + LjI5OTk5MjM4IDAuMTQ5OTk2MTkg + MC42MDAwMDAwMgAQBYAE0hgRGRpU + TlNJRBADgAPSHB0eEFgkY2xhc3Nl + c1okY2xhc3NuYW1lohAfWE5TT2Jq + ZWN00hwdISKjIiMfV05TQ29sb3JY + JU5TQ29sb3IACAARABsAJAApADIA + RABJAEwAUQBTAFkAXwBqAH8AjACT + AKAApwCpAOIBHAEeASABJQEqASwB + LgEzATwBRwFKAVMBWAFcAWQAAAAA + AAACAQAAAAAAAAAkAAAAAAAAAAAA + AAAAAAABbQ== + + b + 0.374853 + g + 0.313649 + r + 0.501686 + + + + Text + + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc\pardirnatural + +\f0\b\fs24 \cf0 HTTP origin server} + + + + Bounds + {{8.9999, 285.201}, {212.598, 212.598}} + Class + ShapedGraphic + ID + 67 + Shape + RoundedRectangle + Style + + fill + + FillType + 2 + + + Text + + Align + 0 + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\ql\qnatural\pardirnatural + +\f0\b\fs24 \cf0 Server domain} + + TextPlacement + 0 + + + Class + Group + Graphics + + + Class + LineGraphic + Head + + ID + 65 + + ID + 59 + Points + + {435.677, 461.28} + {456.85, 461.28} + + Style + + shadow + + Color + + a + 0.5 + b + 0 + g + 0 + r + 0 + + Draws + YES + Fuzziness + 4 + ShadowVector + {0, 2} + + stroke + + Cap + 0 + Color + + b + 0.222222 + g + 0.222222 + r + 0.222222 + + HeadArrow + 0 + Join + 0 + LineType + 1 + TailArrow + FilledArrow + Width + 2 + + + Tail + + ID + 63 + + + + Class + LineGraphic + Head + + ID + 63 + + ID + 60 + Points + + {318.497, 411.466} + {361.605, 440.228} + + Style + + shadow + + Color + + a + 0.5 + b + 0 + g + 0 + r + 0 + + Draws + YES + Fuzziness + 4 + ShadowVector + {0, 2} + + stroke + + Cap + 0 + Color + + b + 0.222222 + g + 0.222222 + r + 0.222222 + + HeadArrow + 0 + Join + 0 + LineType + 1 + TailArrow + FilledArrow + Width + 2 + + + Tail + + ID + 62 + + + + Class + LineGraphic + Head + + ID + 64 + + ID + 61 + Points + + {329.464, 390.414} + {456.85, 390.414} + + Style + + shadow + + Color + + a + 0.5 + b + 0 + g + 0 + r + 0 + + Draws + YES + Fuzziness + 4 + ShadowVector + {0, 2} + + stroke + + Cap + 0 + Color + + b + 0.222222 + g + 0.222222 + r + 0.222222 + + HeadArrow + 0 + Join + 0 + LineType + 1 + TailArrow + FilledArrow + Width + 2 + + + Tail + + ID + 62 + + + + Bounds + {{244.425, 369.154}, {85.0394, 42.5197}} + Class + ShapedGraphic + ID + 62 + Shape + RoundRect + Style + + fill + + FillType + 2 + GradientColor + + a + 0.6 + archive + + YnBsaXN0MDDU + AQIDBAUGBwpZ + JGFyY2hpdmVy + WCR2ZXJzaW9u + VCR0b3BYJG9i + amVjdHNfEA9O + U0tleWVkQXJj + aGl2ZXISAAGG + oNEICVRyb290 + gAGlCwwXGyBV + JG51bGzVDQ4P + EBESExQVFl8Q + Ek5TQ3VzdG9t + Q29sb3JTcGFj + ZVxOU0NvbXBv + bmVudHNWTlND + TVlLXE5TQ29s + b3JTcGFjZVYk + Y2xhc3OAAk8Q + NjAuNTk5OTk5 + OTYgMC4yOTk5 + OTk5OCAwLjI5 + OTk5OTk4IDAu + MTQ5OTk5OTkg + MC42MDAwMDAw + Mk8QNzAuNjAw + MDAwMDIgMC4y + OTk5OTIzOCAw + LjI5OTk5MjM4 + IDAuMTQ5OTk2 + MTkgMC42MDAw + MDAwMgAQBYAE + 0hgRGRpUTlNJ + RBADgAPSHB0e + EFgkY2xhc3Nl + c1okY2xhc3Nu + YW1lohAfWE5T + T2JqZWN00hwd + ISKjIiMfV05T + Q29sb3JYJU5T + Q29sb3IACAAR + ABsAJAApADIA + RABJAEwAUQBT + AFkAXwBqAH8A + jACTAKAApwCp + AOIBHAEeASAB + JQEqASwBLgEz + ATwBRwFKAVMB + WAFcAWQAAAAA + AAACAQAAAAAA + AAAkAAAAAAAA + AAAAAAAAAAAB + bQ== + + b + 0.451133 + g + 0.428122 + r + 0.303029 + + + + Text + + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc\pardirnatural + +\f0\b\fs24 \cf0 Shared HTTP Cache} + + + + Bounds + {{350.638, 440.02}, {85.0394, 42.5197}} + Class + ShapedGraphic + ID + 63 + Shape + RoundRect + Style + + fill + + FillType + 2 + GradientColor + + a + 0.6 + archive + + YnBsaXN0MDDU + AQIDBAUGBwpZ + JGFyY2hpdmVy + WCR2ZXJzaW9u + VCR0b3BYJG9i + amVjdHNfEA9O + U0tleWVkQXJj + aGl2ZXISAAGG + oNEICVRyb290 + gAGlCwwXGyBV + JG51bGzVDQ4P + EBESExQVFl8Q + Ek5TQ3VzdG9t + Q29sb3JTcGFj + ZVxOU0NvbXBv + bmVudHNWTlND + TVlLXE5TQ29s + b3JTcGFjZVYk + Y2xhc3OAAk8Q + NjAuNTk5OTk5 + OTYgMC4yOTk5 + OTk5OCAwLjI5 + OTk5OTk4IDAu + MTQ5OTk5OTkg + MC42MDAwMDAw + Mk8QNzAuNjAw + MDAwMDIgMC4y + OTk5OTIzOCAw + LjI5OTk5MjM4 + IDAuMTQ5OTk2 + MTkgMC42MDAw + MDAwMgAQBYAE + 0hgRGRpUTlNJ + RBADgAPSHB0e + EFgkY2xhc3Nl + c1okY2xhc3Nu + YW1lohAfWE5T + T2JqZWN00hwd + ISKjIiMfV05T + Q29sb3JYJU5T + Q29sb3IACAAR + ABsAJAApADIA + RABJAEwAUQBT + AFkAXwBqAH8A + jACTAKAApwCp + AOIBHAEeASAB + JQEqASwBLgEz + ATwBRwFKAVMB + WAFcAWQAAAAA + AAACAQAAAAAA + AAAkAAAAAAAA + AAAAAAAAAAAB + bQ== + + b + 0.451133 + g + 0.428122 + r + 0.303029 + + + + Text + + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc\pardirnatural + +\f0\b\fs24 \cf0 Private HTTP Cache} + + + + Bounds + {{456.85, 369.154}, {85.0394, 42.5197}} + Class + ShapedGraphic + ID + 64 + Shape + RoundRect + Style + + fill + + FillType + 2 + GradientColor + + a + 0.6 + archive + + YnBsaXN0MDDU + AQIDBAUGBwpZ + JGFyY2hpdmVy + WCR2ZXJzaW9u + VCR0b3BYJG9i + amVjdHNfEA9O + U0tleWVkQXJj + aGl2ZXISAAGG + oNEICVRyb290 + gAGlCwwXGyBV + JG51bGzVDQ4P + EBESExQVFl8Q + Ek5TQ3VzdG9t + Q29sb3JTcGFj + ZVxOU0NvbXBv + bmVudHNWTlND + TVlLXE5TQ29s + b3JTcGFjZVYk + Y2xhc3OAAk8Q + NjAuNTk5OTk5 + OTYgMC4yOTk5 + OTk5OCAwLjI5 + OTk5OTk4IDAu + MTQ5OTk5OTkg + MC42MDAwMDAw + Mk8QNzAuNjAw + MDAwMDIgMC4y + OTk5OTIzOCAw + LjI5OTk5MjM4 + IDAuMTQ5OTk2 + MTkgMC42MDAw + MDAwMgAQBYAE + 0hgRGRpUTlNJ + RBADgAPSHB0e + EFgkY2xhc3Nl + c1okY2xhc3Nu + YW1lohAfWE5T + T2JqZWN00hwd + ISKjIiMfV05T + Q29sb3JYJU5T + Q29sb3IACAAR + ABsAJAApADIA + RABJAEwAUQBT + AFkAXwBqAH8A + jACTAKAApwCp + AOIBHAEeASAB + JQEqASwBLgEz + ATwBRwFKAVMB + WAFcAWQAAAAA + AAACAQAAAAAA + AAAkAAAAAAAA + AAAAAAAAAAAB + bQ== + + b + 0.451133 + g + 0.428122 + r + 0.303029 + + + + Text + + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc\pardirnatural + +\f0\b\fs24 \cf0 HTTP Client} + + + + Bounds + {{456.85, 440.02}, {85.0394, 42.5197}} + Class + ShapedGraphic + ID + 65 + Shape + RoundRect + Style + + fill + + FillType + 2 + GradientColor + + a + 0.6 + archive + + YnBsaXN0MDDU + AQIDBAUGBwpZ + JGFyY2hpdmVy + WCR2ZXJzaW9u + VCR0b3BYJG9i + amVjdHNfEA9O + U0tleWVkQXJj + aGl2ZXISAAGG + oNEICVRyb290 + gAGlCwwXGyBV + JG51bGzVDQ4P + EBESExQVFl8Q + Ek5TQ3VzdG9t + Q29sb3JTcGFj + ZVxOU0NvbXBv + bmVudHNWTlND + TVlLXE5TQ29s + b3JTcGFjZVYk + Y2xhc3OAAk8Q + NjAuNTk5OTk5 + OTYgMC4yOTk5 + OTk5OCAwLjI5 + OTk5OTk4IDAu + MTQ5OTk5OTkg + MC42MDAwMDAw + Mk8QNzAuNjAw + MDAwMDIgMC4y + OTk5OTIzOCAw + LjI5OTk5MjM4 + IDAuMTQ5OTk2 + MTkgMC42MDAw + MDAwMgAQBYAE + 0hgRGRpUTlNJ + RBADgAPSHB0e + EFgkY2xhc3Nl + c1okY2xhc3Nu + YW1lohAfWE5T + T2JqZWN00hwd + ISKjIiMfV05T + Q29sb3JYJU5T + Q29sb3IACAAR + ABsAJAApADIA + RABJAEwAUQBT + AFkAXwBqAH8A + jACTAKAApwCp + AOIBHAEeASAB + JQEqASwBLgEz + ATwBRwFKAVMB + WAFcAWQAAAAA + AAACAQAAAAAA + AAAkAAAAAAAA + AAAAAAAAAAAB + bQ== + + b + 0.451133 + g + 0.428122 + r + 0.303029 + + + + Text + + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc\pardirnatural + +\f0\b\fs24 \cf0 HTTP Client} + + + + Bounds + {{456.85, 298.288}, {85.0394, 42.5197}} + Class + ShapedGraphic + ID + 66 + Shape + RoundRect + Style + + fill + + FillType + 2 + GradientColor + + a + 0.6 + archive + + YnBsaXN0MDDU + AQIDBAUGBwpZ + JGFyY2hpdmVy + WCR2ZXJzaW9u + VCR0b3BYJG9i + amVjdHNfEA9O + U0tleWVkQXJj + aGl2ZXISAAGG + oNEICVRyb290 + gAGlCwwXGyBV + JG51bGzVDQ4P + EBESExQVFl8Q + Ek5TQ3VzdG9t + Q29sb3JTcGFj + ZVxOU0NvbXBv + bmVudHNWTlND + TVlLXE5TQ29s + b3JTcGFjZVYk + Y2xhc3OAAk8Q + NjAuNTk5OTk5 + OTYgMC4yOTk5 + OTk5OCAwLjI5 + OTk5OTk4IDAu + MTQ5OTk5OTkg + MC42MDAwMDAw + Mk8QNzAuNjAw + MDAwMDIgMC4y + OTk5OTIzOCAw + LjI5OTk5MjM4 + IDAuMTQ5OTk2 + MTkgMC42MDAw + MDAwMgAQBYAE + 0hgRGRpUTlNJ + RBADgAPSHB0e + EFgkY2xhc3Nl + c1okY2xhc3Nu + YW1lohAfWE5T + T2JqZWN00hwd + ISKjIiMfV05T + Q29sb3JYJU5T + Q29sb3IACAAR + ABsAJAApADIA + RABJAEwAUQBT + AFkAXwBqAH8A + jACTAKAApwCp + AOIBHAEeASAB + JQEqASwBLgEz + ATwBRwFKAVMB + WAFcAWQAAAAA + AAACAQAAAAAA + AAAkAAAAAAAA + AAAAAAAAAAAB + bQ== + + b + 0.451133 + g + 0.428122 + r + 0.303029 + + + + Text + + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc\pardirnatural + +\f0\b\fs24 \cf0 HTTP Client} + + + + ID + 58 + + + Bounds + {{237.252, 285.201}, {311.811, 212.598}} + Class + ShapedGraphic + ID + 57 + Shape + RoundedRectangle + Style + + fill + + FillType + 2 + + + Text + + Align + 0 + Text + {\rtf1\mac\ansicpg10000\cocoartf824\cocoasubrtf410 +{\fonttbl\f0\fswiss\fcharset77 Helvetica-Bold;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\ql\qnatural\pardirnatural + +\f0\b\fs24 \cf0 Client domain} + + TextPlacement + 0 + + + GridInfo + + GridSpacing + 14.17322826385498 + MajorGridSpacing + 10 + + GuidesLocked + NO + GuidesVisible + YES + HPages + 1 + ImageCounter + 1 + IsPalette + NO + Layers + + + Lock + NO + Name + Layer 1 + Print + YES + View + YES + + + LayoutInfo + + ChildOrdering + 0 + + LinksVisible + NO + MagnetsVisible + NO + ModificationDate + 2006-07-17 08:52:11 +0200 + Modifier + Stig Sandbeck Mathisen + Orientation + 2 + PageBreaks + YES + PageSetup + + BAt0eXBlZHN0cmVhbYED6IQBQISEhAtOU1ByaW50SW5mbwGEhAhOU09iamVjdACFkoSE + hBNOU011dGFibGVEaWN0aW9uYXJ5AISEDE5TRGljdGlvbmFyeQCUhAFpF5KEhIQITlNT + dHJpbmcBlIQBKxBOU0pvYkRpc3Bvc2l0aW9uhpKEmZkPTlNQcmludFNwb29sSm9ihpKE + mZkLTlNQYXBlclNpemWGkoSEhAdOU1ZhbHVlAJSEASqEhAx7X05TU2l6ZT1mZn2cgQJT + gQNKhpKEmZkUTlNWZXJ0aWNhbFBhZ2luYXRpb26GkoSEhAhOU051bWJlcgCdm4SXlwCG + koSZmQ9OU1BhZ2VzUGVyU2hlZXSGkoShm6KXAYaShJmZFE5TVmVydGljYWxseUNlbnRl + cmVkhpKkkoSZmQ5OU1BNUGFnZUZvcm1hdIaShISEDU5TTXV0YWJsZURhdGEAhIQGTlNE + YXRhAJSXgR2mhAdbNzU5MGNdPD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRG + LTgiPz4KPCFET0NUWVBFIHBsaXN0IFBVQkxJQyAiLS8vQXBwbGUgQ29tcHV0ZXIvL0RU + RCBQTElTVCAxLjAvL0VOIiAiaHR0cDovL3d3dy5hcHBsZS5jb20vRFREcy9Qcm9wZXJ0 + eUxpc3QtMS4wLmR0ZCI+CjxwbGlzdCB2ZXJzaW9uPSIxLjAiPgo8ZGljdD4KCTxrZXk+ + Y29tLmFwcGxlLnByaW50LlBhZ2VGb3JtYXQuUE1Ib3Jpem9udGFsUmVzPC9rZXk+Cgk8 + ZGljdD4KCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQuY3JlYXRvcjwva2V5PgoJ + CTxzdHJpbmc+Y29tLmFwcGxlLnByaW50aW5nbWFuYWdlcjwvc3RyaW5nPgoJCTxrZXk+ + Y29tLmFwcGxlLnByaW50LnRpY2tldC5pdGVtQXJyYXk8L2tleT4KCQk8YXJyYXk+CgkJ + CTxkaWN0PgoJCQkJPGtleT5jb20uYXBwbGUucHJpbnQuUGFnZUZvcm1hdC5QTUhvcml6 + b250YWxSZXM8L2tleT4KCQkJCTxyZWFsPjcyPC9yZWFsPgoJCQkJPGtleT5jb20uYXBw + bGUucHJpbnQudGlja2V0LmNsaWVudDwva2V5PgoJCQkJPHN0cmluZz5jb20uYXBwbGUu + cHJpbnRpbmdtYW5hZ2VyPC9zdHJpbmc+CgkJCQk8a2V5PmNvbS5hcHBsZS5wcmludC50 + aWNrZXQubW9kRGF0ZTwva2V5PgoJCQkJPGRhdGU+MjAwMy0wMS0yNFQxNjoyODozMVo8 + L2RhdGU+CgkJCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQuc3RhdGVGbGFnPC9r + ZXk+CgkJCQk8aW50ZWdlcj4wPC9pbnRlZ2VyPgoJCQk8L2RpY3Q+CgkJPC9hcnJheT4K + CTwvZGljdD4KCTxrZXk+Y29tLmFwcGxlLnByaW50LlBhZ2VGb3JtYXQuUE1PcmllbnRh + dGlvbjwva2V5PgoJPGRpY3Q+CgkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0LmNy + ZWF0b3I8L2tleT4KCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmludGluZ21hbmFnZXI8L3N0 + cmluZz4KCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQuaXRlbUFycmF5PC9rZXk+ + CgkJPGFycmF5PgoJCQk8ZGljdD4KCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LlBhZ2VG + b3JtYXQuUE1PcmllbnRhdGlvbjwva2V5PgoJCQkJPGludGVnZXI+MTwvaW50ZWdlcj4K + CQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5jbGllbnQ8L2tleT4KCQkJCTxz + dHJpbmc+Y29tLmFwcGxlLnByaW50aW5nbWFuYWdlcjwvc3RyaW5nPgoJCQkJPGtleT5j + b20uYXBwbGUucHJpbnQudGlja2V0Lm1vZERhdGU8L2tleT4KCQkJCTxkYXRlPjIwMDMt + MDEtMjRUMTY6Mjg6MzFaPC9kYXRlPgoJCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlj + a2V0LnN0YXRlRmxhZzwva2V5PgoJCQkJPGludGVnZXI+MDwvaW50ZWdlcj4KCQkJPC9k + aWN0PgoJCTwvYXJyYXk+Cgk8L2RpY3Q+Cgk8a2V5PmNvbS5hcHBsZS5wcmludC5QYWdl + Rm9ybWF0LlBNU2NhbGluZzwva2V5PgoJPGRpY3Q+CgkJPGtleT5jb20uYXBwbGUucHJp + bnQudGlja2V0LmNyZWF0b3I8L2tleT4KCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmludGlu + Z21hbmFnZXI8L3N0cmluZz4KCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQuaXRl + bUFycmF5PC9rZXk+CgkJPGFycmF5PgoJCQk8ZGljdD4KCQkJCTxrZXk+Y29tLmFwcGxl + LnByaW50LlBhZ2VGb3JtYXQuUE1TY2FsaW5nPC9rZXk+CgkJCQk8cmVhbD4xPC9yZWFs + PgoJCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0LmNsaWVudDwva2V5PgoJCQkJ + PHN0cmluZz5jb20uYXBwbGUucHJpbnRpbmdtYW5hZ2VyPC9zdHJpbmc+CgkJCQk8a2V5 + PmNvbS5hcHBsZS5wcmludC50aWNrZXQubW9kRGF0ZTwva2V5PgoJCQkJPGRhdGU+MjAw + My0wMS0yNFQxNjoyODozMVo8L2RhdGU+CgkJCQk8a2V5PmNvbS5hcHBsZS5wcmludC50 + aWNrZXQuc3RhdGVGbGFnPC9rZXk+CgkJCQk8aW50ZWdlcj4wPC9pbnRlZ2VyPgoJCQk8 + L2RpY3Q+CgkJPC9hcnJheT4KCTwvZGljdD4KCTxrZXk+Y29tLmFwcGxlLnByaW50LlBh + Z2VGb3JtYXQuUE1WZXJ0aWNhbFJlczwva2V5PgoJPGRpY3Q+CgkJPGtleT5jb20uYXBw + bGUucHJpbnQudGlja2V0LmNyZWF0b3I8L2tleT4KCQk8c3RyaW5nPmNvbS5hcHBsZS5w + cmludGluZ21hbmFnZXI8L3N0cmluZz4KCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNr + ZXQuaXRlbUFycmF5PC9rZXk+CgkJPGFycmF5PgoJCQk8ZGljdD4KCQkJCTxrZXk+Y29t + LmFwcGxlLnByaW50LlBhZ2VGb3JtYXQuUE1WZXJ0aWNhbFJlczwva2V5PgoJCQkJPHJl + YWw+NzI8L3JlYWw+CgkJCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQuY2xpZW50 + PC9rZXk+CgkJCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmludGluZ21hbmFnZXI8L3N0cmlu + Zz4KCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5tb2REYXRlPC9rZXk+CgkJ + CQk8ZGF0ZT4yMDAzLTAxLTI0VDE2OjI4OjMxWjwvZGF0ZT4KCQkJCTxrZXk+Y29tLmFw + cGxlLnByaW50LnRpY2tldC5zdGF0ZUZsYWc8L2tleT4KCQkJCTxpbnRlZ2VyPjA8L2lu + dGVnZXI+CgkJCTwvZGljdD4KCQk8L2FycmF5PgoJPC9kaWN0PgoJPGtleT5jb20uYXBw + bGUucHJpbnQuUGFnZUZvcm1hdC5QTVZlcnRpY2FsU2NhbGluZzwva2V5PgoJPGRpY3Q+ + CgkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0LmNyZWF0b3I8L2tleT4KCQk8c3Ry + aW5nPmNvbS5hcHBsZS5wcmludGluZ21hbmFnZXI8L3N0cmluZz4KCQk8a2V5PmNvbS5h + cHBsZS5wcmludC50aWNrZXQuaXRlbUFycmF5PC9rZXk+CgkJPGFycmF5PgoJCQk8ZGlj + dD4KCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LlBhZ2VGb3JtYXQuUE1WZXJ0aWNhbFNj + YWxpbmc8L2tleT4KCQkJCTxyZWFsPjE8L3JlYWw+CgkJCQk8a2V5PmNvbS5hcHBsZS5w + cmludC50aWNrZXQuY2xpZW50PC9rZXk+CgkJCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmlu + dGluZ21hbmFnZXI8L3N0cmluZz4KCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tl + dC5tb2REYXRlPC9rZXk+CgkJCQk8ZGF0ZT4yMDAzLTAxLTI0VDE2OjI4OjMxWjwvZGF0 + ZT4KCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5zdGF0ZUZsYWc8L2tleT4K + CQkJCTxpbnRlZ2VyPjA8L2ludGVnZXI+CgkJCTwvZGljdD4KCQk8L2FycmF5PgoJPC9k + aWN0PgoJPGtleT5jb20uYXBwbGUucHJpbnQuc3ViVGlja2V0LnBhcGVyX2luZm9fdGlj + a2V0PC9rZXk+Cgk8ZGljdD4KCQk8a2V5PmNvbS5hcHBsZS5wcmludC5QYWdlRm9ybWF0 + LlBNQWRqdXN0ZWRQYWdlUmVjdDwva2V5PgoJCTxkaWN0PgoJCQk8a2V5PmNvbS5hcHBs + ZS5wcmludC50aWNrZXQuY3JlYXRvcjwva2V5PgoJCQk8c3RyaW5nPmNvbS5hcHBsZS5w + cmludGluZ21hbmFnZXI8L3N0cmluZz4KCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlj + a2V0Lml0ZW1BcnJheTwva2V5PgoJCQk8YXJyYXk+CgkJCQk8ZGljdD4KCQkJCQk8a2V5 + PmNvbS5hcHBsZS5wcmludC5QYWdlRm9ybWF0LlBNQWRqdXN0ZWRQYWdlUmVjdDwva2V5 + PgoJCQkJCTxhcnJheT4KCQkJCQkJPHJlYWw+MC4wPC9yZWFsPgoJCQkJCQk8cmVhbD4w + LjA8L3JlYWw+CgkJCQkJCTxyZWFsPjc4MzwvcmVhbD4KCQkJCQkJPHJlYWw+NTU5PC9y + ZWFsPgoJCQkJCTwvYXJyYXk+CgkJCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0 + LmNsaWVudDwva2V5PgoJCQkJCTxzdHJpbmc+Y29tLmFwcGxlLnByaW50aW5nbWFuYWdl + cjwvc3RyaW5nPgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5tb2REYXRl + PC9rZXk+CgkJCQkJPGRhdGU+MjAwNi0wNy0xN1QwNjoyMTozNlo8L2RhdGU+CgkJCQkJ + PGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0LnN0YXRlRmxhZzwva2V5PgoJCQkJCTxp + bnRlZ2VyPjA8L2ludGVnZXI+CgkJCQk8L2RpY3Q+CgkJCTwvYXJyYXk+CgkJPC9kaWN0 + PgoJCTxrZXk+Y29tLmFwcGxlLnByaW50LlBhZ2VGb3JtYXQuUE1BZGp1c3RlZFBhcGVy + UmVjdDwva2V5PgoJCTxkaWN0PgoJCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQu + Y3JlYXRvcjwva2V5PgoJCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmludGluZ21hbmFnZXI8 + L3N0cmluZz4KCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0Lml0ZW1BcnJheTwv + a2V5PgoJCQk8YXJyYXk+CgkJCQk8ZGljdD4KCQkJCQk8a2V5PmNvbS5hcHBsZS5wcmlu + dC5QYWdlRm9ybWF0LlBNQWRqdXN0ZWRQYXBlclJlY3Q8L2tleT4KCQkJCQk8YXJyYXk+ + CgkJCQkJCTxyZWFsPi0xODwvcmVhbD4KCQkJCQkJPHJlYWw+LTE4PC9yZWFsPgoJCQkJ + CQk8cmVhbD44MjQ8L3JlYWw+CgkJCQkJCTxyZWFsPjU3NzwvcmVhbD4KCQkJCQk8L2Fy + cmF5PgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5jbGllbnQ8L2tleT4K + CQkJCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmludGluZ21hbmFnZXI8L3N0cmluZz4KCQkJ + CQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQubW9kRGF0ZTwva2V5PgoJCQkJCTxk + YXRlPjIwMDYtMDctMTdUMDY6MjE6MzZaPC9kYXRlPgoJCQkJCTxrZXk+Y29tLmFwcGxl + LnByaW50LnRpY2tldC5zdGF0ZUZsYWc8L2tleT4KCQkJCQk8aW50ZWdlcj4wPC9pbnRl + Z2VyPgoJCQkJPC9kaWN0PgoJCQk8L2FycmF5PgoJCTwvZGljdD4KCQk8a2V5PmNvbS5h + cHBsZS5wcmludC5QYXBlckluZm8uUE1QYXBlck5hbWU8L2tleT4KCQk8ZGljdD4KCQkJ + PGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0LmNyZWF0b3I8L2tleT4KCQkJPHN0cmlu + Zz5jb20uYXBwbGUucHJpbnQucG0uUG9zdFNjcmlwdDwvc3RyaW5nPgoJCQk8a2V5PmNv + bS5hcHBsZS5wcmludC50aWNrZXQuaXRlbUFycmF5PC9rZXk+CgkJCTxhcnJheT4KCQkJ + CTxkaWN0PgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LlBhcGVySW5mby5QTVBhcGVy + TmFtZTwva2V5PgoJCQkJCTxzdHJpbmc+aXNvLWE0PC9zdHJpbmc+CgkJCQkJPGtleT5j + b20uYXBwbGUucHJpbnQudGlja2V0LmNsaWVudDwva2V5PgoJCQkJCTxzdHJpbmc+Y29t + LmFwcGxlLnByaW50LnBtLlBvc3RTY3JpcHQ8L3N0cmluZz4KCQkJCQk8a2V5PmNvbS5h + cHBsZS5wcmludC50aWNrZXQubW9kRGF0ZTwva2V5PgoJCQkJCTxkYXRlPjIwMDAtMDct + MjhUMjI6NTc6MDRaPC9kYXRlPgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tl + dC5zdGF0ZUZsYWc8L2tleT4KCQkJCQk8aW50ZWdlcj4xPC9pbnRlZ2VyPgoJCQkJPC9k + aWN0PgoJCQk8L2FycmF5PgoJCTwvZGljdD4KCQk8a2V5PmNvbS5hcHBsZS5wcmludC5Q + YXBlckluZm8uUE1VbmFkanVzdGVkUGFnZVJlY3Q8L2tleT4KCQk8ZGljdD4KCQkJPGtl + eT5jb20uYXBwbGUucHJpbnQudGlja2V0LmNyZWF0b3I8L2tleT4KCQkJPHN0cmluZz5j + b20uYXBwbGUucHJpbnQucG0uUG9zdFNjcmlwdDwvc3RyaW5nPgoJCQk8a2V5PmNvbS5h + cHBsZS5wcmludC50aWNrZXQuaXRlbUFycmF5PC9rZXk+CgkJCTxhcnJheT4KCQkJCTxk + aWN0PgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LlBhcGVySW5mby5QTVVuYWRqdXN0 + ZWRQYWdlUmVjdDwva2V5PgoJCQkJCTxhcnJheT4KCQkJCQkJPHJlYWw+MC4wPC9yZWFs + PgoJCQkJCQk8cmVhbD4wLjA8L3JlYWw+CgkJCQkJCTxyZWFsPjc4MzwvcmVhbD4KCQkJ + CQkJPHJlYWw+NTU5PC9yZWFsPgoJCQkJCTwvYXJyYXk+CgkJCQkJPGtleT5jb20uYXBw + bGUucHJpbnQudGlja2V0LmNsaWVudDwva2V5PgoJCQkJCTxzdHJpbmc+Y29tLmFwcGxl + LnByaW50aW5nbWFuYWdlcjwvc3RyaW5nPgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50 + LnRpY2tldC5tb2REYXRlPC9rZXk+CgkJCQkJPGRhdGU+MjAwMy0wMS0yNFQxNjoyODoz + MVo8L2RhdGU+CgkJCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0LnN0YXRlRmxh + Zzwva2V5PgoJCQkJCTxpbnRlZ2VyPjA8L2ludGVnZXI+CgkJCQk8L2RpY3Q+CgkJCTwv + YXJyYXk+CgkJPC9kaWN0PgoJCTxrZXk+Y29tLmFwcGxlLnByaW50LlBhcGVySW5mby5Q + TVVuYWRqdXN0ZWRQYXBlclJlY3Q8L2tleT4KCQk8ZGljdD4KCQkJPGtleT5jb20uYXBw + bGUucHJpbnQudGlja2V0LmNyZWF0b3I8L2tleT4KCQkJPHN0cmluZz5jb20uYXBwbGUu + cHJpbnQucG0uUG9zdFNjcmlwdDwvc3RyaW5nPgoJCQk8a2V5PmNvbS5hcHBsZS5wcmlu + dC50aWNrZXQuaXRlbUFycmF5PC9rZXk+CgkJCTxhcnJheT4KCQkJCTxkaWN0PgoJCQkJ + CTxrZXk+Y29tLmFwcGxlLnByaW50LlBhcGVySW5mby5QTVVuYWRqdXN0ZWRQYXBlclJl + Y3Q8L2tleT4KCQkJCQk8YXJyYXk+CgkJCQkJCTxyZWFsPi0xODwvcmVhbD4KCQkJCQkJ + PHJlYWw+LTE4PC9yZWFsPgoJCQkJCQk8cmVhbD44MjQ8L3JlYWw+CgkJCQkJCTxyZWFs + PjU3NzwvcmVhbD4KCQkJCQk8L2FycmF5PgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50 + LnRpY2tldC5jbGllbnQ8L2tleT4KCQkJCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmludGlu + Z21hbmFnZXI8L3N0cmluZz4KCQkJCQk8a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQu + bW9kRGF0ZTwva2V5PgoJCQkJCTxkYXRlPjIwMDMtMDEtMjRUMTY6Mjg6MzFaPC9kYXRl + PgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5zdGF0ZUZsYWc8L2tleT4K + CQkJCQk8aW50ZWdlcj4wPC9pbnRlZ2VyPgoJCQkJPC9kaWN0PgoJCQk8L2FycmF5PgoJ + CTwvZGljdD4KCQk8a2V5PmNvbS5hcHBsZS5wcmludC5QYXBlckluZm8ucHBkLlBNUGFw + ZXJOYW1lPC9rZXk+CgkJPGRpY3Q+CgkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tl + dC5jcmVhdG9yPC9rZXk+CgkJCTxzdHJpbmc+Y29tLmFwcGxlLnByaW50LnBtLlBvc3RT + Y3JpcHQ8L3N0cmluZz4KCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0Lml0ZW1B + cnJheTwva2V5PgoJCQk8YXJyYXk+CgkJCQk8ZGljdD4KCQkJCQk8a2V5PmNvbS5hcHBs + ZS5wcmludC5QYXBlckluZm8ucHBkLlBNUGFwZXJOYW1lPC9rZXk+CgkJCQkJPHN0cmlu + Zz5BNDwvc3RyaW5nPgoJCQkJCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5jbGll + bnQ8L2tleT4KCQkJCQk8c3RyaW5nPmNvbS5hcHBsZS5wcmludC5wbS5Qb3N0U2NyaXB0 + PC9zdHJpbmc+CgkJCQkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0Lm1vZERhdGU8 + L2tleT4KCQkJCQk8ZGF0ZT4yMDAwLTA3LTI4VDIyOjU3OjA0WjwvZGF0ZT4KCQkJCQk8 + a2V5PmNvbS5hcHBsZS5wcmludC50aWNrZXQuc3RhdGVGbGFnPC9rZXk+CgkJCQkJPGlu + dGVnZXI+MTwvaW50ZWdlcj4KCQkJCTwvZGljdD4KCQkJPC9hcnJheT4KCQk8L2RpY3Q+ + CgkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0LkFQSVZlcnNpb248L2tleT4KCQk8 + c3RyaW5nPjAwLjIwPC9zdHJpbmc+CgkJPGtleT5jb20uYXBwbGUucHJpbnQudGlja2V0 + LnByaXZhdGVMb2NrPC9rZXk+CgkJPGZhbHNlLz4KCQk8a2V5PmNvbS5hcHBsZS5wcmlu + dC50aWNrZXQudHlwZTwva2V5PgoJCTxzdHJpbmc+Y29tLmFwcGxlLnByaW50LlBhcGVy + SW5mb1RpY2tldDwvc3RyaW5nPgoJPC9kaWN0PgoJPGtleT5jb20uYXBwbGUucHJpbnQu + dGlja2V0LkFQSVZlcnNpb248L2tleT4KCTxzdHJpbmc+MDAuMjA8L3N0cmluZz4KCTxr + ZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC5wcml2YXRlTG9jazwva2V5PgoJPGZhbHNl + Lz4KCTxrZXk+Y29tLmFwcGxlLnByaW50LnRpY2tldC50eXBlPC9rZXk+Cgk8c3RyaW5n + PmNvbS5hcHBsZS5wcmludC5QYWdlRm9ybWF0VGlja2V0PC9zdHJpbmc+CjwvZGljdD4K + PC9wbGlzdD4KhpKEmZkPTlNQcmludEFsbFBhZ2VzhpKgkoSZmQhOU0NvcGllc4aSpJKE + mZkLTlNQYXBlck5hbWWGkoSZmQJBNIaShJmZD05TU2NhbGluZ0ZhY3RvcoaShKGbhIQB + ZqEBhpKEmZkJTlNQcmludGVyhpKEhIQJTlNQcmludGVyAJSShJmZCU9LSSBCNjEwMIaG + koSZmQ1OU1JpZ2h0TWFyZ2luhpKEoZuwoQCGkoSZmRZOU0hvcml6b250YWxseUNlbnRl + cmVkhpKkkoSZmRlOU1ByaW50UmV2ZXJzZU9yaWVudGF0aW9uhpKgkoSZmQ1OU0pvYkZl + YXR1cmVzhpKElpcAhpKEmZkMTlNMZWZ0TWFyZ2luhpKEoZuwoQCGkoSZmRVOU0hvcml6 + b25hbFBhZ2luYXRpb26GkqCShJmZEk5TUmV2ZXJzZVBhZ2VPcmRlcoaSoJKEmZkLTlNU + b3BNYXJnaW6GkoShm7ChAIaShJmZDk5TQm90dG9tTWFyZ2luhpKEoZuwoQCGkoSZmQpO + U0xhc3RQYWdlhpKEoZuil4J/////hpKEmZkLTlNGaXJzdFBhZ2WGkqSShJmZDU5TT3Jp + ZW50YXRpb26GkqCGhg== + + ReadOnly + NO + RowAlign + 1 + RowSpacing + 36 + SheetTitle + Canvas 1 + SmartAlignmentGuidesActive + YES + SmartDistanceGuidesActive + YES + UseEntirePage + + VPages + 1 + WindowInfo + + CurrentSheet + 0 + Frame + {{351, 5}, {626, 827}} + ShowRuler + + ShowStatusBar + + VisibleRegion + {{-26, 0}, {611, 729}} + Zoom + 1 + + + Added: trunk/varnish-doc/images/varnish-rfc2616-on-http-caches.png =================================================================== (Binary files differ) Property changes on: trunk/varnish-doc/images/varnish-rfc2616-on-http-caches.png ___________________________________________________________________ Name: svn:mime-type + application/octet-stream From phk at projects.linpro.no Thu Jul 20 13:29:13 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 15:29:13 +0200 (CEST) Subject: r523 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060720132913.020D51EC395@projects.linpro.no> Author: phk Date: 2006-07-20 15:29:13 +0200 (Thu, 20 Jul 2006) New Revision: 523 Modified: trunk/varnish-cache/bin/varnishd/cache_session.c trunk/varnish-cache/include/stat_field.h Log: Keep track of active source addresses Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-20 12:15:49 UTC (rev 522) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-20 13:29:13 UTC (rev 523) @@ -73,11 +73,18 @@ c3 = NULL; TAILQ_FOREACH_SAFE(c, ch, list, c2) { if (c->sum == u && !strcmp(c->addr, sp->addr)) { + if (c->nsess == 0) + VSL_stats->n_srcaddr_act++; c->nsess++; c->ttl = now + CLIENT_TTL; sp->srcaddr = c; TAILQ_REMOVE(ch, c, list); TAILQ_INSERT_TAIL(ch, c, list); + if (0 && c3 != NULL) { + TAILQ_REMOVE(ch, c3, list); + VSL_stats->n_srcaddr--; + free(c3); + } AZ(pthread_mutex_unlock(&ses_mtx)); return; } @@ -93,6 +100,7 @@ } if (c3 == NULL) { c3 = malloc(sizeof *c3); + assert(c3 != NULL); if (c3 != NULL) VSL_stats->n_srcaddr++; } else @@ -105,6 +113,7 @@ c3->ttl = now + CLIENT_TTL; c3->nsess = 1; c3->sah = ch; + VSL_stats->n_srcaddr_act++; TAILQ_INSERT_TAIL(ch, c3, list); } sp->srcaddr = c3; @@ -135,14 +144,17 @@ SES_RelSrcAddr(struct sess *sp) { - if (sp->srcaddr == NULL) { + if (0 && sp->srcaddr == NULL) { /* XXX who comes this way ? */ VSL(SLT_Debug, sp->fd, "had no srcaddr"); return; } assert(sp->srcaddr != NULL); AZ(pthread_mutex_lock(&ses_mtx)); + assert(sp->srcaddr->nsess > 0); sp->srcaddr->nsess--; + if (sp->srcaddr->nsess == 0) + VSL_stats->n_srcaddr_act--; sp->srcaddr = NULL; AZ(pthread_mutex_unlock(&ses_mtx)); } Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2006-07-20 12:15:49 UTC (rev 522) +++ trunk/varnish-cache/include/stat_field.h 2006-07-20 13:29:13 UTC (rev 523) @@ -11,6 +11,7 @@ MAC_STAT(backend_recycle, uint64_t, "u", "Backend connections recyles") MAC_STAT(n_srcaddr, uint64_t, "u", "N struct srcaddr") +MAC_STAT(n_srcaddr_act, uint64_t, "u", "N active struct srcaddr") MAC_STAT(n_sess, uint64_t, "u", "N struct sess") MAC_STAT(n_object, uint64_t, "u", "N struct object") MAC_STAT(n_objecthead, uint64_t, "u", "N struct objecthead") From phk at projects.linpro.no Thu Jul 20 13:33:22 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 15:33:22 +0200 (CEST) Subject: r524 - trunk/varnish-cache/bin/varnishd Message-ID: <20060720133322.9912B1EC396@projects.linpro.no> Author: phk Date: 2006-07-20 15:33:22 +0200 (Thu, 20 Jul 2006) New Revision: 524 Modified: trunk/varnish-cache/bin/varnishd/cache_session.c Log: upd Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-20 13:29:13 UTC (rev 523) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-20 13:33:22 UTC (rev 524) @@ -144,9 +144,8 @@ SES_RelSrcAddr(struct sess *sp) { - if (0 && sp->srcaddr == NULL) { - /* XXX who comes this way ? */ - VSL(SLT_Debug, sp->fd, "had no srcaddr"); + if (sp->srcaddr == NULL) { + /* If we never get to work pool (illegal req) */ return; } assert(sp->srcaddr != NULL); From phk at projects.linpro.no Thu Jul 20 13:39:47 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 15:39:47 +0200 (CEST) Subject: r525 - trunk/varnish-cache/bin/varnishd Message-ID: <20060720133947.EA5E51EC396@projects.linpro.no> Author: phk Date: 2006-07-20 15:39:47 +0200 (Thu, 20 Jul 2006) New Revision: 525 Modified: trunk/varnish-cache/bin/varnishd/cache_session.c Log: Be more paranoid about srcaddr Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-20 13:33:22 UTC (rev 524) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-20 13:39:47 UTC (rev 525) @@ -63,6 +63,7 @@ struct srcaddrhead *ch; time_t now; + assert(sp->srcaddr == NULL); for (u = 0, p = sp->addr; *p; p++) u += u + *p; v = u % CLIENT_HASH; @@ -105,6 +106,7 @@ VSL_stats->n_srcaddr++; } else TAILQ_REMOVE(ch, c3, list); + assert (c3 != NULL); if (c3 != NULL) { memset(c3, 0, sizeof *c3); strcpy(c3->addr, sp->addr); @@ -115,8 +117,8 @@ c3->sah = ch; VSL_stats->n_srcaddr_act++; TAILQ_INSERT_TAIL(ch, c3, list); + sp->srcaddr = c3; } - sp->srcaddr = c3; AZ(pthread_mutex_unlock(&ses_mtx)); } From phk at projects.linpro.no Thu Jul 20 13:42:56 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 15:42:56 +0200 (CEST) Subject: r526 - trunk/varnish-cache/bin/varnishd Message-ID: <20060720134256.C725B1EC3A2@projects.linpro.no> Author: phk Date: 2006-07-20 15:42:56 +0200 (Thu, 20 Jul 2006) New Revision: 526 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Only reference srcaddr on first request on session Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-20 13:39:47 UTC (rev 525) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-20 13:42:56 UTC (rev 526) @@ -516,7 +516,6 @@ sp->t0 = time(NULL); sp->vcl = VCL_Get(); - SES_RefSrcAddr(sp); assert(sp->obj == NULL); Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-20 13:39:47 UTC (rev 525) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-20 13:42:56 UTC (rev 526) @@ -37,9 +37,10 @@ TAILQ_REMOVE(&wrk_reqhead, wrq, list); VSL_stats->n_wrk_queue--; AZ(pthread_mutex_unlock(&wrk_mtx)); - assert(wrq->sess != NULL); + CHECK_OBJ_NOTNULL(wrq->sess, SESS_MAGIC); wrq->sess->wrk = w; - CHECK_OBJ_NOTNULL(wrq->sess, SESS_MAGIC); + if (wrq->sess->srcaddr == NULL) + SES_RefSrcAddr(wrq->sess); if (w->nobj != NULL) CHECK_OBJ(w->nobj, OBJECT_MAGIC); if (w->nobjhead != NULL) From phk at projects.linpro.no Thu Jul 20 14:24:00 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 16:24:00 +0200 (CEST) Subject: r527 - trunk/varnish-cache/bin/varnishd Message-ID: <20060720142400.10D291EC3A2@projects.linpro.no> Author: phk Date: 2006-07-20 16:23:59 +0200 (Thu, 20 Jul 2006) New Revision: 527 Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c Log: We need to check the TTL here also, if a (sequence of) slow client(s) manages to hold the document referenced, the prefetcher may never get lucky with it and it will linger here much past last sell date. Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-20 13:42:56 UTC (rev 526) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-20 14:23:59 UTC (rev 527) @@ -101,12 +101,13 @@ return (NULL); } were_back: - /* XXX: check ttl */ /* XXX: check Vary: */ if (!o->cacheable) { /* ignore */ } else if (o->ttl == 0) { /* Object banned but not reaped yet */ + } else if (o->ttl < sp->t_req) { + /* Object expired */ } else if (BAN_CheckObject(o, h->url)) { o->ttl = 0; VSL(SLT_ExpBan, 0, "%u was banned", o->xid); From phk at projects.linpro.no Thu Jul 20 14:40:54 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 16:40:54 +0200 (CEST) Subject: r528 - trunk/varnish-cache/bin/varnishd Message-ID: <20060720144054.EB69E1EC3A2@projects.linpro.no> Author: phk Date: 2006-07-20 16:40:54 +0200 (Thu, 20 Jul 2006) New Revision: 528 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_response.c Log: More asserts Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-20 14:23:59 UTC (rev 527) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-20 14:40:54 UTC (rev 528) @@ -41,6 +41,7 @@ cl = strtoumax(b, NULL, 0); st = stevedore->alloc(stevedore, cl); + assert(st->stevedore != NULL); TAILQ_INSERT_TAIL(&sp->obj->store, st, list); st->len = cl; sp->obj->len = cl; @@ -105,6 +106,7 @@ } else { st = stevedore->alloc(stevedore, stevedore->trim == NULL ? u : CHUNK_PREALLOC); + assert(st->stevedore != NULL); TAILQ_INSERT_TAIL(&sp->obj->store, st, list); p = st->ptr; } @@ -177,6 +179,7 @@ while (1) { if (v == 0) { st = stevedore->alloc(stevedore, CHUNK_PREALLOC); + assert(st->stevedore != NULL); TAILQ_INSERT_TAIL(&sp->obj->store, st, list); p = st->ptr + st->len; v = st->space - st->len; Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-20 14:23:59 UTC (rev 527) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-20 14:40:54 UTC (rev 528) @@ -196,6 +196,7 @@ /* XXX: conditional request handling */ if (!strcmp(sp->http->req, "GET")) { TAILQ_FOREACH(st, &sp->obj->store, list) { + assert(st->stevedore != NULL); u += st->len; if (st->stevedore->send == NULL) { RES_Write(sp, st->ptr, st->len); From phk at projects.linpro.no Thu Jul 20 14:46:49 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 16:46:49 +0200 (CEST) Subject: r529 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060720144649.594A51EC3A2@projects.linpro.no> Author: phk Date: 2006-07-20 16:46:49 +0200 (Thu, 20 Jul 2006) New Revision: 529 Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/include/stat_field.h Log: Keep an eye on deathrow Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-20 14:40:54 UTC (rev 528) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-20 14:46:49 UTC (rev 529) @@ -76,8 +76,10 @@ continue; } TAILQ_REMOVE(&exp_deathrow, o, deathrow); + VSL_stats->n_deathrow--; + VSL_stats->n_expired++; AZ(pthread_mutex_unlock(&exp_mtx)); - VSL(SLT_ExpKill, 0, "%u", o->xid); + VSL(SLT_ExpKill, 0, "%u %d", o->xid, (int)(o->ttl - t)); HSH_Deref(o); } } @@ -124,6 +126,7 @@ if (sp->handling == VCL_RET_DISCARD) { AZ(pthread_mutex_lock(&exp_mtx)); TAILQ_INSERT_TAIL(&exp_deathrow, o, deathrow); + VSL_stats->n_deathrow++; AZ(pthread_mutex_unlock(&exp_mtx)); continue; } Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2006-07-20 14:40:54 UTC (rev 528) +++ trunk/varnish-cache/include/stat_field.h 2006-07-20 14:46:49 UTC (rev 529) @@ -26,4 +26,7 @@ MAC_STAT(n_wrk_busy, uint64_t, "u", "N busy worker threads") MAC_STAT(n_wrk_queue, uint64_t, "u", "N queued work requests") +MAC_STAT(n_expired, uint64_t, "u", "N expired objects") +MAC_STAT(n_deathrow, uint64_t, "u", "N objects on deathrow") + MAC_STAT(losthdr, uint64_t, "u", "HTTP header overflows") From phk at projects.linpro.no Thu Jul 20 15:10:36 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 20 Jul 2006 17:10:36 +0200 (CEST) Subject: r530 - trunk/varnish-cache/bin/varnishd Message-ID: <20060720151036.DE6A31EC3A2@projects.linpro.no> Author: phk Date: 2006-07-20 17:10:36 +0200 (Thu, 20 Jul 2006) New Revision: 530 Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c Log: Add XXX comment Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-20 14:46:49 UTC (rev 529) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-20 15:10:36 UTC (rev 530) @@ -2,6 +2,13 @@ * $Id$ * * Expiry of cached objects and execution of prefetcher + * + * XXX: Objects can linger on deathrow as long as a slow client + * XXX: tickles data away from it. With many slow clients this could + * XXX: possibly make deathrow very long and make the hangman waste + * XXX: time. The solution is to have another queue for such "pending + * XXX: cases" and have HSH_Deref() move them to deathrow when they + * XXX: are ready. */ #include From phk at projects.linpro.no Thu Jul 20 22:08:44 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 00:08:44 +0200 (CEST) Subject: r531 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060720220844.4DBDE1EC3A5@projects.linpro.no> Author: phk Date: 2006-07-21 00:08:43 +0200 (Fri, 21 Jul 2006) New Revision: 531 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_main.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_response.c trunk/varnish-cache/bin/varnishd/cache_session.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishd/heritage.h trunk/varnish-cache/bin/varnishd/rfc2616.c trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/include/shmlog_tags.h trunk/varnish-cache/lib/libvcl/vcl_compile.c Log: Yet another refinement to the way we store and deal with HTTP headers. Record a triplet of {start, data, end} for all HTTP data items. This represents a regrettable uglification of the sourcecode, but most of it compiles out to constants and the runtime benefits will be worth it. Generate H_FOO magic strings for all the headers we know about. These strings have a length as first char and always ends in ':'. Also genereate H_FOO format strings in VCL compiler. Mandate (with assert) that header references happen using H_FOO strings. Make number of allowed HTTP headers a compile time constant (32) but make the workspace a run-time variable (4096). Introduce new SHM tag for dumping aborted HTTP protocol requests. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-20 22:08:43 UTC (rev 531) @@ -17,6 +17,19 @@ #define MAX_IOVS 10 +#define MAX_HTTP_HDRS 32 + +#define HTTP_HDR_REQ 0 +#define HTTP_HDR_URL 1 +#define HTTP_HDR_PROTO 2 +#define HTTP_HDR_STATUS 3 +#define HTTP_HDR_RESPONSE 4 +#define HTTP_HDR_FIRST 5 + +#define HTTP_START 0 +#define HTTP_DATA 1 +#define HTTP_END 2 + struct event_base; struct cli; struct sbuf; @@ -46,21 +59,16 @@ http_callback_f *callback; void *arg; - char *s; /* start of buffer */ - char *t; /* start of trailing data */ - char *v; /* end of valid bytes */ - char *e; /* end of buffer */ + char *s; /* (S)tart of buffer */ + char *t; /* start of (T)railing data */ + char *v; /* end of (V)alid bytes */ + char *f; /* first (F)ree byte */ + char *e; /* (E)nd of buffer */ - char *req; - char *url; - char *proto; - char *status; - char *response; + unsigned conds; /* If-* headers present */ - unsigned conds; /* If-* headers present */ - - unsigned nhdr; - char **hdr; + char *hd[MAX_HTTP_HDRS][HTTP_END + 1]; + unsigned nhd; }; /*--------------------------------------------------------------------*/ @@ -287,7 +295,8 @@ void HSH_Init(void); /* cache_http.c */ -void http_Init(struct http *ht, void *space); +void HTTP_Init(void); +void http_Init(struct http *ht, void *space, unsigned len); int http_GetHdr(struct http *hp, const char *hdr, char **ptr); int http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr); int http_GetStatus(struct http *hp); @@ -304,6 +313,9 @@ Build_Reply }; void http_BuildSbuf(int fd, enum http_build mode, struct sbuf *sb, struct http *hp); +#define HTTPH(a, b, c, d, e, f, g) extern char b[]; +#include "http_headers.h" +#undef HTTPH /* cache_pass.c */ void PassSession(struct worker *w, struct sess *sp); Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-20 22:08:43 UTC (rev 531) @@ -44,7 +44,6 @@ #define VBC_MEM_MAGIC 0x2fd7af01 struct vbe_conn vbe; struct http http; - char *http_hdr; }; /* A backend IP */ @@ -74,12 +73,7 @@ { struct vbc_mem *vbcm; - vbcm = calloc( - sizeof *vbcm + - heritage.mem_http_headers * sizeof vbcm->http_hdr + - heritage.mem_http_headerspace + - heritage.mem_workspace, - 1); + vbcm = calloc(sizeof *vbcm + heritage.mem_workspace, 1); if (vbcm == NULL) return (NULL); vbcm->magic = VBC_MEM_MAGIC; @@ -87,7 +81,7 @@ vbcm->vbe.magic = VBE_CONN_MAGIC; vbcm->vbe.vbcm = vbcm; vbcm->vbe.http = &vbcm->http; - http_Init(&vbcm->http, (void *)(vbcm + 1)); + http_Init(&vbcm->http, (void *)(vbcm + 1), heritage.mem_workspace); return (&vbcm->vbe); } Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-20 22:08:43 UTC (rev 531) @@ -87,10 +87,11 @@ assert(sp->obj == NULL); if (sp->fd < 0) { /* Allready closed */ - } else if (http_GetHdr(sp->http, "Connection", &b) && + } else if (http_GetHdr(sp->http, H_Connection, &b) && !strcmp(b, "close")) { vca_close_session(sp, "Connection header"); - } else if (strcmp(sp->http->proto, "HTTP/1.1")) { + } else if (strcmp(sp->http->hd[HTTP_HDR_PROTO][HTTP_START], + "HTTP/1.1")) { vca_close_session(sp, "not HTTP/1.1"); } VCL_Rel(sp->vcl); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-20 22:08:43 UTC (rev 531) @@ -216,13 +216,13 @@ vc = sp->vbc; hp = sp->bkd_http; - if (http_GetHdr(hp, "Last-Modified", &b)) + if (http_GetHdr(hp, H_Last_Modified, &b)) sp->obj->last_modified = TIM_parse(b); http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); if (body) { - if (http_GetHdr(hp, "Content-Length", &b)) + if (http_GetHdr(hp, H_Content_Length, &b)) cls = fetch_straight(sp, vc->fd, hp, b); - else if (http_HdrIs(hp, "Transfer-Encoding", "chunked")) + else if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) cls = fetch_chunked(sp, vc->fd, hp); else cls = fetch_eof(sp, vc->fd, hp); @@ -233,7 +233,7 @@ sp->obj->header = strdup(sbuf_data(w->sb)); VSL_stats->n_header++; - if (http_GetHdr(hp, "Connection", &b) && !strcasecmp(b, "close")) + if (http_GetHdr(hp, H_Connection, &b) && !strcasecmp(b, "close")) cls = 1; if (cls) Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-20 22:08:43 UTC (rev 531) @@ -46,7 +46,7 @@ struct http *h; struct objhead *oh; struct object *o; - char *c; + char *url, *host; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); @@ -77,8 +77,9 @@ } else CHECK_OBJ_NOTNULL(w->nobj, OBJECT_MAGIC); - if (!http_GetHdr(h, "Host", &c)) - c = h->url; + url = h->hd[HTTP_HDR_URL][HTTP_START]; + if (!http_GetHdr(h, H_Host, &host)) + host = url; if (sp->obj != NULL) { CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); o = sp->obj; @@ -87,7 +88,7 @@ AZ(pthread_mutex_lock(&oh->mtx)); goto were_back; } - oh = hash->lookup(h->url, c, w->nobjhead); + oh = hash->lookup(url, host, w->nobjhead); CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); if (oh == w->nobjhead) w->nobjhead = NULL; @@ -108,7 +109,7 @@ /* Object banned but not reaped yet */ } else if (o->ttl < sp->t_req) { /* Object expired */ - } else if (BAN_CheckObject(o, h->url)) { + } else if (BAN_CheckObject(o, url)) { o->ttl = 0; VSL(SLT_ExpBan, 0, "%u was banned", o->xid); EXP_TTLchange(o); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-20 22:08:43 UTC (rev 531) @@ -15,19 +15,28 @@ #include "heritage.h" #include "cache.h" +#define HTTPH(a, b, c, d, e, f, g) char b[] = "*" a ":"; +#include "http_headers.h" +#undef HTTPH + +#define VSLH(a, b, c, d) \ + VSLR((a), (b), (c)->hd[d][HTTP_START], (c)->hd[d][HTTP_END]); + /*--------------------------------------------------------------------*/ void -http_Init(struct http *hp, void *space) +http_Init(struct http *hp, void *space, unsigned len) { char *sp = space; + assert(len > 0); memset(hp, 0, sizeof *hp); hp->magic = HTTP_MAGIC; - hp->hdr = (void *)sp; - sp += heritage.mem_http_headers * sizeof hp->hdr; hp->s = sp; - hp->e = hp->s + heritage.mem_http_headerspace; + hp->t = sp; + hp->v = sp; + hp->f = sp; + hp->e = sp + len; } /*--------------------------------------------------------------------*/ @@ -38,19 +47,29 @@ unsigned u, l; char *p; - l = strlen(hdr); - for (u = 0; u < hp->nhdr; u++) { - if (strncasecmp(hdr, hp->hdr[u], l)) + l = hdr[0]; + assert(l == strlen(hdr + 1)); + assert(hdr[l] == ':'); + hdr++; + for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) { + assert(hp->hd[u][HTTP_START] != NULL); + assert(hp->hd[u][HTTP_END] != NULL); + if (hp->hd[u][HTTP_END] < hp->hd[u][HTTP_START] + l) continue; - p = hp->hdr[u]; - if (p[l] != ':') + if (hp->hd[u][HTTP_START][l-1] != ':') continue; - p += l + 1; - while (isspace(*p)) - p++; - *ptr = p; + if (strncasecmp(hdr, hp->hd[u][HTTP_START], l)) + continue; + if (hp->hd[u][HTTP_DATA] == NULL) { + p = hp->hd[u][HTTP_START] + l; + while (isspace(*p)) + p++; + hp->hd[u][HTTP_DATA] = p; + } + *ptr = hp->hd[u][HTTP_DATA]; return (1); } + *ptr = NULL; return (0); } @@ -153,7 +172,9 @@ http_GetStatus(struct http *hp) { - return (strtoul(hp->status, NULL /* XXX */, 10)); + assert(hp->hd[HTTP_HDR_STATUS][HTTP_START] != NULL); + return (strtoul(hp->hd[HTTP_HDR_STATUS][HTTP_START], + NULL /* XXX */, 10)); } /*-------------------------------------------------------------------- @@ -169,11 +190,12 @@ if (*p == '\r') p++; - hp->nhdr = 0; + hp->nhd = HTTP_HDR_FIRST; hp->conds = 0; r = NULL; /* For FlexeLint */ assert(p < hp->v); /* http_header_complete() guarantees this */ for (; p < hp->v; p = r) { + /* XXX: handle continuation lines */ q = strchr(p, '\n'); assert(q != NULL); r = q + 1; @@ -188,17 +210,17 @@ p[2] == '-') hp->conds = 1; - if (hp->nhdr < heritage.mem_http_headers) { - hp->hdr[hp->nhdr++] = p; - VSLR(SLT_Header, fd, p, q); + if (hp->nhd < MAX_HTTP_HDRS) { + hp->hd[hp->nhd][HTTP_START] = p; + hp->hd[hp->nhd][HTTP_END] = q; + VSLH(SLT_Header, fd, hp, hp->nhd); + hp->nhd++; } else { VSL_stats->losthdr++; VSLR(SLT_LostHeader, fd, p, q); } } assert(hp->t <= hp->v); - if (hp->t != r) - printf("hp->t %p r %p\n", hp->t, r); assert(hp->t == r); return (0); } @@ -217,25 +239,27 @@ continue; /* First, the request type (GET/HEAD etc) */ - hp->req = p; + hp->hd[HTTP_HDR_REQ][HTTP_START] = p; for (; isalpha(*p); p++) ; - VSLR(SLT_Request, fd, hp->req, p); + hp->hd[HTTP_HDR_REQ][HTTP_END] = p; + VSLH(SLT_Request, fd, hp, HTTP_HDR_REQ); *p++ = '\0'; /* Next find the URI */ while (isspace(*p) && *p != '\n') p++; if (*p == '\n') { - VSLR(SLT_Debug, fd, hp->s, hp->v); + VSLR(SLT_HttpGarbage, fd, hp->s, hp->v); return (400); } - hp->url = p; + hp->hd[HTTP_HDR_URL][HTTP_START] = p; while (!isspace(*p)) p++; - VSLR(SLT_URL, fd, hp->url, p); + hp->hd[HTTP_HDR_URL][HTTP_END] = p; + VSLH(SLT_URL, fd, hp, HTTP_HDR_URL); if (*p == '\n') { - VSLR(SLT_Debug, fd, hp->s, hp->v); + VSLR(SLT_HttpGarbage, fd, hp->s, hp->v); return (400); } *p++ = '\0'; @@ -244,19 +268,20 @@ while (isspace(*p) && *p != '\n') p++; if (*p == '\n') { - VSLR(SLT_Debug, fd, hp->s, hp->v); + VSLR(SLT_HttpGarbage, fd, hp->s, hp->v); return (400); } - hp->proto = p; + hp->hd[HTTP_HDR_PROTO][HTTP_START] = p; while (!isspace(*p)) p++; - VSLR(SLT_Protocol, fd, hp->proto, p); + hp->hd[HTTP_HDR_PROTO][HTTP_END] = p; + VSLH(SLT_Protocol, fd, hp, HTTP_HDR_PROTO); if (*p != '\n') *p++ = '\0'; while (isspace(*p) && *p != '\n') p++; if (*p != '\n') { - VSLR(SLT_Debug, fd, hp->s, hp->v); + VSLR(SLT_HttpGarbage, fd, hp->s, hp->v); return (400); } *p++ = '\0'; @@ -278,31 +303,35 @@ continue; /* First, protocol */ - hp->proto = p; + hp->hd[HTTP_HDR_PROTO][HTTP_START] = p; while (!isspace(*p)) p++; - VSLR(SLT_Protocol, fd, hp->proto, p); + hp->hd[HTTP_HDR_PROTO][HTTP_END] = p; + VSLH(SLT_Protocol, fd, hp, HTTP_HDR_PROTO); *p++ = '\0'; /* Next find the status */ while (isspace(*p)) p++; - hp->status = p; + hp->hd[HTTP_HDR_STATUS][HTTP_START] = p; while (!isspace(*p)) p++; - VSLR(SLT_Status, fd, hp->status, p); + hp->hd[HTTP_HDR_STATUS][HTTP_END] = p; + VSLH(SLT_Status, fd, hp, HTTP_HDR_STATUS); *p++ = '\0'; /* Next find the response */ while (isspace(*p)) p++; - hp->response = p; + hp->hd[HTTP_HDR_RESPONSE][HTTP_START] = p; while (*p != '\n') p++; - for (q = p; q > hp->response && isspace(q[-1]); q--) + for (q = p; q > hp->hd[HTTP_HDR_RESPONSE][HTTP_START] && + isspace(q[-1]); q--) continue; *q = '\0'; - VSLR(SLT_Response, fd, hp->response, q); + hp->hd[HTTP_HDR_RESPONSE][HTTP_END] = q; + VSLH(SLT_Response, fd, hp, HTTP_HDR_RESPONSE); p++; return (http_dissect_hdrs(hp, fd, p)); @@ -360,7 +389,7 @@ l = hp->e - hp->v; if (l <= 1) { VSL(SLT_HttpError, fd, "Received too much"); - VSLR(SLT_Debug, fd, hp->s, hp->v); + VSLR(SLT_HttpGarbage, fd, hp->s, hp->v); hp->t = NULL; ret = 1; } else { @@ -458,55 +487,47 @@ void http_BuildSbuf(int fd, enum http_build mode, struct sbuf *sb, struct http *hp) { - unsigned u, sup; + unsigned u, sup, rr; sbuf_clear(sb); assert(sb != NULL); switch (mode) { - case Build_Reply: - sbuf_cat(sb, hp->proto); - sbuf_cat(sb, " "); - sbuf_cat(sb, hp->status); - sbuf_cat(sb, " "); - sbuf_cat(sb, hp->response); - sup = 2; - break; - case Build_Pipe: - sbuf_cat(sb, hp->req); - sbuf_cat(sb, " "); - sbuf_cat(sb, hp->url); - sbuf_cat(sb, " "); - sbuf_cat(sb, hp->proto); - sup = 0; - break; - case Build_Pass: - sbuf_cat(sb, hp->req); - sbuf_cat(sb, " "); - sbuf_cat(sb, hp->url); - sbuf_cat(sb, " "); - sbuf_cat(sb, hp->proto); - sup = 2; - break; - case Build_Fetch: - sbuf_cat(sb, "GET "); - sbuf_cat(sb, hp->url); - sbuf_cat(sb, " "); - sbuf_cat(sb, hp->proto); - sup = 1; - break; + case Build_Reply: rr = 0; sup = 2; break; + case Build_Pipe: rr = 1; sup = 0; break; + case Build_Pass: rr = 1; sup = 2; break; + case Build_Fetch: rr = 2; sup = 1; break; default: sup = 0; /* for flexelint */ + rr = 0; /* for flexelint */ printf("mode = %d\n", mode); - assert(mode == 1 || mode == 2); + assert(__LINE__ == 0); } + if (rr == 0) { + sbuf_cat(sb, hp->hd[HTTP_HDR_PROTO][HTTP_START]); + sbuf_cat(sb, " "); + sbuf_cat(sb, hp->hd[HTTP_HDR_STATUS][HTTP_START]); + sbuf_cat(sb, " "); + sbuf_cat(sb, hp->hd[HTTP_HDR_RESPONSE][HTTP_START]); + } else { + if (rr == 2) { + sbuf_cat(sb, "GET "); + } else { + sbuf_cat(sb, hp->hd[HTTP_HDR_REQ][HTTP_START]); + sbuf_cat(sb, " "); + } + sbuf_cat(sb, hp->hd[HTTP_HDR_URL][HTTP_START]); + sbuf_cat(sb, " "); + sbuf_cat(sb, hp->hd[HTTP_HDR_PROTO][HTTP_START]); + } + sbuf_cat(sb, "\r\n"); - for (u = 0; u < hp->nhdr; u++) { - if (http_supress(hp->hdr[u], sup)) + for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) { + if (http_supress(hp->hd[u][HTTP_START], sup)) continue; if (1) - VSL(SLT_BldHdr, fd, "%s", hp->hdr[u]); - sbuf_cat(sb, hp->hdr[u]); + VSL(SLT_BldHdr, fd, "%s", hp->hd[u][HTTP_START]); + sbuf_cat(sb, hp->hd[u][HTTP_START]); sbuf_cat(sb, "\r\n"); } if (mode != Build_Reply) { @@ -514,3 +535,11 @@ sbuf_finish(sb); } } + +void +HTTP_Init(void) +{ +#define HTTPH(a, b, c, d, e, f, g) b[0] = strlen(b + 1); +#include "http_headers.h" +#undef HTTPH +} Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-07-20 22:08:43 UTC (rev 531) @@ -105,6 +105,7 @@ VCL_Init(); VCL_Load(heritage.vcl_file, "boot", NULL); + HTTP_Init(); SES_Init(); VBE_Init(); Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-20 22:08:43 UTC (rev 531) @@ -152,18 +152,18 @@ sbuf_finish(w->sb); RES_Write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); - if (http_GetHdr(hp, "Content-Length", &b)) + if (http_GetHdr(hp, H_Content_Length, &b)) cls = pass_straight(sp, vc->fd, hp, b); - else if (http_HdrIs(hp, "Connection", "close")) + else if (http_HdrIs(hp, H_Connection, "close")) cls = pass_straight(sp, vc->fd, hp, NULL); - else if (http_HdrIs(hp, "Transfer-Encoding", "chunked")) + else if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) cls = pass_chunked(sp, vc->fd, hp); else { cls = pass_straight(sp, vc->fd, hp, NULL); } RES_Flush(sp); - if (http_GetHdr(hp, "Connection", &b) && !strcasecmp(b, "close")) + if (http_GetHdr(hp, H_Connection, &b) && !strcasecmp(b, "close")) cls = 1; if (cls) Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-20 22:08:43 UTC (rev 531) @@ -126,7 +126,7 @@ RES_Write(sp, "Last-Modified: ", -1); RES_Write(sp, p, -1); RES_Write(sp, "\r\n", -1); - if (strcmp(sp->http->proto, "HTTP/1.1")) + if (strcmp(sp->http->hd[HTTP_HDR_PROTO][HTTP_START], "HTTP/1.1")) RES_Write(sp, "Connection: close\r\n", -1); sbuf_printf(sb, "X-Varnish: xid %u\r\n", sp->obj->xid); sbuf_printf(sb, "\r\n"); @@ -144,7 +144,7 @@ time_t ims; if (sp->obj->last_modified > 0 && - http_GetHdr(sp->http, "If-Modified-Since", &p)) { + http_GetHdr(sp->http, H_If_Modified_Since, &p)) { ims = TIM_parse(p); if (ims > sp->t_req) /* [RFC2616 14.25] */ return (0); @@ -187,14 +187,14 @@ sp->obj->age + sp->t_req - sp->obj->entered); sbuf_printf(sb, "Via: 1.1 varnish\r\n"); sbuf_printf(sb, "X-Varnish: xid %u\r\n", sp->obj->xid); - if (strcmp(sp->http->proto, "HTTP/1.1")) + if (strcmp(sp->http->hd[HTTP_HDR_PROTO][HTTP_START], "HTTP/1.1")) sbuf_printf(sb, "Connection: close\r\n"); sbuf_printf(sb, "\r\n"); sbuf_finish(sb); RES_Write(sp, sbuf_data(sb), sbuf_len(sb)); bytes += sbuf_len(sb); /* XXX: conditional request handling */ - if (!strcmp(sp->http->req, "GET")) { + if (!strcmp(sp->http->hd[HTTP_HDR_REQ][HTTP_START], "GET")) { TAILQ_FOREACH(st, &sp->obj->store, list) { assert(st->stevedore != NULL); u += st->len; Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-20 22:08:43 UTC (rev 531) @@ -32,7 +32,6 @@ struct sess sess; struct http http; - char *http_hdr; }; /*--------------------------------------------------------------------*/ @@ -170,10 +169,7 @@ (void)addr; /* XXX */ (void)len; /* XXX */ sm = calloc( - sizeof *sm + - heritage.mem_http_headers * sizeof sm->http_hdr + - heritage.mem_http_headerspace + - heritage.mem_workspace, + sizeof *sm + heritage.mem_workspace, 1); if (sm == NULL) return (NULL); @@ -182,7 +178,7 @@ sm->sess.magic = SESS_MAGIC; sm->sess.mem = sm; sm->sess.http = &sm->http; - http_Init(&sm->http, (void *)(sm + 1)); + http_Init(&sm->http, (void *)(sm + 1), heritage.mem_workspace); return (&sm->sess); } Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-20 22:08:43 UTC (rev 531) @@ -56,11 +56,12 @@ char * VRT_GetReq(struct sess *sp) { + char *p; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); assert(sp != NULL); assert(sp->http != NULL); - return (sp->http->req); + return (sp->http->hd[HTTP_HDR_REQ][HTTP_START]); } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/heritage.h =================================================================== --- trunk/varnish-cache/bin/varnishd/heritage.h 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/bin/varnishd/heritage.h 2006-07-20 22:08:43 UTC (rev 531) @@ -42,8 +42,6 @@ unsigned wthread_timeout; /* Memory allocation hints */ - unsigned mem_http_headerspace; - unsigned mem_http_headers; unsigned mem_workspace; }; Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-20 22:08:43 UTC (rev 531) @@ -83,10 +83,10 @@ retirement_age = INT_MAX; u1 = u2 = 0; - if (http_GetHdrField(hp, "Cache-Control", "max-age", &p)) { + if (http_GetHdrField(hp, H_Cache_Control, "max-age", &p)) { u1 = strtoul(p, NULL, 0); u2 = 0; - if (http_GetHdr(hp, "Age", &p)) { + if (http_GetHdr(hp, H_Age, &p)) { u2 = strtoul(p, NULL, 0); obj->age = u2; } @@ -95,11 +95,11 @@ } h_date = 0; - if (http_GetHdr(hp, "Date", &p)) + if (http_GetHdr(hp, H_Date, &p)) h_date = TIM_parse(p); h_expires = 0; - if (http_GetHdr(hp, "Expires", &p)) + if (http_GetHdr(hp, H_Expires, &p)) h_expires = TIM_parse(p); if (h_date < t_req && h_expires > t_req) { Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-20 22:08:43 UTC (rev 531) @@ -543,9 +543,7 @@ heritage.wthread_min = 1; heritage.wthread_max = UINT_MAX; heritage.wthread_timeout = 10; - heritage.mem_http_headerspace= 4096; - heritage.mem_http_headers= 32; - heritage.mem_workspace = 0; + heritage.mem_workspace = 4096; while ((o = getopt(argc, argv, "b:df:h:p:s:t:w:")) != -1) switch (o) { Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-07-20 22:08:43 UTC (rev 531) @@ -20,6 +20,7 @@ SLTM(BackendReuse) SLTM(BackendClose) SLTM(HttpError) +SLTM(HttpGarbage) SLTM(ClientAddr) SLTM(Backend) SLTM(Request) Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-07-20 15:10:36 UTC (rev 530) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-07-20 22:08:43 UTC (rev 531) @@ -689,7 +689,8 @@ p[i] = '\0'; v->name = p; v->fmt = STRING; - asprintf(&p, "VRT_GetHdr(sp, \"%s\")", v->name + vh->len); + asprintf(&p, "VRT_GetHdr(sp, \"\\%03o%s:\")", + strlen(v->name + vh->len) + 1, v->name + vh->len); assert(p != NULL); v->rname = p; return (v); From phk at projects.linpro.no Fri Jul 21 07:18:40 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 09:18:40 +0200 (CEST) Subject: r532 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060721071840.EF7D61EC38D@projects.linpro.no> Author: phk Date: 2006-07-21 09:18:40 +0200 (Fri, 21 Jul 2006) New Revision: 532 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_session.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/include/shmlog_tags.h Log: Rename shmlog tags for headers to RxHeader and TxHeader that's more logical. Rename http_Init() to http_Setup() to avoid clash with HTTP_Init(). Remove unused variable Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-20 22:08:43 UTC (rev 531) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-21 07:18:40 UTC (rev 532) @@ -296,7 +296,7 @@ /* cache_http.c */ void HTTP_Init(void); -void http_Init(struct http *ht, void *space, unsigned len); +void http_Setup(struct http *ht, void *space, unsigned len); int http_GetHdr(struct http *hp, const char *hdr, char **ptr); int http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr); int http_GetStatus(struct http *hp); Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-20 22:08:43 UTC (rev 531) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-21 07:18:40 UTC (rev 532) @@ -81,7 +81,7 @@ vbcm->vbe.magic = VBE_CONN_MAGIC; vbcm->vbe.vbcm = vbcm; vbcm->vbe.http = &vbcm->http; - http_Init(&vbcm->http, (void *)(vbcm + 1), heritage.mem_workspace); + http_Setup(&vbcm->http, (void *)(vbcm + 1), heritage.mem_workspace); return (&vbcm->vbe); } Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-20 22:08:43 UTC (rev 531) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-21 07:18:40 UTC (rev 532) @@ -25,7 +25,7 @@ /*--------------------------------------------------------------------*/ void -http_Init(struct http *hp, void *space, unsigned len) +http_Setup(struct http *hp, void *space, unsigned len) { char *sp = space; @@ -213,7 +213,7 @@ if (hp->nhd < MAX_HTTP_HDRS) { hp->hd[hp->nhd][HTTP_START] = p; hp->hd[hp->nhd][HTTP_END] = q; - VSLH(SLT_Header, fd, hp, hp->nhd); + VSLH(SLT_RxHeader, fd, hp, hp->nhd); hp->nhd++; } else { VSL_stats->losthdr++; @@ -526,7 +526,7 @@ if (http_supress(hp->hd[u][HTTP_START], sup)) continue; if (1) - VSL(SLT_BldHdr, fd, "%s", hp->hd[u][HTTP_START]); + VSL(SLT_TxHeader, fd, "%s", hp->hd[u][HTTP_START]); sbuf_cat(sb, hp->hd[u][HTTP_START]); sbuf_cat(sb, "\r\n"); } Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-20 22:08:43 UTC (rev 531) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-21 07:18:40 UTC (rev 532) @@ -178,7 +178,7 @@ sm->sess.magic = SESS_MAGIC; sm->sess.mem = sm; sm->sess.http = &sm->http; - http_Init(&sm->http, (void *)(sm + 1), heritage.mem_workspace); + http_Setup(&sm->http, (void *)(sm + 1), heritage.mem_workspace); return (&sm->sess); } Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-20 22:08:43 UTC (rev 531) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-21 07:18:40 UTC (rev 532) @@ -56,7 +56,6 @@ char * VRT_GetReq(struct sess *sp) { - char *p; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); assert(sp != NULL); Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-07-20 22:08:43 UTC (rev 531) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-07-21 07:18:40 UTC (rev 532) @@ -29,8 +29,8 @@ SLTM(Status) SLTM(URL) SLTM(Protocol) -SLTM(Header) -SLTM(BldHdr) +SLTM(RxHeader) +SLTM(TxHeader) SLTM(LostHeader) SLTM(VCL_call) SLTM(VCL_trace) From phk at projects.linpro.no Fri Jul 21 09:32:10 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 11:32:10 +0200 (CEST) Subject: r533 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060721093210.7C0C61EC396@projects.linpro.no> Author: phk Date: 2006-07-21 11:32:09 +0200 (Fri, 21 Jul 2006) New Revision: 533 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_response.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/include/http_headers.h Log: HTTP header munging part (N of M) NB: Only pass mode (lightly) tested right now. Give up on the three element array per header and use a two element struct instead, it reduces obfuscation and removes risk of pointer fandango. Introduce #defined filtercontrol in http_headers.h, use them in a new field. Only Pass is there for now. Use the http-workspace for building headers instead of sbuf. Move uiovec handling to cache_pool.c where it more naturally belongs and so we can use it on both backends and sessions. Add http header munging functiosn for copying, printf'ing, filtering and writing headers. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-21 07:18:40 UTC (rev 532) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-21 09:32:09 UTC (rev 533) @@ -15,10 +15,10 @@ #include "common.h" #include "miniobj.h" -#define MAX_IOVS 10 - #define MAX_HTTP_HDRS 32 +#define MAX_IOVS (MAX_HTTP_HDRS * 2) + #define HTTP_HDR_REQ 0 #define HTTP_HDR_URL 1 #define HTTP_HDR_PROTO 2 @@ -26,10 +26,6 @@ #define HTTP_HDR_RESPONSE 4 #define HTTP_HDR_FIRST 5 -#define HTTP_START 0 -#define HTTP_DATA 1 -#define HTTP_END 2 - struct event_base; struct cli; struct sbuf; @@ -52,6 +48,11 @@ typedef void http_callback_f(void *, int bad); +struct http_hdr { + char *b; + char *e; +}; + struct http { unsigned magic; #define HTTP_MAGIC 0x6428b5c9 @@ -67,8 +68,9 @@ unsigned conds; /* If-* headers present */ - char *hd[MAX_HTTP_HDRS][HTTP_END + 1]; + struct http_hdr hd[MAX_HTTP_HDRS]; unsigned nhd; + unsigned char hdf[MAX_HTTP_HDRS]; }; /*--------------------------------------------------------------------*/ @@ -85,6 +87,8 @@ pthread_cond_t cv; TAILQ_ENTRY(worker) list; + int *wfd; + unsigned werr; /* valid after WRK_Flush() */ struct iovec iov[MAX_IOVS]; unsigned niov; size_t liov; @@ -296,6 +300,13 @@ /* cache_http.c */ void HTTP_Init(void); +void http_Write(struct worker *w, struct http *hp, int resp); +void http_CopyReq(int fd, struct http *to, struct http *fm); +void http_CopyResp(int fd, struct http *to, struct http *fm); +void http_FilterHeader(int fd, struct http *to, struct http *fm, unsigned how); +void http_CopyHeader(int fd, struct http *to, struct http *fm, unsigned n); +void http_PrintfHeader(int fd, struct http *to, const char *fmt, ...); +int http_IsHdr(struct http_hdr *hh, char *hdr); void http_Setup(struct http *ht, void *space, unsigned len); int http_GetHdr(struct http *hp, const char *hdr, char **ptr); int http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr); @@ -318,7 +329,7 @@ #undef HTTPH /* cache_pass.c */ -void PassSession(struct worker *w, struct sess *sp); +void PassSession(struct sess *sp); void PassBody(struct worker *w, struct sess *sp); /* cache_pipe.c */ @@ -327,6 +338,10 @@ /* cache_pool.c */ void WRK_Init(void); void WRK_QueueSession(struct sess *sp); +void WRK_Reset(struct worker *w, int *fd); +int WRK_Flush(struct worker *w); +void WRK_Write(struct worker *w, const void *ptr, size_t len); +void WRK_WriteH(struct worker *w, struct http_hdr *hh, const char *suf); /* cache_session.c [SES] */ void SES_Init(void); @@ -352,8 +367,6 @@ /* cache_response.c */ void RES_Error(struct sess *sp, int error, const char *msg); -void RES_Flush(struct sess *sp); -void RES_Write(struct sess *sp, const void *ptr, size_t len); void RES_WriteObj(struct sess *sp); /* cache_vcl.c */ Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-21 07:18:40 UTC (rev 532) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-21 09:32:09 UTC (rev 533) @@ -90,8 +90,7 @@ } else if (http_GetHdr(sp->http, H_Connection, &b) && !strcmp(b, "close")) { vca_close_session(sp, "Connection header"); - } else if (strcmp(sp->http->hd[HTTP_HDR_PROTO][HTTP_START], - "HTTP/1.1")) { + } else if (strcmp(sp->http->hd[HTTP_HDR_PROTO].b, "HTTP/1.1")) { vca_close_session(sp, "not HTTP/1.1"); } VCL_Rel(sp->vcl); @@ -263,7 +262,7 @@ if (sp->handling == VCL_RET_PASS) { HSH_Deref(sp->obj); sp->obj = NULL; - PassSession(sp->wrk, sp); + PassSession(sp); sp->step = STP_PASSBODY; return (0); } @@ -398,7 +397,7 @@ HSH_Unbusy(sp->obj); HSH_Deref(sp->obj); sp->obj = 0; - PassSession(sp->wrk, sp); + PassSession(sp); sp->step = STP_PASSBODY; return (0); } @@ -431,7 +430,7 @@ cnt_pass(struct sess *sp) { - PassSession(sp->wrk, sp); + PassSession(sp); sp->step = STP_PASSBODY; return (0); } Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-21 07:18:40 UTC (rev 532) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-21 09:32:09 UTC (rev 533) @@ -77,7 +77,7 @@ } else CHECK_OBJ_NOTNULL(w->nobj, OBJECT_MAGIC); - url = h->hd[HTTP_HDR_URL][HTTP_START]; + url = h->hd[HTTP_HDR_URL].b; if (!http_GetHdr(h, H_Host, &host)) host = url; if (sp->obj != NULL) { Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-21 07:18:40 UTC (rev 532) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-21 09:32:09 UTC (rev 533) @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -19,8 +20,8 @@ #include "http_headers.h" #undef HTTPH -#define VSLH(a, b, c, d) \ - VSLR((a), (b), (c)->hd[d][HTTP_START], (c)->hd[d][HTTP_END]); +#define VSLH(ax, bx, cx, dx) \ + VSLR((ax), (bx), (cx)->hd[(dx)].b, (cx)->hd[(dx)].e); /*--------------------------------------------------------------------*/ @@ -37,11 +38,30 @@ hp->v = sp; hp->f = sp; hp->e = sp + len; + hp->nhd = HTTP_HDR_FIRST; } /*--------------------------------------------------------------------*/ + int +http_IsHdr(struct http_hdr *hh, char *hdr) +{ + unsigned l; + + assert(hh->b != NULL); + assert(hh->e != NULL); + assert(hdr != NULL); + l = hdr[0]; + assert(l == strlen(hdr + 1)); + assert(hdr[l] == ':'); + hdr++; + return (!strncasecmp(hdr, hh->b, l)); +} + +/*--------------------------------------------------------------------*/ + +int http_GetHdr(struct http *hp, const char *hdr, char **ptr) { unsigned u, l; @@ -52,27 +72,26 @@ assert(hdr[l] == ':'); hdr++; for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) { - assert(hp->hd[u][HTTP_START] != NULL); - assert(hp->hd[u][HTTP_END] != NULL); - if (hp->hd[u][HTTP_END] < hp->hd[u][HTTP_START] + l) + assert(hp->hd[u].b != NULL); + assert(hp->hd[u].e != NULL); + if (hp->hd[u].e < hp->hd[u].b + l) continue; - if (hp->hd[u][HTTP_START][l-1] != ':') + if (hp->hd[u].b[l-1] != ':') continue; - if (strncasecmp(hdr, hp->hd[u][HTTP_START], l)) + if (strncasecmp(hdr, hp->hd[u].b, l)) continue; - if (hp->hd[u][HTTP_DATA] == NULL) { - p = hp->hd[u][HTTP_START] + l; - while (isspace(*p)) - p++; - hp->hd[u][HTTP_DATA] = p; - } - *ptr = hp->hd[u][HTTP_DATA]; + p = hp->hd[u].b + l; + while (isspace(*p)) + p++; + *ptr = p; return (1); } *ptr = NULL; return (0); } +/*--------------------------------------------------------------------*/ + int http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr) { @@ -106,6 +125,8 @@ return (0); } +/*--------------------------------------------------------------------*/ + int http_HdrIs(struct http *hp, const char *hdr, const char *val) { @@ -119,6 +140,8 @@ return (0); } +/*--------------------------------------------------------------------*/ + int http_GetTail(struct http *hp, unsigned len, char **b, char **e) { @@ -140,6 +163,7 @@ return (1); } +/*--------------------------------------------------------------------*/ /* Read from fd, but soak up any tail first */ int @@ -168,12 +192,14 @@ return (u); } +/*--------------------------------------------------------------------*/ + int http_GetStatus(struct http *hp) { - assert(hp->hd[HTTP_HDR_STATUS][HTTP_START] != NULL); - return (strtoul(hp->hd[HTTP_HDR_STATUS][HTTP_START], + assert(hp->hd[HTTP_HDR_STATUS].b != NULL); + return (strtoul(hp->hd[HTTP_HDR_STATUS].b, NULL /* XXX */, 10)); } @@ -211,8 +237,8 @@ hp->conds = 1; if (hp->nhd < MAX_HTTP_HDRS) { - hp->hd[hp->nhd][HTTP_START] = p; - hp->hd[hp->nhd][HTTP_END] = q; + hp->hd[hp->nhd].b = p; + hp->hd[hp->nhd].e = q; VSLH(SLT_RxHeader, fd, hp, hp->nhd); hp->nhd++; } else { @@ -239,10 +265,10 @@ continue; /* First, the request type (GET/HEAD etc) */ - hp->hd[HTTP_HDR_REQ][HTTP_START] = p; + hp->hd[HTTP_HDR_REQ].b = p; for (; isalpha(*p); p++) ; - hp->hd[HTTP_HDR_REQ][HTTP_END] = p; + hp->hd[HTTP_HDR_REQ].e = p; VSLH(SLT_Request, fd, hp, HTTP_HDR_REQ); *p++ = '\0'; @@ -253,10 +279,10 @@ VSLR(SLT_HttpGarbage, fd, hp->s, hp->v); return (400); } - hp->hd[HTTP_HDR_URL][HTTP_START] = p; + hp->hd[HTTP_HDR_URL].b = p; while (!isspace(*p)) p++; - hp->hd[HTTP_HDR_URL][HTTP_END] = p; + hp->hd[HTTP_HDR_URL].e = p; VSLH(SLT_URL, fd, hp, HTTP_HDR_URL); if (*p == '\n') { VSLR(SLT_HttpGarbage, fd, hp->s, hp->v); @@ -271,10 +297,10 @@ VSLR(SLT_HttpGarbage, fd, hp->s, hp->v); return (400); } - hp->hd[HTTP_HDR_PROTO][HTTP_START] = p; + hp->hd[HTTP_HDR_PROTO].b = p; while (!isspace(*p)) p++; - hp->hd[HTTP_HDR_PROTO][HTTP_END] = p; + hp->hd[HTTP_HDR_PROTO].e = p; VSLH(SLT_Protocol, fd, hp, HTTP_HDR_PROTO); if (*p != '\n') *p++ = '\0'; @@ -303,34 +329,34 @@ continue; /* First, protocol */ - hp->hd[HTTP_HDR_PROTO][HTTP_START] = p; + hp->hd[HTTP_HDR_PROTO].b = p; while (!isspace(*p)) p++; - hp->hd[HTTP_HDR_PROTO][HTTP_END] = p; + hp->hd[HTTP_HDR_PROTO].e = p; VSLH(SLT_Protocol, fd, hp, HTTP_HDR_PROTO); *p++ = '\0'; /* Next find the status */ while (isspace(*p)) p++; - hp->hd[HTTP_HDR_STATUS][HTTP_START] = p; + hp->hd[HTTP_HDR_STATUS].b = p; while (!isspace(*p)) p++; - hp->hd[HTTP_HDR_STATUS][HTTP_END] = p; + hp->hd[HTTP_HDR_STATUS].e = p; VSLH(SLT_Status, fd, hp, HTTP_HDR_STATUS); *p++ = '\0'; /* Next find the response */ while (isspace(*p)) p++; - hp->hd[HTTP_HDR_RESPONSE][HTTP_START] = p; + hp->hd[HTTP_HDR_RESPONSE].b = p; while (*p != '\n') p++; - for (q = p; q > hp->hd[HTTP_HDR_RESPONSE][HTTP_START] && + for (q = p; q > hp->hd[HTTP_HDR_RESPONSE].b && isspace(q[-1]); q--) continue; *q = '\0'; - hp->hd[HTTP_HDR_RESPONSE][HTTP_END] = q; + hp->hd[HTTP_HDR_RESPONSE].e = q; VSLH(SLT_Response, fd, hp, HTTP_HDR_RESPONSE); p++; @@ -375,8 +401,6 @@ /*--------------------------------------------------------------------*/ -#include - static void http_read_f(int fd, short event, void *arg) { @@ -484,7 +508,131 @@ /*--------------------------------------------------------------------*/ +static void +http_copyh(int fd, struct http *to, struct http *fm, unsigned n, enum shmlogtag tag) +{ + + assert(n < MAX_HTTP_HDRS); + assert(fm->hd[n].b != NULL); + to->hd[n].b = fm->hd[n].b; + to->hd[n].e = fm->hd[n].e; + VSLH(tag, fd, to, n); +} + void +http_CopyReq(int fd, struct http *to, struct http *fm) +{ + + CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC); + CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); + http_copyh(fd, to, fm, HTTP_HDR_REQ, SLT_Request); + http_copyh(fd, to, fm, HTTP_HDR_URL, SLT_URL); + http_copyh(fd, to, fm, HTTP_HDR_PROTO, SLT_Protocol); +} + + +void +http_CopyResp(int fd, struct http *to, struct http *fm) +{ + + CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC); + CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); + http_copyh(fd, to, fm, HTTP_HDR_PROTO, SLT_Protocol); + http_copyh(fd, to, fm, HTTP_HDR_STATUS, SLT_Status); + http_copyh(fd, to, fm, HTTP_HDR_RESPONSE, SLT_Response); +} + +void +http_CopyHeader(int fd, struct http *to, struct http *fm, unsigned n) +{ + + CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC); + CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); + assert(n < MAX_HTTP_HDRS); + assert(fm->hd[n].b != NULL); + if (to->nhd < MAX_HTTP_HDRS) { + to->hd[to->nhd].b = fm->hd[n].b; + to->hd[to->nhd].e = fm->hd[n].e; + VSLH(SLT_TxHeader, fd, to, to->nhd); + to->nhd++; + } else { + VSL_stats->losthdr++; + VSLH(SLT_LostHeader, fd, fm, n); + } +} + +/*--------------------------------------------------------------------*/ + +void +http_FilterHeader(int fd, struct http *to, struct http *fm, unsigned how) +{ + unsigned u; + + to->nhd = HTTP_HDR_FIRST; + for (u = HTTP_HDR_FIRST; u < fm->nhd; u++) { +#define HTTPH(a, b, c, d, e, f, g) \ + if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \ + continue; +#include "http_headers.h" +#undef HTTPH + http_CopyHeader(fd, to, fm, u); + } +} + +/*--------------------------------------------------------------------*/ + +void +http_PrintfHeader(int fd, struct http *to, const char *fmt, ...) +{ + va_list ap; + unsigned l, n; + + va_start(ap, fmt); + l = to->e - to->f; + n = vsnprintf(to->f, l, fmt, ap); + if (n + 1 > l || to->nhd >= MAX_HTTP_HDRS) { + VSL_stats->losthdr++; + VSL(SLT_LostHeader, fd, "%s", to->f); + } else { + assert(to->f < to->e); + to->hd[to->nhd].b = to->f; + to->hd[to->nhd].e = to->f + n; + to->f += n + 1; + VSLH(SLT_TxHeader, fd, to, to->nhd); + to->nhd++; + } + va_end(ap); +} + +/*--------------------------------------------------------------------*/ + +void +http_Write(struct worker *w, struct http *hp, int resp) +{ + unsigned u; + + if (resp) { + assert(hp->hd[HTTP_HDR_STATUS].b != NULL); + WRK_WriteH(w, &hp->hd[HTTP_HDR_PROTO], "\r\n"); + WRK_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " "); + WRK_WriteH(w, &hp->hd[HTTP_HDR_RESPONSE], " "); + } else { + assert(hp->hd[HTTP_HDR_URL].b != NULL); + WRK_WriteH(w, &hp->hd[HTTP_HDR_REQ], " "); + WRK_WriteH(w, &hp->hd[HTTP_HDR_URL], " "); + WRK_WriteH(w, &hp->hd[HTTP_HDR_PROTO], "\r\n"); + } + for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) { + assert(hp->hd[u].b != NULL); + assert(hp->hd[u].e != NULL); + WRK_WriteH(w, &hp->hd[u], "\r\n"); + } + WRK_Write(w, "\r\n", -1); +} + +/*--------------------------------------------------------------------*/ + +void http_BuildSbuf(int fd, enum http_build mode, struct sbuf *sb, struct http *hp) { unsigned u, sup, rr; @@ -503,31 +651,29 @@ assert(__LINE__ == 0); } if (rr == 0) { - sbuf_cat(sb, hp->hd[HTTP_HDR_PROTO][HTTP_START]); + sbuf_cat(sb, hp->hd[HTTP_HDR_PROTO].b); sbuf_cat(sb, " "); - sbuf_cat(sb, hp->hd[HTTP_HDR_STATUS][HTTP_START]); + sbuf_cat(sb, hp->hd[HTTP_HDR_STATUS].b); sbuf_cat(sb, " "); - sbuf_cat(sb, hp->hd[HTTP_HDR_RESPONSE][HTTP_START]); + sbuf_cat(sb, hp->hd[HTTP_HDR_RESPONSE].b); } else { if (rr == 2) { sbuf_cat(sb, "GET "); } else { - sbuf_cat(sb, hp->hd[HTTP_HDR_REQ][HTTP_START]); + sbuf_cat(sb, hp->hd[HTTP_HDR_REQ].b); sbuf_cat(sb, " "); } - sbuf_cat(sb, hp->hd[HTTP_HDR_URL][HTTP_START]); + sbuf_cat(sb, hp->hd[HTTP_HDR_URL].b); sbuf_cat(sb, " "); - sbuf_cat(sb, hp->hd[HTTP_HDR_PROTO][HTTP_START]); + sbuf_cat(sb, hp->hd[HTTP_HDR_PROTO].b); } sbuf_cat(sb, "\r\n"); for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) { - if (http_supress(hp->hd[u][HTTP_START], sup)) + if (http_supress(hp->hd[u].b, sup)) continue; - if (1) - VSL(SLT_TxHeader, fd, "%s", hp->hd[u][HTTP_START]); - sbuf_cat(sb, hp->hd[u][HTTP_START]); + sbuf_cat(sb, hp->hd[u].b); sbuf_cat(sb, "\r\n"); } if (mode != Build_Reply) { Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-21 07:18:40 UTC (rev 532) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-21 09:32:09 UTC (rev 533) @@ -46,8 +46,9 @@ if (i == 0 && bi == NULL) return (1); assert(i > 0); - RES_Write(sp, buf, i); - RES_Flush(sp); + WRK_Write(sp->wrk, buf, i); + if (WRK_Flush(sp->wrk)) + vca_close_session(sp, "remote closed"); cl -= i; } return (0); @@ -92,7 +93,7 @@ if (u == 0) break; - RES_Write(sp, p, q - p); + WRK_Write(sp->wrk, p, q - p); p = q; @@ -104,28 +105,30 @@ } if (bp - p < j) j = bp - p; - RES_Write(sp, p, j); + WRK_Write(sp->wrk, p, j); p += j; u -= j; } while (u > 0) { if (http_GetTail(hp, u, &b, &e)) { j = e - b; - RES_Write(sp, q, j); + WRK_Write(sp->wrk, q, j); u -= j; } else break; } - RES_Flush(sp); + if (WRK_Flush(sp->wrk)) + vca_close_session(sp, "remote closed"); while (u > 0) { j = u; if (j > sizeof buf) j = sizeof buf; i = read(fd, buf, j); assert(i > 0); - RES_Write(sp, buf, i); + WRK_Write(sp->wrk, buf, i); u -= i; - RES_Flush(sp); + if (WRK_Flush(sp->wrk)) + vca_close_session(sp, "remote closed"); } } return (0); @@ -138,32 +141,34 @@ PassBody(struct worker *w, struct sess *sp) { struct vbe_conn *vc; - struct http *hp; char *b; int cls; - hp = sp->bkd_http; - assert(hp != NULL); vc = sp->vbc; assert(vc != NULL); - http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); - sbuf_cat(w->sb, "\r\n"); - sbuf_finish(w->sb); - RES_Write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); + sp->http->f = sp->http->v; + sp->http->nhd = HTTP_HDR_FIRST; + http_CopyResp(sp->fd, sp->http, vc->http); + http_FilterHeader(sp->fd, sp->http, vc->http, HTTPH_A_PASS); + http_PrintfHeader(sp->fd, sp->http, "X-Varnish: %u", sp->xid); + WRK_Reset(w, &sp->fd); + http_Write(w, sp->http, 1); - if (http_GetHdr(hp, H_Content_Length, &b)) - cls = pass_straight(sp, vc->fd, hp, b); - else if (http_HdrIs(hp, H_Connection, "close")) - cls = pass_straight(sp, vc->fd, hp, NULL); - else if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) - cls = pass_chunked(sp, vc->fd, hp); + if (http_GetHdr(vc->http, H_Content_Length, &b)) + cls = pass_straight(sp, vc->fd, vc->http, b); + else if (http_HdrIs(vc->http, H_Connection, "close")) + cls = pass_straight(sp, vc->fd, vc->http, NULL); + else if (http_HdrIs(vc->http, H_Transfer_Encoding, "chunked")) + cls = pass_chunked(sp, vc->fd, vc->http); else { - cls = pass_straight(sp, vc->fd, hp, NULL); + cls = pass_straight(sp, vc->fd, vc->http, NULL); } - RES_Flush(sp); - if (http_GetHdr(hp, H_Connection, &b) && !strcasecmp(b, "close")) + if (WRK_Flush(w)) + vca_close_session(sp, "remote closed"); + + if (http_GetHdr(vc->http, H_Connection, &b) && !strcasecmp(b, "close")) cls = 1; if (cls) @@ -172,21 +177,31 @@ VBE_RecycleFd(vc); } + /*--------------------------------------------------------------------*/ + void -PassSession(struct worker *w, struct sess *sp) +PassSession(struct sess *sp) { int i; struct vbe_conn *vc; struct http *hp; + struct worker *w; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); + w = sp->wrk; vc = VBE_GetFd(sp->backend, sp->xid); assert(vc != NULL); VSL(SLT_Backend, sp->fd, "%d %s", vc->fd, sp->backend->vcl_name); - http_BuildSbuf(vc->fd, Build_Pass, w->sb, sp->http); - i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); - assert(i == sbuf_len(w->sb)); + http_CopyReq(vc->fd, vc->http, sp->http); + http_FilterHeader(vc->fd, vc->http, sp->http, HTTPH_R_PASS); + http_PrintfHeader(vc->fd, vc->http, "X-Varnish: %u", sp->xid); + WRK_Reset(w, &vc->fd); + http_Write(w, vc->http, 0); + i = WRK_Flush(w); + assert(i == 0); /* XXX: copy any contents */ Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-21 07:18:40 UTC (rev 532) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-21 09:32:09 UTC (rev 533) @@ -24,6 +24,73 @@ static TAILQ_HEAD(, worker) wrk_head = TAILQ_HEAD_INITIALIZER(wrk_head); static TAILQ_HEAD(, workreq) wrk_reqhead = TAILQ_HEAD_INITIALIZER(wrk_reqhead); +/*-------------------------------------------------------------------- + * Write data to fd + * We try to use writev() if possible in order to minimize number of + * syscalls made and packets sent. It also just might allow the worker + * thread to complete the request without holding stuff locked. + */ + +void +WRK_Reset(struct worker *w, int *fd) +{ + + CHECK_OBJ_NOTNULL(w, WORKER_MAGIC); + w->werr = 0; + w->liov = 0; + w->niov = 0; + w->wfd = fd; +} + +int +WRK_Flush(struct worker *w) +{ + int i; + + CHECK_OBJ_NOTNULL(w, WORKER_MAGIC); + if (*w->wfd < 0 || w->niov == 0 || w->werr) + return (w->werr); +VSL(SLT_Debug, 0, "%s %d", __func__, *w->wfd); + i = writev(*w->wfd, w->iov, w->niov); + if (i != w->liov) + w->werr++; + else { + w->liov = 0; + w->niov = 0; + } + return (w->werr); +} + +void +WRK_WriteH(struct worker *w, struct http_hdr *hh, const char *suf) +{ + + CHECK_OBJ_NOTNULL(w, WORKER_MAGIC); + assert(w != NULL); + assert(hh != NULL); + assert(hh->b != NULL); + assert(hh->e != NULL); + WRK_Write(w, hh->b, hh->e - hh->b); + if (suf != NULL) + WRK_Write(w, suf, -1); +} + +void +WRK_Write(struct worker *w, const void *ptr, size_t len) +{ + + CHECK_OBJ_NOTNULL(w, WORKER_MAGIC); + if (len == 0 || *w->wfd < 0) + return; + if (len == -1) + len = strlen(ptr); + if (w->niov == MAX_IOVS) + WRK_Flush(w); + w->iov[w->niov].iov_base = (void*)(uintptr_t)ptr; + w->iov[w->niov++].iov_len = len; + w->liov += len; +} + /*--------------------------------------------------------------------*/ static void Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-21 07:18:40 UTC (rev 532) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-21 09:32:09 UTC (rev 533) @@ -65,50 +65,12 @@ " \r\n" "\r\n"); sbuf_finish(sb); - RES_Write(sp, sbuf_data(sb), sbuf_len(sb)); - RES_Flush(sp); + WRK_Write(sp->wrk, sbuf_data(sb), sbuf_len(sb)); + WRK_Flush(sp->wrk); vca_close_session(sp, msg); } -/*-------------------------------------------------------------------- - * Write data to client - * We try to use writev() if possible in order to minimize number of - * syscalls made and packets sent. It also just might allow the worker - * thread to complete the request without holding stuff locked. - */ - -void -RES_Flush(struct sess *sp) -{ - int i; - - if (sp->fd < 0 || sp->wrk->niov == 0) - return; - i = writev(sp->fd, sp->wrk->iov, sp->wrk->niov); - if (i != sp->wrk->liov) - vca_close_session(sp, "remote closed"); - sp->wrk->liov = 0; - sp->wrk->niov = 0; -} - -void -RES_Write(struct sess *sp, const void *ptr, size_t len) -{ - - if (sp->fd < 0 || len == 0) - return; - if (len == -1) - len = strlen(ptr); - if (sp->wrk->niov == MAX_IOVS) - RES_Flush(sp); - if (sp->fd < 0) - return; - sp->wrk->iov[sp->wrk->niov].iov_base = (void*)(uintptr_t)ptr; - sp->wrk->iov[sp->wrk->niov++].iov_len = len; - sp->wrk->liov += len; -} - /*--------------------------------------------------------------------*/ static void @@ -121,18 +83,18 @@ VSL(SLT_Status, sp->fd, "%u", 304); VSL(SLT_Length, sp->fd, "%u", 0); - RES_Write(sp, "HTTP/1.1 304 Not Modified\r\n", -1); - RES_Write(sp, "Via: 1.1 varnish\r\n", -1); - RES_Write(sp, "Last-Modified: ", -1); - RES_Write(sp, p, -1); - RES_Write(sp, "\r\n", -1); - if (strcmp(sp->http->hd[HTTP_HDR_PROTO][HTTP_START], "HTTP/1.1")) - RES_Write(sp, "Connection: close\r\n", -1); + WRK_Write(sp->wrk, "HTTP/1.1 304 Not Modified\r\n", -1); + WRK_Write(sp->wrk, "Via: 1.1 varnish\r\n", -1); + WRK_Write(sp->wrk, "Last-Modified: ", -1); + WRK_Write(sp->wrk, p, -1); + WRK_Write(sp->wrk, "\r\n", -1); + if (strcmp(sp->http->hd[HTTP_HDR_PROTO].b, "HTTP/1.1")) + WRK_Write(sp->wrk, "Connection: close\r\n", -1); sbuf_printf(sb, "X-Varnish: xid %u\r\n", sp->obj->xid); sbuf_printf(sb, "\r\n"); sbuf_finish(sb); - RES_Write(sp, sbuf_data(sb), sbuf_len(sb)); - RES_Flush(sp); + WRK_Write(sp->wrk, sbuf_data(sb), sbuf_len(sb)); + WRK_Flush(sp->wrk); } /*--------------------------------------------------------------------*/ @@ -180,26 +142,26 @@ VSL(SLT_Status, sp->fd, "%u", sp->obj->response); VSL(SLT_Length, sp->fd, "%u", sp->obj->len); - RES_Write(sp, sp->obj->header, strlen(sp->obj->header)); + WRK_Write(sp->wrk, sp->obj->header, strlen(sp->obj->header)); sbuf_clear(sb); sbuf_printf(sb, "Age: %u\r\n", sp->obj->age + sp->t_req - sp->obj->entered); sbuf_printf(sb, "Via: 1.1 varnish\r\n"); sbuf_printf(sb, "X-Varnish: xid %u\r\n", sp->obj->xid); - if (strcmp(sp->http->hd[HTTP_HDR_PROTO][HTTP_START], "HTTP/1.1")) + if (strcmp(sp->http->hd[HTTP_HDR_PROTO].b, "HTTP/1.1")) sbuf_printf(sb, "Connection: close\r\n"); sbuf_printf(sb, "\r\n"); sbuf_finish(sb); - RES_Write(sp, sbuf_data(sb), sbuf_len(sb)); + WRK_Write(sp->wrk, sbuf_data(sb), sbuf_len(sb)); bytes += sbuf_len(sb); /* XXX: conditional request handling */ - if (!strcmp(sp->http->hd[HTTP_HDR_REQ][HTTP_START], "GET")) { + if (!strcmp(sp->http->hd[HTTP_HDR_REQ].b, "GET")) { TAILQ_FOREACH(st, &sp->obj->store, list) { assert(st->stevedore != NULL); u += st->len; if (st->stevedore->send == NULL) { - RES_Write(sp, st->ptr, st->len); + WRK_Write(sp->wrk, st->ptr, st->len); continue; } st->stevedore->send(st, sp, @@ -210,5 +172,5 @@ assert(u == sp->obj->len); } SES_ChargeBytes(sp, bytes + u); - RES_Flush(sp); + WRK_Flush(sp->wrk); } Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-21 07:18:40 UTC (rev 532) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-21 09:32:09 UTC (rev 533) @@ -60,7 +60,7 @@ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); assert(sp != NULL); assert(sp->http != NULL); - return (sp->http->hd[HTTP_HDR_REQ][HTTP_START]); + return (sp->http->hd[HTTP_HDR_REQ].b); } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/include/http_headers.h =================================================================== --- trunk/varnish-cache/include/http_headers.h 2006-07-21 07:18:40 UTC (rev 532) +++ trunk/varnish-cache/include/http_headers.h 2006-07-21 09:32:09 UTC (rev 533) @@ -5,7 +5,7 @@ * b session field name * c Request(1)/Response(2) bitfield * d Supress header to backend (1) / Supress header to client (2) - * e unused + * e Supress header in pass from client to backend * f unused * g unused * @@ -15,24 +15,29 @@ *-------------------------------------------------------------------- */ -HTTPH("Keep-Alive", H_Keep_Alive, 3, 3, 0, 0, 0) /* RFC2068 */ +#ifndef HTTPH_R_PASS +#define HTTPH_R_PASS (1 << 0) +#define HTTPH_A_PASS (1 << 1) +#endif +HTTPH("Keep-Alive", H_Keep_Alive, 3, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2068 */ + HTTPH("Accept", H_Accept, 1, 0, 0, 0, 0) /* RFC2616 14.1 */ HTTPH("Accept-Charset", H_Accept_Charset, 1, 0, 0, 0, 0) /* RFC2616 14.2 */ HTTPH("Accept-Encoding", H_Accept_Encoding, 1, 0, 0, 0, 0) /* RFC2616 14.3 */ HTTPH("Accept-Language", H_Accept_Language, 1, 0, 0, 0, 0) /* RFC2616 14.4 */ -HTTPH("Accept-Ranges", H_Accept_Ranges, 2, 3, 0, 0, 0) /* RFC2616 14.5 */ +HTTPH("Accept-Ranges", H_Accept_Ranges, 2, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.5 */ HTTPH("Age", H_Age, 2, 0, 0, 0, 0) /* RFC2616 14.6 */ HTTPH("Allow", H_Allow, 2, 0, 0, 0, 0) /* RFC2616 14.7 */ HTTPH("Authorization", H_Authorization, 1, 0, 0, 0, 0) /* RFC2616 14.8 */ -HTTPH("Cache-Control", H_Cache_Control, 3, 3, 0, 0, 0) /* RFC2616 14.9 */ -HTTPH("Connection", H_Connection, 3, 3, 0, 0, 0) /* RFC2616 14.10 */ +HTTPH("Cache-Control", H_Cache_Control, 3, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.9 */ +HTTPH("Connection", H_Connection, 3, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.10 */ HTTPH("Content-Encoding", H_Content_Encoding, 2, 0, 0, 0, 0) /* RFC2616 14.11 */ HTTPH("Content-Langugae", H_Content_Language, 2, 0, 0, 0, 0) /* RFC2616 14.12 */ HTTPH("Content-Length", H_Content_Length, 2, 2, 0, 0, 0) /* RFC2616 14.13 */ HTTPH("Content-Location", H_Content_Location, 2, 0, 0, 0, 0) /* RFC2616 14.14 */ HTTPH("Content-MD5", H_Content_MD5, 2, 0, 0, 0, 0) /* RFC2616 14.15 */ -HTTPH("Content-Range", H_Content_Range, 2, 3, 0, 0, 0) /* RFC2616 14.16 */ +HTTPH("Content-Range", H_Content_Range, 2, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.16 */ HTTPH("Content-Type", H_Content_Type, 2, 0, 0, 0, 0) /* RFC2616 14.17 */ HTTPH("Date", H_Date, 2, 0, 0, 0, 0) /* RFC2616 14.18 */ HTTPH("ETag", H_ETag, 2, 0, 0, 0, 0) /* RFC2616 14.19 */ @@ -55,10 +60,10 @@ HTTPH("Referer", H_Referer, 1, 0, 0, 0, 0) /* RFC2616 14.36 */ HTTPH("Retry-After", H_Retry_After, 2, 0, 0, 0, 0) /* RFC2616 14.37 */ HTTPH("Server", H_Server, 2, 0, 0, 0, 0) /* RFC2616 14.38 */ -HTTPH("TE", H_TE, 1, 3, 0, 0, 0) /* RFC2616 14.39 */ -HTTPH("Trailer", H_Trailer, 1, 3, 0, 0, 0) /* RFC2616 14.40 */ -HTTPH("Transfer-Encoding", H_Transfer_Encoding, 2, 3, 0, 0, 0) /* RFC2616 14.41 */ -HTTPH("Upgrade", H_Upgrade, 2, 3, 0, 0, 0) /* RFC2616 14.42 */ +HTTPH("TE", H_TE, 1, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.39 */ +HTTPH("Trailer", H_Trailer, 1, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.40 */ +HTTPH("Transfer-Encoding", H_Transfer_Encoding, 2, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.41 */ +HTTPH("Upgrade", H_Upgrade, 2, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.42 */ HTTPH("User-Agent", H_User_Agent, 1, 0, 0, 0, 0) /* RFC2616 14.43 */ HTTPH("Vary", H_Vary, 2, 0, 0, 0, 0) /* RFC2616 14.44 */ HTTPH("Via", H_Via, 2, 0, 0, 0, 0) /* RFC2616 14.45 */ From phk at projects.linpro.no Fri Jul 21 10:44:12 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 12:44:12 +0200 (CEST) Subject: r534 - trunk/varnish-cache/bin/varnishd Message-ID: <20060721104412.3947B1EC38D@projects.linpro.no> Author: phk Date: 2006-07-21 12:44:12 +0200 (Fri, 21 Jul 2006) New Revision: 534 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Make pipe use the new http manipulation. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-21 09:32:09 UTC (rev 533) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-21 10:44:12 UTC (rev 534) @@ -333,7 +333,7 @@ void PassBody(struct worker *w, struct sess *sp); /* cache_pipe.c */ -void PipeSession(struct worker *w, struct sess *sp); +void PipeSession(struct sess *sp); /* cache_pool.c */ void WRK_Init(void); Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-21 09:32:09 UTC (rev 533) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-21 10:44:12 UTC (rev 534) @@ -477,7 +477,7 @@ cnt_pipe(struct sess *sp) { - PipeSession(sp->wrk, sp); + PipeSession(sp); sp->step = STP_DONE; return (0); } Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-21 09:32:09 UTC (rev 533) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-21 10:44:12 UTC (rev 534) @@ -191,6 +191,7 @@ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); w = sp->wrk; + vc = VBE_GetFd(sp->backend, sp->xid); assert(vc != NULL); VSL(SLT_Backend, sp->fd, "%d %s", vc->fd, sp->backend->vcl_name); @@ -214,6 +215,5 @@ (void)event_base_loop(w->eb, 0); http_DissectResponse(hp, vc->fd); - sp->bkd_http = hp; sp->vbc = vc; } Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-21 09:32:09 UTC (rev 533) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-07-21 10:44:12 UTC (rev 534) @@ -45,27 +45,34 @@ } void -PipeSession(struct worker *w, struct sess *sp) +PipeSession(struct sess *sp) { - int i; struct vbe_conn *vc; struct edir e1, e2; char *b, *e; + struct worker *w; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); + w = sp->wrk; + vc = VBE_GetFd(sp->backend, sp->xid); assert(vc != NULL); VSL(SLT_Backend, sp->fd, "%d %s", vc->fd, sp->backend->vcl_name); - http_BuildSbuf(vc->fd, Build_Pipe, w->sb, sp->http); - i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); - assert(i == sbuf_len(w->sb)); - if (http_GetTail(sp->http, 0, &b, &e) && b != e) { - i = write(vc->fd, b, e - b); - if (i != e - b) { - close (vc->fd); - vca_close_session(sp, "pipe"); - VBE_ClosedFd(vc); - } + http_CopyReq(vc->fd, vc->http, sp->http); + http_FilterHeader(vc->fd, vc->http, sp->http, HTTPH_R_PIPE); + http_PrintfHeader(vc->fd, vc->http, "X-Varnish: %u", sp->xid); + WRK_Reset(w, &vc->fd); + http_Write(w, vc->http, 0); + + if (http_GetTail(sp->http, 0, &b, &e) && b != e) + WRK_Write(w, b, e - b); + + if (WRK_Flush(w)) { + vca_close_session(sp, "pipe"); + VBE_ClosedFd(vc); + return; } e1.fd = vc->fd; Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-21 09:32:09 UTC (rev 533) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-21 10:44:12 UTC (rev 534) @@ -50,7 +50,6 @@ CHECK_OBJ_NOTNULL(w, WORKER_MAGIC); if (*w->wfd < 0 || w->niov == 0 || w->werr) return (w->werr); -VSL(SLT_Debug, 0, "%s %d", __func__, *w->wfd); i = writev(*w->wfd, w->iov, w->niov); if (i != w->liov) w->werr++; From phk at projects.linpro.no Fri Jul 21 11:55:18 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 13:55:18 +0200 (CEST) Subject: r535 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060721115518.404331EC38E@projects.linpro.no> Author: phk Date: 2006-07-21 13:55:17 +0200 (Fri, 21 Jul 2006) New Revision: 535 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_response.c trunk/varnish-cache/include/http_headers.h trunk/varnish-cache/include/stat_field.h Log: Convert fetch, insert and deliver to use new HTTP header munging code. Remove sbuf from workerthread, it is only used in the Error handling now and it will probably not even survive that in the long run. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-21 10:44:12 UTC (rev 534) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-21 11:55:17 UTC (rev 535) @@ -70,7 +70,6 @@ struct http_hdr hd[MAX_HTTP_HDRS]; unsigned nhd; - unsigned char hdf[MAX_HTTP_HDRS]; }; /*--------------------------------------------------------------------*/ @@ -79,7 +78,6 @@ unsigned magic; #define WORKER_MAGIC 0x6391adcf struct event_base *eb; - struct sbuf *sb; struct objhead *nobjhead; struct object *nobj; @@ -167,7 +165,7 @@ time_t last_modified; - char *header; + struct http http; TAILQ_ENTRY(object) list; TAILQ_ENTRY(object) deathrow; @@ -215,6 +213,7 @@ struct srcaddr *srcaddr; /* HTTP request */ + const char *doclose; struct http *http; time_t t_req; @@ -226,7 +225,6 @@ TAILQ_ENTRY(sess) list; struct vbe_conn *vbc; - struct http *bkd_http; struct backend *backend; struct object *obj; struct VCL_conf *vcl; @@ -288,8 +286,8 @@ void EXP_TTLchange(struct object *o); /* cache_fetch.c */ -int FetchBody(struct worker *w, struct sess *sp); -int FetchHeaders(struct worker *w, struct sess *sp); +int FetchBody(struct sess *sp); +int FetchHeaders(struct sess *sp); /* cache_hash.c */ struct object *HSH_Lookup(struct sess *sp); @@ -300,6 +298,7 @@ /* cache_http.c */ void HTTP_Init(void); +void http_CopyHttp(struct http *to, struct http *fm); void http_Write(struct worker *w, struct http *hp, int resp); void http_CopyReq(int fd, struct http *to, struct http *fm); void http_CopyResp(int fd, struct http *to, struct http *fm); Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-21 10:44:12 UTC (rev 534) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-21 11:55:17 UTC (rev 535) @@ -82,17 +82,10 @@ static int cnt_done(struct sess *sp) { - char *b; assert(sp->obj == NULL); - if (sp->fd < 0) { - /* Allready closed */ - } else if (http_GetHdr(sp->http, H_Connection, &b) && - !strcmp(b, "close")) { - vca_close_session(sp, "Connection header"); - } else if (strcmp(sp->http->hd[HTTP_HDR_PROTO].b, "HTTP/1.1")) { - vca_close_session(sp, "not HTTP/1.1"); - } + if (sp->fd >= 0 && sp->doclose != NULL) + vca_close_session(sp, sp->doclose); VCL_Rel(sp->vcl); sp->vcl = NULL; @@ -169,7 +162,7 @@ cnt_fetch(struct sess *sp) { - RFC2616_cache_policy(sp, sp->bkd_http); + RFC2616_cache_policy(sp, sp->vbc->http); VCL_fetch_method(sp); @@ -194,7 +187,7 @@ } if (sp->handling == VCL_RET_INSERT) { sp->obj->cacheable = 1; - FetchBody(sp->wrk, sp); + FetchBody(sp); HSH_Ref(sp->obj); /* get another, STP_DELIVER will deref */ HSH_Unbusy(sp->obj); sp->step = STP_DELIVER; @@ -404,7 +397,7 @@ if (sp->handling == VCL_RET_LOOKUP) INCOMPL(); if (sp->handling == VCL_RET_FETCH) { - FetchHeaders(sp->wrk, sp); + FetchHeaders(sp); sp->step = STP_FETCH; return (0); } @@ -513,6 +506,7 @@ cnt_recv(struct sess *sp) { int done; + char *b; sp->t0 = time(NULL); sp->vcl = VCL_Get(); @@ -526,6 +520,11 @@ return (0); } + if (http_GetHdr(sp->http, H_Connection, &b) && !strcmp(b, "close")) + sp->doclose = "Connection:"; + else if (strcmp(sp->http->hd[HTTP_HDR_PROTO].b, "HTTP/1.1")) + sp->doclose = "not HTTP/1.1"; + sp->backend = sp->vcl->backend[0]; /* XXX: Handle TRACE & OPTIONS of Max-Forwards = 0 */ Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-21 10:44:12 UTC (rev 534) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-21 11:55:17 UTC (rev 535) @@ -205,35 +205,43 @@ /*--------------------------------------------------------------------*/ int -FetchBody(struct worker *w, struct sess *sp) +FetchBody(struct sess *sp) { int cls; struct vbe_conn *vc; - struct http *hp; + struct worker *w; char *b; int body = 1; /* XXX */ + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); + assert(sp->obj->busy != 0); + w = sp->wrk; + vc = sp->vbc; - hp = sp->bkd_http; - if (http_GetHdr(hp, H_Last_Modified, &b)) + if (http_GetHdr(vc->http, H_Last_Modified, &b)) sp->obj->last_modified = TIM_parse(b); - http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp); + + sp->http->f = sp->http->v; + sp->http->nhd = HTTP_HDR_FIRST; + http_CopyResp(sp->fd, sp->http, vc->http); + http_FilterHeader(sp->fd, sp->http, vc->http, HTTPH_A_INS); + if (body) { - if (http_GetHdr(hp, H_Content_Length, &b)) - cls = fetch_straight(sp, vc->fd, hp, b); - else if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) - cls = fetch_chunked(sp, vc->fd, hp); + if (http_GetHdr(vc->http, H_Content_Length, &b)) + cls = fetch_straight(sp, vc->fd, vc->http, b); + else if (http_HdrIs(vc->http, H_Transfer_Encoding, "chunked")) + cls = fetch_chunked(sp, vc->fd, vc->http); else - cls = fetch_eof(sp, vc->fd, hp); - sbuf_printf(w->sb, "Content-Length: %u\r\n", sp->obj->len); + cls = fetch_eof(sp, vc->fd, vc->http); + http_PrintfHeader(sp->fd, sp->http, "Content-Length: %u", sp->obj->len); } else cls = 0; - sbuf_finish(w->sb); - sp->obj->header = strdup(sbuf_data(w->sb)); - VSL_stats->n_header++; + http_CopyHttp(&sp->obj->http, sp->http); - if (http_GetHdr(hp, H_Connection, &b) && !strcasecmp(b, "close")) + if (http_GetHdr(vc->http, H_Connection, &b) && !strcasecmp(b, "close")) cls = 1; if (cls) @@ -247,12 +255,18 @@ /*--------------------------------------------------------------------*/ int -FetchHeaders(struct worker *w, struct sess *sp) +FetchHeaders(struct sess *sp) { int i; struct vbe_conn *vc; - struct http *hp; + struct worker *w; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); + assert(sp->obj->busy != 0); + w = sp->wrk; + sp->obj->xid = sp->xid; vc = VBE_GetFd(sp->backend, sp->xid); @@ -261,23 +275,24 @@ assert(vc != NULL); /* XXX: handle this */ VSL(SLT_Backend, sp->fd, "%d %s", vc->fd, sp->backend->vcl_name); - hp = vc->http; - http_BuildSbuf(vc->fd, Build_Fetch, w->sb, sp->http); - i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb)); - assert(i == sbuf_len(w->sb)); + http_CopyReq(vc->fd, vc->http, sp->http); + http_FilterHeader(vc->fd, vc->http, sp->http, HTTPH_R_FETCH); + http_PrintfHeader(vc->fd, vc->http, "X-Varnish: %u", sp->xid); + sp->t_req = time(NULL); + WRK_Reset(w, &vc->fd); + http_Write(w, vc->http, 0); + i = WRK_Flush(w); + assert(i == 0); - /* XXX: copy any body ?? */ - /* * XXX: It might be cheaper to avoid the event_engine and simply * XXX: read(2) the header */ - http_RecvHead(hp, vc->fd, w->eb, NULL, NULL); + http_RecvHead(vc->http, vc->fd, w->eb, NULL, NULL); (void)event_base_loop(w->eb, 0); sp->t_resp = time(NULL); - assert(http_DissectResponse(hp, vc->fd) == 0); + assert(http_DissectResponse(vc->http, vc->fd) == 0); sp->vbc = vc; - sp->bkd_http = hp; return (0); } Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-21 10:44:12 UTC (rev 534) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-21 11:55:17 UTC (rev 535) @@ -69,6 +69,7 @@ w->nobj = calloc(sizeof *w->nobj, 1); assert(w->nobj != NULL); w->nobj->magic = OBJECT_MAGIC; + w->nobj->http.magic = HTTP_MAGIC; w->nobj->busy = 1; w->nobj->refcnt = 1; TAILQ_INIT(&w->nobj->store); @@ -187,9 +188,8 @@ if (r != 0) return; - if (o->header != NULL) { - free(o->header); - VSL_stats->n_header--; + if (o->http.s != NULL) { + free(o->http.s); } TAILQ_FOREACH_SAFE(st, &o->store, list, stn) { Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-21 10:44:12 UTC (rev 534) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-21 11:55:17 UTC (rev 535) @@ -483,27 +483,36 @@ /*--------------------------------------------------------------------*/ -static int -http_supress(const char *hdr, int flag) +void +http_CopyHttp(struct http *to, struct http *fm) { + unsigned u, l; -#define HTTPH_0(a,d) -#define HTTPH_1(a,d) \ - if ((flag & d) && !strncasecmp(hdr, a, strlen(a))) { \ - return (1); \ + l = 0; + for (u = 0; u < fm->nhd; u++) { + if (fm->hd[u].b == NULL) + continue; + assert(fm->hd[u].e != NULL); + l += (fm->hd[u].e - fm->hd[u].b) + 1; } -#define HTTPH_2(a,d) HTTPH_1(a,d) -#define HTTPH_3(a,d) HTTPH_1(a,d) - -#define HTTPH(a,b,c,d,e,f,g) HTTPH_ ## d(a ":",d) -#include "http_headers.h" -#undef HTTPH -#undef HTTPH_0 -#undef HTTPH_1 -#undef HTTPH_2 -#undef HTTPH_3 - - return (0); + to->s = malloc(l); + assert(to->s != NULL); + to->e = to->s + l; + to->f = to->s; + for (u = 0; u < fm->nhd; u++) { + if (fm->hd[u].b == NULL) + continue; + assert(fm->hd[u].e != NULL); + assert(*fm->hd[u].e == '\0'); + l = fm->hd[u].e - fm->hd[u].b; + assert(l == strlen(fm->hd[u].b)); + memcpy(to->f, fm->hd[u].b, l); + to->hd[u].b = to->f; + to->hd[u].e = to->f + l; + *to->hd[u].e = '\0'; + to->f += l + 1; + } + to->nhd = fm->nhd; } /*--------------------------------------------------------------------*/ @@ -633,56 +642,6 @@ /*--------------------------------------------------------------------*/ void -http_BuildSbuf(int fd, enum http_build mode, struct sbuf *sb, struct http *hp) -{ - unsigned u, sup, rr; - - sbuf_clear(sb); - assert(sb != NULL); - switch (mode) { - case Build_Reply: rr = 0; sup = 2; break; - case Build_Pipe: rr = 1; sup = 0; break; - case Build_Pass: rr = 1; sup = 2; break; - case Build_Fetch: rr = 2; sup = 1; break; - default: - sup = 0; /* for flexelint */ - rr = 0; /* for flexelint */ - printf("mode = %d\n", mode); - assert(__LINE__ == 0); - } - if (rr == 0) { - sbuf_cat(sb, hp->hd[HTTP_HDR_PROTO].b); - sbuf_cat(sb, " "); - sbuf_cat(sb, hp->hd[HTTP_HDR_STATUS].b); - sbuf_cat(sb, " "); - sbuf_cat(sb, hp->hd[HTTP_HDR_RESPONSE].b); - } else { - if (rr == 2) { - sbuf_cat(sb, "GET "); - } else { - sbuf_cat(sb, hp->hd[HTTP_HDR_REQ].b); - sbuf_cat(sb, " "); - } - sbuf_cat(sb, hp->hd[HTTP_HDR_URL].b); - sbuf_cat(sb, " "); - sbuf_cat(sb, hp->hd[HTTP_HDR_PROTO].b); - } - - sbuf_cat(sb, "\r\n"); - - for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) { - if (http_supress(hp->hd[u].b, sup)) - continue; - sbuf_cat(sb, hp->hd[u].b); - sbuf_cat(sb, "\r\n"); - } - if (mode != Build_Reply) { - sbuf_cat(sb, "\r\n"); - sbuf_finish(sb); - } -} - -void HTTP_Init(void) { #define HTTPH(a, b, c, d, e, f, g) b[0] = strlen(b + 1); Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-21 10:44:12 UTC (rev 534) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-21 11:55:17 UTC (rev 535) @@ -185,7 +185,6 @@ { int i; struct vbe_conn *vc; - struct http *hp; struct worker *w; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); @@ -210,10 +209,9 @@ * XXX: It might be cheaper to avoid the event_engine and simply * XXX: read(2) the header */ - hp = vc->http; - http_RecvHead(hp, vc->fd, w->eb, NULL, NULL); + http_RecvHead(vc->http, vc->fd, w->eb, NULL, NULL); (void)event_base_loop(w->eb, 0); - http_DissectResponse(hp, vc->fd); + http_DissectResponse(vc->http, vc->fd); sp->vbc = vc; } Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-21 10:44:12 UTC (rev 534) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-21 11:55:17 UTC (rev 535) @@ -135,9 +135,6 @@ w->eb = event_init(); assert(w->eb != NULL); - w->sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); - assert(w->sb != NULL); - AZ(pthread_mutex_lock(&wrk_mtx)); w->nbr = VSL_stats->n_wrk; if (priv == NULL) { @@ -168,7 +165,6 @@ TAILQ_REMOVE(&wrk_head, w, list); AZ(pthread_mutex_unlock(&wrk_mtx)); VSL(SLT_WorkThread, 0, "%u suicide", w->nbr); - sbuf_delete(w->sb); event_base_free(w->eb); AZ(pthread_cond_destroy(&w->cv)); return (NULL); Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-21 10:44:12 UTC (rev 534) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-21 11:55:17 UTC (rev 535) @@ -20,7 +20,8 @@ char buf[40]; struct sbuf *sb; - sb = sp->wrk->sb; + sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); + assert(sb != NULL); if (msg == NULL) { switch (error) { @@ -68,6 +69,7 @@ WRK_Write(sp->wrk, sbuf_data(sb), sbuf_len(sb)); WRK_Flush(sp->wrk); vca_close_session(sp, msg); + sbuf_delete(sb); } @@ -76,25 +78,31 @@ static void res_do_304(struct sess *sp, char *p) { - struct sbuf *sb; - sb = sp->wrk->sb; - sbuf_clear(sb); - VSL(SLT_Status, sp->fd, "%u", 304); VSL(SLT_Length, sp->fd, "%u", 0); - WRK_Write(sp->wrk, "HTTP/1.1 304 Not Modified\r\n", -1); - WRK_Write(sp->wrk, "Via: 1.1 varnish\r\n", -1); - WRK_Write(sp->wrk, "Last-Modified: ", -1); - WRK_Write(sp->wrk, p, -1); - WRK_Write(sp->wrk, "\r\n", -1); - if (strcmp(sp->http->hd[HTTP_HDR_PROTO].b, "HTTP/1.1")) - WRK_Write(sp->wrk, "Connection: close\r\n", -1); - sbuf_printf(sb, "X-Varnish: xid %u\r\n", sp->obj->xid); - sbuf_printf(sb, "\r\n"); - sbuf_finish(sb); - WRK_Write(sp->wrk, sbuf_data(sb), sbuf_len(sb)); - WRK_Flush(sp->wrk); + + sp->http->f = sp->http->v; + + sp->http->nhd = HTTP_HDR_PROTO; + http_PrintfHeader(sp->fd, sp->http, "HTTP/1.1"); + + sp->http->nhd = HTTP_HDR_STATUS; + http_PrintfHeader(sp->fd, sp->http, "304"); + + sp->http->nhd = HTTP_HDR_RESPONSE; + http_PrintfHeader(sp->fd, sp->http, "Not Modified"); + + sp->http->nhd = HTTP_HDR_FIRST; + http_PrintfHeader(sp->fd, sp->http, "X-Varnish: %u", sp->xid); + http_PrintfHeader(sp->fd, sp->http, "Via: 1.1 varnish"); + http_PrintfHeader(sp->fd, sp->http, "Last-Modified: %s", p); + if (sp->doclose != NULL) + http_PrintfHeader(sp->fd, sp->http, "Connection: close"); + WRK_Reset(sp->wrk, &sp->fd); + http_Write(sp->wrk, sp->http, 1); + if (WRK_Flush(sp->wrk)) + vca_close_session(sp, "remote closed"); } /*--------------------------------------------------------------------*/ @@ -129,32 +137,34 @@ RES_WriteObj(struct sess *sp) { struct storage *st; - struct sbuf *sb; unsigned u = 0; uint64_t bytes = 0; - - sb = sp->wrk->sb; - if (sp->obj->response == 200 && sp->http->conds && res_do_conds(sp)) return; VSL(SLT_Status, sp->fd, "%u", sp->obj->response); VSL(SLT_Length, sp->fd, "%u", sp->obj->len); - WRK_Write(sp->wrk, sp->obj->header, strlen(sp->obj->header)); - - sbuf_clear(sb); - sbuf_printf(sb, "Age: %u\r\n", - sp->obj->age + sp->t_req - sp->obj->entered); - sbuf_printf(sb, "Via: 1.1 varnish\r\n"); - sbuf_printf(sb, "X-Varnish: xid %u\r\n", sp->obj->xid); - if (strcmp(sp->http->hd[HTTP_HDR_PROTO].b, "HTTP/1.1")) - sbuf_printf(sb, "Connection: close\r\n"); - sbuf_printf(sb, "\r\n"); - sbuf_finish(sb); - WRK_Write(sp->wrk, sbuf_data(sb), sbuf_len(sb)); + sp->http->f = sp->http->v; + sp->http->nhd = HTTP_HDR_FIRST; + http_CopyResp(sp->fd, sp->http, &sp->obj->http); + http_FilterHeader(sp->fd, sp->http, &sp->obj->http, HTTPH_A_DELIVER); + if (sp->xid != sp->obj->xid) + http_PrintfHeader(sp->fd, sp->http, "X-Varnish: %u %u", sp->xid, sp->obj->xid); + else + http_PrintfHeader(sp->fd, sp->http, "X-Varnish: %u", sp->xid); + http_PrintfHeader(sp->fd, sp->http, "Age: %u", + sp->obj->age + sp->t_req - sp->obj->entered); + http_PrintfHeader(sp->fd, sp->http, "Via: 1.1 varnish"); + if (sp->doclose != NULL) + http_PrintfHeader(sp->fd, sp->http, "Connection: close"); + WRK_Reset(sp->wrk, &sp->fd); + http_Write(sp->wrk, sp->http, 1); + +#if 0 /* XXX */ bytes += sbuf_len(sb); +#endif /* XXX: conditional request handling */ if (!strcmp(sp->http->hd[HTTP_HDR_REQ].b, "GET")) { TAILQ_FOREACH(st, &sp->obj->store, list) { @@ -172,5 +182,6 @@ assert(u == sp->obj->len); } SES_ChargeBytes(sp, bytes + u); - WRK_Flush(sp->wrk); + if (WRK_Flush(sp->wrk)) + vca_close_session(sp, "remote closed"); } Modified: trunk/varnish-cache/include/http_headers.h =================================================================== --- trunk/varnish-cache/include/http_headers.h 2006-07-21 10:44:12 UTC (rev 534) +++ trunk/varnish-cache/include/http_headers.h 2006-07-21 11:55:17 UTC (rev 535) @@ -16,28 +16,32 @@ */ #ifndef HTTPH_R_PASS -#define HTTPH_R_PASS (1 << 0) -#define HTTPH_A_PASS (1 << 1) +#define HTTPH_R_PASS (1 << 0) /* Request (c->b) in pass mode */ +#define HTTPH_A_PASS (1 << 1) /* Response (b->c)in pass mode */ +#define HTTPH_R_PIPE (1 << 2) /* Request (c->b) in pipe mode */ +#define HTTPH_R_FETCH (1 << 3) /* Request (c->b) for fetch */ +#define HTTPH_A_INS (1 << 4) /* Response (b->o) for insert */ +#define HTTPH_A_DELIVER (1 << 5) /* Response (o->c) for deliver */ #endif -HTTPH("Keep-Alive", H_Keep_Alive, 3, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2068 */ +HTTPH("Keep-Alive", H_Keep_Alive, 3, 3, HTTPH_R_PASS|HTTPH_A_PASS|HTTPH_R_FETCH, 0, 0) /* RFC2068 */ HTTPH("Accept", H_Accept, 1, 0, 0, 0, 0) /* RFC2616 14.1 */ HTTPH("Accept-Charset", H_Accept_Charset, 1, 0, 0, 0, 0) /* RFC2616 14.2 */ HTTPH("Accept-Encoding", H_Accept_Encoding, 1, 0, 0, 0, 0) /* RFC2616 14.3 */ HTTPH("Accept-Language", H_Accept_Language, 1, 0, 0, 0, 0) /* RFC2616 14.4 */ -HTTPH("Accept-Ranges", H_Accept_Ranges, 2, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.5 */ +HTTPH("Accept-Ranges", H_Accept_Ranges, 2, 3, HTTPH_R_PASS|HTTPH_A_PASS|HTTPH_R_FETCH|HTTPH_A_INS, 0, 0) /* RFC2616 14.5 */ HTTPH("Age", H_Age, 2, 0, 0, 0, 0) /* RFC2616 14.6 */ HTTPH("Allow", H_Allow, 2, 0, 0, 0, 0) /* RFC2616 14.7 */ HTTPH("Authorization", H_Authorization, 1, 0, 0, 0, 0) /* RFC2616 14.8 */ -HTTPH("Cache-Control", H_Cache_Control, 3, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.9 */ -HTTPH("Connection", H_Connection, 3, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.10 */ +HTTPH("Cache-Control", H_Cache_Control, 3, 3, HTTPH_R_PASS|HTTPH_A_PASS|HTTPH_R_FETCH|HTTPH_A_INS, 0, 0) /* RFC2616 14.9 */ +HTTPH("Connection", H_Connection, 3, 3, HTTPH_R_PASS|HTTPH_A_PASS|HTTPH_R_FETCH|HTTPH_A_INS, 0, 0) /* RFC2616 14.10 */ HTTPH("Content-Encoding", H_Content_Encoding, 2, 0, 0, 0, 0) /* RFC2616 14.11 */ HTTPH("Content-Langugae", H_Content_Language, 2, 0, 0, 0, 0) /* RFC2616 14.12 */ -HTTPH("Content-Length", H_Content_Length, 2, 2, 0, 0, 0) /* RFC2616 14.13 */ +HTTPH("Content-Length", H_Content_Length, 2, 2, HTTPH_A_INS, 0, 0) /* RFC2616 14.13 */ HTTPH("Content-Location", H_Content_Location, 2, 0, 0, 0, 0) /* RFC2616 14.14 */ HTTPH("Content-MD5", H_Content_MD5, 2, 0, 0, 0, 0) /* RFC2616 14.15 */ -HTTPH("Content-Range", H_Content_Range, 2, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.16 */ +HTTPH("Content-Range", H_Content_Range, 2, 3, HTTPH_R_PASS|HTTPH_A_PASS|HTTPH_R_FETCH|HTTPH_A_INS, 0, 0) /* RFC2616 14.16 */ HTTPH("Content-Type", H_Content_Type, 2, 0, 0, 0, 0) /* RFC2616 14.17 */ HTTPH("Date", H_Date, 2, 0, 0, 0, 0) /* RFC2616 14.18 */ HTTPH("ETag", H_ETag, 2, 0, 0, 0, 0) /* RFC2616 14.19 */ @@ -45,25 +49,25 @@ HTTPH("Expires", H_Expires, 2, 0, 0, 0, 0) /* RFC2616 14.21 */ HTTPH("From", H_From, 1, 0, 0, 0, 0) /* RFC2616 14.22 */ HTTPH("Host", H_Host, 1, 0, 0, 0, 0) /* RFC2616 14.23 */ -HTTPH("If-Match", H_If_Match, 1, 1, 0, 0, 0) /* RFC2616 14.24 */ -HTTPH("If-Modified-Since", H_If_Modified_Since, 1, 1, 0, 0, 0) /* RFC2616 14.25 */ -HTTPH("If-None-Match", H_If_None_Match, 1, 1, 0, 0, 0) /* RFC2616 14.26 */ -HTTPH("If-Range", H_If_Range, 1, 1, 0, 0, 0) /* RFC2616 14.27 */ -HTTPH("If-Unmodified-Since", H_If_Unmodifed_Since, 1, 1, 0, 0, 0) /* RFC2616 14.28 */ +HTTPH("If-Match", H_If_Match, 1, 1, HTTPH_R_FETCH, 0, 0) /* RFC2616 14.24 */ +HTTPH("If-Modified-Since", H_If_Modified_Since, 1, 1, HTTPH_R_FETCH, 0, 0) /* RFC2616 14.25 */ +HTTPH("If-None-Match", H_If_None_Match, 1, 1, HTTPH_R_FETCH, 0, 0) /* RFC2616 14.26 */ +HTTPH("If-Range", H_If_Range, 1, 1, HTTPH_R_FETCH, 0, 0) /* RFC2616 14.27 */ +HTTPH("If-Unmodified-Since", H_If_Unmodifed_Since, 1, 1, HTTPH_R_FETCH, 0, 0) /* RFC2616 14.28 */ HTTPH("Last-Modified", H_Last_Modified, 2, 0, 0, 0, 0) /* RFC2616 14.29 */ HTTPH("Location", H_Location, 2, 0, 0, 0, 0) /* RFC2616 14.30 */ HTTPH("Max-Forwards", H_Max_Forwards, 1, 0, 0, 0, 0) /* RFC2616 14.31 */ HTTPH("Pragma", H_Pragma, 1, 0, 0, 0, 0) /* RFC2616 14.32 */ -HTTPH("Proxy-Authenticate", H_Proxy_Authenticate, 2, 3, 0, 0, 0) /* RFC2616 14.33 */ -HTTPH("Proxy-Authorization", H_Proxy_Authorization, 1, 3, 0, 0, 0) /* RFC2616 14.34 */ +HTTPH("Proxy-Authenticate", H_Proxy_Authenticate, 2, 3, HTTPH_R_FETCH|HTTPH_A_INS, 0, 0) /* RFC2616 14.33 */ +HTTPH("Proxy-Authorization", H_Proxy_Authorization, 1, 3, HTTPH_R_FETCH|HTTPH_A_INS, 0, 0) /* RFC2616 14.34 */ HTTPH("Range", H_Range, 1, 0, 0, 0, 0) /* RFC2616 14.35 */ HTTPH("Referer", H_Referer, 1, 0, 0, 0, 0) /* RFC2616 14.36 */ HTTPH("Retry-After", H_Retry_After, 2, 0, 0, 0, 0) /* RFC2616 14.37 */ HTTPH("Server", H_Server, 2, 0, 0, 0, 0) /* RFC2616 14.38 */ -HTTPH("TE", H_TE, 1, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.39 */ -HTTPH("Trailer", H_Trailer, 1, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.40 */ -HTTPH("Transfer-Encoding", H_Transfer_Encoding, 2, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.41 */ -HTTPH("Upgrade", H_Upgrade, 2, 3, HTTPH_R_PASS|HTTPH_A_PASS, 0, 0) /* RFC2616 14.42 */ +HTTPH("TE", H_TE, 1, 3, HTTPH_R_PASS|HTTPH_A_PASS|HTTPH_R_FETCH|HTTPH_A_INS, 0, 0) /* RFC2616 14.39 */ +HTTPH("Trailer", H_Trailer, 1, 3, HTTPH_R_PASS|HTTPH_A_PASS|HTTPH_R_FETCH|HTTPH_A_INS, 0, 0) /* RFC2616 14.40 */ +HTTPH("Transfer-Encoding", H_Transfer_Encoding, 2, 3, HTTPH_R_PASS|HTTPH_A_PASS|HTTPH_R_FETCH|HTTPH_A_INS, 0, 0) /* RFC2616 14.41 */ +HTTPH("Upgrade", H_Upgrade, 2, 3, HTTPH_R_PASS|HTTPH_A_PASS|HTTPH_R_FETCH|HTTPH_A_INS, 0, 0) /* RFC2616 14.42 */ HTTPH("User-Agent", H_User_Agent, 1, 0, 0, 0, 0) /* RFC2616 14.43 */ HTTPH("Vary", H_Vary, 2, 0, 0, 0, 0) /* RFC2616 14.44 */ HTTPH("Via", H_Via, 2, 0, 0, 0, 0) /* RFC2616 14.45 */ Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2006-07-21 10:44:12 UTC (rev 534) +++ trunk/varnish-cache/include/stat_field.h 2006-07-21 11:55:17 UTC (rev 535) @@ -15,7 +15,6 @@ MAC_STAT(n_sess, uint64_t, "u", "N struct sess") MAC_STAT(n_object, uint64_t, "u", "N struct object") MAC_STAT(n_objecthead, uint64_t, "u", "N struct objecthead") -MAC_STAT(n_header, uint64_t, "u", "N struct header") MAC_STAT(n_smf, uint64_t, "u", "N struct smf") MAC_STAT(n_vbe, uint64_t, "u", "N struct vbe") MAC_STAT(n_vbe_conn, uint64_t, "u", "N struct vbe_conn") From phk at projects.linpro.no Fri Jul 21 12:06:36 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 14:06:36 +0200 (CEST) Subject: r536 - trunk/varnish-cache/bin/varnishd Message-ID: <20060721120636.F302C1EC38D@projects.linpro.no> Author: phk Date: 2006-07-21 14:06:36 +0200 (Fri, 21 Jul 2006) New Revision: 536 Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c Log: Better name Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-21 11:55:17 UTC (rev 535) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-21 12:06:36 UTC (rev 536) @@ -42,7 +42,7 @@ struct vbc_mem { unsigned magic; #define VBC_MEM_MAGIC 0x2fd7af01 - struct vbe_conn vbe; + struct vbe_conn vbc; struct http http; }; @@ -78,11 +78,11 @@ return (NULL); vbcm->magic = VBC_MEM_MAGIC; VSL_stats->n_vbe_conn++; - vbcm->vbe.magic = VBE_CONN_MAGIC; - vbcm->vbe.vbcm = vbcm; - vbcm->vbe.http = &vbcm->http; + vbcm->vbc.magic = VBE_CONN_MAGIC; + vbcm->vbc.vbcm = vbcm; + vbcm->vbc.http = &vbcm->http; http_Setup(&vbcm->http, (void *)(vbcm + 1), heritage.mem_workspace); - return (&vbcm->vbe); + return (&vbcm->vbc); } static void From phk at projects.linpro.no Fri Jul 21 12:08:57 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 14:08:57 +0200 (CEST) Subject: r537 - trunk/varnish-cache/bin/varnishd Message-ID: <20060721120857.66DDE1EC39A@projects.linpro.no> Author: phk Date: 2006-07-21 14:08:57 +0200 (Fri, 21 Jul 2006) New Revision: 537 Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c Log: Magic check on struct vbe Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-21 12:06:36 UTC (rev 536) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-07-21 12:08:57 UTC (rev 537) @@ -312,12 +312,14 @@ if (vp == NULL) { vp = calloc(sizeof *vp, 1); assert(vp != NULL); + vp->magic = VBE_MAGIC; TAILQ_INIT(&vp->fconn); TAILQ_INIT(&vp->bconn); vp->ip = bp->ip; bp->vbe = vp; TAILQ_INSERT_TAIL(&vbe_head, vp, list); - } + } else + CHECK_OBJ(vp, VBE_MAGIC); /* XXX: check nconn vs backend->maxcon */ vc = TAILQ_FIRST(&vp->fconn); if (vc != NULL) { From phk at projects.linpro.no Fri Jul 21 12:18:02 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 14:18:02 +0200 (CEST) Subject: r538 - trunk/varnish-cache/bin/varnishd Message-ID: <20060721121802.109821EC396@projects.linpro.no> Author: phk Date: 2006-07-21 14:18:01 +0200 (Fri, 21 Jul 2006) New Revision: 538 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_session.c trunk/varnish-cache/bin/varnishd/flint.lnt Log: Cleanup Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-21 12:08:57 UTC (rev 537) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-21 12:18:01 UTC (rev 538) @@ -303,7 +303,6 @@ void http_CopyReq(int fd, struct http *to, struct http *fm); void http_CopyResp(int fd, struct http *to, struct http *fm); void http_FilterHeader(int fd, struct http *to, struct http *fm, unsigned how); -void http_CopyHeader(int fd, struct http *to, struct http *fm, unsigned n); void http_PrintfHeader(int fd, struct http *to, const char *fmt, ...); int http_IsHdr(struct http_hdr *hh, char *hdr); void http_Setup(struct http *ht, void *space, unsigned len); @@ -316,13 +315,7 @@ void http_RecvHead(struct http *hp, int fd, struct event_base *eb, http_callback_f *func, void *arg); int http_DissectRequest(struct http *sp, int fd); int http_DissectResponse(struct http *sp, int fd); -enum http_build { - Build_Pipe, - Build_Pass, - Build_Fetch, - Build_Reply -}; -void http_BuildSbuf(int fd, enum http_build mode, struct sbuf *sb, struct http *hp); + #define HTTPH(a, b, c, d, e, f, g) extern char b[]; #include "http_headers.h" #undef HTTPH @@ -339,7 +332,7 @@ void WRK_QueueSession(struct sess *sp); void WRK_Reset(struct worker *w, int *fd); int WRK_Flush(struct worker *w); -void WRK_Write(struct worker *w, const void *ptr, size_t len); +void WRK_Write(struct worker *w, const void *ptr, int len); void WRK_WriteH(struct worker *w, struct http_hdr *hh, const char *suf); /* cache_session.c [SES] */ @@ -347,7 +340,6 @@ struct sess *SES_New(struct sockaddr *addr, unsigned len); void SES_Delete(struct sess *sp); void SES_RefSrcAddr(struct sess *sp); -void SES_RelSrcAddr(struct sess *sp); void SES_ChargeBytes(struct sess *sp, uint64_t bytes); /* cache_shmlog.c */ Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-21 12:08:57 UTC (rev 537) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-07-21 12:18:01 UTC (rev 538) @@ -110,6 +110,7 @@ (void)arg; sp = SES_New(NULL, 0); + assert(sp != NULL); while (1) { t = time(NULL); AZ(pthread_mutex_lock(&exp_mtx)); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-21 12:08:57 UTC (rev 537) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-21 12:18:01 UTC (rev 538) @@ -209,7 +209,6 @@ { int cls; struct vbe_conn *vc; - struct worker *w; char *b; int body = 1; /* XXX */ @@ -217,7 +216,6 @@ CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); assert(sp->obj->busy != 0); - w = sp->wrk; vc = sp->vbc; Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-21 12:08:57 UTC (rev 537) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-21 12:18:01 UTC (rev 538) @@ -551,8 +551,8 @@ http_copyh(fd, to, fm, HTTP_HDR_RESPONSE, SLT_Response); } -void -http_CopyHeader(int fd, struct http *to, struct http *fm, unsigned n) +static void +http_copyheader(int fd, struct http *to, struct http *fm, unsigned n) { CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC); @@ -584,7 +584,7 @@ continue; #include "http_headers.h" #undef HTTPH - http_CopyHeader(fd, to, fm, u); + http_copyheader(fd, to, fm, u); } } @@ -644,7 +644,7 @@ void HTTP_Init(void) { -#define HTTPH(a, b, c, d, e, f, g) b[0] = strlen(b + 1); +#define HTTPH(a, b, c, d, e, f, g) b[0] = (char)strlen(b + 1); #include "http_headers.h" #undef HTTPH } Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-21 12:08:57 UTC (rev 537) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-21 12:18:01 UTC (rev 538) @@ -75,7 +75,7 @@ } void -WRK_Write(struct worker *w, const void *ptr, size_t len) +WRK_Write(struct worker *w, const void *ptr, int len) { CHECK_OBJ_NOTNULL(w, WORKER_MAGIC); Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-21 12:08:57 UTC (rev 537) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-21 12:18:01 UTC (rev 538) @@ -141,8 +141,8 @@ sa->addr, (intmax_t)(bytes), now - sa->first); } -void -SES_RelSrcAddr(struct sess *sp) +static void +ses_relsrcaddr(struct sess *sp) { if (sp->srcaddr == NULL) { @@ -188,7 +188,7 @@ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); VSL_stats->n_sess--; - SES_RelSrcAddr(sp); + ses_relsrcaddr(sp); CHECK_OBJ_NOTNULL(sp->mem, SESSMEM_MAGIC); free(sp->mem); } Modified: trunk/varnish-cache/bin/varnishd/flint.lnt =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.lnt 2006-07-21 12:08:57 UTC (rev 537) +++ trunk/varnish-cache/bin/varnishd/flint.lnt 2006-07-21 12:18:01 UTC (rev 538) @@ -28,6 +28,8 @@ // cache_center.c -efunc(525, CNT_Session) // Negative indentation from line +-efunc(525, http_FilterHeader) // Negative indentation from line +-efunc(539, http_FilterHeader) // Positive indentation from line // cache_vcl.c -efunc(525, vcl_handlingname) // Negative indentation from line From phk at phk.freebsd.dk Fri Jul 21 12:22:03 2006 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Fri, 21 Jul 2006 12:22:03 +0000 Subject: r538 - trunk/varnish-cache/bin/varnishd In-Reply-To: Your message of "Fri, 21 Jul 2006 14:18:02 +0200." <20060721121802.109821EC396@projects.linpro.no> Message-ID: <89873.1153484523@critter.freebsd.dk> In message <20060721121802.109821EC396 at projects.linpro.no>, phk at projects.linpro .no writes: >Log: >Cleanup OK, I think the dust has settled :-) Now we have almost all the tools for mucking about with headers that we will need. Right now the space for such mucking about is 4K ( which can be changed) but the intent is to use overflow to malloc rather than just loosing the headers like we do now. That bit of code is on the queue somewhere. This also means that we can start to decide and audit our header behaviour for good. Once I've played with it a bit here, I would like to take a live shot on c21 if that's OK with you Anders ? Please set it down to weight 1, the object is not performance but correctness. Next: TTL modification -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at projects.linpro.no Fri Jul 21 15:25:10 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 17:25:10 +0200 (CEST) Subject: r539 - in trunk/varnish-cache: bin/varnishlog bin/varnishtop include lib/libvarnishapi Message-ID: <20060721152510.123E31EC38E@projects.linpro.no> Author: phk Date: 2006-07-21 17:25:09 +0200 (Fri, 21 Jul 2006) New Revision: 539 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c trunk/varnish-cache/bin/varnishtop/varnishtop.c trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Work on logtailer api a bit: By default, start at the last entry in shared memory. To dump the entire segment from the start, specify '-d' option. Terminate programs when '-r $file' reaches EOF Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-21 12:18:01 UTC (rev 538) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-21 15:25:09 UTC (rev 539) @@ -240,8 +240,10 @@ v = 0; while (1) { - p = VSL_NextLog(vd); - if (p == NULL) { + i = VSL_NextLog(vd, &p); + if (i < 0) + break; + if (i == 0) { if (w_opt == NULL) { if (o_flag && ++v == 100) clean_order(); Modified: trunk/varnish-cache/bin/varnishtop/varnishtop.c =================================================================== --- trunk/varnish-cache/bin/varnishtop/varnishtop.c 2006-07-21 12:18:01 UTC (rev 538) +++ trunk/varnish-cache/bin/varnishtop/varnishtop.c 2006-07-21 15:25:09 UTC (rev 539) @@ -110,8 +110,10 @@ initscr(); v = 0; while (1) { - p = VSL_NextLog(vd); - if (p == NULL) { + i = VSL_NextLog(vd, &p); + if (i < 0) + break; + if (i == 0) { upd(); usleep(50000); continue; Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2006-07-21 12:18:01 UTC (rev 538) +++ trunk/varnish-cache/include/varnishapi.h 2006-07-21 15:25:09 UTC (rev 539) @@ -8,11 +8,11 @@ #define V_DEAD __attribute__ ((noreturn)) /* shmlog.c */ -#define VSL_ARGS "bcr:i:x:CI:X:" +#define VSL_ARGS "bcdr:i:x:CI:X:" struct VSL_data; struct VSL_data *VSL_New(void); int VSL_OpenLog(struct VSL_data *vd); -unsigned char *VSL_NextLog(struct VSL_data *lh); +int VSL_NextLog(struct VSL_data *lh, unsigned char **pp); int VSL_Arg(struct VSL_data *vd, int arg, const char *opt); struct varnish_stats *VSL_OpenStats(void); const char *VSL_tags[256]; Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-21 12:18:01 UTC (rev 538) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-21 15:25:09 UTC (rev 539) @@ -26,6 +26,7 @@ int b_opt; int c_opt; + int d_opt; int ix_opt; unsigned char supr[256]; @@ -44,6 +45,7 @@ static int vsl_fd; static struct shmloghead *vsl_lh; +static int vsl_nextlog(struct VSL_data *vd, unsigned char **pp); /*--------------------------------------------------------------------*/ @@ -107,8 +109,8 @@ int VSL_OpenLog(struct VSL_data *vd) { + unsigned char *p; - if (vd->fi != NULL) return (0); @@ -119,13 +121,17 @@ vd->logstart = (unsigned char *)vsl_lh + vsl_lh->start; vd->logend = vd->logstart + vsl_lh->size; vd->ptr = vd->logstart; + + if (!vd->d_opt) + while (vsl_nextlog(vd, &p) == 1) + continue; return (0); } /*--------------------------------------------------------------------*/ -static unsigned char * -vsl_nextlog(struct VSL_data *vd) +static int +vsl_nextlog(struct VSL_data *vd, unsigned char **pp) { unsigned char *p; int i; @@ -133,13 +139,14 @@ if (vd->fi != NULL) { i = fread(vd->rbuf, 4, 1, vd->fi); if (i != 1) - return (NULL); + return (-1); if (vd->rbuf[1] > 0) { i = fread(vd->rbuf + 4, vd->rbuf[1], 1, vd->fi); if (i != 1) - return (NULL); + return (-1); } - return (vd->rbuf); + *pp = vd->rbuf; + return (1); } p = vd->ptr; @@ -150,15 +157,16 @@ } if (*p == SLT_ENDMARKER) { vd->ptr = p; - return (NULL); + return (0); } vd->ptr = p + p[1] + 4; - return (p); + *pp = p; + return (1); } } -unsigned char * -VSL_NextLog(struct VSL_data *vd) +int +VSL_NextLog(struct VSL_data *vd, unsigned char **pp) { unsigned char *p; regmatch_t rm; @@ -166,9 +174,9 @@ int i; while (1) { - p = vsl_nextlog(vd); - if (p == NULL) - return (p); + i = vsl_nextlog(vd, &p); + if (i != 1) + return (i); u = (p[2] << 8) | p[3]; switch(p[0]) { case SLT_SessionOpen: @@ -200,7 +208,8 @@ if (i != REG_NOMATCH) continue; } - return (p); + *pp = p; + return (1); } } @@ -310,6 +319,7 @@ switch (arg) { case 'b': vd->b_opt = !vd->b_opt; return (1); case 'c': vd->c_opt = !vd->c_opt; return (1); + case 'd': vd->d_opt = !vd->d_opt; return (1); case 'i': case 'x': return (vsl_ix_arg(vd, opt, arg)); case 'r': return (vsl_r_arg(vd, opt)); case 'I': case 'X': return (vsl_IX_arg(vd, opt, arg)); From phk at projects.linpro.no Fri Jul 21 16:05:53 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 18:05:53 +0200 (CEST) Subject: r540 - trunk/varnish-cache/bin/varnishd Message-ID: <20060721160553.8F83A1EC396@projects.linpro.no> Author: phk Date: 2006-07-21 18:05:53 +0200 (Fri, 21 Jul 2006) New Revision: 540 Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c Log: Log TTL calculation on the right fd Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-21 15:25:09 UTC (rev 539) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-21 16:05:53 UTC (rev 540) @@ -71,7 +71,7 @@ #endif static time_t -RFC2616_Ttl(struct http *hp, time_t t_req, time_t t_resp, struct object *obj) +RFC2616_Ttl(int fd, struct http *hp, time_t t_req, time_t t_resp, struct object *obj) { int retirement_age; unsigned u1, u2; @@ -117,7 +117,7 @@ ttd = t_req + retirement_age; } - VSL(SLT_Debug, 0, + VSL(SLT_Debug, fd, "TTD: max-age %u Age: %u Date: %d (%d) Expires %d (%d) our_clock %d" " -> ttd %d (%d)", u1, u2, h_date, h_date - t_req, h_expires, h_expires - t_req, t_req, ttd, ttd - t_req); @@ -153,7 +153,7 @@ break; } - sp->obj->ttl = RFC2616_Ttl(hp, sp->t_req, sp->t_resp, sp->obj); + sp->obj->ttl = RFC2616_Ttl(sp->fd, hp, sp->t_req, sp->t_resp, sp->obj); sp->obj->entered = sp->t_req; if (sp->obj->ttl == 0) { sp->obj->cacheable = 0; From phk at projects.linpro.no Fri Jul 21 16:06:07 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 18:06:07 +0200 (CEST) Subject: r541 - trunk/varnish-cache/bin/varnishd Message-ID: <20060721160607.71AC71EC396@projects.linpro.no> Author: phk Date: 2006-07-21 18:06:07 +0200 (Fri, 21 Jul 2006) New Revision: 541 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Fix formatting of responses. Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-21 16:05:53 UTC (rev 540) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-21 16:06:07 UTC (rev 541) @@ -622,9 +622,9 @@ if (resp) { assert(hp->hd[HTTP_HDR_STATUS].b != NULL); - WRK_WriteH(w, &hp->hd[HTTP_HDR_PROTO], "\r\n"); + WRK_WriteH(w, &hp->hd[HTTP_HDR_PROTO], " "); WRK_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " "); - WRK_WriteH(w, &hp->hd[HTTP_HDR_RESPONSE], " "); + WRK_WriteH(w, &hp->hd[HTTP_HDR_RESPONSE], "\r\n"); } else { assert(hp->hd[HTTP_HDR_URL].b != NULL); WRK_WriteH(w, &hp->hd[HTTP_HDR_REQ], " "); From phk at projects.linpro.no Fri Jul 21 16:15:58 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 18:15:58 +0200 (CEST) Subject: r542 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060721161558.EB7D11EC38E@projects.linpro.no> Author: phk Date: 2006-07-21 18:15:58 +0200 (Fri, 21 Jul 2006) New Revision: 542 Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c trunk/varnish-cache/include/shmlog_tags.h Log: Properly log TTL calculation to shmem Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-21 16:06:07 UTC (rev 541) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-21 16:15:58 UTC (rev 542) @@ -117,10 +117,12 @@ ttd = t_req + retirement_age; } - VSL(SLT_Debug, fd, - "TTD: max-age %u Age: %u Date: %d (%d) Expires %d (%d) our_clock %d" - " -> ttd %d (%d)", - u1, u2, h_date, h_date - t_req, h_expires, h_expires - t_req, t_req, ttd, ttd - t_req); + + /* calculated TTL, Our time, Date, Expires, max-age, age */ + VSL(SLT_TTL, fd, "%d %d %d %d %d %d", + (int)(ttd - t_req), (int)t_req, (int)h_date, (int)h_expires, + (int)u1, (int)u2); + return (ttd); } Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-07-21 16:06:07 UTC (rev 541) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-07-21 16:15:58 UTC (rev 542) @@ -32,6 +32,7 @@ SLTM(RxHeader) SLTM(TxHeader) SLTM(LostHeader) +SLTM(TTL) SLTM(VCL_call) SLTM(VCL_trace) SLTM(VCL_return) From phk at projects.linpro.no Fri Jul 21 16:25:24 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 18:25:24 +0200 (CEST) Subject: r543 - trunk/varnish-cache/include Message-ID: <20060721162524.8098D1EC396@projects.linpro.no> Author: phk Date: 2006-07-21 18:25:24 +0200 (Fri, 21 Jul 2006) New Revision: 543 Modified: trunk/varnish-cache/include/http_headers.h Log: update comment Modified: trunk/varnish-cache/include/http_headers.h =================================================================== --- trunk/varnish-cache/include/http_headers.h 2006-07-21 16:15:58 UTC (rev 542) +++ trunk/varnish-cache/include/http_headers.h 2006-07-21 16:25:24 UTC (rev 543) @@ -4,7 +4,7 @@ * a Http header name * b session field name * c Request(1)/Response(2) bitfield - * d Supress header to backend (1) / Supress header to client (2) + * d (obsolete) * e Supress header in pass from client to backend * f unused * g unused From phk at projects.linpro.no Fri Jul 21 18:12:59 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 20:12:59 +0200 (CEST) Subject: r544 - trunk/varnish-cache/lib/libvcl Message-ID: <20060721181259.51A5D1EC396@projects.linpro.no> Author: phk Date: 2006-07-21 20:12:56 +0200 (Fri, 21 Jul 2006) New Revision: 544 Added: trunk/varnish-cache/lib/libvcl/vcc_compile.c trunk/varnish-cache/lib/libvcl/vcc_compile.h trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl trunk/varnish-cache/lib/libvcl/vcc_obj.c trunk/varnish-cache/lib/libvcl/vcc_priv.h trunk/varnish-cache/lib/libvcl/vcc_token.c trunk/varnish-cache/lib/libvcl/vcc_token_defs.h Removed: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl trunk/varnish-cache/lib/libvcl/vcl_priv.h trunk/varnish-cache/lib/libvcl/vcl_token_defs.h Modified: trunk/varnish-cache/lib/libvcl/Makefile.am Log: Move things over to the correct "VCC" prefix. Split some stuff into separate files while we're at it. Modified: trunk/varnish-cache/lib/libvcl/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvcl/Makefile.am 2006-07-21 16:25:24 UTC (rev 543) +++ trunk/varnish-cache/lib/libvcl/Makefile.am 2006-07-21 18:12:56 UTC (rev 544) @@ -5,8 +5,11 @@ lib_LTLIBRARIES = libvcl.la libvcl_la_SOURCES = \ - vcl_priv.h \ - vcl_token_defs.h \ + vcc_priv.h \ + vcc_compile.h \ + vcc_token_defs.h \ \ - vcl_compile.c \ - vcl_fixed_token.c + vcc_compile.c \ + vcc_fixed_token.c \ + vcc_obj.c \ + vcc_token.c Added: trunk/varnish-cache/lib/libvcl/vcc_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.c 2006-07-21 16:25:24 UTC (rev 543) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.c 2006-07-21 18:12:56 UTC (rev 544) @@ -0,0 +1,1529 @@ +/* + * $Id: vcl_compile.c 531 2006-07-20 22:08:43Z phk $ + */ + +/* + * XXX: + * generate interface structure + * + * XXX: + * Better error messages, throughout. + * >It also accured to me that we could link the errors to the error + * >documentation. + * > + * >Unreferenced function 'request_policy', first mention is + * > Line 8 Pos 4 + * > sub request_policy { + * > ----##############-- + * >Read more about this type of error: + * >http://varnish/doc/error.html#Unreferenced%20function + * > + * > + * > Unknown variable 'obj.bandwidth' + * > At: Line 88 Pos 12 + * > if (obj.bandwidth < 1 kb/h) { + * > ------------#############------------ + * >Read more about this type of error: + * >http://varnish/doc/error.html#Unknown%20variable + * + * XXX: + * Create proper tmp filenames for .h, .c and .o + * + * XXX: + * and all the rest... + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vcc_priv.h" +#include "vcc_compile.h" + +#include "libvcl.h" + +#define ERRCHK(tl) do { if ((tl)->err) return; } while (0) + +/*--------------------------------------------------------------------*/ + +static void Compound(struct tokenlist *tl); +static void Cond_0(struct tokenlist *tl); +static struct proc *AddProc(struct tokenlist *tl, struct token *t, int def); +static void AddCall(struct tokenlist *tl, struct token *t); +const char *vcc_default_vcl_b, *vcc_default_vcl_e; + +/*--------------------------------------------------------------------*/ + +#define ErrInternal(tl) vcc__ErrInternal(tl, __func__, __LINE__) + +#define Expect(a, b) vcc__Expect(a, b, __LINE__) +#define ExpectErr(a, b) do { vcc__Expect(a, b, __LINE__); ERRCHK(a);} while (0) + +#define L(tl, foo) do { \ + tl->indent += INDENT; \ + foo; \ + tl->indent -= INDENT; \ +} while (0) + +#define C(tl, sep) do { \ + Fc(tl, 1, "VRT_count(sp, %u)%s\n", ++tl->cnt, sep); \ + tl->t->cnt = tl->cnt; \ +} while (0) + +/*-------------------------------------------------------------------- + * Printf output to the two sbufs, possibly indented + */ + +static void +Fh(struct tokenlist *tl, int indent, const char *fmt, ...) +{ + va_list ap; + + if (indent) + sbuf_printf(tl->fh, "%*.*s", tl->indent, tl->indent, ""); + va_start(ap, fmt); + sbuf_vprintf(tl->fh, fmt, ap); + va_end(ap); +} + +static void +Fc(struct tokenlist *tl, int indent, const char *fmt, ...) +{ + va_list ap; + + if (indent) + sbuf_printf(tl->fc, "%*.*s", tl->indent, tl->indent, ""); + va_start(ap, fmt); + sbuf_vprintf(tl->fc, fmt, ap); + va_end(ap); +} + +/*--------------------------------------------------------------------*/ + +static char * +EncString(struct token *t) +{ + char *p, *q; + const char *r; + unsigned u; + + assert(t->tok == CSTR); + p = malloc(t->e - t->b); + assert(p != NULL); + q = p; + for (r = t->b + 1; r < t->e - 1; ) { + if (*r != '\\') { + *q++ = *r++; + continue; + } + switch (r[1]) { + case 'n': *q++ = '\n'; r += 2; break; + case 'r': *q++ = '\r'; r += 2; break; + case 'v': *q++ = '\v'; r += 2; break; + case 'f': *q++ = '\f'; r += 2; break; + case 't': *q++ = '\t'; r += 2; break; + case 'b': *q++ = '\b'; r += 2; break; + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + u = r[1] - '0'; + r += 2; + if (isdigit(r[0]) && (r[0] - '0') < 8) { + u <<= 3; + u |= r[0] - '0'; + r++; + if (isdigit(r[0]) && (r[0] - '0') < 8) { + u <<= 3; + u |= r[0] - '0'; + r++; + } + } + *q++ = u; + break; + default: + *q++ = r[1]; + r += 2; + break; + } + } + *q = '\0'; + return (p); +} + + +/*-------------------------------------------------------------------- + * Keep track of definitions and references + */ + +static struct ref * +FindRef(struct tokenlist *tl, struct token *t, enum ref_type type) +{ + struct ref *r; + + TAILQ_FOREACH(r, &tl->refs, list) { + if (r->type != type) + continue; + if (vcc_Teq(r->name, t)) + return (r); + } + r = calloc(sizeof *r, 1); + assert(r != NULL); + r->name = t; + r->type = type; + TAILQ_INSERT_TAIL(&tl->refs, r, list); + return (r); +} + +static int +FindRefStr(struct tokenlist *tl, const char *s, enum ref_type type) +{ + struct ref *r; + + TAILQ_FOREACH(r, &tl->refs, list) { + if (r->type != type) + continue; + if (vcc_IdIs(r->name, s)) + return (1); + } + return (0); +} + +static void +AddRef(struct tokenlist *tl, struct token *t, enum ref_type type) +{ + + FindRef(tl, t, type)->refcnt++; +} + +static void +AddRefStr(struct tokenlist *tl, const char *s, enum ref_type type) +{ + struct token *t; + + t = calloc(sizeof *t, 1); + t->b = s; + t->e = strchr(s, '\0'); + t->tok = METHOD; + AddRef(tl, t, type); +} + +static void +AddDef(struct tokenlist *tl, struct token *t, enum ref_type type) +{ + struct ref *r; + + r = FindRef(tl, t, type); + r->defcnt++; + r->name = t; +} + +/*-------------------------------------------------------------------- + * Recognize and convert units of time, return seconds. + */ + +static double +TimeUnit(struct tokenlist *tl) +{ + double sc = 1.0; + + assert(tl->t->tok == ID); + if (vcc_IdIs(tl->t, "ms")) + sc = 1e-3; + else if (vcc_IdIs(tl->t, "s")) + sc = 1.0; + else if (vcc_IdIs(tl->t, "m")) + sc = 60.0; + else if (vcc_IdIs(tl->t, "h")) + sc = 60.0 * 60.0; + else if (vcc_IdIs(tl->t, "d")) + sc = 60.0 * 60.0 * 24.0; + else { + sbuf_printf(tl->sb, "Unknown time unit "); + vcc_ErrToken(tl, tl->t); + sbuf_printf(tl->sb, ". Legal are 's', 'm', 'h' and 'd'\n"); + vcc_ErrWhere(tl, tl->t); + return (1.0); + } + vcc_NextToken(tl); + return (sc); +} + +/*-------------------------------------------------------------------- + * Recognize and convert units of size, return bytes. + */ + +static double +SizeUnit(struct tokenlist *tl) +{ + double sc = 1.0; + + assert(tl->t->tok == ID); + if (vcc_IdIs(tl->t, "b")) + sc = 1.0; + else if (vcc_IdIs(tl->t, "kb")) + sc = 1024.0; + else if (vcc_IdIs(tl->t, "mb") || vcc_IdIs(tl->t, "Mb")) + sc = 1024.0 * 1024.0; + else if (vcc_IdIs(tl->t, "gb") || vcc_IdIs(tl->t, "Gb")) + sc = 1024.0 * 1024.0 * 1024.0; + else { + sbuf_printf(tl->sb, "Unknown size unit "); + vcc_ErrToken(tl, tl->t); + sbuf_printf(tl->sb, ". Legal are 'kb', 'mb' and 'gb'\n"); + vcc_ErrWhere(tl, tl->t); + return (1.0); + } + vcc_NextToken(tl); + return (sc); +} + +/*-------------------------------------------------------------------- + * Recognize and convert units of rate as { space '/' time } + */ + +static double +RateUnit(struct tokenlist *tl) +{ + double sc = 1.0; + + assert(tl->t->tok == ID); + sc = SizeUnit(tl); + Expect(tl, '/'); + vcc_NextToken(tl); + sc /= TimeUnit(tl); + return (sc); +} + +/*-------------------------------------------------------------------- + * Recognize and convert { CNUM } to unsigned value + */ + +static unsigned +UintVal(struct tokenlist *tl) +{ + unsigned d = 0; + const char *p; + + Expect(tl, CNUM); + for (p = tl->t->b; p < tl->t->e; p++) { + d *= 10; + d += *p - '0'; + } + vcc_NextToken(tl); + return (d); +} + +/*-------------------------------------------------------------------- + * Recognize and convert { CNUM [ '.' [ CNUM ] ] } to double value + */ + +static double +DoubleVal(struct tokenlist *tl) +{ + double d = 0.0, e = 0.1; + const char *p; + + Expect(tl, CNUM); + for (p = tl->t->b; p < tl->t->e; p++) { + d *= 10; + d += *p - '0'; + } + vcc_NextToken(tl); + if (tl->t->tok != '.') + return (d); + vcc_NextToken(tl); + if (tl->t->tok != CNUM) + return (d); + for (p = tl->t->b; p < tl->t->e; p++) { + d += (*p - '0') * e; + e *= 0.1; + } + vcc_NextToken(tl); + return (d); +} + +/*--------------------------------------------------------------------*/ + +static unsigned +IpVal(struct tokenlist *tl) +{ + unsigned u, v; + struct token *t; + + t = tl->t; + u = UintVal(tl); + if (u < 256) { + v = u << 24; + Expect(tl, '.'); + vcc_NextToken(tl); + t = tl->t; + u = UintVal(tl); + if (u < 256) { + v |= u << 16; + Expect(tl, '.'); + vcc_NextToken(tl); + t = tl->t; + u = UintVal(tl); + if (u < 256) { + v |= u << 8; + Expect(tl, '.'); + vcc_NextToken(tl); + t = tl->t; + u = UintVal(tl); + if (u < 256) { + v |= u; + return (v); + } + } + } + } + sbuf_printf(tl->sb, "Illegal octet in IP number\n"); + vcc_ErrWhere(tl, t); + return (0); +} + +/*--------------------------------------------------------------------*/ + +static struct var * +HeaderVar(struct tokenlist *tl __unused, struct token *t, struct var *vh) +{ + char *p; + struct var *v; + int i; + + v = calloc(sizeof *v, 1); + assert(v != NULL); + i = t->e - t->b; + p = malloc(i + 1); + assert(p != NULL); + memcpy(p, t->b, i); + p[i] = '\0'; + v->name = p; + v->fmt = STRING; + asprintf(&p, "VRT_GetHdr(sp, \"\\%03o%s:\")", + strlen(v->name + vh->len) + 1, v->name + vh->len); + assert(p != NULL); + v->rname = p; + return (v); +} + +/*--------------------------------------------------------------------*/ + +static struct var * +FindVar(struct tokenlist *tl, struct token *t, struct var *vl) +{ + struct var *v; + + for (v = vl; v->name != NULL; v++) { + if (v->fmt == HEADER && t->e - t->b <= v->len) + continue; + if (v->fmt != HEADER && t->e - t->b != v->len) + continue; + if (memcmp(t->b, v->name, v->len)) + continue; + if (v->fmt != HEADER) + return (v); + return (HeaderVar(tl, t, v)); + } + sbuf_printf(tl->sb, "Unknown variable "); + vcc_ErrToken(tl, t); + sbuf_cat(tl->sb, "\nAt: "); + vcc_ErrWhere(tl, t); + return (NULL); +} + + +/*--------------------------------------------------------------------*/ + +static void +TimeVal(struct tokenlist *tl) +{ + double v, sc; + + v = DoubleVal(tl); + ExpectErr(tl, ID); + sc = TimeUnit(tl); + Fc(tl, 0, "(%g * %g)", v, sc); +} + +static void +SizeVal(struct tokenlist *tl) +{ + double v, sc; + + v = DoubleVal(tl); + ExpectErr(tl, ID); + sc = SizeUnit(tl); + Fc(tl, 0, "(%g * %g)", v, sc); +} + +static void +RateVal(struct tokenlist *tl) +{ + double v, sc; + + v = DoubleVal(tl); + ExpectErr(tl, ID); + sc = RateUnit(tl); + Fc(tl, 0, "(%g * %g)", v, sc); +} + +/*--------------------------------------------------------------------*/ + +static void +Cond_Ip(struct var *vp, struct tokenlist *tl) +{ + unsigned u; + + switch (tl->t->tok) { + case '~': + vcc_NextToken(tl); + ExpectErr(tl, ID); + AddRef(tl, tl->t, R_ACL); + Fc(tl, 1, "ip_match(%s, acl_%T)\n", vp->rname, tl->t); + vcc_NextToken(tl); + break; + case T_EQ: + case T_NEQ: + Fc(tl, 1, "%s %T ", vp->rname, tl->t); + vcc_NextToken(tl); + u = IpVal(tl); + Fc(tl, 0, "%uU /* %u.%u.%u.%u */\n", u, + (u >> 24) & 0xff, (u >> 16) & 0xff, + (u >> 8) & 0xff, (u) & 0xff); + break; + default: + sbuf_printf(tl->sb, "Illegal condition "); + vcc_ErrToken(tl, tl->t); + sbuf_printf(tl->sb, " on IP number variable\n"); + sbuf_printf(tl->sb, " only '==', '!=' and '~' are legal\n"); + vcc_ErrWhere(tl, tl->t); + break; + } +} + +static void +Cond_String(struct var *vp, struct tokenlist *tl) +{ + + switch (tl->t->tok) { + case '~': + Fc(tl, 1, "string_match(%s, ", vp->rname); + vcc_NextToken(tl); + ExpectErr(tl, CSTR); + Fc(tl, 0, "%T)\n", tl->t); + vcc_NextToken(tl); + break; + case T_EQ: + case T_NEQ: + Fc(tl, 1, "%sstrcmp(%s, ", + tl->t->tok == T_EQ ? "!" : "", vp->rname); + vcc_NextToken(tl); + ExpectErr(tl, CSTR); + Fc(tl, 0, "%T)\n", tl->t); + vcc_NextToken(tl); + break; + default: + Fc(tl, 1, "%s != (void*)0", vp->rname); + break; + } +} + +static void +Cond_Int(struct var *vp, struct tokenlist *tl) +{ + + Fc(tl, 1, "%s ", vp->rname); + switch (tl->t->tok) { + case T_EQ: + case T_NEQ: + case T_LEQ: + case T_GEQ: + case '>': + case '<': + Fc(tl, 0, "%T ", tl->t); + vcc_NextToken(tl); + switch(vp->fmt) { + case TIME: + TimeVal(tl); + break; + case INT: + ExpectErr(tl, CNUM); + Fc(tl, 0, "%T ", tl->t); + vcc_NextToken(tl); + break; + case SIZE: + SizeVal(tl); + break; + default: + sbuf_printf(tl->sb, + "No conditions available for variable '%s'\n", + vp->name); + vcc_ErrWhere(tl, tl->t); + return; + } + Fc(tl, 0, "\n"); + break; + default: + sbuf_printf(tl->sb, "Illegal condition "); + vcc_ErrToken(tl, tl->t); + sbuf_printf(tl->sb, " on integer variable\n"); + sbuf_printf(tl->sb, + " only '==', '!=', '<', '>', '<=' and '>=' are legal\n"); + vcc_ErrWhere(tl, tl->t); + break; + } +} + +static void +Cond_Bool(struct var *vp, struct tokenlist *tl) +{ + + Fc(tl, 1, "%s\n", vp->rname); +} + +static void +Cond_2(struct tokenlist *tl) +{ + struct var *vp; + + C(tl, ","); + if (tl->t->tok == '!') { + Fc(tl, 1, "!(\n"); + vcc_NextToken(tl); + } else { + Fc(tl, 1, "(\n"); + } + if (tl->t->tok == '(') { + vcc_NextToken(tl); + Cond_0(tl); + ExpectErr(tl, ')'); + vcc_NextToken(tl); + } else if (tl->t->tok == VAR) { + vp = FindVar(tl, tl->t, vcc_vars); + ERRCHK(tl); + assert(vp != NULL); + vcc_NextToken(tl); + switch (vp->fmt) { + case INT: L(tl, Cond_Int(vp, tl)); break; + case SIZE: L(tl, Cond_Int(vp, tl)); break; + case BOOL: L(tl, Cond_Bool(vp, tl)); break; + case IP: L(tl, Cond_Ip(vp, tl)); break; + case STRING: L(tl, Cond_String(vp, tl)); break; + case TIME: L(tl, Cond_Int(vp, tl)); break; + /* XXX backend == */ + default: + sbuf_printf(tl->sb, + "Variable '%s'" + " has no conditions that can be checked\n", + vp->name); + vcc_ErrWhere(tl, tl->t); + return; + } + } else { + sbuf_printf(tl->sb, + "Syntax error in condition, expected '(', '!' or" + " variable name, found "); + vcc_ErrToken(tl, tl->t); + sbuf_printf(tl->sb, "\n"); + vcc_ErrWhere(tl, tl->t); + return; + } + Fc(tl, 1, ")\n"); +} + +static void +Cond_1(struct tokenlist *tl) +{ + + Fc(tl, 1, "(\n"); + L(tl, Cond_2(tl)); + while (tl->t->tok == T_CAND) { + vcc_NextToken(tl); + Fc(tl, 1, ") && (\n"); + L(tl, Cond_2(tl)); + } + Fc(tl, 1, ")\n"); +} + +static void +Cond_0(struct tokenlist *tl) +{ + + Fc(tl, 1, "(\n"); + L(tl, Cond_1(tl)); + while (tl->t->tok == T_COR) { + vcc_NextToken(tl); + Fc(tl, 1, ") || (\n"); + L(tl, Cond_1(tl)); + } + Fc(tl, 1, ")\n"); +} + +static void +Conditional(struct tokenlist *tl) +{ + + ExpectErr(tl, '('); + vcc_NextToken(tl); + Fc(tl, 1, "(\n"); + L(tl, Cond_0(tl)); + ERRCHK(tl); + Fc(tl, 1, ")\n"); + ExpectErr(tl, ')'); + vcc_NextToken(tl); +} + +/*--------------------------------------------------------------------*/ + +static void +IfStmt(struct tokenlist *tl) +{ + + ExpectErr(tl, T_IF); + Fc(tl, 1, "if \n"); + vcc_NextToken(tl); + L(tl, Conditional(tl)); + ERRCHK(tl); + L(tl, Compound(tl)); + ERRCHK(tl); + while (1) { + switch (tl->t->tok) { + case T_ELSE: + vcc_NextToken(tl); + if (tl->t->tok != T_IF) { + Fc(tl, 1, "else \n"); + L(tl, Compound(tl)); + ERRCHK(tl); + return; + } + /* FALLTHROUGH */ + case T_ELSEIF: + case T_ELSIF: + Fc(tl, 1, "else if \n"); + vcc_NextToken(tl); + L(tl, Conditional(tl)); + ERRCHK(tl); + L(tl, Compound(tl)); + ERRCHK(tl); + break; + default: + C(tl, ";"); + return; + } + } +} + +/*--------------------------------------------------------------------*/ + +static void +Action(struct tokenlist *tl) +{ + unsigned a, u; + struct var *vp; + struct token *at; + + at = tl->t; + vcc_NextToken(tl); + switch (at->tok) { + case T_NO_NEW_CACHE: + Fc(tl, 1, "VCL_no_new_cache(sp);\n"); + return; + case T_NO_CACHE: + Fc(tl, 1, "VCL_no_cache(sp);\n"); + return; +#define VCL_RET_MAC(a,b,c) case T_##b: \ + Fc(tl, 1, "VRT_done(sp, VCL_RET_%s);\n", #b); \ + tl->curproc->returns |= VCL_RET_##b; \ + tl->curproc->returnt[c] = at; \ + return; +#include "vcl_returns.h" +#undef VCL_RET_MAC + case T_ERROR: + if (tl->t->tok == CNUM) + a = UintVal(tl); + else + a = 0; + Fc(tl, 1, "VRT_error(sp, %u, ", a); + if (tl->t->tok == CSTR) { + Fc(tl, 0, "%T);\n", tl->t); + vcc_NextToken(tl); + } else + Fc(tl, 0, "(const char *)0);\n"); + Fc(tl, 1, "VRT_done(sp, VCL_RET_ERROR);\n"); + return; + case T_SWITCH_CONFIG: + ExpectErr(tl, ID); + Fc(tl, 1, "VCL_switch_config(\"%T\");\n", tl->t); + vcc_NextToken(tl); + return; + case T_CALL: + ExpectErr(tl, ID); + AddCall(tl, tl->t); + AddRef(tl, tl->t, R_FUNC); + Fc(tl, 1, "if (VGC_function_%T(sp))\n", tl->t); + Fc(tl, 1, "\treturn (1);\n"); + vcc_NextToken(tl); + return; + case T_REWRITE: + ExpectErr(tl, CSTR); + Fc(tl, 1, "VCL_rewrite(%T", tl->t); + vcc_NextToken(tl); + ExpectErr(tl, CSTR); + Fc(tl, 0, ", %T);\n", tl->t); + vcc_NextToken(tl); + return; + case T_SET: + ExpectErr(tl, VAR); + vp = FindVar(tl, tl->t, vcc_vars); + ERRCHK(tl); + assert(vp != NULL); + Fc(tl, 1, "%s ", vp->rname); + vcc_NextToken(tl); + switch (vp->fmt) { + case INT: + case SIZE: + case RATE: + case TIME: + case FLOAT: + Fc(tl, 0, "%T ", tl->t); + a = tl->t->tok; + vcc_NextToken(tl); + if (a == T_MUL || a == T_DIV) + Fc(tl, 0, "%g", DoubleVal(tl)); + else if (vp->fmt == TIME) + TimeVal(tl); + else if (vp->fmt == SIZE) + SizeVal(tl); + else if (vp->fmt == RATE) + RateVal(tl); + else + Fc(tl, 0, "%g", DoubleVal(tl)); + Fc(tl, 0, ";\n"); + break; + case IP: + if (tl->t->tok == '=') { + vcc_NextToken(tl); + u = IpVal(tl); + Fc(tl, 0, "= %uU; /* %u.%u.%u.%u */\n", + u, + (u >> 24) & 0xff, + (u >> 16) & 0xff, + (u >> 8) & 0xff, + u & 0xff); + break; + } + sbuf_printf(tl->sb, "Illegal assignment operator "); + vcc_ErrToken(tl, tl->t); + sbuf_printf(tl->sb, + " only '=' is legal for IP numbers\n"); + vcc_ErrWhere(tl, tl->t); + return; + case BACKEND: + if (tl->t->tok == '=') { + vcc_NextToken(tl); + AddRef(tl, tl->t, R_BACKEND); + Fc(tl, 0, "= &VGC_backend_%T;\n", tl->t); + vcc_NextToken(tl); + break; + } + sbuf_printf(tl->sb, "Illegal assignment operator "); + vcc_ErrToken(tl, tl->t); + sbuf_printf(tl->sb, + " only '=' is legal for backend\n"); + vcc_ErrWhere(tl, tl->t); + return; + default: + sbuf_printf(tl->sb, + "Assignments not possible for '%s'\n", vp->name); + vcc_ErrWhere(tl, tl->t); + return; + } + return; + default: + sbuf_printf(tl->sb, "Expected action, 'if' or '}'\n"); + vcc_ErrWhere(tl, at); + return; + } +} + +/*--------------------------------------------------------------------*/ + +static void +Acl(struct tokenlist *tl) +{ + unsigned u, m; + + vcc_NextToken(tl); + + ExpectErr(tl, ID); + AddDef(tl, tl->t, R_ACL); + Fh(tl, 0, "static struct vcl_acl acl_%T[];\n", tl->t); + Fc(tl, 1, "static struct vcl_acl acl_%T[] = {\n", tl->t); + vcc_NextToken(tl); + + tl->indent += INDENT; + + ExpectErr(tl, '{'); + vcc_NextToken(tl); + + while (tl->t->tok == CNUM) { + u = IpVal(tl); + if (tl->t->tok == '/') { + vcc_NextToken(tl); + ExpectErr(tl, CNUM); + m = UintVal(tl); + } else + m = 32; + ExpectErr(tl, ';'); + vcc_NextToken(tl); + Fc(tl, 1, "{ %11uU, %3uU }, /* %u.%u.%u.%u/%u */\n", + u, m, + (u >> 24) & 0xff, (u >> 16) & 0xff, + (u >> 8) & 0xff, (u) & 0xff, m); + } + ExpectErr(tl, '}'); + Fc(tl, 1, "{ %11uU, %3uU }\n", 0, 0); + + tl->indent -= INDENT; + + Fc(tl, 1, "};\n\n"); + vcc_NextToken(tl); +} + +/*--------------------------------------------------------------------*/ + +static void +Compound(struct tokenlist *tl) +{ + + ExpectErr(tl, '{'); + Fc(tl, 1, "{\n"); + tl->indent += INDENT; + C(tl, ";"); + vcc_NextToken(tl); + while (1) { + ERRCHK(tl); + switch (tl->t->tok) { + case '{': + Compound(tl); + break; + case T_IF: + IfStmt(tl); + break; + case '}': + vcc_NextToken(tl); + tl->indent -= INDENT; + Fc(tl, 1, "}\n"); + return; + case EOI: + sbuf_printf(tl->sb, + "End of input while in compound statement\n"); + tl->err = 1; + return; + default: + Action(tl); + ERRCHK(tl); + ExpectErr(tl, ';'); + vcc_NextToken(tl); + break; + } + } +} + +/*--------------------------------------------------------------------*/ + +static const char * +CheckHostPort(const char *host, const char *port) +{ + struct addrinfo *res, hint; + int error; + + memset(&hint, 0, sizeof hint); + hint.ai_family = PF_UNSPEC; + hint.ai_socktype = SOCK_STREAM; + error = getaddrinfo(host, port, &hint, &res); + if (error) + return (gai_strerror(error)); + freeaddrinfo(res); + return (NULL); +} + +static void +Backend(struct tokenlist *tl) +{ + struct var *vp; + struct token *t_be = NULL; + struct token *t_host = NULL; + struct token *t_port = NULL; + char *host = NULL; + char *port = NULL; + const char *ep; + + vcc_NextToken(tl); + ExpectErr(tl, ID); + t_be = tl->t; + AddDef(tl, tl->t, R_BACKEND); + if (tl->nbackend == 0) + AddRef(tl, tl->t, R_BACKEND); + Fh(tl, 1, "#define VGC_backend_%T (VCL_conf.backend[%d])\n", + tl->t, tl->nbackend); + Fc(tl, 0, "static void\n"); + Fc(tl, 1, "VGC_init_backend_%T (void)\n", tl->t); + Fc(tl, 1, "{\n"); + Fc(tl, 1, "\tstruct backend *backend = VGC_backend_%T;\n", tl->t); + Fc(tl, 1, "\tconst char *p;\n"); + Fc(tl, 1, "\n"); + Fc(tl, 1, "\tVRT_set_backend_name(backend, \"%T\");\n", tl->t); + vcc_NextToken(tl); + ExpectErr(tl, '{'); + vcc_NextToken(tl); + while (1) { + if (tl->t->tok == '}') + break; + ExpectErr(tl, T_SET); + vcc_NextToken(tl); + ExpectErr(tl, VAR); + vp = FindVar(tl, tl->t, vcc_be_vars); + ERRCHK(tl); + assert(vp != NULL); + vcc_NextToken(tl); + ExpectErr(tl, '='); + vcc_NextToken(tl); + switch (vp->fmt) { + case HOSTNAME: + ExpectErr(tl, CSTR); + t_host = tl->t; + Fc(tl, 1, "\tp = %T;\n", tl->t); + Fc(tl, 1, "\t"); + Fc(tl, 0, vp->lname, "p"); + Fc(tl, 0, ";\n"); + vcc_NextToken(tl); + break; + case PORTNAME: + ExpectErr(tl, CSTR); + t_port = tl->t; + Fc(tl, 1, "\tp = %T;\n", tl->t); + Fc(tl, 1, "\t"); + Fc(tl, 0, vp->lname, "p"); + Fc(tl, 0, ";\n"); + vcc_NextToken(tl); + break; + default: + sbuf_printf(tl->sb, + "Assignments not possible for '%s'\n", vp->name); + vcc_ErrWhere(tl, tl->t); + return; + } + ExpectErr(tl, ';'); + vcc_NextToken(tl); + } + ExpectErr(tl, '}'); + if (t_host == NULL) { + sbuf_printf(tl->sb, "Backend '%T' has no hostname\n", t_be); + vcc_ErrWhere(tl, tl->t); + return; + } + host = EncString(t_host); + ep = CheckHostPort(host, "80"); + if (ep != NULL) { + sbuf_printf(tl->sb, "Backend '%T': %s\n", t_be, ep); + vcc_ErrWhere(tl, t_host); + return; + } + if (t_port != NULL) { + port = EncString(t_port); + ep = CheckHostPort(host, port); + if (ep != NULL) { + sbuf_printf(tl->sb, "Backend '%T': %s\n", t_be, ep); + vcc_ErrWhere(tl, t_port); + return; + } + } + + vcc_NextToken(tl); + Fc(tl, 1, "}\n"); + Fc(tl, 0, "\n"); + tl->nbackend++; +} + +/*--------------------------------------------------------------------*/ + +static void +Function(struct tokenlist *tl) +{ + + vcc_NextToken(tl); + ExpectErr(tl, ID); + tl->curproc = AddProc(tl, tl->t, 1); + tl->curproc->exists++; + AddDef(tl, tl->t, R_FUNC); + Fh(tl, 0, "static int VGC_function_%T (struct sess *sp);\n", tl->t); + Fc(tl, 1, "static int\n"); + Fc(tl, 1, "VGC_function_%T (struct sess *sp)\n", tl->t); + vcc_NextToken(tl); + L(tl, Compound(tl)); + Fc(tl, 0, "\n"); +} + +/*-------------------------------------------------------------------- + * Top level of parser, recognize: + * Function definitions + * Backend definitions + * End of input + */ + +static void +Parse(struct tokenlist *tl) +{ + + while (tl->t->tok != EOI) { + ERRCHK(tl); + switch (tl->t->tok) { + case T_ACL: + Acl(tl); + break; + case T_SUB: + Function(tl); + break; + case T_BACKEND: + Backend(tl); + break; + case EOI: + break; + default: + sbuf_printf(tl->sb, + "Expected 'acl', 'sub' or 'backend', found "); + vcc_ErrToken(tl, tl->t); + sbuf_printf(tl->sb, " at\n"); + vcc_ErrWhere(tl, tl->t); + return; + } + } +} + +/*-------------------------------------------------------------------- + * Consistency check + */ + +static struct proc * +AddProc(struct tokenlist *tl, struct token *t, int def) +{ + struct proc *p; + + TAILQ_FOREACH(p, &tl->procs, list) { + if (!vcc_Teq(p->name, t)) + continue; + if (def) + p->name = t; + return (p); + } + p = calloc(sizeof *p, 1); + assert(p != NULL); + p->name = t; + TAILQ_INIT(&p->calls); + TAILQ_INSERT_TAIL(&tl->procs, p, list); + return (p); +} + +static void +AddCall(struct tokenlist *tl, struct token *t) +{ + struct proccall *pc; + struct proc *p; + + p = AddProc(tl, t, 0); + TAILQ_FOREACH(pc, &tl->curproc->calls, list) { + if (pc->p == p) + return; + } + pc = calloc(sizeof *pc, 1); + assert(pc != NULL); + pc->p = p; + pc->t = t; + TAILQ_INSERT_TAIL(&tl->curproc->calls, pc, list); +} + +static int +Consist_Decend(struct tokenlist *tl, struct proc *p, unsigned returns) +{ + unsigned u; + struct proccall *pc; + + if (!p->exists) { + sbuf_printf(tl->sb, "Function %T does not exist\n", p->name); + return (1); + } + if (p->active) { + sbuf_printf(tl->sb, "Function recurses on\n"); + vcc_ErrWhere(tl, p->name); + return (1); + } + u = p->returns & ~returns; + if (u) { +#define VCL_RET_MAC(a, b, c) \ + if (u & VCL_RET_##b) { \ + sbuf_printf(tl->sb, "Illegal return for method\n"); \ + vcc_ErrWhere(tl, p->returnt[c]); \ + } +#include "vcl_returns.h" +#undef VCL_RET_MAC + sbuf_printf(tl->sb, "In function\n"); + vcc_ErrWhere(tl, p->name); + return (1); + } + p->active = 1; + TAILQ_FOREACH(pc, &p->calls, list) { + if (Consist_Decend(tl, pc->p, returns)) { + sbuf_printf(tl->sb, "\nCalled from\n"); + vcc_ErrWhere(tl, p->name); + sbuf_printf(tl->sb, "at\n"); + vcc_ErrWhere(tl, pc->t); + return (1); + } + } + p->active = 0; + p->called++; + return (0); +} + +static int +Consistency(struct tokenlist *tl) +{ + struct proc *p; + struct method *m; + + TAILQ_FOREACH(p, &tl->procs, list) { + for(m = method_tab; m->name != NULL; m++) { + if (vcc_IdIs(p->name, m->defname)) + p->called = 1; + if (vcc_IdIs(p->name, m->name)) + break; + } + if (m->name == NULL) + continue; + if (Consist_Decend(tl, p, m->returns)) { + sbuf_printf(tl->sb, + "\nwhich is a %s method\n", m->name); + return (1); + } + } + TAILQ_FOREACH(p, &tl->procs, list) { + if (p->called) + continue; + sbuf_printf(tl->sb, "Function unused\n"); + vcc_ErrWhere(tl, p->name); + return (1); + } + return (0); +} + +/*--------------------------------------------------------------------*/ + +static int +CheckRefs(struct tokenlist *tl) +{ + struct ref *r; + const char *type; + int nerr = 0; + + TAILQ_FOREACH(r, &tl->refs, list) { + if (r->defcnt != 0 && r->refcnt != 0) + continue; + nerr++; + + switch(r->type) { + case R_FUNC: + type = "function"; + break; + case R_ACL: + type = "acl"; + break; + case R_BACKEND: + type = "backend"; + break; + default: + ErrInternal(tl); + sbuf_printf(tl->sb, "Ref "); + vcc_ErrToken(tl, r->name); + sbuf_printf(tl->sb, " has unknown type %d\n", + r->type); + continue; + } + if (r->defcnt == 0 && r->name->tok == METHOD) { + sbuf_printf(tl->sb, + "No definition for method %T\n", r->name); + continue; + } + + if (r->defcnt == 0) { + sbuf_printf(tl->sb, + "Undefined %s %T, first reference:\n", + type, r->name); + vcc_ErrWhere(tl, r->name); + continue; + } + + sbuf_printf(tl->sb, "Unused %s %T, defined:\n", type, r->name); + vcc_ErrWhere(tl, r->name); + } + return (nerr); +} + +/*--------------------------------------------------------------------*/ + +static void +LocTable(struct tokenlist *tl) +{ + struct token *t; + unsigned lin, pos; + const char *p; + + Fh(tl, 0, "#define VGC_NREFS %u\n", tl->cnt + 1); + Fh(tl, 0, "static struct vrt_ref VGC_ref[VGC_NREFS];\n"); + Fc(tl, 0, "static struct vrt_ref VGC_ref[VGC_NREFS] = {\n"); + lin = 1; + pos = 0; + p = tl->b; + TAILQ_FOREACH(t, &tl->tokens, list) { + if (t->cnt == 0) + continue; + for (;p < t->b; p++) { + if (*p == '\n') { + lin++; + pos = 0; + } else if (*p == '\t') { + pos &= ~7; + pos += 8; + } else + pos++; + + } + Fc(tl, 0, " [%3u] = { %4u, %3u, 0, \"%T\" },\n", + t->cnt, lin, pos + 1, t); + } + Fc(tl, 0, "};\n"); +} + + +/*--------------------------------------------------------------------*/ + +static void +EmitInitFunc(struct tokenlist *tl) +{ + struct ref *r; + + Fc(tl, 0, "\nstatic void\nVGC_Init(void)\n{\n\n"); + Fc(tl, 0, "\tVRT_alloc_backends(&VCL_conf);\n"); + + TAILQ_FOREACH(r, &tl->refs, list) { + switch(r->type) { + case R_FUNC: + break; + case R_ACL: + break; + case R_BACKEND: + Fc(tl, 0, "\tVGC_init_backend_%T();\n", r->name); + break; + } + } + Fc(tl, 0, "}\n"); +} + +/*--------------------------------------------------------------------*/ + +static void +EmitStruct(struct tokenlist *tl) +{ + + Fc(tl, 0, "\nstruct VCL_conf VCL_conf = {\n"); + Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n"); + Fc(tl, 0, "\t.init_func = VGC_Init,\n"); + Fc(tl, 0, "\t.nbackend = %d,\n", tl->nbackend); + Fc(tl, 0, "\t.ref = VGC_ref,\n"); + Fc(tl, 0, "\t.nref = VGC_NREFS,\n"); +#define VCL_RET_MAC(l,u,b) +#define VCL_MET_MAC(l,u,b) \ + if (FindRefStr(tl, "vcl_" #l, R_FUNC)) { \ + Fc(tl, 0, "\t." #l "_func = VGC_function_vcl_" #l ",\n"); \ + AddRefStr(tl, "vcl_" #l, R_FUNC); \ + } else { \ + Fc(tl, 0, "\t." #l "_func = VGC_function_default_vcl_" #l ",\n"); \ + } \ + AddRefStr(tl, "default_vcl_" #l, R_FUNC); +#include "vcl_returns.h" +#undef VCL_MET_MAC +#undef VCL_RET_MAC + Fc(tl, 0, "};\n"); +} + +/*--------------------------------------------------------------------*/ + +char * +VCC_Compile(struct sbuf *sb, const char *b, const char *e) +{ + struct tokenlist tokens; + struct ref *r; + struct token *t; + FILE *fo; + char *of = NULL; + char buf[BUFSIZ]; + + memset(&tokens, 0, sizeof tokens); + TAILQ_INIT(&tokens.tokens); + TAILQ_INIT(&tokens.refs); + TAILQ_INIT(&tokens.procs); + tokens.sb = sb; + + tokens.fc = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); + assert(tokens.fc != NULL); + + tokens.fh = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); + assert(tokens.fh != NULL); + + Fh(&tokens, 0, "extern struct VCL_conf VCL_conf;\n"); + + tokens.b = b; + if (e == NULL) + e = strchr(b, '\0'); + assert(e != NULL); + tokens.e = e; + vcc_Lexer(&tokens, b, e); + vcc_Lexer(&tokens, vcc_default_vcl_b, vcc_default_vcl_e); + vcc_AddToken(&tokens, EOI, e, e); + if (tokens.err) + goto done; + tokens.t = TAILQ_FIRST(&tokens.tokens); + Parse(&tokens); + if (tokens.err) + goto done; + Consistency(&tokens); + if (tokens.err) + goto done; + LocTable(&tokens); + + EmitInitFunc(&tokens); + + EmitStruct(&tokens); + + if (CheckRefs(&tokens)) + goto done; + + of = strdup("/tmp/vcl.XXXXXXXX"); + assert(of != NULL); + mktemp(of); + + sprintf(buf, + "tee /tmp/_.c |" + "cc -fpic -shared -Wl,-x -o %s -x c - ", of); + + fo = popen(buf, "w"); + assert(fo != NULL); + + vcl_output_lang_h(fo); + + sbuf_finish(tokens.fh); + fputs(sbuf_data(tokens.fh), fo); + sbuf_delete(tokens.fh); + + sbuf_finish(tokens.fc); + fputs(sbuf_data(tokens.fc), fo); + sbuf_delete(tokens.fc); + + pclose(fo); +done: + + /* Free References */ + while (!TAILQ_EMPTY(&tokens.refs)) { + r = TAILQ_FIRST(&tokens.refs); + TAILQ_REMOVE(&tokens.refs, r, list); + free(r); + } + + /* Free Tokens */ + while (!TAILQ_EMPTY(&tokens.tokens)) { + t = TAILQ_FIRST(&tokens.tokens); + TAILQ_REMOVE(&tokens.tokens, t, list); + free(t); + } + return (of); +} + +/*--------------------------------------------------------------------*/ + +char * +VCC_CompileFile(struct sbuf *sb, const char *fn) +{ + char *f, *r; + int fd, i; + struct stat st; + + fd = open(fn, O_RDONLY); + if (fd < 0) { + sbuf_printf(sb, "Cannot open file '%s': %s", + fn, strerror(errno)); + return (NULL); + } + assert(0 == fstat(fd, &st)); + f = malloc(st.st_size + 1); + assert(f != NULL); + i = read(fd, f, st.st_size); + assert(i == st.st_size); + f[i] = '\0'; + r = VCC_Compile(sb, f, NULL); + free(f); + return (r); +} + +/*--------------------------------------------------------------------*/ + +static int +VCC_T_render(FILE *f, const struct printf_info *info __unused, const void *const *args) +{ + const struct token *t; + + t = *((const struct token * const*) (args[0])); + return (fprintf(f, "%*.*s", + t->e - t->b, t->e - t->b, t->b)); +} + +static int +VCC_T_arginfo(const struct printf_info *info __unused, size_t n, int *argtypes) +{ + + if (n > 0) + argtypes[0] = PA_POINTER; + return 1; +} + +/*--------------------------------------------------------------------*/ + +void +VCC_InitCompile(const char *default_vcl) +{ + struct var *v; + + vcc_default_vcl_b = default_vcl; + vcc_default_vcl_e = strchr(default_vcl, '\0'); + assert(vcc_default_vcl_e != NULL); + + register_printf_function ('T', VCC_T_render, VCC_T_arginfo); + vcl_init_tnames(); + for (v = vcc_vars; v->name != NULL; v++) + v->len = strlen(v->name); + for (v = vcc_be_vars; v->name != NULL; v++) + v->len = strlen(v->name); +} Added: trunk/varnish-cache/lib/libvcl/vcc_compile.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.h 2006-07-21 16:25:24 UTC (rev 543) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.h 2006-07-21 18:12:56 UTC (rev 544) @@ -0,0 +1,120 @@ +/* + * $Id$ + */ + +#include "queue.h" +#include "vcl_returns.h" + +#define INDENT 2 + +struct token { + unsigned tok; + const char *b; + const char *e; + TAILQ_ENTRY(token) list; + unsigned cnt; +}; + +struct tokenlist { + TAILQ_HEAD(, token) tokens; + const char *b; + const char *e; + struct token *t; + int indent; + unsigned cnt; + struct sbuf *fc, *fh; + TAILQ_HEAD(, ref) refs; + struct sbuf *sb; + int err; + int nbackend; + TAILQ_HEAD(, proc) procs; + struct proc *curproc; +}; + +enum var_type { + BACKEND, + BOOL, + INT, + FLOAT, + SIZE, + RATE, + TIME, + STRING, + IP, + HOSTNAME, + PORTNAME, + HEADER +}; + +enum ref_type { + R_FUNC, + R_ACL, + R_BACKEND +}; + +struct ref { + enum ref_type type; + struct token *name; + unsigned defcnt; + unsigned refcnt; + TAILQ_ENTRY(ref) list; +}; + +struct var { + const char *name; + enum var_type fmt; + int len; + const char *rname; + const char *lname; +}; + +static struct method { + const char *name; + const char *defname; + unsigned returns; +} method_tab[] = { +#define VCL_RET_MAC(a,b,c) +#define VCL_MET_MAC(a,b,c) { "vcl_"#a, "default_vcl_"#a, c }, +#include "vcl_returns.h" +#undef VCL_MET_MAC +#undef VCL_RET_MAC + { NULL, 0U } +}; + +struct proccall { + TAILQ_ENTRY(proccall) list; + struct proc *p; + struct token *t; +}; + +struct proc { + TAILQ_ENTRY(proc) list; + TAILQ_HEAD(,proccall) calls; + struct token *name; + unsigned returns; + unsigned exists; + unsigned called; + unsigned active; + struct token *returnt[VCL_RET_MAX]; +}; + + +/*--------------------------------------------------------------------*/ + +/* vcc_compile.c */ +extern const char *vcc_default_vcl_b, *vcc_default_vcl_e; + +/* vcc_obj.c */ +extern struct var vcc_be_vars[]; +extern struct var vcc_vars[]; + +/* vcc_token.c */ +void vcc_ErrToken(struct tokenlist *tl, struct token *t); +void vcc_ErrWhere(struct tokenlist *tl, struct token *t); +void vcc__Expect(struct tokenlist *tl, unsigned tok, int line); +int vcc_Teq(struct token *t1, struct token *t2); +int vcc_IdIs(struct token *t, const char *p); +void vcc_Lexer(struct tokenlist *tl, const char *b, const char *e); +void vcc_NextToken(struct tokenlist *tl); +void vcc__ErrInternal(struct tokenlist *tl, const char *func, unsigned line); +void vcc_AddToken(struct tokenlist *tl, unsigned tok, const char *b, const char *e); Copied: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c (from rev 480, trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c) =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-07-14 13:54:41 UTC (rev 480) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2006-07-21 18:12:56 UTC (rev 544) @@ -0,0 +1,518 @@ +/* + * $Id$ + * + * NB: This file is machine generated, DO NOT EDIT! + * + * Edit vcc_gen_fixed_token.tcl instead + */ + +#include +#include +#include "vcc_priv.h" + +unsigned +vcl_fixed_token(const char *p, const char **q) +{ + + switch (p[0]) { + case '!': + if (p[0] == '!' && p[1] == '=') { + *q = p + 2; + return (T_NEQ); + } + if (p[0] == '!') { + *q = p + 1; + return ('!'); + } + return (0); + case '%': + if (p[0] == '%') { + *q = p + 1; + return ('%'); + } + return (0); + case '&': + if (p[0] == '&' && p[1] == '&') { + *q = p + 2; + return (T_CAND); + } + if (p[0] == '&') { + *q = p + 1; + return ('&'); + } + return (0); + case '(': + if (p[0] == '(') { + *q = p + 1; + return ('('); + } + return (0); + case ')': + if (p[0] == ')') { + *q = p + 1; + return (')'); + } + return (0); + case '*': + if (p[0] == '*' && p[1] == '=') { + *q = p + 2; + return (T_MUL); + } + if (p[0] == '*') { + *q = p + 1; + return ('*'); + } + return (0); + case '+': + if (p[0] == '+' && p[1] == '=') { + *q = p + 2; + return (T_INCR); + } + if (p[0] == '+' && p[1] == '+') { + *q = p + 2; + return (T_INC); + } + if (p[0] == '+') { + *q = p + 1; + return ('+'); + } + return (0); + case ',': + if (p[0] == ',') { + *q = p + 1; + return (','); + } + return (0); + case '-': + if (p[0] == '-' && p[1] == '-') { + *q = p + 2; + return (T_DEC); + } + if (p[0] == '-') { + *q = p + 1; + return ('-'); + } + return (0); + case '.': + if (p[0] == '.') { + *q = p + 1; + return ('.'); + } + return (0); + case '/': + if (p[0] == '/' && p[1] == '=') { + *q = p + 2; + return (T_DECR); + } + if (p[0] == '/' && p[1] == '=') { + *q = p + 2; + return (T_DIV); + } + if (p[0] == '/') { + *q = p + 1; + return ('/'); + } + return (0); + case ';': + if (p[0] == ';') { + *q = p + 1; + return (';'); + } + return (0); + case '<': + if (p[0] == '<' && p[1] == '=') { + *q = p + 2; + return (T_LEQ); + } + if (p[0] == '<' && p[1] == '<') { + *q = p + 2; + return (T_SHL); + } + if (p[0] == '<') { + *q = p + 1; + return ('<'); + } + return (0); + case '=': + if (p[0] == '=' && p[1] == '=') { + *q = p + 2; + return (T_EQ); + } + if (p[0] == '=') { + *q = p + 1; + return ('='); + } + return (0); + case '>': + if (p[0] == '>' && p[1] == '>') { + *q = p + 2; + return (T_SHR); + } + if (p[0] == '>' && p[1] == '=') { + *q = p + 2; + return (T_GEQ); + } + if (p[0] == '>') { + *q = p + 1; + return ('>'); + } + return (0); + case 'a': + if (p[0] == 'a' && p[1] == 'c' && p[2] == 'l' + && !isvar(p[3])) { + *q = p + 3; + return (T_ACL); + } + return (0); + case 'b': + if (p[0] == 'b' && p[1] == 'a' && p[2] == 'c' && + p[3] == 'k' && p[4] == 'e' && p[5] == 'n' && + p[6] == 'd' && !isvar(p[7])) { + *q = p + 7; + return (T_BACKEND); + } + return (0); + case 'c': + if (p[0] == 'c' && p[1] == 'a' && p[2] == 'l' && + p[3] == 'l' && !isvar(p[4])) { + *q = p + 4; + return (T_CALL); + } + return (0); + case 'd': + if (p[0] == 'd' && p[1] == 'i' && p[2] == 's' && + p[3] == 'c' && p[4] == 'a' && p[5] == 'r' && + p[6] == 'd' && !isvar(p[7])) { + *q = p + 7; + return (T_DISCARD); + } + if (p[0] == 'd' && p[1] == 'e' && p[2] == 'l' && + p[3] == 'i' && p[4] == 'v' && p[5] == 'e' && + p[6] == 'r' && !isvar(p[7])) { + *q = p + 7; + return (T_DELIVER); + } + return (0); + case 'e': + if (p[0] == 'e' && p[1] == 'r' && p[2] == 'r' && + p[3] == 'o' && p[4] == 'r' && !isvar(p[5])) { + *q = p + 5; + return (T_ERROR); + } + if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && + p[3] == 'i' && p[4] == 'f' && !isvar(p[5])) { + *q = p + 5; + return (T_ELSIF); + } + if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && + p[3] == 'e' && p[4] == 'i' && p[5] == 'f' + && !isvar(p[6])) { + *q = p + 6; + return (T_ELSEIF); + } + if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && + p[3] == 'e' && !isvar(p[4])) { + *q = p + 4; + return (T_ELSE); + } + return (0); + case 'f': + if (p[0] == 'f' && p[1] == 'u' && p[2] == 'n' && + p[3] == 'c' && !isvar(p[4])) { + *q = p + 4; + return (T_FUNC); + } + if (p[0] == 'f' && p[1] == 'e' && p[2] == 't' && + p[3] == 'c' && p[4] == 'h' && !isvar(p[5])) { + *q = p + 5; + return (T_FETCH); + } + return (0); + case 'i': + if (p[0] == 'i' && p[1] == 'n' && p[2] == 's' && + p[3] == 'e' && p[4] == 'r' && p[5] == 't' && + p[6] == '_' && p[7] == 'p' && p[8] == 'a' && + p[9] == 's' && p[10] == 's' && !isvar(p[11])) { + *q = p + 11; + return (T_INSERT_PASS); + } + if (p[0] == 'i' && p[1] == 'n' && p[2] == 's' && + p[3] == 'e' && p[4] == 'r' && p[5] == 't' + && !isvar(p[6])) { + *q = p + 6; + return (T_INSERT); + } + if (p[0] == 'i' && p[1] == 'f' && !isvar(p[2])) { + *q = p + 2; + return (T_IF); + } + return (0); + case 'l': + if (p[0] == 'l' && p[1] == 'o' && p[2] == 'o' && + p[3] == 'k' && p[4] == 'u' && p[5] == 'p' + && !isvar(p[6])) { + *q = p + 6; + return (T_LOOKUP); + } + return (0); + case 'n': + if (p[0] == 'n' && p[1] == 'o' && p[2] == '_' && + p[3] == 'n' && p[4] == 'e' && p[5] == 'w' && + p[6] == '_' && p[7] == 'c' && p[8] == 'a' && + p[9] == 'c' && p[10] == 'h' && p[11] == 'e' + && !isvar(p[12])) { + *q = p + 12; + return (T_NO_NEW_CACHE); + } + if (p[0] == 'n' && p[1] == 'o' && p[2] == '_' && + p[3] == 'c' && p[4] == 'a' && p[5] == 'c' && + p[6] == 'h' && p[7] == 'e' && !isvar(p[8])) { + *q = p + 8; + return (T_NO_CACHE); + } + return (0); + case 'p': + if (p[0] == 'p' && p[1] == 'r' && p[2] == 'o' && + p[3] == 'c' && !isvar(p[4])) { + *q = p + 4; + return (T_PROC); + } + if (p[0] == 'p' && p[1] == 'i' && p[2] == 'p' && + p[3] == 'e' && !isvar(p[4])) { + *q = p + 4; + return (T_PIPE); + } + if (p[0] == 'p' && p[1] == 'a' && p[2] == 's' && + p[3] == 's' && !isvar(p[4])) { + *q = p + 4; + return (T_PASS); + } + return (0); + case 'r': + if (p[0] == 'r' && p[1] == 'e' && p[2] == 'w' && + p[3] == 'r' && p[4] == 'i' && p[5] == 't' && + p[6] == 'e' && !isvar(p[7])) { + *q = p + 7; + return (T_REWRITE); + } + return (0); + case 's': + if (p[0] == 's' && p[1] == 'w' && p[2] == 'i' && + p[3] == 't' && p[4] == 'c' && p[5] == 'h' && + p[6] == '_' && p[7] == 'c' && p[8] == 'o' && + p[9] == 'n' && p[10] == 'f' && p[11] == 'i' && + p[12] == 'g' && !isvar(p[13])) { + *q = p + 13; + return (T_SWITCH_CONFIG); + } + if (p[0] == 's' && p[1] == 'u' && p[2] == 'b' + && !isvar(p[3])) { + *q = p + 3; + return (T_SUB); + } + if (p[0] == 's' && p[1] == 'e' && p[2] == 't' + && !isvar(p[3])) { + *q = p + 3; + return (T_SET); + } + return (0); + case '{': + if (p[0] == '{') { + *q = p + 1; + return ('{'); + } + return (0); + case '|': + if (p[0] == '|' && p[1] == '|') { + *q = p + 2; + return (T_COR); + } + if (p[0] == '|') { + *q = p + 1; + return ('|'); + } + return (0); + case '}': + if (p[0] == '}') { + *q = p + 1; + return ('}'); + } + return (0); + case '~': + if (p[0] == '~') { + *q = p + 1; + return ('~'); + } + return (0); + default: + return (0); + } +} + +const char *vcl_tnames[256]; + +void +vcl_init_tnames(void) +{ + vcl_tnames['!'] = "'!'"; + vcl_tnames['%'] = "'%'"; + vcl_tnames['&'] = "'&'"; + vcl_tnames['('] = "'('"; + vcl_tnames[')'] = "')'"; + vcl_tnames['*'] = "'*'"; + vcl_tnames['+'] = "'+'"; + vcl_tnames[','] = "','"; + vcl_tnames['-'] = "'-'"; + vcl_tnames['.'] = "'.'"; + vcl_tnames['/'] = "'/'"; + vcl_tnames['<'] = "'<'"; + vcl_tnames['='] = "'='"; + vcl_tnames['>'] = "'>'"; + vcl_tnames['{'] = "'{'"; + vcl_tnames['}'] = "'}'"; + vcl_tnames['|'] = "'|'"; + vcl_tnames['~'] = "'~'"; + vcl_tnames[';'] = "';'"; + vcl_tnames[CNUM] = "CNUM"; + vcl_tnames[CSTR] = "CSTR"; + vcl_tnames[EOI] = "EOI"; + vcl_tnames[ID] = "ID"; + vcl_tnames[METHOD] = "METHOD"; + vcl_tnames[T_ACL] = "acl"; + vcl_tnames[T_BACKEND] = "backend"; + vcl_tnames[T_CALL] = "call"; + vcl_tnames[T_CAND] = "&&"; + vcl_tnames[T_COR] = "||"; + vcl_tnames[T_DEC] = "--"; + vcl_tnames[T_DECR] = "/="; + vcl_tnames[T_DELIVER] = "deliver"; + vcl_tnames[T_DISCARD] = "discard"; + vcl_tnames[T_DIV] = "/="; + vcl_tnames[T_ELSE] = "else"; + vcl_tnames[T_ELSEIF] = "elseif"; + vcl_tnames[T_ELSIF] = "elsif"; + vcl_tnames[T_EQ] = "=="; + vcl_tnames[T_ERROR] = "error"; + vcl_tnames[T_FETCH] = "fetch"; + vcl_tnames[T_FUNC] = "func"; + vcl_tnames[T_GEQ] = ">="; + vcl_tnames[T_IF] = "if"; + vcl_tnames[T_INC] = "++"; + vcl_tnames[T_INCR] = "+="; + vcl_tnames[T_INSERT] = "insert"; + vcl_tnames[T_INSERT_PASS] = "insert_pass"; + vcl_tnames[T_LEQ] = "<="; + vcl_tnames[T_LOOKUP] = "lookup"; + vcl_tnames[T_MUL] = "*="; + vcl_tnames[T_NEQ] = "!="; + vcl_tnames[T_NO_CACHE] = "no_cache"; + vcl_tnames[T_NO_NEW_CACHE] = "no_new_cache"; + vcl_tnames[T_PASS] = "pass"; + vcl_tnames[T_PIPE] = "pipe"; + vcl_tnames[T_PROC] = "proc"; + vcl_tnames[T_REWRITE] = "rewrite"; + vcl_tnames[T_SET] = "set"; + vcl_tnames[T_SHL] = "<<"; + vcl_tnames[T_SHR] = ">>"; + vcl_tnames[T_SUB] = "sub"; + vcl_tnames[T_SWITCH_CONFIG] = "switch_config"; + vcl_tnames[VAR] = "VAR"; +} + +void +vcl_output_lang_h(FILE *f) +{ + fputs("#define VCL_RET_ERROR (1 << 0)\n", f); + fputs("#define VCL_RET_LOOKUP (1 << 1)\n", f); + fputs("#define VCL_RET_PIPE (1 << 2)\n", f); + fputs("#define VCL_RET_PASS (1 << 3)\n", f); + fputs("#define VCL_RET_INSERT_PASS (1 << 4)\n", f); + fputs("#define VCL_RET_FETCH (1 << 5)\n", f); + fputs("#define VCL_RET_INSERT (1 << 6)\n", f); + fputs("#define VCL_RET_DELIVER (1 << 7)\n", f); + fputs("#define VCL_RET_DISCARD (1 << 8)\n", f); + fputs("/*\n", f); + fputs(" * $Id$\n", f); + fputs(" *\n", f); + fputs(" * NB: This file is machine generated, DO NOT EDIT!\n", f); + fputs(" *\n", f); + fputs(" * Edit vcc_gen_fixed_token.tcl instead\n", f); + fputs(" */\n", f); + fputs("\n", f); + fputs("struct sess;\n", f); + fputs("\n", f); + fputs("typedef void vcl_init_f(void);\n", f); + fputs("typedef int vcl_func_f(struct sess *sp);\n", f); + fputs("\n", f); + fputs("struct VCL_conf {\n", f); + fputs(" unsigned magic;\n", f); + fputs("#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */\n", f); + fputs("\n", f); + fputs(" struct backend **backend;\n", f); + fputs(" unsigned nbackend;\n", f); + fputs(" struct vrt_ref *ref;\n", f); + fputs(" unsigned nref;\n", f); + fputs(" unsigned busy;\n", f); + fputs("\n", f); + fputs(" vcl_init_f *init_func;\n", f); + fputs("\n", f); + fputs(" vcl_func_f *recv_func;\n", f); + fputs(" vcl_func_f *miss_func;\n", f); + fputs(" vcl_func_f *hit_func;\n", f); + fputs(" vcl_func_f *fetch_func;\n", f); + fputs(" vcl_func_f *timeout_func;\n", f); + fputs("};\n", f); + fputs("/*\n", f); + fputs(" * $Id$ \n", f); + fputs(" *\n", f); + fputs(" * Runtime support for compiled VCL programs.\n", f); + fputs(" *\n", f); + fputs(" * XXX: When this file is changed, lib/libvcl/vcl_gen_fixed_token.tcl\n", f); + fputs(" * XXX: *MUST* be rerun.\n", f); + fputs(" */\n", f); + fputs("\n", f); + fputs("struct sess;\n", f); + fputs("struct backend;\n", f); + fputs("struct VCL_conf;\n", f); + fputs("\n", f); + fputs("struct vrt_ref {\n", f); + fputs(" unsigned line;\n", f); + fputs(" unsigned pos;\n", f); + fputs(" unsigned count;\n", f); + fputs(" const char *token;\n", f); + fputs("};\n", f); + fputs("\n", f); + fputs("struct vrt_acl {\n", f); + fputs(" unsigned ip;\n", f); + fputs(" unsigned mask;\n", f); + fputs("};\n", f); + fputs("\n", f); + fputs("void VRT_count(struct sess *, unsigned);\n", f); + fputs("void VRT_no_cache(struct sess *);\n", f); + fputs("void VRT_no_new_cache(struct sess *);\n", f); + fputs("#if 0\n", f); + fputs("int ip_match(unsigned, struct vcl_acl *);\n", f); + fputs("int string_match(const char *, const char *);\n", f); + fputs("#endif\n", f); + fputs("int VRT_rewrite(const char *, const char *);\n", f); + fputs("void VRT_error(struct sess *, unsigned, const char *);\n", f); + fputs("int VRT_switch_config(const char *);\n", f); + fputs("\n", f); + fputs("char *VRT_GetHdr(struct sess *, const char *);\n", f); + fputs("char *VRT_GetReq(struct sess *);\n", f); + fputs("void VRT_handling(struct sess *sp, unsigned hand);\n", f); + fputs("int VRT_obj_valid(struct sess *);\n", f); + fputs("int VRT_obj_cacheable(struct sess *);\n", f); + fputs("\n", f); + fputs("void VRT_set_backend_name(struct backend *, const char *);\n", f); + fputs("void VRT_set_backend_hostname(struct backend *, const char *);\n", f); + fputs("void VRT_set_backend_portname(struct backend *, const char *);\n", f); + fputs("\n", f); + fputs("void VRT_alloc_backends(struct VCL_conf *cp);\n", f); + fputs("\n", f); + fputs("#define VRT_done(sp, hand) \\\n", f); + fputs(" do { \\\n", f); + fputs(" VRT_handling(sp, hand); \\\n", f); + fputs(" return (1); \\\n", f); + fputs(" } while (0)\n", f); +} Copied: trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl (from rev 480, trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl) =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-07-14 13:54:41 UTC (rev 480) +++ trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2006-07-21 18:12:56 UTC (rev 544) @@ -0,0 +1,319 @@ +#!/usr/local/bin/tclsh8.4 +# +# Generate various .c and .h files for the VCL compiler and the interfaces +# for it. + +# These are the metods which can be called in the VCL program. +# Second element is list of valid return actions. +# +set methods { + {recv {error pass pipe lookup}} + {miss {error pass pipe fetch}} + {hit {error pass pipe deliver}} + {fetch {error pass pipe insert insert_pass}} + {timeout {fetch discard}} +} + +# These are the return actions +# +set returns { + error + lookup + pipe + pass + insert_pass + fetch + insert + deliver + discard +} + +# Language keywords +# +set keywords { + if else elseif elsif + + func proc sub + + acl + + backend + + call + no_cache + no_new_cache + set + rewrite + switch_config +} + +# Non-word tokens +# +set magic { + {"++" INC} + {"--" DEC} + {"&&" CAND} + {"||" COR} + {"<=" LEQ} + {"==" EQ} + {"!=" NEQ} + {">=" GEQ} + {">>" SHR} + {"<<" SHL} + {"+=" INCR} + {"/=" DECR} + {"*=" MUL} + {"/=" DIV} +} + +# Single char tokens +# +set char {{}()*+-/%><=;!&.|~,} + +# Other token identifiers +# +set extras {ID VAR CNUM CSTR EOI METHOD} + +#---------------------------------------------------------------------- +# Boilerplate warning for all generated files. + +proc warns {fd} { + + puts $fd "/*" + puts $fd { * $Id$} + puts $fd " *" + puts $fd " * NB: This file is machine generated, DO NOT EDIT!" + puts $fd " *" + puts $fd " * Edit vcc_gen_fixed_token.tcl instead" + puts $fd " */" + puts $fd "" +} + +#---------------------------------------------------------------------- +# Build the vcl.h #include file + +set fo [open ../../include/vcl.h w] +warns $fo +puts $fo {struct sess; + +typedef void vcl_init_f(void); +typedef int vcl_func_f(struct sess *sp); +} +puts $fo "struct VCL_conf {" +puts $fo { unsigned magic; +#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */ + + struct backend **backend; + unsigned nbackend; + struct vrt_ref *ref; + unsigned nref; + unsigned busy; + + vcl_init_f *init_func; +} +foreach m $methods { + puts $fo "\tvcl_func_f\t*[lindex $m 0]_func;" +} +puts $fo "};" + +close $fo + +#---------------------------------------------------------------------- +# Build the vcl_returns.h #include file + +set for [open "../../include/vcl_returns.h" w] +warns $for +puts $for "#ifdef VCL_RET_MAC" +set i 0 +foreach k $returns { + if {$k == "error"} { + puts $for "#ifdef VCL_RET_MAC_E" + puts $for "VCL_RET_MAC_E($k, [string toupper $k], $i)" + puts $for "#endif" + } else { + puts $for "VCL_RET_MAC($k, [string toupper $k], (1 << $i))" + } + incr i +} +puts $for "#else" +set i 0 +foreach k $returns { + puts $for "#define VCL_RET_[string toupper $k] (1 << $i)" + incr i +} +puts $for "#define VCL_RET_MAX $i" +puts $for "#endif" +puts $for "" +puts $for "#ifdef VCL_MET_MAC" +foreach m $methods { + puts -nonewline $for "VCL_MET_MAC([lindex $m 0]" + puts -nonewline $for ",[string toupper [lindex $m 0]]" + set l [lindex $m 1] + puts -nonewline $for ",(VCL_RET_[string toupper [lindex $l 0]]" + foreach r [lrange $l 1 end] { + puts -nonewline $for "|VCL_RET_[string toupper $r]" + } + puts -nonewline $for ")" + puts $for ")" +} +puts $for "#endif" +close $for + +#---------------------------------------------------------------------- +# Build the compiler token table and recognizers + +set fo [open "vcc_fixed_token.c" w] +warns $fo + +set foh [open "vcc_token_defs.h" w] +warns $foh + +puts $fo "#include " +puts $fo "#include " +puts $fo "#include \"vcc_priv.h\"" + +set tn 128 +puts $foh "#define LOW_TOKEN $tn" + + +proc add_token {tok str alpha} { + global tokens tn fixed foh + + lappend tokens [list $tok $str] + puts $foh "#define $tok $tn" + incr tn + lappend fixed [list $str $tok $alpha] +} + +proc mk_token {tok str alpha} { + set tok T_[string toupper $tok] + add_token $tok $str $alpha +} + +foreach k $keywords { mk_token $k $k 1 } +foreach k $returns { mk_token $k $k 1 } +foreach k $magic { mk_token [lindex $k 1] [lindex $k 0] 0 } +foreach k $extras { + set t [string toupper $k] + lappend tokens [list $t $t] + puts $foh "#define [string toupper $k] $tn" + incr tn +} +for {set i 0} {$i < [string length $char]} {incr i} { + set t [string index $char $i] + lappend token2 [list '$t' T$t] + lappend fixed [list "$t" '$t' 0] +} + +set tokens [lsort $tokens] +set token2 [lsort $token2] + +# We want to output in ascii order: create sorted first char list +foreach t $fixed { + set xx([string index [lindex $t 0] 0]) 1 +} +set seq [lsort [array names xx]] + +set ll 0 + +puts $fo { +unsigned +vcl_fixed_token(const char *p, const char **q)} +puts $fo "{" +puts $fo "" +puts $fo " switch (p\[0\]) {" + +foreach ch "$seq" { + # Now find all tokens starting with ch + set l "" + foreach t $fixed { + if {[string index [lindex $t 0] 0] == $ch} { + lappend l $t + } + } + # And do then in reverse order to match longest first + set l [lsort -index 0 -decreasing $l] + scan "$ch" "%c" cx + if {$cx != $ll} { + if {$ll} { + puts $fo " return (0);" + } + + puts $fo " case '$ch':" + set ll $cx + } + foreach tt $l { + set k [lindex $tt 0] + puts -nonewline $fo " if (" + for {set i 0} {$i < [string length $k]} {incr i} { + if {$i > 0} { + puts -nonewline $fo " && " + if {![expr $i % 3]} { + puts -nonewline $fo "\n\t\t " + } + } + puts -nonewline $fo "p\[$i\] == '[string index $k $i]'" + } + if {[lindex $tt 2]} { + if {![expr $i % 3]} { + puts -nonewline $fo "\n\t\t " + } + puts -nonewline $fo " && !isvar(p\[$i\])" + } + puts $fo ") {" + puts $fo " *q = p + [string length $k];" + puts $fo " return ([lindex $tt 1]);" + puts $fo " }" + } +} +puts $fo " return (0);" +puts $fo " default:" +puts $fo " return (0);" +puts $fo " }" +puts $fo "}" + +puts $fo "" +puts $fo "const char *vcl_tnames\[256\];\n" +puts $fo "void" +puts $fo "vcl_init_tnames(void)" +puts $fo "{" +foreach i $token2 { + puts $fo "\tvcl_tnames\[[lindex $i 0]\] = \"[lindex $i 0]\";" +} +foreach i $tokens { + puts $fo "\tvcl_tnames\[[lindex $i 0]\] = \"[lindex $i 1]\";" +} +puts $fo "}" + +#---------------------------------------------------------------------- +# Create the C-code which emits the boilerplate definitions for the +# generated C code output + +proc copy_include {n} { + global fo + + set fi [open $n] + while {[gets $fi a] >= 0} { + regsub -all {\\} $a {\\\\} a + puts $fo "\tfputs(\"$a\\n\", f);" + } + close $fi +} + +puts $fo "" +puts $fo "void" +puts $fo "vcl_output_lang_h(FILE *f)" +puts $fo "{" +set i 0 +foreach k $returns { + puts $fo "\tfputs(\"#define VCL_RET_[string toupper $k] (1 << $i)\\n\", f);" + incr i +} + +copy_include ../../include/vcl.h +copy_include ../../include/vrt.h + +puts $fo "}" + +close $foh +close $fo Added: trunk/varnish-cache/lib/libvcl/vcc_obj.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_obj.c 2006-07-21 16:25:24 UTC (rev 543) +++ trunk/varnish-cache/lib/libvcl/vcc_obj.c 2006-07-21 18:12:56 UTC (rev 544) @@ -0,0 +1,1843 @@ +/* + * $Id$ + */ + +#include + +#include "vcc_compile.h" + +struct var vcc_be_vars[] = { + { "backend.host", + HOSTNAME, 0, NULL, "VRT_set_backend_hostname(backend, %s)" }, + { "backend.port", + PORTNAME, 0, NULL, "VRT_set_backend_portname(backend, %s)" }, +}; + + +struct var vcc_vars[] = { + { "req.request", STRING, 0, "VRT_GetReq(sp)" }, + { "obj.valid", BOOL, 0, "VRT_obj_valid(sp)" }, + { "obj.cacheable", BOOL, 0, "VRT_obj_cacheable(sp)" }, + { "obj.backend", BACKEND, 0, "VRT_obj_backend(sp)" }, + { "req.http.", HEADER, 0, NULL }, +#if 0 + { "req.ttlfactor", FLOAT, 0, "req->ttlfactor" }, + { "req.url.host", STRING, 0, "req->url.host" }, + { "req.url.path", STRING, 0, "req->url.path" }, + { "req.useragent", STRING, 0, "req->useragent" }, + { "req.backend", BACKEND, 0, "req->backend" }, + { "client.ip", IP, 0, "client->ip" }, + { "backend.response_time", TIME, 0, "backend->responsetime" }, + { "backend.down", BOOL, 0, "backend->down" }, + { "backend.timeout", TIME, 0, "backend->timeout" }, + { "backend.bandwidth", RATE, 0, "backend->bandwidth" }, + { "obj.exist", BOOL, 0, "obj->exists" }, + { "obj.ttl", TIME, 0, "obj->ttl" }, + { "obj.result", INT, 0, "obj->result" }, + { "obj.size", SIZE, 0, "obj->size" }, + { "obj.usage", INT, 0, "obj->usage" }, +#endif + { NULL, INT, 0, "NULL" } +}; + + +#if 0 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vcl_priv.h" +#include "vcl_returns.h" +#include "vcl_compile.h" + +#include "libvcl.h" + +#define ERRCHK(tl) do { if ((tl)->err) return; } while (0) + +#define INDENT 2 + +/*-------------------------------------------------------------------- + * Consistency check + */ + +static struct method { + const char *name; + const char *defname; + unsigned returns; +} method_tab[] = { +#define VCL_RET_MAC(a,b,c) +#define VCL_MET_MAC(a,b,c) { "vcl_"#a, "default_vcl_"#a, c }, +#include "vcl_returns.h" +#undef VCL_MET_MAC +#undef VCL_RET_MAC + { NULL, 0U } +}; + +struct proccall { + TAILQ_ENTRY(proccall) list; + struct proc *p; + struct token *t; +}; + +struct proc { + TAILQ_ENTRY(proc) list; + TAILQ_HEAD(,proccall) calls; + struct token *name; + unsigned returns; + unsigned exists; + unsigned called; + unsigned active; + struct token *returnt[VCL_RET_MAX]; +}; + +/*--------------------------------------------------------------------*/ + + +static void Compound(struct tokenlist *tl); +static void Cond_0(struct tokenlist *tl); +static struct proc *AddProc(struct tokenlist *tl, struct token *t, int def); +static void AddCall(struct tokenlist *tl, struct token *t); +static const char *vcc_default_vcl_b, *vcc_default_vcl_e; + +/*--------------------------------------------------------------------*/ + +static void +ErrToken(struct tokenlist *tl, struct token *t) +{ + + if (t->tok == EOI) + sbuf_printf(tl->sb, "end of input"); + else + sbuf_printf(tl->sb, "'%T'", t); +} + +static void +_ErrInternal(struct tokenlist *tl, const char *func, unsigned line) +{ + + sbuf_printf(tl->sb, "VCL compiler internal error at %s():%u\n", + func, line); + tl->err = 1; +} + +#define ErrInternal(tl) _ErrInternal(tl, __func__, __LINE__) + +static void +ErrWhere(struct tokenlist *tl, struct token *t) +{ + unsigned lin, pos, x, y; + const char *p, *l, *f, *b, *e; + + lin = 1; + pos = 0; + if (t->tok == METHOD) + return; + if (t->b >= vcc_default_vcl_b && t->b < vcc_default_vcl_e) { + f = "Default VCL code (compiled in)"; + b = vcc_default_vcl_b; + e = vcc_default_vcl_e; + } else { + f = "VCL code"; + b = tl->b; + e = tl->e; + } + for (l = p = b; p < t->b; p++) { + if (*p == '\n') { + lin++; + pos = 0; + l = p + 1; + } else if (*p == '\t') { + pos &= ~7; + pos += 8; + } else + pos++; + } + sbuf_printf(tl->sb, "In %s Line %d Pos %d\n", f, lin, pos); + x = y = 0; + for (p = l; p < e && *p != '\n'; p++) { + if (*p == '\t') { + y &= ~7; + y += 8; + while (x < y) { + sbuf_bcat(tl->sb, " ", 1); + x++; + } + } else { + x++; + y++; + sbuf_bcat(tl->sb, p, 1); + } + } + sbuf_cat(tl->sb, "\n"); + x = y = 0; + for (p = l; p < e && *p != '\n'; p++) { + if (p >= t->b && p < t->e) { + sbuf_bcat(tl->sb, "#", 1); + x++; + y++; + continue; + } + if (*p == '\t') { + y &= ~7; + y += 8; + } else + y++; + while (x < y) { + sbuf_bcat(tl->sb, "-", 1); + x++; + } + } + sbuf_cat(tl->sb, "\n"); + tl->err = 1; +} + +/*--------------------------------------------------------------------*/ + +static void +NextToken(struct tokenlist *tl) +{ + tl->t = TAILQ_NEXT(tl->t, list); + if (tl->t == NULL) { + sbuf_printf(tl->sb, + "Ran out of input, something is missing or" + " maybe unbalanced (...) or {...}\n"); + tl->err = 1; + return; + } +} + +static void +_Expect(struct tokenlist *tl, unsigned tok, int line) +{ + if (tl->t->tok == tok) + return; + sbuf_printf(tl->sb, "Expected %s got ", vcl_tnames[tok]); + ErrToken(tl, tl->t); + sbuf_printf(tl->sb, "\n(program line %u), at\n", line); + ErrWhere(tl, tl->t); +} + +#define Expect(a, b) _Expect(a, b, __LINE__) +#define ExpectErr(a, b) do { _Expect(a, b, __LINE__); ERRCHK(a);} while (0) + +#define L(tl, foo) do { \ + tl->indent += INDENT; \ + foo; \ + tl->indent -= INDENT; \ +} while (0) + +#define C(tl, sep) do { \ + Fc(tl, 1, "VRT_count(sp, %u)%s\n", ++tl->cnt, sep); \ + tl->t->cnt = tl->cnt; \ +} while (0) + +/*-------------------------------------------------------------------- + * Printf output to the two sbufs, possibly indented + */ + +static void +Fh(struct tokenlist *tl, int indent, const char *fmt, ...) +{ + va_list ap; + + if (indent) + sbuf_printf(tl->fh, "%*.*s", tl->indent, tl->indent, ""); + va_start(ap, fmt); + sbuf_vprintf(tl->fh, fmt, ap); + va_end(ap); +} + +static void +Fc(struct tokenlist *tl, int indent, const char *fmt, ...) +{ + va_list ap; + + if (indent) + sbuf_printf(tl->fc, "%*.*s", tl->indent, tl->indent, ""); + va_start(ap, fmt); + sbuf_vprintf(tl->fc, fmt, ap); + va_end(ap); +} + +/*-------------------------------------------------------------------- + * Compare token to token + */ + +static int +Teq(struct token *t1, struct token *t2) +{ + if (t1->e - t1->b != t2->e - t2->b) + return (0); + return (!memcmp(t1->b, t2->b, t1->e - t1->b)); +} + +/*-------------------------------------------------------------------- + * Compare ID token to string, return true of match + */ + +static int +IdIs(struct token *t, const char *p) +{ + const char *q; + + assert(t->tok == ID); + for (q = t->b; q < t->e && *p != '\0'; p++, q++) + if (*q != *p) + return (0); + if (q != t->e || *p != '\0') + return (0); + return (1); +} + +/*--------------------------------------------------------------------*/ + +static char * +EncString(struct token *t) +{ + char *p, *q; + const char *r; + unsigned u; + + assert(t->tok == CSTR); + p = malloc(t->e - t->b); + assert(p != NULL); + q = p; + for (r = t->b + 1; r < t->e - 1; ) { + if (*r != '\\') { + *q++ = *r++; + continue; + } + switch (r[1]) { + case 'n': *q++ = '\n'; r += 2; break; + case 'r': *q++ = '\r'; r += 2; break; + case 'v': *q++ = '\v'; r += 2; break; + case 'f': *q++ = '\f'; r += 2; break; + case 't': *q++ = '\t'; r += 2; break; + case 'b': *q++ = '\b'; r += 2; break; + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + u = r[1] - '0'; + r += 2; + if (isdigit(r[0]) && (r[0] - '0') < 8) { + u <<= 3; + u |= r[0] - '0'; + r++; + if (isdigit(r[0]) && (r[0] - '0') < 8) { + u <<= 3; + u |= r[0] - '0'; + r++; + } + } + *q++ = u; + break; + default: + *q++ = r[1]; + r += 2; + break; + } + } + *q = '\0'; + return (p); +} + + +/*-------------------------------------------------------------------- + * Keep track of definitions and references + */ + +static struct ref * +FindRef(struct tokenlist *tl, struct token *t, enum ref_type type) +{ + struct ref *r; + + TAILQ_FOREACH(r, &tl->refs, list) { + if (r->type != type) + continue; + if (Teq(r->name, t)) + return (r); + } + r = calloc(sizeof *r, 1); + assert(r != NULL); + r->name = t; + r->type = type; + TAILQ_INSERT_TAIL(&tl->refs, r, list); + return (r); +} + +static int +FindRefStr(struct tokenlist *tl, const char *s, enum ref_type type) +{ + struct ref *r; + + TAILQ_FOREACH(r, &tl->refs, list) { + if (r->type != type) + continue; + if (IdIs(r->name, s)) + return (1); + } + return (0); +} + +static void +AddRef(struct tokenlist *tl, struct token *t, enum ref_type type) +{ + + FindRef(tl, t, type)->refcnt++; +} + +static void +AddRefStr(struct tokenlist *tl, const char *s, enum ref_type type) +{ + struct token *t; + + t = calloc(sizeof *t, 1); + t->b = s; + t->e = strchr(s, '\0'); + t->tok = METHOD; + AddRef(tl, t, type); +} + +static void +AddDef(struct tokenlist *tl, struct token *t, enum ref_type type) +{ + struct ref *r; + + r = FindRef(tl, t, type); + r->defcnt++; + r->name = t; +} + +/*-------------------------------------------------------------------- + * Recognize and convert units of time, return seconds. + */ + +static double +TimeUnit(struct tokenlist *tl) +{ + double sc = 1.0; + + assert(tl->t->tok == ID); + if (IdIs(tl->t, "ms")) + sc = 1e-3; + else if (IdIs(tl->t, "s")) + sc = 1.0; + else if (IdIs(tl->t, "m")) + sc = 60.0; + else if (IdIs(tl->t, "h")) + sc = 60.0 * 60.0; + else if (IdIs(tl->t, "d")) + sc = 60.0 * 60.0 * 24.0; + else { + sbuf_printf(tl->sb, "Unknown time unit "); + ErrToken(tl, tl->t); + sbuf_printf(tl->sb, ". Legal are 's', 'm', 'h' and 'd'\n"); + ErrWhere(tl, tl->t); + return (1.0); + } + NextToken(tl); + return (sc); +} + +/*-------------------------------------------------------------------- + * Recognize and convert units of size, return bytes. + */ + +static double +SizeUnit(struct tokenlist *tl) +{ + double sc = 1.0; + + assert(tl->t->tok == ID); + if (IdIs(tl->t, "b")) + sc = 1.0; + else if (IdIs(tl->t, "kb")) + sc = 1024.0; + else if (IdIs(tl->t, "mb") || IdIs(tl->t, "Mb")) + sc = 1024.0 * 1024.0; + else if (IdIs(tl->t, "gb") || IdIs(tl->t, "Gb")) + sc = 1024.0 * 1024.0 * 1024.0; + else { + sbuf_printf(tl->sb, "Unknown size unit "); + ErrToken(tl, tl->t); + sbuf_printf(tl->sb, ". Legal are 'kb', 'mb' and 'gb'\n"); + ErrWhere(tl, tl->t); + return (1.0); + } + NextToken(tl); + return (sc); +} + +/*-------------------------------------------------------------------- + * Recognize and convert units of rate as { space '/' time } + */ + +static double +RateUnit(struct tokenlist *tl) +{ + double sc = 1.0; + + assert(tl->t->tok == ID); + sc = SizeUnit(tl); + Expect(tl, '/'); + NextToken(tl); + sc /= TimeUnit(tl); + return (sc); +} + +/*-------------------------------------------------------------------- + * Recognize and convert { CNUM } to unsigned value + */ + +static unsigned +UintVal(struct tokenlist *tl) +{ + unsigned d = 0; + const char *p; + + Expect(tl, CNUM); + for (p = tl->t->b; p < tl->t->e; p++) { + d *= 10; + d += *p - '0'; + } + NextToken(tl); + return (d); +} + +/*-------------------------------------------------------------------- + * Recognize and convert { CNUM [ '.' [ CNUM ] ] } to double value + */ + +static double +DoubleVal(struct tokenlist *tl) +{ + double d = 0.0, e = 0.1; + const char *p; + + Expect(tl, CNUM); + for (p = tl->t->b; p < tl->t->e; p++) { + d *= 10; + d += *p - '0'; + } + NextToken(tl); + if (tl->t->tok != '.') + return (d); + NextToken(tl); + if (tl->t->tok != CNUM) + return (d); + for (p = tl->t->b; p < tl->t->e; p++) { + d += (*p - '0') * e; + e *= 0.1; + } + NextToken(tl); + return (d); +} + +/*--------------------------------------------------------------------*/ + +static unsigned +IpVal(struct tokenlist *tl) +{ + unsigned u, v; + struct token *t; + + t = tl->t; + u = UintVal(tl); + if (u < 256) { + v = u << 24; + Expect(tl, '.'); + NextToken(tl); + t = tl->t; + u = UintVal(tl); + if (u < 256) { + v |= u << 16; + Expect(tl, '.'); + NextToken(tl); + t = tl->t; + u = UintVal(tl); + if (u < 256) { + v |= u << 8; + Expect(tl, '.'); + NextToken(tl); + t = tl->t; + u = UintVal(tl); + if (u < 256) { + v |= u; + return (v); + } + } + } + } + sbuf_printf(tl->sb, "Illegal octet in IP number\n"); + ErrWhere(tl, t); + return (0); +} + +/*--------------------------------------------------------------------*/ + +static struct var * +HeaderVar(struct tokenlist *tl __unused, struct token *t, struct var *vh) +{ + char *p; + struct var *v; + int i; + + v = calloc(sizeof *v, 1); + assert(v != NULL); + i = t->e - t->b; + p = malloc(i + 1); + assert(p != NULL); + memcpy(p, t->b, i); + p[i] = '\0'; + v->name = p; + v->fmt = STRING; + asprintf(&p, "VRT_GetHdr(sp, \"\\%03o%s:\")", + strlen(v->name + vh->len) + 1, v->name + vh->len); + assert(p != NULL); + v->rname = p; + return (v); +} + +/*--------------------------------------------------------------------*/ + +static struct var * +FindVar(struct tokenlist *tl, struct token *t, struct var *vl) +{ + struct var *v; + + for (v = vl; v->name != NULL; v++) { + if (v->fmt == HEADER && t->e - t->b <= v->len) + continue; + if (v->fmt != HEADER && t->e - t->b != v->len) + continue; + if (memcmp(t->b, v->name, v->len)) + continue; + if (v->fmt != HEADER) + return (v); + return (HeaderVar(tl, t, v)); + } + sbuf_printf(tl->sb, "Unknown variable "); + ErrToken(tl, t); + sbuf_cat(tl->sb, "\nAt: "); + ErrWhere(tl, t); + return (NULL); +} + + +/*--------------------------------------------------------------------*/ + +static void +TimeVal(struct tokenlist *tl) +{ + double v, sc; + + v = DoubleVal(tl); + ExpectErr(tl, ID); + sc = TimeUnit(tl); + Fc(tl, 0, "(%g * %g)", v, sc); +} + +static void +SizeVal(struct tokenlist *tl) +{ + double v, sc; + + v = DoubleVal(tl); + ExpectErr(tl, ID); + sc = SizeUnit(tl); + Fc(tl, 0, "(%g * %g)", v, sc); +} + +static void +RateVal(struct tokenlist *tl) +{ + double v, sc; + + v = DoubleVal(tl); + ExpectErr(tl, ID); + sc = RateUnit(tl); + Fc(tl, 0, "(%g * %g)", v, sc); +} + +/*--------------------------------------------------------------------*/ + +static void +Cond_Ip(struct var *vp, struct tokenlist *tl) +{ + unsigned u; + + switch (tl->t->tok) { + case '~': + NextToken(tl); + ExpectErr(tl, ID); + AddRef(tl, tl->t, R_ACL); + Fc(tl, 1, "ip_match(%s, acl_%T)\n", vp->rname, tl->t); + NextToken(tl); + break; + case T_EQ: + case T_NEQ: + Fc(tl, 1, "%s %T ", vp->rname, tl->t); + NextToken(tl); + u = IpVal(tl); + Fc(tl, 0, "%uU /* %u.%u.%u.%u */\n", u, + (u >> 24) & 0xff, (u >> 16) & 0xff, + (u >> 8) & 0xff, (u) & 0xff); + break; + default: + sbuf_printf(tl->sb, "Illegal condition "); + ErrToken(tl, tl->t); + sbuf_printf(tl->sb, " on IP number variable\n"); + sbuf_printf(tl->sb, " only '==', '!=' and '~' are legal\n"); + ErrWhere(tl, tl->t); + break; + } +} + +static void +Cond_String(struct var *vp, struct tokenlist *tl) +{ + + switch (tl->t->tok) { + case '~': + Fc(tl, 1, "string_match(%s, ", vp->rname); + NextToken(tl); + ExpectErr(tl, CSTR); + Fc(tl, 0, "%T)\n", tl->t); + NextToken(tl); + break; + case T_EQ: + case T_NEQ: + Fc(tl, 1, "%sstrcmp(%s, ", + tl->t->tok == T_EQ ? "!" : "", vp->rname); + NextToken(tl); + ExpectErr(tl, CSTR); + Fc(tl, 0, "%T)\n", tl->t); + NextToken(tl); + break; + default: + Fc(tl, 1, "%s != (void*)0", vp->rname); + break; + } +} + +static void +Cond_Int(struct var *vp, struct tokenlist *tl) +{ + + Fc(tl, 1, "%s ", vp->rname); + switch (tl->t->tok) { + case T_EQ: + case T_NEQ: + case T_LEQ: + case T_GEQ: + case '>': + case '<': + Fc(tl, 0, "%T ", tl->t); + NextToken(tl); + switch(vp->fmt) { + case TIME: + TimeVal(tl); + break; + case INT: + ExpectErr(tl, CNUM); + Fc(tl, 0, "%T ", tl->t); + NextToken(tl); + break; + case SIZE: + SizeVal(tl); + break; + default: + sbuf_printf(tl->sb, + "No conditions available for variable '%s'\n", + vp->name); + ErrWhere(tl, tl->t); + return; + } + Fc(tl, 0, "\n"); + break; + default: + sbuf_printf(tl->sb, "Illegal condition "); + ErrToken(tl, tl->t); + sbuf_printf(tl->sb, " on integer variable\n"); + sbuf_printf(tl->sb, + " only '==', '!=', '<', '>', '<=' and '>=' are legal\n"); + ErrWhere(tl, tl->t); + break; + } +} + +static void +Cond_Bool(struct var *vp, struct tokenlist *tl) +{ + + Fc(tl, 1, "%s\n", vp->rname); +} + +static void +Cond_2(struct tokenlist *tl) +{ + struct var *vp; + + C(tl, ","); + if (tl->t->tok == '!') { + Fc(tl, 1, "!(\n"); + NextToken(tl); + } else { + Fc(tl, 1, "(\n"); + } + if (tl->t->tok == '(') { + NextToken(tl); + Cond_0(tl); + ExpectErr(tl, ')'); + NextToken(tl); + } else if (tl->t->tok == VAR) { + vp = FindVar(tl, tl->t, vars); + ERRCHK(tl); + assert(vp != NULL); + NextToken(tl); + switch (vp->fmt) { + case INT: L(tl, Cond_Int(vp, tl)); break; + case SIZE: L(tl, Cond_Int(vp, tl)); break; + case BOOL: L(tl, Cond_Bool(vp, tl)); break; + case IP: L(tl, Cond_Ip(vp, tl)); break; + case STRING: L(tl, Cond_String(vp, tl)); break; + case TIME: L(tl, Cond_Int(vp, tl)); break; + /* XXX backend == */ + default: + sbuf_printf(tl->sb, + "Variable '%s'" + " has no conditions that can be checked\n", + vp->name); + ErrWhere(tl, tl->t); + return; + } + } else { + sbuf_printf(tl->sb, + "Syntax error in condition, expected '(', '!' or" + " variable name, found "); + ErrToken(tl, tl->t); + sbuf_printf(tl->sb, "\n"); + ErrWhere(tl, tl->t); + return; + } + Fc(tl, 1, ")\n"); +} + +static void +Cond_1(struct tokenlist *tl) +{ + + Fc(tl, 1, "(\n"); + L(tl, Cond_2(tl)); + while (tl->t->tok == T_CAND) { + NextToken(tl); + Fc(tl, 1, ") && (\n"); + L(tl, Cond_2(tl)); + } + Fc(tl, 1, ")\n"); +} + +static void +Cond_0(struct tokenlist *tl) +{ + + Fc(tl, 1, "(\n"); + L(tl, Cond_1(tl)); + while (tl->t->tok == T_COR) { + NextToken(tl); + Fc(tl, 1, ") || (\n"); + L(tl, Cond_1(tl)); + } + Fc(tl, 1, ")\n"); +} + +static void +Conditional(struct tokenlist *tl) +{ + + ExpectErr(tl, '('); + NextToken(tl); + Fc(tl, 1, "(\n"); + L(tl, Cond_0(tl)); + ERRCHK(tl); + Fc(tl, 1, ")\n"); + ExpectErr(tl, ')'); + NextToken(tl); +} + +/*--------------------------------------------------------------------*/ + +static void +IfStmt(struct tokenlist *tl) +{ + + ExpectErr(tl, T_IF); + Fc(tl, 1, "if \n"); + NextToken(tl); + L(tl, Conditional(tl)); + ERRCHK(tl); + L(tl, Compound(tl)); + ERRCHK(tl); + while (1) { + switch (tl->t->tok) { + case T_ELSE: + NextToken(tl); + if (tl->t->tok != T_IF) { + Fc(tl, 1, "else \n"); + L(tl, Compound(tl)); + ERRCHK(tl); + return; + } + /* FALLTHROUGH */ + case T_ELSEIF: + case T_ELSIF: + Fc(tl, 1, "else if \n"); + NextToken(tl); + L(tl, Conditional(tl)); + ERRCHK(tl); + L(tl, Compound(tl)); + ERRCHK(tl); + break; + default: + C(tl, ";"); + return; + } + } +} + +/*--------------------------------------------------------------------*/ + +static void +Action(struct tokenlist *tl) +{ + unsigned a, u; + struct var *vp; + struct token *at; + + at = tl->t; + NextToken(tl); + switch (at->tok) { + case T_NO_NEW_CACHE: + Fc(tl, 1, "VCL_no_new_cache(sp);\n"); + return; + case T_NO_CACHE: + Fc(tl, 1, "VCL_no_cache(sp);\n"); + return; +#define VCL_RET_MAC(a,b,c) case T_##b: \ + Fc(tl, 1, "VRT_done(sp, VCL_RET_%s);\n", #b); \ + tl->curproc->returns |= VCL_RET_##b; \ + tl->curproc->returnt[c] = at; \ + return; +#include "vcl_returns.h" +#undef VCL_RET_MAC + case T_ERROR: + if (tl->t->tok == CNUM) + a = UintVal(tl); + else + a = 0; + Fc(tl, 1, "VRT_error(sp, %u, ", a); + if (tl->t->tok == CSTR) { + Fc(tl, 0, "%T);\n", tl->t); + NextToken(tl); + } else + Fc(tl, 0, "(const char *)0);\n"); + Fc(tl, 1, "VRT_done(sp, VCL_RET_ERROR);\n"); + return; + case T_SWITCH_CONFIG: + ExpectErr(tl, ID); + Fc(tl, 1, "VCL_switch_config(\"%T\");\n", tl->t); + NextToken(tl); + return; + case T_CALL: + ExpectErr(tl, ID); + AddCall(tl, tl->t); + AddRef(tl, tl->t, R_FUNC); + Fc(tl, 1, "if (VGC_function_%T(sp))\n", tl->t); + Fc(tl, 1, "\treturn (1);\n"); + NextToken(tl); + return; + case T_REWRITE: + ExpectErr(tl, CSTR); + Fc(tl, 1, "VCL_rewrite(%T", tl->t); + NextToken(tl); + ExpectErr(tl, CSTR); + Fc(tl, 0, ", %T);\n", tl->t); + NextToken(tl); + return; + case T_SET: + ExpectErr(tl, VAR); + vp = FindVar(tl, tl->t, vars); + ERRCHK(tl); + assert(vp != NULL); + Fc(tl, 1, "%s ", vp->rname); + NextToken(tl); + switch (vp->fmt) { + case INT: + case SIZE: + case RATE: + case TIME: + case FLOAT: + Fc(tl, 0, "%T ", tl->t); + a = tl->t->tok; + NextToken(tl); + if (a == T_MUL || a == T_DIV) + Fc(tl, 0, "%g", DoubleVal(tl)); + else if (vp->fmt == TIME) + TimeVal(tl); + else if (vp->fmt == SIZE) + SizeVal(tl); + else if (vp->fmt == RATE) + RateVal(tl); + else + Fc(tl, 0, "%g", DoubleVal(tl)); + Fc(tl, 0, ";\n"); + break; + case IP: + if (tl->t->tok == '=') { + NextToken(tl); + u = IpVal(tl); + Fc(tl, 0, "= %uU; /* %u.%u.%u.%u */\n", + u, + (u >> 24) & 0xff, + (u >> 16) & 0xff, + (u >> 8) & 0xff, + u & 0xff); + break; + } + sbuf_printf(tl->sb, "Illegal assignment operator "); + ErrToken(tl, tl->t); + sbuf_printf(tl->sb, + " only '=' is legal for IP numbers\n"); + ErrWhere(tl, tl->t); + return; + case BACKEND: + if (tl->t->tok == '=') { + NextToken(tl); + AddRef(tl, tl->t, R_BACKEND); + Fc(tl, 0, "= &VGC_backend_%T;\n", tl->t); + NextToken(tl); + break; + } + sbuf_printf(tl->sb, "Illegal assignment operator "); + ErrToken(tl, tl->t); + sbuf_printf(tl->sb, + " only '=' is legal for backend\n"); + ErrWhere(tl, tl->t); + return; + default: + sbuf_printf(tl->sb, + "Assignments not possible for '%s'\n", vp->name); + ErrWhere(tl, tl->t); + return; + } + return; + default: + sbuf_printf(tl->sb, "Expected action, 'if' or '}'\n"); + ErrWhere(tl, at); + return; + } +} + +/*--------------------------------------------------------------------*/ + +static void +Acl(struct tokenlist *tl) +{ + unsigned u, m; + + NextToken(tl); + + ExpectErr(tl, ID); + AddDef(tl, tl->t, R_ACL); + Fh(tl, 0, "static struct vcl_acl acl_%T[];\n", tl->t); + Fc(tl, 1, "static struct vcl_acl acl_%T[] = {\n", tl->t); + NextToken(tl); + + tl->indent += INDENT; + + ExpectErr(tl, '{'); + NextToken(tl); + + while (tl->t->tok == CNUM) { + u = IpVal(tl); + if (tl->t->tok == '/') { + NextToken(tl); + ExpectErr(tl, CNUM); + m = UintVal(tl); + } else + m = 32; + ExpectErr(tl, ';'); + NextToken(tl); + Fc(tl, 1, "{ %11uU, %3uU }, /* %u.%u.%u.%u/%u */\n", + u, m, + (u >> 24) & 0xff, (u >> 16) & 0xff, + (u >> 8) & 0xff, (u) & 0xff, m); + } + ExpectErr(tl, '}'); + Fc(tl, 1, "{ %11uU, %3uU }\n", 0, 0); + + tl->indent -= INDENT; + + Fc(tl, 1, "};\n\n"); + NextToken(tl); +} + +/*--------------------------------------------------------------------*/ + +static void +Compound(struct tokenlist *tl) +{ + + ExpectErr(tl, '{'); + Fc(tl, 1, "{\n"); + tl->indent += INDENT; + C(tl, ";"); + NextToken(tl); + while (1) { + ERRCHK(tl); + switch (tl->t->tok) { + case '{': + Compound(tl); + break; + case T_IF: + IfStmt(tl); + break; + case '}': + NextToken(tl); + tl->indent -= INDENT; + Fc(tl, 1, "}\n"); + return; + case EOI: + sbuf_printf(tl->sb, + "End of input while in compound statement\n"); + tl->err = 1; + return; + default: + Action(tl); + ERRCHK(tl); + ExpectErr(tl, ';'); + NextToken(tl); + break; + } + } +} + +/*--------------------------------------------------------------------*/ + +static const char * +CheckHostPort(const char *host, const char *port) +{ + struct addrinfo *res, hint; + int error; + + memset(&hint, 0, sizeof hint); + hint.ai_family = PF_UNSPEC; + hint.ai_socktype = SOCK_STREAM; + error = getaddrinfo(host, port, &hint, &res); + if (error) + return (gai_strerror(error)); + freeaddrinfo(res); + return (NULL); +} + +static void +Backend(struct tokenlist *tl) +{ + struct var *vp; + struct token *t_be = NULL; + struct token *t_host = NULL; + struct token *t_port = NULL; + char *host = NULL; + char *port = NULL; + const char *ep; + + NextToken(tl); + ExpectErr(tl, ID); + t_be = tl->t; + AddDef(tl, tl->t, R_BACKEND); + if (tl->nbackend == 0) + AddRef(tl, tl->t, R_BACKEND); + Fh(tl, 1, "#define VGC_backend_%T (VCL_conf.backend[%d])\n", + tl->t, tl->nbackend); + Fc(tl, 0, "static void\n"); + Fc(tl, 1, "VGC_init_backend_%T (void)\n", tl->t); + Fc(tl, 1, "{\n"); + Fc(tl, 1, "\tstruct backend *backend = VGC_backend_%T;\n", tl->t); + Fc(tl, 1, "\tconst char *p;\n"); + Fc(tl, 1, "\n"); + Fc(tl, 1, "\tVRT_set_backend_name(backend, \"%T\");\n", tl->t); + NextToken(tl); + ExpectErr(tl, '{'); + NextToken(tl); + while (1) { + if (tl->t->tok == '}') + break; + ExpectErr(tl, T_SET); + NextToken(tl); + ExpectErr(tl, VAR); + vp = FindVar(tl, tl->t, be_vars); + ERRCHK(tl); + assert(vp != NULL); + NextToken(tl); + ExpectErr(tl, '='); + NextToken(tl); + switch (vp->fmt) { + case HOSTNAME: + ExpectErr(tl, CSTR); + t_host = tl->t; + Fc(tl, 1, "\tp = %T;\n", tl->t); + Fc(tl, 1, "\t"); + Fc(tl, 0, vp->lname, "p"); + Fc(tl, 0, ";\n"); + NextToken(tl); + break; + case PORTNAME: + ExpectErr(tl, CSTR); + t_port = tl->t; + Fc(tl, 1, "\tp = %T;\n", tl->t); + Fc(tl, 1, "\t"); + Fc(tl, 0, vp->lname, "p"); + Fc(tl, 0, ";\n"); + NextToken(tl); + break; + default: + sbuf_printf(tl->sb, + "Assignments not possible for '%s'\n", vp->name); + ErrWhere(tl, tl->t); + return; + } + ExpectErr(tl, ';'); + NextToken(tl); + } + ExpectErr(tl, '}'); + if (t_host == NULL) { + sbuf_printf(tl->sb, "Backend '%T' has no hostname\n", t_be); + ErrWhere(tl, tl->t); + return; + } + host = EncString(t_host); + ep = CheckHostPort(host, "80"); + if (ep != NULL) { + sbuf_printf(tl->sb, "Backend '%T': %s\n", t_be, ep); + ErrWhere(tl, t_host); + return; + } + if (t_port != NULL) { + port = EncString(t_port); + ep = CheckHostPort(host, port); + if (ep != NULL) { + sbuf_printf(tl->sb, "Backend '%T': %s\n", t_be, ep); + ErrWhere(tl, t_port); + return; + } + } + + NextToken(tl); + Fc(tl, 1, "}\n"); + Fc(tl, 0, "\n"); + tl->nbackend++; +} + +/*--------------------------------------------------------------------*/ + +static void +Function(struct tokenlist *tl) +{ + + NextToken(tl); + ExpectErr(tl, ID); + tl->curproc = AddProc(tl, tl->t, 1); + tl->curproc->exists++; + AddDef(tl, tl->t, R_FUNC); + Fh(tl, 0, "static int VGC_function_%T (struct sess *sp);\n", tl->t); + Fc(tl, 1, "static int\n"); + Fc(tl, 1, "VGC_function_%T (struct sess *sp)\n", tl->t); + NextToken(tl); + L(tl, Compound(tl)); + Fc(tl, 0, "\n"); +} + +/*-------------------------------------------------------------------- + * Top level of parser, recognize: + * Function definitions + * Backend definitions + * End of input + */ + +static void +Parse(struct tokenlist *tl) +{ + + while (tl->t->tok != EOI) { + ERRCHK(tl); + switch (tl->t->tok) { + case T_ACL: + Acl(tl); + break; + case T_SUB: + Function(tl); + break; + case T_BACKEND: + Backend(tl); + break; + case EOI: + break; + default: + sbuf_printf(tl->sb, + "Expected 'acl', 'sub' or 'backend', found "); + ErrToken(tl, tl->t); + sbuf_printf(tl->sb, " at\n"); + ErrWhere(tl, tl->t); + return; + } + } +} + +/*-------------------------------------------------------------------- + * Add a token to the token list. + */ + +static void +AddToken(struct tokenlist *tl, unsigned tok, const char *b, const char *e) +{ + struct token *t; + + t = calloc(sizeof *t, 1); + assert(t != NULL); + t->tok = tok; + t->b = b; + t->e = e; + TAILQ_INSERT_TAIL(&tl->tokens, t, list); + tl->t = t; + if (0) { + fprintf(stderr, "[%s %*.*s] ", + vcl_tnames[tok], + e - b, e - b, b); + if (tok == EOI) + fprintf(stderr, "\n"); + } +} + +/*-------------------------------------------------------------------- + * Lexical analysis and token generation + */ + +static void +Lexer(struct tokenlist *tl, const char *b, const char *e) +{ + const char *p, *q; + unsigned u; + + for (p = b; p < e; ) { + + /* Skip any whitespace */ + if (isspace(*p)) { + p++; + continue; + } + + /* Skip '#.*\n' comments */ + if (*p == '#') { + while (p < e && *p != '\n') + p++; + continue; + } + + /* Skip C-style comments */ + if (*p == '/' && p[1] == '*') { + p += 2; + for (p += 2; p < e; p++) { + if (*p == '*' && p[1] == '/') { + p += 2; + break; + } + } + continue; + } + + /* Match for the fixed tokens (see token.tcl) */ + u = vcl_fixed_token(p, &q); + if (u != 0) { + AddToken(tl, u, p, q); + p = q; + continue; + } + + /* Match strings, with \\ and \" escapes */ + if (*p == '"') { + for (q = p + 1; q < e; q++) { + if (*q == '\\' && q[1] == '\\') + q++; + else if (*q == '\\' && q[1] == '"') + q++; + else if (*q == '"') { + q++; + break; + } + } + AddToken(tl, CSTR, p, q); + p = q; + continue; + } + + /* Match Identifiers */ + if (isident1(*p)) { + for (q = p; q < e; q++) + if (!isident(*q)) + break; + if (isvar(*q)) { + for (; q < e; q++) + if (!isvar(*q)) + break; + AddToken(tl, VAR, p, q); + } else { + AddToken(tl, ID, p, q); + } + p = q; + continue; + } + + /* Match numbers { [0-9]+ } */ + if (isdigit(*p)) { + for (q = p; q < e; q++) + if (!isdigit(*q)) + break; + AddToken(tl, CNUM, p, q); + p = q; + continue; + } + AddToken(tl, EOI, p, p + 1); + sbuf_printf(tl->sb, "Syntax error at\n"); + ErrWhere(tl, tl->t); + return; + } +} + +/*-------------------------------------------------------------------- + * Consistency check + */ + +static struct proc * +AddProc(struct tokenlist *tl, struct token *t, int def) +{ + struct proc *p; + + TAILQ_FOREACH(p, &tl->procs, list) { + if (!Teq(p->name, t)) + continue; + if (def) + p->name = t; + return (p); + } + p = calloc(sizeof *p, 1); + assert(p != NULL); + p->name = t; + TAILQ_INIT(&p->calls); + TAILQ_INSERT_TAIL(&tl->procs, p, list); + return (p); +} + +static void +AddCall(struct tokenlist *tl, struct token *t) +{ + struct proccall *pc; + struct proc *p; + + p = AddProc(tl, t, 0); + TAILQ_FOREACH(pc, &tl->curproc->calls, list) { + if (pc->p == p) + return; + } + pc = calloc(sizeof *pc, 1); + assert(pc != NULL); + pc->p = p; + pc->t = t; + TAILQ_INSERT_TAIL(&tl->curproc->calls, pc, list); +} + +static int +Consist_Decend(struct tokenlist *tl, struct proc *p, unsigned returns) +{ + unsigned u; + struct proccall *pc; + + if (!p->exists) { + sbuf_printf(tl->sb, "Function %T does not exist\n", p->name); + return (1); + } + if (p->active) { + sbuf_printf(tl->sb, "Function recurses on\n"); + ErrWhere(tl, p->name); + return (1); + } + u = p->returns & ~returns; + if (u) { +#define VCL_RET_MAC(a, b, c) \ + if (u & VCL_RET_##b) { \ + sbuf_printf(tl->sb, "Illegal return for method\n"); \ + ErrWhere(tl, p->returnt[c]); \ + } +#include "vcl_returns.h" +#undef VCL_RET_MAC + sbuf_printf(tl->sb, "In function\n"); + ErrWhere(tl, p->name); + return (1); + } + p->active = 1; + TAILQ_FOREACH(pc, &p->calls, list) { + if (Consist_Decend(tl, pc->p, returns)) { + sbuf_printf(tl->sb, "\nCalled from\n"); + ErrWhere(tl, p->name); + sbuf_printf(tl->sb, "at\n"); + ErrWhere(tl, pc->t); + return (1); + } + } + p->active = 0; + p->called++; + return (0); +} + +static int +Consistency(struct tokenlist *tl) +{ + struct proc *p; + struct method *m; + + TAILQ_FOREACH(p, &tl->procs, list) { + for(m = method_tab; m->name != NULL; m++) { + if (IdIs(p->name, m->defname)) + p->called = 1; + if (IdIs(p->name, m->name)) + break; + } + if (m->name == NULL) + continue; + if (Consist_Decend(tl, p, m->returns)) { + sbuf_printf(tl->sb, + "\nwhich is a %s method\n", m->name); + return (1); + } + } + TAILQ_FOREACH(p, &tl->procs, list) { + if (p->called) + continue; + sbuf_printf(tl->sb, "Function unused\n"); + ErrWhere(tl, p->name); + return (1); + } + return (0); +} + +/*--------------------------------------------------------------------*/ + +static int +CheckRefs(struct tokenlist *tl) +{ + struct ref *r; + const char *type; + int nerr = 0; + + TAILQ_FOREACH(r, &tl->refs, list) { + if (r->defcnt != 0 && r->refcnt != 0) + continue; + nerr++; + + switch(r->type) { + case R_FUNC: + type = "function"; + break; + case R_ACL: + type = "acl"; + break; + case R_BACKEND: + type = "backend"; + break; + default: + ErrInternal(tl); + sbuf_printf(tl->sb, "Ref "); + ErrToken(tl, r->name); + sbuf_printf(tl->sb, " has unknown type %d\n", + r->type); + continue; + } + if (r->defcnt == 0 && r->name->tok == METHOD) { + sbuf_printf(tl->sb, + "No definition for method %T\n", r->name); + continue; + } + + if (r->defcnt == 0) { + sbuf_printf(tl->sb, + "Undefined %s %T, first reference:\n", + type, r->name); + ErrWhere(tl, r->name); + continue; + } + + sbuf_printf(tl->sb, "Unused %s %T, defined:\n", type, r->name); + ErrWhere(tl, r->name); + } + return (nerr); +} + +/*--------------------------------------------------------------------*/ + +static void +LocTable(struct tokenlist *tl) +{ + struct token *t; + unsigned lin, pos; + const char *p; + + Fh(tl, 0, "#define VGC_NREFS %u\n", tl->cnt + 1); + Fh(tl, 0, "static struct vrt_ref VGC_ref[VGC_NREFS];\n"); + Fc(tl, 0, "static struct vrt_ref VGC_ref[VGC_NREFS] = {\n"); + lin = 1; + pos = 0; + p = tl->b; + TAILQ_FOREACH(t, &tl->tokens, list) { + if (t->cnt == 0) + continue; + for (;p < t->b; p++) { + if (*p == '\n') { + lin++; + pos = 0; + } else if (*p == '\t') { + pos &= ~7; + pos += 8; + } else + pos++; + + } + Fc(tl, 0, " [%3u] = { %4u, %3u, 0, \"%T\" },\n", + t->cnt, lin, pos + 1, t); + } + Fc(tl, 0, "};\n"); +} + + +/*--------------------------------------------------------------------*/ + +static void +EmitInitFunc(struct tokenlist *tl) +{ + struct ref *r; + + Fc(tl, 0, "\nstatic void\nVGC_Init(void)\n{\n\n"); + Fc(tl, 0, "\tVRT_alloc_backends(&VCL_conf);\n"); + + TAILQ_FOREACH(r, &tl->refs, list) { + switch(r->type) { + case R_FUNC: + break; + case R_ACL: + break; + case R_BACKEND: + Fc(tl, 0, "\tVGC_init_backend_%T();\n", r->name); + break; + } + } + Fc(tl, 0, "}\n"); +} + +/*--------------------------------------------------------------------*/ + +static void +EmitStruct(struct tokenlist *tl) +{ + + Fc(tl, 0, "\nstruct VCL_conf VCL_conf = {\n"); + Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n"); + Fc(tl, 0, "\t.init_func = VGC_Init,\n"); + Fc(tl, 0, "\t.nbackend = %d,\n", tl->nbackend); + Fc(tl, 0, "\t.ref = VGC_ref,\n"); + Fc(tl, 0, "\t.nref = VGC_NREFS,\n"); +#define VCL_RET_MAC(l,u,b) +#define VCL_MET_MAC(l,u,b) \ + if (FindRefStr(tl, "vcl_" #l, R_FUNC)) { \ + Fc(tl, 0, "\t." #l "_func = VGC_function_vcl_" #l ",\n"); \ + AddRefStr(tl, "vcl_" #l, R_FUNC); \ + } else { \ + Fc(tl, 0, "\t." #l "_func = VGC_function_default_vcl_" #l ",\n"); \ + } \ + AddRefStr(tl, "default_vcl_" #l, R_FUNC); +#include "vcl_returns.h" +#undef VCL_MET_MAC +#undef VCL_RET_MAC + Fc(tl, 0, "};\n"); +} + +/*--------------------------------------------------------------------*/ + +char * +VCC_Compile(struct sbuf *sb, const char *b, const char *e) +{ + struct tokenlist tokens; + struct ref *r; + struct token *t; + FILE *fo; + char *of = NULL; + char buf[BUFSIZ]; + + memset(&tokens, 0, sizeof tokens); + TAILQ_INIT(&tokens.tokens); + TAILQ_INIT(&tokens.refs); + TAILQ_INIT(&tokens.procs); + tokens.sb = sb; + + tokens.fc = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); + assert(tokens.fc != NULL); + + tokens.fh = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); + assert(tokens.fh != NULL); + + Fh(&tokens, 0, "extern struct VCL_conf VCL_conf;\n"); + + tokens.b = b; + if (e == NULL) + e = strchr(b, '\0'); + assert(e != NULL); + tokens.e = e; + Lexer(&tokens, b, e); + Lexer(&tokens, vcc_default_vcl_b, vcc_default_vcl_e); + AddToken(&tokens, EOI, e, e); + if (tokens.err) + goto done; + tokens.t = TAILQ_FIRST(&tokens.tokens); + Parse(&tokens); + if (tokens.err) + goto done; + Consistency(&tokens); + if (tokens.err) + goto done; + LocTable(&tokens); + + EmitInitFunc(&tokens); + + EmitStruct(&tokens); + + if (CheckRefs(&tokens)) + goto done; + + of = strdup("/tmp/vcl.XXXXXXXX"); + assert(of != NULL); + mktemp(of); + + sprintf(buf, + "tee /tmp/_.c |" + "cc -fpic -shared -Wl,-x -o %s -x c - ", of); + + fo = popen(buf, "w"); + assert(fo != NULL); + + vcl_output_lang_h(fo); + + sbuf_finish(tokens.fh); + fputs(sbuf_data(tokens.fh), fo); + sbuf_delete(tokens.fh); + + sbuf_finish(tokens.fc); + fputs(sbuf_data(tokens.fc), fo); + sbuf_delete(tokens.fc); + + pclose(fo); +done: + + /* Free References */ + while (!TAILQ_EMPTY(&tokens.refs)) { + r = TAILQ_FIRST(&tokens.refs); + TAILQ_REMOVE(&tokens.refs, r, list); + free(r); + } + + /* Free Tokens */ + while (!TAILQ_EMPTY(&tokens.tokens)) { + t = TAILQ_FIRST(&tokens.tokens); + TAILQ_REMOVE(&tokens.tokens, t, list); + free(t); + } + return (of); +} + +/*--------------------------------------------------------------------*/ + +char * +VCC_CompileFile(struct sbuf *sb, const char *fn) +{ + char *f, *r; + int fd, i; + struct stat st; + + fd = open(fn, O_RDONLY); + if (fd < 0) { + sbuf_printf(sb, "Cannot open file '%s': %s", + fn, strerror(errno)); + return (NULL); + } + assert(0 == fstat(fd, &st)); + f = malloc(st.st_size + 1); + assert(f != NULL); + i = read(fd, f, st.st_size); + assert(i == st.st_size); + f[i] = '\0'; + r = VCC_Compile(sb, f, NULL); + free(f); + return (r); +} + +/*--------------------------------------------------------------------*/ + +static int +VCC_T_render(FILE *f, const struct printf_info *info __unused, const void *const *args) +{ + const struct token *t; + + t = *((const struct token * const*) (args[0])); + return (fprintf(f, "%*.*s", + t->e - t->b, t->e - t->b, t->b)); +} + +static int +VCC_T_arginfo(const struct printf_info *info __unused, size_t n, int *argtypes) +{ + + if (n > 0) + argtypes[0] = PA_POINTER; + return 1; +} + +/*--------------------------------------------------------------------*/ + +void +VCC_InitCompile(const char *default_vcl) +{ + struct var *v; + + vcc_default_vcl_b = default_vcl; + vcc_default_vcl_e = strchr(default_vcl, '\0'); + assert(vcc_default_vcl_e != NULL); + + register_printf_function ('T', VCC_T_render, VCC_T_arginfo); + vcl_init_tnames(); + for (v = vars; v->name != NULL; v++) + v->len = strlen(v->name); + for (v = be_vars; v->name != NULL; v++) + v->len = strlen(v->name); +} + +#endif Copied: trunk/varnish-cache/lib/libvcl/vcc_priv.h (from rev 480, trunk/varnish-cache/lib/libvcl/vcl_priv.h) =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_priv.h 2006-07-14 13:54:41 UTC (rev 480) +++ trunk/varnish-cache/lib/libvcl/vcc_priv.h 2006-07-21 18:12:56 UTC (rev 544) @@ -0,0 +1,13 @@ +/* + * Stuff shared between main.c and fixed_token.c + */ + +#include "vcc_token_defs.h" + +#define isident1(c) (isalpha(c)) +#define isident(c) (isalpha(c) || isdigit(c) || (c) == '_') +#define isvar(c) (isident(c) || (c) == '.') +unsigned vcl_fixed_token(const char *p, const char **q); +extern const char *vcl_tnames[256]; +void vcl_init_tnames(void); +void vcl_output_lang_h(FILE *f); Added: trunk/varnish-cache/lib/libvcl/vcc_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_token.c 2006-07-21 16:25:24 UTC (rev 543) +++ trunk/varnish-cache/lib/libvcl/vcc_token.c 2006-07-21 18:12:56 UTC (rev 544) @@ -0,0 +1,312 @@ +/* + * $Id$ + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vcc_priv.h" +#include "vcl_returns.h" +#include "vcc_compile.h" + +#include "libvcl.h" + +#define ERRCHK(tl) do { if ((tl)->err) return; } while (0) + +#define INDENT 2 + +/*--------------------------------------------------------------------*/ + +void +vcc_ErrToken(struct tokenlist *tl, struct token *t) +{ + + if (t->tok == EOI) + sbuf_printf(tl->sb, "end of input"); + else + sbuf_printf(tl->sb, "'%T'", t); +} + +void +vcc__ErrInternal(struct tokenlist *tl, const char *func, unsigned line) +{ + + sbuf_printf(tl->sb, "VCL compiler internal error at %s():%u\n", + func, line); + tl->err = 1; +} + +void +vcc_ErrWhere(struct tokenlist *tl, struct token *t) +{ + unsigned lin, pos, x, y; + const char *p, *l, *f, *b, *e; + + lin = 1; + pos = 0; + if (t->tok == METHOD) + return; + if (t->b >= vcc_default_vcl_b && t->b < vcc_default_vcl_e) { + f = "Default VCL code (compiled in)"; + b = vcc_default_vcl_b; + e = vcc_default_vcl_e; + } else { + f = "VCL code"; + b = tl->b; + e = tl->e; + } + for (l = p = b; p < t->b; p++) { + if (*p == '\n') { + lin++; + pos = 0; + l = p + 1; + } else if (*p == '\t') { + pos &= ~7; + pos += 8; + } else + pos++; + } + sbuf_printf(tl->sb, "In %s Line %d Pos %d\n", f, lin, pos); + x = y = 0; + for (p = l; p < e && *p != '\n'; p++) { + if (*p == '\t') { + y &= ~7; + y += 8; + while (x < y) { + sbuf_bcat(tl->sb, " ", 1); + x++; + } + } else { + x++; + y++; + sbuf_bcat(tl->sb, p, 1); + } + } + sbuf_cat(tl->sb, "\n"); + x = y = 0; + for (p = l; p < e && *p != '\n'; p++) { + if (p >= t->b && p < t->e) { + sbuf_bcat(tl->sb, "#", 1); + x++; + y++; + continue; + } + if (*p == '\t') { + y &= ~7; + y += 8; + } else + y++; + while (x < y) { + sbuf_bcat(tl->sb, "-", 1); + x++; + } + } + sbuf_cat(tl->sb, "\n"); + tl->err = 1; +} + +/*--------------------------------------------------------------------*/ + +void +vcc_NextToken(struct tokenlist *tl) +{ + tl->t = TAILQ_NEXT(tl->t, list); + if (tl->t == NULL) { + sbuf_printf(tl->sb, + "Ran out of input, something is missing or" + " maybe unbalanced (...) or {...}\n"); + tl->err = 1; + return; + } +} + +void +vcc__Expect(struct tokenlist *tl, unsigned tok, int line) +{ + if (tl->t->tok == tok) + return; + sbuf_printf(tl->sb, "Expected %s got ", vcl_tnames[tok]); + vcc_ErrToken(tl, tl->t); + sbuf_printf(tl->sb, "\n(program line %u), at\n", line); + vcc_ErrWhere(tl, tl->t); +} + +#define Expect(a, b) _Expect(a, b, __LINE__) +#define ExpectErr(a, b) do { _Expect(a, b, __LINE__); ERRCHK(a);} while (0) + +#define L(tl, foo) do { \ + tl->indent += INDENT; \ + foo; \ + tl->indent -= INDENT; \ +} while (0) + +#define C(tl, sep) do { \ + Fc(tl, 1, "VRT_count(sp, %u)%s\n", ++tl->cnt, sep); \ + tl->t->cnt = tl->cnt; \ +} while (0) + + +/*-------------------------------------------------------------------- + * Compare token to token + */ + +int +vcc_Teq(struct token *t1, struct token *t2) +{ + if (t1->e - t1->b != t2->e - t2->b) + return (0); + return (!memcmp(t1->b, t2->b, t1->e - t1->b)); +} + +/*-------------------------------------------------------------------- + * Compare ID token to string, return true of match + */ + +int +vcc_IdIs(struct token *t, const char *p) +{ + const char *q; + + assert(t->tok == ID); + for (q = t->b; q < t->e && *p != '\0'; p++, q++) + if (*q != *p) + return (0); + if (q != t->e || *p != '\0') + return (0); + return (1); +} + +/*-------------------------------------------------------------------- + * Add a token to the token list. + */ + +void +vcc_AddToken(struct tokenlist *tl, unsigned tok, const char *b, const char *e) +{ + struct token *t; + + t = calloc(sizeof *t, 1); + assert(t != NULL); + t->tok = tok; + t->b = b; + t->e = e; + TAILQ_INSERT_TAIL(&tl->tokens, t, list); + tl->t = t; + if (0) { + fprintf(stderr, "[%s %*.*s] ", + vcl_tnames[tok], + e - b, e - b, b); + if (tok == EOI) + fprintf(stderr, "\n"); + } +} + +/*-------------------------------------------------------------------- + * Lexical analysis and token generation + */ + +void +vcc_Lexer(struct tokenlist *tl, const char *b, const char *e) +{ + const char *p, *q; + unsigned u; + + for (p = b; p < e; ) { + + /* Skip any whitespace */ + if (isspace(*p)) { + p++; + continue; + } + + /* Skip '#.*\n' comments */ + if (*p == '#') { + while (p < e && *p != '\n') + p++; + continue; + } + + /* Skip C-style comments */ + if (*p == '/' && p[1] == '*') { + p += 2; + for (p += 2; p < e; p++) { + if (*p == '*' && p[1] == '/') { + p += 2; + break; + } + } + continue; + } + + /* Match for the fixed tokens (see token.tcl) */ + u = vcl_fixed_token(p, &q); + if (u != 0) { + vcc_AddToken(tl, u, p, q); + p = q; + continue; + } + + /* Match strings, with \\ and \" escapes */ + if (*p == '"') { + for (q = p + 1; q < e; q++) { + if (*q == '\\' && q[1] == '\\') + q++; + else if (*q == '\\' && q[1] == '"') + q++; + else if (*q == '"') { + q++; + break; + } + } + vcc_AddToken(tl, CSTR, p, q); + p = q; + continue; + } + + /* Match Identifiers */ + if (isident1(*p)) { + for (q = p; q < e; q++) + if (!isident(*q)) + break; + if (isvar(*q)) { + for (; q < e; q++) + if (!isvar(*q)) + break; + vcc_AddToken(tl, VAR, p, q); + } else { + vcc_AddToken(tl, ID, p, q); + } + p = q; + continue; + } + + /* Match numbers { [0-9]+ } */ + if (isdigit(*p)) { + for (q = p; q < e; q++) + if (!isdigit(*q)) + break; + vcc_AddToken(tl, CNUM, p, q); + p = q; + continue; + } + vcc_AddToken(tl, EOI, p, p + 1); + sbuf_printf(tl->sb, "Syntax error at\n"); + vcc_ErrWhere(tl, tl->t); + return; + } +} Copied: trunk/varnish-cache/lib/libvcl/vcc_token_defs.h (from rev 480, trunk/varnish-cache/lib/libvcl/vcl_token_defs.h) =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-07-14 13:54:41 UTC (rev 480) +++ trunk/varnish-cache/lib/libvcl/vcc_token_defs.h 2006-07-21 18:12:56 UTC (rev 544) @@ -0,0 +1,53 @@ +/* + * $Id$ + * + * NB: This file is machine generated, DO NOT EDIT! + * + * Edit vcc_gen_fixed_token.tcl instead + */ + +#define LOW_TOKEN 128 +#define T_IF 128 +#define T_ELSE 129 +#define T_ELSEIF 130 +#define T_ELSIF 131 +#define T_FUNC 132 +#define T_PROC 133 +#define T_SUB 134 +#define T_ACL 135 +#define T_BACKEND 136 +#define T_CALL 137 +#define T_NO_CACHE 138 +#define T_NO_NEW_CACHE 139 +#define T_SET 140 +#define T_REWRITE 141 +#define T_SWITCH_CONFIG 142 +#define T_ERROR 143 +#define T_LOOKUP 144 +#define T_PIPE 145 +#define T_PASS 146 +#define T_INSERT_PASS 147 +#define T_FETCH 148 +#define T_INSERT 149 +#define T_DELIVER 150 +#define T_DISCARD 151 +#define T_INC 152 +#define T_DEC 153 +#define T_CAND 154 +#define T_COR 155 +#define T_LEQ 156 +#define T_EQ 157 +#define T_NEQ 158 +#define T_GEQ 159 +#define T_SHR 160 +#define T_SHL 161 +#define T_INCR 162 +#define T_DECR 163 +#define T_MUL 164 +#define T_DIV 165 +#define ID 166 +#define VAR 167 +#define CNUM 168 +#define CSTR 169 +#define EOI 170 +#define METHOD 171 Deleted: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-07-21 16:25:24 UTC (rev 543) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-07-21 18:12:56 UTC (rev 544) @@ -1,518 +0,0 @@ -/* - * $Id$ - * - * NB: This file is machine generated, DO NOT EDIT! - * - * Edit vcl_gen_fixed_token.tcl instead - */ - -#include -#include -#include "vcl_priv.h" - -unsigned -vcl_fixed_token(const char *p, const char **q) -{ - - switch (p[0]) { - case '!': - if (p[0] == '!' && p[1] == '=') { - *q = p + 2; - return (T_NEQ); - } - if (p[0] == '!') { - *q = p + 1; - return ('!'); - } - return (0); - case '%': - if (p[0] == '%') { - *q = p + 1; - return ('%'); - } - return (0); - case '&': - if (p[0] == '&' && p[1] == '&') { - *q = p + 2; - return (T_CAND); - } - if (p[0] == '&') { - *q = p + 1; - return ('&'); - } - return (0); - case '(': - if (p[0] == '(') { - *q = p + 1; - return ('('); - } - return (0); - case ')': - if (p[0] == ')') { - *q = p + 1; - return (')'); - } - return (0); - case '*': - if (p[0] == '*' && p[1] == '=') { - *q = p + 2; - return (T_MUL); - } - if (p[0] == '*') { - *q = p + 1; - return ('*'); - } - return (0); - case '+': - if (p[0] == '+' && p[1] == '=') { - *q = p + 2; - return (T_INCR); - } - if (p[0] == '+' && p[1] == '+') { - *q = p + 2; - return (T_INC); - } - if (p[0] == '+') { - *q = p + 1; - return ('+'); - } - return (0); - case ',': - if (p[0] == ',') { - *q = p + 1; - return (','); - } - return (0); - case '-': - if (p[0] == '-' && p[1] == '-') { - *q = p + 2; - return (T_DEC); - } - if (p[0] == '-') { - *q = p + 1; - return ('-'); - } - return (0); - case '.': - if (p[0] == '.') { - *q = p + 1; - return ('.'); - } - return (0); - case '/': - if (p[0] == '/' && p[1] == '=') { - *q = p + 2; - return (T_DECR); - } - if (p[0] == '/' && p[1] == '=') { - *q = p + 2; - return (T_DIV); - } - if (p[0] == '/') { - *q = p + 1; - return ('/'); - } - return (0); - case ';': - if (p[0] == ';') { - *q = p + 1; - return (';'); - } - return (0); - case '<': - if (p[0] == '<' && p[1] == '=') { - *q = p + 2; - return (T_LEQ); - } - if (p[0] == '<' && p[1] == '<') { - *q = p + 2; - return (T_SHL); - } - if (p[0] == '<') { - *q = p + 1; - return ('<'); - } - return (0); - case '=': - if (p[0] == '=' && p[1] == '=') { - *q = p + 2; - return (T_EQ); - } - if (p[0] == '=') { - *q = p + 1; - return ('='); - } - return (0); - case '>': - if (p[0] == '>' && p[1] == '>') { - *q = p + 2; - return (T_SHR); - } - if (p[0] == '>' && p[1] == '=') { - *q = p + 2; - return (T_GEQ); - } - if (p[0] == '>') { - *q = p + 1; - return ('>'); - } - return (0); - case 'a': - if (p[0] == 'a' && p[1] == 'c' && p[2] == 'l' - && !isvar(p[3])) { - *q = p + 3; - return (T_ACL); - } - return (0); - case 'b': - if (p[0] == 'b' && p[1] == 'a' && p[2] == 'c' && - p[3] == 'k' && p[4] == 'e' && p[5] == 'n' && - p[6] == 'd' && !isvar(p[7])) { - *q = p + 7; - return (T_BACKEND); - } - return (0); - case 'c': - if (p[0] == 'c' && p[1] == 'a' && p[2] == 'l' && - p[3] == 'l' && !isvar(p[4])) { - *q = p + 4; - return (T_CALL); - } - return (0); - case 'd': - if (p[0] == 'd' && p[1] == 'i' && p[2] == 's' && - p[3] == 'c' && p[4] == 'a' && p[5] == 'r' && - p[6] == 'd' && !isvar(p[7])) { - *q = p + 7; - return (T_DISCARD); - } - if (p[0] == 'd' && p[1] == 'e' && p[2] == 'l' && - p[3] == 'i' && p[4] == 'v' && p[5] == 'e' && - p[6] == 'r' && !isvar(p[7])) { - *q = p + 7; - return (T_DELIVER); - } - return (0); - case 'e': - if (p[0] == 'e' && p[1] == 'r' && p[2] == 'r' && - p[3] == 'o' && p[4] == 'r' && !isvar(p[5])) { - *q = p + 5; - return (T_ERROR); - } - if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && - p[3] == 'i' && p[4] == 'f' && !isvar(p[5])) { - *q = p + 5; - return (T_ELSIF); - } - if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && - p[3] == 'e' && p[4] == 'i' && p[5] == 'f' - && !isvar(p[6])) { - *q = p + 6; - return (T_ELSEIF); - } - if (p[0] == 'e' && p[1] == 'l' && p[2] == 's' && - p[3] == 'e' && !isvar(p[4])) { - *q = p + 4; - return (T_ELSE); - } - return (0); - case 'f': - if (p[0] == 'f' && p[1] == 'u' && p[2] == 'n' && - p[3] == 'c' && !isvar(p[4])) { - *q = p + 4; - return (T_FUNC); - } - if (p[0] == 'f' && p[1] == 'e' && p[2] == 't' && - p[3] == 'c' && p[4] == 'h' && !isvar(p[5])) { - *q = p + 5; - return (T_FETCH); - } - return (0); - case 'i': - if (p[0] == 'i' && p[1] == 'n' && p[2] == 's' && - p[3] == 'e' && p[4] == 'r' && p[5] == 't' && - p[6] == '_' && p[7] == 'p' && p[8] == 'a' && - p[9] == 's' && p[10] == 's' && !isvar(p[11])) { - *q = p + 11; - return (T_INSERT_PASS); - } - if (p[0] == 'i' && p[1] == 'n' && p[2] == 's' && - p[3] == 'e' && p[4] == 'r' && p[5] == 't' - && !isvar(p[6])) { - *q = p + 6; - return (T_INSERT); - } - if (p[0] == 'i' && p[1] == 'f' && !isvar(p[2])) { - *q = p + 2; - return (T_IF); - } - return (0); - case 'l': - if (p[0] == 'l' && p[1] == 'o' && p[2] == 'o' && - p[3] == 'k' && p[4] == 'u' && p[5] == 'p' - && !isvar(p[6])) { - *q = p + 6; - return (T_LOOKUP); - } - return (0); - case 'n': - if (p[0] == 'n' && p[1] == 'o' && p[2] == '_' && - p[3] == 'n' && p[4] == 'e' && p[5] == 'w' && - p[6] == '_' && p[7] == 'c' && p[8] == 'a' && - p[9] == 'c' && p[10] == 'h' && p[11] == 'e' - && !isvar(p[12])) { - *q = p + 12; - return (T_NO_NEW_CACHE); - } - if (p[0] == 'n' && p[1] == 'o' && p[2] == '_' && - p[3] == 'c' && p[4] == 'a' && p[5] == 'c' && - p[6] == 'h' && p[7] == 'e' && !isvar(p[8])) { - *q = p + 8; - return (T_NO_CACHE); - } - return (0); - case 'p': - if (p[0] == 'p' && p[1] == 'r' && p[2] == 'o' && - p[3] == 'c' && !isvar(p[4])) { - *q = p + 4; - return (T_PROC); - } - if (p[0] == 'p' && p[1] == 'i' && p[2] == 'p' && - p[3] == 'e' && !isvar(p[4])) { - *q = p + 4; - return (T_PIPE); - } - if (p[0] == 'p' && p[1] == 'a' && p[2] == 's' && - p[3] == 's' && !isvar(p[4])) { - *q = p + 4; - return (T_PASS); - } - return (0); - case 'r': - if (p[0] == 'r' && p[1] == 'e' && p[2] == 'w' && - p[3] == 'r' && p[4] == 'i' && p[5] == 't' && - p[6] == 'e' && !isvar(p[7])) { - *q = p + 7; - return (T_REWRITE); - } - return (0); - case 's': - if (p[0] == 's' && p[1] == 'w' && p[2] == 'i' && - p[3] == 't' && p[4] == 'c' && p[5] == 'h' && - p[6] == '_' && p[7] == 'c' && p[8] == 'o' && - p[9] == 'n' && p[10] == 'f' && p[11] == 'i' && - p[12] == 'g' && !isvar(p[13])) { - *q = p + 13; - return (T_SWITCH_CONFIG); - } - if (p[0] == 's' && p[1] == 'u' && p[2] == 'b' - && !isvar(p[3])) { - *q = p + 3; - return (T_SUB); - } - if (p[0] == 's' && p[1] == 'e' && p[2] == 't' - && !isvar(p[3])) { - *q = p + 3; - return (T_SET); - } - return (0); - case '{': - if (p[0] == '{') { - *q = p + 1; - return ('{'); - } - return (0); - case '|': - if (p[0] == '|' && p[1] == '|') { - *q = p + 2; - return (T_COR); - } - if (p[0] == '|') { - *q = p + 1; - return ('|'); - } - return (0); - case '}': - if (p[0] == '}') { - *q = p + 1; - return ('}'); - } - return (0); - case '~': - if (p[0] == '~') { - *q = p + 1; - return ('~'); - } - return (0); - default: - return (0); - } -} - -const char *vcl_tnames[256]; - -void -vcl_init_tnames(void) -{ - vcl_tnames['!'] = "'!'"; - vcl_tnames['%'] = "'%'"; - vcl_tnames['&'] = "'&'"; - vcl_tnames['('] = "'('"; - vcl_tnames[')'] = "')'"; - vcl_tnames['*'] = "'*'"; - vcl_tnames['+'] = "'+'"; - vcl_tnames[','] = "','"; - vcl_tnames['-'] = "'-'"; - vcl_tnames['.'] = "'.'"; - vcl_tnames['/'] = "'/'"; - vcl_tnames['<'] = "'<'"; - vcl_tnames['='] = "'='"; - vcl_tnames['>'] = "'>'"; - vcl_tnames['{'] = "'{'"; - vcl_tnames['}'] = "'}'"; - vcl_tnames['|'] = "'|'"; - vcl_tnames['~'] = "'~'"; - vcl_tnames[';'] = "';'"; - vcl_tnames[CNUM] = "CNUM"; - vcl_tnames[CSTR] = "CSTR"; - vcl_tnames[EOI] = "EOI"; - vcl_tnames[ID] = "ID"; - vcl_tnames[METHOD] = "METHOD"; - vcl_tnames[T_ACL] = "acl"; - vcl_tnames[T_BACKEND] = "backend"; - vcl_tnames[T_CALL] = "call"; - vcl_tnames[T_CAND] = "&&"; - vcl_tnames[T_COR] = "||"; - vcl_tnames[T_DEC] = "--"; - vcl_tnames[T_DECR] = "/="; - vcl_tnames[T_DELIVER] = "deliver"; - vcl_tnames[T_DISCARD] = "discard"; - vcl_tnames[T_DIV] = "/="; - vcl_tnames[T_ELSE] = "else"; - vcl_tnames[T_ELSEIF] = "elseif"; - vcl_tnames[T_ELSIF] = "elsif"; - vcl_tnames[T_EQ] = "=="; - vcl_tnames[T_ERROR] = "error"; - vcl_tnames[T_FETCH] = "fetch"; - vcl_tnames[T_FUNC] = "func"; - vcl_tnames[T_GEQ] = ">="; - vcl_tnames[T_IF] = "if"; - vcl_tnames[T_INC] = "++"; - vcl_tnames[T_INCR] = "+="; - vcl_tnames[T_INSERT] = "insert"; - vcl_tnames[T_INSERT_PASS] = "insert_pass"; - vcl_tnames[T_LEQ] = "<="; - vcl_tnames[T_LOOKUP] = "lookup"; - vcl_tnames[T_MUL] = "*="; - vcl_tnames[T_NEQ] = "!="; - vcl_tnames[T_NO_CACHE] = "no_cache"; - vcl_tnames[T_NO_NEW_CACHE] = "no_new_cache"; - vcl_tnames[T_PASS] = "pass"; - vcl_tnames[T_PIPE] = "pipe"; - vcl_tnames[T_PROC] = "proc"; - vcl_tnames[T_REWRITE] = "rewrite"; - vcl_tnames[T_SET] = "set"; - vcl_tnames[T_SHL] = "<<"; - vcl_tnames[T_SHR] = ">>"; - vcl_tnames[T_SUB] = "sub"; - vcl_tnames[T_SWITCH_CONFIG] = "switch_config"; - vcl_tnames[VAR] = "VAR"; -} - -void -vcl_output_lang_h(FILE *f) -{ - fputs("#define VCL_RET_ERROR (1 << 0)\n", f); - fputs("#define VCL_RET_LOOKUP (1 << 1)\n", f); - fputs("#define VCL_RET_PIPE (1 << 2)\n", f); - fputs("#define VCL_RET_PASS (1 << 3)\n", f); - fputs("#define VCL_RET_INSERT_PASS (1 << 4)\n", f); - fputs("#define VCL_RET_FETCH (1 << 5)\n", f); - fputs("#define VCL_RET_INSERT (1 << 6)\n", f); - fputs("#define VCL_RET_DELIVER (1 << 7)\n", f); - fputs("#define VCL_RET_DISCARD (1 << 8)\n", f); - fputs("/*\n", f); - fputs(" * $Id$\n", f); - fputs(" *\n", f); - fputs(" * NB: This file is machine generated, DO NOT EDIT!\n", f); - fputs(" *\n", f); - fputs(" * Edit vcl_gen_fixed_token.tcl instead\n", f); - fputs(" */\n", f); - fputs("\n", f); - fputs("struct sess;\n", f); - fputs("\n", f); - fputs("typedef void vcl_init_f(void);\n", f); - fputs("typedef int vcl_func_f(struct sess *sp);\n", f); - fputs("\n", f); - fputs("struct VCL_conf {\n", f); - fputs(" unsigned magic;\n", f); - fputs("#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */\n", f); - fputs("\n", f); - fputs(" struct backend **backend;\n", f); - fputs(" unsigned nbackend;\n", f); - fputs(" struct vrt_ref *ref;\n", f); - fputs(" unsigned nref;\n", f); - fputs(" unsigned busy;\n", f); - fputs("\n", f); - fputs(" vcl_init_f *init_func;\n", f); - fputs("\n", f); - fputs(" vcl_func_f *recv_func;\n", f); - fputs(" vcl_func_f *miss_func;\n", f); - fputs(" vcl_func_f *hit_func;\n", f); - fputs(" vcl_func_f *fetch_func;\n", f); - fputs(" vcl_func_f *timeout_func;\n", f); - fputs("};\n", f); - fputs("/*\n", f); - fputs(" * $Id$ \n", f); - fputs(" *\n", f); - fputs(" * Runtime support for compiled VCL programs.\n", f); - fputs(" *\n", f); - fputs(" * XXX: When this file is changed, lib/libvcl/vcl_gen_fixed_token.tcl\n", f); - fputs(" * XXX: *MUST* be rerun.\n", f); - fputs(" */\n", f); - fputs("\n", f); - fputs("struct sess;\n", f); - fputs("struct backend;\n", f); - fputs("struct VCL_conf;\n", f); - fputs("\n", f); - fputs("struct vrt_ref {\n", f); - fputs(" unsigned line;\n", f); - fputs(" unsigned pos;\n", f); - fputs(" unsigned count;\n", f); - fputs(" const char *token;\n", f); - fputs("};\n", f); - fputs("\n", f); - fputs("struct vrt_acl {\n", f); - fputs(" unsigned ip;\n", f); - fputs(" unsigned mask;\n", f); - fputs("};\n", f); - fputs("\n", f); - fputs("void VRT_count(struct sess *, unsigned);\n", f); - fputs("void VRT_no_cache(struct sess *);\n", f); - fputs("void VRT_no_new_cache(struct sess *);\n", f); - fputs("#if 0\n", f); - fputs("int ip_match(unsigned, struct vcl_acl *);\n", f); - fputs("int string_match(const char *, const char *);\n", f); - fputs("#endif\n", f); - fputs("int VRT_rewrite(const char *, const char *);\n", f); - fputs("void VRT_error(struct sess *, unsigned, const char *);\n", f); - fputs("int VRT_switch_config(const char *);\n", f); - fputs("\n", f); - fputs("char *VRT_GetHdr(struct sess *, const char *);\n", f); - fputs("char *VRT_GetReq(struct sess *);\n", f); - fputs("void VRT_handling(struct sess *sp, unsigned hand);\n", f); - fputs("int VRT_obj_valid(struct sess *);\n", f); - fputs("int VRT_obj_cacheable(struct sess *);\n", f); - fputs("\n", f); - fputs("void VRT_set_backend_name(struct backend *, const char *);\n", f); - fputs("void VRT_set_backend_hostname(struct backend *, const char *);\n", f); - fputs("void VRT_set_backend_portname(struct backend *, const char *);\n", f); - fputs("\n", f); - fputs("void VRT_alloc_backends(struct VCL_conf *cp);\n", f); - fputs("\n", f); - fputs("#define VRT_done(sp, hand) \\\n", f); - fputs(" do { \\\n", f); - fputs(" VRT_handling(sp, hand); \\\n", f); - fputs(" return (1); \\\n", f); - fputs(" } while (0)\n", f); -} Deleted: trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-07-21 16:25:24 UTC (rev 543) +++ trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-07-21 18:12:56 UTC (rev 544) @@ -1,319 +0,0 @@ -#!/usr/local/bin/tclsh8.4 -# -# Generate various .c and .h files for the VCL compiler and the interfaces -# for it. - -# These are the metods which can be called in the VCL program. -# Second element is list of valid return actions. -# -set methods { - {recv {error pass pipe lookup}} - {miss {error pass pipe fetch}} - {hit {error pass pipe deliver}} - {fetch {error pass pipe insert insert_pass}} - {timeout {fetch discard}} -} - -# These are the return actions -# -set returns { - error - lookup - pipe - pass - insert_pass - fetch - insert - deliver - discard -} - -# Language keywords -# -set keywords { - if else elseif elsif - - func proc sub - - acl - - backend - - call - no_cache - no_new_cache - set - rewrite - switch_config -} - -# Non-word tokens -# -set magic { - {"++" INC} - {"--" DEC} - {"&&" CAND} - {"||" COR} - {"<=" LEQ} - {"==" EQ} - {"!=" NEQ} - {">=" GEQ} - {">>" SHR} - {"<<" SHL} - {"+=" INCR} - {"/=" DECR} - {"*=" MUL} - {"/=" DIV} -} - -# Single char tokens -# -set char {{}()*+-/%><=;!&.|~,} - -# Other token identifiers -# -set extras {ID VAR CNUM CSTR EOI METHOD} - -#---------------------------------------------------------------------- -# Boilerplate warning for all generated files. - -proc warns {fd} { - - puts $fd "/*" - puts $fd { * $Id$} - puts $fd " *" - puts $fd " * NB: This file is machine generated, DO NOT EDIT!" - puts $fd " *" - puts $fd " * Edit vcl_gen_fixed_token.tcl instead" - puts $fd " */" - puts $fd "" -} - -#---------------------------------------------------------------------- -# Build the vcl.h #include file - -set fo [open ../../include/vcl.h w] -warns $fo -puts $fo {struct sess; - -typedef void vcl_init_f(void); -typedef int vcl_func_f(struct sess *sp); -} -puts $fo "struct VCL_conf {" -puts $fo { unsigned magic; -#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */ - - struct backend **backend; - unsigned nbackend; - struct vrt_ref *ref; - unsigned nref; - unsigned busy; - - vcl_init_f *init_func; -} -foreach m $methods { - puts $fo "\tvcl_func_f\t*[lindex $m 0]_func;" -} -puts $fo "};" - -close $fo - -#---------------------------------------------------------------------- -# Build the vcl_returns.h #include file - -set for [open "../../include/vcl_returns.h" w] -warns $for -puts $for "#ifdef VCL_RET_MAC" -set i 0 -foreach k $returns { - if {$k == "error"} { - puts $for "#ifdef VCL_RET_MAC_E" - puts $for "VCL_RET_MAC_E($k, [string toupper $k], $i)" - puts $for "#endif" - } else { - puts $for "VCL_RET_MAC($k, [string toupper $k], (1 << $i))" - } - incr i -} -puts $for "#else" -set i 0 -foreach k $returns { - puts $for "#define VCL_RET_[string toupper $k] (1 << $i)" - incr i -} -puts $for "#define VCL_RET_MAX $i" -puts $for "#endif" -puts $for "" -puts $for "#ifdef VCL_MET_MAC" -foreach m $methods { - puts -nonewline $for "VCL_MET_MAC([lindex $m 0]" - puts -nonewline $for ",[string toupper [lindex $m 0]]" - set l [lindex $m 1] - puts -nonewline $for ",(VCL_RET_[string toupper [lindex $l 0]]" - foreach r [lrange $l 1 end] { - puts -nonewline $for "|VCL_RET_[string toupper $r]" - } - puts -nonewline $for ")" - puts $for ")" -} -puts $for "#endif" -close $for - -#---------------------------------------------------------------------- -# Build the compiler token table and recognizers - -set fo [open "vcl_fixed_token.c" w] -warns $fo - -set foh [open "vcl_token_defs.h" w] -warns $foh - -puts $fo "#include " -puts $fo "#include " -puts $fo "#include \"vcl_priv.h\"" - -set tn 128 -puts $foh "#define LOW_TOKEN $tn" - - -proc add_token {tok str alpha} { - global tokens tn fixed foh - - lappend tokens [list $tok $str] - puts $foh "#define $tok $tn" - incr tn - lappend fixed [list $str $tok $alpha] -} - -proc mk_token {tok str alpha} { - set tok T_[string toupper $tok] - add_token $tok $str $alpha -} - -foreach k $keywords { mk_token $k $k 1 } -foreach k $returns { mk_token $k $k 1 } -foreach k $magic { mk_token [lindex $k 1] [lindex $k 0] 0 } -foreach k $extras { - set t [string toupper $k] - lappend tokens [list $t $t] - puts $foh "#define [string toupper $k] $tn" - incr tn -} -for {set i 0} {$i < [string length $char]} {incr i} { - set t [string index $char $i] - lappend token2 [list '$t' T$t] - lappend fixed [list "$t" '$t' 0] -} - -set tokens [lsort $tokens] -set token2 [lsort $token2] - -# We want to output in ascii order: create sorted first char list -foreach t $fixed { - set xx([string index [lindex $t 0] 0]) 1 -} -set seq [lsort [array names xx]] - -set ll 0 - -puts $fo { -unsigned -vcl_fixed_token(const char *p, const char **q)} -puts $fo "{" -puts $fo "" -puts $fo " switch (p\[0\]) {" - -foreach ch "$seq" { - # Now find all tokens starting with ch - set l "" - foreach t $fixed { - if {[string index [lindex $t 0] 0] == $ch} { - lappend l $t - } - } - # And do then in reverse order to match longest first - set l [lsort -index 0 -decreasing $l] - scan "$ch" "%c" cx - if {$cx != $ll} { - if {$ll} { - puts $fo " return (0);" - } - - puts $fo " case '$ch':" - set ll $cx - } - foreach tt $l { - set k [lindex $tt 0] - puts -nonewline $fo " if (" - for {set i 0} {$i < [string length $k]} {incr i} { - if {$i > 0} { - puts -nonewline $fo " && " - if {![expr $i % 3]} { - puts -nonewline $fo "\n\t\t " - } - } - puts -nonewline $fo "p\[$i\] == '[string index $k $i]'" - } - if {[lindex $tt 2]} { - if {![expr $i % 3]} { - puts -nonewline $fo "\n\t\t " - } - puts -nonewline $fo " && !isvar(p\[$i\])" - } - puts $fo ") {" - puts $fo " *q = p + [string length $k];" - puts $fo " return ([lindex $tt 1]);" - puts $fo " }" - } -} -puts $fo " return (0);" -puts $fo " default:" -puts $fo " return (0);" -puts $fo " }" -puts $fo "}" - -puts $fo "" -puts $fo "const char *vcl_tnames\[256\];\n" -puts $fo "void" -puts $fo "vcl_init_tnames(void)" -puts $fo "{" -foreach i $token2 { - puts $fo "\tvcl_tnames\[[lindex $i 0]\] = \"[lindex $i 0]\";" -} -foreach i $tokens { - puts $fo "\tvcl_tnames\[[lindex $i 0]\] = \"[lindex $i 1]\";" -} -puts $fo "}" - -#---------------------------------------------------------------------- -# Create the C-code which emits the boilerplate definitions for the -# generated C code output - -proc copy_include {n} { - global fo - - set fi [open $n] - while {[gets $fi a] >= 0} { - regsub -all {\\} $a {\\\\} a - puts $fo "\tfputs(\"$a\\n\", f);" - } - close $fi -} - -puts $fo "" -puts $fo "void" -puts $fo "vcl_output_lang_h(FILE *f)" -puts $fo "{" -set i 0 -foreach k $returns { - puts $fo "\tfputs(\"#define VCL_RET_[string toupper $k] (1 << $i)\\n\", f);" - incr i -} - -copy_include ../../include/vcl.h -copy_include ../../include/vrt.h - -puts $fo "}" - -close $foh -close $fo Deleted: trunk/varnish-cache/lib/libvcl/vcl_priv.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_priv.h 2006-07-21 16:25:24 UTC (rev 543) +++ trunk/varnish-cache/lib/libvcl/vcl_priv.h 2006-07-21 18:12:56 UTC (rev 544) @@ -1,13 +0,0 @@ -/* - * Stuff shared between main.c and fixed_token.c - */ - -#include "vcl_token_defs.h" - -#define isident1(c) (isalpha(c)) -#define isident(c) (isalpha(c) || isdigit(c) || (c) == '_') -#define isvar(c) (isident(c) || (c) == '.') -unsigned vcl_fixed_token(const char *p, const char **q); -extern const char *vcl_tnames[256]; -void vcl_init_tnames(void); -void vcl_output_lang_h(FILE *f); Deleted: trunk/varnish-cache/lib/libvcl/vcl_token_defs.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-07-21 16:25:24 UTC (rev 543) +++ trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-07-21 18:12:56 UTC (rev 544) @@ -1,53 +0,0 @@ -/* - * $Id$ - * - * NB: This file is machine generated, DO NOT EDIT! - * - * Edit vcl_gen_fixed_token.tcl instead - */ - -#define LOW_TOKEN 128 -#define T_IF 128 -#define T_ELSE 129 -#define T_ELSEIF 130 -#define T_ELSIF 131 -#define T_FUNC 132 -#define T_PROC 133 -#define T_SUB 134 -#define T_ACL 135 -#define T_BACKEND 136 -#define T_CALL 137 -#define T_NO_CACHE 138 -#define T_NO_NEW_CACHE 139 -#define T_SET 140 -#define T_REWRITE 141 -#define T_SWITCH_CONFIG 142 -#define T_ERROR 143 -#define T_LOOKUP 144 -#define T_PIPE 145 -#define T_PASS 146 -#define T_INSERT_PASS 147 -#define T_FETCH 148 -#define T_INSERT 149 -#define T_DELIVER 150 -#define T_DISCARD 151 -#define T_INC 152 -#define T_DEC 153 -#define T_CAND 154 -#define T_COR 155 -#define T_LEQ 156 -#define T_EQ 157 -#define T_NEQ 158 -#define T_GEQ 159 -#define T_SHR 160 -#define T_SHL 161 -#define T_INCR 162 -#define T_DECR 163 -#define T_MUL 164 -#define T_DIV 165 -#define ID 166 -#define VAR 167 -#define CNUM 168 -#define CSTR 169 -#define EOI 170 -#define METHOD 171 From phk at projects.linpro.no Fri Jul 21 20:43:56 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 22:43:56 +0200 (CEST) Subject: r545 - trunk/varnish-cache/lib/libvcl Message-ID: <20060721204356.ED48C1EC38D@projects.linpro.no> Author: phk Date: 2006-07-21 22:43:56 +0200 (Fri, 21 Jul 2006) New Revision: 545 Added: trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl Modified: trunk/varnish-cache/lib/libvcl/Makefile.am trunk/varnish-cache/lib/libvcl/vcc_compile.c trunk/varnish-cache/lib/libvcl/vcc_compile.h trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl trunk/varnish-cache/lib/libvcl/vcc_obj.c trunk/varnish-cache/lib/libvcl/vcc_token.c Log: Automate generation of tables and prototypes for the objects which VCL programs can manipulate. Property changes on: trunk/varnish-cache/lib/libvcl/Makefile.am ___________________________________________________________________ Name: svn:keywords + Id Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.c 2006-07-21 18:12:56 UTC (rev 544) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.c 2006-07-21 20:43:56 UTC (rev 545) @@ -1,5 +1,5 @@ /* - * $Id: vcl_compile.c 531 2006-07-20 22:08:43Z phk $ + * $Id$ */ /* @@ -791,7 +791,7 @@ vp = FindVar(tl, tl->t, vcc_vars); ERRCHK(tl); assert(vp != NULL); - Fc(tl, 1, "%s ", vp->rname); + Fc(tl, 1, "%s", vp->lname); vcc_NextToken(tl); switch (vp->fmt) { case INT: @@ -799,7 +799,8 @@ case RATE: case TIME: case FLOAT: - Fc(tl, 0, "%T ", tl->t); + if (tl->t->tok != '=') + Fc(tl, 0, "%s %c ", vp->rname, *tl->t->b); a = tl->t->tok; vcc_NextToken(tl); if (a == T_MUL || a == T_DIV) @@ -812,7 +813,7 @@ RateVal(tl); else Fc(tl, 0, "%g", DoubleVal(tl)); - Fc(tl, 0, ";\n"); + Fc(tl, 0, ");\n"); break; case IP: if (tl->t->tok == '=') { @@ -1007,19 +1008,13 @@ case HOSTNAME: ExpectErr(tl, CSTR); t_host = tl->t; - Fc(tl, 1, "\tp = %T;\n", tl->t); - Fc(tl, 1, "\t"); - Fc(tl, 0, vp->lname, "p"); - Fc(tl, 0, ";\n"); + Fc(tl, 1, "\t%s %T);\n", vp->lname, tl->t); vcc_NextToken(tl); break; case PORTNAME: ExpectErr(tl, CSTR); t_port = tl->t; - Fc(tl, 1, "\tp = %T;\n", tl->t); - Fc(tl, 1, "\t"); - Fc(tl, 0, vp->lname, "p"); - Fc(tl, 0, ";\n"); + Fc(tl, 1, "\t%s %T);\n", vp->lname, tl->t); vcc_NextToken(tl); break; default: @@ -1381,6 +1376,7 @@ FILE *fo; char *of = NULL; char buf[BUFSIZ]; + int i; memset(&tokens, 0, sizeof tokens); TAILQ_INIT(&tokens.tokens); @@ -1434,6 +1430,7 @@ assert(fo != NULL); vcl_output_lang_h(fo); + fputs(vrt_obj_h, fo); sbuf_finish(tokens.fh); fputs(sbuf_data(tokens.fh), fo); @@ -1443,7 +1440,14 @@ fputs(sbuf_data(tokens.fc), fo); sbuf_delete(tokens.fc); - pclose(fo); + i = pclose(fo); + fprintf(stderr, "pclose=%d\n", i); + if (i) { + sbuf_printf(sb, "Internal error: GCC returned 0x%04x\n", i); + unlink(of); + free(of); + return (NULL); + } done: /* Free References */ Property changes on: trunk/varnish-cache/lib/libvcl/vcc_compile.c ___________________________________________________________________ Name: svn:keywords + Id Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.h 2006-07-21 18:12:56 UTC (rev 544) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.h 2006-07-21 20:43:56 UTC (rev 545) @@ -63,7 +63,7 @@ struct var { const char *name; enum var_type fmt; - int len; + unsigned len; const char *rname; const char *lname; }; @@ -107,6 +107,7 @@ /* vcc_obj.c */ extern struct var vcc_be_vars[]; extern struct var vcc_vars[]; +const char *vrt_obj_h; /* vcc_token.c */ void vcc_ErrToken(struct tokenlist *tl, struct token *t); Property changes on: trunk/varnish-cache/lib/libvcl/vcc_compile.h ___________________________________________________________________ Name: svn:keywords + Id Property changes on: trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl ___________________________________________________________________ Name: svn:keywords + Id Added: trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl 2006-07-21 18:12:56 UTC (rev 544) +++ trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl 2006-07-21 20:43:56 UTC (rev 545) @@ -0,0 +1,95 @@ +#!/usr/local/bin/tclsh8.4 +# +# Generate various .c and .h files for the VCL compiler and the interfaces +# for it. + +# Objects which operate on backends +set beobj { + { backend.host HOSTNAME } + { backend.port PORTNAME } +} + +# Objects which operate on sessions + +set spobj { + { req.request STRING } + { req.url STRING } + { obj.valid BOOL } + { obj.cacheable BOOL } + { obj.backend BACKEND } + { obj.ttl TIME } + { req.http. HEADER } +} + +set tt(STRING) "char *" +set tt(BOOL) "double" +set tt(BACKEND) "struct backend *" +set tt(TIME) "double" +set tt(HEADER) "char *" +set tt(HOSTNAME) "char *" +set tt(PORTNAME) "char *" + +#---------------------------------------------------------------------- +# Boilerplate warning for all generated files. + +proc warns {fd} { + + puts $fd "/*" + puts $fd { * $Id$} + puts $fd " *" + puts $fd " * NB: This file is machine generated, DO NOT EDIT!" + puts $fd " *" + puts $fd " * Edit vcc_gen_obj.tcl instead" + puts $fd " */" + puts $fd "" +} + +set fo [open vcc_obj.c w] +warns $fo +set fp [open ../../include/vrt_obj.h w] +warns $fp + +proc vars {v ty pa} { + global tt fo fp + + foreach v $v { + set n [lindex $v 0] + regsub -all {[.]} $n "_" m + set t [lindex $v 1] + puts $fo "\t\{ \"$n\", $t, [string length $n]," + puts $fo "\t \"VRT_r_${m}($pa)\"," + puts $fo "\t \"VRT_l_${m}($pa, \"," + puts $fo "\t\}," + + puts $fp "$tt($t) VRT_r_${m}($ty);" + puts $fp "void VRT_l_${m}($ty, $tt($t));" + } + puts $fo "\t{ NULL }" +} + +puts $fo "#include " +puts $fo "#include \"vcc_compile.h\"" +puts $fo "" + +puts $fo "struct var vcc_be_vars\[\] = {" +vars $beobj "struct backend *" "backend" +puts $fo "};" + +puts $fo "" + +puts $fo "struct var vcc_vars\[\] = {" +vars $spobj "struct sess *" "sp" +puts $fo "};" + +close $fp +set fp [open ../../include/vrt_obj.h] + +puts $fo "" +puts $fo "const char *vrt_obj_h = " +while {[gets $fp a] >= 0} { + puts $fo "\t\"$a\\n\"" +} +puts $fo ";" + +close $fo +close $fp Property changes on: trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl ___________________________________________________________________ Name: svn:executable + * Name: svn:keywords + Id Modified: trunk/varnish-cache/lib/libvcl/vcc_obj.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_obj.c 2006-07-21 18:12:56 UTC (rev 544) +++ trunk/varnish-cache/lib/libvcl/vcc_obj.c 2006-07-21 20:43:56 UTC (rev 545) @@ -1,1843 +1,83 @@ /* * $Id$ + * + * NB: This file is machine generated, DO NOT EDIT! + * + * Edit vcc_gen_obj.tcl instead */ #include - #include "vcc_compile.h" struct var vcc_be_vars[] = { - { "backend.host", - HOSTNAME, 0, NULL, "VRT_set_backend_hostname(backend, %s)" }, - { "backend.port", - PORTNAME, 0, NULL, "VRT_set_backend_portname(backend, %s)" }, + { "backend.host", HOSTNAME, 12, + "VRT_r_backend_host(backend)", + "VRT_l_backend_host(backend, ", + }, + { "backend.port", PORTNAME, 12, + "VRT_r_backend_port(backend)", + "VRT_l_backend_port(backend, ", + }, + { NULL } }; - struct var vcc_vars[] = { - { "req.request", STRING, 0, "VRT_GetReq(sp)" }, - { "obj.valid", BOOL, 0, "VRT_obj_valid(sp)" }, - { "obj.cacheable", BOOL, 0, "VRT_obj_cacheable(sp)" }, - { "obj.backend", BACKEND, 0, "VRT_obj_backend(sp)" }, - { "req.http.", HEADER, 0, NULL }, -#if 0 - { "req.ttlfactor", FLOAT, 0, "req->ttlfactor" }, - { "req.url.host", STRING, 0, "req->url.host" }, - { "req.url.path", STRING, 0, "req->url.path" }, - { "req.useragent", STRING, 0, "req->useragent" }, - { "req.backend", BACKEND, 0, "req->backend" }, - { "client.ip", IP, 0, "client->ip" }, - { "backend.response_time", TIME, 0, "backend->responsetime" }, - { "backend.down", BOOL, 0, "backend->down" }, - { "backend.timeout", TIME, 0, "backend->timeout" }, - { "backend.bandwidth", RATE, 0, "backend->bandwidth" }, - { "obj.exist", BOOL, 0, "obj->exists" }, - { "obj.ttl", TIME, 0, "obj->ttl" }, - { "obj.result", INT, 0, "obj->result" }, - { "obj.size", SIZE, 0, "obj->size" }, - { "obj.usage", INT, 0, "obj->usage" }, -#endif - { NULL, INT, 0, "NULL" } + { "req.request", STRING, 11, + "VRT_r_req_request(sp)", + "VRT_l_req_request(sp, ", + }, + { "req.url", STRING, 7, + "VRT_r_req_url(sp)", + "VRT_l_req_url(sp, ", + }, + { "obj.valid", BOOL, 9, + "VRT_r_obj_valid(sp)", + "VRT_l_obj_valid(sp, ", + }, + { "obj.cacheable", BOOL, 13, + "VRT_r_obj_cacheable(sp)", + "VRT_l_obj_cacheable(sp, ", + }, + { "obj.backend", BACKEND, 11, + "VRT_r_obj_backend(sp)", + "VRT_l_obj_backend(sp, ", + }, + { "obj.ttl", TIME, 7, + "VRT_r_obj_ttl(sp)", + "VRT_l_obj_ttl(sp, ", + }, + { "req.http.", HEADER, 9, + "VRT_r_req_http_(sp)", + "VRT_l_req_http_(sp, ", + }, + { NULL } }; - -#if 0 - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "vcl_priv.h" -#include "vcl_returns.h" -#include "vcl_compile.h" - -#include "libvcl.h" - -#define ERRCHK(tl) do { if ((tl)->err) return; } while (0) - -#define INDENT 2 - -/*-------------------------------------------------------------------- - * Consistency check - */ - -static struct method { - const char *name; - const char *defname; - unsigned returns; -} method_tab[] = { -#define VCL_RET_MAC(a,b,c) -#define VCL_MET_MAC(a,b,c) { "vcl_"#a, "default_vcl_"#a, c }, -#include "vcl_returns.h" -#undef VCL_MET_MAC -#undef VCL_RET_MAC - { NULL, 0U } -}; - -struct proccall { - TAILQ_ENTRY(proccall) list; - struct proc *p; - struct token *t; -}; - -struct proc { - TAILQ_ENTRY(proc) list; - TAILQ_HEAD(,proccall) calls; - struct token *name; - unsigned returns; - unsigned exists; - unsigned called; - unsigned active; - struct token *returnt[VCL_RET_MAX]; -}; - -/*--------------------------------------------------------------------*/ - - -static void Compound(struct tokenlist *tl); -static void Cond_0(struct tokenlist *tl); -static struct proc *AddProc(struct tokenlist *tl, struct token *t, int def); -static void AddCall(struct tokenlist *tl, struct token *t); -static const char *vcc_default_vcl_b, *vcc_default_vcl_e; - -/*--------------------------------------------------------------------*/ - -static void -ErrToken(struct tokenlist *tl, struct token *t) -{ - - if (t->tok == EOI) - sbuf_printf(tl->sb, "end of input"); - else - sbuf_printf(tl->sb, "'%T'", t); -} - -static void -_ErrInternal(struct tokenlist *tl, const char *func, unsigned line) -{ - - sbuf_printf(tl->sb, "VCL compiler internal error at %s():%u\n", - func, line); - tl->err = 1; -} - -#define ErrInternal(tl) _ErrInternal(tl, __func__, __LINE__) - -static void -ErrWhere(struct tokenlist *tl, struct token *t) -{ - unsigned lin, pos, x, y; - const char *p, *l, *f, *b, *e; - - lin = 1; - pos = 0; - if (t->tok == METHOD) - return; - if (t->b >= vcc_default_vcl_b && t->b < vcc_default_vcl_e) { - f = "Default VCL code (compiled in)"; - b = vcc_default_vcl_b; - e = vcc_default_vcl_e; - } else { - f = "VCL code"; - b = tl->b; - e = tl->e; - } - for (l = p = b; p < t->b; p++) { - if (*p == '\n') { - lin++; - pos = 0; - l = p + 1; - } else if (*p == '\t') { - pos &= ~7; - pos += 8; - } else - pos++; - } - sbuf_printf(tl->sb, "In %s Line %d Pos %d\n", f, lin, pos); - x = y = 0; - for (p = l; p < e && *p != '\n'; p++) { - if (*p == '\t') { - y &= ~7; - y += 8; - while (x < y) { - sbuf_bcat(tl->sb, " ", 1); - x++; - } - } else { - x++; - y++; - sbuf_bcat(tl->sb, p, 1); - } - } - sbuf_cat(tl->sb, "\n"); - x = y = 0; - for (p = l; p < e && *p != '\n'; p++) { - if (p >= t->b && p < t->e) { - sbuf_bcat(tl->sb, "#", 1); - x++; - y++; - continue; - } - if (*p == '\t') { - y &= ~7; - y += 8; - } else - y++; - while (x < y) { - sbuf_bcat(tl->sb, "-", 1); - x++; - } - } - sbuf_cat(tl->sb, "\n"); - tl->err = 1; -} - -/*--------------------------------------------------------------------*/ - -static void -NextToken(struct tokenlist *tl) -{ - tl->t = TAILQ_NEXT(tl->t, list); - if (tl->t == NULL) { - sbuf_printf(tl->sb, - "Ran out of input, something is missing or" - " maybe unbalanced (...) or {...}\n"); - tl->err = 1; - return; - } -} - -static void -_Expect(struct tokenlist *tl, unsigned tok, int line) -{ - if (tl->t->tok == tok) - return; - sbuf_printf(tl->sb, "Expected %s got ", vcl_tnames[tok]); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, "\n(program line %u), at\n", line); - ErrWhere(tl, tl->t); -} - -#define Expect(a, b) _Expect(a, b, __LINE__) -#define ExpectErr(a, b) do { _Expect(a, b, __LINE__); ERRCHK(a);} while (0) - -#define L(tl, foo) do { \ - tl->indent += INDENT; \ - foo; \ - tl->indent -= INDENT; \ -} while (0) - -#define C(tl, sep) do { \ - Fc(tl, 1, "VRT_count(sp, %u)%s\n", ++tl->cnt, sep); \ - tl->t->cnt = tl->cnt; \ -} while (0) - -/*-------------------------------------------------------------------- - * Printf output to the two sbufs, possibly indented - */ - -static void -Fh(struct tokenlist *tl, int indent, const char *fmt, ...) -{ - va_list ap; - - if (indent) - sbuf_printf(tl->fh, "%*.*s", tl->indent, tl->indent, ""); - va_start(ap, fmt); - sbuf_vprintf(tl->fh, fmt, ap); - va_end(ap); -} - -static void -Fc(struct tokenlist *tl, int indent, const char *fmt, ...) -{ - va_list ap; - - if (indent) - sbuf_printf(tl->fc, "%*.*s", tl->indent, tl->indent, ""); - va_start(ap, fmt); - sbuf_vprintf(tl->fc, fmt, ap); - va_end(ap); -} - -/*-------------------------------------------------------------------- - * Compare token to token - */ - -static int -Teq(struct token *t1, struct token *t2) -{ - if (t1->e - t1->b != t2->e - t2->b) - return (0); - return (!memcmp(t1->b, t2->b, t1->e - t1->b)); -} - -/*-------------------------------------------------------------------- - * Compare ID token to string, return true of match - */ - -static int -IdIs(struct token *t, const char *p) -{ - const char *q; - - assert(t->tok == ID); - for (q = t->b; q < t->e && *p != '\0'; p++, q++) - if (*q != *p) - return (0); - if (q != t->e || *p != '\0') - return (0); - return (1); -} - -/*--------------------------------------------------------------------*/ - -static char * -EncString(struct token *t) -{ - char *p, *q; - const char *r; - unsigned u; - - assert(t->tok == CSTR); - p = malloc(t->e - t->b); - assert(p != NULL); - q = p; - for (r = t->b + 1; r < t->e - 1; ) { - if (*r != '\\') { - *q++ = *r++; - continue; - } - switch (r[1]) { - case 'n': *q++ = '\n'; r += 2; break; - case 'r': *q++ = '\r'; r += 2; break; - case 'v': *q++ = '\v'; r += 2; break; - case 'f': *q++ = '\f'; r += 2; break; - case 't': *q++ = '\t'; r += 2; break; - case 'b': *q++ = '\b'; r += 2; break; - case '0': case '1': case '2': case '3': - case '4': case '5': case '6': case '7': - u = r[1] - '0'; - r += 2; - if (isdigit(r[0]) && (r[0] - '0') < 8) { - u <<= 3; - u |= r[0] - '0'; - r++; - if (isdigit(r[0]) && (r[0] - '0') < 8) { - u <<= 3; - u |= r[0] - '0'; - r++; - } - } - *q++ = u; - break; - default: - *q++ = r[1]; - r += 2; - break; - } - } - *q = '\0'; - return (p); -} - - -/*-------------------------------------------------------------------- - * Keep track of definitions and references - */ - -static struct ref * -FindRef(struct tokenlist *tl, struct token *t, enum ref_type type) -{ - struct ref *r; - - TAILQ_FOREACH(r, &tl->refs, list) { - if (r->type != type) - continue; - if (Teq(r->name, t)) - return (r); - } - r = calloc(sizeof *r, 1); - assert(r != NULL); - r->name = t; - r->type = type; - TAILQ_INSERT_TAIL(&tl->refs, r, list); - return (r); -} - -static int -FindRefStr(struct tokenlist *tl, const char *s, enum ref_type type) -{ - struct ref *r; - - TAILQ_FOREACH(r, &tl->refs, list) { - if (r->type != type) - continue; - if (IdIs(r->name, s)) - return (1); - } - return (0); -} - -static void -AddRef(struct tokenlist *tl, struct token *t, enum ref_type type) -{ - - FindRef(tl, t, type)->refcnt++; -} - -static void -AddRefStr(struct tokenlist *tl, const char *s, enum ref_type type) -{ - struct token *t; - - t = calloc(sizeof *t, 1); - t->b = s; - t->e = strchr(s, '\0'); - t->tok = METHOD; - AddRef(tl, t, type); -} - -static void -AddDef(struct tokenlist *tl, struct token *t, enum ref_type type) -{ - struct ref *r; - - r = FindRef(tl, t, type); - r->defcnt++; - r->name = t; -} - -/*-------------------------------------------------------------------- - * Recognize and convert units of time, return seconds. - */ - -static double -TimeUnit(struct tokenlist *tl) -{ - double sc = 1.0; - - assert(tl->t->tok == ID); - if (IdIs(tl->t, "ms")) - sc = 1e-3; - else if (IdIs(tl->t, "s")) - sc = 1.0; - else if (IdIs(tl->t, "m")) - sc = 60.0; - else if (IdIs(tl->t, "h")) - sc = 60.0 * 60.0; - else if (IdIs(tl->t, "d")) - sc = 60.0 * 60.0 * 24.0; - else { - sbuf_printf(tl->sb, "Unknown time unit "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, ". Legal are 's', 'm', 'h' and 'd'\n"); - ErrWhere(tl, tl->t); - return (1.0); - } - NextToken(tl); - return (sc); -} - -/*-------------------------------------------------------------------- - * Recognize and convert units of size, return bytes. - */ - -static double -SizeUnit(struct tokenlist *tl) -{ - double sc = 1.0; - - assert(tl->t->tok == ID); - if (IdIs(tl->t, "b")) - sc = 1.0; - else if (IdIs(tl->t, "kb")) - sc = 1024.0; - else if (IdIs(tl->t, "mb") || IdIs(tl->t, "Mb")) - sc = 1024.0 * 1024.0; - else if (IdIs(tl->t, "gb") || IdIs(tl->t, "Gb")) - sc = 1024.0 * 1024.0 * 1024.0; - else { - sbuf_printf(tl->sb, "Unknown size unit "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, ". Legal are 'kb', 'mb' and 'gb'\n"); - ErrWhere(tl, tl->t); - return (1.0); - } - NextToken(tl); - return (sc); -} - -/*-------------------------------------------------------------------- - * Recognize and convert units of rate as { space '/' time } - */ - -static double -RateUnit(struct tokenlist *tl) -{ - double sc = 1.0; - - assert(tl->t->tok == ID); - sc = SizeUnit(tl); - Expect(tl, '/'); - NextToken(tl); - sc /= TimeUnit(tl); - return (sc); -} - -/*-------------------------------------------------------------------- - * Recognize and convert { CNUM } to unsigned value - */ - -static unsigned -UintVal(struct tokenlist *tl) -{ - unsigned d = 0; - const char *p; - - Expect(tl, CNUM); - for (p = tl->t->b; p < tl->t->e; p++) { - d *= 10; - d += *p - '0'; - } - NextToken(tl); - return (d); -} - -/*-------------------------------------------------------------------- - * Recognize and convert { CNUM [ '.' [ CNUM ] ] } to double value - */ - -static double -DoubleVal(struct tokenlist *tl) -{ - double d = 0.0, e = 0.1; - const char *p; - - Expect(tl, CNUM); - for (p = tl->t->b; p < tl->t->e; p++) { - d *= 10; - d += *p - '0'; - } - NextToken(tl); - if (tl->t->tok != '.') - return (d); - NextToken(tl); - if (tl->t->tok != CNUM) - return (d); - for (p = tl->t->b; p < tl->t->e; p++) { - d += (*p - '0') * e; - e *= 0.1; - } - NextToken(tl); - return (d); -} - -/*--------------------------------------------------------------------*/ - -static unsigned -IpVal(struct tokenlist *tl) -{ - unsigned u, v; - struct token *t; - - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v = u << 24; - Expect(tl, '.'); - NextToken(tl); - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v |= u << 16; - Expect(tl, '.'); - NextToken(tl); - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v |= u << 8; - Expect(tl, '.'); - NextToken(tl); - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v |= u; - return (v); - } - } - } - } - sbuf_printf(tl->sb, "Illegal octet in IP number\n"); - ErrWhere(tl, t); - return (0); -} - -/*--------------------------------------------------------------------*/ - -static struct var * -HeaderVar(struct tokenlist *tl __unused, struct token *t, struct var *vh) -{ - char *p; - struct var *v; - int i; - - v = calloc(sizeof *v, 1); - assert(v != NULL); - i = t->e - t->b; - p = malloc(i + 1); - assert(p != NULL); - memcpy(p, t->b, i); - p[i] = '\0'; - v->name = p; - v->fmt = STRING; - asprintf(&p, "VRT_GetHdr(sp, \"\\%03o%s:\")", - strlen(v->name + vh->len) + 1, v->name + vh->len); - assert(p != NULL); - v->rname = p; - return (v); -} - -/*--------------------------------------------------------------------*/ - -static struct var * -FindVar(struct tokenlist *tl, struct token *t, struct var *vl) -{ - struct var *v; - - for (v = vl; v->name != NULL; v++) { - if (v->fmt == HEADER && t->e - t->b <= v->len) - continue; - if (v->fmt != HEADER && t->e - t->b != v->len) - continue; - if (memcmp(t->b, v->name, v->len)) - continue; - if (v->fmt != HEADER) - return (v); - return (HeaderVar(tl, t, v)); - } - sbuf_printf(tl->sb, "Unknown variable "); - ErrToken(tl, t); - sbuf_cat(tl->sb, "\nAt: "); - ErrWhere(tl, t); - return (NULL); -} - - -/*--------------------------------------------------------------------*/ - -static void -TimeVal(struct tokenlist *tl) -{ - double v, sc; - - v = DoubleVal(tl); - ExpectErr(tl, ID); - sc = TimeUnit(tl); - Fc(tl, 0, "(%g * %g)", v, sc); -} - -static void -SizeVal(struct tokenlist *tl) -{ - double v, sc; - - v = DoubleVal(tl); - ExpectErr(tl, ID); - sc = SizeUnit(tl); - Fc(tl, 0, "(%g * %g)", v, sc); -} - -static void -RateVal(struct tokenlist *tl) -{ - double v, sc; - - v = DoubleVal(tl); - ExpectErr(tl, ID); - sc = RateUnit(tl); - Fc(tl, 0, "(%g * %g)", v, sc); -} - -/*--------------------------------------------------------------------*/ - -static void -Cond_Ip(struct var *vp, struct tokenlist *tl) -{ - unsigned u; - - switch (tl->t->tok) { - case '~': - NextToken(tl); - ExpectErr(tl, ID); - AddRef(tl, tl->t, R_ACL); - Fc(tl, 1, "ip_match(%s, acl_%T)\n", vp->rname, tl->t); - NextToken(tl); - break; - case T_EQ: - case T_NEQ: - Fc(tl, 1, "%s %T ", vp->rname, tl->t); - NextToken(tl); - u = IpVal(tl); - Fc(tl, 0, "%uU /* %u.%u.%u.%u */\n", u, - (u >> 24) & 0xff, (u >> 16) & 0xff, - (u >> 8) & 0xff, (u) & 0xff); - break; - default: - sbuf_printf(tl->sb, "Illegal condition "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, " on IP number variable\n"); - sbuf_printf(tl->sb, " only '==', '!=' and '~' are legal\n"); - ErrWhere(tl, tl->t); - break; - } -} - -static void -Cond_String(struct var *vp, struct tokenlist *tl) -{ - - switch (tl->t->tok) { - case '~': - Fc(tl, 1, "string_match(%s, ", vp->rname); - NextToken(tl); - ExpectErr(tl, CSTR); - Fc(tl, 0, "%T)\n", tl->t); - NextToken(tl); - break; - case T_EQ: - case T_NEQ: - Fc(tl, 1, "%sstrcmp(%s, ", - tl->t->tok == T_EQ ? "!" : "", vp->rname); - NextToken(tl); - ExpectErr(tl, CSTR); - Fc(tl, 0, "%T)\n", tl->t); - NextToken(tl); - break; - default: - Fc(tl, 1, "%s != (void*)0", vp->rname); - break; - } -} - -static void -Cond_Int(struct var *vp, struct tokenlist *tl) -{ - - Fc(tl, 1, "%s ", vp->rname); - switch (tl->t->tok) { - case T_EQ: - case T_NEQ: - case T_LEQ: - case T_GEQ: - case '>': - case '<': - Fc(tl, 0, "%T ", tl->t); - NextToken(tl); - switch(vp->fmt) { - case TIME: - TimeVal(tl); - break; - case INT: - ExpectErr(tl, CNUM); - Fc(tl, 0, "%T ", tl->t); - NextToken(tl); - break; - case SIZE: - SizeVal(tl); - break; - default: - sbuf_printf(tl->sb, - "No conditions available for variable '%s'\n", - vp->name); - ErrWhere(tl, tl->t); - return; - } - Fc(tl, 0, "\n"); - break; - default: - sbuf_printf(tl->sb, "Illegal condition "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, " on integer variable\n"); - sbuf_printf(tl->sb, - " only '==', '!=', '<', '>', '<=' and '>=' are legal\n"); - ErrWhere(tl, tl->t); - break; - } -} - -static void -Cond_Bool(struct var *vp, struct tokenlist *tl) -{ - - Fc(tl, 1, "%s\n", vp->rname); -} - -static void -Cond_2(struct tokenlist *tl) -{ - struct var *vp; - - C(tl, ","); - if (tl->t->tok == '!') { - Fc(tl, 1, "!(\n"); - NextToken(tl); - } else { - Fc(tl, 1, "(\n"); - } - if (tl->t->tok == '(') { - NextToken(tl); - Cond_0(tl); - ExpectErr(tl, ')'); - NextToken(tl); - } else if (tl->t->tok == VAR) { - vp = FindVar(tl, tl->t, vars); - ERRCHK(tl); - assert(vp != NULL); - NextToken(tl); - switch (vp->fmt) { - case INT: L(tl, Cond_Int(vp, tl)); break; - case SIZE: L(tl, Cond_Int(vp, tl)); break; - case BOOL: L(tl, Cond_Bool(vp, tl)); break; - case IP: L(tl, Cond_Ip(vp, tl)); break; - case STRING: L(tl, Cond_String(vp, tl)); break; - case TIME: L(tl, Cond_Int(vp, tl)); break; - /* XXX backend == */ - default: - sbuf_printf(tl->sb, - "Variable '%s'" - " has no conditions that can be checked\n", - vp->name); - ErrWhere(tl, tl->t); - return; - } - } else { - sbuf_printf(tl->sb, - "Syntax error in condition, expected '(', '!' or" - " variable name, found "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, "\n"); - ErrWhere(tl, tl->t); - return; - } - Fc(tl, 1, ")\n"); -} - -static void -Cond_1(struct tokenlist *tl) -{ - - Fc(tl, 1, "(\n"); - L(tl, Cond_2(tl)); - while (tl->t->tok == T_CAND) { - NextToken(tl); - Fc(tl, 1, ") && (\n"); - L(tl, Cond_2(tl)); - } - Fc(tl, 1, ")\n"); -} - -static void -Cond_0(struct tokenlist *tl) -{ - - Fc(tl, 1, "(\n"); - L(tl, Cond_1(tl)); - while (tl->t->tok == T_COR) { - NextToken(tl); - Fc(tl, 1, ") || (\n"); - L(tl, Cond_1(tl)); - } - Fc(tl, 1, ")\n"); -} - -static void -Conditional(struct tokenlist *tl) -{ - - ExpectErr(tl, '('); - NextToken(tl); - Fc(tl, 1, "(\n"); - L(tl, Cond_0(tl)); - ERRCHK(tl); - Fc(tl, 1, ")\n"); - ExpectErr(tl, ')'); - NextToken(tl); -} - -/*--------------------------------------------------------------------*/ - -static void -IfStmt(struct tokenlist *tl) -{ - - ExpectErr(tl, T_IF); - Fc(tl, 1, "if \n"); - NextToken(tl); - L(tl, Conditional(tl)); - ERRCHK(tl); - L(tl, Compound(tl)); - ERRCHK(tl); - while (1) { - switch (tl->t->tok) { - case T_ELSE: - NextToken(tl); - if (tl->t->tok != T_IF) { - Fc(tl, 1, "else \n"); - L(tl, Compound(tl)); - ERRCHK(tl); - return; - } - /* FALLTHROUGH */ - case T_ELSEIF: - case T_ELSIF: - Fc(tl, 1, "else if \n"); - NextToken(tl); - L(tl, Conditional(tl)); - ERRCHK(tl); - L(tl, Compound(tl)); - ERRCHK(tl); - break; - default: - C(tl, ";"); - return; - } - } -} - -/*--------------------------------------------------------------------*/ - -static void -Action(struct tokenlist *tl) -{ - unsigned a, u; - struct var *vp; - struct token *at; - - at = tl->t; - NextToken(tl); - switch (at->tok) { - case T_NO_NEW_CACHE: - Fc(tl, 1, "VCL_no_new_cache(sp);\n"); - return; - case T_NO_CACHE: - Fc(tl, 1, "VCL_no_cache(sp);\n"); - return; -#define VCL_RET_MAC(a,b,c) case T_##b: \ - Fc(tl, 1, "VRT_done(sp, VCL_RET_%s);\n", #b); \ - tl->curproc->returns |= VCL_RET_##b; \ - tl->curproc->returnt[c] = at; \ - return; -#include "vcl_returns.h" -#undef VCL_RET_MAC - case T_ERROR: - if (tl->t->tok == CNUM) - a = UintVal(tl); - else - a = 0; - Fc(tl, 1, "VRT_error(sp, %u, ", a); - if (tl->t->tok == CSTR) { - Fc(tl, 0, "%T);\n", tl->t); - NextToken(tl); - } else - Fc(tl, 0, "(const char *)0);\n"); - Fc(tl, 1, "VRT_done(sp, VCL_RET_ERROR);\n"); - return; - case T_SWITCH_CONFIG: - ExpectErr(tl, ID); - Fc(tl, 1, "VCL_switch_config(\"%T\");\n", tl->t); - NextToken(tl); - return; - case T_CALL: - ExpectErr(tl, ID); - AddCall(tl, tl->t); - AddRef(tl, tl->t, R_FUNC); - Fc(tl, 1, "if (VGC_function_%T(sp))\n", tl->t); - Fc(tl, 1, "\treturn (1);\n"); - NextToken(tl); - return; - case T_REWRITE: - ExpectErr(tl, CSTR); - Fc(tl, 1, "VCL_rewrite(%T", tl->t); - NextToken(tl); - ExpectErr(tl, CSTR); - Fc(tl, 0, ", %T);\n", tl->t); - NextToken(tl); - return; - case T_SET: - ExpectErr(tl, VAR); - vp = FindVar(tl, tl->t, vars); - ERRCHK(tl); - assert(vp != NULL); - Fc(tl, 1, "%s ", vp->rname); - NextToken(tl); - switch (vp->fmt) { - case INT: - case SIZE: - case RATE: - case TIME: - case FLOAT: - Fc(tl, 0, "%T ", tl->t); - a = tl->t->tok; - NextToken(tl); - if (a == T_MUL || a == T_DIV) - Fc(tl, 0, "%g", DoubleVal(tl)); - else if (vp->fmt == TIME) - TimeVal(tl); - else if (vp->fmt == SIZE) - SizeVal(tl); - else if (vp->fmt == RATE) - RateVal(tl); - else - Fc(tl, 0, "%g", DoubleVal(tl)); - Fc(tl, 0, ";\n"); - break; - case IP: - if (tl->t->tok == '=') { - NextToken(tl); - u = IpVal(tl); - Fc(tl, 0, "= %uU; /* %u.%u.%u.%u */\n", - u, - (u >> 24) & 0xff, - (u >> 16) & 0xff, - (u >> 8) & 0xff, - u & 0xff); - break; - } - sbuf_printf(tl->sb, "Illegal assignment operator "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, - " only '=' is legal for IP numbers\n"); - ErrWhere(tl, tl->t); - return; - case BACKEND: - if (tl->t->tok == '=') { - NextToken(tl); - AddRef(tl, tl->t, R_BACKEND); - Fc(tl, 0, "= &VGC_backend_%T;\n", tl->t); - NextToken(tl); - break; - } - sbuf_printf(tl->sb, "Illegal assignment operator "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, - " only '=' is legal for backend\n"); - ErrWhere(tl, tl->t); - return; - default: - sbuf_printf(tl->sb, - "Assignments not possible for '%s'\n", vp->name); - ErrWhere(tl, tl->t); - return; - } - return; - default: - sbuf_printf(tl->sb, "Expected action, 'if' or '}'\n"); - ErrWhere(tl, at); - return; - } -} - -/*--------------------------------------------------------------------*/ - -static void -Acl(struct tokenlist *tl) -{ - unsigned u, m; - - NextToken(tl); - - ExpectErr(tl, ID); - AddDef(tl, tl->t, R_ACL); - Fh(tl, 0, "static struct vcl_acl acl_%T[];\n", tl->t); - Fc(tl, 1, "static struct vcl_acl acl_%T[] = {\n", tl->t); - NextToken(tl); - - tl->indent += INDENT; - - ExpectErr(tl, '{'); - NextToken(tl); - - while (tl->t->tok == CNUM) { - u = IpVal(tl); - if (tl->t->tok == '/') { - NextToken(tl); - ExpectErr(tl, CNUM); - m = UintVal(tl); - } else - m = 32; - ExpectErr(tl, ';'); - NextToken(tl); - Fc(tl, 1, "{ %11uU, %3uU }, /* %u.%u.%u.%u/%u */\n", - u, m, - (u >> 24) & 0xff, (u >> 16) & 0xff, - (u >> 8) & 0xff, (u) & 0xff, m); - } - ExpectErr(tl, '}'); - Fc(tl, 1, "{ %11uU, %3uU }\n", 0, 0); - - tl->indent -= INDENT; - - Fc(tl, 1, "};\n\n"); - NextToken(tl); -} - -/*--------------------------------------------------------------------*/ - -static void -Compound(struct tokenlist *tl) -{ - - ExpectErr(tl, '{'); - Fc(tl, 1, "{\n"); - tl->indent += INDENT; - C(tl, ";"); - NextToken(tl); - while (1) { - ERRCHK(tl); - switch (tl->t->tok) { - case '{': - Compound(tl); - break; - case T_IF: - IfStmt(tl); - break; - case '}': - NextToken(tl); - tl->indent -= INDENT; - Fc(tl, 1, "}\n"); - return; - case EOI: - sbuf_printf(tl->sb, - "End of input while in compound statement\n"); - tl->err = 1; - return; - default: - Action(tl); - ERRCHK(tl); - ExpectErr(tl, ';'); - NextToken(tl); - break; - } - } -} - -/*--------------------------------------------------------------------*/ - -static const char * -CheckHostPort(const char *host, const char *port) -{ - struct addrinfo *res, hint; - int error; - - memset(&hint, 0, sizeof hint); - hint.ai_family = PF_UNSPEC; - hint.ai_socktype = SOCK_STREAM; - error = getaddrinfo(host, port, &hint, &res); - if (error) - return (gai_strerror(error)); - freeaddrinfo(res); - return (NULL); -} - -static void -Backend(struct tokenlist *tl) -{ - struct var *vp; - struct token *t_be = NULL; - struct token *t_host = NULL; - struct token *t_port = NULL; - char *host = NULL; - char *port = NULL; - const char *ep; - - NextToken(tl); - ExpectErr(tl, ID); - t_be = tl->t; - AddDef(tl, tl->t, R_BACKEND); - if (tl->nbackend == 0) - AddRef(tl, tl->t, R_BACKEND); - Fh(tl, 1, "#define VGC_backend_%T (VCL_conf.backend[%d])\n", - tl->t, tl->nbackend); - Fc(tl, 0, "static void\n"); - Fc(tl, 1, "VGC_init_backend_%T (void)\n", tl->t); - Fc(tl, 1, "{\n"); - Fc(tl, 1, "\tstruct backend *backend = VGC_backend_%T;\n", tl->t); - Fc(tl, 1, "\tconst char *p;\n"); - Fc(tl, 1, "\n"); - Fc(tl, 1, "\tVRT_set_backend_name(backend, \"%T\");\n", tl->t); - NextToken(tl); - ExpectErr(tl, '{'); - NextToken(tl); - while (1) { - if (tl->t->tok == '}') - break; - ExpectErr(tl, T_SET); - NextToken(tl); - ExpectErr(tl, VAR); - vp = FindVar(tl, tl->t, be_vars); - ERRCHK(tl); - assert(vp != NULL); - NextToken(tl); - ExpectErr(tl, '='); - NextToken(tl); - switch (vp->fmt) { - case HOSTNAME: - ExpectErr(tl, CSTR); - t_host = tl->t; - Fc(tl, 1, "\tp = %T;\n", tl->t); - Fc(tl, 1, "\t"); - Fc(tl, 0, vp->lname, "p"); - Fc(tl, 0, ";\n"); - NextToken(tl); - break; - case PORTNAME: - ExpectErr(tl, CSTR); - t_port = tl->t; - Fc(tl, 1, "\tp = %T;\n", tl->t); - Fc(tl, 1, "\t"); - Fc(tl, 0, vp->lname, "p"); - Fc(tl, 0, ";\n"); - NextToken(tl); - break; - default: - sbuf_printf(tl->sb, - "Assignments not possible for '%s'\n", vp->name); - ErrWhere(tl, tl->t); - return; - } - ExpectErr(tl, ';'); - NextToken(tl); - } - ExpectErr(tl, '}'); - if (t_host == NULL) { - sbuf_printf(tl->sb, "Backend '%T' has no hostname\n", t_be); - ErrWhere(tl, tl->t); - return; - } - host = EncString(t_host); - ep = CheckHostPort(host, "80"); - if (ep != NULL) { - sbuf_printf(tl->sb, "Backend '%T': %s\n", t_be, ep); - ErrWhere(tl, t_host); - return; - } - if (t_port != NULL) { - port = EncString(t_port); - ep = CheckHostPort(host, port); - if (ep != NULL) { - sbuf_printf(tl->sb, "Backend '%T': %s\n", t_be, ep); - ErrWhere(tl, t_port); - return; - } - } - - NextToken(tl); - Fc(tl, 1, "}\n"); - Fc(tl, 0, "\n"); - tl->nbackend++; -} - -/*--------------------------------------------------------------------*/ - -static void -Function(struct tokenlist *tl) -{ - - NextToken(tl); - ExpectErr(tl, ID); - tl->curproc = AddProc(tl, tl->t, 1); - tl->curproc->exists++; - AddDef(tl, tl->t, R_FUNC); - Fh(tl, 0, "static int VGC_function_%T (struct sess *sp);\n", tl->t); - Fc(tl, 1, "static int\n"); - Fc(tl, 1, "VGC_function_%T (struct sess *sp)\n", tl->t); - NextToken(tl); - L(tl, Compound(tl)); - Fc(tl, 0, "\n"); -} - -/*-------------------------------------------------------------------- - * Top level of parser, recognize: - * Function definitions - * Backend definitions - * End of input - */ - -static void -Parse(struct tokenlist *tl) -{ - - while (tl->t->tok != EOI) { - ERRCHK(tl); - switch (tl->t->tok) { - case T_ACL: - Acl(tl); - break; - case T_SUB: - Function(tl); - break; - case T_BACKEND: - Backend(tl); - break; - case EOI: - break; - default: - sbuf_printf(tl->sb, - "Expected 'acl', 'sub' or 'backend', found "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, " at\n"); - ErrWhere(tl, tl->t); - return; - } - } -} - -/*-------------------------------------------------------------------- - * Add a token to the token list. - */ - -static void -AddToken(struct tokenlist *tl, unsigned tok, const char *b, const char *e) -{ - struct token *t; - - t = calloc(sizeof *t, 1); - assert(t != NULL); - t->tok = tok; - t->b = b; - t->e = e; - TAILQ_INSERT_TAIL(&tl->tokens, t, list); - tl->t = t; - if (0) { - fprintf(stderr, "[%s %*.*s] ", - vcl_tnames[tok], - e - b, e - b, b); - if (tok == EOI) - fprintf(stderr, "\n"); - } -} - -/*-------------------------------------------------------------------- - * Lexical analysis and token generation - */ - -static void -Lexer(struct tokenlist *tl, const char *b, const char *e) -{ - const char *p, *q; - unsigned u; - - for (p = b; p < e; ) { - - /* Skip any whitespace */ - if (isspace(*p)) { - p++; - continue; - } - - /* Skip '#.*\n' comments */ - if (*p == '#') { - while (p < e && *p != '\n') - p++; - continue; - } - - /* Skip C-style comments */ - if (*p == '/' && p[1] == '*') { - p += 2; - for (p += 2; p < e; p++) { - if (*p == '*' && p[1] == '/') { - p += 2; - break; - } - } - continue; - } - - /* Match for the fixed tokens (see token.tcl) */ - u = vcl_fixed_token(p, &q); - if (u != 0) { - AddToken(tl, u, p, q); - p = q; - continue; - } - - /* Match strings, with \\ and \" escapes */ - if (*p == '"') { - for (q = p + 1; q < e; q++) { - if (*q == '\\' && q[1] == '\\') - q++; - else if (*q == '\\' && q[1] == '"') - q++; - else if (*q == '"') { - q++; - break; - } - } - AddToken(tl, CSTR, p, q); - p = q; - continue; - } - - /* Match Identifiers */ - if (isident1(*p)) { - for (q = p; q < e; q++) - if (!isident(*q)) - break; - if (isvar(*q)) { - for (; q < e; q++) - if (!isvar(*q)) - break; - AddToken(tl, VAR, p, q); - } else { - AddToken(tl, ID, p, q); - } - p = q; - continue; - } - - /* Match numbers { [0-9]+ } */ - if (isdigit(*p)) { - for (q = p; q < e; q++) - if (!isdigit(*q)) - break; - AddToken(tl, CNUM, p, q); - p = q; - continue; - } - AddToken(tl, EOI, p, p + 1); - sbuf_printf(tl->sb, "Syntax error at\n"); - ErrWhere(tl, tl->t); - return; - } -} - -/*-------------------------------------------------------------------- - * Consistency check - */ - -static struct proc * -AddProc(struct tokenlist *tl, struct token *t, int def) -{ - struct proc *p; - - TAILQ_FOREACH(p, &tl->procs, list) { - if (!Teq(p->name, t)) - continue; - if (def) - p->name = t; - return (p); - } - p = calloc(sizeof *p, 1); - assert(p != NULL); - p->name = t; - TAILQ_INIT(&p->calls); - TAILQ_INSERT_TAIL(&tl->procs, p, list); - return (p); -} - -static void -AddCall(struct tokenlist *tl, struct token *t) -{ - struct proccall *pc; - struct proc *p; - - p = AddProc(tl, t, 0); - TAILQ_FOREACH(pc, &tl->curproc->calls, list) { - if (pc->p == p) - return; - } - pc = calloc(sizeof *pc, 1); - assert(pc != NULL); - pc->p = p; - pc->t = t; - TAILQ_INSERT_TAIL(&tl->curproc->calls, pc, list); -} - -static int -Consist_Decend(struct tokenlist *tl, struct proc *p, unsigned returns) -{ - unsigned u; - struct proccall *pc; - - if (!p->exists) { - sbuf_printf(tl->sb, "Function %T does not exist\n", p->name); - return (1); - } - if (p->active) { - sbuf_printf(tl->sb, "Function recurses on\n"); - ErrWhere(tl, p->name); - return (1); - } - u = p->returns & ~returns; - if (u) { -#define VCL_RET_MAC(a, b, c) \ - if (u & VCL_RET_##b) { \ - sbuf_printf(tl->sb, "Illegal return for method\n"); \ - ErrWhere(tl, p->returnt[c]); \ - } -#include "vcl_returns.h" -#undef VCL_RET_MAC - sbuf_printf(tl->sb, "In function\n"); - ErrWhere(tl, p->name); - return (1); - } - p->active = 1; - TAILQ_FOREACH(pc, &p->calls, list) { - if (Consist_Decend(tl, pc->p, returns)) { - sbuf_printf(tl->sb, "\nCalled from\n"); - ErrWhere(tl, p->name); - sbuf_printf(tl->sb, "at\n"); - ErrWhere(tl, pc->t); - return (1); - } - } - p->active = 0; - p->called++; - return (0); -} - -static int -Consistency(struct tokenlist *tl) -{ - struct proc *p; - struct method *m; - - TAILQ_FOREACH(p, &tl->procs, list) { - for(m = method_tab; m->name != NULL; m++) { - if (IdIs(p->name, m->defname)) - p->called = 1; - if (IdIs(p->name, m->name)) - break; - } - if (m->name == NULL) - continue; - if (Consist_Decend(tl, p, m->returns)) { - sbuf_printf(tl->sb, - "\nwhich is a %s method\n", m->name); - return (1); - } - } - TAILQ_FOREACH(p, &tl->procs, list) { - if (p->called) - continue; - sbuf_printf(tl->sb, "Function unused\n"); - ErrWhere(tl, p->name); - return (1); - } - return (0); -} - -/*--------------------------------------------------------------------*/ - -static int -CheckRefs(struct tokenlist *tl) -{ - struct ref *r; - const char *type; - int nerr = 0; - - TAILQ_FOREACH(r, &tl->refs, list) { - if (r->defcnt != 0 && r->refcnt != 0) - continue; - nerr++; - - switch(r->type) { - case R_FUNC: - type = "function"; - break; - case R_ACL: - type = "acl"; - break; - case R_BACKEND: - type = "backend"; - break; - default: - ErrInternal(tl); - sbuf_printf(tl->sb, "Ref "); - ErrToken(tl, r->name); - sbuf_printf(tl->sb, " has unknown type %d\n", - r->type); - continue; - } - if (r->defcnt == 0 && r->name->tok == METHOD) { - sbuf_printf(tl->sb, - "No definition for method %T\n", r->name); - continue; - } - - if (r->defcnt == 0) { - sbuf_printf(tl->sb, - "Undefined %s %T, first reference:\n", - type, r->name); - ErrWhere(tl, r->name); - continue; - } - - sbuf_printf(tl->sb, "Unused %s %T, defined:\n", type, r->name); - ErrWhere(tl, r->name); - } - return (nerr); -} - -/*--------------------------------------------------------------------*/ - -static void -LocTable(struct tokenlist *tl) -{ - struct token *t; - unsigned lin, pos; - const char *p; - - Fh(tl, 0, "#define VGC_NREFS %u\n", tl->cnt + 1); - Fh(tl, 0, "static struct vrt_ref VGC_ref[VGC_NREFS];\n"); - Fc(tl, 0, "static struct vrt_ref VGC_ref[VGC_NREFS] = {\n"); - lin = 1; - pos = 0; - p = tl->b; - TAILQ_FOREACH(t, &tl->tokens, list) { - if (t->cnt == 0) - continue; - for (;p < t->b; p++) { - if (*p == '\n') { - lin++; - pos = 0; - } else if (*p == '\t') { - pos &= ~7; - pos += 8; - } else - pos++; - - } - Fc(tl, 0, " [%3u] = { %4u, %3u, 0, \"%T\" },\n", - t->cnt, lin, pos + 1, t); - } - Fc(tl, 0, "};\n"); -} - - -/*--------------------------------------------------------------------*/ - -static void -EmitInitFunc(struct tokenlist *tl) -{ - struct ref *r; - - Fc(tl, 0, "\nstatic void\nVGC_Init(void)\n{\n\n"); - Fc(tl, 0, "\tVRT_alloc_backends(&VCL_conf);\n"); - - TAILQ_FOREACH(r, &tl->refs, list) { - switch(r->type) { - case R_FUNC: - break; - case R_ACL: - break; - case R_BACKEND: - Fc(tl, 0, "\tVGC_init_backend_%T();\n", r->name); - break; - } - } - Fc(tl, 0, "}\n"); -} - -/*--------------------------------------------------------------------*/ - -static void -EmitStruct(struct tokenlist *tl) -{ - - Fc(tl, 0, "\nstruct VCL_conf VCL_conf = {\n"); - Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n"); - Fc(tl, 0, "\t.init_func = VGC_Init,\n"); - Fc(tl, 0, "\t.nbackend = %d,\n", tl->nbackend); - Fc(tl, 0, "\t.ref = VGC_ref,\n"); - Fc(tl, 0, "\t.nref = VGC_NREFS,\n"); -#define VCL_RET_MAC(l,u,b) -#define VCL_MET_MAC(l,u,b) \ - if (FindRefStr(tl, "vcl_" #l, R_FUNC)) { \ - Fc(tl, 0, "\t." #l "_func = VGC_function_vcl_" #l ",\n"); \ - AddRefStr(tl, "vcl_" #l, R_FUNC); \ - } else { \ - Fc(tl, 0, "\t." #l "_func = VGC_function_default_vcl_" #l ",\n"); \ - } \ - AddRefStr(tl, "default_vcl_" #l, R_FUNC); -#include "vcl_returns.h" -#undef VCL_MET_MAC -#undef VCL_RET_MAC - Fc(tl, 0, "};\n"); -} - -/*--------------------------------------------------------------------*/ - -char * -VCC_Compile(struct sbuf *sb, const char *b, const char *e) -{ - struct tokenlist tokens; - struct ref *r; - struct token *t; - FILE *fo; - char *of = NULL; - char buf[BUFSIZ]; - - memset(&tokens, 0, sizeof tokens); - TAILQ_INIT(&tokens.tokens); - TAILQ_INIT(&tokens.refs); - TAILQ_INIT(&tokens.procs); - tokens.sb = sb; - - tokens.fc = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); - assert(tokens.fc != NULL); - - tokens.fh = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); - assert(tokens.fh != NULL); - - Fh(&tokens, 0, "extern struct VCL_conf VCL_conf;\n"); - - tokens.b = b; - if (e == NULL) - e = strchr(b, '\0'); - assert(e != NULL); - tokens.e = e; - Lexer(&tokens, b, e); - Lexer(&tokens, vcc_default_vcl_b, vcc_default_vcl_e); - AddToken(&tokens, EOI, e, e); - if (tokens.err) - goto done; - tokens.t = TAILQ_FIRST(&tokens.tokens); - Parse(&tokens); - if (tokens.err) - goto done; - Consistency(&tokens); - if (tokens.err) - goto done; - LocTable(&tokens); - - EmitInitFunc(&tokens); - - EmitStruct(&tokens); - - if (CheckRefs(&tokens)) - goto done; - - of = strdup("/tmp/vcl.XXXXXXXX"); - assert(of != NULL); - mktemp(of); - - sprintf(buf, - "tee /tmp/_.c |" - "cc -fpic -shared -Wl,-x -o %s -x c - ", of); - - fo = popen(buf, "w"); - assert(fo != NULL); - - vcl_output_lang_h(fo); - - sbuf_finish(tokens.fh); - fputs(sbuf_data(tokens.fh), fo); - sbuf_delete(tokens.fh); - - sbuf_finish(tokens.fc); - fputs(sbuf_data(tokens.fc), fo); - sbuf_delete(tokens.fc); - - pclose(fo); -done: - - /* Free References */ - while (!TAILQ_EMPTY(&tokens.refs)) { - r = TAILQ_FIRST(&tokens.refs); - TAILQ_REMOVE(&tokens.refs, r, list); - free(r); - } - - /* Free Tokens */ - while (!TAILQ_EMPTY(&tokens.tokens)) { - t = TAILQ_FIRST(&tokens.tokens); - TAILQ_REMOVE(&tokens.tokens, t, list); - free(t); - } - return (of); -} - -/*--------------------------------------------------------------------*/ - -char * -VCC_CompileFile(struct sbuf *sb, const char *fn) -{ - char *f, *r; - int fd, i; - struct stat st; - - fd = open(fn, O_RDONLY); - if (fd < 0) { - sbuf_printf(sb, "Cannot open file '%s': %s", - fn, strerror(errno)); - return (NULL); - } - assert(0 == fstat(fd, &st)); - f = malloc(st.st_size + 1); - assert(f != NULL); - i = read(fd, f, st.st_size); - assert(i == st.st_size); - f[i] = '\0'; - r = VCC_Compile(sb, f, NULL); - free(f); - return (r); -} - -/*--------------------------------------------------------------------*/ - -static int -VCC_T_render(FILE *f, const struct printf_info *info __unused, const void *const *args) -{ - const struct token *t; - - t = *((const struct token * const*) (args[0])); - return (fprintf(f, "%*.*s", - t->e - t->b, t->e - t->b, t->b)); -} - -static int -VCC_T_arginfo(const struct printf_info *info __unused, size_t n, int *argtypes) -{ - - if (n > 0) - argtypes[0] = PA_POINTER; - return 1; -} - -/*--------------------------------------------------------------------*/ - -void -VCC_InitCompile(const char *default_vcl) -{ - struct var *v; - - vcc_default_vcl_b = default_vcl; - vcc_default_vcl_e = strchr(default_vcl, '\0'); - assert(vcc_default_vcl_e != NULL); - - register_printf_function ('T', VCC_T_render, VCC_T_arginfo); - vcl_init_tnames(); - for (v = vars; v->name != NULL; v++) - v->len = strlen(v->name); - for (v = be_vars; v->name != NULL; v++) - v->len = strlen(v->name); -} - -#endif +const char *vrt_obj_h = + "/*\n" + " * $Id$\n" + " *\n" + " * NB: This file is machine generated, DO NOT EDIT!\n" + " *\n" + " * Edit vcc_gen_obj.tcl instead\n" + " */\n" + "\n" + "char * VRT_r_backend_host(struct backend *);\n" + "void VRT_l_backend_host(struct backend *, char *);\n" + "char * VRT_r_backend_port(struct backend *);\n" + "void VRT_l_backend_port(struct backend *, char *);\n" + "char * VRT_r_req_request(struct sess *);\n" + "void VRT_l_req_request(struct sess *, char *);\n" + "char * VRT_r_req_url(struct sess *);\n" + "void VRT_l_req_url(struct sess *, char *);\n" + "double VRT_r_obj_valid(struct sess *);\n" + "void VRT_l_obj_valid(struct sess *, double);\n" + "double VRT_r_obj_cacheable(struct sess *);\n" + "void VRT_l_obj_cacheable(struct sess *, double);\n" + "struct backend * VRT_r_obj_backend(struct sess *);\n" + "void VRT_l_obj_backend(struct sess *, struct backend *);\n" + "double VRT_r_obj_ttl(struct sess *);\n" + "void VRT_l_obj_ttl(struct sess *, double);\n" + "char * VRT_r_req_http_(struct sess *);\n" + "void VRT_l_req_http_(struct sess *, char *);\n" +; Property changes on: trunk/varnish-cache/lib/libvcl/vcc_obj.c ___________________________________________________________________ Name: svn:keywords + Id Property changes on: trunk/varnish-cache/lib/libvcl/vcc_token.c ___________________________________________________________________ Name: svn:keywords + Id From phk at projects.linpro.no Fri Jul 21 20:44:58 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 22:44:58 +0200 (CEST) Subject: r546 - trunk/varnish-cache/include Message-ID: <20060721204458.738B21EC38E@projects.linpro.no> Author: phk Date: 2006-07-21 22:44:58 +0200 (Fri, 21 Jul 2006) New Revision: 546 Added: trunk/varnish-cache/include/vrt_obj.h Log: This file is generated. Added: trunk/varnish-cache/include/vrt_obj.h =================================================================== --- trunk/varnish-cache/include/vrt_obj.h 2006-07-21 20:43:56 UTC (rev 545) +++ trunk/varnish-cache/include/vrt_obj.h 2006-07-21 20:44:58 UTC (rev 546) @@ -0,0 +1,26 @@ +/* + * $Id$ + * + * NB: This file is machine generated, DO NOT EDIT! + * + * Edit vcc_gen_obj.tcl instead + */ + +char * VRT_r_backend_host(struct backend *); +void VRT_l_backend_host(struct backend *, char *); +char * VRT_r_backend_port(struct backend *); +void VRT_l_backend_port(struct backend *, char *); +char * VRT_r_req_request(struct sess *); +void VRT_l_req_request(struct sess *, char *); +char * VRT_r_req_url(struct sess *); +void VRT_l_req_url(struct sess *, char *); +double VRT_r_obj_valid(struct sess *); +void VRT_l_obj_valid(struct sess *, double); +double VRT_r_obj_cacheable(struct sess *); +void VRT_l_obj_cacheable(struct sess *, double); +struct backend * VRT_r_obj_backend(struct sess *); +void VRT_l_obj_backend(struct sess *, struct backend *); +double VRT_r_obj_ttl(struct sess *); +void VRT_l_obj_ttl(struct sess *, double); +char * VRT_r_req_http_(struct sess *); +void VRT_l_req_http_(struct sess *, char *); From phk at projects.linpro.no Fri Jul 21 20:45:13 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 22:45:13 +0200 (CEST) Subject: r547 - trunk/varnish-cache/include Message-ID: <20060721204513.CD34D1EC38D@projects.linpro.no> Author: phk Date: 2006-07-21 22:45:13 +0200 (Fri, 21 Jul 2006) New Revision: 547 Modified: trunk/varnish-cache/include/binary_heap.h trunk/varnish-cache/include/miniobj.h trunk/varnish-cache/include/stat_field.h trunk/varnish-cache/include/stats.h trunk/varnish-cache/include/vcl.h trunk/varnish-cache/include/vcl_returns.h Log: Update Property changes on: trunk/varnish-cache/include/binary_heap.h ___________________________________________________________________ Name: svn:keywords + Id Property changes on: trunk/varnish-cache/include/miniobj.h ___________________________________________________________________ Name: svn:keywords + Id Property changes on: trunk/varnish-cache/include/stat_field.h ___________________________________________________________________ Name: svn:keywords + Id Property changes on: trunk/varnish-cache/include/stats.h ___________________________________________________________________ Name: svn:keywords + Id Modified: trunk/varnish-cache/include/vcl.h =================================================================== --- trunk/varnish-cache/include/vcl.h 2006-07-21 20:44:58 UTC (rev 546) +++ trunk/varnish-cache/include/vcl.h 2006-07-21 20:45:13 UTC (rev 547) @@ -3,7 +3,7 @@ * * NB: This file is machine generated, DO NOT EDIT! * - * Edit vcl_gen_fixed_token.tcl instead + * Edit vcc_gen_fixed_token.tcl instead */ struct sess; Modified: trunk/varnish-cache/include/vcl_returns.h =================================================================== --- trunk/varnish-cache/include/vcl_returns.h 2006-07-21 20:44:58 UTC (rev 546) +++ trunk/varnish-cache/include/vcl_returns.h 2006-07-21 20:45:13 UTC (rev 547) @@ -3,7 +3,7 @@ * * NB: This file is machine generated, DO NOT EDIT! * - * Edit vcl_gen_fixed_token.tcl instead + * Edit vcc_gen_fixed_token.tcl instead */ #ifdef VCL_RET_MAC From phk at projects.linpro.no Fri Jul 21 20:51:24 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 22:51:24 +0200 (CEST) Subject: r548 - in trunk/varnish-cache: include lib/libvcl Message-ID: <20060721205124.EAFB91EC396@projects.linpro.no> Author: phk Date: 2006-07-21 22:51:24 +0200 (Fri, 21 Jul 2006) New Revision: 548 Modified: trunk/varnish-cache/include/vrt_obj.h trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl trunk/varnish-cache/lib/libvcl/vcc_obj.c Log: Use const char* for safety Modified: trunk/varnish-cache/include/vrt_obj.h =================================================================== --- trunk/varnish-cache/include/vrt_obj.h 2006-07-21 20:45:13 UTC (rev 547) +++ trunk/varnish-cache/include/vrt_obj.h 2006-07-21 20:51:24 UTC (rev 548) @@ -1,19 +1,19 @@ /* - * $Id$ + * $Id: vcc_gen_obj.tcl 545 2006-07-21 20:43:56Z phk $ * * NB: This file is machine generated, DO NOT EDIT! * * Edit vcc_gen_obj.tcl instead */ -char * VRT_r_backend_host(struct backend *); -void VRT_l_backend_host(struct backend *, char *); -char * VRT_r_backend_port(struct backend *); -void VRT_l_backend_port(struct backend *, char *); -char * VRT_r_req_request(struct sess *); -void VRT_l_req_request(struct sess *, char *); -char * VRT_r_req_url(struct sess *); -void VRT_l_req_url(struct sess *, char *); +const char * VRT_r_backend_host(struct backend *); +void VRT_l_backend_host(struct backend *, const char *); +const char * VRT_r_backend_port(struct backend *); +void VRT_l_backend_port(struct backend *, const char *); +const char * VRT_r_req_request(struct sess *); +void VRT_l_req_request(struct sess *, const char *); +const char * VRT_r_req_url(struct sess *); +void VRT_l_req_url(struct sess *, const char *); double VRT_r_obj_valid(struct sess *); void VRT_l_obj_valid(struct sess *, double); double VRT_r_obj_cacheable(struct sess *); @@ -22,5 +22,5 @@ void VRT_l_obj_backend(struct sess *, struct backend *); double VRT_r_obj_ttl(struct sess *); void VRT_l_obj_ttl(struct sess *, double); -char * VRT_r_req_http_(struct sess *); -void VRT_l_req_http_(struct sess *, char *); +const char * VRT_r_req_http_(struct sess *); +void VRT_l_req_http_(struct sess *, const char *); Modified: trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl 2006-07-21 20:45:13 UTC (rev 547) +++ trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl 2006-07-21 20:51:24 UTC (rev 548) @@ -21,13 +21,13 @@ { req.http. HEADER } } -set tt(STRING) "char *" +set tt(STRING) "const char *" set tt(BOOL) "double" set tt(BACKEND) "struct backend *" set tt(TIME) "double" -set tt(HEADER) "char *" -set tt(HOSTNAME) "char *" -set tt(PORTNAME) "char *" +set tt(HEADER) "const char *" +set tt(HOSTNAME) "const char *" +set tt(PORTNAME) "const char *" #---------------------------------------------------------------------- # Boilerplate warning for all generated files. Modified: trunk/varnish-cache/lib/libvcl/vcc_obj.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_obj.c 2006-07-21 20:45:13 UTC (rev 547) +++ trunk/varnish-cache/lib/libvcl/vcc_obj.c 2006-07-21 20:51:24 UTC (rev 548) @@ -62,14 +62,14 @@ " * Edit vcc_gen_obj.tcl instead\n" " */\n" "\n" - "char * VRT_r_backend_host(struct backend *);\n" - "void VRT_l_backend_host(struct backend *, char *);\n" - "char * VRT_r_backend_port(struct backend *);\n" - "void VRT_l_backend_port(struct backend *, char *);\n" - "char * VRT_r_req_request(struct sess *);\n" - "void VRT_l_req_request(struct sess *, char *);\n" - "char * VRT_r_req_url(struct sess *);\n" - "void VRT_l_req_url(struct sess *, char *);\n" + "const char * VRT_r_backend_host(struct backend *);\n" + "void VRT_l_backend_host(struct backend *, const char *);\n" + "const char * VRT_r_backend_port(struct backend *);\n" + "void VRT_l_backend_port(struct backend *, const char *);\n" + "const char * VRT_r_req_request(struct sess *);\n" + "void VRT_l_req_request(struct sess *, const char *);\n" + "const char * VRT_r_req_url(struct sess *);\n" + "void VRT_l_req_url(struct sess *, const char *);\n" "double VRT_r_obj_valid(struct sess *);\n" "void VRT_l_obj_valid(struct sess *, double);\n" "double VRT_r_obj_cacheable(struct sess *);\n" @@ -78,6 +78,6 @@ "void VRT_l_obj_backend(struct sess *, struct backend *);\n" "double VRT_r_obj_ttl(struct sess *);\n" "void VRT_l_obj_ttl(struct sess *, double);\n" - "char * VRT_r_req_http_(struct sess *);\n" - "void VRT_l_req_http_(struct sess *, char *);\n" + "const char * VRT_r_req_http_(struct sess *);\n" + "void VRT_l_req_http_(struct sess *, const char *);\n" ; From phk at projects.linpro.no Fri Jul 21 21:01:45 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 23:01:45 +0200 (CEST) Subject: r549 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060721210145.ECBCA1EC396@projects.linpro.no> Author: phk Date: 2006-07-21 23:01:45 +0200 (Fri, 21 Jul 2006) New Revision: 549 Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/include/vrt.h Log: Update VRT to minimal functional level again Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-21 20:51:24 UTC (rev 548) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-21 21:01:45 UTC (rev 549) @@ -11,6 +11,7 @@ #include "shmlog.h" #include "vrt.h" +#include "vrt_obj.h" #include "vcl.h" #include "cache.h" @@ -74,32 +75,34 @@ sp->handling = hand; } -int -VRT_obj_valid(struct sess *sp) +/*--------------------------------------------------------------------*/ + +void +VRT_l_backend_host(struct backend *be, const char *h) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - return (sp->obj->valid); + CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); + be->hostname = h; } -int -VRT_obj_cacheable(struct sess *sp) +const char * +VRT_r_backend_host(struct backend *be) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - return (sp->obj->cacheable); + CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); + return (be->hostname); } void -VRT_set_backend_hostname(struct backend *be, const char *h) +VRT_l_backend_port(struct backend *be, const char *p) { CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); - be->hostname = h; + be->portname = p; } -void -VRT_set_backend_portname(struct backend *be, const char *p) +const char * +VRT_r_backend_port(struct backend *be) { CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); - be->portname = p; + return (be->portname); } void @@ -122,3 +125,59 @@ cp->backend[i]->magic = BACKEND_MAGIC; } } + +/*--------------------------------------------------------------------*/ + +void +VRT_l_obj_ttl(struct sess *sp, double a) +{ + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + (void)a; +} + +double +VRT_r_obj_ttl(struct sess *sp) +{ + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); /* XXX */ + return (sp->obj->ttl - sp->t_req); +} + + +double +VRT_r_obj_valid(struct sess *sp) +{ + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); /* XXX */ + return (sp->obj->valid); +} + + +double +VRT_r_obj_cacheable(struct sess *sp) +{ + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); /* XXX */ + return (sp->obj->cacheable); +} + +/*--------------------------------------------------------------------*/ + +const char * +VRT_r_req_request(struct sess *sp) +{ + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC); + return (sp->http->hd[HTTP_HDR_REQ].b); +} + + +const char * +VRT_r_req_url(struct sess *sp) +{ + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC); + return (sp->http->hd[HTTP_HDR_URL].b); +} + Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2006-07-21 20:51:24 UTC (rev 548) +++ trunk/varnish-cache/include/vrt.h 2006-07-21 21:01:45 UTC (rev 549) @@ -24,8 +24,6 @@ }; void VRT_count(struct sess *, unsigned); -void VRT_no_cache(struct sess *); -void VRT_no_new_cache(struct sess *); #if 0 int ip_match(unsigned, struct vcl_acl *); int string_match(const char *, const char *); @@ -35,14 +33,9 @@ int VRT_switch_config(const char *); char *VRT_GetHdr(struct sess *, const char *); -char *VRT_GetReq(struct sess *); void VRT_handling(struct sess *sp, unsigned hand); -int VRT_obj_valid(struct sess *); -int VRT_obj_cacheable(struct sess *); void VRT_set_backend_name(struct backend *, const char *); -void VRT_set_backend_hostname(struct backend *, const char *); -void VRT_set_backend_portname(struct backend *, const char *); void VRT_alloc_backends(struct VCL_conf *cp); From phk at projects.linpro.no Fri Jul 21 21:13:44 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 23:13:44 +0200 (CEST) Subject: r550 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060721211344.081D41EC396@projects.linpro.no> Author: phk Date: 2006-07-21 23:13:43 +0200 (Fri, 21 Jul 2006) New Revision: 550 Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/include/vrt_obj.h trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl trunk/varnish-cache/lib/libvcl/vcc_obj.c Log: More VRT work. Use macros for trivial objects which are just a field in a struct. Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-21 21:01:45 UTC (rev 549) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-21 21:13:43 UTC (rev 550) @@ -54,18 +54,6 @@ /*--------------------------------------------------------------------*/ -char * -VRT_GetReq(struct sess *sp) -{ - - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - assert(sp != NULL); - assert(sp->http != NULL); - return (sp->http->hd[HTTP_HDR_REQ].b); -} - -/*--------------------------------------------------------------------*/ - void VRT_handling(struct sess *sp, unsigned hand) { @@ -78,34 +66,6 @@ /*--------------------------------------------------------------------*/ void -VRT_l_backend_host(struct backend *be, const char *h) -{ - CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); - be->hostname = h; -} - -const char * -VRT_r_backend_host(struct backend *be) -{ - CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); - return (be->hostname); -} - -void -VRT_l_backend_port(struct backend *be, const char *p) -{ - CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); - be->portname = p; -} - -const char * -VRT_r_backend_port(struct backend *be) -{ - CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); - return (be->portname); -} - -void VRT_set_backend_name(struct backend *be, const char *p) { CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); @@ -128,6 +88,26 @@ /*--------------------------------------------------------------------*/ +#define VBACKEND(type,onm,field) \ +void \ +VRT_l_backend_##onm(struct backend *be, type a) \ +{ \ + CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); \ + be->field = a; \ +} \ + \ +type \ +VRT_r_backend_##onm(struct backend *be) \ +{ \ + CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); \ + return (be->field); \ +} + +VBACKEND(const char *, host, hostname) +VBACKEND(const char *, port, portname) + +/*--------------------------------------------------------------------*/ + void VRT_l_obj_ttl(struct sess *sp, double a) { @@ -144,40 +124,39 @@ return (sp->obj->ttl - sp->t_req); } +/*--------------------------------------------------------------------*/ -double -VRT_r_obj_valid(struct sess *sp) -{ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); /* XXX */ - return (sp->obj->valid); +#define VOBJ(type,onm,field) \ +void \ +VRT_l_obj_##onm(struct sess *sp, type a) \ +{ \ + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); /* XXX */ \ + sp->obj->field = a; \ +} \ + \ +type \ +VRT_r_obj_##onm(struct sess *sp) \ +{ \ + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); /* XXX */ \ + return (sp->obj->field); \ } +VOBJ(double, valid, valid) +VOBJ(double, cacheable, cacheable) -double -VRT_r_obj_cacheable(struct sess *sp) -{ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); /* XXX */ - return (sp->obj->cacheable); -} - /*--------------------------------------------------------------------*/ -const char * -VRT_r_req_request(struct sess *sp) -{ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC); - return (sp->http->hd[HTTP_HDR_REQ].b); +#define VREQ(n1, n2) \ +const char * \ +VRT_r_req_##n1(struct sess *sp) \ +{ \ + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ + CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC); \ + return (sp->http->hd[n2].b); \ } - -const char * -VRT_r_req_url(struct sess *sp) -{ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC); - return (sp->http->hd[HTTP_HDR_URL].b); -} - +VREQ(request, HTTP_HDR_REQ) +VREQ(url, HTTP_HDR_URL) +VREQ(proto, HTTP_HDR_PROTO) Modified: trunk/varnish-cache/include/vrt_obj.h =================================================================== --- trunk/varnish-cache/include/vrt_obj.h 2006-07-21 21:01:45 UTC (rev 549) +++ trunk/varnish-cache/include/vrt_obj.h 2006-07-21 21:13:43 UTC (rev 550) @@ -1,5 +1,5 @@ /* - * $Id: vcc_gen_obj.tcl 545 2006-07-21 20:43:56Z phk $ + * $Id: vcc_gen_obj.tcl 548 2006-07-21 20:51:24Z phk $ * * NB: This file is machine generated, DO NOT EDIT! * @@ -14,6 +14,8 @@ void VRT_l_req_request(struct sess *, const char *); const char * VRT_r_req_url(struct sess *); void VRT_l_req_url(struct sess *, const char *); +const char * VRT_r_req_proto(struct sess *); +void VRT_l_req_proto(struct sess *, const char *); double VRT_r_obj_valid(struct sess *); void VRT_l_obj_valid(struct sess *, double); double VRT_r_obj_cacheable(struct sess *); Modified: trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl 2006-07-21 21:01:45 UTC (rev 549) +++ trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl 2006-07-21 21:13:43 UTC (rev 550) @@ -14,6 +14,7 @@ set spobj { { req.request STRING } { req.url STRING } + { req.proto STRING } { obj.valid BOOL } { obj.cacheable BOOL } { obj.backend BACKEND } Modified: trunk/varnish-cache/lib/libvcl/vcc_obj.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_obj.c 2006-07-21 21:01:45 UTC (rev 549) +++ trunk/varnish-cache/lib/libvcl/vcc_obj.c 2006-07-21 21:13:43 UTC (rev 550) @@ -30,6 +30,10 @@ "VRT_r_req_url(sp)", "VRT_l_req_url(sp, ", }, + { "req.proto", STRING, 9, + "VRT_r_req_proto(sp)", + "VRT_l_req_proto(sp, ", + }, { "obj.valid", BOOL, 9, "VRT_r_obj_valid(sp)", "VRT_l_obj_valid(sp, ", @@ -70,6 +74,8 @@ "void VRT_l_req_request(struct sess *, const char *);\n" "const char * VRT_r_req_url(struct sess *);\n" "void VRT_l_req_url(struct sess *, const char *);\n" + "const char * VRT_r_req_proto(struct sess *);\n" + "void VRT_l_req_proto(struct sess *, const char *);\n" "double VRT_r_obj_valid(struct sess *);\n" "void VRT_l_obj_valid(struct sess *, double);\n" "double VRT_r_obj_cacheable(struct sess *);\n" From phk at projects.linpro.no Fri Jul 21 21:28:51 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 23:28:51 +0200 (CEST) Subject: r551 - trunk/varnish-cache/bin/varnishd Message-ID: <20060721212851.25C8F1EC396@projects.linpro.no> Author: phk Date: 2006-07-21 23:28:51 +0200 (Fri, 21 Jul 2006) New Revision: 551 Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishd/rfc2616.c Log: Implement TTL adjustment from VCL Log in shmem where the TTL came from (doc-candidate): 696613561 RFC 900 1153517009 1153517014 1153517914 900 0 | | | | | | | | | | | | | | | age | | | | | | max-age | | | | | Expires: header | | | | Date: header | | | "now" | | TTL relative to "now" | who set the TTL xid of object or 696613561 VCL 20 1153517009 | | | | | | | "now" | | TTL relative to "now" | who set the TTL xid of object Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-21 21:13:43 UTC (rev 550) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-21 21:28:51 UTC (rev 551) @@ -113,7 +113,11 @@ { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - (void)a; + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); /* XXX */ + VSL(SLT_TTL, sp->fd, "%u VCL %.0f %u", sp->obj->xid, a, sp->t_req); + sp->obj->ttl = sp->t_req + a; + if (sp->obj->heap_idx != 0) + EXP_TTLchange(sp->obj); } double Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-21 21:13:43 UTC (rev 550) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-21 21:28:51 UTC (rev 551) @@ -71,7 +71,7 @@ #endif static time_t -RFC2616_Ttl(int fd, struct http *hp, time_t t_req, time_t t_resp, struct object *obj) +RFC2616_Ttl(struct sess *sp, struct http *hp, time_t t_req, time_t t_resp, struct object *obj) { int retirement_age; unsigned u1, u2; @@ -119,7 +119,7 @@ } /* calculated TTL, Our time, Date, Expires, max-age, age */ - VSL(SLT_TTL, fd, "%d %d %d %d %d %d", + VSL(SLT_TTL, sp->fd, "%u RFC %d %d %d %d %d %d", sp->xid, (int)(ttd - t_req), (int)t_req, (int)h_date, (int)h_expires, (int)u1, (int)u2); @@ -155,7 +155,7 @@ break; } - sp->obj->ttl = RFC2616_Ttl(sp->fd, hp, sp->t_req, sp->t_resp, sp->obj); + sp->obj->ttl = RFC2616_Ttl(sp, hp, sp->t_req, sp->t_resp, sp->obj); sp->obj->entered = sp->t_req; if (sp->obj->ttl == 0) { sp->obj->cacheable = 0; From phk at projects.linpro.no Fri Jul 21 21:42:00 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 23:42:00 +0200 (CEST) Subject: r552 - trunk/varnish-cache Message-ID: <20060721214200.79BFE1EC396@projects.linpro.no> Author: phk Date: 2006-07-21 23:42:00 +0200 (Fri, 21 Jul 2006) New Revision: 552 Removed: trunk/varnish-cache/autogen.phk Log: Drop this one now. Deleted: trunk/varnish-cache/autogen.phk =================================================================== --- trunk/varnish-cache/autogen.phk 2006-07-21 21:28:51 UTC (rev 551) +++ trunk/varnish-cache/autogen.phk 2006-07-21 21:42:00 UTC (rev 552) @@ -1,35 +0,0 @@ -#!/bin/sh -# -# $Id$ -# - -set -ex - -if [ -d /usr/local/gnu-autotools/bin ] ; then - PATH=${PATH}:/usr/local/gnu-autotools/bin - export PATH -fi - -base=$(cd $(dirname $0) && pwd) -for dir in $base $base/contrib/libevent ; do - ( - echo $dir - cd $dir - aclocal - libtoolize --copy --force - aclocal - autoheader - automake --add-missing --copy --foreign - autoconf - ) -done - -sh configure \ - --enable-pedantic \ - --enable-wall \ - --enable-werror \ - --enable-dependency-tracking \ - "CFLAGS=-std=c99 -Wno-format" - -# This is a safety-measure during development -( cd lib/libvcl && ./*.tcl ) From phk at projects.linpro.no Fri Jul 21 21:57:44 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 21 Jul 2006 23:57:44 +0200 (CEST) Subject: r553 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060721215744.1A7A71EC396@projects.linpro.no> Author: phk Date: 2006-07-21 23:57:43 +0200 (Fri, 21 Jul 2006) New Revision: 553 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_vcl.c trunk/varnish-cache/include/vcl_returns.h trunk/varnish-cache/lib/libvcl/vcc_compile.c trunk/varnish-cache/lib/libvcl/vcc_compile.h trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl trunk/varnish-cache/lib/libvcl/vcc_token.c Log: Make FlexeLint happier Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-21 21:42:00 UTC (rev 552) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-21 21:57:43 UTC (rev 553) @@ -366,7 +366,7 @@ struct VCL_conf *VCL_Get(void); int VCL_Load(const char *fn, const char *name, struct cli *cli); -#define VCL_RET_MAC(l,u,b) +#define VCL_RET_MAC(l,u,b,n) #define VCL_MET_MAC(l,u,b) void VCL_##l##_method(struct sess *); #include "vcl_returns.h" #undef VCL_MET_MAC Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-07-21 21:42:00 UTC (rev 552) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-07-21 21:57:43 UTC (rev 553) @@ -206,8 +206,8 @@ { switch (u) { -#define VCL_RET_MAC(a, b, c) case VCL_RET_##b: return(#a); -#define VCL_RET_MAC_E(a, b, c) case VCL_RET_##b: return(#a); +#define VCL_RET_MAC(a, b, c,d) case VCL_RET_##b: return(#a); +#define VCL_RET_MAC_E(a, b, c,d) case VCL_RET_##b: return(#a); #include "vcl_returns.h" #undef VCL_RET_MAC #undef VCL_RET_MAC_E @@ -216,7 +216,7 @@ } } -#define VCL_RET_MAC(l,u,b) +#define VCL_RET_MAC(l,u,b,n) #define VCL_MET_MAC(func, xxx, bitmap) \ void \ Modified: trunk/varnish-cache/include/vcl_returns.h =================================================================== --- trunk/varnish-cache/include/vcl_returns.h 2006-07-21 21:42:00 UTC (rev 552) +++ trunk/varnish-cache/include/vcl_returns.h 2006-07-21 21:57:43 UTC (rev 553) @@ -1,5 +1,5 @@ /* - * $Id$ + * $Id: vcc_gen_fixed_token.tcl 545 2006-07-21 20:43:56Z phk $ * * NB: This file is machine generated, DO NOT EDIT! * @@ -8,16 +8,16 @@ #ifdef VCL_RET_MAC #ifdef VCL_RET_MAC_E -VCL_RET_MAC_E(error, ERROR, 0) +VCL_RET_MAC_E(error, ERROR, (1 << 0), 0) #endif -VCL_RET_MAC(lookup, LOOKUP, (1 << 1)) -VCL_RET_MAC(pipe, PIPE, (1 << 2)) -VCL_RET_MAC(pass, PASS, (1 << 3)) -VCL_RET_MAC(insert_pass, INSERT_PASS, (1 << 4)) -VCL_RET_MAC(fetch, FETCH, (1 << 5)) -VCL_RET_MAC(insert, INSERT, (1 << 6)) -VCL_RET_MAC(deliver, DELIVER, (1 << 7)) -VCL_RET_MAC(discard, DISCARD, (1 << 8)) +VCL_RET_MAC(lookup, LOOKUP, (1 << 1), 1) +VCL_RET_MAC(pipe, PIPE, (1 << 2), 2) +VCL_RET_MAC(pass, PASS, (1 << 3), 3) +VCL_RET_MAC(insert_pass, INSERT_PASS, (1 << 4), 4) +VCL_RET_MAC(fetch, FETCH, (1 << 5), 5) +VCL_RET_MAC(insert, INSERT, (1 << 6), 6) +VCL_RET_MAC(deliver, DELIVER, (1 << 7), 7) +VCL_RET_MAC(discard, DISCARD, (1 << 8), 8) #else #define VCL_RET_ERROR (1 << 0) #define VCL_RET_LOOKUP (1 << 1) Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.c 2006-07-21 21:42:00 UTC (rev 552) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.c 2006-07-21 21:57:43 UTC (rev 553) @@ -58,6 +58,15 @@ #define ERRCHK(tl) do { if ((tl)->err) return; } while (0) +static struct method method_tab[] = { +#define VCL_RET_MAC(a,b,c,d) +#define VCL_MET_MAC(a,b,c) { "vcl_"#a, "default_vcl_"#a, c }, +#include "vcl_returns.h" +#undef VCL_MET_MAC +#undef VCL_RET_MAC + { NULL, 0U } +}; + /*--------------------------------------------------------------------*/ static void Compound(struct tokenlist *tl); @@ -214,6 +223,7 @@ struct token *t; t = calloc(sizeof *t, 1); + assert(t != NULL); t->b = s; t->e = strchr(s, '\0'); t->tok = METHOD; @@ -745,10 +755,10 @@ case T_NO_CACHE: Fc(tl, 1, "VCL_no_cache(sp);\n"); return; -#define VCL_RET_MAC(a,b,c) case T_##b: \ +#define VCL_RET_MAC(a,b,c,d) case T_##b: \ Fc(tl, 1, "VRT_done(sp, VCL_RET_%s);\n", #b); \ tl->curproc->returns |= VCL_RET_##b; \ - tl->curproc->returnt[c] = at; \ + tl->curproc->returnt[d] = at; \ return; #include "vcl_returns.h" #undef VCL_RET_MAC @@ -1169,10 +1179,10 @@ } u = p->returns & ~returns; if (u) { -#define VCL_RET_MAC(a, b, c) \ +#define VCL_RET_MAC(a, b, c, d) \ if (u & VCL_RET_##b) { \ sbuf_printf(tl->sb, "Illegal return for method\n"); \ - vcc_ErrWhere(tl, p->returnt[c]); \ + vcc_ErrWhere(tl, p->returnt[d]); \ } #include "vcl_returns.h" #undef VCL_RET_MAC @@ -1350,7 +1360,7 @@ Fc(tl, 0, "\t.nbackend = %d,\n", tl->nbackend); Fc(tl, 0, "\t.ref = VGC_ref,\n"); Fc(tl, 0, "\t.nref = VGC_NREFS,\n"); -#define VCL_RET_MAC(l,u,b) +#define VCL_RET_MAC(l,u,b,n) #define VCL_MET_MAC(l,u,b) \ if (FindRefStr(tl, "vcl_" #l, R_FUNC)) { \ Fc(tl, 0, "\t." #l "_func = VGC_function_vcl_" #l ",\n"); \ Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.h 2006-07-21 21:42:00 UTC (rev 552) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.h 2006-07-21 21:57:43 UTC (rev 553) @@ -68,17 +68,10 @@ const char *lname; }; -static struct method { +struct method { const char *name; const char *defname; unsigned returns; -} method_tab[] = { -#define VCL_RET_MAC(a,b,c) -#define VCL_MET_MAC(a,b,c) { "vcl_"#a, "default_vcl_"#a, c }, -#include "vcl_returns.h" -#undef VCL_MET_MAC -#undef VCL_RET_MAC - { NULL, 0U } }; struct proccall { @@ -107,7 +100,7 @@ /* vcc_obj.c */ extern struct var vcc_be_vars[]; extern struct var vcc_vars[]; -const char *vrt_obj_h; +extern const char *vrt_obj_h; /* vcc_token.c */ void vcc_ErrToken(struct tokenlist *tl, struct token *t); Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2006-07-21 21:42:00 UTC (rev 552) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2006-07-21 21:57:43 UTC (rev 553) @@ -488,8 +488,6 @@ fputs("};\n", f); fputs("\n", f); fputs("void VRT_count(struct sess *, unsigned);\n", f); - fputs("void VRT_no_cache(struct sess *);\n", f); - fputs("void VRT_no_new_cache(struct sess *);\n", f); fputs("#if 0\n", f); fputs("int ip_match(unsigned, struct vcl_acl *);\n", f); fputs("int string_match(const char *, const char *);\n", f); @@ -499,14 +497,9 @@ fputs("int VRT_switch_config(const char *);\n", f); fputs("\n", f); fputs("char *VRT_GetHdr(struct sess *, const char *);\n", f); - fputs("char *VRT_GetReq(struct sess *);\n", f); fputs("void VRT_handling(struct sess *sp, unsigned hand);\n", f); - fputs("int VRT_obj_valid(struct sess *);\n", f); - fputs("int VRT_obj_cacheable(struct sess *);\n", f); fputs("\n", f); fputs("void VRT_set_backend_name(struct backend *, const char *);\n", f); - fputs("void VRT_set_backend_hostname(struct backend *, const char *);\n", f); - fputs("void VRT_set_backend_portname(struct backend *, const char *);\n", f); fputs("\n", f); fputs("void VRT_alloc_backends(struct VCL_conf *cp);\n", f); fputs("\n", f); Modified: trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2006-07-21 21:42:00 UTC (rev 552) +++ trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2006-07-21 21:57:43 UTC (rev 553) @@ -128,10 +128,10 @@ foreach k $returns { if {$k == "error"} { puts $for "#ifdef VCL_RET_MAC_E" - puts $for "VCL_RET_MAC_E($k, [string toupper $k], $i)" + puts $for "VCL_RET_MAC_E($k, [string toupper $k], (1 << $i), $i)" puts $for "#endif" } else { - puts $for "VCL_RET_MAC($k, [string toupper $k], (1 << $i))" + puts $for "VCL_RET_MAC($k, [string toupper $k], (1 << $i), $i)" } incr i } Modified: trunk/varnish-cache/lib/libvcl/vcc_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_token.c 2006-07-21 21:42:00 UTC (rev 552) +++ trunk/varnish-cache/lib/libvcl/vcc_token.c 2006-07-21 21:57:43 UTC (rev 553) @@ -26,10 +26,6 @@ #include "libvcl.h" -#define ERRCHK(tl) do { if ((tl)->err) return; } while (0) - -#define INDENT 2 - /*--------------------------------------------------------------------*/ void @@ -146,21 +142,6 @@ vcc_ErrWhere(tl, tl->t); } -#define Expect(a, b) _Expect(a, b, __LINE__) -#define ExpectErr(a, b) do { _Expect(a, b, __LINE__); ERRCHK(a);} while (0) - -#define L(tl, foo) do { \ - tl->indent += INDENT; \ - foo; \ - tl->indent -= INDENT; \ -} while (0) - -#define C(tl, sep) do { \ - Fc(tl, 1, "VRT_count(sp, %u)%s\n", ++tl->cnt, sep); \ - tl->t->cnt = tl->cnt; \ -} while (0) - - /*-------------------------------------------------------------------- * Compare token to token */ From phk at projects.linpro.no Fri Jul 21 22:12:38 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 22 Jul 2006 00:12:38 +0200 (CEST) Subject: r554 - trunk/varnish-cache/bin/varnishd Message-ID: <20060721221238.2C4521EC38D@projects.linpro.no> Author: phk Date: 2006-07-22 00:12:38 +0200 (Sat, 22 Jul 2006) New Revision: 554 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishd/storage_file.c Log: Unless the user specifies an explicit size, don't use more than 2GB on 32 bit architectures to avoid running out of address room Make FlexeLint happy. Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-21 21:57:43 UTC (rev 553) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-21 22:12:38 UTC (rev 554) @@ -13,7 +13,6 @@ #include "libvarnish.h" #include "shmlog.h" -#include "heritage.h" #include "cache.h" #define HTTPH(a, b, c, d, e, f, g) char b[] = "*" a ":"; Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-21 21:57:43 UTC (rev 553) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-21 22:12:38 UTC (rev 554) @@ -115,7 +115,9 @@ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); /* XXX */ VSL(SLT_TTL, sp->fd, "%u VCL %.0f %u", sp->obj->xid, a, sp->t_req); - sp->obj->ttl = sp->t_req + a; + if (a < 0) + a = 0; + sp->obj->ttl = sp->t_req + (int)a; if (sp->obj->heap_idx != 0) EXP_TTLchange(sp->obj); } Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-21 21:57:43 UTC (rev 553) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-21 22:12:38 UTC (rev 554) @@ -70,7 +70,7 @@ uintmax_t l; unsigned bs; char suff[2]; - int i; + int i, expl; off_t o; struct statfs fsst; struct stat st; @@ -87,6 +87,7 @@ i = sscanf(size, "%ju%1s", &l, suff); /* can return -1, 0, 1 or 2 */ + expl = i; if (i == 0) { fprintf(stderr, "Error: (-sfile) size \"%s\" not understood\n", size); @@ -151,13 +152,22 @@ /* round down to of filesystem blocksize or pagesize */ l -= (l % bs); - if (l < MINPAGES * sc->pagesize) { + if (l < MINPAGES * (uintmax_t)sc->pagesize) { fprintf(stderr, "Error: size too small, at least %ju needed\n", (uintmax_t)MINPAGES * sc->pagesize); exit (2); } + if (expl < 0 && sizeof(void *) == 4 && l > (1ULL << 31)) { + fprintf(stderr, + "NB: Limiting size to 2GB on 32 bit architecture to" + " prevent running out of\naddress space." + " Specifiy explicit size to override.\n" + ); + l = 1ULL << 31; + } + printf("file %s size %ju bytes (%ju fs-blocks, %ju pages)\n", sc->filename, l, l / fsst.f_bsize, l / sc->pagesize); @@ -471,7 +481,7 @@ (uintmax_t)sum, sc->filesize); /* XXX */ - if (sum < MINPAGES * getpagesize()) + if (sum < MINPAGES * (uintmax_t)getpagesize()) exit (2); AZ(pthread_mutex_init(&sc->mtx, NULL)); } From phk at projects.linpro.no Sat Jul 22 08:02:47 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 22 Jul 2006 10:02:47 +0200 (CEST) Subject: r555 - in trunk/varnish-cache: include lib/libvcl Message-ID: <20060722080247.558F11EC3AB@projects.linpro.no> Author: phk Date: 2006-07-22 10:02:47 +0200 (Sat, 22 Jul 2006) New Revision: 555 Added: trunk/varnish-cache/lib/libvcl/vcc_acl.c Modified: trunk/varnish-cache/include/vrt_obj.h trunk/varnish-cache/lib/libvcl/Makefile.am trunk/varnish-cache/lib/libvcl/vcc_compile.c trunk/varnish-cache/lib/libvcl/vcc_compile.h trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl trunk/varnish-cache/lib/libvcl/vcc_obj.c Log: Split IP/ACL compilation into vcc_acl.c Modified: trunk/varnish-cache/include/vrt_obj.h =================================================================== --- trunk/varnish-cache/include/vrt_obj.h 2006-07-21 22:12:38 UTC (rev 554) +++ trunk/varnish-cache/include/vrt_obj.h 2006-07-22 08:02:47 UTC (rev 555) @@ -1,5 +1,5 @@ /* - * $Id: vcc_gen_obj.tcl 548 2006-07-21 20:51:24Z phk $ + * $Id: vcc_gen_obj.tcl 550 2006-07-21 21:13:43Z phk $ * * NB: This file is machine generated, DO NOT EDIT! * @@ -10,6 +10,8 @@ void VRT_l_backend_host(struct backend *, const char *); const char * VRT_r_backend_port(struct backend *); void VRT_l_backend_port(struct backend *, const char *); +const unsigned char * VRT_r_client_ip(struct sess *); +void VRT_l_client_ip(struct sess *, const unsigned char *); const char * VRT_r_req_request(struct sess *); void VRT_l_req_request(struct sess *, const char *); const char * VRT_r_req_url(struct sess *); Modified: trunk/varnish-cache/lib/libvcl/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvcl/Makefile.am 2006-07-21 22:12:38 UTC (rev 554) +++ trunk/varnish-cache/lib/libvcl/Makefile.am 2006-07-22 08:02:47 UTC (rev 555) @@ -9,6 +9,7 @@ vcc_compile.h \ vcc_token_defs.h \ \ + vcc_acl.c \ vcc_compile.c \ vcc_fixed_token.c \ vcc_obj.c \ Added: trunk/varnish-cache/lib/libvcl/vcc_acl.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_acl.c 2006-07-21 22:12:38 UTC (rev 554) +++ trunk/varnish-cache/lib/libvcl/vcc_acl.c 2006-07-22 08:02:47 UTC (rev 555) @@ -0,0 +1,135 @@ +/* + * $Id$ + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vcc_priv.h" +#include "vcc_compile.h" + +#include "libvcl.h" + +unsigned +vcc_IpVal(struct tokenlist *tl) +{ + unsigned u, v; + struct token *t; + + t = tl->t; + u = UintVal(tl); + if (u < 256) { + v = u << 24; + Expect(tl, '.'); + vcc_NextToken(tl); + t = tl->t; + u = UintVal(tl); + if (u < 256) { + v |= u << 16; + Expect(tl, '.'); + vcc_NextToken(tl); + t = tl->t; + u = UintVal(tl); + if (u < 256) { + v |= u << 8; + Expect(tl, '.'); + vcc_NextToken(tl); + t = tl->t; + u = UintVal(tl); + if (u < 256) { + v |= u; + return (v); + } + } + } + } + sbuf_printf(tl->sb, "Illegal octet in IP number\n"); + vcc_ErrWhere(tl, t); + return (0); +} + +void +vcc_Cond_Ip(struct var *vp, struct tokenlist *tl) +{ + unsigned u; + + switch (tl->t->tok) { + case '~': + vcc_NextToken(tl); + ExpectErr(tl, ID); + AddRef(tl, tl->t, R_ACL); + Fc(tl, 1, "ip_match(%s, acl_%T)\n", vp->rname, tl->t); + vcc_NextToken(tl); + break; + case T_EQ: + case T_NEQ: + Fc(tl, 1, "%s %T ", vp->rname, tl->t); + vcc_NextToken(tl); + u = vcc_IpVal(tl); + Fc(tl, 0, "%uU /* %u.%u.%u.%u */\n", u, + (u >> 24) & 0xff, (u >> 16) & 0xff, + (u >> 8) & 0xff, (u) & 0xff); + break; + default: + sbuf_printf(tl->sb, "Illegal condition "); + vcc_ErrToken(tl, tl->t); + sbuf_printf(tl->sb, " on IP number variable\n"); + sbuf_printf(tl->sb, " only '==', '!=' and '~' are legal\n"); + vcc_ErrWhere(tl, tl->t); + break; + } +} + +void +vcc_Acl(struct tokenlist *tl) +{ + unsigned u, m; + + vcc_NextToken(tl); + + ExpectErr(tl, ID); + AddDef(tl, tl->t, R_ACL); + Fh(tl, 0, "static struct vcl_acl acl_%T[];\n", tl->t); + Fc(tl, 1, "static struct vcl_acl acl_%T[] = {\n", tl->t); + vcc_NextToken(tl); + + tl->indent += INDENT; + + ExpectErr(tl, '{'); + vcc_NextToken(tl); + + while (tl->t->tok == CNUM) { + u = vcc_IpVal(tl); + if (tl->t->tok == '/') { + vcc_NextToken(tl); + ExpectErr(tl, CNUM); + m = UintVal(tl); + } else + m = 32; + ExpectErr(tl, ';'); + vcc_NextToken(tl); + Fc(tl, 1, "{ %11uU, %3uU }, /* %u.%u.%u.%u/%u */\n", + u, m, + (u >> 24) & 0xff, (u >> 16) & 0xff, + (u >> 8) & 0xff, (u) & 0xff, m); + } + ExpectErr(tl, '}'); + Fc(tl, 1, "{ %11uU, %3uU }\n", 0, 0); + + tl->indent -= INDENT; + + Fc(tl, 1, "};\n\n"); + vcc_NextToken(tl); +} Property changes on: trunk/varnish-cache/lib/libvcl/vcc_acl.c ___________________________________________________________________ Name: svn:keywords + Id Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.c 2006-07-21 22:12:38 UTC (rev 554) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.c 2006-07-22 08:02:47 UTC (rev 555) @@ -56,8 +56,6 @@ #include "libvcl.h" -#define ERRCHK(tl) do { if ((tl)->err) return; } while (0) - static struct method method_tab[] = { #define VCL_RET_MAC(a,b,c,d) #define VCL_MET_MAC(a,b,c) { "vcl_"#a, "default_vcl_"#a, c }, @@ -77,11 +75,6 @@ /*--------------------------------------------------------------------*/ -#define ErrInternal(tl) vcc__ErrInternal(tl, __func__, __LINE__) - -#define Expect(a, b) vcc__Expect(a, b, __LINE__) -#define ExpectErr(a, b) do { vcc__Expect(a, b, __LINE__); ERRCHK(a);} while (0) - #define L(tl, foo) do { \ tl->indent += INDENT; \ foo; \ @@ -97,7 +90,7 @@ * Printf output to the two sbufs, possibly indented */ -static void +void Fh(struct tokenlist *tl, int indent, const char *fmt, ...) { va_list ap; @@ -109,7 +102,7 @@ va_end(ap); } -static void +void Fc(struct tokenlist *tl, int indent, const char *fmt, ...) { va_list ap; @@ -210,7 +203,7 @@ return (0); } -static void +void AddRef(struct tokenlist *tl, struct token *t, enum ref_type type) { @@ -230,7 +223,7 @@ AddRef(tl, t, type); } -static void +void AddDef(struct tokenlist *tl, struct token *t, enum ref_type type) { struct ref *r; @@ -321,7 +314,7 @@ * Recognize and convert { CNUM } to unsigned value */ -static unsigned +unsigned UintVal(struct tokenlist *tl) { unsigned d = 0; @@ -367,46 +360,6 @@ /*--------------------------------------------------------------------*/ -static unsigned -IpVal(struct tokenlist *tl) -{ - unsigned u, v; - struct token *t; - - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v = u << 24; - Expect(tl, '.'); - vcc_NextToken(tl); - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v |= u << 16; - Expect(tl, '.'); - vcc_NextToken(tl); - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v |= u << 8; - Expect(tl, '.'); - vcc_NextToken(tl); - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v |= u; - return (v); - } - } - } - } - sbuf_printf(tl->sb, "Illegal octet in IP number\n"); - vcc_ErrWhere(tl, t); - return (0); -} - -/*--------------------------------------------------------------------*/ - static struct var * HeaderVar(struct tokenlist *tl __unused, struct token *t, struct var *vh) { @@ -494,38 +447,6 @@ /*--------------------------------------------------------------------*/ static void -Cond_Ip(struct var *vp, struct tokenlist *tl) -{ - unsigned u; - - switch (tl->t->tok) { - case '~': - vcc_NextToken(tl); - ExpectErr(tl, ID); - AddRef(tl, tl->t, R_ACL); - Fc(tl, 1, "ip_match(%s, acl_%T)\n", vp->rname, tl->t); - vcc_NextToken(tl); - break; - case T_EQ: - case T_NEQ: - Fc(tl, 1, "%s %T ", vp->rname, tl->t); - vcc_NextToken(tl); - u = IpVal(tl); - Fc(tl, 0, "%uU /* %u.%u.%u.%u */\n", u, - (u >> 24) & 0xff, (u >> 16) & 0xff, - (u >> 8) & 0xff, (u) & 0xff); - break; - default: - sbuf_printf(tl->sb, "Illegal condition "); - vcc_ErrToken(tl, tl->t); - sbuf_printf(tl->sb, " on IP number variable\n"); - sbuf_printf(tl->sb, " only '==', '!=' and '~' are legal\n"); - vcc_ErrWhere(tl, tl->t); - break; - } -} - -static void Cond_String(struct var *vp, struct tokenlist *tl) { @@ -631,7 +552,7 @@ case INT: L(tl, Cond_Int(vp, tl)); break; case SIZE: L(tl, Cond_Int(vp, tl)); break; case BOOL: L(tl, Cond_Bool(vp, tl)); break; - case IP: L(tl, Cond_Ip(vp, tl)); break; + case IP: L(tl, vcc_Cond_Ip(vp, tl)); break; case STRING: L(tl, Cond_String(vp, tl)); break; case TIME: L(tl, Cond_Int(vp, tl)); break; /* XXX backend == */ @@ -742,7 +663,7 @@ static void Action(struct tokenlist *tl) { - unsigned a, u; + unsigned a; struct var *vp; struct token *at; @@ -825,10 +746,11 @@ Fc(tl, 0, "%g", DoubleVal(tl)); Fc(tl, 0, ");\n"); break; +#if 0 /* XXX: enable if we find a legit use */ case IP: if (tl->t->tok == '=') { vcc_NextToken(tl); - u = IpVal(tl); + u = vcc_IpVal(tl); Fc(tl, 0, "= %uU; /* %u.%u.%u.%u */\n", u, (u >> 24) & 0xff, @@ -843,6 +765,7 @@ " only '=' is legal for IP numbers\n"); vcc_ErrWhere(tl, tl->t); return; +#endif case BACKEND: if (tl->t->tok == '=') { vcc_NextToken(tl); @@ -874,50 +797,6 @@ /*--------------------------------------------------------------------*/ static void -Acl(struct tokenlist *tl) -{ - unsigned u, m; - - vcc_NextToken(tl); - - ExpectErr(tl, ID); - AddDef(tl, tl->t, R_ACL); - Fh(tl, 0, "static struct vcl_acl acl_%T[];\n", tl->t); - Fc(tl, 1, "static struct vcl_acl acl_%T[] = {\n", tl->t); - vcc_NextToken(tl); - - tl->indent += INDENT; - - ExpectErr(tl, '{'); - vcc_NextToken(tl); - - while (tl->t->tok == CNUM) { - u = IpVal(tl); - if (tl->t->tok == '/') { - vcc_NextToken(tl); - ExpectErr(tl, CNUM); - m = UintVal(tl); - } else - m = 32; - ExpectErr(tl, ';'); - vcc_NextToken(tl); - Fc(tl, 1, "{ %11uU, %3uU }, /* %u.%u.%u.%u/%u */\n", - u, m, - (u >> 24) & 0xff, (u >> 16) & 0xff, - (u >> 8) & 0xff, (u) & 0xff, m); - } - ExpectErr(tl, '}'); - Fc(tl, 1, "{ %11uU, %3uU }\n", 0, 0); - - tl->indent -= INDENT; - - Fc(tl, 1, "};\n\n"); - vcc_NextToken(tl); -} - -/*--------------------------------------------------------------------*/ - -static void Compound(struct tokenlist *tl) { @@ -1099,7 +978,7 @@ ERRCHK(tl); switch (tl->t->tok) { case T_ACL: - Acl(tl); + vcc_Acl(tl); break; case T_SUB: Function(tl); Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.h 2006-07-21 22:12:38 UTC (rev 554) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.h 2006-07-22 08:02:47 UTC (rev 555) @@ -94,14 +94,26 @@ /*--------------------------------------------------------------------*/ +/* vcc_acl.c */ + +void vcc_Acl(struct tokenlist *tl); +unsigned vcc_IpVal(struct tokenlist *tl); +void vcc_Cond_Ip(struct var *vp, struct tokenlist *tl); + /* vcc_compile.c */ extern const char *vcc_default_vcl_b, *vcc_default_vcl_e; +void Fh(struct tokenlist *tl, int indent, const char *fmt, ...); +void Fc(struct tokenlist *tl, int indent, const char *fmt, ...); +unsigned UintVal(struct tokenlist *tl); +void AddDef(struct tokenlist *tl, struct token *t, enum ref_type type); +void AddRef(struct tokenlist *tl, struct token *t, enum ref_type type); /* vcc_obj.c */ extern struct var vcc_be_vars[]; extern struct var vcc_vars[]; extern const char *vrt_obj_h; + /* vcc_token.c */ void vcc_ErrToken(struct tokenlist *tl, struct token *t); void vcc_ErrWhere(struct tokenlist *tl, struct token *t); @@ -112,3 +124,9 @@ void vcc_NextToken(struct tokenlist *tl); void vcc__ErrInternal(struct tokenlist *tl, const char *func, unsigned line); void vcc_AddToken(struct tokenlist *tl, unsigned tok, const char *b, const char *e); + +#define ERRCHK(tl) do { if ((tl)->err) return; } while (0) +#define ErrInternal(tl) vcc__ErrInternal(tl, __func__, __LINE__) +#define Expect(a, b) vcc__Expect(a, b, __LINE__) +#define ExpectErr(a, b) do { vcc__Expect(a, b, __LINE__); ERRCHK(a);} while (0) + Modified: trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl 2006-07-21 22:12:38 UTC (rev 554) +++ trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl 2006-07-22 08:02:47 UTC (rev 555) @@ -12,6 +12,7 @@ # Objects which operate on sessions set spobj { + { client.ip IP } { req.request STRING } { req.url STRING } { req.proto STRING } @@ -22,6 +23,7 @@ { req.http. HEADER } } +set tt(IP) "const unsigned char *" set tt(STRING) "const char *" set tt(BOOL) "double" set tt(BACKEND) "struct backend *" Modified: trunk/varnish-cache/lib/libvcl/vcc_obj.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_obj.c 2006-07-21 22:12:38 UTC (rev 554) +++ trunk/varnish-cache/lib/libvcl/vcc_obj.c 2006-07-22 08:02:47 UTC (rev 555) @@ -22,6 +22,10 @@ }; struct var vcc_vars[] = { + { "client.ip", IP, 9, + "VRT_r_client_ip(sp)", + "VRT_l_client_ip(sp, ", + }, { "req.request", STRING, 11, "VRT_r_req_request(sp)", "VRT_l_req_request(sp, ", @@ -70,6 +74,8 @@ "void VRT_l_backend_host(struct backend *, const char *);\n" "const char * VRT_r_backend_port(struct backend *);\n" "void VRT_l_backend_port(struct backend *, const char *);\n" + "const unsigned char * VRT_r_client_ip(struct sess *);\n" + "void VRT_l_client_ip(struct sess *, const unsigned char *);\n" "const char * VRT_r_req_request(struct sess *);\n" "void VRT_l_req_request(struct sess *, const char *);\n" "const char * VRT_r_req_url(struct sess *);\n" From phk at projects.linpro.no Sat Jul 22 09:38:10 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 22 Jul 2006 11:38:10 +0200 (CEST) Subject: r556 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060722093810.475751EC3AB@projects.linpro.no> Author: phk Date: 2006-07-22 11:38:09 +0200 (Sat, 22 Jul 2006) New Revision: 556 Added: trunk/varnish-cache/bin/varnishd/cache_vrt_acl.c Modified: trunk/varnish-cache/bin/varnishd/Makefile.am trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/include/vcl.h trunk/varnish-cache/include/vcl_returns.h trunk/varnish-cache/include/vrt.h trunk/varnish-cache/include/vrt_obj.h trunk/varnish-cache/lib/libvcl/vcc_acl.c trunk/varnish-cache/lib/libvcl/vcc_compile.c trunk/varnish-cache/lib/libvcl/vcc_compile.h trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl Log: VCL compiler: add two sbufs for "init" and "fini" actions. VCL ACLs: Change syntax and implementation as follows. ACL Syntax now works the following way: acl $name { ! ( "myhost.com" ) ; "10.0.0.1" /8 ; } The '!' means not. If the address matches the rest of the rule the address does NOT match the acl and the search terminates here. Enclosing the string in paranthesis means that the rule will be ignored if the string cannot be converted to an address (with getaddrinfo). When a string can not be looked up, and is not enclosed in a paranthesis, a positive rule (ie: without !) will not match and a negative rule (with !) will match. A mask can always be supplied, no matter the style of the string given, so it is possible to do things like: { "fw.ourcompany.dom" / 24 } Which means "any host on the same /24 subnet as fw.ourcompany.dom". Unfortunately getaddrinfo() does not return a TTL for the results, in the future we may want to use some kind of timeout to refresh the lookups. Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-22 08:02:47 UTC (rev 555) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-22 09:38:09 UTC (rev 556) @@ -30,6 +30,7 @@ cache_session.c \ cache_vcl.c \ cache_vrt.c \ + cache_vrt_acl.c \ cli_event.c \ hash_simple_list.c \ hash_classic.c \ Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-22 08:02:47 UTC (rev 555) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-07-22 09:38:09 UTC (rev 556) @@ -86,6 +86,19 @@ } } +void +VRT_free_backends(struct VCL_conf *cp) +{ + + (void)cp; +} + +void +VRT_fini_backend(struct backend *be) +{ + (void)be; +} + /*--------------------------------------------------------------------*/ #define VBACKEND(type,onm,field) \ Added: trunk/varnish-cache/bin/varnishd/cache_vrt_acl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt_acl.c 2006-07-22 08:02:47 UTC (rev 555) +++ trunk/varnish-cache/bin/varnishd/cache_vrt_acl.c 2006-07-22 09:38:09 UTC (rev 556) @@ -0,0 +1,69 @@ +/* + * $Id$ + * + * Runtime support for compiled VCL programs, ACLs + * + * XXX: getaddrinfo() does not return a TTL. We might want to add + * XXX: a refresh facility. + */ + +#include +#include +#include + +#include "shmlog.h" +#include "vrt.h" +#include "vrt_obj.h" +#include "vcl.h" +#include "cache.h" +#include +#include +#include + + +int +VRT_acl_match(struct sess *sp, struct vrt_acl *ap) +{ + (void)sp; + (void)ap; + return (0); +} + +void +VRT_acl_init(struct vrt_acl *ap) +{ + struct addrinfo a0, *a1; + int i; + + memset(&a0, 0, sizeof a0); + a0.ai_socktype = SOCK_STREAM; + + for ( ; ap->name != NULL; ap++) { + a1 = NULL; + i = getaddrinfo(ap->name, NULL, &a0, &a1); + if (i != 0) { + fprintf(stderr, "getaddrinfo(%s) = %s\n", + ap->name, gai_strerror(i)); + if (a1 != NULL) + freeaddrinfo(a1); + a1 = NULL; + } + ap->priv = a1; + } +} + +void +VRT_acl_fini(struct vrt_acl *ap) +{ + struct addrinfo *a1; + + for ( ; ap->name != NULL; ap++) { + if (ap->priv == NULL) + continue; + a1 = ap->priv; + ap->priv = NULL; + freeaddrinfo(a1); + } +} + + Property changes on: trunk/varnish-cache/bin/varnishd/cache_vrt_acl.c ___________________________________________________________________ Name: svn:keywords + Id Modified: trunk/varnish-cache/include/vcl.h =================================================================== --- trunk/varnish-cache/include/vcl.h 2006-07-22 08:02:47 UTC (rev 555) +++ trunk/varnish-cache/include/vcl.h 2006-07-22 09:38:09 UTC (rev 556) @@ -9,6 +9,7 @@ struct sess; typedef void vcl_init_f(void); +typedef void vcl_fini_f(void); typedef int vcl_func_f(struct sess *sp); struct VCL_conf { @@ -22,6 +23,7 @@ unsigned busy; vcl_init_f *init_func; + vcl_fini_f *fini_func; vcl_func_f *recv_func; vcl_func_f *miss_func; Modified: trunk/varnish-cache/include/vcl_returns.h =================================================================== --- trunk/varnish-cache/include/vcl_returns.h 2006-07-22 08:02:47 UTC (rev 555) +++ trunk/varnish-cache/include/vcl_returns.h 2006-07-22 09:38:09 UTC (rev 556) @@ -1,5 +1,5 @@ /* - * $Id: vcc_gen_fixed_token.tcl 545 2006-07-21 20:43:56Z phk $ + * $Id: vcc_gen_fixed_token.tcl 553 2006-07-21 21:57:43Z phk $ * * NB: This file is machine generated, DO NOT EDIT! * Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2006-07-22 08:02:47 UTC (rev 555) +++ trunk/varnish-cache/include/vrt.h 2006-07-22 09:38:09 UTC (rev 556) @@ -19,15 +19,19 @@ }; struct vrt_acl { - unsigned ip; - unsigned mask; + unsigned char not; + unsigned char paren; + unsigned char mask; + const char *name; + void *priv; }; +/* ACL related */ +int VRT_acl_match(struct sess *, struct vrt_acl *); +void VRT_acl_init(struct vrt_acl *); +void VRT_acl_fini(struct vrt_acl *); + void VRT_count(struct sess *, unsigned); -#if 0 -int ip_match(unsigned, struct vcl_acl *); -int string_match(const char *, const char *); -#endif int VRT_rewrite(const char *, const char *); void VRT_error(struct sess *, unsigned, const char *); int VRT_switch_config(const char *); @@ -35,10 +39,13 @@ char *VRT_GetHdr(struct sess *, const char *); void VRT_handling(struct sess *sp, unsigned hand); +/* Backend related */ void VRT_set_backend_name(struct backend *, const char *); - void VRT_alloc_backends(struct VCL_conf *cp); +void VRT_free_backends(struct VCL_conf *cp); +void VRT_fini_backend(struct backend *be); + #define VRT_done(sp, hand) \ do { \ VRT_handling(sp, hand); \ Modified: trunk/varnish-cache/include/vrt_obj.h =================================================================== --- trunk/varnish-cache/include/vrt_obj.h 2006-07-22 08:02:47 UTC (rev 555) +++ trunk/varnish-cache/include/vrt_obj.h 2006-07-22 09:38:09 UTC (rev 556) @@ -1,5 +1,5 @@ /* - * $Id: vcc_gen_obj.tcl 550 2006-07-21 21:13:43Z phk $ + * $Id: vcc_gen_obj.tcl 555 2006-07-22 08:02:47Z phk $ * * NB: This file is machine generated, DO NOT EDIT! * Modified: trunk/varnish-cache/lib/libvcl/vcc_acl.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_acl.c 2006-07-22 08:02:47 UTC (rev 555) +++ trunk/varnish-cache/lib/libvcl/vcc_acl.c 2006-07-22 09:38:09 UTC (rev 556) @@ -22,66 +22,20 @@ #include "libvcl.h" -unsigned -vcc_IpVal(struct tokenlist *tl) -{ - unsigned u, v; - struct token *t; - - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v = u << 24; - Expect(tl, '.'); - vcc_NextToken(tl); - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v |= u << 16; - Expect(tl, '.'); - vcc_NextToken(tl); - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v |= u << 8; - Expect(tl, '.'); - vcc_NextToken(tl); - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v |= u; - return (v); - } - } - } - } - sbuf_printf(tl->sb, "Illegal octet in IP number\n"); - vcc_ErrWhere(tl, t); - return (0); -} - void vcc_Cond_Ip(struct var *vp, struct tokenlist *tl) { - unsigned u; + (void)vp; /* only client.ip at this time */ + switch (tl->t->tok) { case '~': vcc_NextToken(tl); ExpectErr(tl, ID); AddRef(tl, tl->t, R_ACL); - Fc(tl, 1, "ip_match(%s, acl_%T)\n", vp->rname, tl->t); + Fc(tl, 1, "VRT_acl_match(sp, acl_%T)\n", tl->t); vcc_NextToken(tl); break; - case T_EQ: - case T_NEQ: - Fc(tl, 1, "%s %T ", vp->rname, tl->t); - vcc_NextToken(tl); - u = vcc_IpVal(tl); - Fc(tl, 0, "%uU /* %u.%u.%u.%u */\n", u, - (u >> 24) & 0xff, (u >> 16) & 0xff, - (u >> 8) & 0xff, (u) & 0xff); - break; default: sbuf_printf(tl->sb, "Illegal condition "); vcc_ErrToken(tl, tl->t); @@ -95,41 +49,63 @@ void vcc_Acl(struct tokenlist *tl) { - unsigned u, m; + unsigned mask, para, not; + struct token *t, *an; vcc_NextToken(tl); ExpectErr(tl, ID); - AddDef(tl, tl->t, R_ACL); - Fh(tl, 0, "static struct vcl_acl acl_%T[];\n", tl->t); - Fc(tl, 1, "static struct vcl_acl acl_%T[] = {\n", tl->t); + an = tl->t; vcc_NextToken(tl); + AddDef(tl, an, R_ACL); + Fh(tl, 0, "static struct vrt_acl acl_%T[];\n", an); + Fc(tl, 1, "static struct vrt_acl acl_%T[] = {\n", an); + tl->indent += INDENT; ExpectErr(tl, '{'); vcc_NextToken(tl); - while (tl->t->tok == CNUM) { - u = vcc_IpVal(tl); + while (tl->t->tok != '}') { + + not = para = mask = 0; + + if (tl->t->tok == '!') { + not = 1; + vcc_NextToken(tl); + } + + if (tl->t->tok == '(') { + para = 1; + vcc_NextToken(tl); + } + + ExpectErr(tl, CSTR); + /* XXX: try to look it up, warn if failure */ + t = tl->t; + vcc_NextToken(tl); if (tl->t->tok == '/') { vcc_NextToken(tl); ExpectErr(tl, CNUM); - m = UintVal(tl); - } else - m = 32; + mask = UintVal(tl); + } + Fc(tl, 1, "{ %u, %u, %u, %T },\n", not, mask, para, t); + + if (para) { + ExpectErr(tl, ')'); + vcc_NextToken(tl); + } ExpectErr(tl, ';'); vcc_NextToken(tl); - Fc(tl, 1, "{ %11uU, %3uU }, /* %u.%u.%u.%u/%u */\n", - u, m, - (u >> 24) & 0xff, (u >> 16) & 0xff, - (u >> 8) & 0xff, (u) & 0xff, m); } - ExpectErr(tl, '}'); - Fc(tl, 1, "{ %11uU, %3uU }\n", 0, 0); - + Fc(tl, 1, "{ 0, 0, 0, (void*)0}\n", 0, 0); tl->indent -= INDENT; - Fc(tl, 1, "};\n\n"); + + ExpectErr(tl, '}'); vcc_NextToken(tl); + + Fi(tl, 1, "\tVRT_acl_init(acl_%T);\n", an); + Ff(tl, 1, "\tVRT_acl_fini(acl_%T);\n", an); } Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.c 2006-07-22 08:02:47 UTC (rev 555) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.c 2006-07-22 09:38:09 UTC (rev 556) @@ -114,9 +114,33 @@ va_end(ap); } +void +Fi(struct tokenlist *tl, int indent, const char *fmt, ...) +{ + va_list ap; + + if (indent) + sbuf_printf(tl->fi, "%*.*s", tl->indent, tl->indent, ""); + va_start(ap, fmt); + sbuf_vprintf(tl->fi, fmt, ap); + va_end(ap); +} + +void +Ff(struct tokenlist *tl, int indent, const char *fmt, ...) +{ + va_list ap; + + if (indent) + sbuf_printf(tl->ff, "%*.*s", tl->indent, tl->indent, ""); + va_start(ap, fmt); + sbuf_vprintf(tl->ff, fmt, ap); + va_end(ap); +} + /*--------------------------------------------------------------------*/ -static char * +char * EncString(struct token *t) { char *p, *q; @@ -871,11 +895,11 @@ AddRef(tl, tl->t, R_BACKEND); Fh(tl, 1, "#define VGC_backend_%T (VCL_conf.backend[%d])\n", tl->t, tl->nbackend); + Fc(tl, 0, "\n"); Fc(tl, 0, "static void\n"); Fc(tl, 1, "VGC_init_backend_%T (void)\n", tl->t); Fc(tl, 1, "{\n"); Fc(tl, 1, "\tstruct backend *backend = VGC_backend_%T;\n", tl->t); - Fc(tl, 1, "\tconst char *p;\n"); Fc(tl, 1, "\n"); Fc(tl, 1, "\tVRT_set_backend_name(backend, \"%T\");\n", tl->t); vcc_NextToken(tl); @@ -941,6 +965,8 @@ vcc_NextToken(tl); Fc(tl, 1, "}\n"); Fc(tl, 0, "\n"); + Fi(tl, 0, "\tVGC_init_backend_%T();\n", t_be); + Ff(tl, 0, "\tVRT_fini_backend(VGC_backend_%T);\n", t_be); tl->nbackend++; } @@ -1208,25 +1234,23 @@ static void EmitInitFunc(struct tokenlist *tl) { - struct ref *r; Fc(tl, 0, "\nstatic void\nVGC_Init(void)\n{\n\n"); - Fc(tl, 0, "\tVRT_alloc_backends(&VCL_conf);\n"); - - TAILQ_FOREACH(r, &tl->refs, list) { - switch(r->type) { - case R_FUNC: - break; - case R_ACL: - break; - case R_BACKEND: - Fc(tl, 0, "\tVGC_init_backend_%T();\n", r->name); - break; - } - } + sbuf_finish(tl->fi); + sbuf_cat(tl->fc, sbuf_data(tl->fi)); Fc(tl, 0, "}\n"); } +static void +EmitFiniFunc(struct tokenlist *tl) +{ + + Fc(tl, 0, "\nstatic void\nVGC_Fini(void)\n{\n\n"); + sbuf_finish(tl->ff); + sbuf_cat(tl->fc, sbuf_data(tl->ff)); + Fc(tl, 0, "}\n"); +} + /*--------------------------------------------------------------------*/ static void @@ -1236,6 +1260,7 @@ Fc(tl, 0, "\nstruct VCL_conf VCL_conf = {\n"); Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n"); Fc(tl, 0, "\t.init_func = VGC_Init,\n"); + Fc(tl, 0, "\t.fini_func = VGC_Fini,\n"); Fc(tl, 0, "\t.nbackend = %d,\n", tl->nbackend); Fc(tl, 0, "\t.ref = VGC_ref,\n"); Fc(tl, 0, "\t.nref = VGC_NREFS,\n"); @@ -1279,8 +1304,16 @@ tokens.fh = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(tokens.fh != NULL); + tokens.fi = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); + assert(tokens.fi != NULL); + + tokens.ff = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); + assert(tokens.ff != NULL); + Fh(&tokens, 0, "extern struct VCL_conf VCL_conf;\n"); + Fi(&tokens, 0, "\tVRT_alloc_backends(&VCL_conf);\n"); + tokens.b = b; if (e == NULL) e = strchr(b, '\0'); @@ -1300,8 +1333,12 @@ goto done; LocTable(&tokens); + Ff(&tokens, 0, "\tVRT_free_backends(&VCL_conf);\n"); + EmitInitFunc(&tokens); + EmitFiniFunc(&tokens); + EmitStruct(&tokens); if (CheckRefs(&tokens)) Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.h 2006-07-22 08:02:47 UTC (rev 555) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.h 2006-07-22 09:38:09 UTC (rev 556) @@ -22,7 +22,7 @@ struct token *t; int indent; unsigned cnt; - struct sbuf *fc, *fh; + struct sbuf *fc, *fh, *fi, *ff; TAILQ_HEAD(, ref) refs; struct sbuf *sb; int err; @@ -97,17 +97,20 @@ /* vcc_acl.c */ void vcc_Acl(struct tokenlist *tl); -unsigned vcc_IpVal(struct tokenlist *tl); void vcc_Cond_Ip(struct var *vp, struct tokenlist *tl); /* vcc_compile.c */ extern const char *vcc_default_vcl_b, *vcc_default_vcl_e; void Fh(struct tokenlist *tl, int indent, const char *fmt, ...); void Fc(struct tokenlist *tl, int indent, const char *fmt, ...); +void Fi(struct tokenlist *tl, int indent, const char *fmt, ...); +void Ff(struct tokenlist *tl, int indent, const char *fmt, ...); unsigned UintVal(struct tokenlist *tl); void AddDef(struct tokenlist *tl, struct token *t, enum ref_type type); void AddRef(struct tokenlist *tl, struct token *t, enum ref_type type); +char *EncString(struct token *t); + /* vcc_obj.c */ extern struct var vcc_be_vars[]; extern struct var vcc_vars[]; Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2006-07-22 08:02:47 UTC (rev 555) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2006-07-22 09:38:09 UTC (rev 556) @@ -442,6 +442,7 @@ fputs("struct sess;\n", f); fputs("\n", f); fputs("typedef void vcl_init_f(void);\n", f); + fputs("typedef void vcl_fini_f(void);\n", f); fputs("typedef int vcl_func_f(struct sess *sp);\n", f); fputs("\n", f); fputs("struct VCL_conf {\n", f); @@ -455,6 +456,7 @@ fputs(" unsigned busy;\n", f); fputs("\n", f); fputs(" vcl_init_f *init_func;\n", f); + fputs(" vcl_fini_f *fini_func;\n", f); fputs("\n", f); fputs(" vcl_func_f *recv_func;\n", f); fputs(" vcl_func_f *miss_func;\n", f); @@ -483,15 +485,19 @@ fputs("};\n", f); fputs("\n", f); fputs("struct vrt_acl {\n", f); - fputs(" unsigned ip;\n", f); - fputs(" unsigned mask;\n", f); + fputs(" unsigned char not;\n", f); + fputs(" unsigned char paren;\n", f); + fputs(" unsigned char mask;\n", f); + fputs(" const char *name;\n", f); + fputs(" void *priv;\n", f); fputs("};\n", f); fputs("\n", f); + fputs("/* ACL related */\n", f); + fputs("int VRT_acl_match(struct sess *, struct vrt_acl *);\n", f); + fputs("void VRT_acl_init(struct vrt_acl *);\n", f); + fputs("void VRT_acl_fini(struct vrt_acl *);\n", f); + fputs("\n", f); fputs("void VRT_count(struct sess *, unsigned);\n", f); - fputs("#if 0\n", f); - fputs("int ip_match(unsigned, struct vcl_acl *);\n", f); - fputs("int string_match(const char *, const char *);\n", f); - fputs("#endif\n", f); fputs("int VRT_rewrite(const char *, const char *);\n", f); fputs("void VRT_error(struct sess *, unsigned, const char *);\n", f); fputs("int VRT_switch_config(const char *);\n", f); @@ -499,10 +505,13 @@ fputs("char *VRT_GetHdr(struct sess *, const char *);\n", f); fputs("void VRT_handling(struct sess *sp, unsigned hand);\n", f); fputs("\n", f); + fputs("/* Backend related */\n", f); fputs("void VRT_set_backend_name(struct backend *, const char *);\n", f); - fputs("\n", f); fputs("void VRT_alloc_backends(struct VCL_conf *cp);\n", f); + fputs("void VRT_free_backends(struct VCL_conf *cp);\n", f); + fputs("void VRT_fini_backend(struct backend *be);\n", f); fputs("\n", f); + fputs("\n", f); fputs("#define VRT_done(sp, hand) \\\n", f); fputs(" do { \\\n", f); fputs(" VRT_handling(sp, hand); \\\n", f); Modified: trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2006-07-22 08:02:47 UTC (rev 555) +++ trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2006-07-22 09:38:09 UTC (rev 556) @@ -97,6 +97,7 @@ puts $fo {struct sess; typedef void vcl_init_f(void); +typedef void vcl_fini_f(void); typedef int vcl_func_f(struct sess *sp); } puts $fo "struct VCL_conf {" @@ -110,6 +111,7 @@ unsigned busy; vcl_init_f *init_func; + vcl_fini_f *fini_func; } foreach m $methods { puts $fo "\tvcl_func_f\t*[lindex $m 0]_func;" From phk at projects.linpro.no Sat Jul 22 10:35:39 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 22 Jul 2006 12:35:39 +0200 (CEST) Subject: r557 - trunk/varnish-cache/bin/varnishd Message-ID: <20060722103539.2552A1EC38D@projects.linpro.no> Author: phk Date: 2006-07-22 12:35:39 +0200 (Sat, 22 Jul 2006) New Revision: 557 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_session.c Log: Store the socket address in the session Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-22 09:38:09 UTC (rev 556) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-22 10:35:39 UTC (rev 557) @@ -207,6 +207,9 @@ struct worker *wrk; + unsigned sockaddrlen; + struct sockaddr *sockaddr; + /* formatted ascii client address */ char addr[TCP_ADDRBUFSIZE]; char port[TCP_PORTBUFSIZE]; Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-22 09:38:09 UTC (rev 556) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-22 10:35:39 UTC (rev 557) @@ -15,6 +15,7 @@ #include #include #include +#include #include "libvarnish.h" #include "heritage.h" @@ -27,11 +28,12 @@ /*--------------------------------------------------------------------*/ struct sessmem { - unsigned magic; -#define SESSMEM_MAGIC 0x555859c5 + unsigned magic; +#define SESSMEM_MAGIC 0x555859c5 - struct sess sess; - struct http http; + struct sess sess; + struct http http; + struct sockaddr sockaddr[2]; /* INET6 hack */ }; /*--------------------------------------------------------------------*/ @@ -166,8 +168,6 @@ { struct sessmem *sm; - (void)addr; /* XXX */ - (void)len; /* XXX */ sm = calloc( sizeof *sm + heritage.mem_workspace, 1); @@ -178,6 +178,11 @@ sm->sess.magic = SESS_MAGIC; sm->sess.mem = sm; sm->sess.http = &sm->http; + + sm->sess.sockaddr = sm->sockaddr; + assert(len < sizeof(sm->sockaddr)); + memcpy(sm->sess.sockaddr, addr, len); + sm->sess.sockaddrlen = len; http_Setup(&sm->http, (void *)(sm + 1), heritage.mem_workspace); return (&sm->sess); } From phk at projects.linpro.no Sat Jul 22 10:41:00 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 22 Jul 2006 12:41:00 +0200 (CEST) Subject: r558 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060722104100.711B71EC39A@projects.linpro.no> Author: phk Date: 2006-07-22 12:41:00 +0200 (Sat, 22 Jul 2006) New Revision: 558 Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_acl.c trunk/varnish-cache/include/shmlog_tags.h trunk/varnish-cache/include/vcl_returns.h trunk/varnish-cache/include/vrt.h trunk/varnish-cache/lib/libvcl/vcc_acl.c trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c Log: Change the acl syntax slightly: the ( ... ) should enclose all of the rule (ie: also ! and /mask if present). Implement matching for IPv4. Acl tests are shmlogged as follows (doc candidate): shmlog tag: VCL_actl "NO_MATCH $acl" client did not match access list $acl "FAIL $acl $rule" getaddrinfo(3) failed on $rule which had a '!' "MATCH $acl $rule" client matched $rule "NEG_MATCH $acl $rule" client matched negated (!) $rule Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_acl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt_acl.c 2006-07-22 10:35:39 UTC (rev 557) +++ trunk/varnish-cache/bin/varnishd/cache_vrt_acl.c 2006-07-22 10:41:00 UTC (rev 558) @@ -19,14 +19,85 @@ #include #include #include +#include +static unsigned ipv4mask[] = { + [0] = 0xffffffff, +#define M(n) [n] = (0xffffffff << (32 - n)) + M( 1), M( 2), M( 3), M( 4), M( 5), M( 6), M( 7), M( 8), M( 9), M(10), + M(11), M(12), M(13), M(14), M(15), M(16), M(17), M(18), M(19), M(20), + M(21), M(22), M(23), M(24), M(25), M(26), M(27), M(28), M(29), M(30), + M(31), M(32) +}; + +static int +vrt_acl_vsl(struct sess *sp, const char *acl, struct vrt_acl *ap, int r) +{ + + assert(ap != NULL); + if (ap->name == NULL) { + assert(r == 0); + VSL(SLT_VCL_acl, sp->fd, "NO_MATCH %s", acl); + return (r); + } + if (ap->priv == NULL) { + assert(r == 0); + VSL(SLT_VCL_acl, sp->fd, "FAIL %s %s", acl, ap->desc); + return (r); + } + + VSL(SLT_VCL_acl, sp->fd, "%s %s %s", + r ? "MATCH" : "NEG_MATCH", acl, ap->desc); + return (r); +} + int -VRT_acl_match(struct sess *sp, struct vrt_acl *ap) +VRT_acl_match(struct sess *sp, const char *acl, struct vrt_acl *ap) { - (void)sp; - (void)ap; - return (0); + struct addrinfo *a1; + struct sockaddr_in *sin1, *sin2; + + if (sp->sockaddr->sa_family == AF_INET) { + assert(sp->sockaddrlen >= sizeof *sin1); + sin1 = (void*)sp->sockaddr; + } else { + sin1 = NULL; + } + + for ( ; ap->name != NULL; ap++) { + if (ap->priv == NULL && ap->paren) + continue; + if (ap->priv == NULL && ap->not) { + return (vrt_acl_vsl(sp, acl, ap, 0)); + } + if (ap->priv == NULL) + continue; + for (a1 = ap->priv; a1 != NULL; a1 = a1->ai_next) { + + /* only match the right family */ + if (a1->ai_family != sp->sockaddr->sa_family) + continue; + + if (a1->ai_family == AF_INET) { + assert(sin1 != NULL); + assert(a1->ai_addrlen >= sizeof (*sin2)); + sin2 = (void*)a1->ai_addr; + if (0 == (( + htonl(sin1->sin_addr.s_addr) ^ + htonl(sin2->sin_addr.s_addr)) & + ipv4mask[ap->mask > 32 ? 32 : ap->mask])) + return ( + vrt_acl_vsl(sp, acl, ap, !ap->not)); + continue; + } + + /* Not rules for unknown protos match */ + if (ap->not) + return (vrt_acl_vsl(sp, acl, ap, 0)); + } + } + return (vrt_acl_vsl(sp, acl, ap, 0)); } void Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-07-22 10:35:39 UTC (rev 557) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-07-22 10:41:00 UTC (rev 558) @@ -33,6 +33,7 @@ SLTM(TxHeader) SLTM(LostHeader) SLTM(TTL) +SLTM(VCL_acl) SLTM(VCL_call) SLTM(VCL_trace) SLTM(VCL_return) Modified: trunk/varnish-cache/include/vcl_returns.h =================================================================== --- trunk/varnish-cache/include/vcl_returns.h 2006-07-22 10:35:39 UTC (rev 557) +++ trunk/varnish-cache/include/vcl_returns.h 2006-07-22 10:41:00 UTC (rev 558) @@ -1,5 +1,5 @@ /* - * $Id: vcc_gen_fixed_token.tcl 553 2006-07-21 21:57:43Z phk $ + * $Id: vcc_gen_fixed_token.tcl 556 2006-07-22 09:38:09Z phk $ * * NB: This file is machine generated, DO NOT EDIT! * Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2006-07-22 10:35:39 UTC (rev 557) +++ trunk/varnish-cache/include/vrt.h 2006-07-22 10:41:00 UTC (rev 558) @@ -20,14 +20,15 @@ struct vrt_acl { unsigned char not; + unsigned char mask; unsigned char paren; - unsigned char mask; const char *name; + const char *desc; void *priv; }; /* ACL related */ -int VRT_acl_match(struct sess *, struct vrt_acl *); +int VRT_acl_match(struct sess *, const char *, struct vrt_acl *); void VRT_acl_init(struct vrt_acl *); void VRT_acl_fini(struct vrt_acl *); Modified: trunk/varnish-cache/lib/libvcl/vcc_acl.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_acl.c 2006-07-22 10:35:39 UTC (rev 557) +++ trunk/varnish-cache/lib/libvcl/vcc_acl.c 2006-07-22 10:41:00 UTC (rev 558) @@ -33,7 +33,7 @@ vcc_NextToken(tl); ExpectErr(tl, ID); AddRef(tl, tl->t, R_ACL); - Fc(tl, 1, "VRT_acl_match(sp, acl_%T)\n", tl->t); + Fc(tl, 1, "VRT_acl_match(sp, \"%T\", acl_%T)\n", tl->t, tl->t); vcc_NextToken(tl); break; default: @@ -51,6 +51,7 @@ { unsigned mask, para, not; struct token *t, *an; + char *p; vcc_NextToken(tl); @@ -71,13 +72,13 @@ not = para = mask = 0; - if (tl->t->tok == '!') { - not = 1; + if (tl->t->tok == '(') { + para = 1; vcc_NextToken(tl); } - if (tl->t->tok == '(') { - para = 1; + if (tl->t->tok == '!') { + not = 1; vcc_NextToken(tl); } @@ -90,7 +91,19 @@ ExpectErr(tl, CNUM); mask = UintVal(tl); } - Fc(tl, 1, "{ %u, %u, %u, %T },\n", not, mask, para, t); + Fc(tl, 1, "{ %u, %u, %u, %T, \"", not, mask, para, t); + if (para) + Fc(tl, 0, "("); + if (not) + Fc(tl, 0, "!"); + p = EncString(t); + Fc(tl, 0, "%s", p); + free(p); + if (mask) + Fc(tl, 0, "/%u", mask); + if (para) + Fc(tl, 0, ")"); + Fc(tl, 0, "\" },\n"); if (para) { ExpectErr(tl, ')'); @@ -99,7 +112,7 @@ ExpectErr(tl, ';'); vcc_NextToken(tl); } - Fc(tl, 1, "{ 0, 0, 0, (void*)0}\n", 0, 0); + Fc(tl, 1, "{ 0, 0, 0, (void*)0, ""}\n", 0, 0); tl->indent -= INDENT; Fc(tl, 1, "};\n\n"); Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2006-07-22 10:35:39 UTC (rev 557) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2006-07-22 10:41:00 UTC (rev 558) @@ -486,14 +486,15 @@ fputs("\n", f); fputs("struct vrt_acl {\n", f); fputs(" unsigned char not;\n", f); + fputs(" unsigned char mask;\n", f); fputs(" unsigned char paren;\n", f); - fputs(" unsigned char mask;\n", f); fputs(" const char *name;\n", f); + fputs(" const char *desc;\n", f); fputs(" void *priv;\n", f); fputs("};\n", f); fputs("\n", f); fputs("/* ACL related */\n", f); - fputs("int VRT_acl_match(struct sess *, struct vrt_acl *);\n", f); + fputs("int VRT_acl_match(struct sess *, const char *, struct vrt_acl *);\n", f); fputs("void VRT_acl_init(struct vrt_acl *);\n", f); fputs("void VRT_acl_fini(struct vrt_acl *);\n", f); fputs("\n", f); From phk at projects.linpro.no Sat Jul 22 12:00:18 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 22 Jul 2006 14:00:18 +0200 (CEST) Subject: r559 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060722120018.B42851EC3A8@projects.linpro.no> Author: phk Date: 2006-07-22 14:00:18 +0200 (Sat, 22 Jul 2006) New Revision: 559 Added: trunk/varnish-cache/bin/varnishd/cache_vrt_re.c Modified: trunk/varnish-cache/bin/varnishd/Makefile.am trunk/varnish-cache/include/vrt.h trunk/varnish-cache/lib/libvcl/vcc_compile.c trunk/varnish-cache/lib/libvcl/vcc_compile.h trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c Log: Implement regexp matching of strings in VCL. For now we default to REG_EXTENDED, but it might make sense to let the user control this flag and the case sensitivity. Another concern is the stringification of regexps, it may lead to backslash madness. Maybe we should define '...' string types also and do no backslash substitution in those at all. Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-22 10:41:00 UTC (rev 558) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-22 12:00:18 UTC (rev 559) @@ -31,6 +31,7 @@ cache_vcl.c \ cache_vrt.c \ cache_vrt_acl.c \ + cache_vrt_re.c \ cli_event.c \ hash_simple_list.c \ hash_classic.c \ Added: trunk/varnish-cache/bin/varnishd/cache_vrt_re.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt_re.c 2006-07-22 10:41:00 UTC (rev 558) +++ trunk/varnish-cache/bin/varnishd/cache_vrt_re.c 2006-07-22 12:00:18 UTC (rev 559) @@ -0,0 +1,72 @@ +/* + * $Id$ + * + * Runtime support for compiled VCL programs, regexps + * + */ + +#include +#include +#include +#include + +#include "shmlog.h" +#include "vrt.h" +#include "vrt_obj.h" +#include "sbuf.h" +#include "vcl.h" +#include "cache.h" + +void +VRT_re_init(void **rep, const char *re) +{ + regex_t *t; + int i; + + t = calloc(sizeof *t, 1); + assert(t != NULL); + i = regcomp(t, re, REG_EXTENDED | REG_NOSUB); + assert(i == 0); + *rep = t; +} + +void +VRT_re_fini(void *rep) +{ + + if (rep != NULL) + regfree(rep); +} + +int +VRT_re_match(const char *s, void *re) +{ + regex_t *t; + int i; + + t = re; + i = regexec(t, s, 0, NULL, 0); + if (i == 0) + return (1); + assert(i == REG_NOMATCH); + return (0); +} + +int +VRT_re_test(struct sbuf *sb, const char *re) +{ + int i, j; + regex_t t; + char buf[BUFSIZ]; + + memset(&t, 0, sizeof t); + i = regcomp(&t, re, REG_EXTENDED | REG_NOSUB); + if (i == 0) { + regfree(&t); + return (0); + } + j = regerror(i, &t, buf, sizeof buf); + sbuf_printf(sb, "Regexp compilation error:\n\n%s\n\n", buf); + regfree(&t); + return (1); +} Property changes on: trunk/varnish-cache/bin/varnishd/cache_vrt_re.c ___________________________________________________________________ Name: svn:keywords + Id Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2006-07-22 10:41:00 UTC (rev 558) +++ trunk/varnish-cache/include/vrt.h 2006-07-22 12:00:18 UTC (rev 559) @@ -8,6 +8,7 @@ */ struct sess; +struct sbuf; struct backend; struct VCL_conf; @@ -32,6 +33,12 @@ void VRT_acl_init(struct vrt_acl *); void VRT_acl_fini(struct vrt_acl *); +/* Regexp related */ +void VRT_re_init(void **, const char *); +void VRT_re_fini(void *); +int VRT_re_match(const char *, void *re); +int VRT_re_test(struct sbuf *, const char *); + void VRT_count(struct sess *, unsigned); int VRT_rewrite(const char *, const char *); void VRT_error(struct sess *, unsigned, const char *); Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.c 2006-07-22 10:41:00 UTC (rev 558) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.c 2006-07-22 12:00:18 UTC (rev 559) @@ -54,6 +54,7 @@ #include "vcc_priv.h" #include "vcc_compile.h" +#include "vrt.h" #include "libvcl.h" static struct method method_tab[] = { @@ -471,15 +472,36 @@ /*--------------------------------------------------------------------*/ static void +vcc_re(struct tokenlist *tl, const char *str, struct token *re) +{ + char buf[32], *p; + + p = EncString(re); + if (VRT_re_test(tl->sb, p)) { + vcc_ErrWhere(tl, re); + return; + } + free(p); + sprintf(buf, "VGC_re_%u", tl->recnt++); + + Fc(tl, 1, "VRT_re_match(%s, %s)\n", str, buf); + Fh(tl, 0, "void *%s;\n", buf); + Fi(tl, 0, "\tVRT_re_init(&%s, %T);\n", buf, re); + Ff(tl, 0, "\tVRT_re_fini(%s);\n", buf); +} + + +/*--------------------------------------------------------------------*/ + +static void Cond_String(struct var *vp, struct tokenlist *tl) { switch (tl->t->tok) { case '~': - Fc(tl, 1, "string_match(%s, ", vp->rname); vcc_NextToken(tl); ExpectErr(tl, CSTR); - Fc(tl, 0, "%T)\n", tl->t); + vcc_re(tl, vp->rname, tl->t); vcc_NextToken(tl); break; case T_EQ: Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.h 2006-07-22 10:41:00 UTC (rev 558) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.h 2006-07-22 12:00:18 UTC (rev 559) @@ -29,6 +29,8 @@ int nbackend; TAILQ_HEAD(, proc) procs; struct proc *curproc; + + unsigned recnt; }; enum var_type { Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2006-07-22 10:41:00 UTC (rev 558) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2006-07-22 12:00:18 UTC (rev 559) @@ -474,6 +474,7 @@ fputs(" */\n", f); fputs("\n", f); fputs("struct sess;\n", f); + fputs("struct sbuf;\n", f); fputs("struct backend;\n", f); fputs("struct VCL_conf;\n", f); fputs("\n", f); @@ -498,6 +499,12 @@ fputs("void VRT_acl_init(struct vrt_acl *);\n", f); fputs("void VRT_acl_fini(struct vrt_acl *);\n", f); fputs("\n", f); + fputs("/* Regexp related */\n", f); + fputs("void VRT_re_init(void **, const char *);\n", f); + fputs("void VRT_re_fini(void *);\n", f); + fputs("int VRT_re_match(const char *, void *re);\n", f); + fputs("int VRT_re_test(struct sbuf *, const char *);\n", f); + fputs("\n", f); fputs("void VRT_count(struct sess *, unsigned);\n", f); fputs("int VRT_rewrite(const char *, const char *);\n", f); fputs("void VRT_error(struct sess *, unsigned, const char *);\n", f); From phk at projects.linpro.no Sat Jul 22 13:58:31 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 22 Jul 2006 15:58:31 +0200 (CEST) Subject: r560 - trunk/varnish-cache/bin/varnishlog Message-ID: <20060722135831.DF6651EC3A8@projects.linpro.no> Author: phk Date: 2006-07-22 15:58:31 +0200 (Sat, 22 Jul 2006) New Revision: 560 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c Log: exit after error Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-22 12:00:18 UTC (rev 559) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-22 13:58:31 UTC (rev 560) @@ -233,8 +233,10 @@ if (w_opt != NULL) { wfile = fopen(w_opt, "w"); - if (wfile == NULL) + if (wfile == NULL) { perror(w_opt); + exit (1); + } } u = 0; v = 0; From phk at projects.linpro.no Sat Jul 22 16:15:17 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 22 Jul 2006 18:15:17 +0200 (CEST) Subject: r561 - trunk/varnish-cache/bin/varnishd Message-ID: <20060722161517.3784A1EC3A8@projects.linpro.no> Author: phk Date: 2006-07-22 18:15:17 +0200 (Sat, 22 Jul 2006) New Revision: 561 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c Log: Always use GET and HTTP/1.1 against the backend for fetch Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-22 13:58:31 UTC (rev 560) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-22 16:15:17 UTC (rev 561) @@ -303,6 +303,7 @@ void HTTP_Init(void); void http_CopyHttp(struct http *to, struct http *fm); void http_Write(struct worker *w, struct http *hp, int resp); +void http_GetReq(int fd, struct http *to, struct http *fm); void http_CopyReq(int fd, struct http *to, struct http *fm); void http_CopyResp(int fd, struct http *to, struct http *fm); void http_FilterHeader(int fd, struct http *to, struct http *fm, unsigned how); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-22 13:58:31 UTC (rev 560) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-22 16:15:17 UTC (rev 561) @@ -51,7 +51,7 @@ i &= ~O_NONBLOCK; i = fcntl(fd, F_SETFL, i); - while (cl != 0) { + while (cl > 0) { i = http_Read(hp, fd, p, cl); assert(i > 0); /* XXX seen */ p += i; @@ -273,7 +273,7 @@ assert(vc != NULL); /* XXX: handle this */ VSL(SLT_Backend, sp->fd, "%d %s", vc->fd, sp->backend->vcl_name); - http_CopyReq(vc->fd, vc->http, sp->http); + http_GetReq(vc->fd, vc->http, sp->http); http_FilterHeader(vc->fd, vc->http, sp->http, HTTPH_R_FETCH); http_PrintfHeader(vc->fd, vc->http, "X-Varnish: %u", sp->xid); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-22 13:58:31 UTC (rev 560) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-22 16:15:17 UTC (rev 561) @@ -517,6 +517,16 @@ /*--------------------------------------------------------------------*/ static void +http_seth(int fd, struct http *to, unsigned n, enum shmlogtag tag, const char *fm) +{ + assert(n < MAX_HTTP_HDRS); + assert(fm != NULL); + to->hd[n].b = (void*)(uintptr_t)fm; + to->hd[n].e = strchr(fm, '\0'); + VSLH(tag, fd, to, n); +} + +static void http_copyh(int fd, struct http *to, struct http *fm, unsigned n, enum shmlogtag tag) { @@ -528,6 +538,17 @@ } void +http_GetReq(int fd, struct http *to, struct http *fm) +{ + + CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC); + CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); + http_seth(fd, to, HTTP_HDR_REQ, SLT_Request, "GET"); + http_copyh(fd, to, fm, HTTP_HDR_URL, SLT_URL); + http_seth(fd, to, HTTP_HDR_PROTO, SLT_Protocol, "HTTP/1.1"); +} + +void http_CopyReq(int fd, struct http *to, struct http *fm) { From phk at projects.linpro.no Sat Jul 22 16:26:45 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 22 Jul 2006 18:26:45 +0200 (CEST) Subject: r562 - trunk/varnish-cache/bin/varnishd Message-ID: <20060722162645.5E9B11EC3A8@projects.linpro.no> Author: phk Date: 2006-07-22 18:26:45 +0200 (Sat, 22 Jul 2006) New Revision: 562 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: Action pass from vcl_hit() needs to go to STP_PASS Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-22 16:15:17 UTC (rev 561) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-22 16:26:45 UTC (rev 562) @@ -256,7 +256,7 @@ HSH_Deref(sp->obj); sp->obj = NULL; PassSession(sp); - sp->step = STP_PASSBODY; + sp->step = STP_PASS; return (0); } From phk at projects.linpro.no Sat Jul 22 16:55:17 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 22 Jul 2006 18:55:17 +0200 (CEST) Subject: r563 - trunk/varnish-cache/bin/varnishd Message-ID: <20060722165517.B06361EC399@projects.linpro.no> Author: phk Date: 2006-07-22 18:55:17 +0200 (Sat, 22 Jul 2006) New Revision: 563 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c Log: Make sure there always is a Host: header in fetch requests. We fill it in with backend.hostname, but this may not be optimal (direct IP# etc etc) so VCL should be able to override it. Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-22 16:26:45 UTC (rev 562) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-22 16:55:17 UTC (rev 563) @@ -258,6 +258,7 @@ int i; struct vbe_conn *vc; struct worker *w; + char *b; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); @@ -276,6 +277,10 @@ http_GetReq(vc->fd, vc->http, sp->http); http_FilterHeader(vc->fd, vc->http, sp->http, HTTPH_R_FETCH); http_PrintfHeader(vc->fd, vc->http, "X-Varnish: %u", sp->xid); + if (!http_GetHdr(vc->http, H_Host, &b)) { + http_PrintfHeader(vc->fd, vc->http, "Host: %s", + sp->backend->hostname); + } sp->t_req = time(NULL); WRK_Reset(w, &vc->fd); From phk at projects.linpro.no Sat Jul 22 20:57:03 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 22 Jul 2006 22:57:03 +0200 (CEST) Subject: r564 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060722205703.6378E1EC2FF@projects.linpro.no> Author: phk Date: 2006-07-22 22:57:02 +0200 (Sat, 22 Jul 2006) New Revision: 564 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_response.c trunk/varnish-cache/bin/varnishd/cache_session.c trunk/varnish-cache/include/shmlog_tags.h trunk/varnish-cache/include/stat_field.h Log: Add per address, per session and total statistics. We (will) collect data in unlocked per workerthread accumulators and whenever the workerthread leaves the session, we charge the bill to the srcaddr (issuing a StatAddr shmrecord), to the session and to the global counters in the stats struct. When sessions die we issue a StatSess shmrecord. StatAddr and StatSess has the same format: address port (always zero for StatAddr) duration (seconds) #sessions #requests #pipe #pass #fetch #hdrbytes #bodybytes Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-22 16:55:17 UTC (rev 563) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-22 20:57:02 UTC (rev 564) @@ -74,6 +74,19 @@ /*--------------------------------------------------------------------*/ +struct acct { + time_t first; + uint64_t sess; + uint64_t req; + uint64_t pipe; + uint64_t pass; + uint64_t fetch; + uint64_t hdrbytes; + uint64_t bodybytes; +}; + +/*--------------------------------------------------------------------*/ + struct worker { unsigned magic; #define WORKER_MAGIC 0x6391adcf @@ -90,6 +103,8 @@ struct iovec iov[MAX_IOVS]; unsigned niov; size_t liov; + + struct acct acct; }; struct workreq { @@ -189,20 +204,24 @@ struct srcaddr { unsigned magic; #define SRCADDR_MAGIC 0x375111db + + unsigned hash; TAILQ_ENTRY(srcaddr) list; - unsigned nsess; + struct srcaddrhead *sah; + char addr[TCP_ADDRBUFSIZE]; - unsigned sum; - time_t first; + unsigned nref; + time_t ttl; - uint64_t bytes; - struct srcaddrhead *sah; + + struct acct acct; }; struct sess { unsigned magic; #define SESS_MAGIC 0x2c2f9c5a int fd; + int id; unsigned xid; struct worker *wrk; @@ -237,6 +256,7 @@ time_t t0; struct workreq workreq; + struct acct acct; }; struct backend { @@ -344,7 +364,7 @@ struct sess *SES_New(struct sockaddr *addr, unsigned len); void SES_Delete(struct sess *sp); void SES_RefSrcAddr(struct sess *sp); -void SES_ChargeBytes(struct sess *sp, uint64_t bytes); +void SES_Charge(struct sess *sp); /* cache_shmlog.c */ Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-22 16:55:17 UTC (rev 563) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-07-22 20:57:02 UTC (rev 564) @@ -118,6 +118,7 @@ assert(sp != NULL); /* XXX handle */ sp->fd = i; + sp->id = i; #ifdef SO_NOSIGPIPE /* XXX Linux */ i = 1; Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-22 16:55:17 UTC (rev 563) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-22 20:57:02 UTC (rev 564) @@ -89,6 +89,7 @@ VCL_Rel(sp->vcl); sp->vcl = NULL; + SES_Charge(sp); vca_return_session(sp); return (1); } @@ -316,6 +317,7 @@ if (o == NULL) { VSL(SLT_Debug, sp->fd, "on waiting list on obj %u", sp->obj->xid); + SES_Charge(sp); return (1); } @@ -584,7 +586,7 @@ switch (sp->step) { #define STEP(l,u) \ case STP_##u: \ - VSL(SLT_Debug, sp->fd, "State " #u); \ + VSL(SLT_Debug, sp->id, "State " #u); \ done = cnt_##l(sp); \ break; #include "steps.h" Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-22 16:55:17 UTC (rev 563) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-22 20:57:02 UTC (rev 564) @@ -105,8 +105,10 @@ AZ(pthread_mutex_unlock(&wrk_mtx)); CHECK_OBJ_NOTNULL(wrq->sess, SESS_MAGIC); wrq->sess->wrk = w; - if (wrq->sess->srcaddr == NULL) + if (wrq->sess->srcaddr == NULL) { + w->acct.sess++; SES_RefSrcAddr(wrq->sess); + } if (w->nobj != NULL) CHECK_OBJ(w->nobj, OBJECT_MAGIC); if (w->nobjhead != NULL) Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-22 16:55:17 UTC (rev 563) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-22 20:57:02 UTC (rev 564) @@ -138,7 +138,6 @@ { struct storage *st; unsigned u = 0; - uint64_t bytes = 0; if (sp->obj->response == 200 && sp->http->conds && res_do_conds(sp)) return; @@ -162,9 +161,6 @@ WRK_Reset(sp->wrk, &sp->fd); http_Write(sp->wrk, sp->http, 1); -#if 0 /* XXX */ - bytes += sbuf_len(sb); -#endif /* XXX: conditional request handling */ if (!strcmp(sp->http->hd[HTTP_HDR_REQ].b, "GET")) { TAILQ_FOREACH(st, &sp->obj->store, list) { @@ -181,7 +177,6 @@ } assert(u == sp->obj->len); } - SES_ChargeBytes(sp, bytes + u); if (WRK_Flush(sp->wrk)) vca_close_session(sp, "remote closed"); } Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-22 16:55:17 UTC (rev 563) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-22 20:57:02 UTC (rev 564) @@ -74,10 +74,10 @@ AZ(pthread_mutex_lock(&ses_mtx)); c3 = NULL; TAILQ_FOREACH_SAFE(c, ch, list, c2) { - if (c->sum == u && !strcmp(c->addr, sp->addr)) { - if (c->nsess == 0) + if (c->hash == u && !strcmp(c->addr, sp->addr)) { + if (c->nref == 0) VSL_stats->n_srcaddr_act++; - c->nsess++; + c->nref++; c->ttl = now + CLIENT_TTL; sp->srcaddr = c; TAILQ_REMOVE(ch, c, list); @@ -90,7 +90,7 @@ AZ(pthread_mutex_unlock(&ses_mtx)); return; } - if (c->nsess > 0 || c->ttl > now) + if (c->nref > 0 || c->ttl > now) continue; if (c3 == NULL) { c3 = c; @@ -111,10 +111,10 @@ if (c3 != NULL) { memset(c3, 0, sizeof *c3); strcpy(c3->addr, sp->addr); - c3->sum = u; - c3->first = now; + c3->hash = u; + c3->acct.first = now; c3->ttl = now + CLIENT_TTL; - c3->nsess = 1; + c3->nref = 1; c3->sah = ch; VSL_stats->n_srcaddr_act++; TAILQ_INSERT_TAIL(ch, c3, list); @@ -123,24 +123,42 @@ AZ(pthread_mutex_unlock(&ses_mtx)); } +static void +ses_sum_acct(struct acct *sum, struct acct *inc) +{ + + sum->sess += inc->sess; + sum->req += inc->req; + sum->pipe += inc->pipe; + sum->pass += inc->pass; + sum->fetch += inc->fetch; + sum->hdrbytes += inc->hdrbytes; + sum->bodybytes += inc->bodybytes; +} + void -SES_ChargeBytes(struct sess *sp, uint64_t bytes) +SES_Charge(struct sess *sp) { - struct srcaddr *sa; - time_t now; + struct acct *a = &sp->wrk->acct; + struct acct *b = &sp->srcaddr->acct; - assert(sp->srcaddr != NULL); - sa = sp->srcaddr; - now = time(NULL); + ses_sum_acct(&sp->acct, a); + AZ(pthread_mutex_lock(&ses_mtx)); - sa->bytes += bytes; - sa->ttl = now + CLIENT_TTL; - TAILQ_REMOVE(sa->sah, sa, list); - TAILQ_INSERT_TAIL(sa->sah, sa, list); - bytes = sa->bytes; + ses_sum_acct(b, a); + VSL(SLT_StatAddr, sp->id, "%s 0 %d %ju %ju %ju %ju %ju %ju %ju", + sp->srcaddr->addr, time(NULL) - b->first, + b->sess, b->req, b->pipe, b->pass, + b->fetch, b->hdrbytes, b->bodybytes); + VSL_stats->s_sess += a->sess; + VSL_stats->s_req += a->req; + VSL_stats->s_pipe += a->pipe; + VSL_stats->s_pass += a->pass; + VSL_stats->s_fetch += a->fetch; + VSL_stats->s_hdrbytes += a->hdrbytes; + VSL_stats->s_bodybytes += a->bodybytes; AZ(pthread_mutex_unlock(&ses_mtx)); - VSL(SLT_SrcAddr, sp->fd, "%s %jd %d", - sa->addr, (intmax_t)(bytes), now - sa->first); + memset(a, 0, sizeof *a); } static void @@ -153,9 +171,9 @@ } assert(sp->srcaddr != NULL); AZ(pthread_mutex_lock(&ses_mtx)); - assert(sp->srcaddr->nsess > 0); - sp->srcaddr->nsess--; - if (sp->srcaddr->nsess == 0) + assert(sp->srcaddr->nref > 0); + sp->srcaddr->nref--; + if (sp->srcaddr->nref == 0) VSL_stats->n_srcaddr_act--; sp->srcaddr = NULL; AZ(pthread_mutex_unlock(&ses_mtx)); @@ -183,17 +201,26 @@ assert(len < sizeof(sm->sockaddr)); memcpy(sm->sess.sockaddr, addr, len); sm->sess.sockaddrlen = len; + http_Setup(&sm->http, (void *)(sm + 1), heritage.mem_workspace); + + sm->sess.acct.first = time(NULL); + return (&sm->sess); } void SES_Delete(struct sess *sp) { + struct acct *b = &sp->acct; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); VSL_stats->n_sess--; ses_relsrcaddr(sp); + VSL(SLT_StatSess, sp->id, "%s %s %d %ju %ju %ju %ju %ju %ju %ju", + sp->addr, sp->port, time(NULL) - b->first, + b->sess, b->req, b->pipe, b->pass, + b->fetch, b->hdrbytes, b->bodybytes); CHECK_OBJ_NOTNULL(sp->mem, SESSMEM_MAGIC); free(sp->mem); } Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-07-22 16:55:17 UTC (rev 563) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-07-22 20:57:02 UTC (rev 564) @@ -11,7 +11,8 @@ SLTM(Debug) SLTM(Error) SLTM(CLI) -SLTM(SrcAddr) +SLTM(StatAddr) +SLTM(StatSess) SLTM(SessionOpen) SLTM(SessionReuse) SLTM(SessionClose) Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2006-07-22 16:55:17 UTC (rev 563) +++ trunk/varnish-cache/include/stat_field.h 2006-07-22 20:57:02 UTC (rev 564) @@ -29,3 +29,11 @@ MAC_STAT(n_deathrow, uint64_t, "u", "N objects on deathrow") MAC_STAT(losthdr, uint64_t, "u", "HTTP header overflows") + +MAC_STAT(s_sess, uint64_t, "u", "Total Sessions") +MAC_STAT(s_req, uint64_t, "u", "Total Requests") +MAC_STAT(s_pipe, uint64_t, "u", "Total pipe") +MAC_STAT(s_pass, uint64_t, "u", "Total pass") +MAC_STAT(s_fetch, uint64_t, "u", "Total fetch") +MAC_STAT(s_hdrbytes, uint64_t, "u", "Total header bytes") +MAC_STAT(s_bodybytes, uint64_t, "u", "Total body bytes") From phk at projects.linpro.no Sat Jul 22 21:20:08 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 22 Jul 2006 23:20:08 +0200 (CEST) Subject: r565 - trunk/varnish-cache/bin/varnishd Message-ID: <20060722212008.7B0261EC39A@projects.linpro.no> Author: phk Date: 2006-07-22 23:20:08 +0200 (Sat, 22 Jul 2006) New Revision: 565 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_response.c trunk/varnish-cache/bin/varnishd/stevedore.h trunk/varnish-cache/bin/varnishd/storage_file.c trunk/varnish-cache/bin/varnishd/storage_malloc.c Log: Eliminate redundant args from stevedore->send() Have WRK_Write() and friends return number of bytes (we can't use WRK_Flush() as that may act on both header and body). Collect more stats. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-22 20:57:02 UTC (rev 564) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-22 21:20:08 UTC (rev 565) @@ -322,7 +322,7 @@ /* cache_http.c */ void HTTP_Init(void); void http_CopyHttp(struct http *to, struct http *fm); -void http_Write(struct worker *w, struct http *hp, int resp); +unsigned http_Write(struct worker *w, struct http *hp, int resp); void http_GetReq(int fd, struct http *to, struct http *fm); void http_CopyReq(int fd, struct http *to, struct http *fm); void http_CopyResp(int fd, struct http *to, struct http *fm); @@ -346,7 +346,7 @@ /* cache_pass.c */ void PassSession(struct sess *sp); -void PassBody(struct worker *w, struct sess *sp); +void PassBody(struct sess *sp); /* cache_pipe.c */ void PipeSession(struct sess *sp); @@ -356,8 +356,8 @@ void WRK_QueueSession(struct sess *sp); void WRK_Reset(struct worker *w, int *fd); int WRK_Flush(struct worker *w); -void WRK_Write(struct worker *w, const void *ptr, int len); -void WRK_WriteH(struct worker *w, struct http_hdr *hh, const char *suf); +unsigned WRK_Write(struct worker *w, const void *ptr, int len); +unsigned WRK_WriteH(struct worker *w, struct http_hdr *hh, const char *suf); /* cache_session.c [SES] */ void SES_Init(void); Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-22 20:57:02 UTC (rev 564) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-22 21:20:08 UTC (rev 565) @@ -191,6 +191,7 @@ FetchBody(sp); HSH_Ref(sp->obj); /* get another, STP_DELIVER will deref */ HSH_Unbusy(sp->obj); + sp->wrk->acct.fetch++; sp->step = STP_DELIVER; return (0); } @@ -448,7 +449,9 @@ static int cnt_passbody(struct sess *sp) { - PassBody(sp->wrk, sp); + + sp->wrk->acct.pass++; + PassBody(sp); sp->step = STP_DONE; return (0); } @@ -472,6 +475,7 @@ cnt_pipe(struct sess *sp) { + sp->wrk->acct.pipe++; PipeSession(sp); sp->step = STP_DONE; return (0); @@ -515,6 +519,7 @@ assert(sp->obj == NULL); + sp->wrk->acct.req++; done = http_DissectRequest(sp->http, sp->fd); if (done != 0) { RES_Error(sp, done, NULL); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-22 20:57:02 UTC (rev 564) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-22 21:20:08 UTC (rev 565) @@ -635,28 +635,29 @@ /*--------------------------------------------------------------------*/ -void +unsigned http_Write(struct worker *w, struct http *hp, int resp) { - unsigned u; + unsigned u, l; if (resp) { assert(hp->hd[HTTP_HDR_STATUS].b != NULL); - WRK_WriteH(w, &hp->hd[HTTP_HDR_PROTO], " "); - WRK_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " "); - WRK_WriteH(w, &hp->hd[HTTP_HDR_RESPONSE], "\r\n"); + l = WRK_WriteH(w, &hp->hd[HTTP_HDR_PROTO], " "); + l += WRK_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " "); + l += WRK_WriteH(w, &hp->hd[HTTP_HDR_RESPONSE], "\r\n"); } else { assert(hp->hd[HTTP_HDR_URL].b != NULL); - WRK_WriteH(w, &hp->hd[HTTP_HDR_REQ], " "); - WRK_WriteH(w, &hp->hd[HTTP_HDR_URL], " "); - WRK_WriteH(w, &hp->hd[HTTP_HDR_PROTO], "\r\n"); + l = WRK_WriteH(w, &hp->hd[HTTP_HDR_REQ], " "); + l += WRK_WriteH(w, &hp->hd[HTTP_HDR_URL], " "); + l += WRK_WriteH(w, &hp->hd[HTTP_HDR_PROTO], "\r\n"); } for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) { assert(hp->hd[u].b != NULL); assert(hp->hd[u].e != NULL); - WRK_WriteH(w, &hp->hd[u], "\r\n"); + l += WRK_WriteH(w, &hp->hd[u], "\r\n"); } - WRK_Write(w, "\r\n", -1); + l += WRK_Write(w, "\r\n", -1); + return (l); } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-22 20:57:02 UTC (rev 564) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-22 21:20:08 UTC (rev 565) @@ -46,7 +46,7 @@ if (i == 0 && bi == NULL) return (1); assert(i > 0); - WRK_Write(sp->wrk, buf, i); + sp->wrk->acct.bodybytes += WRK_Write(sp->wrk, buf, i); if (WRK_Flush(sp->wrk)) vca_close_session(sp, "remote closed"); cl -= i; @@ -93,7 +93,7 @@ if (u == 0) break; - WRK_Write(sp->wrk, p, q - p); + sp->wrk->acct.bodybytes += WRK_Write(sp->wrk, p, q - p); p = q; @@ -105,14 +105,15 @@ } if (bp - p < j) j = bp - p; - WRK_Write(sp->wrk, p, j); + sp->wrk->acct.bodybytes += WRK_Write(sp->wrk, p, j); p += j; u -= j; } while (u > 0) { if (http_GetTail(hp, u, &b, &e)) { j = e - b; - WRK_Write(sp->wrk, q, j); + sp->wrk->acct.bodybytes += + WRK_Write(sp->wrk, q, j); u -= j; } else break; @@ -125,7 +126,7 @@ j = sizeof buf; i = read(fd, buf, j); assert(i > 0); - WRK_Write(sp->wrk, buf, i); + sp->wrk->acct.bodybytes += WRK_Write(sp->wrk, buf, i); u -= i; if (WRK_Flush(sp->wrk)) vca_close_session(sp, "remote closed"); @@ -138,7 +139,7 @@ /*--------------------------------------------------------------------*/ void -PassBody(struct worker *w, struct sess *sp) +PassBody(struct sess *sp) { struct vbe_conn *vc; char *b; @@ -152,8 +153,8 @@ http_CopyResp(sp->fd, sp->http, vc->http); http_FilterHeader(sp->fd, sp->http, vc->http, HTTPH_A_PASS); http_PrintfHeader(sp->fd, sp->http, "X-Varnish: %u", sp->xid); - WRK_Reset(w, &sp->fd); - http_Write(w, sp->http, 1); + WRK_Reset(sp->wrk, &sp->fd); + sp->wrk->acct.hdrbytes += http_Write(sp->wrk, sp->http, 1); if (http_GetHdr(vc->http, H_Content_Length, &b)) cls = pass_straight(sp, vc->fd, vc->http, b); @@ -165,7 +166,7 @@ cls = pass_straight(sp, vc->fd, vc->http, NULL); } - if (WRK_Flush(w)) + if (WRK_Flush(sp->wrk)) vca_close_session(sp, "remote closed"); if (http_GetHdr(vc->http, H_Connection, &b) && !strcasecmp(b, "close")) Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-22 20:57:02 UTC (rev 564) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-22 21:20:08 UTC (rev 565) @@ -60,27 +60,29 @@ return (w->werr); } -void +unsigned WRK_WriteH(struct worker *w, struct http_hdr *hh, const char *suf) { + unsigned u; CHECK_OBJ_NOTNULL(w, WORKER_MAGIC); assert(w != NULL); assert(hh != NULL); assert(hh->b != NULL); assert(hh->e != NULL); - WRK_Write(w, hh->b, hh->e - hh->b); + u = WRK_Write(w, hh->b, hh->e - hh->b); if (suf != NULL) - WRK_Write(w, suf, -1); + u += WRK_Write(w, suf, -1); + return (u); } -void +unsigned WRK_Write(struct worker *w, const void *ptr, int len) { CHECK_OBJ_NOTNULL(w, WORKER_MAGIC); if (len == 0 || *w->wfd < 0) - return; + return (0); if (len == -1) len = strlen(ptr); if (w->niov == MAX_IOVS) @@ -88,6 +90,7 @@ w->iov[w->niov].iov_base = (void*)(uintptr_t)ptr; w->iov[w->niov++].iov_len = len; w->liov += len; + return (len); } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-22 20:57:02 UTC (rev 564) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-22 21:20:08 UTC (rev 565) @@ -159,21 +159,21 @@ if (sp->doclose != NULL) http_PrintfHeader(sp->fd, sp->http, "Connection: close"); WRK_Reset(sp->wrk, &sp->fd); - http_Write(sp->wrk, sp->http, 1); + sp->wrk->acct.hdrbytes += http_Write(sp->wrk, sp->http, 1); /* XXX: conditional request handling */ if (!strcmp(sp->http->hd[HTTP_HDR_REQ].b, "GET")) { TAILQ_FOREACH(st, &sp->obj->store, list) { assert(st->stevedore != NULL); u += st->len; + sp->wrk->acct.bodybytes += st->len; if (st->stevedore->send == NULL) { WRK_Write(sp->wrk, st->ptr, st->len); - continue; + } else { + st->stevedore->send(st, sp); + sp->wrk->niov = 0; + sp->wrk->liov = 0; } - st->stevedore->send(st, sp, - sp->wrk->iov, sp->wrk->niov, sp->wrk->liov); - sp->wrk->niov = 0; - sp->wrk->liov = 0; } assert(u == sp->obj->len); } Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2006-07-22 20:57:02 UTC (rev 564) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2006-07-22 21:20:08 UTC (rev 565) @@ -11,7 +11,7 @@ typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); typedef void storage_trim_f(struct storage *, size_t size); typedef void storage_free_f(struct storage *); -typedef void storage_send_f(struct storage *, struct sess *, struct iovec *, int niovec, size_t liovec); +typedef void storage_send_f(struct storage *, struct sess *); struct stevedore { const char *name; Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-22 20:57:02 UTC (rev 564) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-07-22 21:20:08 UTC (rev 565) @@ -555,7 +555,7 @@ /*--------------------------------------------------------------------*/ static void -smf_send(struct storage *st, struct sess *sp, struct iovec *iov, int niov, size_t liov) +smf_send(struct storage *st, struct sess *sp) { struct smf *smf; int i; @@ -565,32 +565,33 @@ smf = st->priv; memset(&sfh, 0, sizeof sfh); - sfh.headers = iov; - sfh.hdr_cnt = niov; + sfh.headers = sp->wrk->iov; + sfh.hdr_cnt = sp->wrk->niov; i = sendfile(smf->sc->fd, sp->fd, smf->offset, st->len, &sfh, &sent, 0); - if (sent == st->len + liov) + if (sent == st->len + sp->wrk->liov) return; vca_close_session(sp, "remote closed"); if (errno == EPIPE || errno == ENOTCONN) return; VSL(SLT_Debug, sp->fd, "sent i=%d sent=%ju size=%ju liov=%ju errno=%d\n", - i, (uintmax_t)sent, (uintmax_t)st->len, (uintmax_t)liov, errno); + i, (uintmax_t)sent, (uintmax_t)st->len, + (uintmax_t)sp->wrk->liov, errno); } /*--------------------------------------------------------------------*/ struct stevedore smf_stevedore = { - "file", - smf_init, - smf_open, - smf_alloc, - smf_trim, - smf_free, - smf_send + .name = "file", + .init = smf_init, + .open = smf_open, + .alloc = smf_alloc, + .trim = smf_trim, + .free = smf_free, + .send = smf_send }; #ifdef INCLUDE_TEST_DRIVER Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-07-22 20:57:02 UTC (rev 564) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-07-22 21:20:08 UTC (rev 565) @@ -39,10 +39,7 @@ } struct stevedore sma_stevedore = { - "malloc", - NULL, /* init */ - NULL, /* open */ - sma_alloc, - NULL, /* trim */ - sma_free + .name = "malloc", + .alloc = sma_alloc, + .free = sma_free }; From des at linpro.no Sat Jul 22 21:22:34 2006 From: des at linpro.no (Dag-Erling =?iso-8859-1?Q?Sm=F8rgrav?=) Date: Sat, 22 Jul 2006 23:22:34 +0200 Subject: r559 - in trunk/varnish-cache: bin/varnishd include lib/libvcl References: <20060722120018.B42851EC3A8@projects.linpro.no> Message-ID: phk at projects.linpro.no writes: > Log: > Implement regexp matching of strings in VCL. > > For now we default to REG_EXTENDED, but it might make sense > to let the user control this flag and the case sensitivity. > > Another concern is the stringification of regexps, it may lead > to backslash madness. Maybe we should define '...' string types > also and do no backslash substitution in those at all. How about using the syntax used in ed / sed / ex / vi / perl etc, i.e. /regex/? DES -- Dag-Erling Sm?rgrav Senior Software Developer Linpro AS - www.linpro.no From phk at projects.linpro.no Sat Jul 22 22:01:04 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 23 Jul 2006 00:01:04 +0200 (CEST) Subject: r566 - trunk/varnish-cache/bin/varnishd Message-ID: <20060722220104.A93BA1EC399@projects.linpro.no> Author: phk Date: 2006-07-23 00:01:04 +0200 (Sun, 23 Jul 2006) New Revision: 566 Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c Log: reorg a little bit. Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-22 21:20:08 UTC (rev 565) +++ trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-22 22:01:04 UTC (rev 566) @@ -177,16 +177,16 @@ CAST_OBJ_NOTNULL(he, oh->hashpriv, HCL_ENTRY_MAGIC); mtx = he->mtx; AZ(pthread_mutex_lock(&hcl_mutex[mtx])); - if (--he->refcnt == 0) { - free(he->key1); - free(he->key2); - TAILQ_REMOVE(&hcl_head[he->hash], he, list); - free(he); - ret = 0; - } else - ret = 1; + if (--he->refcnt >= 0) { + AZ(pthread_mutex_unlock(&hcl_mutex[mtx])); + return (1) + } + TAILQ_REMOVE(&hcl_head[he->hash], he, list); AZ(pthread_mutex_unlock(&hcl_mutex[mtx])); - return (ret); + free(he->key1); + free(he->key2); + free(he); + return (0); } /*--------------------------------------------------------------------*/ From phk at phk.freebsd.dk Sat Jul 22 22:18:11 2006 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Sat, 22 Jul 2006 22:18:11 +0000 Subject: r559 - in trunk/varnish-cache: bin/varnishd include lib/libvcl In-Reply-To: Your message of "Sat, 22 Jul 2006 23:22:34 +0200." Message-ID: <81582.1153606691@critter.freebsd.dk> In message , Dag-Erling =?iso-8859-1?Q?Sm=F8rgra v?= writes: >phk at projects.linpro.no writes: >> Log: >> Implement regexp matching of strings in VCL. >> >> For now we default to REG_EXTENDED, but it might make sense >> to let the user control this flag and the case sensitivity. >> >> Another concern is the stringification of regexps, it may lead >> to backslash madness. Maybe we should define '...' string types >> also and do no backslash substitution in those at all. > >How about using the syntax used in ed / sed / ex / vi / perl etc, >i.e. /regex/? That would force '/' to be escaped and it is heavily used in URLS. I may have overlooked the obvious however: the current backslash processing on CSTR ("...") is not actually called for as far as I can see, so maybe removing that is a much better idea. The reason I put it in there was for HTML error messages, but they need to be handled more sensibly than that. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at projects.linpro.no Mon Jul 24 10:13:29 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 24 Jul 2006 12:13:29 +0200 (CEST) Subject: r567 - trunk/varnish-cache/lib/libvcl Message-ID: <20060724101329.4A6431EC38E@projects.linpro.no> Author: phk Date: 2006-07-24 12:13:29 +0200 (Mon, 24 Jul 2006) New Revision: 567 Removed: trunk/varnish-cache/lib/libvcl/vcl_compile.c Log: remove this file (again) Deleted: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-07-22 22:01:04 UTC (rev 566) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-07-24 10:13:29 UTC (rev 567) @@ -1,1925 +0,0 @@ -/* - * $Id$ - */ - -/* - * XXX: - * generate interface structure - * - * XXX: - * Better error messages, throughout. - * >It also accured to me that we could link the errors to the error - * >documentation. - * > - * >Unreferenced function 'request_policy', first mention is - * > Line 8 Pos 4 - * > sub request_policy { - * > ----##############-- - * >Read more about this type of error: - * >http://varnish/doc/error.html#Unreferenced%20function - * > - * > - * > Unknown variable 'obj.bandwidth' - * > At: Line 88 Pos 12 - * > if (obj.bandwidth < 1 kb/h) { - * > ------------#############------------ - * >Read more about this type of error: - * >http://varnish/doc/error.html#Unknown%20variable - * - * XXX: - * Create proper tmp filenames for .h, .c and .o - * - * XXX: - * and all the rest... - */ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "vcl_priv.h" -#include "vcl_returns.h" - -#include "libvcl.h" - -#define ERRCHK(tl) do { if ((tl)->err) return; } while (0) - -#define INDENT 2 - -struct token { - unsigned tok; - const char *b; - const char *e; - TAILQ_ENTRY(token) list; - unsigned cnt; -}; - -struct tokenlist { - TAILQ_HEAD(, token) tokens; - const char *b; - const char *e; - struct token *t; - int indent; - unsigned cnt; - struct sbuf *fc, *fh; - TAILQ_HEAD(, ref) refs; - struct sbuf *sb; - int err; - int nbackend; - TAILQ_HEAD(, proc) procs; - struct proc *curproc; -}; - -enum var_type { - BACKEND, - BOOL, - INT, - FLOAT, - SIZE, - RATE, - TIME, - STRING, - IP, - HOSTNAME, - PORTNAME, - HEADER -}; - - -enum ref_type { - R_FUNC, - R_ACL, - R_BACKEND -}; - -struct ref { - enum ref_type type; - struct token *name; - unsigned defcnt; - unsigned refcnt; - TAILQ_ENTRY(ref) list; -}; - -struct var { - const char *name; - enum var_type fmt; - int len; - const char *rname; - const char *lname; -}; - -/*-------------------------------------------------------------------- - * Consistency check - */ - -static struct method { - const char *name; - const char *defname; - unsigned returns; -} method_tab[] = { -#define VCL_RET_MAC(a,b,c) -#define VCL_MET_MAC(a,b,c) { "vcl_"#a, "default_vcl_"#a, c }, -#include "vcl_returns.h" -#undef VCL_MET_MAC -#undef VCL_RET_MAC - { NULL, 0U } -}; - -struct proccall { - TAILQ_ENTRY(proccall) list; - struct proc *p; - struct token *t; -}; - -struct proc { - TAILQ_ENTRY(proc) list; - TAILQ_HEAD(,proccall) calls; - struct token *name; - unsigned returns; - unsigned exists; - unsigned called; - unsigned active; - struct token *returnt[VCL_RET_MAX]; -}; - -/*--------------------------------------------------------------------*/ - -static struct var be_vars[] = { - { "backend.host", - HOSTNAME, 0, NULL, "VRT_set_backend_hostname(backend, %s)" }, - { "backend.port", - PORTNAME, 0, NULL, "VRT_set_backend_portname(backend, %s)" }, -}; - - -static struct var vars[] = { - { "req.request", STRING, 0, "VRT_GetReq(sp)" }, - { "obj.valid", BOOL, 0, "VRT_obj_valid(sp)" }, - { "obj.cacheable", BOOL, 0, "VRT_obj_cacheable(sp)" }, - { "obj.backend", BACKEND, 0, "VRT_obj_backend(sp)" }, - { "req.http.", HEADER, 0, NULL }, -#if 0 - { "req.ttlfactor", FLOAT, 0, "req->ttlfactor" }, - { "req.url.host", STRING, 0, "req->url.host" }, - { "req.url.path", STRING, 0, "req->url.path" }, - { "req.useragent", STRING, 0, "req->useragent" }, - { "req.backend", BACKEND, 0, "req->backend" }, - { "client.ip", IP, 0, "client->ip" }, - { "backend.response_time", TIME, 0, "backend->responsetime" }, - { "backend.down", BOOL, 0, "backend->down" }, - { "backend.timeout", TIME, 0, "backend->timeout" }, - { "backend.bandwidth", RATE, 0, "backend->bandwidth" }, - { "obj.exist", BOOL, 0, "obj->exists" }, - { "obj.ttl", TIME, 0, "obj->ttl" }, - { "obj.result", INT, 0, "obj->result" }, - { "obj.size", SIZE, 0, "obj->size" }, - { "obj.usage", INT, 0, "obj->usage" }, -#endif - { NULL, INT, 0, "NULL" } -}; - -static void Compound(struct tokenlist *tl); -static void Cond_0(struct tokenlist *tl); -static struct proc *AddProc(struct tokenlist *tl, struct token *t, int def); -static void AddCall(struct tokenlist *tl, struct token *t); -static const char *vcc_default_vcl_b, *vcc_default_vcl_e; - -/*--------------------------------------------------------------------*/ - -static void -ErrToken(struct tokenlist *tl, struct token *t) -{ - - if (t->tok == EOI) - sbuf_printf(tl->sb, "end of input"); - else - sbuf_printf(tl->sb, "'%T'", t); -} - -static void -_ErrInternal(struct tokenlist *tl, const char *func, unsigned line) -{ - - sbuf_printf(tl->sb, "VCL compiler internal error at %s():%u\n", - func, line); - tl->err = 1; -} - -#define ErrInternal(tl) _ErrInternal(tl, __func__, __LINE__) - -static void -ErrWhere(struct tokenlist *tl, struct token *t) -{ - unsigned lin, pos, x, y; - const char *p, *l, *f, *b, *e; - - lin = 1; - pos = 0; - if (t->tok == METHOD) - return; - if (t->b >= vcc_default_vcl_b && t->b < vcc_default_vcl_e) { - f = "Default VCL code (compiled in)"; - b = vcc_default_vcl_b; - e = vcc_default_vcl_e; - } else { - f = "VCL code"; - b = tl->b; - e = tl->e; - } - for (l = p = b; p < t->b; p++) { - if (*p == '\n') { - lin++; - pos = 0; - l = p + 1; - } else if (*p == '\t') { - pos &= ~7; - pos += 8; - } else - pos++; - } - sbuf_printf(tl->sb, "In %s Line %d Pos %d\n", f, lin, pos); - x = y = 0; - for (p = l; p < e && *p != '\n'; p++) { - if (*p == '\t') { - y &= ~7; - y += 8; - while (x < y) { - sbuf_bcat(tl->sb, " ", 1); - x++; - } - } else { - x++; - y++; - sbuf_bcat(tl->sb, p, 1); - } - } - sbuf_cat(tl->sb, "\n"); - x = y = 0; - for (p = l; p < e && *p != '\n'; p++) { - if (p >= t->b && p < t->e) { - sbuf_bcat(tl->sb, "#", 1); - x++; - y++; - continue; - } - if (*p == '\t') { - y &= ~7; - y += 8; - } else - y++; - while (x < y) { - sbuf_bcat(tl->sb, "-", 1); - x++; - } - } - sbuf_cat(tl->sb, "\n"); - tl->err = 1; -} - -/*--------------------------------------------------------------------*/ - -static void -NextToken(struct tokenlist *tl) -{ - tl->t = TAILQ_NEXT(tl->t, list); - if (tl->t == NULL) { - sbuf_printf(tl->sb, - "Ran out of input, something is missing or" - " maybe unbalanced (...) or {...}\n"); - tl->err = 1; - return; - } -} - -static void -_Expect(struct tokenlist *tl, unsigned tok, int line) -{ - if (tl->t->tok == tok) - return; - sbuf_printf(tl->sb, "Expected %s got ", vcl_tnames[tok]); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, "\n(program line %u), at\n", line); - ErrWhere(tl, tl->t); -} - -#define Expect(a, b) _Expect(a, b, __LINE__) -#define ExpectErr(a, b) do { _Expect(a, b, __LINE__); ERRCHK(a);} while (0) - -#define L(tl, foo) do { \ - tl->indent += INDENT; \ - foo; \ - tl->indent -= INDENT; \ -} while (0) - -#define C(tl, sep) do { \ - Fc(tl, 1, "VRT_count(sp, %u)%s\n", ++tl->cnt, sep); \ - tl->t->cnt = tl->cnt; \ -} while (0) - -/*-------------------------------------------------------------------- - * Printf output to the two sbufs, possibly indented - */ - -static void -Fh(struct tokenlist *tl, int indent, const char *fmt, ...) -{ - va_list ap; - - if (indent) - sbuf_printf(tl->fh, "%*.*s", tl->indent, tl->indent, ""); - va_start(ap, fmt); - sbuf_vprintf(tl->fh, fmt, ap); - va_end(ap); -} - -static void -Fc(struct tokenlist *tl, int indent, const char *fmt, ...) -{ - va_list ap; - - if (indent) - sbuf_printf(tl->fc, "%*.*s", tl->indent, tl->indent, ""); - va_start(ap, fmt); - sbuf_vprintf(tl->fc, fmt, ap); - va_end(ap); -} - -/*-------------------------------------------------------------------- - * Compare token to token - */ - -static int -Teq(struct token *t1, struct token *t2) -{ - if (t1->e - t1->b != t2->e - t2->b) - return (0); - return (!memcmp(t1->b, t2->b, t1->e - t1->b)); -} - -/*-------------------------------------------------------------------- - * Compare ID token to string, return true of match - */ - -static int -IdIs(struct token *t, const char *p) -{ - const char *q; - - assert(t->tok == ID); - for (q = t->b; q < t->e && *p != '\0'; p++, q++) - if (*q != *p) - return (0); - if (q != t->e || *p != '\0') - return (0); - return (1); -} - -/*--------------------------------------------------------------------*/ - -static char * -EncString(struct token *t) -{ - char *p, *q; - const char *r; - unsigned u; - - assert(t->tok == CSTR); - p = malloc(t->e - t->b); - assert(p != NULL); - q = p; - for (r = t->b + 1; r < t->e - 1; ) { - if (*r != '\\') { - *q++ = *r++; - continue; - } - switch (r[1]) { - case 'n': *q++ = '\n'; r += 2; break; - case 'r': *q++ = '\r'; r += 2; break; - case 'v': *q++ = '\v'; r += 2; break; - case 'f': *q++ = '\f'; r += 2; break; - case 't': *q++ = '\t'; r += 2; break; - case 'b': *q++ = '\b'; r += 2; break; - case '0': case '1': case '2': case '3': - case '4': case '5': case '6': case '7': - u = r[1] - '0'; - r += 2; - if (isdigit(r[0]) && (r[0] - '0') < 8) { - u <<= 3; - u |= r[0] - '0'; - r++; - if (isdigit(r[0]) && (r[0] - '0') < 8) { - u <<= 3; - u |= r[0] - '0'; - r++; - } - } - *q++ = u; - break; - default: - *q++ = r[1]; - r += 2; - break; - } - } - *q = '\0'; - return (p); -} - - -/*-------------------------------------------------------------------- - * Keep track of definitions and references - */ - -static struct ref * -FindRef(struct tokenlist *tl, struct token *t, enum ref_type type) -{ - struct ref *r; - - TAILQ_FOREACH(r, &tl->refs, list) { - if (r->type != type) - continue; - if (Teq(r->name, t)) - return (r); - } - r = calloc(sizeof *r, 1); - assert(r != NULL); - r->name = t; - r->type = type; - TAILQ_INSERT_TAIL(&tl->refs, r, list); - return (r); -} - -static int -FindRefStr(struct tokenlist *tl, const char *s, enum ref_type type) -{ - struct ref *r; - - TAILQ_FOREACH(r, &tl->refs, list) { - if (r->type != type) - continue; - if (IdIs(r->name, s)) - return (1); - } - return (0); -} - -static void -AddRef(struct tokenlist *tl, struct token *t, enum ref_type type) -{ - - FindRef(tl, t, type)->refcnt++; -} - -static void -AddRefStr(struct tokenlist *tl, const char *s, enum ref_type type) -{ - struct token *t; - - t = calloc(sizeof *t, 1); - t->b = s; - t->e = strchr(s, '\0'); - t->tok = METHOD; - AddRef(tl, t, type); -} - -static void -AddDef(struct tokenlist *tl, struct token *t, enum ref_type type) -{ - struct ref *r; - - r = FindRef(tl, t, type); - r->defcnt++; - r->name = t; -} - -/*-------------------------------------------------------------------- - * Recognize and convert units of time, return seconds. - */ - -static double -TimeUnit(struct tokenlist *tl) -{ - double sc = 1.0; - - assert(tl->t->tok == ID); - if (IdIs(tl->t, "ms")) - sc = 1e-3; - else if (IdIs(tl->t, "s")) - sc = 1.0; - else if (IdIs(tl->t, "m")) - sc = 60.0; - else if (IdIs(tl->t, "h")) - sc = 60.0 * 60.0; - else if (IdIs(tl->t, "d")) - sc = 60.0 * 60.0 * 24.0; - else { - sbuf_printf(tl->sb, "Unknown time unit "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, ". Legal are 's', 'm', 'h' and 'd'\n"); - ErrWhere(tl, tl->t); - return (1.0); - } - NextToken(tl); - return (sc); -} - -/*-------------------------------------------------------------------- - * Recognize and convert units of size, return bytes. - */ - -static double -SizeUnit(struct tokenlist *tl) -{ - double sc = 1.0; - - assert(tl->t->tok == ID); - if (IdIs(tl->t, "b")) - sc = 1.0; - else if (IdIs(tl->t, "kb")) - sc = 1024.0; - else if (IdIs(tl->t, "mb") || IdIs(tl->t, "Mb")) - sc = 1024.0 * 1024.0; - else if (IdIs(tl->t, "gb") || IdIs(tl->t, "Gb")) - sc = 1024.0 * 1024.0 * 1024.0; - else { - sbuf_printf(tl->sb, "Unknown size unit "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, ". Legal are 'kb', 'mb' and 'gb'\n"); - ErrWhere(tl, tl->t); - return (1.0); - } - NextToken(tl); - return (sc); -} - -/*-------------------------------------------------------------------- - * Recognize and convert units of rate as { space '/' time } - */ - -static double -RateUnit(struct tokenlist *tl) -{ - double sc = 1.0; - - assert(tl->t->tok == ID); - sc = SizeUnit(tl); - Expect(tl, '/'); - NextToken(tl); - sc /= TimeUnit(tl); - return (sc); -} - -/*-------------------------------------------------------------------- - * Recognize and convert { CNUM } to unsigned value - */ - -static unsigned -UintVal(struct tokenlist *tl) -{ - unsigned d = 0; - const char *p; - - Expect(tl, CNUM); - for (p = tl->t->b; p < tl->t->e; p++) { - d *= 10; - d += *p - '0'; - } - NextToken(tl); - return (d); -} - -/*-------------------------------------------------------------------- - * Recognize and convert { CNUM [ '.' [ CNUM ] ] } to double value - */ - -static double -DoubleVal(struct tokenlist *tl) -{ - double d = 0.0, e = 0.1; - const char *p; - - Expect(tl, CNUM); - for (p = tl->t->b; p < tl->t->e; p++) { - d *= 10; - d += *p - '0'; - } - NextToken(tl); - if (tl->t->tok != '.') - return (d); - NextToken(tl); - if (tl->t->tok != CNUM) - return (d); - for (p = tl->t->b; p < tl->t->e; p++) { - d += (*p - '0') * e; - e *= 0.1; - } - NextToken(tl); - return (d); -} - -/*--------------------------------------------------------------------*/ - -static unsigned -IpVal(struct tokenlist *tl) -{ - unsigned u, v; - struct token *t; - - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v = u << 24; - Expect(tl, '.'); - NextToken(tl); - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v |= u << 16; - Expect(tl, '.'); - NextToken(tl); - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v |= u << 8; - Expect(tl, '.'); - NextToken(tl); - t = tl->t; - u = UintVal(tl); - if (u < 256) { - v |= u; - return (v); - } - } - } - } - sbuf_printf(tl->sb, "Illegal octet in IP number\n"); - ErrWhere(tl, t); - return (0); -} - -/*--------------------------------------------------------------------*/ - -static struct var * -HeaderVar(struct tokenlist *tl __unused, struct token *t, struct var *vh) -{ - char *p; - struct var *v; - int i; - - v = calloc(sizeof *v, 1); - assert(v != NULL); - i = t->e - t->b; - p = malloc(i + 1); - assert(p != NULL); - memcpy(p, t->b, i); - p[i] = '\0'; - v->name = p; - v->fmt = STRING; - asprintf(&p, "VRT_GetHdr(sp, \"\\%03o%s:\")", - strlen(v->name + vh->len) + 1, v->name + vh->len); - assert(p != NULL); - v->rname = p; - return (v); -} - -/*--------------------------------------------------------------------*/ - -static struct var * -FindVar(struct tokenlist *tl, struct token *t, struct var *vl) -{ - struct var *v; - - for (v = vl; v->name != NULL; v++) { - if (v->fmt == HEADER && t->e - t->b <= v->len) - continue; - if (v->fmt != HEADER && t->e - t->b != v->len) - continue; - if (memcmp(t->b, v->name, v->len)) - continue; - if (v->fmt != HEADER) - return (v); - return (HeaderVar(tl, t, v)); - } - sbuf_printf(tl->sb, "Unknown variable "); - ErrToken(tl, t); - sbuf_cat(tl->sb, "\nAt: "); - ErrWhere(tl, t); - return (NULL); -} - - -/*--------------------------------------------------------------------*/ - -static void -TimeVal(struct tokenlist *tl) -{ - double v, sc; - - v = DoubleVal(tl); - ExpectErr(tl, ID); - sc = TimeUnit(tl); - Fc(tl, 0, "(%g * %g)", v, sc); -} - -static void -SizeVal(struct tokenlist *tl) -{ - double v, sc; - - v = DoubleVal(tl); - ExpectErr(tl, ID); - sc = SizeUnit(tl); - Fc(tl, 0, "(%g * %g)", v, sc); -} - -static void -RateVal(struct tokenlist *tl) -{ - double v, sc; - - v = DoubleVal(tl); - ExpectErr(tl, ID); - sc = RateUnit(tl); - Fc(tl, 0, "(%g * %g)", v, sc); -} - -/*--------------------------------------------------------------------*/ - -static void -Cond_Ip(struct var *vp, struct tokenlist *tl) -{ - unsigned u; - - switch (tl->t->tok) { - case '~': - NextToken(tl); - ExpectErr(tl, ID); - AddRef(tl, tl->t, R_ACL); - Fc(tl, 1, "ip_match(%s, acl_%T)\n", vp->rname, tl->t); - NextToken(tl); - break; - case T_EQ: - case T_NEQ: - Fc(tl, 1, "%s %T ", vp->rname, tl->t); - NextToken(tl); - u = IpVal(tl); - Fc(tl, 0, "%uU /* %u.%u.%u.%u */\n", u, - (u >> 24) & 0xff, (u >> 16) & 0xff, - (u >> 8) & 0xff, (u) & 0xff); - break; - default: - sbuf_printf(tl->sb, "Illegal condition "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, " on IP number variable\n"); - sbuf_printf(tl->sb, " only '==', '!=' and '~' are legal\n"); - ErrWhere(tl, tl->t); - break; - } -} - -static void -Cond_String(struct var *vp, struct tokenlist *tl) -{ - - switch (tl->t->tok) { - case '~': - Fc(tl, 1, "string_match(%s, ", vp->rname); - NextToken(tl); - ExpectErr(tl, CSTR); - Fc(tl, 0, "%T)\n", tl->t); - NextToken(tl); - break; - case T_EQ: - case T_NEQ: - Fc(tl, 1, "%sstrcmp(%s, ", - tl->t->tok == T_EQ ? "!" : "", vp->rname); - NextToken(tl); - ExpectErr(tl, CSTR); - Fc(tl, 0, "%T)\n", tl->t); - NextToken(tl); - break; - default: - Fc(tl, 1, "%s != (void*)0", vp->rname); - break; - } -} - -static void -Cond_Int(struct var *vp, struct tokenlist *tl) -{ - - Fc(tl, 1, "%s ", vp->rname); - switch (tl->t->tok) { - case T_EQ: - case T_NEQ: - case T_LEQ: - case T_GEQ: - case '>': - case '<': - Fc(tl, 0, "%T ", tl->t); - NextToken(tl); - switch(vp->fmt) { - case TIME: - TimeVal(tl); - break; - case INT: - ExpectErr(tl, CNUM); - Fc(tl, 0, "%T ", tl->t); - NextToken(tl); - break; - case SIZE: - SizeVal(tl); - break; - default: - sbuf_printf(tl->sb, - "No conditions available for variable '%s'\n", - vp->name); - ErrWhere(tl, tl->t); - return; - } - Fc(tl, 0, "\n"); - break; - default: - sbuf_printf(tl->sb, "Illegal condition "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, " on integer variable\n"); - sbuf_printf(tl->sb, - " only '==', '!=', '<', '>', '<=' and '>=' are legal\n"); - ErrWhere(tl, tl->t); - break; - } -} - -static void -Cond_Bool(struct var *vp, struct tokenlist *tl) -{ - - Fc(tl, 1, "%s\n", vp->rname); -} - -static void -Cond_2(struct tokenlist *tl) -{ - struct var *vp; - - C(tl, ","); - if (tl->t->tok == '!') { - Fc(tl, 1, "!(\n"); - NextToken(tl); - } else { - Fc(tl, 1, "(\n"); - } - if (tl->t->tok == '(') { - NextToken(tl); - Cond_0(tl); - ExpectErr(tl, ')'); - NextToken(tl); - } else if (tl->t->tok == VAR) { - vp = FindVar(tl, tl->t, vars); - ERRCHK(tl); - assert(vp != NULL); - NextToken(tl); - switch (vp->fmt) { - case INT: L(tl, Cond_Int(vp, tl)); break; - case SIZE: L(tl, Cond_Int(vp, tl)); break; - case BOOL: L(tl, Cond_Bool(vp, tl)); break; - case IP: L(tl, Cond_Ip(vp, tl)); break; - case STRING: L(tl, Cond_String(vp, tl)); break; - case TIME: L(tl, Cond_Int(vp, tl)); break; - /* XXX backend == */ - default: - sbuf_printf(tl->sb, - "Variable '%s'" - " has no conditions that can be checked\n", - vp->name); - ErrWhere(tl, tl->t); - return; - } - } else { - sbuf_printf(tl->sb, - "Syntax error in condition, expected '(', '!' or" - " variable name, found "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, "\n"); - ErrWhere(tl, tl->t); - return; - } - Fc(tl, 1, ")\n"); -} - -static void -Cond_1(struct tokenlist *tl) -{ - - Fc(tl, 1, "(\n"); - L(tl, Cond_2(tl)); - while (tl->t->tok == T_CAND) { - NextToken(tl); - Fc(tl, 1, ") && (\n"); - L(tl, Cond_2(tl)); - } - Fc(tl, 1, ")\n"); -} - -static void -Cond_0(struct tokenlist *tl) -{ - - Fc(tl, 1, "(\n"); - L(tl, Cond_1(tl)); - while (tl->t->tok == T_COR) { - NextToken(tl); - Fc(tl, 1, ") || (\n"); - L(tl, Cond_1(tl)); - } - Fc(tl, 1, ")\n"); -} - -static void -Conditional(struct tokenlist *tl) -{ - - ExpectErr(tl, '('); - NextToken(tl); - Fc(tl, 1, "(\n"); - L(tl, Cond_0(tl)); - ERRCHK(tl); - Fc(tl, 1, ")\n"); - ExpectErr(tl, ')'); - NextToken(tl); -} - -/*--------------------------------------------------------------------*/ - -static void -IfStmt(struct tokenlist *tl) -{ - - ExpectErr(tl, T_IF); - Fc(tl, 1, "if \n"); - NextToken(tl); - L(tl, Conditional(tl)); - ERRCHK(tl); - L(tl, Compound(tl)); - ERRCHK(tl); - while (1) { - switch (tl->t->tok) { - case T_ELSE: - NextToken(tl); - if (tl->t->tok != T_IF) { - Fc(tl, 1, "else \n"); - L(tl, Compound(tl)); - ERRCHK(tl); - return; - } - /* FALLTHROUGH */ - case T_ELSEIF: - case T_ELSIF: - Fc(tl, 1, "else if \n"); - NextToken(tl); - L(tl, Conditional(tl)); - ERRCHK(tl); - L(tl, Compound(tl)); - ERRCHK(tl); - break; - default: - C(tl, ";"); - return; - } - } -} - -/*--------------------------------------------------------------------*/ - -static void -Action(struct tokenlist *tl) -{ - unsigned a, u; - struct var *vp; - struct token *at; - - at = tl->t; - NextToken(tl); - switch (at->tok) { - case T_NO_NEW_CACHE: - Fc(tl, 1, "VCL_no_new_cache(sp);\n"); - return; - case T_NO_CACHE: - Fc(tl, 1, "VCL_no_cache(sp);\n"); - return; -#define VCL_RET_MAC(a,b,c) case T_##b: \ - Fc(tl, 1, "VRT_done(sp, VCL_RET_%s);\n", #b); \ - tl->curproc->returns |= VCL_RET_##b; \ - tl->curproc->returnt[c] = at; \ - return; -#include "vcl_returns.h" -#undef VCL_RET_MAC - case T_ERROR: - if (tl->t->tok == CNUM) - a = UintVal(tl); - else - a = 0; - Fc(tl, 1, "VRT_error(sp, %u, ", a); - if (tl->t->tok == CSTR) { - Fc(tl, 0, "%T);\n", tl->t); - NextToken(tl); - } else - Fc(tl, 0, "(const char *)0);\n"); - Fc(tl, 1, "VRT_done(sp, VCL_RET_ERROR);\n"); - return; - case T_SWITCH_CONFIG: - ExpectErr(tl, ID); - Fc(tl, 1, "VCL_switch_config(\"%T\");\n", tl->t); - NextToken(tl); - return; - case T_CALL: - ExpectErr(tl, ID); - AddCall(tl, tl->t); - AddRef(tl, tl->t, R_FUNC); - Fc(tl, 1, "if (VGC_function_%T(sp))\n", tl->t); - Fc(tl, 1, "\treturn (1);\n"); - NextToken(tl); - return; - case T_REWRITE: - ExpectErr(tl, CSTR); - Fc(tl, 1, "VCL_rewrite(%T", tl->t); - NextToken(tl); - ExpectErr(tl, CSTR); - Fc(tl, 0, ", %T);\n", tl->t); - NextToken(tl); - return; - case T_SET: - ExpectErr(tl, VAR); - vp = FindVar(tl, tl->t, vars); - ERRCHK(tl); - assert(vp != NULL); - Fc(tl, 1, "%s ", vp->rname); - NextToken(tl); - switch (vp->fmt) { - case INT: - case SIZE: - case RATE: - case TIME: - case FLOAT: - Fc(tl, 0, "%T ", tl->t); - a = tl->t->tok; - NextToken(tl); - if (a == T_MUL || a == T_DIV) - Fc(tl, 0, "%g", DoubleVal(tl)); - else if (vp->fmt == TIME) - TimeVal(tl); - else if (vp->fmt == SIZE) - SizeVal(tl); - else if (vp->fmt == RATE) - RateVal(tl); - else - Fc(tl, 0, "%g", DoubleVal(tl)); - Fc(tl, 0, ";\n"); - break; - case IP: - if (tl->t->tok == '=') { - NextToken(tl); - u = IpVal(tl); - Fc(tl, 0, "= %uU; /* %u.%u.%u.%u */\n", - u, - (u >> 24) & 0xff, - (u >> 16) & 0xff, - (u >> 8) & 0xff, - u & 0xff); - break; - } - sbuf_printf(tl->sb, "Illegal assignment operator "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, - " only '=' is legal for IP numbers\n"); - ErrWhere(tl, tl->t); - return; - case BACKEND: - if (tl->t->tok == '=') { - NextToken(tl); - AddRef(tl, tl->t, R_BACKEND); - Fc(tl, 0, "= &VGC_backend_%T;\n", tl->t); - NextToken(tl); - break; - } - sbuf_printf(tl->sb, "Illegal assignment operator "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, - " only '=' is legal for backend\n"); - ErrWhere(tl, tl->t); - return; - default: - sbuf_printf(tl->sb, - "Assignments not possible for '%s'\n", vp->name); - ErrWhere(tl, tl->t); - return; - } - return; - default: - sbuf_printf(tl->sb, "Expected action, 'if' or '}'\n"); - ErrWhere(tl, at); - return; - } -} - -/*--------------------------------------------------------------------*/ - -static void -Acl(struct tokenlist *tl) -{ - unsigned u, m; - - NextToken(tl); - - ExpectErr(tl, ID); - AddDef(tl, tl->t, R_ACL); - Fh(tl, 0, "static struct vcl_acl acl_%T[];\n", tl->t); - Fc(tl, 1, "static struct vcl_acl acl_%T[] = {\n", tl->t); - NextToken(tl); - - tl->indent += INDENT; - - ExpectErr(tl, '{'); - NextToken(tl); - - while (tl->t->tok == CNUM) { - u = IpVal(tl); - if (tl->t->tok == '/') { - NextToken(tl); - ExpectErr(tl, CNUM); - m = UintVal(tl); - } else - m = 32; - ExpectErr(tl, ';'); - NextToken(tl); - Fc(tl, 1, "{ %11uU, %3uU }, /* %u.%u.%u.%u/%u */\n", - u, m, - (u >> 24) & 0xff, (u >> 16) & 0xff, - (u >> 8) & 0xff, (u) & 0xff, m); - } - ExpectErr(tl, '}'); - Fc(tl, 1, "{ %11uU, %3uU }\n", 0, 0); - - tl->indent -= INDENT; - - Fc(tl, 1, "};\n\n"); - NextToken(tl); -} - -/*--------------------------------------------------------------------*/ - -static void -Compound(struct tokenlist *tl) -{ - - ExpectErr(tl, '{'); - Fc(tl, 1, "{\n"); - tl->indent += INDENT; - C(tl, ";"); - NextToken(tl); - while (1) { - ERRCHK(tl); - switch (tl->t->tok) { - case '{': - Compound(tl); - break; - case T_IF: - IfStmt(tl); - break; - case '}': - NextToken(tl); - tl->indent -= INDENT; - Fc(tl, 1, "}\n"); - return; - case EOI: - sbuf_printf(tl->sb, - "End of input while in compound statement\n"); - tl->err = 1; - return; - default: - Action(tl); - ERRCHK(tl); - ExpectErr(tl, ';'); - NextToken(tl); - break; - } - } -} - -/*--------------------------------------------------------------------*/ - -static const char * -CheckHostPort(const char *host, const char *port) -{ - struct addrinfo *res, hint; - int error; - - memset(&hint, 0, sizeof hint); - hint.ai_family = PF_UNSPEC; - hint.ai_socktype = SOCK_STREAM; - error = getaddrinfo(host, port, &hint, &res); - if (error) - return (gai_strerror(error)); - freeaddrinfo(res); - return (NULL); -} - -static void -Backend(struct tokenlist *tl) -{ - struct var *vp; - struct token *t_be = NULL; - struct token *t_host = NULL; - struct token *t_port = NULL; - char *host = NULL; - char *port = NULL; - const char *ep; - - NextToken(tl); - ExpectErr(tl, ID); - t_be = tl->t; - AddDef(tl, tl->t, R_BACKEND); - if (tl->nbackend == 0) - AddRef(tl, tl->t, R_BACKEND); - Fh(tl, 1, "#define VGC_backend_%T (VCL_conf.backend[%d])\n", - tl->t, tl->nbackend); - Fc(tl, 0, "static void\n"); - Fc(tl, 1, "VGC_init_backend_%T (void)\n", tl->t); - Fc(tl, 1, "{\n"); - Fc(tl, 1, "\tstruct backend *backend = VGC_backend_%T;\n", tl->t); - Fc(tl, 1, "\tconst char *p;\n"); - Fc(tl, 1, "\n"); - Fc(tl, 1, "\tVRT_set_backend_name(backend, \"%T\");\n", tl->t); - NextToken(tl); - ExpectErr(tl, '{'); - NextToken(tl); - while (1) { - if (tl->t->tok == '}') - break; - ExpectErr(tl, T_SET); - NextToken(tl); - ExpectErr(tl, VAR); - vp = FindVar(tl, tl->t, be_vars); - ERRCHK(tl); - assert(vp != NULL); - NextToken(tl); - ExpectErr(tl, '='); - NextToken(tl); - switch (vp->fmt) { - case HOSTNAME: - ExpectErr(tl, CSTR); - t_host = tl->t; - Fc(tl, 1, "\tp = %T;\n", tl->t); - Fc(tl, 1, "\t"); - Fc(tl, 0, vp->lname, "p"); - Fc(tl, 0, ";\n"); - NextToken(tl); - break; - case PORTNAME: - ExpectErr(tl, CSTR); - t_port = tl->t; - Fc(tl, 1, "\tp = %T;\n", tl->t); - Fc(tl, 1, "\t"); - Fc(tl, 0, vp->lname, "p"); - Fc(tl, 0, ";\n"); - NextToken(tl); - break; - default: - sbuf_printf(tl->sb, - "Assignments not possible for '%s'\n", vp->name); - ErrWhere(tl, tl->t); - return; - } - ExpectErr(tl, ';'); - NextToken(tl); - } - ExpectErr(tl, '}'); - if (t_host == NULL) { - sbuf_printf(tl->sb, "Backend '%T' has no hostname\n", t_be); - ErrWhere(tl, tl->t); - return; - } - host = EncString(t_host); - ep = CheckHostPort(host, "80"); - if (ep != NULL) { - sbuf_printf(tl->sb, "Backend '%T': %s\n", t_be, ep); - ErrWhere(tl, t_host); - return; - } - if (t_port != NULL) { - port = EncString(t_port); - ep = CheckHostPort(host, port); - if (ep != NULL) { - sbuf_printf(tl->sb, "Backend '%T': %s\n", t_be, ep); - ErrWhere(tl, t_port); - return; - } - } - - NextToken(tl); - Fc(tl, 1, "}\n"); - Fc(tl, 0, "\n"); - tl->nbackend++; -} - -/*--------------------------------------------------------------------*/ - -static void -Function(struct tokenlist *tl) -{ - - NextToken(tl); - ExpectErr(tl, ID); - tl->curproc = AddProc(tl, tl->t, 1); - tl->curproc->exists++; - AddDef(tl, tl->t, R_FUNC); - Fh(tl, 0, "static int VGC_function_%T (struct sess *sp);\n", tl->t); - Fc(tl, 1, "static int\n"); - Fc(tl, 1, "VGC_function_%T (struct sess *sp)\n", tl->t); - NextToken(tl); - L(tl, Compound(tl)); - Fc(tl, 0, "\n"); -} - -/*-------------------------------------------------------------------- - * Top level of parser, recognize: - * Function definitions - * Backend definitions - * End of input - */ - -static void -Parse(struct tokenlist *tl) -{ - - while (tl->t->tok != EOI) { - ERRCHK(tl); - switch (tl->t->tok) { - case T_ACL: - Acl(tl); - break; - case T_SUB: - Function(tl); - break; - case T_BACKEND: - Backend(tl); - break; - case EOI: - break; - default: - sbuf_printf(tl->sb, - "Expected 'acl', 'sub' or 'backend', found "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, " at\n"); - ErrWhere(tl, tl->t); - return; - } - } -} - -/*-------------------------------------------------------------------- - * Add a token to the token list. - */ - -static void -AddToken(struct tokenlist *tl, unsigned tok, const char *b, const char *e) -{ - struct token *t; - - t = calloc(sizeof *t, 1); - assert(t != NULL); - t->tok = tok; - t->b = b; - t->e = e; - TAILQ_INSERT_TAIL(&tl->tokens, t, list); - tl->t = t; - if (0) { - fprintf(stderr, "[%s %*.*s] ", - vcl_tnames[tok], - e - b, e - b, b); - if (tok == EOI) - fprintf(stderr, "\n"); - } -} - -/*-------------------------------------------------------------------- - * Lexical analysis and token generation - */ - -static void -Lexer(struct tokenlist *tl, const char *b, const char *e) -{ - const char *p, *q; - unsigned u; - - for (p = b; p < e; ) { - - /* Skip any whitespace */ - if (isspace(*p)) { - p++; - continue; - } - - /* Skip '#.*\n' comments */ - if (*p == '#') { - while (p < e && *p != '\n') - p++; - continue; - } - - /* Skip C-style comments */ - if (*p == '/' && p[1] == '*') { - p += 2; - for (p += 2; p < e; p++) { - if (*p == '*' && p[1] == '/') { - p += 2; - break; - } - } - continue; - } - - /* Match for the fixed tokens (see token.tcl) */ - u = vcl_fixed_token(p, &q); - if (u != 0) { - AddToken(tl, u, p, q); - p = q; - continue; - } - - /* Match strings, with \\ and \" escapes */ - if (*p == '"') { - for (q = p + 1; q < e; q++) { - if (*q == '\\' && q[1] == '\\') - q++; - else if (*q == '\\' && q[1] == '"') - q++; - else if (*q == '"') { - q++; - break; - } - } - AddToken(tl, CSTR, p, q); - p = q; - continue; - } - - /* Match Identifiers */ - if (isident1(*p)) { - for (q = p; q < e; q++) - if (!isident(*q)) - break; - if (isvar(*q)) { - for (; q < e; q++) - if (!isvar(*q)) - break; - AddToken(tl, VAR, p, q); - } else { - AddToken(tl, ID, p, q); - } - p = q; - continue; - } - - /* Match numbers { [0-9]+ } */ - if (isdigit(*p)) { - for (q = p; q < e; q++) - if (!isdigit(*q)) - break; - AddToken(tl, CNUM, p, q); - p = q; - continue; - } - AddToken(tl, EOI, p, p + 1); - sbuf_printf(tl->sb, "Syntax error at\n"); - ErrWhere(tl, tl->t); - return; - } -} - -/*-------------------------------------------------------------------- - * Consistency check - */ - -static struct proc * -AddProc(struct tokenlist *tl, struct token *t, int def) -{ - struct proc *p; - - TAILQ_FOREACH(p, &tl->procs, list) { - if (!Teq(p->name, t)) - continue; - if (def) - p->name = t; - return (p); - } - p = calloc(sizeof *p, 1); - assert(p != NULL); - p->name = t; - TAILQ_INIT(&p->calls); - TAILQ_INSERT_TAIL(&tl->procs, p, list); - return (p); -} - -static void -AddCall(struct tokenlist *tl, struct token *t) -{ - struct proccall *pc; - struct proc *p; - - p = AddProc(tl, t, 0); - TAILQ_FOREACH(pc, &tl->curproc->calls, list) { - if (pc->p == p) - return; - } - pc = calloc(sizeof *pc, 1); - assert(pc != NULL); - pc->p = p; - pc->t = t; - TAILQ_INSERT_TAIL(&tl->curproc->calls, pc, list); -} - -static int -Consist_Decend(struct tokenlist *tl, struct proc *p, unsigned returns) -{ - unsigned u; - struct proccall *pc; - - if (!p->exists) { - sbuf_printf(tl->sb, "Function %T does not exist\n", p->name); - return (1); - } - if (p->active) { - sbuf_printf(tl->sb, "Function recurses on\n"); - ErrWhere(tl, p->name); - return (1); - } - u = p->returns & ~returns; - if (u) { -#define VCL_RET_MAC(a, b, c) \ - if (u & VCL_RET_##b) { \ - sbuf_printf(tl->sb, "Illegal return for method\n"); \ - ErrWhere(tl, p->returnt[c]); \ - } -#include "vcl_returns.h" -#undef VCL_RET_MAC - sbuf_printf(tl->sb, "In function\n"); - ErrWhere(tl, p->name); - return (1); - } - p->active = 1; - TAILQ_FOREACH(pc, &p->calls, list) { - if (Consist_Decend(tl, pc->p, returns)) { - sbuf_printf(tl->sb, "\nCalled from\n"); - ErrWhere(tl, p->name); - sbuf_printf(tl->sb, "at\n"); - ErrWhere(tl, pc->t); - return (1); - } - } - p->active = 0; - p->called++; - return (0); -} - -static int -Consistency(struct tokenlist *tl) -{ - struct proc *p; - struct method *m; - - TAILQ_FOREACH(p, &tl->procs, list) { - for(m = method_tab; m->name != NULL; m++) { - if (IdIs(p->name, m->defname)) - p->called = 1; - if (IdIs(p->name, m->name)) - break; - } - if (m->name == NULL) - continue; - if (Consist_Decend(tl, p, m->returns)) { - sbuf_printf(tl->sb, - "\nwhich is a %s method\n", m->name); - return (1); - } - } - TAILQ_FOREACH(p, &tl->procs, list) { - if (p->called) - continue; - sbuf_printf(tl->sb, "Function unused\n"); - ErrWhere(tl, p->name); - return (1); - } - return (0); -} - -/*--------------------------------------------------------------------*/ - -static int -CheckRefs(struct tokenlist *tl) -{ - struct ref *r; - const char *type; - int nerr = 0; - - TAILQ_FOREACH(r, &tl->refs, list) { - if (r->defcnt != 0 && r->refcnt != 0) - continue; - nerr++; - - switch(r->type) { - case R_FUNC: - type = "function"; - break; - case R_ACL: - type = "acl"; - break; - case R_BACKEND: - type = "backend"; - break; - default: - ErrInternal(tl); - sbuf_printf(tl->sb, "Ref "); - ErrToken(tl, r->name); - sbuf_printf(tl->sb, " has unknown type %d\n", - r->type); - continue; - } - if (r->defcnt == 0 && r->name->tok == METHOD) { - sbuf_printf(tl->sb, - "No definition for method %T\n", r->name); - continue; - } - - if (r->defcnt == 0) { - sbuf_printf(tl->sb, - "Undefined %s %T, first reference:\n", - type, r->name); - ErrWhere(tl, r->name); - continue; - } - - sbuf_printf(tl->sb, "Unused %s %T, defined:\n", type, r->name); - ErrWhere(tl, r->name); - } - return (nerr); -} - -/*--------------------------------------------------------------------*/ - -static void -LocTable(struct tokenlist *tl) -{ - struct token *t; - unsigned lin, pos; - const char *p; - - Fh(tl, 0, "#define VGC_NREFS %u\n", tl->cnt + 1); - Fh(tl, 0, "static struct vrt_ref VGC_ref[VGC_NREFS];\n"); - Fc(tl, 0, "static struct vrt_ref VGC_ref[VGC_NREFS] = {\n"); - lin = 1; - pos = 0; - p = tl->b; - TAILQ_FOREACH(t, &tl->tokens, list) { - if (t->cnt == 0) - continue; - for (;p < t->b; p++) { - if (*p == '\n') { - lin++; - pos = 0; - } else if (*p == '\t') { - pos &= ~7; - pos += 8; - } else - pos++; - - } - Fc(tl, 0, " [%3u] = { %4u, %3u, 0, \"%T\" },\n", - t->cnt, lin, pos + 1, t); - } - Fc(tl, 0, "};\n"); -} - - -/*--------------------------------------------------------------------*/ - -static void -EmitInitFunc(struct tokenlist *tl) -{ - struct ref *r; - - Fc(tl, 0, "\nstatic void\nVGC_Init(void)\n{\n\n"); - Fc(tl, 0, "\tVRT_alloc_backends(&VCL_conf);\n"); - - TAILQ_FOREACH(r, &tl->refs, list) { - switch(r->type) { - case R_FUNC: - break; - case R_ACL: - break; - case R_BACKEND: - Fc(tl, 0, "\tVGC_init_backend_%T();\n", r->name); - break; - } - } - Fc(tl, 0, "}\n"); -} - -/*--------------------------------------------------------------------*/ - -static void -EmitStruct(struct tokenlist *tl) -{ - - Fc(tl, 0, "\nstruct VCL_conf VCL_conf = {\n"); - Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n"); - Fc(tl, 0, "\t.init_func = VGC_Init,\n"); - Fc(tl, 0, "\t.nbackend = %d,\n", tl->nbackend); - Fc(tl, 0, "\t.ref = VGC_ref,\n"); - Fc(tl, 0, "\t.nref = VGC_NREFS,\n"); -#define VCL_RET_MAC(l,u,b) -#define VCL_MET_MAC(l,u,b) \ - if (FindRefStr(tl, "vcl_" #l, R_FUNC)) { \ - Fc(tl, 0, "\t." #l "_func = VGC_function_vcl_" #l ",\n"); \ - AddRefStr(tl, "vcl_" #l, R_FUNC); \ - } else { \ - Fc(tl, 0, "\t." #l "_func = VGC_function_default_vcl_" #l ",\n"); \ - } \ - AddRefStr(tl, "default_vcl_" #l, R_FUNC); -#include "vcl_returns.h" -#undef VCL_MET_MAC -#undef VCL_RET_MAC - Fc(tl, 0, "};\n"); -} - -/*--------------------------------------------------------------------*/ - -char * -VCC_Compile(struct sbuf *sb, const char *b, const char *e) -{ - struct tokenlist tokens; - struct ref *r; - struct token *t; - FILE *fo; - char *of = NULL; - char buf[BUFSIZ]; - - memset(&tokens, 0, sizeof tokens); - TAILQ_INIT(&tokens.tokens); - TAILQ_INIT(&tokens.refs); - TAILQ_INIT(&tokens.procs); - tokens.sb = sb; - - tokens.fc = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); - assert(tokens.fc != NULL); - - tokens.fh = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); - assert(tokens.fh != NULL); - - Fh(&tokens, 0, "extern struct VCL_conf VCL_conf;\n"); - - tokens.b = b; - if (e == NULL) - e = strchr(b, '\0'); - assert(e != NULL); - tokens.e = e; - Lexer(&tokens, b, e); - Lexer(&tokens, vcc_default_vcl_b, vcc_default_vcl_e); - AddToken(&tokens, EOI, e, e); - if (tokens.err) - goto done; - tokens.t = TAILQ_FIRST(&tokens.tokens); - Parse(&tokens); - if (tokens.err) - goto done; - Consistency(&tokens); - if (tokens.err) - goto done; - LocTable(&tokens); - - EmitInitFunc(&tokens); - - EmitStruct(&tokens); - - if (CheckRefs(&tokens)) - goto done; - - of = strdup("/tmp/vcl.XXXXXXXX"); - assert(of != NULL); - mktemp(of); - - sprintf(buf, - "tee /tmp/_.c |" - "cc -fpic -shared -Wl,-x -o %s -x c - ", of); - - fo = popen(buf, "w"); - assert(fo != NULL); - - vcl_output_lang_h(fo); - - sbuf_finish(tokens.fh); - fputs(sbuf_data(tokens.fh), fo); - sbuf_delete(tokens.fh); - - sbuf_finish(tokens.fc); - fputs(sbuf_data(tokens.fc), fo); - sbuf_delete(tokens.fc); - - pclose(fo); -done: - - /* Free References */ - while (!TAILQ_EMPTY(&tokens.refs)) { - r = TAILQ_FIRST(&tokens.refs); - TAILQ_REMOVE(&tokens.refs, r, list); - free(r); - } - - /* Free Tokens */ - while (!TAILQ_EMPTY(&tokens.tokens)) { - t = TAILQ_FIRST(&tokens.tokens); - TAILQ_REMOVE(&tokens.tokens, t, list); - free(t); - } - return (of); -} - -/*--------------------------------------------------------------------*/ - -char * -VCC_CompileFile(struct sbuf *sb, const char *fn) -{ - char *f, *r; - int fd, i; - struct stat st; - - fd = open(fn, O_RDONLY); - if (fd < 0) { - sbuf_printf(sb, "Cannot open file '%s': %s", - fn, strerror(errno)); - return (NULL); - } - assert(0 == fstat(fd, &st)); - f = malloc(st.st_size + 1); - assert(f != NULL); - i = read(fd, f, st.st_size); - assert(i == st.st_size); - f[i] = '\0'; - r = VCC_Compile(sb, f, NULL); - free(f); - return (r); -} - -/*--------------------------------------------------------------------*/ - -static int -VCC_T_render(FILE *f, const struct printf_info *info __unused, const void *const *args) -{ - const struct token *t; - - t = *((const struct token * const*) (args[0])); - return (fprintf(f, "%*.*s", - t->e - t->b, t->e - t->b, t->b)); -} - -static int -VCC_T_arginfo(const struct printf_info *info __unused, size_t n, int *argtypes) -{ - - if (n > 0) - argtypes[0] = PA_POINTER; - return 1; -} - -/*--------------------------------------------------------------------*/ - -void -VCC_InitCompile(const char *default_vcl) -{ - struct var *v; - - vcc_default_vcl_b = default_vcl; - vcc_default_vcl_e = strchr(default_vcl, '\0'); - assert(vcc_default_vcl_e != NULL); - - register_printf_function ('T', VCC_T_render, VCC_T_arginfo); - vcl_init_tnames(); - for (v = vars; v->name != NULL; v++) - v->len = strlen(v->name); - for (v = be_vars; v->name != NULL; v++) - v->len = strlen(v->name); -} From phk at projects.linpro.no Fri Jul 28 13:41:20 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 28 Jul 2006 15:41:20 +0200 (CEST) Subject: r568 - trunk/varnish-cache/bin/varnishlog Message-ID: <20060728134120.D72DD1EC396@projects.linpro.no> Author: phk Date: 2006-07-28 15:41:20 +0200 (Fri, 28 Jul 2006) New Revision: 568 Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c Log: "HEAD" has 4 characters. Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-24 10:13:29 UTC (rev 567) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-28 13:41:20 UTC (rev 568) @@ -129,7 +129,7 @@ case SLT_Request: if (h_opt && p[1] == 3 && !memcmp(p + 4, "GET", 3)) hc[u]++; - if (h_opt && p[1] == 4 && !memcmp(p + 4, "HEAD", 3)) + if (h_opt && p[1] == 4 && !memcmp(p + 4, "HEAD", 4)) hc[u]++; v = 1; break; From des at projects.linpro.no Mon Jul 31 06:24:51 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Mon, 31 Jul 2006 08:24:51 +0200 (CEST) Subject: r569 - trunk/varnish-cache/bin/varnishd Message-ID: <20060731062451.9F86C1EC3BC@projects.linpro.no> Author: des Date: 2006-07-31 08:24:51 +0200 (Mon, 31 Jul 2006) New Revision: 569 Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c Log: Unbreak build. Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-28 13:41:20 UTC (rev 568) +++ trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-31 06:24:51 UTC (rev 569) @@ -179,7 +179,7 @@ AZ(pthread_mutex_lock(&hcl_mutex[mtx])); if (--he->refcnt >= 0) { AZ(pthread_mutex_unlock(&hcl_mutex[mtx])); - return (1) + return (1); } TAILQ_REMOVE(&hcl_head[he->hash], he, list); AZ(pthread_mutex_unlock(&hcl_mutex[mtx])); From des at projects.linpro.no Mon Jul 31 06:36:56 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Mon, 31 Jul 2006 08:36:56 +0200 (CEST) Subject: r570 - in trunk/varnish-cache: . bin bin/varnishncsa Message-ID: <20060731063656.C8A821EC3B1@projects.linpro.no> Author: des Date: 2006-07-31 08:36:56 +0200 (Mon, 31 Jul 2006) New Revision: 570 Added: trunk/varnish-cache/bin/varnishncsa/ trunk/varnish-cache/bin/varnishncsa/Makefile.am trunk/varnish-cache/bin/varnishncsa/varnishncsa.1 trunk/varnish-cache/bin/varnishncsa/varnishncsa.c Modified: trunk/varnish-cache/bin/Makefile.am trunk/varnish-cache/configure.ac Log: Clone varnishncsa off of varnishlog. Anders will hack on it to produce NCSA-style (common / combined) logs. Modified: trunk/varnish-cache/bin/Makefile.am =================================================================== --- trunk/varnish-cache/bin/Makefile.am 2006-07-31 06:24:51 UTC (rev 569) +++ trunk/varnish-cache/bin/Makefile.am 2006-07-31 06:36:56 UTC (rev 570) @@ -1,3 +1,3 @@ # $Id$ -SUBDIRS = varnishd varnishlog varnishstat varnishtester varnishtop +SUBDIRS = varnishd varnishlog varnishncsa varnishstat varnishtester varnishtop Copied: trunk/varnish-cache/bin/varnishncsa/Makefile.am (from rev 347, trunk/varnish-cache/bin/varnishlog/Makefile.am) =================================================================== --- trunk/varnish-cache/bin/varnishlog/Makefile.am 2006-07-06 09:31:45 UTC (rev 347) +++ trunk/varnish-cache/bin/varnishncsa/Makefile.am 2006-07-31 06:36:56 UTC (rev 570) @@ -0,0 +1,13 @@ +# $Id$ + +INCLUDES = -I$(top_srcdir)/include + +bin_PROGRAMS = varnishncsa + +man_MANS = varnishncsa.1 + +varnishncsa_SOURCES = varnishncsa.c + +varnishncsa_LDADD = \ + $(top_builddir)/lib/libvarnishapi/libvarnishapi.la \ + $(top_builddir)/lib/libsbuf/libsbuf.a Copied: trunk/varnish-cache/bin/varnishncsa/varnishncsa.1 (from rev 423, trunk/varnish-cache/bin/varnishlog/varnishlog.1) =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.1 2006-07-11 11:41:31 UTC (rev 423) +++ trunk/varnish-cache/bin/varnishncsa/varnishncsa.1 2006-07-31 06:36:56 UTC (rev 570) @@ -0,0 +1,126 @@ +.\" +.\" $Id$ +.\" +.Dd July 11, 2006 +.Dt VARNISHNCSA 1 +.Os +.Sh NAME +.Nm varnishncsa +.Nd Generate NCSA common / combined logs +.Sh SYNOPSIS +.Nm +.Op Fl C +.Op Fl b +.Op Fl c +.Op Fl o +.Op Fl I Ar regex +.Op Fl X Ar regex +.Op Fl i Ar tag +.Op Fl r Ar file +.Op Fl w Ar file +.Op Fl x Ar tag +.Sh DESCRIPTION +The +.Nm +utility reads and presents +.Xr varnishd 1 +shared memory logs. +.Pp +The following options are available: +.Bl -tag -width Fl +.It Fl C +Ignore case when matching regular expressions. +.It Fl b +Include log entries which result from communication with a backend +server. +If neither +.Fl b +nor +.Fl c +is specified, +.Nm +acts as if they both were. +.It Fl c +Include log entries which result from communication with a client. +If neither +.Fl b +nor +.Fl c +is specified, +.Nm +acts as if they both were. +.It Fl o +Group log entries by request ID. +.It Fl I Ar regex +Include log entries which match the specified regular expression. +If neither +.Fl I +nor +.Fl i +is specified, all log entries are included. +.It Fl X Ar regex +Exclude log entries which match the specified regular expression. +.It Fl i Ar tag +Include log entries with the specified tag. +If neither +.Fl I +nor +.Fl i +is specified, all log entries are included. +.It Fl r Ar file +Read log entries from +.Ar file +instead of shared memory. +.It Fl w Ar file +Write log entries to +.Ar file +instead of displaying them. +.It Fl x Ar tag +Exclude log entries with the specified tag. +.El +.Sh TAGS +The following log entry tags are currently defined: +.\" keep in sync with include/shmlog_tags.h +.\" XXX add descriptions +.Bl -tag -width 16 +.It Dv Backend +.It Dv BackendClose +.It Dv BackendOpen +.It Dv BackendReuse +.It Dv BackendXID +.It Dv BldHdr +.It Dv CLI +.It Dv ClientAddr +.It Dv Debug +.It Dv Error +.It Dv ExpBan +.It Dv ExpKill +.It Dv ExpPick +.It Dv Header +.It Dv Hit +.It Dv HttpError +.It Dv Length +.It Dv LostHeader +.It Dv Protocol +.It Dv Request +.It Dv Response +.It Dv SessionClose +.It Dv SessionOpen +.It Dv SessionReuse +.It Dv Status +.It Dv URL +.It Dv VCL_call +.It Dv VCL_return +.It Dv VCL_trace +.It Dv WorkThread +.It Dv XID +.El +.Sh SEE ALSO +.Xr varnishd 1 , +.Xr varnishstat 1 +.Sh HISTORY +The +.Nm +utility was developed by +.An Poul-Henning Kamp Aq phk at freebsd.dk +in cooperation with Verdens Gang AS and Linpro AS. Copied: trunk/varnish-cache/bin/varnishncsa/varnishncsa.c (from rev 568, trunk/varnish-cache/bin/varnishlog/varnishlog.c) =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-28 13:41:20 UTC (rev 568) +++ trunk/varnish-cache/bin/varnishncsa/varnishncsa.c 2006-07-31 06:36:56 UTC (rev 570) @@ -0,0 +1,295 @@ +/* + * $Id$ + * + * Log tailer for Varnish + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "shmlog.h" +#include "varnishapi.h" + + +static char * +vis_it(unsigned char *p) +{ + static char visbuf[255*4 + 3 + 1]; + + strcpy(visbuf, " ["); + strvisx(visbuf + 2, p + 4, p[1], + VIS_OCTAL | VIS_TAB | VIS_NL); + strcat(visbuf, "]"); + return (visbuf); +} + +/* Ordering-----------------------------------------------------------*/ + +static struct sbuf *ob[65536]; +static int hc[65536]; +static int xrf[65536]; + +static void +clean_order(void) +{ + unsigned u; + + for (u = 0; u < 65536; u++) { + if (ob[u] == NULL) + continue; + sbuf_finish(ob[u]); + if (sbuf_len(ob[u])) + printf("%s\n", sbuf_data(ob[u])); + sbuf_clear(ob[u]); + } +} + +static void +order(unsigned char *p, int h_opt) +{ + unsigned u, v; + + u = (p[2] << 8) | p[3]; + if (ob[u] == NULL) { + ob[u] = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); + assert(ob[u] != NULL); + } + v = 0; + switch (p[0]) { + case SLT_VCL_call: + sbuf_printf(ob[u], "%02x %3d %4d %-12s", + p[0], p[1], u, VSL_tags[p[0]]); + if (p[1] > 0) { + sbuf_cat(ob[u], " <"); + sbuf_bcat(ob[u], p + 4, p[1]); + } + if (h_opt && p[1] == 3 && !memcmp(p + 4, "hit", 3)) + hc[u]++; + break; + case SLT_VCL_trace: + if (p[1] > 0) { + sbuf_cat(ob[u], " "); + sbuf_bcat(ob[u], p + 4, p[1]); + } + break; + case SLT_VCL_return: + if (p[1] > 0) { + sbuf_cat(ob[u], " "); + sbuf_bcat(ob[u], p + 4, p[1]); + sbuf_cat(ob[u], ">\n"); + } + if (h_opt && p[1] == 7 && !memcmp(p + 4, "deliver", 7)) + hc[u]++; + if (h_opt && p[1] == 6 && !memcmp(p + 4, "insert", 6)) { + if (hc[xrf[u]] == 1) { + hc[u] += 2; + hc[xrf[u]] = 4; + } + } + break; + case SLT_Debug: + if (p[1] == 0) + break; + if (!h_opt) + ; + else if (p[1] > 4 && !memcmp(p + 4, "TTD:", 4)) + break; + sbuf_printf(ob[u], "%02x %3d %4d %-12s", + p[0], p[1], u, VSL_tags[p[0]]); + if (p[1] > 0) + sbuf_cat(ob[u], vis_it(p)); + sbuf_cat(ob[u], "\n"); + break; + case SLT_HttpError: + if (!h_opt) + v = 1; + else if (p[1] == 16 && !memcmp(p + 4, "Received nothing", 16)) + ; + else if (p[1] == 17 && !memcmp(p + 4, "Received errno 54", 17)) + ; + else + v = 1; + break; + case SLT_SessionClose: + if (!h_opt) + v = 1; + else if (p[1] == 10 && !memcmp(p + 4, "no request", 10)) + ; + else if (p[1] == 7 && !memcmp(p + 4, "timeout", 7)) + ; + else + v = 1; + break; + case SLT_Request: + if (h_opt && p[1] == 3 && !memcmp(p + 4, "GET", 3)) + hc[u]++; + if (h_opt && p[1] == 4 && !memcmp(p + 4, "HEAD", 4)) + hc[u]++; + v = 1; + break; + case SLT_Backend: + xrf[u] = atoi(p + 4); + v = 1; + break; + case SLT_Status: + if (h_opt && p[1] == 3 && !memcmp(p + 4, "200", 3)) + hc[u]++; + v = 1; + break; + default: + v = 1; + break; + } + if (v) { + sbuf_printf(ob[u], "%02x %3d %4d %-12s", + p[0], p[1], u, VSL_tags[p[0]]); + if (p[1] > 0) { + sbuf_cat(ob[u], " <"); + sbuf_bcat(ob[u], p + 4, p[1]); + sbuf_cat(ob[u], ">"); + } + sbuf_cat(ob[u], "\n"); + } + if (u == 0) { + sbuf_finish(ob[u]); + printf("%s", sbuf_data(ob[u])); + sbuf_clear(ob[u]); + return; + } + switch (p[0]) { + case SLT_SessionClose: + case SLT_SessionReuse: + case SLT_BackendClose: + case SLT_BackendReuse: + sbuf_finish(ob[u]); + if ((hc[u] != 4 || h_opt == 0) && sbuf_len(ob[u]) > 1) + printf("%s\n", sbuf_data(ob[u])); + sbuf_clear(ob[u]); + hc[u] = 0; + xrf[u] = 0; + break; + default: + break; + } +} + + + +/*--------------------------------------------------------------------*/ + +static void +Usage(void) +{ + fprintf(stderr, "Usage: varnishncsa [-o] [-w file] [-r file]\n"); + exit(2); +} + +int +main(int argc, char **argv) +{ + int i, c; + unsigned u, v; + unsigned char *p; + int o_flag = 0; + char *w_opt = NULL; + FILE *wfile = NULL; + int h_opt = 0; + struct VSL_data *vd; + + vd = VSL_New(); + + while ((c = getopt(argc, argv, VSL_ARGS "how:")) != -1) { + i = VSL_Arg(vd, c, optarg); + if (i < 0) + exit (1); + if (i > 0) + continue; + switch (c) { + case 'h': + h_opt = 1; + break; + case 'o': + o_flag = 1; + break; + case 'w': + w_opt = optarg; + break; + default: + Usage(); + } + } + + if (VSL_OpenLog(vd)) + exit (1); + + if (o_flag && w_opt != NULL) + Usage(); + + if (w_opt != NULL) { + wfile = fopen(w_opt, "w"); + if (wfile == NULL) { + perror(w_opt); + exit (1); + } + } + u = 0; + v = 0; + + while (1) { + i = VSL_NextLog(vd, &p); + if (i < 0) + break; + if (i == 0) { + if (w_opt == NULL) { + if (o_flag && ++v == 100) + clean_order(); + fflush(stdout); + } else if (++v == 100) { + fflush(wfile); + printf("\nFlushed\n"); + } + usleep(50000); + continue; + } + v = 0; + if (wfile != NULL) { + i = fwrite(p, 4 + p[1], 1, wfile); + if (i != 1) + perror(w_opt); + u++; + if (!(u % 1000)) { + printf("%u\r", u); + fflush(stdout); + } + continue; + } + if (o_flag) { + order(p, h_opt); + continue; + } + u = (p[2] << 8) | p[3]; + printf("%02x %3d %4d %-12s", + p[0], p[1], u, VSL_tags[p[0]]); + + if (p[1] > 0) { + if (p[0] != SLT_Debug) { + printf(" <"); + fwrite(p + 4, p[1], 1, stdout); + printf(">"); + } else { + fputs(vis_it(p), stdout); + } + + } + printf("\n"); + } + if (o_flag) + clean_order(); + return (0); +} Modified: trunk/varnish-cache/configure.ac =================================================================== --- trunk/varnish-cache/configure.ac 2006-07-31 06:24:51 UTC (rev 569) +++ trunk/varnish-cache/configure.ac 2006-07-31 06:36:56 UTC (rev 570) @@ -69,6 +69,7 @@ bin/Makefile bin/varnishd/Makefile bin/varnishlog/Makefile + bin/varnishncsa/Makefile bin/varnishstat/Makefile bin/varnishtester/Makefile bin/varnishtop/Makefile From phk at projects.linpro.no Mon Jul 31 07:13:45 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 31 Jul 2006 09:13:45 +0200 (CEST) Subject: r571 - trunk/varnish-cache/bin/varnishd Message-ID: <20060731071345.E8B591EC3A5@projects.linpro.no> Author: phk Date: 2006-07-31 09:13:45 +0200 (Mon, 31 Jul 2006) New Revision: 571 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Don't fill more than half the workspace with received data, we need to have space for composing the reply as well. Without this fix, the entire workspace could be filled with pipelined requests and we would have no space to compose the reply. Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-31 06:36:56 UTC (rev 570) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-31 07:13:45 UTC (rev 571) @@ -409,7 +409,11 @@ (void)event; - l = hp->e - hp->v; + l = (hp->e - hp->s) / 2; + if (l < hp->v - hp->s) + l = 0; + else + l -= hp->v - hp->s; if (l <= 1) { VSL(SLT_HttpError, fd, "Received too much"); VSLR(SLT_HttpGarbage, fd, hp->s, hp->v); From phk at projects.linpro.no Mon Jul 31 07:26:25 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 31 Jul 2006 09:26:25 +0200 (CEST) Subject: r572 - trunk/varnish-cache/bin/varnishd Message-ID: <20060731072625.6BB321EC39A@projects.linpro.no> Author: phk Date: 2006-07-31 09:26:25 +0200 (Mon, 31 Jul 2006) New Revision: 572 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_response.c Log: Introduce http_SetHeader() for setting a http header to a const string, no need to waste time printf'ing in this case, and no need to waste workspace. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-31 07:13:45 UTC (rev 571) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-31 07:26:25 UTC (rev 572) @@ -328,6 +328,7 @@ void http_CopyResp(int fd, struct http *to, struct http *fm); void http_FilterHeader(int fd, struct http *to, struct http *fm, unsigned how); void http_PrintfHeader(int fd, struct http *to, const char *fmt, ...); +void http_SetHeader(int fd, struct http *to, unsigned n, const char *hdr); int http_IsHdr(struct http_hdr *hh, char *hdr); void http_Setup(struct http *ht, void *space, unsigned len); int http_GetHdr(struct http *hp, const char *hdr, char **ptr); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-31 07:13:45 UTC (rev 571) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-31 07:26:25 UTC (rev 572) @@ -484,7 +484,9 @@ AZ(event_add(&hp->ev, NULL)); /* XXX: timeout */ } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * Copy HTTP headers into malloc'ed space. + */ void http_CopyHttp(struct http *to, struct http *fm) @@ -615,6 +617,19 @@ /*--------------------------------------------------------------------*/ void +http_SetHeader(int fd, struct http *to, unsigned n, const char *hdr) +{ + + if (n == 0) + n = to->nhd++; + to->hd[n].b = (void*)(uintptr_t)hdr; + to->hd[n].e = strchr(hdr, '\0'); + VSLH(SLT_TxHeader, fd, to, n); +} + +/*--------------------------------------------------------------------*/ + +void http_PrintfHeader(int fd, struct http *to, const char *fmt, ...) { va_list ap; Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-31 07:13:45 UTC (rev 571) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-31 07:26:25 UTC (rev 572) @@ -84,21 +84,16 @@ sp->http->f = sp->http->v; - sp->http->nhd = HTTP_HDR_PROTO; - http_PrintfHeader(sp->fd, sp->http, "HTTP/1.1"); + http_SetHeader(sp->fd, sp->http, HTTP_HDR_PROTO, "HTTP/1.1"); + http_SetHeader(sp->fd, sp->http, HTTP_HDR_STATUS, "304"); + http_SetHeader(sp->fd, sp->http, HTTP_HDR_RESPONSE, "Not Modified"); - sp->http->nhd = HTTP_HDR_STATUS; - http_PrintfHeader(sp->fd, sp->http, "304"); - - sp->http->nhd = HTTP_HDR_RESPONSE; - http_PrintfHeader(sp->fd, sp->http, "Not Modified"); - sp->http->nhd = HTTP_HDR_FIRST; + http_SetHeader(sp->fd, sp->http, 0, "Via: 1.1 varnish"); http_PrintfHeader(sp->fd, sp->http, "X-Varnish: %u", sp->xid); - http_PrintfHeader(sp->fd, sp->http, "Via: 1.1 varnish"); http_PrintfHeader(sp->fd, sp->http, "Last-Modified: %s", p); if (sp->doclose != NULL) - http_PrintfHeader(sp->fd, sp->http, "Connection: close"); + http_SetHeader(sp->fd, sp->http, 0, "Connection: close"); WRK_Reset(sp->wrk, &sp->fd); http_Write(sp->wrk, sp->http, 1); if (WRK_Flush(sp->wrk)) @@ -155,9 +150,9 @@ http_PrintfHeader(sp->fd, sp->http, "X-Varnish: %u", sp->xid); http_PrintfHeader(sp->fd, sp->http, "Age: %u", sp->obj->age + sp->t_req - sp->obj->entered); - http_PrintfHeader(sp->fd, sp->http, "Via: 1.1 varnish"); + http_SetHeader(sp->fd, sp->http, 0, "Via: 1.1 varnish"); if (sp->doclose != NULL) - http_PrintfHeader(sp->fd, sp->http, "Connection: close"); + http_SetHeader(sp->fd, sp->http, 0, "Connection: close"); WRK_Reset(sp->wrk, &sp->fd); sp->wrk->acct.hdrbytes += http_Write(sp->wrk, sp->http, 1); From andersb at projects.linpro.no Mon Jul 31 14:50:37 2006 From: andersb at projects.linpro.no (andersb at projects.linpro.no) Date: Mon, 31 Jul 2006 16:50:37 +0200 (CEST) Subject: r573 - trunk/varnish-cache/bin/varnishncsa Message-ID: <20060731145037.19F7E1EC3AD@projects.linpro.no> Author: andersb Date: 2006-07-31 16:50:36 +0200 (Mon, 31 Jul 2006) New Revision: 573 Modified: trunk/varnish-cache/bin/varnishncsa/varnishncsa.c Log: Initial commit of real structure. This code will print the User-Agent. Modified: trunk/varnish-cache/bin/varnishncsa/varnishncsa.c =================================================================== --- trunk/varnish-cache/bin/varnishncsa/varnishncsa.c 2006-07-31 07:26:25 UTC (rev 572) +++ trunk/varnish-cache/bin/varnishncsa/varnishncsa.c 2006-07-31 14:50:36 UTC (rev 573) @@ -1,7 +1,15 @@ /* - * $Id$ + * $Id:$ * - * Log tailer for Varnish + * Program that will get data from the shared memory log. When it has the data + * it will order the data based on the sessionid. When the data is ordered + * and session is finished it will write the data into disk. Logging will be + * in NCSA extended/combined access log format. + * + * "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" + * + * TODO: - Log in any format one wants + * - Maybe rotate/compress log */ #include @@ -17,24 +25,24 @@ #include "varnishapi.h" -static char * -vis_it(unsigned char *p) -{ - static char visbuf[255*4 + 3 + 1]; +/* Ordering-----------------------------------------------------------*/ - strcpy(visbuf, " ["); - strvisx(visbuf + 2, p + 4, p[1], - VIS_OCTAL | VIS_TAB | VIS_NL); - strcat(visbuf, "]"); - return (visbuf); -} -/* Ordering-----------------------------------------------------------*/ +/* We make a array of pointers to sbuf's. Sbuf is a string buffer. +* The buffer can be made/extended/cleared etc. through a API. +* The array is 65536 long because we will use sessionid as key. +* +*/ static struct sbuf *ob[65536]; -static int hc[65536]; -static int xrf[65536]; + +/* +* Clean order is called once in a while. It clears all the sessions that +* where never finished (SLT_SessionClose). Because the data is not complete +* we disregard the data. +*/ + static void clean_order(void) { @@ -44,17 +52,35 @@ if (ob[u] == NULL) continue; sbuf_finish(ob[u]); - if (sbuf_len(ob[u])) - printf("%s\n", sbuf_data(ob[u])); + + /* XXX delete this code? Probably, since we write data to disk/screen + * as soon as we have all the data we need anyway. If we are here + * we don't have all the data, hence we don't bother to write out. + * + * + * if (sbuf_len(ob[u])) + * printf("%s\n", sbuf_data(ob[u])); + */ + sbuf_clear(ob[u]); } } static void -order(unsigned char *p, int h_opt) +extended_log_format(unsigned char *p, char *w_opt) { unsigned u, v; + int i,j; + char *ans; + char soek[1]; + strcpy(soek," "); + if (w_opt != NULL){ + // printf(" Has w_opt\n"); + } else { + // printf(" Does not have w_opt\n"); + } + u = (p[2] << 8) | p[3]; if (ob[u] == NULL) { ob[u] = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); @@ -62,91 +88,36 @@ } v = 0; switch (p[0]) { - case SLT_VCL_call: - sbuf_printf(ob[u], "%02x %3d %4d %-12s", - p[0], p[1], u, VSL_tags[p[0]]); - if (p[1] > 0) { - sbuf_cat(ob[u], " <"); - sbuf_bcat(ob[u], p + 4, p[1]); - } - if (h_opt && p[1] == 3 && !memcmp(p + 4, "hit", 3)) - hc[u]++; + + case SLT_SessionOpen: + + //ans = strchr(&p[4], (int)soek); + //j = strlen(ans); + //printf("%d\n",j); break; - case SLT_VCL_trace: - if (p[1] > 0) { - sbuf_cat(ob[u], " "); + + case SLT_RxHeader: + + if (p[1] >= 11 && !strncasecmp((void *)&p[4], "user-agent:",11)){ + //printf(" User-Agent: %s\n", p[4]); + //sbuf_printf(ob[u], "%s\n", &p[4]); sbuf_bcat(ob[u], p + 4, p[1]); + sbuf_cat(ob[u], "\n"); + sbuf_finish(ob[u]); + printf("%s", sbuf_data(ob[u])); + sbuf_clear(ob[u]); } break; - case SLT_VCL_return: - if (p[1] > 0) { - sbuf_cat(ob[u], " "); - sbuf_bcat(ob[u], p + 4, p[1]); - sbuf_cat(ob[u], ">\n"); - } - if (h_opt && p[1] == 7 && !memcmp(p + 4, "deliver", 7)) - hc[u]++; - if (h_opt && p[1] == 6 && !memcmp(p + 4, "insert", 6)) { - if (hc[xrf[u]] == 1) { - hc[u] += 2; - hc[xrf[u]] = 4; - } - } - break; - case SLT_Debug: - if (p[1] == 0) - break; - if (!h_opt) - ; - else if (p[1] > 4 && !memcmp(p + 4, "TTD:", 4)) - break; - sbuf_printf(ob[u], "%02x %3d %4d %-12s", - p[0], p[1], u, VSL_tags[p[0]]); - if (p[1] > 0) - sbuf_cat(ob[u], vis_it(p)); - sbuf_cat(ob[u], "\n"); - break; - case SLT_HttpError: - if (!h_opt) - v = 1; - else if (p[1] == 16 && !memcmp(p + 4, "Received nothing", 16)) - ; - else if (p[1] == 17 && !memcmp(p + 4, "Received errno 54", 17)) - ; - else - v = 1; - break; + case SLT_SessionClose: - if (!h_opt) - v = 1; - else if (p[1] == 10 && !memcmp(p + 4, "no request", 10)) - ; - else if (p[1] == 7 && !memcmp(p + 4, "timeout", 7)) - ; - else - v = 1; break; - case SLT_Request: - if (h_opt && p[1] == 3 && !memcmp(p + 4, "GET", 3)) - hc[u]++; - if (h_opt && p[1] == 4 && !memcmp(p + 4, "HEAD", 4)) - hc[u]++; - v = 1; - break; - case SLT_Backend: - xrf[u] = atoi(p + 4); - v = 1; - break; - case SLT_Status: - if (h_opt && p[1] == 3 && !memcmp(p + 4, "200", 3)) - hc[u]++; - v = 1; - break; default: v = 1; break; } if (v) { + + /* XXX Need to write some code to make the logline sbuf_printf(ob[u], "%02x %3d %4d %-12s", p[0], p[1], u, VSL_tags[p[0]]); if (p[1] > 0) { @@ -155,38 +126,27 @@ sbuf_cat(ob[u], ">"); } sbuf_cat(ob[u], "\n"); + */ } + + /* XXX Do I need this? When is u == 0? I can't seem to see + * it used before this place. if (u == 0) { sbuf_finish(ob[u]); printf("%s", sbuf_data(ob[u])); sbuf_clear(ob[u]); return; } - switch (p[0]) { - case SLT_SessionClose: - case SLT_SessionReuse: - case SLT_BackendClose: - case SLT_BackendReuse: - sbuf_finish(ob[u]); - if ((hc[u] != 4 || h_opt == 0) && sbuf_len(ob[u]) > 1) - printf("%s\n", sbuf_data(ob[u])); - sbuf_clear(ob[u]); - hc[u] = 0; - xrf[u] = 0; - break; - default: - break; - } + */ + } - - /*--------------------------------------------------------------------*/ static void Usage(void) { - fprintf(stderr, "Usage: varnishncsa [-o] [-w file] [-r file]\n"); + fprintf(stderr, "Usage: varnishlogfile [-w file] [-r file]\n"); exit(2); } @@ -196,27 +156,22 @@ int i, c; unsigned u, v; unsigned char *p; - int o_flag = 0; + //int o_flag = 0; + //int l_flag = 0; char *w_opt = NULL; FILE *wfile = NULL; - int h_opt = 0; + // int h_opt = 0; struct VSL_data *vd; vd = VSL_New(); - while ((c = getopt(argc, argv, VSL_ARGS "how:")) != -1) { + while ((c = getopt(argc, argv, VSL_ARGS "w:")) != -1) { i = VSL_Arg(vd, c, optarg); if (i < 0) exit (1); if (i > 0) continue; switch (c) { - case 'h': - h_opt = 1; - break; - case 'o': - o_flag = 1; - break; case 'w': w_opt = optarg; break; @@ -228,9 +183,6 @@ if (VSL_OpenLog(vd)) exit (1); - if (o_flag && w_opt != NULL) - Usage(); - if (w_opt != NULL) { wfile = fopen(w_opt, "w"); if (wfile == NULL) { @@ -247,17 +199,24 @@ break; if (i == 0) { if (w_opt == NULL) { - if (o_flag && ++v == 100) + if (++v == 100) clean_order(); fflush(stdout); } else if (++v == 100) { - fflush(wfile); - printf("\nFlushed\n"); - } + + /* Not sure if needed. + * + *fflush(wfile); + */ + + printf("\n Inside the ++v==100 and w_opt is set.\n"); + } usleep(50000); continue; } v = 0; + + /* XXX probably wanna throw this out. Not sure when needed. if (wfile != NULL) { i = fwrite(p, 4 + p[1], 1, wfile); if (i != 1) @@ -269,27 +228,14 @@ } continue; } - if (o_flag) { - order(p, h_opt); - continue; - } - u = (p[2] << 8) | p[3]; - printf("%02x %3d %4d %-12s", - p[0], p[1], u, VSL_tags[p[0]]); + */ - if (p[1] > 0) { - if (p[0] != SLT_Debug) { - printf(" <"); - fwrite(p + 4, p[1], 1, stdout); - printf(">"); - } else { - fputs(vis_it(p), stdout); - } - - } - printf("\n"); + extended_log_format(p, w_opt); + + + //printf("\n"); } - if (o_flag) - clean_order(); + + clean_order(); return (0); } From phk at projects.linpro.no Mon Jul 31 19:18:26 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 31 Jul 2006 21:18:26 +0200 (CEST) Subject: r574 - trunk/varnish-cache/bin/varnishd Message-ID: <20060731191826.EA12C1EC399@projects.linpro.no> Author: phk Date: 2006-07-31 21:18:26 +0200 (Mon, 31 Jul 2006) New Revision: 574 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: Fix the dot-graph Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-31 14:50:36 UTC (rev 573) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-31 19:18:26 UTC (rev 574) @@ -291,7 +291,7 @@ DOT lookup -> lookup3 [style=bold] DOT } DOT lookup -> HIT [label="hit", style=bold] -DOT lookup2 -> MISS [label="miss", style=bold] +DOT lookup3 -> MISS [label="miss", style=bold] */ static int From phk at projects.linpro.no Mon Jul 31 20:27:02 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 31 Jul 2006 22:27:02 +0200 (CEST) Subject: r575 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060731202702.87D3B1EC3B7@projects.linpro.no> Author: phk Date: 2006-07-31 22:27:02 +0200 (Mon, 31 Jul 2006) New Revision: 575 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/include/shmlog_tags.h Log: Log the headers we store in the object under ObjHeader so that we don't get two confusing batches of TxHeader in the sessions logentries. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-31 19:18:26 UTC (rev 574) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-31 20:27:02 UTC (rev 575) @@ -66,7 +66,8 @@ char *f; /* first (F)ree byte */ char *e; /* (E)nd of buffer */ - unsigned conds; /* If-* headers present */ + unsigned char conds; /* If-* headers present */ + unsigned char objlog; struct http_hdr hd[MAX_HTTP_HDRS]; unsigned nhd; Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-31 19:18:26 UTC (rev 574) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-31 20:27:02 UTC (rev 575) @@ -222,8 +222,15 @@ if (http_GetHdr(vc->http, H_Last_Modified, &b)) sp->obj->last_modified = TIM_parse(b); + /* + * We borrow the sessions workspace and http header for building the + * headers to store in the object, then copy them over there. + * The actual headers to reply with are built later on over in + * cache_response.c + */ sp->http->f = sp->http->v; sp->http->nhd = HTTP_HDR_FIRST; + sp->http->objlog = 1; /* log as SLT_ObjHeader */ http_CopyResp(sp->fd, sp->http, vc->http); http_FilterHeader(sp->fd, sp->http, vc->http, HTTPH_A_INS); @@ -234,10 +241,13 @@ cls = fetch_chunked(sp, vc->fd, vc->http); else cls = fetch_eof(sp, vc->fd, vc->http); - http_PrintfHeader(sp->fd, sp->http, "Content-Length: %u", sp->obj->len); + http_PrintfHeader(sp->fd, sp->http, + "Content-Length: %u", sp->obj->len); } else cls = 0; + sp->http->objlog = 0; http_CopyHttp(&sp->obj->http, sp->http); + sp->http->f = sp->http->v; if (http_GetHdr(vc->http, H_Connection, &b) && !strcasecmp(b, "close")) cls = 1; Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-31 19:18:26 UTC (rev 574) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-31 20:27:02 UTC (rev 575) @@ -22,6 +22,9 @@ #define VSLH(ax, bx, cx, dx) \ VSLR((ax), (bx), (cx)->hd[(dx)].b, (cx)->hd[(dx)].e); +#define VSLHT(bx, cx, dx) \ + VSLH((cx)->objlog ? SLT_ObjHeader : SLT_TxHeader, bx, cx, dx) + /*--------------------------------------------------------------------*/ void @@ -588,7 +591,7 @@ if (to->nhd < MAX_HTTP_HDRS) { to->hd[to->nhd].b = fm->hd[n].b; to->hd[to->nhd].e = fm->hd[n].e; - VSLH(SLT_TxHeader, fd, to, to->nhd); + VSLHT(fd, to, to->nhd); to->nhd++; } else { VSL_stats->losthdr++; @@ -624,7 +627,7 @@ n = to->nhd++; to->hd[n].b = (void*)(uintptr_t)hdr; to->hd[n].e = strchr(hdr, '\0'); - VSLH(SLT_TxHeader, fd, to, n); + VSLHT(fd, to, n); } /*--------------------------------------------------------------------*/ @@ -646,7 +649,7 @@ to->hd[to->nhd].b = to->f; to->hd[to->nhd].e = to->f + n; to->f += n + 1; - VSLH(SLT_TxHeader, fd, to, to->nhd); + VSLHT(fd, to, to->nhd); to->nhd++; } va_end(ap); Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-07-31 19:18:26 UTC (rev 574) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-07-31 20:27:02 UTC (rev 575) @@ -32,6 +32,7 @@ SLTM(Protocol) SLTM(RxHeader) SLTM(TxHeader) +SLTM(ObjHeader) SLTM(LostHeader) SLTM(TTL) SLTM(VCL_acl) From phk at projects.linpro.no Mon Jul 31 20:38:46 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 31 Jul 2006 22:38:46 +0200 (CEST) Subject: r576 - trunk/varnish-cache/bin/varnishd Message-ID: <20060731203846.5984E1EC3B7@projects.linpro.no> Author: phk Date: 2006-07-31 22:38:46 +0200 (Mon, 31 Jul 2006) New Revision: 576 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_response.c Log: Add a http_SetResp() function for constructing HTTP responses (like 304). Eliminate the header index from http_SetHeader() which is no unused. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-31 20:27:02 UTC (rev 575) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-31 20:38:46 UTC (rev 576) @@ -327,9 +327,10 @@ void http_GetReq(int fd, struct http *to, struct http *fm); void http_CopyReq(int fd, struct http *to, struct http *fm); void http_CopyResp(int fd, struct http *to, struct http *fm); +void http_SetResp(int fd, struct http *to, const char *proto, const char *status, const char *response); void http_FilterHeader(int fd, struct http *to, struct http *fm, unsigned how); void http_PrintfHeader(int fd, struct http *to, const char *fmt, ...); -void http_SetHeader(int fd, struct http *to, unsigned n, const char *hdr); +void http_SetHeader(int fd, struct http *to, const char *hdr); int http_IsHdr(struct http_hdr *hh, char *hdr); void http_Setup(struct http *ht, void *space, unsigned len); int http_GetHdr(struct http *hp, const char *hdr, char **ptr); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-31 20:27:02 UTC (rev 575) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-31 20:38:46 UTC (rev 576) @@ -580,6 +580,15 @@ http_copyh(fd, to, fm, HTTP_HDR_RESPONSE, SLT_Response); } +void +http_SetResp(int fd, struct http *to, const char *proto, const char *status, const char *response) +{ + CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); + http_seth(fd, to, HTTP_HDR_PROTO, SLT_Protocol, proto); + http_seth(fd, to, HTTP_HDR_STATUS, SLT_Status, status); + http_seth(fd, to, HTTP_HDR_RESPONSE, SLT_Response, response); +} + static void http_copyheader(int fd, struct http *to, struct http *fm, unsigned n) { @@ -620,14 +629,14 @@ /*--------------------------------------------------------------------*/ void -http_SetHeader(int fd, struct http *to, unsigned n, const char *hdr) +http_SetHeader(int fd, struct http *to, const char *hdr) { - if (n == 0) - n = to->nhd++; - to->hd[n].b = (void*)(uintptr_t)hdr; - to->hd[n].e = strchr(hdr, '\0'); - VSLHT(fd, to, n); + to->hd[to->nhd].b = (void*)(uintptr_t)hdr; + to->hd[to->nhd].e = strchr(hdr, '\0'); + assert(to->hd[to->nhd].e != NULL); + VSLHT(fd, to, to->nhd); + to->nhd++; } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-31 20:27:02 UTC (rev 575) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-31 20:38:46 UTC (rev 576) @@ -84,16 +84,14 @@ sp->http->f = sp->http->v; - http_SetHeader(sp->fd, sp->http, HTTP_HDR_PROTO, "HTTP/1.1"); - http_SetHeader(sp->fd, sp->http, HTTP_HDR_STATUS, "304"); - http_SetHeader(sp->fd, sp->http, HTTP_HDR_RESPONSE, "Not Modified"); + http_SetResp(sp->fd, sp->http, "HTTP/1.1", "304", "Not Modified"); sp->http->nhd = HTTP_HDR_FIRST; - http_SetHeader(sp->fd, sp->http, 0, "Via: 1.1 varnish"); + http_SetHeader(sp->fd, sp->http, "Via: 1.1 varnish"); http_PrintfHeader(sp->fd, sp->http, "X-Varnish: %u", sp->xid); http_PrintfHeader(sp->fd, sp->http, "Last-Modified: %s", p); if (sp->doclose != NULL) - http_SetHeader(sp->fd, sp->http, 0, "Connection: close"); + http_SetHeader(sp->fd, sp->http, "Connection: close"); WRK_Reset(sp->wrk, &sp->fd); http_Write(sp->wrk, sp->http, 1); if (WRK_Flush(sp->wrk)) @@ -150,9 +148,9 @@ http_PrintfHeader(sp->fd, sp->http, "X-Varnish: %u", sp->xid); http_PrintfHeader(sp->fd, sp->http, "Age: %u", sp->obj->age + sp->t_req - sp->obj->entered); - http_SetHeader(sp->fd, sp->http, 0, "Via: 1.1 varnish"); + http_SetHeader(sp->fd, sp->http, "Via: 1.1 varnish"); if (sp->doclose != NULL) - http_SetHeader(sp->fd, sp->http, 0, "Connection: close"); + http_SetHeader(sp->fd, sp->http, "Connection: close"); WRK_Reset(sp->wrk, &sp->fd); sp->wrk->acct.hdrbytes += http_Write(sp->wrk, sp->http, 1); From phk at projects.linpro.no Mon Jul 31 21:04:43 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 31 Jul 2006 23:04:43 +0200 (CEST) Subject: r577 - trunk/varnish-cache/bin/varnishd Message-ID: <20060731210443.EEB211EC3B7@projects.linpro.no> Author: phk Date: 2006-07-31 23:04:43 +0200 (Mon, 31 Jul 2006) New Revision: 577 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_response.c Log: Add http_ClrHeader() and cure an unintended bug-oid its use exposes: we checked if the request is a GET long after we should have. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-31 20:38:46 UTC (rev 576) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-31 21:04:43 UTC (rev 577) @@ -244,6 +244,7 @@ enum step step; unsigned handling; + unsigned char wantbody; TAILQ_ENTRY(sess) list; @@ -322,6 +323,7 @@ /* cache_http.c */ void HTTP_Init(void); +void http_ClrHeader(struct http *to); void http_CopyHttp(struct http *to, struct http *fm); unsigned http_Write(struct worker *w, struct http *hp, int resp); void http_GetReq(int fd, struct http *to, struct http *fm); Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-31 20:38:46 UTC (rev 576) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2006-07-31 21:04:43 UTC (rev 577) @@ -543,6 +543,7 @@ switch(sp->handling) { case VCL_RET_LOOKUP: /* XXX: discard req body, if any */ + sp->wantbody = !strcmp(sp->http->hd[HTTP_HDR_REQ].b, "GET"); sp->step = STP_LOOKUP; return (0); case VCL_RET_PIPE: Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-31 20:38:46 UTC (rev 576) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-31 21:04:43 UTC (rev 577) @@ -228,8 +228,7 @@ * The actual headers to reply with are built later on over in * cache_response.c */ - sp->http->f = sp->http->v; - sp->http->nhd = HTTP_HDR_FIRST; + http_ClrHeader(sp->http); sp->http->objlog = 1; /* log as SLT_ObjHeader */ http_CopyResp(sp->fd, sp->http, vc->http); http_FilterHeader(sp->fd, sp->http, vc->http, HTTPH_A_INS); @@ -247,7 +246,6 @@ cls = 0; sp->http->objlog = 0; http_CopyHttp(&sp->obj->http, sp->http); - sp->http->f = sp->http->v; if (http_GetHdr(vc->http, H_Connection, &b) && !strcasecmp(b, "close")) cls = 1; Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-31 20:38:46 UTC (rev 576) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-31 21:04:43 UTC (rev 577) @@ -629,9 +629,22 @@ /*--------------------------------------------------------------------*/ void +http_ClrHeader(struct http *to) +{ + + CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); + to->f = to->v; + to->nhd = HTTP_HDR_FIRST; + memset(to->hd, 0, sizeof to->hd); +} + +/*--------------------------------------------------------------------*/ + +void http_SetHeader(int fd, struct http *to, const char *hdr) { + CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); to->hd[to->nhd].b = (void*)(uintptr_t)hdr; to->hd[to->nhd].e = strchr(hdr, '\0'); assert(to->hd[to->nhd].e != NULL); @@ -647,6 +660,7 @@ va_list ap; unsigned l, n; + CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); va_start(ap, fmt); l = to->e - to->f; n = vsnprintf(to->f, l, fmt, ap); Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-31 20:38:46 UTC (rev 576) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-07-31 21:04:43 UTC (rev 577) @@ -148,8 +148,7 @@ vc = sp->vbc; assert(vc != NULL); - sp->http->f = sp->http->v; - sp->http->nhd = HTTP_HDR_FIRST; + http_ClrHeader(sp->http); http_CopyResp(sp->fd, sp->http, vc->http); http_FilterHeader(sp->fd, sp->http, vc->http, HTTPH_A_PASS); http_PrintfHeader(sp->fd, sp->http, "X-Varnish: %u", sp->xid); Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-31 20:38:46 UTC (rev 576) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-31 21:04:43 UTC (rev 577) @@ -79,14 +79,10 @@ res_do_304(struct sess *sp, char *p) { - VSL(SLT_Status, sp->fd, "%u", 304); VSL(SLT_Length, sp->fd, "%u", 0); - sp->http->f = sp->http->v; - + http_ClrHeader(sp->http); http_SetResp(sp->fd, sp->http, "HTTP/1.1", "304", "Not Modified"); - - sp->http->nhd = HTTP_HDR_FIRST; http_SetHeader(sp->fd, sp->http, "Via: 1.1 varnish"); http_PrintfHeader(sp->fd, sp->http, "X-Varnish: %u", sp->xid); http_PrintfHeader(sp->fd, sp->http, "Last-Modified: %s", p); @@ -135,15 +131,14 @@ if (sp->obj->response == 200 && sp->http->conds && res_do_conds(sp)) return; - VSL(SLT_Status, sp->fd, "%u", sp->obj->response); VSL(SLT_Length, sp->fd, "%u", sp->obj->len); - sp->http->f = sp->http->v; - sp->http->nhd = HTTP_HDR_FIRST; + http_ClrHeader(sp->http); http_CopyResp(sp->fd, sp->http, &sp->obj->http); http_FilterHeader(sp->fd, sp->http, &sp->obj->http, HTTPH_A_DELIVER); if (sp->xid != sp->obj->xid) - http_PrintfHeader(sp->fd, sp->http, "X-Varnish: %u %u", sp->xid, sp->obj->xid); + http_PrintfHeader(sp->fd, sp->http, + "X-Varnish: %u %u", sp->xid, sp->obj->xid); else http_PrintfHeader(sp->fd, sp->http, "X-Varnish: %u", sp->xid); http_PrintfHeader(sp->fd, sp->http, "Age: %u", @@ -155,7 +150,7 @@ sp->wrk->acct.hdrbytes += http_Write(sp->wrk, sp->http, 1); /* XXX: conditional request handling */ - if (!strcmp(sp->http->hd[HTTP_HDR_REQ].b, "GET")) { + if (sp->wantbody) { TAILQ_FOREACH(st, &sp->obj->store, list) { assert(st->stevedore != NULL); u += st->len; From phk at projects.linpro.no Mon Jul 31 21:37:10 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 31 Jul 2006 23:37:10 +0200 (CEST) Subject: r578 - trunk/varnish-cache/bin/varnishd Message-ID: <20060731213710.1DED11EC3B7@projects.linpro.no> Author: phk Date: 2006-07-31 23:37:10 +0200 (Mon, 31 Jul 2006) New Revision: 578 Modified: trunk/varnish-cache/bin/varnishd/cache_session.c Log: addr might be NULL if we are called from the prefetcher. Modified: trunk/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-31 21:04:43 UTC (rev 577) +++ trunk/varnish-cache/bin/varnishd/cache_session.c 2006-07-31 21:37:10 UTC (rev 578) @@ -199,8 +199,10 @@ sm->sess.sockaddr = sm->sockaddr; assert(len < sizeof(sm->sockaddr)); - memcpy(sm->sess.sockaddr, addr, len); - sm->sess.sockaddrlen = len; + if (addr != NULL) { + memcpy(sm->sess.sockaddr, addr, len); + sm->sess.sockaddrlen = len; + } http_Setup(&sm->http, (void *)(sm + 1), heritage.mem_workspace); From phk at projects.linpro.no Mon Jul 31 21:46:32 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 31 Jul 2006 23:46:32 +0200 (CEST) Subject: r579 - trunk/varnish-cache/bin/varnishd Message-ID: <20060731214632.3D4461EC399@projects.linpro.no> Author: phk Date: 2006-07-31 23:46:32 +0200 (Mon, 31 Jul 2006) New Revision: 579 Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c Log: Remove unused variable Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-31 21:37:10 UTC (rev 578) +++ trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-31 21:46:32 UTC (rev 579) @@ -170,7 +170,6 @@ hcl_deref(struct objhead *oh) { struct hcl_entry *he; - int ret; unsigned mtx; CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); From phk at projects.linpro.no Mon Jul 31 21:49:29 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 31 Jul 2006 23:49:29 +0200 (CEST) Subject: r580 - in trunk/varnish-cache: bin/varnishd bin/varnishlog include Message-ID: <20060731214929.95B261EC3AD@projects.linpro.no> Author: phk Date: 2006-07-31 23:49:29 +0200 (Mon, 31 Jul 2006) New Revision: 580 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_response.c trunk/varnish-cache/bin/varnishlog/varnishlog.c trunk/varnish-cache/include/shmlog_tags.h Log: Create three groups of seven SHMlog tags: {Rx,Tx,Obj}{Request,Response,Status,URL,Protocol,Header,LostHeader} And log http header munching accordingly. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-31 21:46:32 UTC (rev 579) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-31 21:49:29 UTC (rev 580) @@ -67,7 +67,11 @@ char *e; /* (E)nd of buffer */ unsigned char conds; /* If-* headers present */ - unsigned char objlog; + enum httpwhence { + HTTP_Rx, + HTTP_Tx, + HTTP_Obj + } logtag; struct http_hdr hd[MAX_HTTP_HDRS]; unsigned nhd; Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-31 21:46:32 UTC (rev 579) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-31 21:49:29 UTC (rev 580) @@ -229,7 +229,7 @@ * cache_response.c */ http_ClrHeader(sp->http); - sp->http->objlog = 1; /* log as SLT_ObjHeader */ + sp->http->logtag = HTTP_Obj; http_CopyResp(sp->fd, sp->http, vc->http); http_FilterHeader(sp->fd, sp->http, vc->http, HTTPH_A_INS); @@ -244,7 +244,6 @@ "Content-Length: %u", sp->obj->len); } else cls = 0; - sp->http->objlog = 0; http_CopyHttp(&sp->obj->http, sp->http); if (http_GetHdr(vc->http, H_Connection, &b) && !strcasecmp(b, "close")) @@ -282,6 +281,8 @@ assert(vc != NULL); /* XXX: handle this */ VSL(SLT_Backend, sp->fd, "%d %s", vc->fd, sp->backend->vcl_name); + http_ClrHeader(vc->http); + vc->http->logtag = HTTP_Tx; http_GetReq(vc->fd, vc->http, sp->http); http_FilterHeader(vc->fd, vc->http, sp->http, HTTPH_R_FETCH); http_PrintfHeader(vc->fd, vc->http, "X-Varnish: %u", sp->xid); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-31 21:46:32 UTC (rev 579) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-07-31 21:49:29 UTC (rev 580) @@ -19,12 +19,48 @@ #include "http_headers.h" #undef HTTPH +enum httptag { + HTTP_T_Request, + HTTP_T_Response, + HTTP_T_Status, + HTTP_T_URL, + HTTP_T_Protocol, + HTTP_T_Header, + HTTP_T_LostHeader, +}; + +#define LOGMTX2(ax, bx) \ + [HTTP_T_##bx] = SLT_##ax##bx + +#define LOGMTX1(ax) { \ + LOGMTX2(ax, Request), \ + LOGMTX2(ax, Response), \ + LOGMTX2(ax, Status), \ + LOGMTX2(ax, URL), \ + LOGMTX2(ax, Protocol), \ + LOGMTX2(ax, Header), \ + LOGMTX2(ax, LostHeader) \ + } + +static enum shmlogtag logmtx[3][7] = { + [HTTP_Rx] = LOGMTX1(Rx), + [HTTP_Tx] = LOGMTX1(Tx), + [HTTP_Obj] = LOGMTX1(Obj) +}; + +static enum shmlogtag +T(struct http *hp, enum httptag t) +{ + + CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); + assert(hp->logtag >= HTTP_Rx && hp->logtag <= HTTP_Obj); + assert(t >= HTTP_T_Request && t <= HTTP_T_LostHeader); + return (logmtx[hp->logtag][t]); +} + #define VSLH(ax, bx, cx, dx) \ - VSLR((ax), (bx), (cx)->hd[(dx)].b, (cx)->hd[(dx)].e); + VSLR(T((cx), (ax)), (bx), (cx)->hd[(dx)].b, (cx)->hd[(dx)].e); -#define VSLHT(bx, cx, dx) \ - VSLH((cx)->objlog ? SLT_ObjHeader : SLT_TxHeader, bx, cx, dx) - /*--------------------------------------------------------------------*/ void @@ -241,11 +277,11 @@ if (hp->nhd < MAX_HTTP_HDRS) { hp->hd[hp->nhd].b = p; hp->hd[hp->nhd].e = q; - VSLH(SLT_RxHeader, fd, hp, hp->nhd); + VSLH(HTTP_T_Header, fd, hp, hp->nhd); hp->nhd++; } else { VSL_stats->losthdr++; - VSLR(SLT_LostHeader, fd, p, q); + VSLR(T(hp, HTTP_T_LostHeader), fd, p, q); } } assert(hp->t <= hp->v); @@ -263,6 +299,8 @@ assert(hp->t != NULL); assert(hp->s < hp->t); assert(hp->t <= hp->v); + hp->logtag = HTTP_Rx; + for (p = hp->s ; isspace(*p); p++) continue; @@ -271,7 +309,7 @@ for (; isalpha(*p); p++) ; hp->hd[HTTP_HDR_REQ].e = p; - VSLH(SLT_Request, fd, hp, HTTP_HDR_REQ); + VSLH(HTTP_T_Request, fd, hp, HTTP_HDR_REQ); *p++ = '\0'; /* Next find the URI */ @@ -285,7 +323,7 @@ while (!isspace(*p)) p++; hp->hd[HTTP_HDR_URL].e = p; - VSLH(SLT_URL, fd, hp, HTTP_HDR_URL); + VSLH(HTTP_T_URL, fd, hp, HTTP_HDR_URL); if (*p == '\n') { VSLR(SLT_HttpGarbage, fd, hp->s, hp->v); return (400); @@ -303,7 +341,7 @@ while (!isspace(*p)) p++; hp->hd[HTTP_HDR_PROTO].e = p; - VSLH(SLT_Protocol, fd, hp, HTTP_HDR_PROTO); + VSLH(HTTP_T_Protocol, fd, hp, HTTP_HDR_PROTO); if (*p != '\n') *p++ = '\0'; while (isspace(*p) && *p != '\n') @@ -327,6 +365,8 @@ assert(hp->t != NULL); assert(hp->s < hp->t); assert(hp->t <= hp->v); + hp->logtag = HTTP_Rx; + for (p = hp->s ; isspace(*p); p++) continue; @@ -335,7 +375,7 @@ while (!isspace(*p)) p++; hp->hd[HTTP_HDR_PROTO].e = p; - VSLH(SLT_Protocol, fd, hp, HTTP_HDR_PROTO); + VSLH(HTTP_T_Protocol, fd, hp, HTTP_HDR_PROTO); *p++ = '\0'; /* Next find the status */ @@ -345,7 +385,7 @@ while (!isspace(*p)) p++; hp->hd[HTTP_HDR_STATUS].e = p; - VSLH(SLT_Status, fd, hp, HTTP_HDR_STATUS); + VSLH(HTTP_T_Status, fd, hp, HTTP_HDR_STATUS); *p++ = '\0'; /* Next find the response */ @@ -359,7 +399,7 @@ continue; *q = '\0'; hp->hd[HTTP_HDR_RESPONSE].e = q; - VSLH(SLT_Response, fd, hp, HTTP_HDR_RESPONSE); + VSLH(HTTP_T_Response, fd, hp, HTTP_HDR_RESPONSE); p++; return (http_dissect_hdrs(hp, fd, p)); @@ -526,17 +566,17 @@ /*--------------------------------------------------------------------*/ static void -http_seth(int fd, struct http *to, unsigned n, enum shmlogtag tag, const char *fm) +http_seth(int fd, struct http *to, unsigned n, enum httptag tag, const char *fm) { assert(n < MAX_HTTP_HDRS); assert(fm != NULL); to->hd[n].b = (void*)(uintptr_t)fm; - to->hd[n].e = strchr(fm, '\0'); + to->hd[n].e = (void*)(uintptr_t)strchr(fm, '\0'); VSLH(tag, fd, to, n); } static void -http_copyh(int fd, struct http *to, struct http *fm, unsigned n, enum shmlogtag tag) +http_copyh(int fd, struct http *to, struct http *fm, unsigned n, enum httptag tag) { assert(n < MAX_HTTP_HDRS); @@ -552,9 +592,9 @@ CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC); CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); - http_seth(fd, to, HTTP_HDR_REQ, SLT_Request, "GET"); - http_copyh(fd, to, fm, HTTP_HDR_URL, SLT_URL); - http_seth(fd, to, HTTP_HDR_PROTO, SLT_Protocol, "HTTP/1.1"); + http_seth(fd, to, HTTP_HDR_REQ, HTTP_T_Request, "GET"); + http_copyh(fd, to, fm, HTTP_HDR_URL, HTTP_T_URL); + http_seth(fd, to, HTTP_HDR_PROTO, HTTP_T_Protocol, "HTTP/1.1"); } void @@ -563,9 +603,9 @@ CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC); CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); - http_copyh(fd, to, fm, HTTP_HDR_REQ, SLT_Request); - http_copyh(fd, to, fm, HTTP_HDR_URL, SLT_URL); - http_copyh(fd, to, fm, HTTP_HDR_PROTO, SLT_Protocol); + http_copyh(fd, to, fm, HTTP_HDR_REQ, HTTP_T_Request); + http_copyh(fd, to, fm, HTTP_HDR_URL, HTTP_T_URL); + http_copyh(fd, to, fm, HTTP_HDR_PROTO, HTTP_T_Protocol); } @@ -575,18 +615,18 @@ CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC); CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); - http_copyh(fd, to, fm, HTTP_HDR_PROTO, SLT_Protocol); - http_copyh(fd, to, fm, HTTP_HDR_STATUS, SLT_Status); - http_copyh(fd, to, fm, HTTP_HDR_RESPONSE, SLT_Response); + http_copyh(fd, to, fm, HTTP_HDR_PROTO, HTTP_T_Protocol); + http_copyh(fd, to, fm, HTTP_HDR_STATUS, HTTP_T_Status); + http_copyh(fd, to, fm, HTTP_HDR_RESPONSE, HTTP_T_Response); } void http_SetResp(int fd, struct http *to, const char *proto, const char *status, const char *response) { CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); - http_seth(fd, to, HTTP_HDR_PROTO, SLT_Protocol, proto); - http_seth(fd, to, HTTP_HDR_STATUS, SLT_Status, status); - http_seth(fd, to, HTTP_HDR_RESPONSE, SLT_Response, response); + http_seth(fd, to, HTTP_HDR_PROTO, HTTP_T_Protocol, proto); + http_seth(fd, to, HTTP_HDR_STATUS, HTTP_T_Status, status); + http_seth(fd, to, HTTP_HDR_RESPONSE, HTTP_T_Response, response); } static void @@ -600,11 +640,11 @@ if (to->nhd < MAX_HTTP_HDRS) { to->hd[to->nhd].b = fm->hd[n].b; to->hd[to->nhd].e = fm->hd[n].e; - VSLHT(fd, to, to->nhd); + VSLH(HTTP_T_Header, fd, to, to->nhd); to->nhd++; } else { VSL_stats->losthdr++; - VSLH(SLT_LostHeader, fd, fm, n); + VSLH(HTTP_T_LostHeader, fd, fm, n); } } @@ -645,11 +685,12 @@ { CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); - to->hd[to->nhd].b = (void*)(uintptr_t)hdr; - to->hd[to->nhd].e = strchr(hdr, '\0'); - assert(to->hd[to->nhd].e != NULL); - VSLHT(fd, to, to->nhd); - to->nhd++; + if (to->nhd >= MAX_HTTP_HDRS) { + VSL_stats->losthdr++; + VSL(T(to, HTTP_T_LostHeader), fd, "%s", hdr); + return; + } + http_seth(fd, to, to->nhd++, HTTP_T_Header, hdr); } /*--------------------------------------------------------------------*/ @@ -666,13 +707,13 @@ n = vsnprintf(to->f, l, fmt, ap); if (n + 1 > l || to->nhd >= MAX_HTTP_HDRS) { VSL_stats->losthdr++; - VSL(SLT_LostHeader, fd, "%s", to->f); + VSL(T(to, HTTP_T_LostHeader), fd, "%s", to->f); } else { assert(to->f < to->e); to->hd[to->nhd].b = to->f; to->hd[to->nhd].e = to->f + n; to->f += n + 1; - VSLHT(fd, to, to->nhd); + VSLH(HTTP_T_Header, fd, to, to->nhd); to->nhd++; } va_end(ap); Modified: trunk/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-31 21:46:32 UTC (rev 579) +++ trunk/varnish-cache/bin/varnishd/cache_response.c 2006-07-31 21:49:29 UTC (rev 580) @@ -82,6 +82,7 @@ VSL(SLT_Length, sp->fd, "%u", 0); http_ClrHeader(sp->http); + sp->http->logtag = HTTP_Tx; http_SetResp(sp->fd, sp->http, "HTTP/1.1", "304", "Not Modified"); http_SetHeader(sp->fd, sp->http, "Via: 1.1 varnish"); http_PrintfHeader(sp->fd, sp->http, "X-Varnish: %u", sp->xid); @@ -134,6 +135,7 @@ VSL(SLT_Length, sp->fd, "%u", sp->obj->len); http_ClrHeader(sp->http); + sp->http->logtag = HTTP_Tx; http_CopyResp(sp->fd, sp->http, &sp->obj->http); http_FilterHeader(sp->fd, sp->http, &sp->obj->http, HTTPH_A_DELIVER); if (sp->xid != sp->obj->xid) Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-31 21:46:32 UTC (rev 579) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-31 21:49:29 UTC (rev 580) @@ -126,7 +126,7 @@ else v = 1; break; - case SLT_Request: + case SLT_RxRequest: if (h_opt && p[1] == 3 && !memcmp(p + 4, "GET", 3)) hc[u]++; if (h_opt && p[1] == 4 && !memcmp(p + 4, "HEAD", 4)) @@ -137,7 +137,7 @@ xrf[u] = atoi(p + 4); v = 1; break; - case SLT_Status: + case SLT_RxStatus: if (h_opt && p[1] == 3 && !memcmp(p + 4, "200", 3)) hc[u]++; v = 1; Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-07-31 21:46:32 UTC (rev 579) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-07-31 21:49:29 UTC (rev 580) @@ -24,16 +24,32 @@ SLTM(HttpGarbage) SLTM(ClientAddr) SLTM(Backend) -SLTM(Request) -SLTM(Response) SLTM(Length) -SLTM(Status) -SLTM(URL) -SLTM(Protocol) + +SLTM(RxRequest) +SLTM(RxResponse) +SLTM(RxStatus) +SLTM(RxURL) +SLTM(RxProtocol) SLTM(RxHeader) +SLTM(RxLostHeader) + +SLTM(TxRequest) +SLTM(TxResponse) +SLTM(TxStatus) +SLTM(TxURL) +SLTM(TxProtocol) SLTM(TxHeader) +SLTM(TxLostHeader) + +SLTM(ObjRequest) +SLTM(ObjResponse) +SLTM(ObjStatus) +SLTM(ObjURL) +SLTM(ObjProtocol) SLTM(ObjHeader) -SLTM(LostHeader) +SLTM(ObjLostHeader) + SLTM(TTL) SLTM(VCL_acl) SLTM(VCL_call) From phk at phk.freebsd.dk Mon Jul 31 21:51:06 2006 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Mon, 31 Jul 2006 21:51:06 +0000 Subject: r580 - in trunk/varnish-cache: bin/varnishd bin/varnishlog include In-Reply-To: Your message of "Mon, 31 Jul 2006 23:49:29 +0200." <20060731214929.95B261EC3AD@projects.linpro.no> Message-ID: <67219.1154382666@critter.freebsd.dk> In message <20060731214929.95B261EC3AD at projects.linpro.no>, phk at projects.linpro .no writes: Anders, you will probably need to make a change similar to this in varnishncsa. >=================================================================== >--- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-31 21:46:32 UTC (rev 579) >+++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-31 21:49:29 UTC (rev 580) >@@ -126,7 +126,7 @@ > else > v = 1; > break; >- case SLT_Request: >+ case SLT_RxRequest: > if (h_opt && p[1] == 3 && !memcmp(p + 4, "GET", 3)) > hc[u]++; > if (h_opt && p[1] == 4 && !memcmp(p + 4, "HEAD", 4)) >@@ -137,7 +137,7 @@ > xrf[u] = atoi(p + 4); > v = 1; > break; >- case SLT_Status: >+ case SLT_RxStatus: > if (h_opt && p[1] == 3 && !memcmp(p + 4, "200", 3)) > hc[u]++; > v = 1; > -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at projects.linpro.no Mon Jul 31 22:09:42 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 1 Aug 2006 00:09:42 +0200 (CEST) Subject: r581 - in trunk/varnish-cache: bin/varnishd bin/varnishlog lib/libvarnishapi Message-ID: <20060731220942.4FD101EC3BB@projects.linpro.no> Author: phk Date: 2006-08-01 00:09:42 +0200 (Tue, 01 Aug 2006) New Revision: 581 Modified: trunk/varnish-cache/bin/varnishd/shmlog.c trunk/varnish-cache/bin/varnishlog/varnishlog.c trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Always NUL terminate shmlog entries. Modified: trunk/varnish-cache/bin/varnishd/shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-31 21:49:29 UTC (rev 580) +++ trunk/varnish-cache/bin/varnishd/shmlog.c 2006-07-31 22:09:42 UTC (rev 581) @@ -66,17 +66,18 @@ assert(loghead->ptr < loghead->size); /* Wrap if necessary */ - if (loghead->ptr + 4 + l + 1 > loghead->size) + if (loghead->ptr + 5 + l + 1 > loghead->size) vsl_wrap(); p = logstart + loghead->ptr; p[1] = l & 0xff; p[2] = (id >> 8) & 0xff; p[3] = id & 0xff; memcpy(p + 4, b, l); - p[4 + l] = SLT_ENDMARKER; + p[4 + l] = '\0'; + p[5 + l] = SLT_ENDMARKER; p[0] = tag; - loghead->ptr += 4 + l; + loghead->ptr += 5 + l; assert(loghead->ptr < loghead->size); AZ(pthread_mutex_unlock(&vsl_mutex)); } @@ -95,7 +96,7 @@ assert(loghead->ptr < loghead->size); /* Wrap if we cannot fit a full size record */ - if (loghead->ptr + 4 + 255 + 1 > loghead->size) + if (loghead->ptr + 5 + 255 + 1 > loghead->size) vsl_wrap(); p = logstart + loghead->ptr; @@ -108,10 +109,11 @@ p[1] = n & 0xff; p[2] = (id >> 8) & 0xff; p[3] = id & 0xff; - p[4 + n] = SLT_ENDMARKER; + p[4 + n] = '\0';; + p[5 + n] = SLT_ENDMARKER; p[0] = tag; - loghead->ptr += 4 + n; + loghead->ptr += 5 + n; assert(loghead->ptr < loghead->size); AZ(pthread_mutex_unlock(&vsl_mutex)); Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-31 21:49:29 UTC (rev 580) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-07-31 22:09:42 UTC (rev 581) @@ -259,7 +259,7 @@ } v = 0; if (wfile != NULL) { - i = fwrite(p, 4 + p[1], 1, wfile); + i = fwrite(p, 5 + p[1], 1, wfile); if (i != 1) perror(w_opt); u++; Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-31 21:49:29 UTC (rev 580) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-07-31 22:09:42 UTC (rev 581) @@ -22,7 +22,7 @@ unsigned char *ptr; FILE *fi; - unsigned char rbuf[4 + 255 + 1]; + unsigned char rbuf[5 + 255 + 1]; int b_opt; int c_opt; @@ -140,11 +140,9 @@ i = fread(vd->rbuf, 4, 1, vd->fi); if (i != 1) return (-1); - if (vd->rbuf[1] > 0) { - i = fread(vd->rbuf + 4, vd->rbuf[1], 1, vd->fi); - if (i != 1) - return (-1); - } + i = fread(vd->rbuf + 4, vd->rbuf[1] + 1, 1, vd->fi); + if (i != 1) + return (-1); *pp = vd->rbuf; return (1); } @@ -159,7 +157,7 @@ vd->ptr = p; return (0); } - vd->ptr = p + p[1] + 4; + vd->ptr = p + p[1] + 5; *pp = p; return (1); } From andersb at projects.linpro.no Mon Jul 31 22:21:02 2006 From: andersb at projects.linpro.no (andersb at projects.linpro.no) Date: Tue, 1 Aug 2006 00:21:02 +0200 (CEST) Subject: r582 - trunk/varnish-cache/bin/varnishncsa Message-ID: <20060731222102.3904F1EC3C8@projects.linpro.no> Author: andersb Date: 2006-08-01 00:21:02 +0200 (Tue, 01 Aug 2006) New Revision: 582 Modified: trunk/varnish-cache/bin/varnishncsa/varnishncsa.c Log: Added some more toying with the data. All is still a mess, and I am not sure of structure yet. PHK is also doing changes in areas that will be needed. Modified: trunk/varnish-cache/bin/varnishncsa/varnishncsa.c =================================================================== --- trunk/varnish-cache/bin/varnishncsa/varnishncsa.c 2006-07-31 22:09:42 UTC (rev 581) +++ trunk/varnish-cache/bin/varnishncsa/varnishncsa.c 2006-07-31 22:21:02 UTC (rev 582) @@ -1,5 +1,5 @@ /* - * $Id:$ + * $Id$ * * Program that will get data from the shared memory log. When it has the data * it will order the data based on the sessionid. When the data is ordered @@ -70,15 +70,29 @@ extended_log_format(unsigned char *p, char *w_opt) { unsigned u, v; - int i,j; - char *ans; - char soek[1]; - strcpy(soek," "); + int i,j,k; + unsigned char *ans; + // Declare the int's that are used to determin if we have all data. + int ll_h = 0; // %h + int ll_l = 0; // %l + int ll_u = 0; // %u + int ll_t = 0; // %t + int ll_r = 0; // %r + int ll_s = 0; // %s + int ll_b = 0; // %b + int ll_R = 0; // %{Referer}i + int ll_U = 0; // %{User-agent}i + // Declare the data where we store the differnt parts + char df_h[4 * (3 + 1)]; // Datafield for %h (IP adress) + char df_l; + char df_u[65536]; + char df_U[65536]; + if (w_opt != NULL){ // printf(" Has w_opt\n"); } else { - // printf(" Does not have w_opt\n"); + //printf(" Does not have w_opt:\n"); } u = (p[2] << 8) | p[3]; @@ -91,21 +105,34 @@ case SLT_SessionOpen: - //ans = strchr(&p[4], (int)soek); - //j = strlen(ans); - //printf("%d\n",j); + + // Finding the IP adress when data is: "XXX.XXX.XXX.XXX somenumber" + + ans = strchr(p + 4, ' '); + j = ans - (p + 4); // length + //printf("Ip address: '%*.*s'\n", j, j, p + 4); + memcpy(df_h, p + 4, j); + df_h[j] = '\0'; + //printf("Ip address: %s\n", df_h); + ll_h = 1; + break; case SLT_RxHeader: if (p[1] >= 11 && !strncasecmp((void *)&p[4], "user-agent:",11)){ - //printf(" User-Agent: %s\n", p[4]); - //sbuf_printf(ob[u], "%s\n", &p[4]); - sbuf_bcat(ob[u], p + 4, p[1]); + // Could actually check for ll_h = 1 also in line above. + // If it is equal 1 we know a new client is in, hence a new User-Agent. + + memcpy(df_U, p + 4, p[1]); + df_U[p[1]] = '\0'; + //printf("Ip address: %s\n", df_U); + /*sbuf_bcat(ob[u], p + 4, p[1]); sbuf_cat(ob[u], "\n"); sbuf_finish(ob[u]); printf("%s", sbuf_data(ob[u])); sbuf_clear(ob[u]); + */ } break; @@ -115,7 +142,10 @@ v = 1; break; } - if (v) { + + + + if (ll_h && ll_U) { /* XXX Need to write some code to make the logline sbuf_printf(ob[u], "%02x %3d %4d %-12s",