From phk at projects.linpro.no Tue Jun 13 07:25:20 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 13 Jun 2006 09:25:20 +0200 (CEST) Subject: r168 - trunk/varnish-cache/bin/varnishd Message-ID: <20060613072520.8DCC81EC226@projects.linpro.no> Author: phk Date: 2006-06-13 09:25:20 +0200 (Tue, 13 Jun 2006) New Revision: 168 Modified: trunk/varnish-cache/bin/varnishd/cache_main.c Log: Allow for NULL init methods for hash and stevedore Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-05-01 12:59:34 UTC (rev 167) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-13 07:25:20 UTC (rev 168) @@ -122,10 +122,12 @@ assert(eb != NULL); hash = &hsl_slinger; - hash->init(); + if (hash->init != NULL) + hash->init(); stevedore = &sma_stevedore; - stevedore->init(); + if (stevedore->init != NULL) + stevedore->init(); CVCL_Load(heritage.vcl_file, "boot"); cli = cli_setup(eb, heritage.fds[2], heritage.fds[1], 0, cli_proto); From phk at projects.linpro.no Tue Jun 13 07:26:20 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 13 Jun 2006 09:26:20 +0200 (CEST) Subject: r169 - trunk/varnish-cache/bin/varnishd Message-ID: <20060613072620.264041EC231@projects.linpro.no> Author: phk Date: 2006-06-13 09:26:20 +0200 (Tue, 13 Jun 2006) New Revision: 169 Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c Log: Use NULL init method Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-06-13 07:25:20 UTC (rev 168) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-06-13 07:26:20 UTC (rev 169) @@ -16,11 +16,6 @@ struct storage s; }; -static void -sma_init(void) -{ -} - static struct storage * sma_alloc(unsigned size) { @@ -47,7 +42,7 @@ struct stevedore sma_stevedore = { "Malloc", - sma_init, + NULL, /* init */ sma_alloc, sma_free }; From phk at projects.linpro.no Tue Jun 13 07:57:33 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 13 Jun 2006 09:57:33 +0200 (CEST) Subject: r170 - trunk/varnish-cache/bin/varnishd Message-ID: <20060613075733.093151EC231@projects.linpro.no> Author: phk Date: 2006-06-13 09:57:32 +0200 (Tue, 13 Jun 2006) New Revision: 170 Added: trunk/varnish-cache/bin/varnishd/_stevedore.h Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_main.c trunk/varnish-cache/bin/varnishd/heritage.h trunk/varnish-cache/bin/varnishd/mgt.h trunk/varnish-cache/bin/varnishd/storage_malloc.c trunk/varnish-cache/bin/varnishd/varnishd.c Log: Put more meat on the stevedore (storage backend) interface: Pull the struct definition into _stevedore.h and include this from cache.h and mgt.h, they both need to be able to see it. Add the stevedore pointer as an argument to the stevedore alloc function so multiple stevedores is possible later on. Add the stevedore pointer to the storage object, so freeing it again is possible. Add -s argument processing to select a given stevedore, call it's ->init method and pass the stevedore in the heritage. In the cache process pick stevedore out from heritage, call its open method. Added: trunk/varnish-cache/bin/varnishd/_stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/_stevedore.h 2006-06-13 07:26:20 UTC (rev 169) +++ trunk/varnish-cache/bin/varnishd/_stevedore.h 2006-06-13 07:57:32 UTC (rev 170) @@ -0,0 +1,21 @@ +/* + * $Id: cache.h 164 2006-05-01 12:45:20Z phk $ + */ + +struct stevedore; + +typedef void storage_init_f(struct stevedore *, const char *spec); +typedef void storage_open_f(struct stevedore *); +typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); +typedef void storage_free_f(struct storage *); + +struct stevedore { + const char *name; + storage_init_f *init; /* called by mgt process */ + storage_open_f *open; /* called by cache process */ + storage_alloc_f *alloc; + storage_free_f *free; + + /* private fields */ + void *priv; +}; Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-13 07:26:20 UTC (rev 169) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-13 07:57:32 UTC (rev 170) @@ -42,21 +42,16 @@ unsigned char *ptr; unsigned len; void *priv; + struct stevedore *stevedore; }; -typedef void storage_init_f(void); -typedef struct storage *storage_alloc_f(unsigned size); -typedef void storage_free_f(struct storage *); +#include "_stevedore.h" -struct stevedore { - const char *name; - storage_init_f *init; - storage_alloc_f *alloc; - storage_free_f *free; -}; - -extern struct stevedore sma_stevedore; - +/* + * XXX: in the longer term, we want to support multiple stevedores, + * XXX: selected by some kind of heuristics based on size, lifetime + * XXX: etc etc. For now we support only one. + */ extern struct stevedore *stevedore; /* Prototypes etc ----------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-13 07:26:20 UTC (rev 169) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-13 07:57:32 UTC (rev 170) @@ -33,7 +33,7 @@ cl = strtoumax(b, NULL, 0); - st = stevedore->alloc(cl); + st = stevedore->alloc(stevedore, cl); TAILQ_INSERT_TAIL(&sp->obj->store, st, list); st->len = cl; sp->obj->len = cl; @@ -104,7 +104,7 @@ q++; if (u == 0) break; - st = stevedore->alloc(u); + st = stevedore->alloc(stevedore, u); TAILQ_INSERT_TAIL(&sp->obj->store, st, list); st->len = u; sp->obj->len += u; Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-13 07:26:20 UTC (rev 169) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-13 07:57:32 UTC (rev 170) @@ -125,9 +125,9 @@ if (hash->init != NULL) hash->init(); - stevedore = &sma_stevedore; - if (stevedore->init != NULL) - stevedore->init(); + stevedore = heritage.stevedore; + if (stevedore->open != NULL) + stevedore->open(stevedore); CVCL_Load(heritage.vcl_file, "boot"); cli = cli_setup(eb, heritage.fds[2], heritage.fds[1], 0, cli_proto); Modified: trunk/varnish-cache/bin/varnishd/heritage.h =================================================================== --- trunk/varnish-cache/bin/varnishd/heritage.h 2006-06-13 07:26:20 UTC (rev 169) +++ trunk/varnish-cache/bin/varnishd/heritage.h 2006-06-13 07:57:32 UTC (rev 170) @@ -18,15 +18,18 @@ * interface IP number). */ #define HERITAGE_NSOCKS 2 /* IPv4 + IPv6 */ - int sock_local[HERITAGE_NSOCKS]; - int sock_remote[HERITAGE_NSOCKS]; + int sock_local[HERITAGE_NSOCKS]; + int sock_remote[HERITAGE_NSOCKS]; /* Share memory log fd and size (incl header) */ - int vsl_fd; - unsigned vsl_size; + int vsl_fd; + unsigned vsl_size; /* Initial VCL file */ - char *vcl_file; + char *vcl_file; + + /* Storage method */ + struct stevedore *stevedore; }; extern struct heritage heritage; Modified: trunk/varnish-cache/bin/varnishd/mgt.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt.h 2006-06-13 07:26:20 UTC (rev 169) +++ trunk/varnish-cache/bin/varnishd/mgt.h 2006-06-13 07:57:32 UTC (rev 170) @@ -13,3 +13,7 @@ /* tcp.c */ int open_tcp(const char *port); + +#include "_stevedore.h" + +extern struct stevedore sma_stevedore; Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-06-13 07:26:20 UTC (rev 169) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-06-13 07:57:32 UTC (rev 170) @@ -17,7 +17,7 @@ }; static struct storage * -sma_alloc(unsigned size) +sma_alloc(struct stevedore *st __unused, unsigned size) { struct sma *sma; @@ -41,8 +41,9 @@ } struct stevedore sma_stevedore = { - "Malloc", + "malloc", NULL, /* init */ + NULL, /* open */ sma_alloc, sma_free }; Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-13 07:26:20 UTC (rev 169) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-13 07:57:32 UTC (rev 170) @@ -277,10 +277,12 @@ usage(void) { fprintf(stderr, "usage: varnishd [options]\n"); - fprintf(stderr, " %-20s # %s\n", "-b", "backend_IP_number"); - fprintf(stderr, " %-20s # %s\n", "-d", "debug"); - fprintf(stderr, " %-20s # %s\n", "-f", "VCL_file"); - fprintf(stderr, " %-20s # %s\n", "-p number", "TCP listen port"); + fprintf(stderr, " %-28s # %s\n", "-b", "backend_IP_number"); + fprintf(stderr, " %-28s # %s\n", "-d", "debug"); + fprintf(stderr, " %-28s # %s\n", "-f", "VCL_file"); + fprintf(stderr, " %-28s # %s\n", "-p number", "TCP listen port"); + fprintf(stderr, " %-28s # %s\n", + "-s kind[,storageoptions]", "Backend storage specification"); #if 0 -c clusterid at cluster_controller -m memory_limit @@ -294,6 +296,37 @@ /*--------------------------------------------------------------------*/ +static int +cmp_storage(struct stevedore *s, const char *p, const char *q) +{ + if (strlen(s->name) != q - p) + return (1); + if (strncmp(s->name, p, q - p)) + return (1); + return (0); +} + +static void +setup_storage(const char *sflag) +{ + const char *p; + + p = strchr(sflag, ','); + if (p == NULL) + p = strchr(sflag, '\0'); + if (!cmp_storage(&sma_stevedore, sflag, p)) { + heritage.stevedore = &sma_stevedore; + } else { + fprintf(stderr, "Unknown storage method \"%*.*s\"\n", + p - sflag, p - sflag, sflag); + exit (2); + } + if (heritage.stevedore->init != NULL) + heritage.stevedore->init(heritage.stevedore, p); +} + +/*--------------------------------------------------------------------*/ + #include "shmlog.h" static void @@ -324,6 +357,7 @@ AZ(ftruncate(heritage.vsl_fd, sizeof slh + size)); heritage.vsl_size = slh.size + slh.start; } + /*--------------------------------------------------------------------*/ /* for development purposes */ @@ -338,6 +372,7 @@ unsigned dflag = 1; /* XXX: debug=on for now */ const char *bflag = NULL; const char *fflag = NULL; + const char *sflag = "malloc"; register_printf_render_std((const unsigned char *)"HVQ"); @@ -357,6 +392,9 @@ case 'p': portnumber = optarg; break; + case 's': + sflag = optarg; + break; default: usage(); } @@ -385,6 +423,8 @@ if (heritage.vcl_file == NULL) exit (1); + setup_storage(sflag); + /* * XXX: Lacking the suspend/resume facility (due to the socket API * missing an unlisten(2) facility) we may want to push this into @@ -398,6 +438,5 @@ testme(); - exit(0); } From phk at projects.linpro.no Tue Jun 13 07:59:20 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 13 Jun 2006 09:59:20 +0200 (CEST) Subject: r171 - trunk/varnish-cache/bin/varnishd Message-ID: <20060613075920.B37CE1EC231@projects.linpro.no> Author: phk Date: 2006-06-13 09:59:20 +0200 (Tue, 13 Jun 2006) New Revision: 171 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Remember to tell getopt about -s Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-13 07:57:32 UTC (rev 170) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-13 07:59:20 UTC (rev 171) @@ -378,7 +378,7 @@ VCL_InitCompile(); - while ((o = getopt(argc, argv, "b:df:p:")) != -1) + while ((o = getopt(argc, argv, "b:df:p:s:")) != -1) switch (o) { case 'b': bflag = optarg; From phk at projects.linpro.no Tue Jun 13 08:02:59 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 13 Jun 2006 10:02:59 +0200 (CEST) Subject: r172 - trunk/varnish-cache/bin/varnishd Message-ID: <20060613080259.8E7A71EC23A@projects.linpro.no> Author: phk Date: 2006-06-13 10:02:59 +0200 (Tue, 13 Jun 2006) New Revision: 172 Added: trunk/varnish-cache/bin/varnishd/storage_file.c Modified: trunk/varnish-cache/bin/varnishd/Makefile.am trunk/varnish-cache/bin/varnishd/mgt.h trunk/varnish-cache/bin/varnishd/varnishd.c Log: Clone the malloc stevedore to the file stevedore Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-06-13 07:59:20 UTC (rev 171) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-06-13 08:02:59 UTC (rev 172) @@ -18,6 +18,7 @@ cli_event.c \ hash_simple_list.c \ mgt_child.c \ + storage_file.c \ storage_malloc.c \ tcp.c \ varnishd.c Modified: trunk/varnish-cache/bin/varnishd/mgt.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt.h 2006-06-13 07:59:20 UTC (rev 171) +++ trunk/varnish-cache/bin/varnishd/mgt.h 2006-06-13 08:02:59 UTC (rev 172) @@ -17,3 +17,4 @@ #include "_stevedore.h" extern struct stevedore sma_stevedore; +extern struct stevedore smf_stevedore; Added: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-13 07:59:20 UTC (rev 171) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-13 08:02:59 UTC (rev 172) @@ -0,0 +1,49 @@ +/* + * $Id: storage_malloc.c 170 2006-06-13 07:57:32Z phk $ + * + * Storage method based on mmap'ed file + */ + +#include +#include +#include +#include + +#include "vcl_lang.h" +#include "cache.h" + +struct smf { + struct storage s; +}; + +static struct storage * +smf_alloc(struct stevedore *st __unused, unsigned size) +{ + struct smf *smf; + + smf = calloc(sizeof *smf, 1); + assert(smf != NULL); + smf->s.priv = smf; + smf->s.ptr = malloc(size); + assert(smf->s.ptr != NULL); + smf->s.len = size; + return (&smf->s); +} + +static void +smf_free(struct storage *s) +{ + struct smf *smf; + + smf = s->priv; + free(smf->s.ptr); + free(smf); +} + +struct stevedore smf_stevedore = { + "file", + NULL, /* init */ + NULL, /* open */ + smf_alloc, + smf_free +}; Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-13 07:59:20 UTC (rev 171) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-13 08:02:59 UTC (rev 172) @@ -316,6 +316,8 @@ p = strchr(sflag, '\0'); if (!cmp_storage(&sma_stevedore, sflag, p)) { heritage.stevedore = &sma_stevedore; + } else if (!cmp_storage(&smf_stevedore, sflag, p)) { + heritage.stevedore = &smf_stevedore; } else { fprintf(stderr, "Unknown storage method \"%*.*s\"\n", p - sflag, p - sflag, sflag); From phk at projects.linpro.no Tue Jun 13 08:05:33 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 13 Jun 2006 10:05:33 +0200 (CEST) Subject: r173 - trunk/varnish-cache/bin/varnishd Message-ID: <20060613080533.7AD4E1EC23A@projects.linpro.no> Author: phk Date: 2006-06-13 10:05:33 +0200 (Tue, 13 Jun 2006) New Revision: 173 Added: trunk/varnish-cache/bin/varnishd/stevedore.h Removed: trunk/varnish-cache/bin/varnishd/_stevedore.h Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/mgt.h Log: After having a strong cup of tea: don't name files with leading underscore even though that's how FreeBSD's kernel does it. In my private world a leading underscore means "junk file, remove at your pleasure". Deleted: trunk/varnish-cache/bin/varnishd/_stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/_stevedore.h 2006-06-13 08:02:59 UTC (rev 172) +++ trunk/varnish-cache/bin/varnishd/_stevedore.h 2006-06-13 08:05:33 UTC (rev 173) @@ -1,21 +0,0 @@ -/* - * $Id: cache.h 164 2006-05-01 12:45:20Z phk $ - */ - -struct stevedore; - -typedef void storage_init_f(struct stevedore *, const char *spec); -typedef void storage_open_f(struct stevedore *); -typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); -typedef void storage_free_f(struct storage *); - -struct stevedore { - const char *name; - storage_init_f *init; /* called by mgt process */ - storage_open_f *open; /* called by cache process */ - storage_alloc_f *alloc; - storage_free_f *free; - - /* private fields */ - void *priv; -}; Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-13 08:02:59 UTC (rev 172) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-13 08:05:33 UTC (rev 173) @@ -45,7 +45,7 @@ struct stevedore *stevedore; }; -#include "_stevedore.h" +#include "stevedore.h" /* * XXX: in the longer term, we want to support multiple stevedores, Modified: trunk/varnish-cache/bin/varnishd/mgt.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt.h 2006-06-13 08:02:59 UTC (rev 172) +++ trunk/varnish-cache/bin/varnishd/mgt.h 2006-06-13 08:05:33 UTC (rev 173) @@ -14,7 +14,7 @@ /* tcp.c */ int open_tcp(const char *port); -#include "_stevedore.h" +#include "stevedore.h" extern struct stevedore sma_stevedore; extern struct stevedore smf_stevedore; Copied: trunk/varnish-cache/bin/varnishd/stevedore.h (from rev 170, trunk/varnish-cache/bin/varnishd/_stevedore.h) From phk at projects.linpro.no Tue Jun 13 13:10:09 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 13 Jun 2006 15:10:09 +0200 (CEST) Subject: r174 - trunk/varnish-cache/bin/varnishd Message-ID: <20060613131009.B3B641EC1CD@projects.linpro.no> Author: phk Date: 2006-06-13 15:10:09 +0200 (Tue, 13 Jun 2006) New Revision: 174 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Clone the stevedore before calling its init function and be more precise about any optional arguments. Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-13 08:05:33 UTC (rev 173) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-13 13:10:09 UTC (rev 174) @@ -309,22 +309,27 @@ static void setup_storage(const char *sflag) { - const char *p; + const char *p, *q; + struct stevedore *stp; - p = strchr(sflag, ','); + q = p = strchr(sflag, ','); if (p == NULL) - p = strchr(sflag, '\0'); + q = p = strchr(sflag, '\0'); + else + q++; if (!cmp_storage(&sma_stevedore, sflag, p)) { - heritage.stevedore = &sma_stevedore; + stp = &sma_stevedore; } else if (!cmp_storage(&smf_stevedore, sflag, p)) { - heritage.stevedore = &smf_stevedore; + stp = &smf_stevedore; } else { fprintf(stderr, "Unknown storage method \"%*.*s\"\n", p - sflag, p - sflag, sflag); exit (2); } - if (heritage.stevedore->init != NULL) - heritage.stevedore->init(heritage.stevedore, p); + heritage.stevedore = malloc(sizeof *heritage.stevedore); + *heritage.stevedore = *stp; + if (stp->init != NULL) + stp->init(heritage.stevedore, q); } /*--------------------------------------------------------------------*/ From phk at projects.linpro.no Tue Jun 13 13:14:12 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 13 Jun 2006 15:14:12 +0200 (CEST) Subject: r175 - trunk/varnish-cache/bin/varnishd Message-ID: <20060613131412.E93F01EC1CF@projects.linpro.no> Author: phk Date: 2006-06-13 15:14:12 +0200 (Tue, 13 Jun 2006) New Revision: 175 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: Calculate the size of the backing store file. A size can be specified in absolute terms (suffix: k, m, g, t supported), but also as a percentage of the filesystems free space (suffix '%'). If the specified size is larger than an off_t can cope, we bisect repeatedly until it can. If the size exceeds the available space of the filesystem, we truncate to 80% of the free space. Then round down to an integral number of blocks, sized by the larger of the filesystem blocksize and the pagesize. This was tricker than I'd expected... Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-13 13:10:09 UTC (rev 174) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-13 13:14:12 UTC (rev 175) @@ -5,17 +5,222 @@ */ #include +#include +#include +#include +#include +#include +#include #include +#include #include +#include #include +#include +#include #include "vcl_lang.h" +#include "libvarnish.h" #include "cache.h" +struct smf_sc { + char *filename; + int fd; + uintmax_t filesize; +}; + struct smf { struct storage s; }; +/*--------------------------------------------------------------------*/ + +static void +smf_calcsize(struct smf_sc *sc, const char *size, int newfile) +{ + uintmax_t l; + unsigned bs; + char suff[2]; + int i; + off_t o; + struct statfs fsst; + struct stat st; + + AZ(fstat(sc->fd, &st)); + AZ(fstatfs(sc->fd, &fsst)); + + /* We use units of the larger of filesystem blocksize and pagesize */ + bs = getpagesize(); + if (bs < fsst.f_bsize) + bs = fsst.f_bsize; + + assert(S_ISREG(st.st_mode)); + + i = sscanf(size, "%ju%1s", &l, suff); /* can return -1, 0, 1 or 2 */ + + if (i == 0) { + fprintf(stderr, + "Error: (-sfile) size \"%s\" not understood\n", size); + exit (2); + } + + if (i >= 1 && l == 0) { + fprintf(stderr, + "Error: (-sfile) zero size not permitted\n"); + exit (2); + } + + if (i == -1 && !newfile) /* Use the existing size of the file */ + l = st.st_size; + + /* We must have at least one block */ + if (l < bs) { + if (i == -1) { + fprintf(stderr, + "Info: (-sfile) default to 80%% size\n"); + l = 80; + suff[0] = '%'; + i = 2; + } + + if (i == 2) { + if (suff[0] == 'k' || suff[0] == 'K') + l *= 1024UL; + else if (suff[0] == 'm' || suff[0] == 'M') + l *= 1024UL * 1024UL; + else if (suff[0] == 'g' || suff[0] == 'G') + l *= 1024UL * 1024UL * 1024UL; + else if (suff[0] == 't' || suff[0] == 'T') + l *= (uintmax_t)(1024UL * 1024UL) * + (uintmax_t)(1024UL * 1024UL); + else if (suff[0] == '%') { + l *= fsst.f_bsize * fsst.f_bavail; + l /= 100; + } + } + + o = l; + if (o != l || o < 0) { + fprintf(stderr, + "Warning: size reduced to system limit (off_t)\n"); + do { + l >>= 1; + o = l; + } while (o != l || o < 0); + } + + if (l < st.st_size) { + AZ(ftruncate(sc->fd, l)); + } else if (l - st.st_size > fsst.f_bsize * fsst.f_bavail) { + fprintf(stderr, + "Warning: size larger than filesystem free space," + " reduced to 80%% of free space.\n"); + l = (fsst.f_bsize * fsst.f_bavail * 80) / 100; + } + } + + /* round down to of filesystem blocksize or pagesize */ + l -= (l % bs); + + assert(l >= bs); + + printf("file %s size %ju bytes (%ju fs-blocks, %ju pages)\n", + sc->filename, l, l / fsst.f_bsize, l / getpagesize()); + + sc->filesize = l; +} + +static void +smf_initfile(struct smf_sc *sc, const char *size, int newfile) +{ + smf_calcsize(sc, size, newfile); +} + +static void +smf_init(struct stevedore *parent, const char *spec) +{ + char *size; + char *p, *q; + struct stat st; + struct smf_sc *sc; + + sc = calloc(sizeof *sc, 1); + assert(sc != NULL); + + /* If no size specified, use 50% of filesystem free space */ + if (spec == NULL || *spec == '\0') + spec = "/tmp,50%"; + + if (strchr(spec, ',') == NULL) + asprintf(&p, "%s,", spec); + else + p = strdup(spec); + assert(p != NULL); + size = strchr(p, ','); + assert(size != NULL); + + *size++ = '\0'; + + /* try to create a new file of this name */ + sc->fd = open(p, O_RDWR | O_CREAT | O_EXCL, 0600); + if (sc->fd >= 0) { + sc->filename = p; + smf_initfile(sc, size, 1); + return; + } + + /* it must exist then */ + if (stat(p, &st)) { + fprintf(stderr, + "Error: (-sfile) \"%s\" " + "does not exist and could not be created\n", p); + exit (2); + } + + /* and it should be a file or directory */ + if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode))) { + fprintf(stderr, + "Error: (-sfile) \"%s\" " + "is neither file nor directory\n", p); + exit (2); + } + + if (S_ISREG(st.st_mode)) { + sc->fd = open(p, O_RDWR); + if (sc->fd < 0) { + fprintf(stderr, + "Error: (-sfile) \"%s\" " + "could not open (%s)\n", p, strerror(errno)); + exit (2); + } + AZ(fstat(sc->fd, &st)); + if (!S_ISREG(st.st_mode)) { + fprintf(stderr, + "Error: (-sfile) \"%s\" " + "was not a file after opening\n", p); + exit (2); + } + sc->filename = p; + smf_initfile(sc, size, 0); + return; + } + + asprintf(&q, "%s/varnish.XXXXXX", p); + assert(q != NULL); + sc->fd = mkstemp(q); + if (sc->fd < 0) { + fprintf(stderr, + "Error: (-sfile) \"%s\" " + "mkstemp(%s) failed (%s)\n", p, q, strerror(errno)); + exit (2); + } + AZ(unlink(q)); + asprintf(&sc->filename, "%s (unlinked)", q); + assert(sc->filename != NULL); + free(q); + smf_initfile(sc, size, 1); +} + static struct storage * smf_alloc(struct stevedore *st __unused, unsigned size) { @@ -42,7 +247,7 @@ struct stevedore smf_stevedore = { "file", - NULL, /* init */ + smf_init, /* init */ NULL, /* open */ smf_alloc, smf_free From phk at phk.freebsd.dk Tue Jun 13 13:22:27 2006 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Tue, 13 Jun 2006 13:22:27 +0000 Subject: r175 - trunk/varnish-cache/bin/varnishd In-Reply-To: Your message of "Tue, 13 Jun 2006 15:14:12 +0200." <20060613131412.E93F01EC1CF@projects.linpro.no> Message-ID: <3253.1150204947@critter.freebsd.dk> In message <20060613131412.E93F01EC1CF at projects.linpro.no>, phk at projects.linpro .no writes: >This was tricker than I'd expected... I had to add CFLAGS=-std=c99 to configure to get it to accept uintmax_t and the %j formats. Is there any way we can make that the default ? -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From des at linpro.no Tue Jun 13 13:40:44 2006 From: des at linpro.no (Dag-Erling =?iso-8859-1?Q?Sm=F8rgrav?=) Date: Tue, 13 Jun 2006 15:40:44 +0200 Subject: r175 - trunk/varnish-cache/bin/varnishd References: <3253.1150204947@critter.freebsd.dk> Message-ID: "Poul-Henning Kamp" writes: > I had to add CFLAGS=-std=c99 to configure to get it to accept > uintmax_t and the %j formats. > > Is there any way we can make that the default ? Easily. In the meantime, you can re-run configure without --enable-pedantic (which implies -std=c89) DES -- Dag-Erling Sm?rgrav Senior Software Developer Linpro AS - www.linpro.no From phk at projects.linpro.no Tue Jun 13 13:47:51 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 13 Jun 2006 15:47:51 +0200 (CEST) Subject: r176 - trunk/varnish-cache/bin/varnishd Message-ID: <20060613134751.91DE31EC1CF@projects.linpro.no> Author: phk Date: 2006-06-13 15:47:51 +0200 (Tue, 13 Jun 2006) New Revision: 176 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: mmap as much as the file as we are able to when the cache process opens the stevedore. This should probably be (fine-)tuned later on as it might be too aggressive when faced with user errors. Also, might it be possible to mmap() more than MAX_SIZE_T bytes if it is done in multiple calls ? Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-13 13:14:12 UTC (rev 175) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-13 13:47:51 UTC (rev 176) @@ -18,11 +18,16 @@ #include #include #include +#include #include "vcl_lang.h" #include "libvarnish.h" #include "cache.h" +#define MINPAGES 128 + +/*--------------------------------------------------------------------*/ + struct smf_sc { char *filename; int fd; @@ -122,7 +127,12 @@ /* round down to of filesystem blocksize or pagesize */ l -= (l % bs); - assert(l >= bs); + if (l < MINPAGES * getpagesize()) { + fprintf(stderr, + "Error: size too small, at least %ju needed\n", + (uintmax_t)MINPAGES * getpagesize()); + exit (2); + } printf("file %s size %ju bytes (%ju fs-blocks, %ju pages)\n", sc->filename, l, l / fsst.f_bsize, l / getpagesize()); @@ -134,6 +144,10 @@ smf_initfile(struct smf_sc *sc, const char *size, int newfile) { smf_calcsize(sc, size, newfile); + + AZ(ftruncate(sc->fd, sc->filesize)); + + /* XXX: force block allocation here or in open ? */ } static void @@ -146,6 +160,7 @@ sc = calloc(sizeof *sc, 1); assert(sc != NULL); + parent->priv = sc; /* If no size specified, use 50% of filesystem free space */ if (spec == NULL || *spec == '\0') @@ -221,6 +236,65 @@ smf_initfile(sc, size, 1); } +/*--------------------------------------------------------------------*/ + +static void +smf_open_chunk(struct smf_sc *sc, size_t sz, size_t off, size_t *fail, size_t *sum) +{ + void *p; + size_t h; + + if (sz < getpagesize() * MINPAGES) + return; + + if (sz > *fail) + return; + + printf("%s(%ju, %ju)\n", __func__, (uintmax_t)sz, (uintmax_t)off); + p = mmap(NULL, sz, PROT_READ|PROT_WRITE, + MAP_NOCORE | MAP_NOSYNC | MAP_PRIVATE, sc->fd, off); + printf("mmap = %p\n", p); + if (p != MAP_FAILED) { + (*sum) += sz; + return; + } + + if (sz < *fail) + *fail = sz; + + h = sz / 2; + h -= (h % getpagesize()); + + smf_open_chunk(sc, h, off, fail, sum); + smf_open_chunk(sc, sz - h, off + h, fail, sum); +} + +static void +smf_open(struct stevedore *st) +{ + struct smf_sc *sc; + size_t fail = SIZE_T_MAX; + size_t sum = 0; + + sc = st->priv; + + if (sc->filesize > SIZE_T_MAX) { + sc->filesize = SIZE_T_MAX; + fprintf(stderr, "Truncating %s to SIZE_T_MAX %ju\n", + sc->filename, sc->filesize); + } + + smf_open_chunk(sc, sc->filesize, 0, &fail, &sum); + printf("managed to mmap %ju bytes of %ju\n", + (uintmax_t)sum, sc->filesize); + + /* XXX */ + if (sum < MINPAGES * getpagesize()) + exit (2); +} + +/*--------------------------------------------------------------------*/ + static struct storage * smf_alloc(struct stevedore *st __unused, unsigned size) { @@ -247,8 +321,8 @@ struct stevedore smf_stevedore = { "file", - smf_init, /* init */ - NULL, /* open */ + smf_init, + smf_open, smf_alloc, smf_free }; From des at linpro.no Tue Jun 13 14:24:53 2006 From: des at linpro.no (Dag-Erling =?iso-8859-1?Q?Sm=F8rgrav?=) Date: Tue, 13 Jun 2006 16:24:53 +0200 Subject: r176 - trunk/varnish-cache/bin/varnishd References: <20060613134751.91DE31EC1CF@projects.linpro.no> Message-ID: phk at projects.linpro.no writes: > Also, might it be possible to mmap() more than MAX_SIZE_T bytes if > it is done in multiple calls ? I suspect you'll run out of address space long before you reach SIZE_T_MAX, since SIZE_T_MAX is equal to the size of the address space minus one on all relevant platforms. I don't quite understand this code, BTW - it discards the return value from mmap(), and it does not attempt to map chunks contiguously. I assume this is because it is unfinished? DES -- Dag-Erling Sm?rgrav Senior Software Developer Linpro AS - www.linpro.no From phk at projects.linpro.no Tue Jun 13 20:06:45 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 13 Jun 2006 22:06:45 +0200 (CEST) Subject: r177 - trunk/varnish-cache/bin/varnishd Message-ID: <20060613200645.9B4D01EC1CF@projects.linpro.no> Author: phk Date: 2006-06-13 22:06:45 +0200 (Tue, 13 Jun 2006) New Revision: 177 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: Be more aggressive about mmap'ing memory. The size_t thing is a bogus constraint in FreeBSD and we shouldn't really listen to it. On my laptop I can mmap 2422MB file this way. This may be so aggressive that it leaves insufficient address space for malloc/threadstacks and other issues. If we find a way to pick up a platform/architecture specific limit, we can enforce that along the way. For now people can just specify a saner and smaller "-sfile" size. Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-13 13:47:51 UTC (rev 176) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-13 20:06:45 UTC (rev 177) @@ -238,31 +238,40 @@ /*--------------------------------------------------------------------*/ +/* + * XXX: This may be too aggressive and soak up too much address room. + * XXX: On the other hand, the user, directly or implicitly asked us to + * XXX: use this much storage, so we should make a decent effort. + * XXX: worst case (I think), malloc will fail. + */ + static void -smf_open_chunk(struct smf_sc *sc, size_t sz, size_t off, size_t *fail, size_t *sum) +smf_open_chunk(struct smf_sc *sc, off_t sz, off_t off, off_t *fail, off_t *sum) { void *p; - size_t h; + off_t h; - if (sz < getpagesize() * MINPAGES) + if (*fail < getpagesize() * MINPAGES) return; - if (sz > *fail) - return; - - printf("%s(%ju, %ju)\n", __func__, (uintmax_t)sz, (uintmax_t)off); - p = mmap(NULL, sz, PROT_READ|PROT_WRITE, - MAP_NOCORE | MAP_NOSYNC | MAP_PRIVATE, sc->fd, off); - printf("mmap = %p\n", p); - if (p != MAP_FAILED) { - (*sum) += sz; - return; + if (sz < *fail && sz < SIZE_T_MAX) { + p = mmap(NULL, sz, PROT_READ|PROT_WRITE, + MAP_NOCORE | MAP_NOSYNC | MAP_PRIVATE, sc->fd, off); + if (p != MAP_FAILED) { + printf("%12p...%12p %12jx...%12jx %12jx\n", + p, (u_char *)p + sz, + off, off + sz, sz); + (*sum) += sz; + return; + } } if (sz < *fail) *fail = sz; h = sz / 2; + if (h > SIZE_T_MAX) + h = SIZE_T_MAX; h -= (h % getpagesize()); smf_open_chunk(sc, h, off, fail, sum); @@ -273,17 +282,11 @@ smf_open(struct stevedore *st) { struct smf_sc *sc; - size_t fail = SIZE_T_MAX; - size_t sum = 0; + off_t fail = SIZE_T_MAX; + off_t sum = 0; sc = st->priv; - if (sc->filesize > SIZE_T_MAX) { - sc->filesize = SIZE_T_MAX; - fprintf(stderr, "Truncating %s to SIZE_T_MAX %ju\n", - sc->filename, sc->filesize); - } - smf_open_chunk(sc, sc->filesize, 0, &fail, &sum); printf("managed to mmap %ju bytes of %ju\n", (uintmax_t)sum, sc->filesize); From phk at projects.linpro.no Tue Jun 13 20:09:29 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 13 Jun 2006 22:09:29 +0200 (CEST) Subject: r178 - trunk/varnish-cache/bin/varnishd Message-ID: <20060613200929.009481EC1CE@projects.linpro.no> Author: phk Date: 2006-06-13 22:09:29 +0200 (Tue, 13 Jun 2006) New Revision: 178 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Default to "file" stevedore from now on. Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-13 20:06:45 UTC (rev 177) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-13 20:09:29 UTC (rev 178) @@ -379,7 +379,7 @@ unsigned dflag = 1; /* XXX: debug=on for now */ const char *bflag = NULL; const char *fflag = NULL; - const char *sflag = "malloc"; + const char *sflag = "file"; register_printf_render_std((const unsigned char *)"HVQ"); From phk at projects.linpro.no Wed Jun 14 06:58:55 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 14 Jun 2006 08:58:55 +0200 (CEST) Subject: r179 - trunk/varnish-cache/bin/varnishd Message-ID: <20060614065855.954D21EC2FA@projects.linpro.no> Author: phk Date: 2006-06-14 08:58:55 +0200 (Wed, 14 Jun 2006) New Revision: 179 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: Complete the storage_file method. Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-13 20:09:29 UTC (rev 178) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-14 06:58:55 UTC (rev 179) @@ -28,16 +28,33 @@ /*--------------------------------------------------------------------*/ +struct smf { + struct storage s; + struct smf_sc *sc; + + int alloc; + time_t age; + + off_t size; + off_t offset; + unsigned char *ptr; + + TAILQ_ENTRY(smf) order; + TAILQ_ENTRY(smf) status; +}; + +TAILQ_HEAD(smfhead, smf); + struct smf_sc { char *filename; int fd; + int pagesize; uintmax_t filesize; + struct smfhead order; + struct smfhead free; + struct smfhead used; }; -struct smf { - struct storage s; -}; - /*--------------------------------------------------------------------*/ static void @@ -55,7 +72,7 @@ AZ(fstatfs(sc->fd, &fsst)); /* We use units of the larger of filesystem blocksize and pagesize */ - bs = getpagesize(); + bs = sc->pagesize; if (bs < fsst.f_bsize) bs = fsst.f_bsize; @@ -127,15 +144,15 @@ /* round down to of filesystem blocksize or pagesize */ l -= (l % bs); - if (l < MINPAGES * getpagesize()) { + if (l < MINPAGES * sc->pagesize) { fprintf(stderr, "Error: size too small, at least %ju needed\n", - (uintmax_t)MINPAGES * getpagesize()); + (uintmax_t)MINPAGES * sc->pagesize); exit (2); } printf("file %s size %ju bytes (%ju fs-blocks, %ju pages)\n", - sc->filename, l, l / fsst.f_bsize, l / getpagesize()); + sc->filename, l, l / fsst.f_bsize, l / sc->pagesize); sc->filesize = l; } @@ -160,6 +177,11 @@ sc = calloc(sizeof *sc, 1); assert(sc != NULL); + TAILQ_INIT(&sc->order); + TAILQ_INIT(&sc->free); + TAILQ_INIT(&sc->used); + sc->pagesize = getpagesize(); + parent->priv = sc; /* If no size specified, use 50% of filesystem free space */ @@ -236,6 +258,137 @@ smf_initfile(sc, size, 1); } +/*-------------------------------------------------------------------- + * Allocate a range from the first free range that is large enough. + */ + +static struct smf * +alloc_smf(struct smf_sc *sc, size_t bytes) +{ + struct smf *sp, *sp2; + + bytes += (sc->pagesize - 1); + bytes &= ~(sc->pagesize - 1); + TAILQ_FOREACH(sp, &sc->free, status) { + if (sp->size >= bytes) + break; + } + if (sp == NULL) + return (sp); + + if (sp->size == bytes) { + TAILQ_REMOVE(&sc->free, sp, status); + sp->alloc = 1; + TAILQ_INSERT_TAIL(&sc->used, sp, status); + printf("ALLOC %12p %12p %12jx %12jx\n", (void*)sp, (void*)sp->ptr, (uintmax_t)sp->offset, (uintmax_t)sp->size); + return (sp); + } + + /* Split from front */ + sp2 = malloc(sizeof *sp2); + assert(sp2 != NULL); + *sp2 = *sp; + + sp->offset += bytes; + sp->ptr += bytes; + sp->size -= bytes; + + sp2->size = bytes; + sp2->alloc = 1; + TAILQ_INSERT_BEFORE(sp, sp2, order); + TAILQ_INSERT_TAIL(&sc->used, sp2, status); + printf("SPLIT %12p %12p %12jx %12jx\n", (void*)sp, (void*)sp->ptr, (uintmax_t)sp->offset, (uintmax_t)sp->size); + printf("ALLOC %12p %12p %12jx %12jx\n", (void*)sp2, (void*)sp2->ptr, (uintmax_t)sp2->offset, (uintmax_t)sp2->size); + return (sp2); +} + +/*-------------------------------------------------------------------- + * Free a range. Attemt merge forward and backward, then sort into + * free list according to age. + */ + +static void +free_smf(struct smf *sp) +{ + struct smf *sp2; + struct smf_sc *sc = sp->sc; + + printf("FREE %12p %12p %12jx %12jx\n", (void*)sp, (void*)sp->ptr, (uintmax_t)sp->offset, (uintmax_t)sp->size); + TAILQ_REMOVE(&sc->used, sp, status); + sp->alloc = 0; + + sp2 = TAILQ_NEXT(sp, order); + if (sp2 != NULL && + sp2->alloc == 0 && + (sp2->ptr == sp->ptr + sp->size) && + (sp2->offset == sp->offset + sp->size)) { + sp->size += sp2->size; + TAILQ_REMOVE(&sc->order, sp2, order); + TAILQ_REMOVE(&sc->free, sp2, status); + free(sp2); + printf("MERGEN %12p %12p %12jx %12jx\n", (void*)sp, (void*)sp->ptr, (uintmax_t)sp->offset, (uintmax_t)sp->size); + } + + sp2 = TAILQ_PREV(sp, smfhead, order); + if (sp2 != NULL && + sp2->alloc == 0 && + (sp->ptr == sp2->ptr + sp2->size) && + (sp->offset == sp2->offset + sp2->size)) { + sp2->size += sp->size; + sp2->age = sp->age; + TAILQ_REMOVE(&sc->order, sp, order); + free(sp); + TAILQ_REMOVE(&sc->free, sp2, status); + sp = sp2; + printf("MERGEP %12p %12p %12jx %12jx\n", (void*)sp, (void*)sp->ptr, (uintmax_t)sp->offset, (uintmax_t)sp->size); + } + + TAILQ_FOREACH(sp2, &sc->free, status) { + if (sp->age > sp2->age || + (sp->age == sp2->age && sp->offset < sp2->offset)) { + TAILQ_INSERT_BEFORE(sp2, sp, order); + break; + } + } + if (sp2 == NULL) + TAILQ_INSERT_TAIL(&sc->free, sp, status); +} + +/*-------------------------------------------------------------------- + * Insert a newly created range as busy, then free it to do any collapses + */ + +static void +new_smf(struct smf_sc *sc, unsigned char *ptr, off_t off, size_t len) +{ + struct smf *sp, *sp2; + + sp = calloc(sizeof *sp, 1); + assert(sp != NULL); + printf("NEW %12p %12p %12jx %12jx\n", (void*)sp, (void*)ptr, (uintmax_t)off, (uintmax_t)len); + + sp->sc = sc; + + sp->size = len; + sp->ptr = ptr; + sp->offset = off; + + sp->alloc = 1; + + TAILQ_FOREACH(sp2, &sc->order, order) { + if (sp->ptr < sp2->ptr) { + TAILQ_INSERT_BEFORE(sp2, sp, order); + break; + } + } + if (sp2 == NULL) + TAILQ_INSERT_TAIL(&sc->order, sp, order); + + TAILQ_INSERT_HEAD(&sc->used, sp, status); + + free_smf(sp); +} + /*--------------------------------------------------------------------*/ /* @@ -251,17 +404,15 @@ void *p; off_t h; - if (*fail < getpagesize() * MINPAGES) + if (*fail < sc->pagesize * MINPAGES) return; if (sz < *fail && sz < SIZE_T_MAX) { p = mmap(NULL, sz, PROT_READ|PROT_WRITE, MAP_NOCORE | MAP_NOSYNC | MAP_PRIVATE, sc->fd, off); if (p != MAP_FAILED) { - printf("%12p...%12p %12jx...%12jx %12jx\n", - p, (u_char *)p + sz, - off, off + sz, sz); (*sum) += sz; + new_smf(sc, p, off, sz); return; } } @@ -272,7 +423,7 @@ h = sz / 2; if (h > SIZE_T_MAX) h = SIZE_T_MAX; - h -= (h % getpagesize()); + h -= (h % sc->pagesize); smf_open_chunk(sc, h, off, fail, sum); smf_open_chunk(sc, sz - h, off + h, fail, sum); @@ -299,15 +450,14 @@ /*--------------------------------------------------------------------*/ static struct storage * -smf_alloc(struct stevedore *st __unused, unsigned size) +smf_alloc(struct stevedore *st, unsigned size) { struct smf *smf; - smf = calloc(sizeof *smf, 1); + smf = alloc_smf(st->priv, size); assert(smf != NULL); smf->s.priv = smf; - smf->s.ptr = malloc(size); - assert(smf->s.ptr != NULL); + smf->s.ptr = smf->ptr; smf->s.len = size; return (&smf->s); } @@ -318,8 +468,7 @@ struct smf *smf; smf = s->priv; - free(smf->s.ptr); - free(smf); + free_smf(smf); } struct stevedore smf_stevedore = { From phk at projects.linpro.no Wed Jun 14 07:21:48 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 14 Jun 2006 09:21:48 +0200 (CEST) Subject: r180 - trunk/varnish-cache/bin/varnishd Message-ID: <20060614072148.8A9231EC2F9@projects.linpro.no> Author: phk Date: 2006-06-14 09:21:48 +0200 (Wed, 14 Jun 2006) New Revision: 180 Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/stevedore.h trunk/varnish-cache/bin/varnishd/storage_file.c Log: Give storage backends a "send" method. Let storage_file use sendfile(2) for it. Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-14 06:58:55 UTC (rev 179) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-14 07:21:48 UTC (rev 180) @@ -78,8 +78,12 @@ "\r\n", sp->obj->len); vca_write(sp, buf, strlen(buf)); - TAILQ_FOREACH(st, &sp->obj->store, list) - vca_write(sp, st->ptr, st->len); + TAILQ_FOREACH(st, &sp->obj->store, list) { + if (st->stevedore->send != NULL) + st->stevedore->send(st, sp); + else + vca_write(sp, st->ptr, st->len); + } vca_flush(sp); return (1); } Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2006-06-14 06:58:55 UTC (rev 179) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2006-06-14 07:21:48 UTC (rev 180) @@ -3,11 +3,13 @@ */ struct stevedore; +struct sess; typedef void storage_init_f(struct stevedore *, const char *spec); typedef void storage_open_f(struct stevedore *); typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); typedef void storage_free_f(struct storage *); +typedef void storage_send_f(struct storage *, struct sess *); struct stevedore { const char *name; @@ -15,6 +17,7 @@ storage_open_f *open; /* called by cache process */ storage_alloc_f *alloc; storage_free_f *free; + storage_send_f *send; /* private fields */ void *priv; Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-14 06:58:55 UTC (rev 179) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-14 07:21:48 UTC (rev 180) @@ -19,6 +19,7 @@ #include #include #include +#include #include "vcl_lang.h" #include "libvarnish.h" @@ -409,7 +410,7 @@ if (sz < *fail && sz < SIZE_T_MAX) { p = mmap(NULL, sz, PROT_READ|PROT_WRITE, - MAP_NOCORE | MAP_NOSYNC | MAP_PRIVATE, sc->fd, off); + MAP_NOCORE | MAP_NOSYNC | MAP_SHARED, sc->fd, off); if (p != MAP_FAILED) { (*sum) += sz; new_smf(sc, p, off, sz); @@ -459,9 +460,12 @@ smf->s.priv = smf; smf->s.ptr = smf->ptr; smf->s.len = size; + smf->s.stevedore = st; return (&smf->s); } +/*--------------------------------------------------------------------*/ + static void smf_free(struct storage *s) { @@ -471,10 +475,34 @@ free_smf(smf); } +/*--------------------------------------------------------------------*/ + +static void +smf_send(struct storage *st, struct sess *sp) +{ + struct smf *smf; + int i; + off_t sent; + + smf = st->priv; + + printf("SEND %12p %12p %12jx %12jx\n", (void*)smf, (void*)smf->ptr, (uintmax_t)smf->offset, (uintmax_t)smf->size); + vca_flush(sp); + i = sendfile(smf->sc->fd, + sp->fd, + smf->offset, + st->len, NULL, &sent, 0); + printf("sent i=%d sent=%ju size=%ju\n", + i, (uintmax_t)sent, (uintmax_t)st->len); +} + +/*--------------------------------------------------------------------*/ + struct stevedore smf_stevedore = { "file", smf_init, smf_open, smf_alloc, - smf_free + smf_free, + smf_send }; From phk at projects.linpro.no Wed Jun 14 08:53:21 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 14 Jun 2006 10:53:21 +0200 (CEST) Subject: r181 - trunk/varnish-cache/bin/varnishd Message-ID: <20060614085321.9B8781EC236@projects.linpro.no> Author: phk Date: 2006-06-14 10:53:21 +0200 (Wed, 14 Jun 2006) New Revision: 181 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_main.c Log: Initialize the cache_acceptor.c/VCA in the same manner as other parts. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-14 07:21:48 UTC (rev 180) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-14 08:53:21 UTC (rev 181) @@ -60,9 +60,9 @@ /* cache_acceptor.c */ void vca_write(struct sess *sp, void *ptr, size_t len); void vca_flush(struct sess *sp); -void *vca_main(void *arg); void vca_retire_session(struct sess *sp); void vca_recycle_session(struct sess *sp); +void VCA_Init(void); /* cache_backend.c */ void VBE_Init(void); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-14 07:21:48 UTC (rev 180) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-14 08:53:21 UTC (rev 181) @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -33,6 +34,8 @@ static struct event pipe_e; static int pipes[2]; +static pthread_t vca_thread; + #define SESS_IOVS 5 static struct event accept_e[2 * HERITAGE_NSOCKS]; @@ -137,7 +140,7 @@ http_RecvHead(sp->http, sp->fd, evb, DealWithSession, sp); } -void * +static void * vca_main(void *arg) { unsigned u; @@ -194,3 +197,12 @@ } free(sp); } + +/*--------------------------------------------------------------------*/ + +void +VCA_Init(void) +{ + + AZ(pthread_create(&vca_thread, NULL, vca_main, NULL)); +} Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-14 07:21:48 UTC (rev 180) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-14 08:53:21 UTC (rev 181) @@ -24,7 +24,6 @@ #include "cli_event.h" static struct event ev_keepalive; -static pthread_t vca_thread; struct hash_slinger *hash; struct stevedore *stevedore; @@ -116,7 +115,7 @@ VSL_Init(); CacheInitPool(); - AZ(pthread_create(&vca_thread, NULL, vca_main, NULL)); + VCA_Init(); eb = event_init(); assert(eb != NULL); From phk at projects.linpro.no Wed Jun 14 09:03:45 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 14 Jun 2006 11:03:45 +0200 (CEST) Subject: r182 - trunk/varnish-cache/bin/varnishd Message-ID: <20060614090345.B8E351EC236@projects.linpro.no> Author: phk Date: 2006-06-14 11:03:45 +0200 (Wed, 14 Jun 2006) New Revision: 182 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Sanitize close/recycle session logic a bit. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-14 08:53:21 UTC (rev 181) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-14 09:03:45 UTC (rev 182) @@ -60,8 +60,8 @@ /* cache_acceptor.c */ void vca_write(struct sess *sp, void *ptr, size_t len); void vca_flush(struct sess *sp); -void vca_retire_session(struct sess *sp); -void vca_recycle_session(struct sess *sp); +void vca_return_session(struct sess *sp); +void vca_close_session(struct sess *sp); void VCA_Init(void); /* cache_backend.c */ Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-14 08:53:21 UTC (rev 181) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-14 09:03:45 UTC (rev 182) @@ -66,8 +66,7 @@ if (i != sp->mem->liov) { VSL(SLT_SessionClose, sp->fd, "Premature %d of %d", i, sp->mem->liov); - close(sp->fd); - sp->fd = -1; + vca_close_session(sp); } sp->mem->liov = 0; sp->mem->niov = 0; @@ -178,24 +177,31 @@ return ("FOOBAR"); } +/*--------------------------------------------------------------------*/ + void -vca_recycle_session(struct sess *sp) +vca_close_session(struct sess *sp) { - VSL(SLT_SessionReuse, sp->fd, "%s", sp->addr); - write(pipes[1], &sp, sizeof sp); + + VSL(SLT_SessionClose, sp->fd, "%s", sp->addr); + close(sp->fd); + sp->fd = -1; } +/*--------------------------------------------------------------------*/ + void -vca_retire_session(struct sess *sp) +vca_return_session(struct sess *sp) { - if (sp->http != NULL) - http_Delete(sp->http); if (sp->fd >= 0) { - VSL(SLT_SessionClose, sp->fd, "%s", sp->addr); - close(sp->fd); + VSL(SLT_SessionReuse, sp->fd, "%s", sp->addr); + write(pipes[1], &sp, sizeof sp); + } else { + if (sp->http != NULL) + http_Delete(sp->http); + free(sp); } - free(sp); } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-06-14 08:53:21 UTC (rev 181) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-06-14 09:03:45 UTC (rev 182) @@ -75,7 +75,6 @@ event_add(&e2.ev, NULL); event_base_loop(w->eb, 0); close (fd); - close (sp->fd); + vca_close_session(sp); VBE_ClosedFd(fd_token); - sp->fd = -1; } Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-14 08:53:21 UTC (rev 181) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-14 09:03:45 UTC (rev 182) @@ -159,17 +159,13 @@ if (http_GetHdr(sp->http, "Connection", &b) && !strcmp(b, "close")) { VSL(SLT_SessionClose, sp->fd, "hdr"); - close(sp->fd); - sp->fd = -1; + vca_close_session(sp); } AZ(pthread_mutex_lock(&sessmtx)); RelVCL(sp->vcl); sp->vcl = NULL; - if (sp->fd < 0) - vca_retire_session(sp); - else - vca_recycle_session(sp); + vca_return_session(sp); } } @@ -179,7 +175,8 @@ struct sess *sp = arg; if (!good) { - vca_retire_session(sp); + vca_close_session(sp); + vca_return_session(sp); return; } AZ(pthread_mutex_lock(&sessmtx)); From phk at projects.linpro.no Wed Jun 14 09:23:50 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 14 Jun 2006 11:23:50 +0200 (CEST) Subject: r183 - trunk/varnish-cache/include Message-ID: <20060614092350.2D0E51EC2FA@projects.linpro.no> Author: phk Date: 2006-06-14 11:23:50 +0200 (Wed, 14 Jun 2006) New Revision: 183 Modified: trunk/varnish-cache/include/shmlog_tags.h Log: Add HttpError tag Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-06-14 09:03:45 UTC (rev 182) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-06-14 09:23:50 UTC (rev 183) @@ -14,6 +14,7 @@ SLTM(BackendOpen) SLTM(BackendReuse) SLTM(BackendClose) +SLTM(HttpError) SLTM(ClientAddr) SLTM(Handling) SLTM(Request) From phk at projects.linpro.no Wed Jun 14 09:25:13 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 14 Jun 2006 11:25:13 +0200 (CEST) Subject: r184 - trunk/varnish-cache/bin/varnishd Message-ID: <20060614092513.C93CC1EC2FA@projects.linpro.no> Author: phk Date: 2006-06-14 11:25:13 +0200 (Wed, 14 Jun 2006) New Revision: 184 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Clean up session messages a bit Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-14 09:23:50 UTC (rev 183) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-14 09:25:13 UTC (rev 184) @@ -61,7 +61,7 @@ void vca_write(struct sess *sp, void *ptr, size_t len); void vca_flush(struct sess *sp); void vca_return_session(struct sess *sp); -void vca_close_session(struct sess *sp); +void vca_close_session(struct sess *sp, const char *why); void VCA_Init(void); /* cache_backend.c */ Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-14 09:23:50 UTC (rev 183) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-14 09:25:13 UTC (rev 184) @@ -63,11 +63,8 @@ if (sp->fd < 0 || sp->mem->niov == 0) return; i = writev(sp->fd, sp->mem->iov, sp->mem->niov); - if (i != sp->mem->liov) { - VSL(SLT_SessionClose, sp->fd, "Premature %d of %d", - i, sp->mem->liov); - vca_close_session(sp); - } + if (i != sp->mem->liov) + vca_close_session(sp, "remote closed"); sp->mem->liov = 0; sp->mem->niov = 0; } @@ -180,10 +177,10 @@ /*--------------------------------------------------------------------*/ void -vca_close_session(struct sess *sp) +vca_close_session(struct sess *sp, const char *why) { - VSL(SLT_SessionClose, sp->fd, "%s", sp->addr); + VSL(SLT_SessionClose, sp->fd, why); close(sp->fd); sp->fd = -1; } Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-14 09:23:50 UTC (rev 183) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-14 09:25:13 UTC (rev 184) @@ -255,10 +255,13 @@ i = read(fd, hp->v, hp->e - hp->v); if (i <= 0) { if (hp->v != hp->s) - VSL(SLT_SessionClose, fd, - "remote had %d bytes errno %d", hp->v - hp->s, errno); + VSL(SLT_HttpError, fd, + "Received (only) %d bytes, errno %d", + hp->v - hp->s, errno); + else if (errno == 0) + VSL(SLT_HttpError, fd, "Received nothing"); else - VSL(SLT_SessionClose, fd, "remote errno %d", errno); + VSL(SLT_HttpError, fd, "Received errno %d", errno); hp->t = NULL; event_del(&hp->ev); if (hp->callback != NULL) Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-06-14 09:23:50 UTC (rev 183) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-06-14 09:25:13 UTC (rev 184) @@ -75,6 +75,6 @@ event_add(&e2.ev, NULL); event_base_loop(w->eb, 0); close (fd); - vca_close_session(sp); + vca_close_session(sp, "pipe"); VBE_ClosedFd(fd_token); } Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-14 09:23:50 UTC (rev 183) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-14 09:25:13 UTC (rev 184) @@ -158,8 +158,7 @@ } if (http_GetHdr(sp->http, "Connection", &b) && !strcmp(b, "close")) { - VSL(SLT_SessionClose, sp->fd, "hdr"); - vca_close_session(sp); + vca_close_session(sp, "Connection header"); } AZ(pthread_mutex_lock(&sessmtx)); @@ -175,7 +174,7 @@ struct sess *sp = arg; if (!good) { - vca_close_session(sp); + vca_close_session(sp, "no request"); vca_return_session(sp); return; } From phk at projects.linpro.no Wed Jun 14 09:39:31 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 14 Jun 2006 11:39:31 +0200 (CEST) Subject: r185 - trunk/varnish-cache/bin/varnishd Message-ID: <20060614093931.5C73B1EC2FA@projects.linpro.no> Author: phk Date: 2006-06-14 11:39:31 +0200 (Wed, 14 Jun 2006) New Revision: 185 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Add vca_write_obj() which writes an sbuf (as) HTTP header and the object from the sessions to the client. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-14 09:25:13 UTC (rev 184) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-14 09:39:31 UTC (rev 185) @@ -59,6 +59,7 @@ /* cache_acceptor.c */ void vca_write(struct sess *sp, void *ptr, size_t len); +void vca_write_obj(struct sess *sp, struct sbuf *hdr); void vca_flush(struct sess *sp); void vca_return_session(struct sess *sp); void vca_close_session(struct sess *sp, const char *why); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-14 09:25:13 UTC (rev 184) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-14 09:39:31 UTC (rev 185) @@ -84,6 +84,21 @@ sp->mem->liov += len; } +void +vca_write_obj(struct sess *sp, struct sbuf *hdr) +{ + struct storage *st; + + vca_write(sp, sbuf_data(hdr), sbuf_len(hdr)); + TAILQ_FOREACH(st, &sp->obj->store, list) { + if (st->stevedore->send != NULL) + st->stevedore->send(st, sp); + else + vca_write(sp, st->ptr, st->len); + } + vca_flush(sp); +} + /*--------------------------------------------------------------------*/ static void Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-14 09:25:13 UTC (rev 184) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-14 09:39:31 UTC (rev 185) @@ -60,10 +60,9 @@ } http_BuildSbuf(2, w->sb, hp); - vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); - vca_write(sp, st->ptr, st->len); - vca_flush(sp); + vca_write_obj(sp, w->sb); + hash->deref(sp->obj); return (0); @@ -126,11 +125,8 @@ } http_BuildSbuf(2, w->sb, hp); - vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); - TAILQ_FOREACH(st, &sp->obj->store, list) - vca_write(sp, st->ptr, st->len); - vca_flush(sp); + vca_write_obj(sp, w->sb); hash->deref(sp->obj); Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-14 09:25:13 UTC (rev 184) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-14 09:39:31 UTC (rev 185) @@ -68,23 +68,15 @@ static int DeliverSession(struct worker *w, struct sess *sp) { - char buf[BUFSIZ]; - struct storage *st; - sprintf(buf, + sbuf_clear(w->sb); + sbuf_printf(w->sb, "HTTP/1.1 200 OK\r\n" "Server: Varnish\r\n" "Content-Length: %u\r\n" "\r\n", sp->obj->len); - vca_write(sp, buf, strlen(buf)); - TAILQ_FOREACH(st, &sp->obj->store, list) { - if (st->stevedore->send != NULL) - st->stevedore->send(st, sp); - else - vca_write(sp, st->ptr, st->len); - } - vca_flush(sp); + vca_write_obj(sp, w->sb); return (1); } From phk at projects.linpro.no Wed Jun 14 09:57:19 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 14 Jun 2006 11:57:19 +0200 (CEST) Subject: r186 - trunk/varnish-cache/bin/varnishd Message-ID: <20060614095719.8D06C1EC2FA@projects.linpro.no> Author: phk Date: 2006-06-14 11:57:19 +0200 (Wed, 14 Jun 2006) New Revision: 186 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_vcl.c Log: start examining HTTP status codes from backend Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-14 09:39:31 UTC (rev 185) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-14 09:57:19 UTC (rev 186) @@ -81,6 +81,7 @@ struct http *http_New(void); void http_Delete(struct http *hp); int http_GetHdr(struct http *hp, const char *hdr, char **ptr); +int http_GetStatus(struct http *hp); int http_HdrIs(struct http *hp, const char *hdr, const char *val); int http_GetTail(struct http *hp, unsigned len, char **b, char **e); int http_GetURL(struct http *hp, char **b); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-14 09:39:31 UTC (rev 185) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-14 09:57:19 UTC (rev 186) @@ -161,9 +161,19 @@ event_base_loop(w->eb, 0); http_Dissect(hp, fd, 2); - /* XXX: fill in object from headers */ - sp->obj->valid = 1; - sp->obj->cacheable = 1; + switch (http_GetStatus(hp)) { + case 200: + /* XXX: fill in object from headers */ + sp->obj->valid = 1; + sp->obj->cacheable = 1; + break; + case 301: + sp->obj->valid = 0; + sp->obj->cacheable = 0; + break; + default: + break; + } sp->handling = HND_Insert; sp->vcl->fetch_func(sp); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-14 09:39:31 UTC (rev 185) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-14 09:57:19 UTC (rev 186) @@ -143,6 +143,13 @@ return (1); } +int +http_GetStatus(struct http *hp) +{ + + return (strtoul(hp->status, NULL /* XXX */, 10)); +} + /*--------------------------------------------------------------------*/ void Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-14 09:39:31 UTC (rev 185) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-14 09:57:19 UTC (rev 186) @@ -205,7 +205,11 @@ sess->done++; } -void VCL_error(VCL_FARGS, unsigned err, const char *str) { +void +VCL_error(VCL_FARGS, unsigned err, const char *str) +{ + + VSL(SLT_Debug, 0, "VCL_error(%u, %s)", err, str); } void From phk at projects.linpro.no Thu Jun 15 08:04:19 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 15 Jun 2006 10:04:19 +0200 (CEST) Subject: r187 - trunk/varnish-cache/bin/varnishd Message-ID: <20060615080419.DAD011EC1D1@projects.linpro.no> Author: phk Date: 2006-06-15 10:04:19 +0200 (Thu, 15 Jun 2006) New Revision: 187 Modified: trunk/varnish-cache/bin/varnishd/cache_main.c Log: Less noise Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-14 09:57:19 UTC (rev 186) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-15 08:04:19 UTC (rev 187) @@ -76,7 +76,6 @@ arm_keepalive(); if (av[2] != NULL) { /* XXX: check clock skew is pointless here */ - printf("Got your ping %s\n", av[2]); } time(&t); cli_out(cli, "PONG %ld\n", t); From phk at projects.linpro.no Thu Jun 15 08:04:46 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 15 Jun 2006 10:04:46 +0200 (CEST) Subject: r188 - trunk/varnish-cache/bin/varnishd Message-ID: <20060615080446.306981EC1FC@projects.linpro.no> Author: phk Date: 2006-06-15 10:04:46 +0200 (Thu, 15 Jun 2006) New Revision: 188 Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c Log: less noise Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_child.c 2006-06-15 08:04:19 UTC (rev 187) +++ trunk/varnish-cache/bin/varnishd/mgt_child.c 2006-06-15 08:04:46 UTC (rev 188) @@ -99,7 +99,8 @@ cr = TAILQ_FIRST(&creqhead); if (cr == NULL) return; - printf("Send Request <%s>\n", cr->req); + if (0) + printf("Send Request <%s>\n", cr->req); evbuffer_add_printf(child_cli1->output, "%s", cr->req); for (u = 0; cr->argv != NULL && cr->argv[u] != NULL; u++) { evbuffer_add_printf(child_cli1->output, " "); @@ -173,7 +174,6 @@ static void child_pingpong_ccb(unsigned u, const char *r, void *priv) { - printf("%s(%u, \"%s\", %p)\n", __func__, u, r, priv); /* XXX: reset keepalive timer */ } @@ -184,7 +184,6 @@ time_t t; struct timeval tv; - printf("%s(%d, %d, %p)\n", __func__, a, b, c); time(&t); mgt_child_request(child_pingpong_ccb, NULL, NULL, "ping %ld", t); if (1) { From phk at projects.linpro.no Fri Jun 16 10:16:00 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 16 Jun 2006 12:16:00 +0200 (CEST) Subject: r189 - in trunk/varnish-cache: include lib/libvarnish Message-ID: <20060616101600.41C5D1EC247@projects.linpro.no> Author: phk Date: 2006-06-16 12:16:00 +0200 (Fri, 16 Jun 2006) New Revision: 189 Added: trunk/varnish-cache/lib/libvarnish/time.c Modified: trunk/varnish-cache/include/libvarnish.h trunk/varnish-cache/lib/libvarnish/Makefile.am Log: Add time parse/format functions to libvarnish Modified: trunk/varnish-cache/include/libvarnish.h =================================================================== --- trunk/varnish-cache/include/libvarnish.h 2006-06-15 08:04:46 UTC (rev 188) +++ trunk/varnish-cache/include/libvarnish.h 2006-06-16 10:16:00 UTC (rev 189) @@ -6,6 +6,11 @@ void FreeArgv(char **argv); char **ParseArgv(const char *s, int comment); +#ifdef CLOCK_MONOTONIC +/* from libvarnish/time.c */ +void TIM_format(time_t t, char *p); +time_t TIM_parse(const char *p); +#endif /* Assert zero return value */ #define AZ(foo) do { assert((foo) == 0); } while (0) Modified: trunk/varnish-cache/lib/libvarnish/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvarnish/Makefile.am 2006-06-15 08:04:46 UTC (rev 188) +++ trunk/varnish-cache/lib/libvarnish/Makefile.am 2006-06-16 10:16:00 UTC (rev 189) @@ -6,4 +6,5 @@ libvarnish_la_SOURCES = \ argv.c \ - cli.c + cli.c \ + time.c Added: trunk/varnish-cache/lib/libvarnish/time.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/time.c 2006-06-15 08:04:46 UTC (rev 188) +++ trunk/varnish-cache/lib/libvarnish/time.c 2006-06-16 10:16:00 UTC (rev 189) @@ -0,0 +1,76 @@ +/* + * $Id$ + * + * Semi-trivial functions to handle HTTP header timestamps according to + * RFC 2616 section 3.3. + * + * In the highly unlikely event of performance trouble, handbuilt versions + * would likely be faster than relying on the OS time functions. + * + * We must parse three different formats: + * 000000000011111111112222222222 + * 012345678901234567890123456789 + * ------------------------------ + * "Sun, 06 Nov 1994 08:49:37 GMT" RFC822 & RFC1123 + * "Sunday, 06-Nov-94 08:49:37 GMT" RFC850 + * "Sun Nov 6 08:49:37 1994" ANSI-C asctime() + * + * And always output the RFC1123 format. + * + */ + +#include +#include +#include + +void +TIM_format(time_t t, char *p) +{ + struct tm tm; + + gmtime_r(&t, &tm); + strftime(p, 30, "%a, %d %b %Y %T GMT", &tm); +} + +/* XXX: add statistics ? */ +static const char *fmts[] = { + "%a, %d %b %Y %T GMT", /* RFC 822 & RFC1123 */ + "%A, %d-%b-%y %T GMT", /* RFC850 */ + "%a %b %d %T %Y", /* ANSI-C asctime() */ + NULL +}; + +time_t +TIM_parse(const char *p) +{ + struct tm tm; + const char **r; + + for (r = fmts; *r != NULL; r++) { + memset(&tm, 0, sizeof tm); + if (strptime(p, *r, &tm) != NULL) + return(timegm(&tm)); + } + return (0); +} + +#ifdef TEST_DRIVER +int +main(int argc, char **argv) +{ + time_t t; + char buf[BUFSIZ]; + + time(&t); + memset(buf, 0x55, sizeof buf); + TIM_format(t, buf); + printf("scan = %d <%s>\n", TIM_parse(buf), buf); + + /* Examples from RFC2616 section 3.3.1 */ + printf("scan = %d\n", TIM_parse("Sun, 06 Nov 1994 08:49:37 GMT")); + printf("scan = %d\n", TIM_parse("Sunday, 06-Nov-94 08:49:37 GMT")); + printf("scan = %d\n", TIM_parse("Sun Nov 6 08:49:37 1994")); + + return (0); +} +#endif From phk at projects.linpro.no Fri Jun 16 10:17:25 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 16 Jun 2006 12:17:25 +0200 (CEST) Subject: r190 - trunk/varnish-cache/bin/varnishd Message-ID: <20060616101725.3D57F1EC241@projects.linpro.no> Author: phk Date: 2006-06-16 12:17:25 +0200 (Fri, 16 Jun 2006) New Revision: 190 Modified: trunk/varnish-cache/bin/varnishd/stevedore.h trunk/varnish-cache/bin/varnishd/storage_file.c trunk/varnish-cache/bin/varnishd/storage_malloc.c Log: Add trim method to storage backends so chunked encoding can be stored efficiently. Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2006-06-16 10:16:00 UTC (rev 189) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2006-06-16 10:17:25 UTC (rev 190) @@ -8,6 +8,7 @@ typedef void storage_init_f(struct stevedore *, const char *spec); typedef void storage_open_f(struct stevedore *); typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); +typedef void storage_trim_f(struct storage *, size_t size); typedef void storage_free_f(struct storage *); typedef void storage_send_f(struct storage *, struct sess *); @@ -16,6 +17,7 @@ storage_init_f *init; /* called by mgt process */ storage_open_f *open; /* called by cache process */ storage_alloc_f *alloc; + storage_trim_f *trim; storage_free_f *free; storage_send_f *send; Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-16 10:16:00 UTC (rev 189) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-16 10:17:25 UTC (rev 190) @@ -268,8 +268,6 @@ { struct smf *sp, *sp2; - bytes += (sc->pagesize - 1); - bytes &= ~(sc->pagesize - 1); TAILQ_FOREACH(sp, &sc->free, status) { if (sp->size >= bytes) break; @@ -454,12 +452,16 @@ smf_alloc(struct stevedore *st, unsigned size) { struct smf *smf; + struct smf_sc *sc = st->priv; - smf = alloc_smf(st->priv, size); + size += (sc->pagesize - 1); + size &= ~(sc->pagesize - 1); + smf = alloc_smf(sc, size); assert(smf != NULL); + smf->s.space = size; smf->s.priv = smf; smf->s.ptr = smf->ptr; - smf->s.len = size; + smf->s.len = 0; smf->s.stevedore = st; return (&smf->s); } @@ -467,6 +469,15 @@ /*--------------------------------------------------------------------*/ static void +smf_trim(struct storage *s, size_t size) +{ + + /* XXX: implement */ +} + +/*--------------------------------------------------------------------*/ + +static void smf_free(struct storage *s) { struct smf *smf; @@ -503,6 +514,7 @@ smf_init, smf_open, smf_alloc, + smf_trim, smf_free, smf_send }; Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-06-16 10:16:00 UTC (rev 189) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-06-16 10:17:25 UTC (rev 190) @@ -27,6 +27,7 @@ sma->s.ptr = malloc(size); assert(sma->s.ptr != NULL); sma->s.len = size; + sma->s.space = size; return (&sma->s); } @@ -45,5 +46,6 @@ NULL, /* init */ NULL, /* open */ sma_alloc, + NULL, /* trim */ sma_free }; From phk at projects.linpro.no Fri Jun 16 10:18:08 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 16 Jun 2006 12:18:08 +0200 (CEST) Subject: r191 - trunk/varnish-cache/bin/varnishd Message-ID: <20060616101808.00FD41EC1F9@projects.linpro.no> Author: phk Date: 2006-06-16 12:18:08 +0200 (Fri, 16 Jun 2006) New Revision: 191 Added: trunk/varnish-cache/bin/varnishd/rfc2616.c Modified: trunk/varnish-cache/bin/varnishd/Makefile.am Log: The beginnings of rfc2616 policy implemenation. Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-06-16 10:17:25 UTC (rev 190) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-06-16 10:18:08 UTC (rev 191) @@ -18,6 +18,7 @@ cli_event.c \ hash_simple_list.c \ mgt_child.c \ + rfc2616.c \ storage_file.c \ storage_malloc.c \ tcp.c \ Added: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-16 10:17:25 UTC (rev 190) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-16 10:18:08 UTC (rev 191) @@ -0,0 +1,76 @@ +/* + * $Id$ + */ + +#include +#include +#include +#include "vcl_lang.h" + +#include "cache.h" +#include "libvarnish.h" + +/*-------------------------------------------------------------------- + * From RFC2616, 13.2.3 Age Calculations + * + * age_value + * is the value of Age: header received by the cache with + * this response. + * date_value + * is the value of the origin server's Date: header + * request_time + * is the (local) time when the cache made the request + * that resulted in this cached response + * response_time + * is the (local) time when the cache received the response + * now + * is the current (local) time + * + * apparent_age = max(0, response_time - date_value); + * corrected_received_age = max(apparent_age, age_value); + * response_delay = response_time - request_time; + * corrected_initial_age = corrected_received_age + response_delay; + * resident_time = now - response_time; + * current_age = corrected_initial_age + resident_time; + * + */ + +void +RFC2616_Age(struct http *hp, time_t t_req, time_t t_resp) +{ + time_t h_date = 0, h_expires = 0, h_age = 0; + time_t apparent_age = 0, corrected_received_age; + time_t response_delay, corrected_initial_age; + time_t max_age = -1; + char *p; + + if (http_GetHdrField(hp, "Cache-Control", "max-age", &p)) + max_age = strtoul(p, NULL, 0); + + if (http_GetHdr(hp, "Date", &p)) + h_date = TIM_parse(p); + + if (h_date < t_resp) + apparent_age = t_resp - h_date; + + if (http_GetHdr(hp, "Age", &p)) + h_age = strtoul(p, NULL, 0); + + if (h_age > apparent_age) + corrected_received_age = h_age; + else + corrected_received_age = apparent_age; + + response_delay = t_resp - t_req; + corrected_initial_age = corrected_received_age + response_delay; + + if (http_GetHdr(hp, "Expires", &p)) + h_expires = TIM_parse(p); + + printf("Date: %d\n", h_date); + printf("Expires: %d\n", h_expires); + printf("Age: %d\n", h_age); + printf("CIAge: %d\n", corrected_initial_age); + + +} From phk at projects.linpro.no Fri Jun 16 10:19:44 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 16 Jun 2006 12:19:44 +0200 (CEST) Subject: r192 - trunk/varnish-cache/include Message-ID: <20060616101944.C62E61EC241@projects.linpro.no> Author: phk Date: 2006-06-16 12:19:44 +0200 (Fri, 16 Jun 2006) New Revision: 192 Modified: trunk/varnish-cache/include/http_headers.h Log: Supress Transfer-Encoding Modified: trunk/varnish-cache/include/http_headers.h =================================================================== --- trunk/varnish-cache/include/http_headers.h 2006-06-16 10:18:08 UTC (rev 191) +++ trunk/varnish-cache/include/http_headers.h 2006-06-16 10:19:44 UTC (rev 192) @@ -55,7 +55,7 @@ HTTPH("Server", H_Server, 2, 0, 0, 0, 0) /* RFC2616 14.38 */ HTTPH("TE", H_TE, 1, 0, 0, 0, 0) /* RFC2616 14.39 */ HTTPH("Trailer", H_Trailer, 1, 0, 0, 0, 0) /* RFC2616 14.40 */ -HTTPH("Transfer-Encoding", H_Transfer_Encoding, 2, 0, 0, 0, 0) /* RFC2616 14.41 */ +HTTPH("Transfer-Encoding", H_Transfer_Encoding, 2, 3, 0, 0, 0) /* RFC2616 14.41 */ HTTPH("Upgrade", H_Upgrade, 2, 0, 0, 0, 0) /* RFC2616 14.42 */ HTTPH("User-Agent", H_User_Agent, 1, 0, 0, 0, 0) /* RFC2616 14.43 */ HTTPH("Vary", H_Vary, 2, 0, 0, 0, 0) /* RFC2616 14.44 */ From phk at projects.linpro.no Fri Jun 16 10:20:12 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 16 Jun 2006 12:20:12 +0200 (CEST) Subject: r193 - in trunk/varnish-cache: include lib/libvcl Message-ID: <20060616102012.D723A1EC1F9@projects.linpro.no> Author: phk Date: 2006-06-16 12:20:12 +0200 (Fri, 16 Jun 2006) New Revision: 193 Modified: trunk/varnish-cache/include/vcl_lang.h trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c Log: Add header field to object Modified: trunk/varnish-cache/include/vcl_lang.h =================================================================== --- trunk/varnish-cache/include/vcl_lang.h 2006-06-16 10:19:44 UTC (rev 192) +++ trunk/varnish-cache/include/vcl_lang.h 2006-06-16 10:20:12 UTC (rev 193) @@ -34,6 +34,8 @@ unsigned busy; unsigned len; + char *header; + TAILQ_HEAD(, storage) store; }; Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-16 10:19:44 UTC (rev 192) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-16 10:20:12 UTC (rev 193) @@ -422,6 +422,8 @@ fputs(" unsigned busy;\n", f); fputs(" unsigned len;\n", f); fputs("\n", f); + fputs(" char *header;\n", f); + fputs("\n", f); fputs(" TAILQ_HEAD(, storage) store;\n", f); fputs("};\n", f); fputs("\n", f); From phk at projects.linpro.no Fri Jun 16 10:22:40 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 16 Jun 2006 12:22:40 +0200 (CEST) Subject: r194 - trunk/varnish-cache/bin/varnishd Message-ID: <20060616102240.542061EC241@projects.linpro.no> Author: phk Date: 2006-06-16 12:22:40 +0200 (Fri, 16 Jun 2006) New Revision: 194 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Initial http_GetHdrField() function. Improve chunked encoding, allocate big storage chunks and trim the last one at the end, instead of one storage chunk for each chunk the remote server sends. Call RFC2616 policy code. Store headers from backend in cache and return to client. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-16 10:20:12 UTC (rev 193) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-16 10:22:40 UTC (rev 194) @@ -41,6 +41,7 @@ TAILQ_ENTRY(storage) list; unsigned char *ptr; unsigned len; + unsigned space; void *priv; struct stevedore *stevedore; }; @@ -81,6 +82,7 @@ struct http *http_New(void); void http_Delete(struct http *hp); int http_GetHdr(struct http *hp, const char *hdr, char **ptr); +int http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr); int http_GetStatus(struct http *hp); int http_HdrIs(struct http *hp, const char *hdr, const char *val); int http_GetTail(struct http *hp, unsigned len, char **b, char **e); @@ -124,3 +126,7 @@ cli_func_t cli_func_config_unload; cli_func_t cli_func_config_use; #endif + +/* rfc2616.c */ +void RFC2616_Age(struct http *hp, time_t, time_t); + Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-16 10:20:12 UTC (rev 193) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-16 10:22:40 UTC (rev 194) @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -22,6 +23,17 @@ #include "vcl_lang.h" #include "cache.h" +/* + * Chunked encoding is a hack. We prefer to have a single or a few + * large storage objects, and a terribly long list of small ones. + * If our stevedore can trim, we alloc big chunks and trim the last one + * at the end know the result. + * + * Good testcase: http://www.washingtonpost.com/ + */ +#define CHUNK_PREALLOC (128 * 1024) + +/*--------------------------------------------------------------------*/ static int fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char *b) { @@ -45,7 +57,6 @@ if (http_GetTail(hp, cl, &b, &e)) { i = e - b; - VSL(SLT_Debug, 0, "Fetch_Tail %jd %d", cl, i); memcpy(p, b, i); p += i; cl -= i; @@ -53,7 +64,6 @@ while (cl != 0) { i = read(fd, p, cl); - VSL(SLT_Debug, 0, "Fetch_Read %jd %d", cl, i); assert(i > 0); p += i; cl -= i; @@ -68,6 +78,9 @@ } +/*--------------------------------------------------------------------*/ +/* XXX: Cleanup. It must be possible somehow :-( */ + static int fetch_chunked(struct worker *w, struct sess *sp, int fd, struct http *hp) { @@ -75,7 +88,7 @@ char *b, *q, *e; unsigned char *p; struct storage *st; - unsigned u; + unsigned u, v; char buf[20]; char *bp, *be; @@ -84,46 +97,92 @@ i = fcntl(fd, F_SETFL, i); be = buf + sizeof buf; + bp = buf; + st = NULL; while (1) { - bp = buf; if (http_GetTail(hp, be - bp, &b, &e)) { memcpy(bp, b, e - b); bp += e - b; + *bp = '\0'; } else { i = read(fd, bp, be - bp); assert(i >= 0); bp += i; + *bp = '\0'; } u = strtoul(buf, &q, 16); - if (q == NULL || (*q != '\n' && *q != '\r')) + if (q == NULL || q == buf) continue; + assert(isspace(*q)); + while (*q == '\t' || *q == ' ') + q++; if (*q == '\r') q++; assert(*q == '\n'); q++; if (u == 0) break; - st = stevedore->alloc(stevedore, u); - TAILQ_INSERT_TAIL(&sp->obj->store, st, list); - st->len = u; sp->obj->len += u; - p = st->ptr; - memcpy(p, q, bp - q); - p += bp - q; - u -= bp - q; - if (http_GetTail(hp, u, &b, &e)) { - memcpy(p, b, e - b); - p += e - b; - u -= e - b; - } + while (u > 0) { - i = read(fd, p, u); - assert(i > 0); - u -= i; - p += i; + if (st != NULL && st->len < st->space) { + p = st->ptr + st->len; + } else { + st = stevedore->alloc(stevedore, + stevedore->trim == NULL ? u : CHUNK_PREALLOC); + TAILQ_INSERT_TAIL(&sp->obj->store, st, list); + p = st->ptr; + } + v = st->space - st->len; + if (v > u) + v = u; + + i = bp - q; + if (i == 0) { + } else if (v > i) { + memcpy(p, q, i); + p += i; + st->len += i; + u -= i; + v -= i; + q = bp = buf; + } else if (i >= v) { + memcpy(p, q, v); + p += v; + st->len += i; + q += v; + u -= v; + v -= v; + if (u == 0 && bp > q) { + memcpy(buf, q, bp - q); + q = bp = buf + (bp - q); + } + } + if (u == 0) + break; + if (v == 0) + continue; + if (http_GetTail(hp, v, &b, &e)) { + memcpy(p, b, e - b); + p += e - b; + st->len += e - b; + v -= e - b; + u -= e - b; + } + while (v > 0) { + i = read(fd, p, v); + assert(i > 0); + st->len += i; + v -= i; + u -= i; + p += i; + } } } + if (st != NULL && stevedore->trim != NULL) + stevedore->trim(st, st->len); + http_BuildSbuf(2, w->sb, hp); vca_write_obj(sp, w->sb); @@ -141,6 +200,7 @@ void *fd_token; struct http *hp; char *b; + time_t t_req, t_resp; fd = VBE_GetFd(sp->backend, &fd_token); assert(fd != -1); @@ -150,6 +210,7 @@ http_BuildSbuf(1, w->sb, sp->http); i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); + time(&t_req); /* XXX: copy any contents */ @@ -159,15 +220,21 @@ */ http_RecvHead(hp, fd, w->eb, NULL, NULL); event_base_loop(w->eb, 0); + time(&t_resp); http_Dissect(hp, fd, 2); + RFC2616_Age(hp, t_req, t_resp); + switch (http_GetStatus(hp)) { case 200: + case 301: + http_BuildSbuf(3, w->sb, hp); /* XXX: fill in object from headers */ sp->obj->valid = 1; sp->obj->cacheable = 1; + sp->obj->header = strdup(sbuf_data(w->sb)); break; - case 301: + case 391: sp->obj->valid = 0; sp->obj->cacheable = 0; break; Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-16 10:20:12 UTC (rev 193) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-16 10:22:40 UTC (rev 194) @@ -102,6 +102,39 @@ } int +http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr) +{ + char *h; + int fl; + + if (!http_GetHdr(hp, hdr, &h)) + return (0); + fl = strlen(field); + while (*h) { + if (isspace(*h)) { + h++; + continue; + } + if (*h == ',') { + h++; + continue; + } + if (memcmp(h, field, fl) || + isalpha(h[fl]) || h[fl] == '-') { + while (*h && !(isspace(*h) || *h == ',')) + h++; + continue; + } + if (h[fl] == '=') + *ptr = &h[fl + 1]; + else + *ptr = NULL; + return (1); + } + return (0); +} + +int http_HdrIs(struct http *hp, const char *hdr, const char *val) { char *p; @@ -356,7 +389,7 @@ sbuf_clear(sb); assert(sb != NULL); - if (resp == 2) { + if (resp == 2 || resp == 3) { sbuf_cat(sb, hp->proto); sbuf_cat(sb, " "); sbuf_cat(sb, hp->status); @@ -382,6 +415,7 @@ sbuf_cat(sb, hp->hdr[u]); sbuf_cat(sb, "\r\n"); } - sbuf_cat(sb, "\r\n"); + if (resp != 3) + sbuf_cat(sb, "\r\n"); sbuf_finish(sb); } Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-16 10:20:12 UTC (rev 193) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-16 10:22:40 UTC (rev 194) @@ -71,10 +71,9 @@ sbuf_clear(w->sb); sbuf_printf(w->sb, - "HTTP/1.1 200 OK\r\n" - "Server: Varnish\r\n" + "%sServer: Varnish\r\n" "Content-Length: %u\r\n" - "\r\n", sp->obj->len); + "\r\n", sp->obj->header, sp->obj->len); vca_write_obj(sp, w->sb); return (1); From phk at projects.linpro.no Sun Jun 18 07:28:20 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 18 Jun 2006 09:28:20 +0200 (CEST) Subject: r195 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060618072820.1AFFC1EC22E@projects.linpro.no> Author: phk Date: 2006-06-18 09:28:19 +0200 (Sun, 18 Jun 2006) New Revision: 195 Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/include/vcl_lang.h trunk/varnish-cache/lib/libvcl/vcl_compile.c trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl trunk/varnish-cache/lib/libvcl/vcl_token_defs.h Log: Add "deliver" keyword to VCL compiler. Split vcl_lookup() in vcl_hit() and vcl_miss() Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-16 10:22:40 UTC (rev 194) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-18 07:28:19 UTC (rev 195) @@ -199,6 +199,7 @@ } void VCL_insert(VCL_FARGS) { } +void VCL_deliver(VCL_FARGS) { } void VCL_fetch(VCL_FARGS) { sess->handling = HND_Fetch; Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-16 10:22:40 UTC (rev 194) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-18 07:28:19 UTC (rev 195) @@ -85,19 +85,24 @@ " }\n" "}\n" "\n" - "sub vcl_lookup {\n" - " if (!obj.valid) {\n" - " fetch;\n" - " }\n" + "sub vcl_hit {\n" " if (!obj.cacheable) {\n" " pass;\n" " }\n" + " deliver;\n" "}\n" "\n" + "sub vcl_miss {\n" + " fetch;\n" + "}\n" + "\n" "sub vcl_fetch {\n" " if (!obj.valid) {\n" " error;\n" " }\n" + " if (!obj.cacheable) {\n" + " pass;\n" + " }\n" " insert;\n" "}\n" "", bflag); Modified: trunk/varnish-cache/include/vcl_lang.h =================================================================== --- trunk/varnish-cache/include/vcl_lang.h 2006-06-16 10:22:40 UTC (rev 194) +++ trunk/varnish-cache/include/vcl_lang.h 2006-06-18 07:28:19 UTC (rev 195) @@ -110,7 +110,8 @@ #define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */ vcl_init_f *init_func; vcl_func_f *recv_func; - vcl_func_f *lookup_func; + vcl_func_f *hit_func; + vcl_func_f *miss_func; vcl_func_f *fetch_func; struct backend *default_backend; struct vcl_ref *ref; Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-16 10:22:40 UTC (rev 194) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-18 07:28:19 UTC (rev 195) @@ -892,7 +892,9 @@ I(tl); sbuf_printf(tl->fc, "VCL_no_cache(VCL_PASS_ARGS);\n"); return; - case T_FINISH: + case T_DELIVER: + I(tl); + sbuf_printf(tl->fc, "VCL_deliver(VCL_PASS_ARGS);\n"); I(tl); sbuf_printf(tl->fc, "sess->done = 1;\n"); I(tl); sbuf_printf(tl->fc, "return;\n"); return; @@ -1564,7 +1566,8 @@ sbuf_printf(tl->fc, "\t.init_func = VCL_Init,\n"); sbuf_printf(tl->fc, "\t.recv_func = VCL_function_vcl_recv,\n"); - sbuf_printf(tl->fc, "\t.lookup_func = VCL_function_vcl_lookup,\n"); + sbuf_printf(tl->fc, "\t.hit_func = VCL_function_vcl_hit,\n"); + sbuf_printf(tl->fc, "\t.miss_func = VCL_function_vcl_miss,\n"); sbuf_printf(tl->fc, "\t.fetch_func = VCL_function_vcl_fetch,\n"); sbuf_printf(tl->fc, "\t.default_backend = &VCL_backend_default,\n"); Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-16 10:22:40 UTC (rev 194) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-18 07:28:19 UTC (rev 195) @@ -176,6 +176,14 @@ return (T_CALL); } return (0); + case 'd': + if (p[0] == 'd' && p[1] == 'e' && p[2] == 'l' && + p[3] == 'i' && p[4] == 'v' && p[5] == 'e' && + p[6] == 'r' && !isvar(p[7])) { + *q = p + 7; + return (T_DELIVER); + } + return (0); case 'e': if (p[0] == 'e' && p[1] == 'r' && p[2] == 'r' && p[3] == 'o' && p[4] == 'r' && !isvar(p[5])) { @@ -205,12 +213,6 @@ *q = p + 4; return (T_FUNC); } - if (p[0] == 'f' && p[1] == 'i' && p[2] == 'n' && - p[3] == 'i' && p[4] == 's' && p[5] == 'h' - && !isvar(p[6])) { - *q = p + 6; - return (T_FINISH); - } if (p[0] == 'f' && p[1] == 'e' && p[2] == 't' && p[3] == 'c' && p[4] == 'h' && !isvar(p[5])) { *q = p + 5; @@ -353,6 +355,7 @@ vcl_tnames[T_COR] = "||"; vcl_tnames[T_DEC] = "--"; vcl_tnames[T_DECR] = "/="; + vcl_tnames[T_DELIVER] = "deliver"; vcl_tnames[T_DIV] = "/="; vcl_tnames[T_ELSE] = "else"; vcl_tnames[T_ELSEIF] = "elseif"; @@ -360,7 +363,6 @@ vcl_tnames[T_EQ] = "=="; vcl_tnames[T_ERROR] = "error"; vcl_tnames[T_FETCH] = "fetch"; - vcl_tnames[T_FINISH] = "finish"; vcl_tnames[T_FUNC] = "func"; vcl_tnames[T_GEQ] = ">="; vcl_tnames[T_IF] = "if"; @@ -498,7 +500,8 @@ fputs("#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */\n", f); fputs(" vcl_init_f *init_func;\n", f); fputs(" vcl_func_f *recv_func;\n", f); - fputs(" vcl_func_f *lookup_func;\n", f); + fputs(" vcl_func_f *hit_func;\n", f); + fputs(" vcl_func_f *miss_func;\n", f); fputs(" vcl_func_f *fetch_func;\n", f); fputs(" struct backend *default_backend;\n", f); fputs(" struct vcl_ref *ref;\n", f); Modified: trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-06-16 10:22:40 UTC (rev 194) +++ trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-06-18 07:28:19 UTC (rev 195) @@ -16,12 +16,13 @@ pass fetch insert + deliver + call no_cache no_new_cache set rewrite - finish switch_config } Modified: trunk/varnish-cache/lib/libvcl/vcl_token_defs.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-06-16 10:22:40 UTC (rev 194) +++ trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-06-18 07:28:19 UTC (rev 195) @@ -17,12 +17,12 @@ #define T_PASS 138 #define T_FETCH 139 #define T_INSERT 140 -#define T_CALL 141 -#define T_NO_CACHE 142 -#define T_NO_NEW_CACHE 143 -#define T_SET 144 -#define T_REWRITE 145 -#define T_FINISH 146 +#define T_DELIVER 141 +#define T_CALL 142 +#define T_NO_CACHE 143 +#define T_NO_NEW_CACHE 144 +#define T_SET 145 +#define T_REWRITE 146 #define T_SWITCH_CONFIG 147 #define T_INC 148 #define T_DEC 149 From phk at projects.linpro.no Sun Jun 18 09:12:00 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 18 Jun 2006 11:12:00 +0200 (CEST) Subject: r196 - in trunk/varnish-cache: include lib/libvcl Message-ID: <20060618091200.0A0A11EC233@projects.linpro.no> Author: phk Date: 2006-06-18 11:11:59 +0200 (Sun, 18 Jun 2006) New Revision: 196 Modified: trunk/varnish-cache/include/vcl_lang.h trunk/varnish-cache/lib/libvcl/vcl_compile.c trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl trunk/varnish-cache/lib/libvcl/vcl_token_defs.h Log: Correctly handle \ sequences in .h files in vcl_gen_fixed_token.tcl Make handling a named enum, and use it as a bitmap. Add "lookup" reserved word Add VCL_done() macro to use in compiled code to set handling and drop the per-handling callbacks (apart from VCL_error()) Modified: trunk/varnish-cache/include/vcl_lang.h =================================================================== --- trunk/varnish-cache/include/vcl_lang.h 2006-06-18 07:28:19 UTC (rev 195) +++ trunk/varnish-cache/include/vcl_lang.h 2006-06-18 09:11:59 UTC (rev 196) @@ -38,6 +38,15 @@ TAILQ_HEAD(, storage) store; }; +enum handling { + HND_Error = (1 << 0), + HND_Pipe = (1 << 1), + HND_Pass = (1 << 2), + HND_Lookup = (1 << 3), + HND_Fetch = (1 << 4), + HND_Insert = (1 << 5), + HND_Deliver = (1 << 6), +}; struct sess { int fd; @@ -48,15 +57,7 @@ /* HTTP request */ struct http *http; - enum { - HND_Unclass, - HND_Deliver, - HND_Pass, - HND_Pipe, - HND_Lookup, - HND_Fetch, - HND_Insert - } handling; + enum handling handling; char done; @@ -97,9 +98,6 @@ int string_match(const char *, const char *); int VCL_rewrite(const char *, const char *); void VCL_error(VCL_FARGS, unsigned, const char *); -void VCL_pass(VCL_FARGS); -void VCL_fetch(VCL_FARGS); -void VCL_insert(VCL_FARGS); int VCL_switch_config(const char *); typedef void vcl_init_f(void); @@ -118,3 +116,10 @@ unsigned nref; unsigned busy; }; + +#define VCL_done(sess, hand) \ + do { \ + sess->handling = hand; \ + sess->done = 1; \ + return; \ + } while (0) Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-18 07:28:19 UTC (rev 195) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-18 09:11:59 UTC (rev 196) @@ -893,28 +893,19 @@ sbuf_printf(tl->fc, "VCL_no_cache(VCL_PASS_ARGS);\n"); return; case T_DELIVER: - I(tl); - sbuf_printf(tl->fc, "VCL_deliver(VCL_PASS_ARGS);\n"); - I(tl); sbuf_printf(tl->fc, "sess->done = 1;\n"); - I(tl); sbuf_printf(tl->fc, "return;\n"); + I(tl); sbuf_printf(tl->fc, "VCL_done(sess, HND_Deliver);\n"); return; + case T_LOOKUP: + I(tl); sbuf_printf(tl->fc, "VCL_done(sess, HND_Lookup);\n"); + return; case T_PASS: - I(tl); - sbuf_printf(tl->fc, "VCL_pass(VCL_PASS_ARGS);\n"); - I(tl); sbuf_printf(tl->fc, "sess->done = 1;\n"); - I(tl); sbuf_printf(tl->fc, "return;\n"); + I(tl); sbuf_printf(tl->fc, "VCL_done(sess, HND_Pass);\n"); return; case T_FETCH: - I(tl); - sbuf_printf(tl->fc, "VCL_fetch(VCL_PASS_ARGS);\n"); - I(tl); sbuf_printf(tl->fc, "sess->done = 1;\n"); - I(tl); sbuf_printf(tl->fc, "return;\n"); + I(tl); sbuf_printf(tl->fc, "VCL_done(sess, HND_Fetch);\n"); return; case T_INSERT: - I(tl); - sbuf_printf(tl->fc, "VCL_insert(VCL_PASS_ARGS);\n"); - I(tl); sbuf_printf(tl->fc, "sess->done = 1;\n"); - I(tl); sbuf_printf(tl->fc, "return;\n"); + I(tl); sbuf_printf(tl->fc, "VCL_done(sess, HND_Insert);\n"); return; case T_ERROR: if (tl->t->tok == CNUM) @@ -930,8 +921,7 @@ NextToken(tl); } else sbuf_printf(tl->fc, "(const char *)0);\n"); - I(tl); sbuf_printf(tl->fc, "sess->done = 1;\n"); - I(tl); sbuf_printf(tl->fc, "return;\n"); + I(tl); sbuf_printf(tl->fc, "VCL_done(sess, HND_Error);\n"); return; case T_SWITCH_CONFIG: ExpectErr(tl, ID); Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-18 07:28:19 UTC (rev 195) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-18 09:11:59 UTC (rev 196) @@ -231,6 +231,14 @@ return (T_IF); } return (0); + case 'l': + if (p[0] == 'l' && p[1] == 'o' && p[2] == 'o' && + p[3] == 'k' && p[4] == 'u' && p[5] == 'p' + && !isvar(p[6])) { + *q = p + 6; + return (T_LOOKUP); + } + return (0); case 'n': if (p[0] == 'n' && p[1] == 'o' && p[2] == '_' && p[3] == 'n' && p[4] == 'e' && p[5] == 'w' && @@ -370,6 +378,7 @@ vcl_tnames[T_INCR] = "+="; vcl_tnames[T_INSERT] = "insert"; vcl_tnames[T_LEQ] = "<="; + vcl_tnames[T_LOOKUP] = "lookup"; vcl_tnames[T_MUL] = "*="; vcl_tnames[T_NEQ] = "!="; vcl_tnames[T_NO_CACHE] = "no_cache"; @@ -428,6 +437,15 @@ fputs("\n", f); fputs(" TAILQ_HEAD(, storage) store;\n", f); fputs("};\n", f); + fputs("enum handling {\n", f); + fputs(" HND_Error = (1 << 0),\n", f); + fputs(" HND_Pipe = (1 << 1),\n", f); + fputs(" HND_Pass = (1 << 2),\n", f); + fputs(" HND_Lookup = (1 << 3),\n", f); + fputs(" HND_Fetch = (1 << 4),\n", f); + fputs(" HND_Insert = (1 << 5),\n", f); + fputs(" HND_Deliver = (1 << 6),\n", f); + fputs("};\n", f); fputs("\n", f); fputs("struct sess {\n", f); fputs(" int fd;\n", f); @@ -438,15 +456,7 @@ fputs(" /* HTTP request */\n", f); fputs(" struct http *http;\n", f); fputs("\n", f); - fputs(" enum {\n", f); - fputs(" HND_Unclass,\n", f); - fputs(" HND_Deliver,\n", f); - fputs(" HND_Pass,\n", f); - fputs(" HND_Pipe,\n", f); - fputs(" HND_Lookup,\n", f); - fputs(" HND_Fetch,\n", f); - fputs(" HND_Insert\n", f); - fputs(" } handling;\n", f); + fputs(" enum handling handling;\n", f); fputs("\n", f); fputs(" char done;\n", f); fputs("\n", f); @@ -487,9 +497,6 @@ fputs("int string_match(const char *, const char *);\n", f); fputs("int VCL_rewrite(const char *, const char *);\n", f); fputs("void VCL_error(VCL_FARGS, unsigned, const char *);\n", f); - fputs("void VCL_pass(VCL_FARGS);\n", f); - fputs("void VCL_fetch(VCL_FARGS);\n", f); - fputs("void VCL_insert(VCL_FARGS);\n", f); fputs("int VCL_switch_config(const char *);\n", f); fputs("\n", f); fputs("typedef void vcl_init_f(void);\n", f); @@ -508,4 +515,11 @@ fputs(" unsigned nref;\n", f); fputs(" unsigned busy;\n", f); fputs("};\n", f); + fputs("\n", f); + fputs("#define VCL_done(sess, hand) \\\n", f); + fputs(" do { \\\n", f); + fputs(" sess->handling = hand; \\\n", f); + fputs(" sess->done = 1; \\\n", f); + fputs(" return; \\\n", f); + fputs(" } while (0)\n", f); } Modified: trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-06-18 07:28:19 UTC (rev 195) +++ trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-06-18 09:11:59 UTC (rev 196) @@ -13,6 +13,7 @@ backend error + lookup pass fetch insert @@ -192,6 +193,7 @@ close $fx continue } + regsub -all {\\} $a {\\\\} a puts $fo "\tfputs(\"$a\\n\", f);" } puts $fo "}" Modified: trunk/varnish-cache/lib/libvcl/vcl_token_defs.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-06-18 07:28:19 UTC (rev 195) +++ trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-06-18 09:11:59 UTC (rev 196) @@ -14,32 +14,33 @@ #define T_ACL 135 #define T_BACKEND 136 #define T_ERROR 137 -#define T_PASS 138 -#define T_FETCH 139 -#define T_INSERT 140 -#define T_DELIVER 141 -#define T_CALL 142 -#define T_NO_CACHE 143 -#define T_NO_NEW_CACHE 144 -#define T_SET 145 -#define T_REWRITE 146 -#define T_SWITCH_CONFIG 147 -#define T_INC 148 -#define T_DEC 149 -#define T_CAND 150 -#define T_COR 151 -#define T_LEQ 152 -#define T_EQ 153 -#define T_NEQ 154 -#define T_GEQ 155 -#define T_SHR 156 -#define T_SHL 157 -#define T_INCR 158 -#define T_DECR 159 -#define T_MUL 160 -#define T_DIV 161 -#define ID 162 -#define VAR 163 -#define CNUM 164 -#define CSTR 165 -#define EOI 166 +#define T_LOOKUP 138 +#define T_PASS 139 +#define T_FETCH 140 +#define T_INSERT 141 +#define T_DELIVER 142 +#define T_CALL 143 +#define T_NO_CACHE 144 +#define T_NO_NEW_CACHE 145 +#define T_SET 146 +#define T_REWRITE 147 +#define T_SWITCH_CONFIG 148 +#define T_INC 149 +#define T_DEC 150 +#define T_CAND 151 +#define T_COR 152 +#define T_LEQ 153 +#define T_EQ 154 +#define T_NEQ 155 +#define T_GEQ 156 +#define T_SHR 157 +#define T_SHL 158 +#define T_INCR 159 +#define T_DECR 160 +#define T_MUL 161 +#define T_DIV 162 +#define ID 163 +#define VAR 164 +#define CNUM 165 +#define CSTR 166 +#define EOI 167 From phk at projects.linpro.no Sun Jun 18 09:14:12 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 18 Jun 2006 11:14:12 +0200 (CEST) Subject: r197 - trunk/varnish-cache/bin/varnishd Message-ID: <20060618091412.CDD501EC233@projects.linpro.no> Author: phk Date: 2006-06-18 11:14:12 +0200 (Sun, 18 Jun 2006) New Revision: 197 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: add explicit "lookup" to recv method Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-18 09:11:59 UTC (rev 196) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-18 09:14:12 UTC (rev 197) @@ -83,6 +83,7 @@ " if (req.request != \"GET\" && req.request != \"HEAD\") {\n" " pass;\n" " }\n" + " lookup;\n" "}\n" "\n" "sub vcl_hit {\n" From phk at projects.linpro.no Sun Jun 18 09:16:26 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 18 Jun 2006 11:16:26 +0200 (CEST) Subject: r198 - trunk/varnish-cache/bin/varnishd Message-ID: <20060618091626.A75141EC236@projects.linpro.no> Author: phk Date: 2006-06-18 11:16:26 +0200 (Sun, 18 Jun 2006) New Revision: 198 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_vcl.c Log: Add wrappers around VCL methos so logging and checking of returned handling can be centralized. Remove old handling callbacks. Call hit/miss methods instead of lookup method. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-18 09:14:12 UTC (rev 197) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-18 09:16:26 UTC (rev 198) @@ -120,6 +120,10 @@ void RelVCL(struct VCL_conf *vc); struct VCL_conf *GetVCL(void); int CVCL_Load(const char *fn, const char *name); +void VCL_recv_method(struct sess *); +void VCL_hit_method(struct sess *); +void VCL_miss_method(struct sess *); +void VCL_fetch_method(struct sess *); #ifdef CLI_PRIV_H cli_func_t cli_func_config_list; cli_func_t cli_func_config_load; Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-18 09:14:12 UTC (rev 197) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-18 09:16:26 UTC (rev 198) @@ -242,11 +242,8 @@ break; } - sp->handling = HND_Insert; - sp->vcl->fetch_func(sp); + VCL_fetch_method(sp); - assert(sp->handling == HND_Insert); - if (http_GetHdr(hp, "Content-Length", &b)) cls = fetch_straight(w, sp, fd, hp, b); else if (http_HdrIs(hp, "Transfer-Encoding", "chunked")) Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-18 09:14:12 UTC (rev 197) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-18 09:16:26 UTC (rev 198) @@ -23,6 +23,8 @@ static pthread_cond_t shdcnd; +/*--------------------------------------------------------------------*/ + static int LookupSession(struct worker *w, struct sess *sp) { @@ -31,6 +33,7 @@ MD5_CTX ctx; char *b; + /* Make sure worker thread has a fresh object at hand */ if (w->nobj == NULL) { w->nobj = calloc(sizeof *w->nobj, 1); assert(w->nobj != NULL); @@ -43,25 +46,16 @@ MD5Update(&ctx, b, strlen(b)); MD5Final(key, &ctx); o = hash->lookup(key, w->nobj); + sp->obj = o; if (o == w->nobj) { VSL(SLT_Debug, 0, "Lookup new %p %s", o, b); w->nobj = NULL; + VCL_miss_method(sp); } else { + /* XXX: wait while obj->busy */ VSL(SLT_Debug, 0, "Lookup found %p %s", o, b); + VCL_hit_method(sp); } - /* - * XXX: if obj is busy, park session on it - */ - - sp->obj = o; - sp->handling = HND_Unclass; - sp->vcl->lookup_func(sp); - if (sp->handling == HND_Unclass) { - if (o->valid && o->cacheable) - sp->handling = HND_Deliver; - else - sp->handling = HND_Pass; - } return (0); } @@ -110,13 +104,7 @@ sp->backend = sp->vcl->default_backend; - /* - * Call the VCL recv function. - * Default action is to lookup - */ - sp->handling = HND_Lookup; - - sp->vcl->recv_func(sp); + VCL_recv_method(sp); for (done = 0; !done; ) { switch(sp->handling) { @@ -142,9 +130,7 @@ done = 1; break; default: - VSL(SLT_Handling, sp->fd, "Unclass"); - assert(sp->handling == HND_Unclass); - assert(sp->handling != HND_Unclass); + INCOMPL(); } } if (http_GetHdr(sp->http, "Connection", &b) && Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-18 09:14:12 UTC (rev 197) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-18 09:16:26 UTC (rev 198) @@ -191,22 +191,6 @@ /*--------------------------------------------------------------------*/ void -VCL_pass(VCL_FARGS) -{ - - sess->handling = HND_Pass; - sess->done++; -} - -void VCL_insert(VCL_FARGS) { } -void VCL_deliver(VCL_FARGS) { } - -void VCL_fetch(VCL_FARGS) { - sess->handling = HND_Fetch; - sess->done++; -} - -void VCL_error(VCL_FARGS, unsigned err, const char *str) { @@ -220,3 +204,58 @@ VSL(SLT_VCL, 0, "%u", u); } +/*--------------------------------------------------------------------*/ + +static const char * +HandlingName(unsigned u) +{ + + switch (u) { + case HND_Error: return ("Error"); + case HND_Pass: return ("Pass"); + case HND_Pipe: return ("Pipe"); + case HND_Lookup: return ("Lookup"); + case HND_Fetch: return ("Fetch"); + case HND_Insert: return ("Insert"); + case HND_Deliver: return ("Deliver"); + default: return (NULL); + } +} + +static void +CheckHandling(struct sess *sp, const char *func, unsigned bitmap) +{ + unsigned u; + const char *n; + + u = sp->handling; + n = HandlingName(u); + if (n != NULL) + VSL(SLT_Handling, sp->fd, "%s(): %s", func, n); + else + VSL(SLT_Handling, sp->fd, "%s(): Illegal: 0x%x", func, u); + if (u & (u - 1)) + VSL(SLT_Debug, sp->fd, + "Illegal handling after %s function: 0x%x", func, u); + else if (!(u & bitmap)) + VSL(SLT_Debug, sp->fd, + "Wrong handling after %s function: 0x%x", func, u); + else + return; + sp->handling = HND_Error; +} + +#define VCL_method(func, bitmap) \ +void \ +VCL_##func##_method(struct sess *sp) \ +{ \ + \ + sp->handling = 0; \ + sp->vcl->func##_func(sp); \ + CheckHandling(sp, #func, (bitmap)); \ +} + +VCL_method(recv, HND_Error|HND_Pass|HND_Pipe|HND_Lookup) +VCL_method(miss, HND_Error|HND_Pass|HND_Pipe|HND_Fetch) +VCL_method(hit, HND_Error|HND_Pass|HND_Pipe|HND_Deliver) +VCL_method(fetch, HND_Error|HND_Pass|HND_Pipe|HND_Insert) From phk at projects.linpro.no Sun Jun 18 10:02:35 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 18 Jun 2006 12:02:35 +0200 (CEST) Subject: r199 - in trunk/varnish-cache: include lib/libvcl Message-ID: <20060618100235.5ED9C1EC223@projects.linpro.no> Author: phk Date: 2006-06-18 12:02:35 +0200 (Sun, 18 Jun 2006) New Revision: 199 Modified: trunk/varnish-cache/include/vcl_lang.h trunk/varnish-cache/lib/libvcl/vcl_compile.c trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c Log: Add support for investigating random HTTP headers. Modified: trunk/varnish-cache/include/vcl_lang.h =================================================================== --- trunk/varnish-cache/include/vcl_lang.h 2006-06-18 09:16:26 UTC (rev 198) +++ trunk/varnish-cache/include/vcl_lang.h 2006-06-18 10:02:35 UTC (rev 199) @@ -100,6 +100,8 @@ void VCL_error(VCL_FARGS, unsigned, const char *); int VCL_switch_config(const char *); +char *VCL_GetHdr(VCL_FARGS, const char *); + typedef void vcl_init_f(void); typedef void vcl_func_f(VCL_FARGS); Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-18 09:16:26 UTC (rev 198) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-18 10:02:35 UTC (rev 199) @@ -89,7 +89,8 @@ STRING, IP, HOSTNAME, - PORTNAME + PORTNAME, + HEADER }; struct var { @@ -124,6 +125,7 @@ { "req.request", STRING, 0, "\"GET\"" }, { "obj.valid", BOOL, 0, "sess->obj->valid" }, { "obj.cacheable", BOOL, 0, "sess->obj->cacheable" }, + { "req.http.", HEADER, 0, NULL }, #if 0 { "req.ttlfactor", FLOAT, 0, "req->ttlfactor" }, { "req.url.host", STRING, 0, "req->url.host" }, @@ -550,17 +552,44 @@ } /*--------------------------------------------------------------------*/ +static struct var * +HeaderVar(struct tokenlist *tl, struct token *t, struct var *vh) +{ + char *p; + struct var *v; + int i; + v = calloc(sizeof *v, 1); + i = t->e - t->b; + p = malloc(i + 1); + assert(p != NULL); + memcpy(p, t->b, i); + p[i] = '\0'; + v->name = p; + v->fmt = STRING; + asprintf(&p, "VCL_GetHdr(VCL_PASS_ARGS, \"%s\")", v->name + vh->len); + assert(p != NULL); + v->cname = p; + return (v); +} + +/*--------------------------------------------------------------------*/ + static struct var * FindVar(struct tokenlist *tl, struct token *t, struct var *vl) { struct var *v; for (v = vl; v->name != NULL; v++) { - if (t->e - t->b != v->len) + if (v->fmt == HEADER && t->e - t->b <= v->len) continue; - if (!memcmp(t->b, v->name, v->len)) + if (v->fmt != HEADER && t->e - t->b != v->len) + continue; + if (memcmp(t->b, v->name, v->len)) + continue; + if (v->fmt != HEADER) return (v); + return (HeaderVar(tl, t, v)); } sbuf_printf(tl->sb, "Unknown variable "); ErrToken(tl, t); @@ -651,7 +680,6 @@ Cond_String(struct var *vp, struct tokenlist *tl) { - (void)vp; switch (tl->t->tok) { case '~': I(tl); sbuf_printf(tl->fc, "string_match(%s, ", vp->cname); @@ -662,8 +690,11 @@ tl->t->e - tl->t->b, tl->t->b); NextToken(tl); break; + case T_EQ: case T_NEQ: - I(tl); sbuf_printf(tl->fc, "strcmp(%s, ", vp->cname); + I(tl); + sbuf_printf(tl->fc, "%sstrcmp(%s, ", + tl->t->tok == T_EQ ? "" : "!", vp->cname); NextToken(tl); ExpectErr(tl, CSTR); sbuf_printf(tl->fc, "%*.*s)\n", @@ -672,11 +703,7 @@ NextToken(tl); break; default: - sbuf_printf(tl->sb, "Illegal condition "); - ErrToken(tl, tl->t); - sbuf_printf(tl->sb, " on string variable\n"); - sbuf_printf(tl->sb, " only '~' is legal\n"); - ErrWhere(tl, tl->t); + I(tl); sbuf_printf(tl->fc, "%s != (void*)0", vp->cname); break; } } Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-18 09:16:26 UTC (rev 198) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-18 10:02:35 UTC (rev 199) @@ -499,6 +499,8 @@ fputs("void VCL_error(VCL_FARGS, unsigned, const char *);\n", f); fputs("int VCL_switch_config(const char *);\n", f); fputs("\n", f); + fputs("char *VCL_GetHdr(VCL_FARGS, const char *);\n", f); + fputs("\n", f); fputs("typedef void vcl_init_f(void);\n", f); fputs("typedef void vcl_func_f(VCL_FARGS);\n", f); fputs("\n", f); From phk at projects.linpro.no Sun Jun 18 10:03:44 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 18 Jun 2006 12:03:44 +0200 (CEST) Subject: r200 - trunk/varnish-cache/bin/varnishd Message-ID: <20060618100344.095F11EC22E@projects.linpro.no> Author: phk Date: 2006-06-18 12:03:43 +0200 (Sun, 18 Jun 2006) New Revision: 200 Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c Log: Add VCL function for getting HTTP header Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-18 10:02:35 UTC (rev 199) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-18 10:03:43 UTC (rev 200) @@ -259,3 +259,17 @@ VCL_method(miss, HND_Error|HND_Pass|HND_Pipe|HND_Fetch) VCL_method(hit, HND_Error|HND_Pass|HND_Pipe|HND_Deliver) VCL_method(fetch, HND_Error|HND_Pass|HND_Pipe|HND_Insert) + +/*--------------------------------------------------------------------*/ + +char * +VCL_GetHdr(VCL_FARGS, const char *n) +{ + char *p; + + assert(sess != NULL); + assert(sess->http != NULL); + if (!http_GetHdr(sess->http, n, &p)) + return (NULL); + return (p); +} From phk at projects.linpro.no Sun Jun 18 10:04:09 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 18 Jun 2006 12:04:09 +0200 (CEST) Subject: r201 - trunk/varnish-cache/bin/varnishd Message-ID: <20060618100409.E94731EC22C@projects.linpro.no> Author: phk Date: 2006-06-18 12:04:09 +0200 (Sun, 18 Jun 2006) New Revision: 201 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Pass if we spot an Authenticate or Cookie header Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-18 10:03:43 UTC (rev 200) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-18 10:04:09 UTC (rev 201) @@ -83,6 +83,9 @@ " if (req.request != \"GET\" && req.request != \"HEAD\") {\n" " pass;\n" " }\n" + " if (req.http.Authenticate || req.http.Cookie) {\n" + " pass;\n" + " }\n" " lookup;\n" "}\n" "\n" From phk at projects.linpro.no Sun Jun 18 10:10:45 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 18 Jun 2006 12:10:45 +0200 (CEST) Subject: r202 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060618101045.5A6D61EC22C@projects.linpro.no> Author: phk Date: 2006-06-18 12:10:45 +0200 (Sun, 18 Jun 2006) New Revision: 202 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_vcl.c trunk/varnish-cache/include/vcl_lang.h trunk/varnish-cache/lib/libvcl/vcl_compile.c trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c Log: Implement req.request properly Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-18 10:04:09 UTC (rev 201) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-18 10:10:45 UTC (rev 202) @@ -83,6 +83,7 @@ void http_Delete(struct http *hp); int http_GetHdr(struct http *hp, const char *hdr, char **ptr); int http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr); +int http_GetReq(struct http *hp, char **b); int http_GetStatus(struct http *hp); int http_HdrIs(struct http *hp, const char *hdr, const char *val); int http_GetTail(struct http *hp, unsigned len, char **b, char **e); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-18 10:04:09 UTC (rev 201) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-18 10:10:45 UTC (rev 202) @@ -148,6 +148,15 @@ } int +http_GetReq(struct http *hp, char **b) +{ + if (hp->req == NULL) + return (0); + *b = hp->req; + return (1); +} + +int http_GetURL(struct http *hp, char **b) { if (hp->url == NULL) Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-18 10:04:09 UTC (rev 201) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-18 10:10:45 UTC (rev 202) @@ -273,3 +273,14 @@ return (NULL); return (p); } + +char * +VCL_GetReq(VCL_FARGS) +{ + char *p; + + assert(sess != NULL); + assert(sess->http != NULL); + assert(http_GetReq(sess->http, &p)); + return (p); +} Modified: trunk/varnish-cache/include/vcl_lang.h =================================================================== --- trunk/varnish-cache/include/vcl_lang.h 2006-06-18 10:04:09 UTC (rev 201) +++ trunk/varnish-cache/include/vcl_lang.h 2006-06-18 10:10:45 UTC (rev 202) @@ -101,6 +101,7 @@ int VCL_switch_config(const char *); char *VCL_GetHdr(VCL_FARGS, const char *); +char *VCL_GetReq(VCL_FARGS); typedef void vcl_init_f(void); typedef void vcl_func_f(VCL_FARGS); Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-18 10:04:09 UTC (rev 201) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-18 10:10:45 UTC (rev 202) @@ -122,7 +122,7 @@ static struct var vars[] = { - { "req.request", STRING, 0, "\"GET\"" }, + { "req.request", STRING, 0, "VCL_GetReq(VCL_PASS_ARGS)" }, { "obj.valid", BOOL, 0, "sess->obj->valid" }, { "obj.cacheable", BOOL, 0, "sess->obj->cacheable" }, { "req.http.", HEADER, 0, NULL }, Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-18 10:04:09 UTC (rev 201) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-18 10:10:45 UTC (rev 202) @@ -500,6 +500,7 @@ fputs("int VCL_switch_config(const char *);\n", f); fputs("\n", f); fputs("char *VCL_GetHdr(VCL_FARGS, const char *);\n", f); + fputs("char *VCL_GetReq(VCL_FARGS);\n", f); fputs("\n", f); fputs("typedef void vcl_init_f(void);\n", f); fputs("typedef void vcl_func_f(VCL_FARGS);\n", f); From phk at projects.linpro.no Sun Jun 18 10:12:38 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 18 Jun 2006 12:12:38 +0200 (CEST) Subject: r203 - trunk/varnish-cache/bin/varnishd Message-ID: <20060618101238.6FCB71EC22E@projects.linpro.no> Author: phk Date: 2006-06-18 12:12:38 +0200 (Sun, 18 Jun 2006) New Revision: 203 Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c Log: debug printf for max-age Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-18 10:10:45 UTC (rev 202) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-18 10:12:38 UTC (rev 203) @@ -71,6 +71,7 @@ printf("Expires: %d\n", h_expires); printf("Age: %d\n", h_age); printf("CIAge: %d\n", corrected_initial_age); + printf("Max-Age: %d\n", max_age); } From phk at projects.linpro.no Sun Jun 18 10:16:38 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 18 Jun 2006 12:16:38 +0200 (CEST) Subject: r204 - trunk/varnish-cache/lib/libvcl Message-ID: <20060618101638.7F0711EC22C@projects.linpro.no> Author: phk Date: 2006-06-18 12:16:38 +0200 (Sun, 18 Jun 2006) New Revision: 204 Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c Log: Get the sense of string compares right. Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-18 10:12:38 UTC (rev 203) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-18 10:16:38 UTC (rev 204) @@ -694,7 +694,7 @@ case T_NEQ: I(tl); sbuf_printf(tl->fc, "%sstrcmp(%s, ", - tl->t->tok == T_EQ ? "" : "!", vp->cname); + tl->t->tok == T_EQ ? "!" : "", vp->cname); NextToken(tl); ExpectErr(tl, CSTR); sbuf_printf(tl->fc, "%*.*s)\n", From phk at projects.linpro.no Sun Jun 18 10:19:54 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 18 Jun 2006 12:19:54 +0200 (CEST) Subject: r205 - trunk/varnish-cache/lib/libvcl Message-ID: <20060618101954.90F051EC22E@projects.linpro.no> Author: phk Date: 2006-06-18 12:19:54 +0200 (Sun, 18 Jun 2006) New Revision: 205 Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c Log: Insert a count-point after each conditional. Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-18 10:16:38 UTC (rev 204) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-18 10:19:54 UTC (rev 205) @@ -894,6 +894,7 @@ ERRCHK(tl); break; default: + C(tl, ";"); return; } } From phk at projects.linpro.no Sun Jun 18 10:28:11 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 18 Jun 2006 12:28:11 +0200 (CEST) Subject: r206 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060618102811.6B4401EC22C@projects.linpro.no> Author: phk Date: 2006-06-18 12:28:11 +0200 (Sun, 18 Jun 2006) New Revision: 206 Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c trunk/varnish-cache/include/vcl_lang.h trunk/varnish-cache/lib/libvcl/vcl_compile.c trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c Log: Output line+pos for counts. Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-18 10:19:54 UTC (rev 205) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-18 10:28:11 UTC (rev 206) @@ -198,10 +198,12 @@ } void -VCL_count(unsigned u) +VCL_count(struct sess *sp, unsigned u) { - VSL(SLT_VCL, 0, "%u", u); + VSL(SLT_VCL, 0, "%u %d.%d", u, + sp->vcl->ref[u].line, + sp->vcl->ref[u].pos); } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/include/vcl_lang.h =================================================================== --- trunk/varnish-cache/include/vcl_lang.h 2006-06-18 10:19:54 UTC (rev 205) +++ trunk/varnish-cache/include/vcl_lang.h 2006-06-18 10:28:11 UTC (rev 206) @@ -91,7 +91,7 @@ #define VCL_FARGS struct sess *sess #define VCL_PASS_ARGS sess -void VCL_count(unsigned); +void VCL_count(struct sess *, unsigned); void VCL_no_cache(VCL_FARGS); void VCL_no_new_cache(VCL_FARGS); int ip_match(unsigned, struct vcl_acl *); Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-18 10:19:54 UTC (rev 205) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-18 10:28:11 UTC (rev 206) @@ -272,7 +272,7 @@ #define C(tl, sep) do { \ I(tl); \ - sbuf_printf(tl->fc, "VCL_count(%u)%s\n", ++tl->cnt, sep); \ + sbuf_printf(tl->fc, "VCL_count(sess, %u)%s\n", ++tl->cnt, sep); \ tl->t->cnt = tl->cnt; \ } while (0) Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-18 10:19:54 UTC (rev 205) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-18 10:28:11 UTC (rev 206) @@ -490,7 +490,7 @@ fputs("#define VCL_FARGS struct sess *sess\n", f); fputs("#define VCL_PASS_ARGS sess\n", f); fputs("\n", f); - fputs("void VCL_count(unsigned);\n", f); + fputs("void VCL_count(struct sess *, unsigned);\n", f); fputs("void VCL_no_cache(VCL_FARGS);\n", f); fputs("void VCL_no_new_cache(VCL_FARGS);\n", f); fputs("int ip_match(unsigned, struct vcl_acl *);\n", f); From phk at projects.linpro.no Tue Jun 20 09:15:39 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 20 Jun 2006 11:15:39 +0200 (CEST) Subject: r207 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060620091539.927571EC2FA@projects.linpro.no> Author: phk Date: 2006-06-20 11:15:39 +0200 (Tue, 20 Jun 2006) New Revision: 207 Added: trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/include/vrt.h Modified: trunk/varnish-cache/bin/varnishd/Makefile.am trunk/varnish-cache/bin/varnishd/cache_vcl.c trunk/varnish-cache/include/vcl_lang.h trunk/varnish-cache/lib/libvcl/vcl_compile.c trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl Log: Start putting some structure in the sources relating to VCL handling: Split the runtime support for compiled varnish programs out and give it the prefix "VRT". Start using the prefix "VGC" for generated code. Prefix "VCC" will be for the compiler and "VCL" for calling the compiled and loaded functions. Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-06-18 10:28:11 UTC (rev 206) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-06-20 09:15:39 UTC (rev 207) @@ -15,6 +15,7 @@ cache_pipe.c \ cache_shmlog.c \ cache_vcl.c \ + cache_vrt.c \ cli_event.c \ hash_simple_list.c \ mgt_child.c \ Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-18 10:28:11 UTC (rev 206) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-20 09:15:39 UTC (rev 207) @@ -190,24 +190,6 @@ /*--------------------------------------------------------------------*/ -void -VCL_error(VCL_FARGS, unsigned err, const char *str) -{ - - VSL(SLT_Debug, 0, "VCL_error(%u, %s)", err, str); -} - -void -VCL_count(struct sess *sp, unsigned u) -{ - - VSL(SLT_VCL, 0, "%u %d.%d", u, - sp->vcl->ref[u].line, - sp->vcl->ref[u].pos); -} - -/*--------------------------------------------------------------------*/ - static const char * HandlingName(unsigned u) { @@ -261,28 +243,3 @@ VCL_method(miss, HND_Error|HND_Pass|HND_Pipe|HND_Fetch) VCL_method(hit, HND_Error|HND_Pass|HND_Pipe|HND_Deliver) VCL_method(fetch, HND_Error|HND_Pass|HND_Pipe|HND_Insert) - -/*--------------------------------------------------------------------*/ - -char * -VCL_GetHdr(VCL_FARGS, const char *n) -{ - char *p; - - assert(sess != NULL); - assert(sess->http != NULL); - if (!http_GetHdr(sess->http, n, &p)) - return (NULL); - return (p); -} - -char * -VCL_GetReq(VCL_FARGS) -{ - char *p; - - assert(sess != NULL); - assert(sess->http != NULL); - assert(http_GetReq(sess->http, &p)); - return (p); -} Added: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-18 10:28:11 UTC (rev 206) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-20 09:15:39 UTC (rev 207) @@ -0,0 +1,69 @@ +/* + * $Id$ + * + * Runtime support for compiled VCL programs + */ + + +#include +#include +#include +#include +#include +#include +#include + +#include "cli.h" +#include "cli_priv.h" +#include "shmlog.h" +#include "vcl_lang.h" +#include "vrt.h" +#include "libvarnish.h" +#include "cache.h" + +/*--------------------------------------------------------------------*/ + +void +VRT_error(VCL_FARGS, unsigned err, const char *str) +{ + + VSL(SLT_Debug, 0, "VCL_error(%u, %s)", err, str); +} + +/*--------------------------------------------------------------------*/ + +void +VRT_count(struct sess *sp, unsigned u) +{ + + VSL(SLT_VCL, 0, "%u %d.%d", u, + sp->vcl->ref[u].line, + sp->vcl->ref[u].pos); +} + +/*--------------------------------------------------------------------*/ + +char * +VRT_GetHdr(VCL_FARGS, const char *n) +{ + char *p; + + assert(sess != NULL); + assert(sess->http != NULL); + if (!http_GetHdr(sess->http, n, &p)) + return (NULL); + return (p); +} + +/*--------------------------------------------------------------------*/ + +char * +VRT_GetReq(VCL_FARGS) +{ + char *p; + + assert(sess != NULL); + assert(sess->http != NULL); + assert(http_GetReq(sess->http, &p)); + return (p); +} Modified: trunk/varnish-cache/include/vcl_lang.h =================================================================== --- trunk/varnish-cache/include/vcl_lang.h 2006-06-18 10:28:11 UTC (rev 206) +++ trunk/varnish-cache/include/vcl_lang.h 2006-06-20 09:15:39 UTC (rev 207) @@ -11,18 +11,6 @@ struct sess; typedef void sesscb_f(struct sess *sp); -struct vcl_ref { - unsigned line; - unsigned pos; - unsigned count; - const char *token; -}; - -struct vcl_acl { - unsigned ip; - unsigned mask; -}; - #define VCA_ADDRBUFSIZE 32 struct object { @@ -91,18 +79,11 @@ #define VCL_FARGS struct sess *sess #define VCL_PASS_ARGS sess -void VCL_count(struct sess *, unsigned); -void VCL_no_cache(VCL_FARGS); -void VCL_no_new_cache(VCL_FARGS); +#if 0 int ip_match(unsigned, struct vcl_acl *); int string_match(const char *, const char *); -int VCL_rewrite(const char *, const char *); -void VCL_error(VCL_FARGS, unsigned, const char *); -int VCL_switch_config(const char *); +#endif -char *VCL_GetHdr(VCL_FARGS, const char *); -char *VCL_GetReq(VCL_FARGS); - typedef void vcl_init_f(void); typedef void vcl_func_f(VCL_FARGS); @@ -115,14 +96,7 @@ vcl_func_f *miss_func; vcl_func_f *fetch_func; struct backend *default_backend; - struct vcl_ref *ref; + struct vrt_ref *ref; unsigned nref; unsigned busy; }; - -#define VCL_done(sess, hand) \ - do { \ - sess->handling = hand; \ - sess->done = 1; \ - return; \ - } while (0) Added: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2006-06-18 10:28:11 UTC (rev 206) +++ trunk/varnish-cache/include/vrt.h 2006-06-20 09:15:39 UTC (rev 207) @@ -0,0 +1,40 @@ +/* $Id$ */ +/* + * Runtime support for compiled VCL programs. + * + * XXX: When this file is changed, lib/libvcl/vcl_gen_fixed_token.tcl + * XXX: *MUST* be rerun. + */ + +struct vrt_ref { + unsigned line; + unsigned pos; + unsigned count; + const char *token; +}; + +struct vrt_acl { + unsigned ip; + unsigned mask; +}; + +void VRT_count(struct sess *, unsigned); +void VRT_no_cache(VCL_FARGS); +void VRT_no_new_cache(VCL_FARGS); +#if 0 +int ip_match(unsigned, struct vcl_acl *); +int string_match(const char *, const char *); +#endif +int VRT_rewrite(const char *, const char *); +void VRT_error(VCL_FARGS, unsigned, const char *); +int VRT_switch_config(const char *); + +char *VRT_GetHdr(VCL_FARGS, const char *); +char *VRT_GetReq(VCL_FARGS); + +#define VRT_done(sess, hand) \ + do { \ + sess->handling = hand; \ + sess->done = 1; \ + return; \ + } while (0) Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-18 10:28:11 UTC (rev 206) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-20 09:15:39 UTC (rev 207) @@ -122,7 +122,7 @@ static struct var vars[] = { - { "req.request", STRING, 0, "VCL_GetReq(VCL_PASS_ARGS)" }, + { "req.request", STRING, 0, "VRT_GetReq(VCL_PASS_ARGS)" }, { "obj.valid", BOOL, 0, "sess->obj->valid" }, { "obj.cacheable", BOOL, 0, "sess->obj->cacheable" }, { "req.http.", HEADER, 0, NULL }, @@ -272,7 +272,7 @@ #define C(tl, sep) do { \ I(tl); \ - sbuf_printf(tl->fc, "VCL_count(sess, %u)%s\n", ++tl->cnt, sep); \ + sbuf_printf(tl->fc, "VRT_count(sess, %u)%s\n", ++tl->cnt, sep); \ tl->t->cnt = tl->cnt; \ } while (0) @@ -567,7 +567,7 @@ p[i] = '\0'; v->name = p; v->fmt = STRING; - asprintf(&p, "VCL_GetHdr(VCL_PASS_ARGS, \"%s\")", v->name + vh->len); + asprintf(&p, "VRT_GetHdr(VCL_PASS_ARGS, \"%s\")", v->name + vh->len); assert(p != NULL); v->cname = p; return (v); @@ -921,19 +921,19 @@ sbuf_printf(tl->fc, "VCL_no_cache(VCL_PASS_ARGS);\n"); return; case T_DELIVER: - I(tl); sbuf_printf(tl->fc, "VCL_done(sess, HND_Deliver);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sess, HND_Deliver);\n"); return; case T_LOOKUP: - I(tl); sbuf_printf(tl->fc, "VCL_done(sess, HND_Lookup);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sess, HND_Lookup);\n"); return; case T_PASS: - I(tl); sbuf_printf(tl->fc, "VCL_done(sess, HND_Pass);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sess, HND_Pass);\n"); return; case T_FETCH: - I(tl); sbuf_printf(tl->fc, "VCL_done(sess, HND_Fetch);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sess, HND_Fetch);\n"); return; case T_INSERT: - I(tl); sbuf_printf(tl->fc, "VCL_done(sess, HND_Insert);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sess, HND_Insert);\n"); return; case T_ERROR: if (tl->t->tok == CNUM) @@ -941,7 +941,7 @@ else a = 0; I(tl); - sbuf_printf(tl->fc, "VCL_error(VCL_PASS_ARGS, %u, ", a); + sbuf_printf(tl->fc, "VRT_error(VCL_PASS_ARGS, %u, ", a); if (tl->t->tok == CSTR) { sbuf_printf(tl->fc, "%*.*s);\n", tl->t->e - tl->t->b, @@ -949,7 +949,7 @@ NextToken(tl); } else sbuf_printf(tl->fc, "(const char *)0);\n"); - I(tl); sbuf_printf(tl->fc, "VCL_done(sess, HND_Error);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sess, HND_Error);\n"); return; case T_SWITCH_CONFIG: ExpectErr(tl, ID); @@ -1508,11 +1508,11 @@ const char *p; sbuf_printf(tl->fh, - "#define VCL_NREFS %u\n", tl->cnt + 1); + "#define VGC_NREFS %u\n", tl->cnt + 1); sbuf_printf(tl->fh, - "static struct vcl_ref VCL_ref[VCL_NREFS];\n"); + "static struct vrt_ref VGC_ref[VGC_NREFS];\n"); sbuf_printf(tl->fc, - "static struct vcl_ref VCL_ref[VCL_NREFS] = {\n"); + "static struct vrt_ref VGC_ref[VGC_NREFS] = {\n"); lin = 1; pos = 0; p = tl->b; @@ -1590,9 +1590,9 @@ sbuf_printf(tl->fc, "\t.default_backend = &VCL_backend_default,\n"); sbuf_printf(tl->fc, - "\t.ref = VCL_ref,\n"); + "\t.ref = VGC_ref,\n"); sbuf_printf(tl->fc, - "\t.nref = VCL_NREFS,\n"); + "\t.nref = VGC_NREFS,\n"); sbuf_printf(tl->fc, "};\n"); } Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-18 10:28:11 UTC (rev 206) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-20 09:15:39 UTC (rev 207) @@ -410,18 +410,6 @@ fputs("struct sess;\n", f); fputs("typedef void sesscb_f(struct sess *sp);\n", f); fputs("\n", f); - fputs("struct vcl_ref {\n", f); - fputs(" unsigned line;\n", f); - fputs(" unsigned pos;\n", f); - fputs(" unsigned count;\n", f); - fputs(" const char *token;\n", f); - fputs("};\n", f); - fputs("\n", f); - fputs("struct vcl_acl {\n", f); - fputs(" unsigned ip;\n", f); - fputs(" unsigned mask;\n", f); - fputs("};\n", f); - fputs("\n", f); fputs("#define VCA_ADDRBUFSIZE 32\n", f); fputs("\n", f); fputs("struct object { \n", f); @@ -490,18 +478,11 @@ fputs("#define VCL_FARGS struct sess *sess\n", f); fputs("#define VCL_PASS_ARGS sess\n", f); fputs("\n", f); - fputs("void VCL_count(struct sess *, unsigned);\n", f); - fputs("void VCL_no_cache(VCL_FARGS);\n", f); - fputs("void VCL_no_new_cache(VCL_FARGS);\n", f); + fputs("#if 0\n", f); fputs("int ip_match(unsigned, struct vcl_acl *);\n", f); fputs("int string_match(const char *, const char *);\n", f); - fputs("int VCL_rewrite(const char *, const char *);\n", f); - fputs("void VCL_error(VCL_FARGS, unsigned, const char *);\n", f); - fputs("int VCL_switch_config(const char *);\n", f); + fputs("#endif\n", f); fputs("\n", f); - fputs("char *VCL_GetHdr(VCL_FARGS, const char *);\n", f); - fputs("char *VCL_GetReq(VCL_FARGS);\n", f); - fputs("\n", f); fputs("typedef void vcl_init_f(void);\n", f); fputs("typedef void vcl_func_f(VCL_FARGS);\n", f); fputs("\n", f); @@ -514,12 +495,45 @@ fputs(" vcl_func_f *miss_func;\n", f); fputs(" vcl_func_f *fetch_func;\n", f); fputs(" struct backend *default_backend;\n", f); - fputs(" struct vcl_ref *ref;\n", f); + fputs(" struct vrt_ref *ref;\n", f); fputs(" unsigned nref;\n", f); fputs(" unsigned busy;\n", f); fputs("};\n", f); + fputs("/* $Id$ */\n", f); + fputs("/*\n", f); + fputs(" * Runtime support for compiled VCL programs.\n", f); + fputs(" *\n", f); + fputs(" * XXX: When this file is changed, lib/libvcl/vcl_gen_fixed_token.tcl\n", f); + fputs(" * XXX: *MUST* be rerun.\n", f); + fputs(" */\n", f); fputs("\n", f); - fputs("#define VCL_done(sess, hand) \\\n", f); + fputs("struct vrt_ref {\n", f); + fputs(" unsigned line;\n", f); + fputs(" unsigned pos;\n", f); + fputs(" unsigned count;\n", f); + fputs(" const char *token;\n", f); + fputs("};\n", f); + fputs("\n", f); + fputs("struct vrt_acl {\n", f); + fputs(" unsigned ip;\n", f); + fputs(" unsigned mask;\n", f); + fputs("};\n", f); + fputs("\n", f); + fputs("void VRT_count(struct sess *, unsigned);\n", f); + fputs("void VRT_no_cache(VCL_FARGS);\n", f); + fputs("void VRT_no_new_cache(VCL_FARGS);\n", f); + fputs("#if 0\n", f); + fputs("int ip_match(unsigned, struct vcl_acl *);\n", f); + fputs("int string_match(const char *, const char *);\n", f); + fputs("#endif\n", f); + fputs("int VRT_rewrite(const char *, const char *);\n", f); + fputs("void VRT_error(VCL_FARGS, unsigned, const char *);\n", f); + fputs("int VRT_switch_config(const char *);\n", f); + fputs("\n", f); + fputs("char *VRT_GetHdr(VCL_FARGS, const char *);\n", f); + fputs("char *VRT_GetReq(VCL_FARGS);\n", f); + fputs("\n", f); + fputs("#define VRT_done(sess, hand) \\\n", f); fputs(" do { \\\n", f); fputs(" sess->handling = hand; \\\n", f); fputs(" sess->done = 1; \\\n", f); Modified: trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-06-18 10:28:11 UTC (rev 206) +++ trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-06-20 09:15:39 UTC (rev 207) @@ -176,28 +176,37 @@ } puts $fo "}" -set fi [open "../../include/vcl_lang.h"] + +proc copy_include {n} { + global fo + + set fi [open $n] + while {[gets $fi a] >= 0} { + if {"$a" == "#include "} { + puts "FOO $a" + set fx [open "../../include/http_headers.h"] + while {[gets $fx b] >= 0} { + regsub -all {"} $b {\"} b + puts $fo "\tfputs(\"$b\\n\", f);" + } + close $fx + continue + } + regsub -all {\\} $a {\\\\} a + puts $fo "\tfputs(\"$a\\n\", f);" + } + close $fi +} + puts $fo "" puts $fo "void" puts $fo "vcl_output_lang_h(FILE *f)" puts $fo "{" -while {[gets $fi a] >= 0} { - if {"$a" == "#include "} { - puts "FOO $a" - set fx [open "../../include/http_headers.h"] - while {[gets $fx b] >= 0} { - regsub -all {"} $b {\"} b - puts $fo "\tfputs(\"$b\\n\", f);" - } - close $fx - continue - } - regsub -all {\\} $a {\\\\} a - puts $fo "\tfputs(\"$a\\n\", f);" -} +copy_include ../../include/vcl_lang.h +copy_include ../../include/vrt.h + puts $fo "}" -close $fi close $foh close $fo From phk at projects.linpro.no Tue Jun 20 09:25:21 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 20 Jun 2006 11:25:21 +0200 (CEST) Subject: r208 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060620092521.7FFED1EC235@projects.linpro.no> Author: phk Date: 2006-06-20 11:25:21 +0200 (Tue, 20 Jun 2006) New Revision: 208 Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/include/vcl_lang.h trunk/varnish-cache/include/vrt.h trunk/varnish-cache/lib/libvcl/vcl_compile.c trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c Log: Get rid of VCL_FARGS and VCL_PASS_ARGS macros. Generate VGC prefixes instead of VCL. Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-20 09:15:39 UTC (rev 207) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-20 09:25:21 UTC (rev 208) @@ -24,7 +24,7 @@ /*--------------------------------------------------------------------*/ void -VRT_error(VCL_FARGS, unsigned err, const char *str) +VRT_error(struct sess *sp, unsigned err, const char *str) { VSL(SLT_Debug, 0, "VCL_error(%u, %s)", err, str); @@ -44,13 +44,13 @@ /*--------------------------------------------------------------------*/ char * -VRT_GetHdr(VCL_FARGS, const char *n) +VRT_GetHdr(struct sess *sp, const char *n) { char *p; - assert(sess != NULL); - assert(sess->http != NULL); - if (!http_GetHdr(sess->http, n, &p)) + assert(sp != NULL); + assert(sp->http != NULL); + if (!http_GetHdr(sp->http, n, &p)) return (NULL); return (p); } @@ -58,12 +58,12 @@ /*--------------------------------------------------------------------*/ char * -VRT_GetReq(VCL_FARGS) +VRT_GetReq(struct sess *sp) { char *p; - assert(sess != NULL); - assert(sess->http != NULL); - assert(http_GetReq(sess->http, &p)); + assert(sp != NULL); + assert(sp->http != NULL); + assert(http_GetReq(sp->http, &p)); return (p); } Modified: trunk/varnish-cache/include/vcl_lang.h =================================================================== --- trunk/varnish-cache/include/vcl_lang.h 2006-06-20 09:15:39 UTC (rev 207) +++ trunk/varnish-cache/include/vcl_lang.h 2006-06-20 09:25:21 UTC (rev 208) @@ -76,16 +76,13 @@ struct vbe *vbe; }; -#define VCL_FARGS struct sess *sess -#define VCL_PASS_ARGS sess - #if 0 int ip_match(unsigned, struct vcl_acl *); int string_match(const char *, const char *); #endif typedef void vcl_init_f(void); -typedef void vcl_func_f(VCL_FARGS); +typedef void vcl_func_f(struct sess *sp); struct VCL_conf { unsigned magic; Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2006-06-20 09:15:39 UTC (rev 207) +++ trunk/varnish-cache/include/vrt.h 2006-06-20 09:25:21 UTC (rev 208) @@ -19,18 +19,18 @@ }; void VRT_count(struct sess *, unsigned); -void VRT_no_cache(VCL_FARGS); -void VRT_no_new_cache(VCL_FARGS); +void VRT_no_cache(struct sess *); +void VRT_no_new_cache(struct sess *); #if 0 int ip_match(unsigned, struct vcl_acl *); int string_match(const char *, const char *); #endif int VRT_rewrite(const char *, const char *); -void VRT_error(VCL_FARGS, unsigned, const char *); +void VRT_error(struct sess *, unsigned, const char *); int VRT_switch_config(const char *); -char *VRT_GetHdr(VCL_FARGS, const char *); -char *VRT_GetReq(VCL_FARGS); +char *VRT_GetHdr(struct sess *, const char *); +char *VRT_GetReq(struct sess *); #define VRT_done(sess, hand) \ do { \ Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-20 09:15:39 UTC (rev 207) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-20 09:25:21 UTC (rev 208) @@ -122,9 +122,9 @@ static struct var vars[] = { - { "req.request", STRING, 0, "VRT_GetReq(VCL_PASS_ARGS)" }, - { "obj.valid", BOOL, 0, "sess->obj->valid" }, - { "obj.cacheable", BOOL, 0, "sess->obj->cacheable" }, + { "req.request", STRING, 0, "VRT_GetReq(sp)" }, + { "obj.valid", BOOL, 0, "sp->obj->valid" }, + { "obj.cacheable", BOOL, 0, "sp->obj->cacheable" }, { "req.http.", HEADER, 0, NULL }, #if 0 { "req.ttlfactor", FLOAT, 0, "req->ttlfactor" }, @@ -272,7 +272,7 @@ #define C(tl, sep) do { \ I(tl); \ - sbuf_printf(tl->fc, "VRT_count(sess, %u)%s\n", ++tl->cnt, sep); \ + sbuf_printf(tl->fc, "VRT_count(sp, %u)%s\n", ++tl->cnt, sep); \ tl->t->cnt = tl->cnt; \ } while (0) @@ -567,7 +567,7 @@ p[i] = '\0'; v->name = p; v->fmt = STRING; - asprintf(&p, "VRT_GetHdr(VCL_PASS_ARGS, \"%s\")", v->name + vh->len); + asprintf(&p, "VRT_GetHdr(sp, \"%s\")", v->name + vh->len); assert(p != NULL); v->cname = p; return (v); @@ -914,26 +914,26 @@ switch (at->tok) { case T_NO_NEW_CACHE: I(tl); - sbuf_printf(tl->fc, "VCL_no_new_cache(VCL_PASS_ARGS);\n"); + sbuf_printf(tl->fc, "VCL_no_new_cache(sp);\n"); return; case T_NO_CACHE: I(tl); - sbuf_printf(tl->fc, "VCL_no_cache(VCL_PASS_ARGS);\n"); + sbuf_printf(tl->fc, "VCL_no_cache(sp);\n"); return; case T_DELIVER: - I(tl); sbuf_printf(tl->fc, "VRT_done(sess, HND_Deliver);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sp, HND_Deliver);\n"); return; case T_LOOKUP: - I(tl); sbuf_printf(tl->fc, "VRT_done(sess, HND_Lookup);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sp, HND_Lookup);\n"); return; case T_PASS: - I(tl); sbuf_printf(tl->fc, "VRT_done(sess, HND_Pass);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sp, HND_Pass);\n"); return; case T_FETCH: - I(tl); sbuf_printf(tl->fc, "VRT_done(sess, HND_Fetch);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sp, HND_Fetch);\n"); return; case T_INSERT: - I(tl); sbuf_printf(tl->fc, "VRT_done(sess, HND_Insert);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sp, HND_Insert);\n"); return; case T_ERROR: if (tl->t->tok == CNUM) @@ -941,7 +941,7 @@ else a = 0; I(tl); - sbuf_printf(tl->fc, "VRT_error(VCL_PASS_ARGS, %u, ", a); + sbuf_printf(tl->fc, "VRT_error(sp, %u, ", a); if (tl->t->tok == CSTR) { sbuf_printf(tl->fc, "%*.*s);\n", tl->t->e - tl->t->b, @@ -949,7 +949,7 @@ NextToken(tl); } else sbuf_printf(tl->fc, "(const char *)0);\n"); - I(tl); sbuf_printf(tl->fc, "VRT_done(sess, HND_Error);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sp, HND_Error);\n"); return; case T_SWITCH_CONFIG: ExpectErr(tl, ID); @@ -963,10 +963,10 @@ ExpectErr(tl, ID); AddRef(tl, tl->t, R_FUNC); I(tl); - sbuf_printf(tl->fc, "VCL_function_%*.*s(VCL_PASS_ARGS);\n", + sbuf_printf(tl->fc, "VGC_function_%*.*s(sp);\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); - I(tl); sbuf_printf(tl->fc, "if (sess->done)\n"); + I(tl); sbuf_printf(tl->fc, "if (sp->done)\n"); I(tl); sbuf_printf(tl->fc, "\treturn;\n"); NextToken(tl); return; @@ -1035,7 +1035,7 @@ case BACKEND: if (tl->t->tok == '=') { NextToken(tl); - sbuf_printf(tl->fc, "= &VCL_backend_%*.*s;\n", + sbuf_printf(tl->fc, "= &VGC_backend_%*.*s;\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); NextToken(tl); @@ -1187,16 +1187,16 @@ t_be = tl->t; AddDef(tl, tl->t, R_BACKEND); I(tl); - sbuf_printf(tl->fh, "static struct backend VCL_backend_%*.*s;\n", + sbuf_printf(tl->fh, "static struct backend VGC_backend_%*.*s;\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); - sbuf_printf(tl->fc, "static struct backend VCL_backend_%*.*s;\n", + sbuf_printf(tl->fc, "static struct backend VGC_backend_%*.*s;\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); sbuf_printf(tl->fc, "static void\n"); I(tl); sbuf_printf(tl->fc, - "VCL_init_backend_%*.*s (struct backend *backend)\n", + "VGC_init_backend_%*.*s (struct backend *backend)\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); I(tl); @@ -1291,13 +1291,13 @@ NextToken(tl); ExpectErr(tl, ID); AddDef(tl, tl->t, R_FUNC); - sbuf_printf(tl->fh, "static void VCL_function_%*.*s (VCL_FARGS);\n", + sbuf_printf(tl->fh, "static void VGC_function_%*.*s (struct sess *sp);\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); I(tl); sbuf_printf(tl->fc, "static void\n"); I(tl); - sbuf_printf(tl->fc, "VCL_function_%*.*s (VCL_FARGS)\n", + sbuf_printf(tl->fc, "VGC_function_%*.*s (struct sess *sp)\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); NextToken(tl); @@ -1550,7 +1550,7 @@ sbuf_printf(tl->fc, "\nstatic void\n" - "VCL_Init(void)\n" + "VGC_Init(void)\n" "{\n\n"); TAILQ_FOREACH(r, &tl->refs, list) { @@ -1561,7 +1561,7 @@ break; case R_BACKEND: sbuf_printf(tl->fc, - "\tVCL_init_backend_%*.*s(&VCL_backend_%*.*s);\n", + "\tVGC_init_backend_%*.*s(&VGC_backend_%*.*s);\n", r->name->e - r->name->b, r->name->e - r->name->b, r->name->b, r->name->e - r->name->b, @@ -1582,13 +1582,13 @@ sbuf_printf(tl->fc, "\t.magic = VCL_CONF_MAGIC,\n"); sbuf_printf(tl->fc, - "\t.init_func = VCL_Init,\n"); - sbuf_printf(tl->fc, "\t.recv_func = VCL_function_vcl_recv,\n"); - sbuf_printf(tl->fc, "\t.hit_func = VCL_function_vcl_hit,\n"); - sbuf_printf(tl->fc, "\t.miss_func = VCL_function_vcl_miss,\n"); - sbuf_printf(tl->fc, "\t.fetch_func = VCL_function_vcl_fetch,\n"); + "\t.init_func = VGC_Init,\n"); + sbuf_printf(tl->fc, "\t.recv_func = VGC_function_vcl_recv,\n"); + sbuf_printf(tl->fc, "\t.hit_func = VGC_function_vcl_hit,\n"); + sbuf_printf(tl->fc, "\t.miss_func = VGC_function_vcl_miss,\n"); + sbuf_printf(tl->fc, "\t.fetch_func = VGC_function_vcl_fetch,\n"); sbuf_printf(tl->fc, - "\t.default_backend = &VCL_backend_default,\n"); + "\t.default_backend = &VGC_backend_default,\n"); sbuf_printf(tl->fc, "\t.ref = VGC_ref,\n"); sbuf_printf(tl->fc, Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-20 09:15:39 UTC (rev 207) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-20 09:25:21 UTC (rev 208) @@ -475,16 +475,13 @@ fputs(" struct vbe *vbe;\n", f); fputs("};\n", f); fputs("\n", f); - fputs("#define VCL_FARGS struct sess *sess\n", f); - fputs("#define VCL_PASS_ARGS sess\n", f); - fputs("\n", f); fputs("#if 0\n", f); fputs("int ip_match(unsigned, struct vcl_acl *);\n", f); fputs("int string_match(const char *, const char *);\n", f); fputs("#endif\n", f); fputs("\n", f); fputs("typedef void vcl_init_f(void);\n", f); - fputs("typedef void vcl_func_f(VCL_FARGS);\n", f); + fputs("typedef void vcl_func_f(struct sess *sp);\n", f); fputs("\n", f); fputs("struct VCL_conf {\n", f); fputs(" unsigned magic;\n", f); @@ -520,18 +517,18 @@ fputs("};\n", f); fputs("\n", f); fputs("void VRT_count(struct sess *, unsigned);\n", f); - fputs("void VRT_no_cache(VCL_FARGS);\n", f); - fputs("void VRT_no_new_cache(VCL_FARGS);\n", f); + fputs("void VRT_no_cache(struct sess *);\n", f); + fputs("void VRT_no_new_cache(struct sess *);\n", f); fputs("#if 0\n", f); fputs("int ip_match(unsigned, struct vcl_acl *);\n", f); fputs("int string_match(const char *, const char *);\n", f); fputs("#endif\n", f); fputs("int VRT_rewrite(const char *, const char *);\n", f); - fputs("void VRT_error(VCL_FARGS, unsigned, const char *);\n", f); + fputs("void VRT_error(struct sess *, unsigned, const char *);\n", f); fputs("int VRT_switch_config(const char *);\n", f); fputs("\n", f); - fputs("char *VRT_GetHdr(VCL_FARGS, const char *);\n", f); - fputs("char *VRT_GetReq(VCL_FARGS);\n", f); + fputs("char *VRT_GetHdr(struct sess *, const char *);\n", f); + fputs("char *VRT_GetReq(struct sess *);\n", f); fputs("\n", f); fputs("#define VRT_done(sess, hand) \\\n", f); fputs(" do { \\\n", f); From phk at projects.linpro.no Tue Jun 20 09:28:00 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 20 Jun 2006 11:28:00 +0200 (CEST) Subject: r209 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060620092800.6E7D41EC235@projects.linpro.no> Author: phk Date: 2006-06-20 11:28:00 +0200 (Tue, 20 Jun 2006) New Revision: 209 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/include/libvcl.h trunk/varnish-cache/lib/libvcl/vcl_compile.c Log: Rename the VCL compilers public functions to VCC prefix Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-20 09:25:21 UTC (rev 208) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-20 09:28:00 UTC (rev 209) @@ -113,7 +113,7 @@ assert(buf != NULL); sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(sb != NULL); - vf = VCL_Compile(sb, buf, NULL); + vf = VCC_Compile(sb, buf, NULL); sbuf_finish(sb); if (sbuf_len(sb) > 0) { fprintf(stderr, "%s", sbuf_data(sb)); @@ -134,7 +134,7 @@ sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(sb != NULL); - vf = VCL_Compile(sb, av[3], NULL); + vf = VCC_Compile(sb, av[3], NULL); sbuf_finish(sb); if (sbuf_len(sb) > 0) { cli_out(cli, "%s", sbuf_data(sb)); @@ -156,7 +156,7 @@ sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(sb != NULL); - vf = VCL_CompileFile(sb, av[3]); + vf = VCC_CompileFile(sb, av[3]); sbuf_finish(sb); if (sbuf_len(sb) > 0) { cli_out(cli, "%s", sbuf_data(sb)); @@ -177,7 +177,7 @@ sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(sb != NULL); - vf = VCL_CompileFile(sb, fflag); + vf = VCC_CompileFile(sb, fflag); sbuf_finish(sb); if (sbuf_len(sb) > 0) { fprintf(stderr, "%s", sbuf_data(sb)); @@ -392,7 +392,7 @@ register_printf_render_std((const unsigned char *)"HVQ"); - VCL_InitCompile(); + VCC_InitCompile(); while ((o = getopt(argc, argv, "b:df:p:s:")) != -1) switch (o) { Modified: trunk/varnish-cache/include/libvcl.h =================================================================== --- trunk/varnish-cache/include/libvcl.h 2006-06-20 09:25:21 UTC (rev 208) +++ trunk/varnish-cache/include/libvcl.h 2006-06-20 09:28:00 UTC (rev 209) @@ -2,8 +2,8 @@ * $Id$ */ -char *VCL_Compile(struct sbuf *sb, const char *b, const char *e); -char *VCL_CompileFile(struct sbuf *sb, const char *fn); -void VCL_InitCompile(void); +char *VCC_Compile(struct sbuf *sb, const char *b, const char *e); +char *VCC_CompileFile(struct sbuf *sb, const char *fn); +void VCC_InitCompile(void); Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-20 09:25:21 UTC (rev 208) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-20 09:28:00 UTC (rev 209) @@ -1599,7 +1599,7 @@ /*--------------------------------------------------------------------*/ char * -VCL_Compile(struct sbuf *sb, const char *b, const char *e) +VCC_Compile(struct sbuf *sb, const char *b, const char *e) { struct tokenlist tokens; struct ref *r; @@ -1684,7 +1684,7 @@ /*--------------------------------------------------------------------*/ char * -VCL_CompileFile(struct sbuf *sb, const char *fn) +VCC_CompileFile(struct sbuf *sb, const char *fn) { char *f, *r; int fd, i; @@ -1702,7 +1702,7 @@ i = read(fd, f, st.st_size); assert(i == st.st_size); f[i] = '\0'; - r = VCL_Compile(sb, f, NULL); + r = VCC_Compile(sb, f, NULL); free(f); return (r); } @@ -1710,7 +1710,7 @@ /*--------------------------------------------------------------------*/ void -VCL_InitCompile(void) +VCC_InitCompile(void) { struct var *v; From phk at projects.linpro.no Tue Jun 20 09:41:43 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 20 Jun 2006 11:41:43 +0200 (CEST) Subject: r210 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060620094143.4A51B1EC235@projects.linpro.no> Author: phk Date: 2006-06-20 11:41:43 +0200 (Tue, 20 Jun 2006) New Revision: 210 Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/include/vcl_lang.h trunk/varnish-cache/include/vrt.h trunk/varnish-cache/lib/libvcl/vcl_compile.c trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c Log: Work towards making struct sess opaque to the generated code. Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-20 09:28:00 UTC (rev 209) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-20 09:41:43 UTC (rev 210) @@ -67,3 +67,12 @@ assert(http_GetReq(sp->http, &p)); return (p); } + +/*--------------------------------------------------------------------*/ + +void +VRT_handling(struct sess *sp, enum handling hand) +{ + + sp->handling = hand; +} Modified: trunk/varnish-cache/include/vcl_lang.h =================================================================== --- trunk/varnish-cache/include/vcl_lang.h 2006-06-20 09:28:00 UTC (rev 209) +++ trunk/varnish-cache/include/vcl_lang.h 2006-06-20 09:41:43 UTC (rev 210) @@ -47,8 +47,6 @@ enum handling handling; - char done; - TAILQ_ENTRY(sess) list; sesscb_f *sesscb; @@ -82,7 +80,7 @@ #endif typedef void vcl_init_f(void); -typedef void vcl_func_f(struct sess *sp); +typedef int vcl_func_f(struct sess *sp); struct VCL_conf { unsigned magic; Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2006-06-20 09:28:00 UTC (rev 209) +++ trunk/varnish-cache/include/vrt.h 2006-06-20 09:41:43 UTC (rev 210) @@ -31,10 +31,10 @@ char *VRT_GetHdr(struct sess *, const char *); char *VRT_GetReq(struct sess *); +void VRT_handling(struct sess *sp, enum handling hand); -#define VRT_done(sess, hand) \ +#define VRT_done(sp, hand) \ do { \ - sess->handling = hand; \ - sess->done = 1; \ - return; \ + VRT_handling(sp, hand); \ + return (1); \ } while (0) Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-20 09:28:00 UTC (rev 209) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-20 09:41:43 UTC (rev 210) @@ -962,12 +962,11 @@ case T_CALL: ExpectErr(tl, ID); AddRef(tl, tl->t, R_FUNC); - I(tl); - sbuf_printf(tl->fc, "VGC_function_%*.*s(sp);\n", + I(tl); sbuf_printf(tl->fc, + "if (VGC_function_%*.*s(sp))\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); - I(tl); sbuf_printf(tl->fc, "if (sp->done)\n"); - I(tl); sbuf_printf(tl->fc, "\treturn;\n"); + I(tl); sbuf_printf(tl->fc, "\treturn (1);\n"); NextToken(tl); return; case T_REWRITE: @@ -1291,13 +1290,11 @@ NextToken(tl); ExpectErr(tl, ID); AddDef(tl, tl->t, R_FUNC); - sbuf_printf(tl->fh, "static void VGC_function_%*.*s (struct sess *sp);\n", + sbuf_printf(tl->fh, "static int VGC_function_%*.*s (struct sess *sp);\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); - I(tl); - sbuf_printf(tl->fc, "static void\n"); - I(tl); - sbuf_printf(tl->fc, "VGC_function_%*.*s (struct sess *sp)\n", + I(tl); sbuf_printf(tl->fc, "static int\n"); + I(tl); sbuf_printf(tl->fc, "VGC_function_%*.*s (struct sess *sp)\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); NextToken(tl); Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-20 09:28:00 UTC (rev 209) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-20 09:41:43 UTC (rev 210) @@ -446,8 +446,6 @@ fputs("\n", f); fputs(" enum handling handling;\n", f); fputs("\n", f); - fputs(" char done;\n", f); - fputs("\n", f); fputs(" TAILQ_ENTRY(sess) list;\n", f); fputs("\n", f); fputs(" sesscb_f *sesscb;\n", f); @@ -481,7 +479,7 @@ fputs("#endif\n", f); fputs("\n", f); fputs("typedef void vcl_init_f(void);\n", f); - fputs("typedef void vcl_func_f(struct sess *sp);\n", f); + fputs("typedef int vcl_func_f(struct sess *sp);\n", f); fputs("\n", f); fputs("struct VCL_conf {\n", f); fputs(" unsigned magic;\n", f); @@ -529,11 +527,11 @@ fputs("\n", f); fputs("char *VRT_GetHdr(struct sess *, const char *);\n", f); fputs("char *VRT_GetReq(struct sess *);\n", f); + fputs("void VRT_handling(struct sess *sp, enum handling hand);\n", f); fputs("\n", f); - fputs("#define VRT_done(sess, hand) \\\n", f); + fputs("#define VRT_done(sp, hand) \\\n", f); fputs(" do { \\\n", f); - fputs(" sess->handling = hand; \\\n", f); - fputs(" sess->done = 1; \\\n", f); - fputs(" return; \\\n", f); + fputs(" VRT_handling(sp, hand); \\\n", f); + fputs(" return (1); \\\n", f); fputs(" } while (0)\n", f); } From phk at projects.linpro.no Tue Jun 20 10:31:50 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 20 Jun 2006 12:31:50 +0200 (CEST) Subject: r211 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060620103150.BDCBE1EC22D@projects.linpro.no> Author: phk Date: 2006-06-20 12:31:50 +0200 (Tue, 20 Jun 2006) New Revision: 211 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/include/vcl_lang.h trunk/varnish-cache/include/vrt.h trunk/varnish-cache/lib/libvcl/vcl_compile.c trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c Log: At the expense of some complexity and a small runtime overhead, isolate the compiled code from the internal structures of the cache process through of VRT functions. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-20 09:41:43 UTC (rev 210) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-20 10:31:50 UTC (rev 211) @@ -2,6 +2,8 @@ * $Id$ */ +#include + struct event_base; struct sbuf; @@ -55,6 +57,73 @@ */ extern struct stevedore *stevedore; +/* Storage -----------------------------------------------------------*/ + +struct sess; +typedef void sesscb_f(struct sess *sp); + +#define VCA_ADDRBUFSIZE 32 + +struct object { + unsigned char hash[16]; + unsigned refcnt; + unsigned valid; + unsigned cacheable; + + unsigned busy; + unsigned len; + + char *header; + + TAILQ_HEAD(, storage) store; +}; + +#define HND_Error (1 << 0) +#define HND_Pipe (1 << 1) +#define HND_Pass (1 << 2) +#define HND_Lookup (1 << 3) +#define HND_Fetch (1 << 4) +#define HND_Insert (1 << 5) +#define HND_Deliver (1 << 6) + +struct sess { + int fd; + + /* formatted ascii client address */ + char addr[VCA_ADDRBUFSIZE]; + + /* HTTP request */ + struct http *http; + + unsigned handling; + + TAILQ_ENTRY(sess) list; + + sesscb_f *sesscb; + + struct backend *backend; + struct object *obj; + struct VCL_conf *vcl; + + /* Various internal stuff */ + struct event *rd_e; + struct sessmem *mem; +}; + +struct backend { + const char *hostname; + const char *portname; + struct addrinfo *addr; + unsigned ip; + double responsetime; + double timeout; + double bandwidth; + int down; + + /* internal stuff */ + struct vbe *vbe; +}; + /* Prototypes etc ----------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-20 09:41:43 UTC (rev 210) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-20 10:31:50 UTC (rev 211) @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -25,7 +26,6 @@ #include "config.h" #include "compat.h" #include "libvarnish.h" -#include "vcl_lang.h" #include "heritage.h" #include "shmlog.h" #include "cache.h" Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-20 09:41:43 UTC (rev 210) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-20 10:31:50 UTC (rev 211) @@ -16,7 +16,6 @@ #include "libvarnish.h" #include "shmlog.h" -#include "vcl_lang.h" #include "cache.h" static unsigned http_bufsize = 4096; Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-20 09:41:43 UTC (rev 210) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-20 10:31:50 UTC (rev 211) @@ -102,7 +102,7 @@ http_Dissect(sp->http, sp->fd, 1); - sp->backend = sp->vcl->default_backend; + sp->backend = sp->vcl->backend[0]; VCL_recv_method(sp); Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-20 09:41:43 UTC (rev 210) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-20 10:31:50 UTC (rev 211) @@ -16,9 +16,9 @@ #include "cli.h" #include "cli_priv.h" #include "shmlog.h" -#include "vcl_lang.h" #include "vrt.h" #include "libvarnish.h" +#include "vcl_lang.h" #include "cache.h" /*--------------------------------------------------------------------*/ @@ -71,8 +71,58 @@ /*--------------------------------------------------------------------*/ void -VRT_handling(struct sess *sp, enum handling hand) +VRT_handling(struct sess *sp, unsigned hand) { - sp->handling = hand; + assert(!(hand & (hand -1))); /* must be power of two */ + switch (hand) { +#define FOO(a,b) case VRT_H_##a: sp->handling = HND_##b; break; + FOO(error, Error); + FOO(pipe, Pipe); + FOO(pass, Pass); + FOO(lookup, Lookup); + FOO(fetch, Fetch); + FOO(insert, Insert); + FOO(deliver, Deliver); +#undef FOO + default: + assert(hand == 0); + } } + +int +VRT_obj_valid(struct sess *sp) +{ + return (sp->obj->valid); +} + +int +VRT_obj_cacheable(struct sess *sp) +{ + return (sp->obj->cacheable); +} + +void +VRT_set_backend_hostname(struct backend *be, const char *h) +{ + be->hostname = h; +} + +void +VRT_set_backend_portname(struct backend *be, const char *p) +{ + be->portname = p; +} + +void +VRT_alloc_backends(struct VCL_conf *cp) +{ + int i; + + cp->backend = calloc(sizeof *cp->backend, cp->nbackend); + assert(cp->backend != NULL); + for (i = 0; i < cp->nbackend; i++) { + cp->backend[i] = calloc(sizeof *cp->backend[i], 1); + assert(cp->backend[i] != NULL); + } +} Modified: trunk/varnish-cache/include/vcl_lang.h =================================================================== --- trunk/varnish-cache/include/vcl_lang.h 2006-06-20 09:41:43 UTC (rev 210) +++ trunk/varnish-cache/include/vcl_lang.h 2006-06-20 10:31:50 UTC (rev 211) @@ -5,80 +5,8 @@ * XXX: *MUST* be rerun. */ -/* XXX: This include is bad. The VCL compiler shouldn't know about it. */ -#include - struct sess; -typedef void sesscb_f(struct sess *sp); -#define VCA_ADDRBUFSIZE 32 - -struct object { - unsigned char hash[16]; - unsigned refcnt; - unsigned valid; - unsigned cacheable; - - unsigned busy; - unsigned len; - - char *header; - - TAILQ_HEAD(, storage) store; -}; -enum handling { - HND_Error = (1 << 0), - HND_Pipe = (1 << 1), - HND_Pass = (1 << 2), - HND_Lookup = (1 << 3), - HND_Fetch = (1 << 4), - HND_Insert = (1 << 5), - HND_Deliver = (1 << 6), -}; - -struct sess { - int fd; - - /* formatted ascii client address */ - char addr[VCA_ADDRBUFSIZE]; - - /* HTTP request */ - struct http *http; - - enum handling handling; - - TAILQ_ENTRY(sess) list; - - sesscb_f *sesscb; - - struct backend *backend; - struct object *obj; - struct VCL_conf *vcl; - - /* Various internal stuff */ - struct event *rd_e; - struct sessmem *mem; -}; - -struct backend { - const char *hostname; - const char *portname; - struct addrinfo *addr; - unsigned ip; - double responsetime; - double timeout; - double bandwidth; - int down; - - /* internal stuff */ - struct vbe *vbe; -}; - -#if 0 -int ip_match(unsigned, struct vcl_acl *); -int string_match(const char *, const char *); -#endif - typedef void vcl_init_f(void); typedef int vcl_func_f(struct sess *sp); @@ -90,7 +18,8 @@ vcl_func_f *hit_func; vcl_func_f *miss_func; vcl_func_f *fetch_func; - struct backend *default_backend; + struct backend **backend; + unsigned nbackend; struct vrt_ref *ref; unsigned nref; unsigned busy; Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2006-06-20 09:41:43 UTC (rev 210) +++ trunk/varnish-cache/include/vrt.h 2006-06-20 10:31:50 UTC (rev 211) @@ -6,6 +6,18 @@ * XXX: *MUST* be rerun. */ +#define VRT_H_error (1 << 0) +#define VRT_H_pipe (1 << 1) +#define VRT_H_pass (1 << 2) +#define VRT_H_lookup (1 << 3) +#define VRT_H_fetch (1 << 4) +#define VRT_H_insert (1 << 5) +#define VRT_H_deliver (1 << 6) + +struct sess; +struct backend; +struct VCL_conf; + struct vrt_ref { unsigned line; unsigned pos; @@ -31,8 +43,15 @@ char *VRT_GetHdr(struct sess *, const char *); char *VRT_GetReq(struct sess *); -void VRT_handling(struct sess *sp, enum handling hand); +void VRT_handling(struct sess *sp, unsigned hand); +int VRT_obj_valid(struct sess *); +int VRT_obj_cachable(struct sess *); +void VRT_set_backend_hostname(struct backend *, const char *); +void VRT_set_backend_portname(struct backend *, const char *); + +void VRT_alloc_backends(struct VCL_conf *cp); + #define VRT_done(sp, hand) \ do { \ VRT_handling(sp, hand); \ Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-20 09:41:43 UTC (rev 210) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-20 10:31:50 UTC (rev 211) @@ -76,6 +76,7 @@ TAILQ_HEAD(, ref) refs; struct sbuf *sb; int err; + int nbackend; }; enum var_type { @@ -93,12 +94,6 @@ HEADER }; -struct var { - const char *name; - enum var_type fmt; - int len; - const char *cname; -}; enum ref_type { R_FUNC, @@ -114,17 +109,26 @@ TAILQ_ENTRY(ref) list; }; +struct var { + const char *name; + enum var_type fmt; + int len; + const char *rname; + const char *lname; +}; static struct var be_vars[] = { - { "backend.host", HOSTNAME, 0, "backend->hostname" }, - { "backend.port", PORTNAME, 0, "backend->portname" }, + { "backend.host", + HOSTNAME, 0, NULL, "VRT_set_backend_hostname(backend, %s)" }, + { "backend.port", + PORTNAME, 0, NULL, "VRT_set_backend_portname(backend, %s)" }, }; static struct var vars[] = { { "req.request", STRING, 0, "VRT_GetReq(sp)" }, - { "obj.valid", BOOL, 0, "sp->obj->valid" }, - { "obj.cacheable", BOOL, 0, "sp->obj->cacheable" }, + { "obj.valid", BOOL, 0, "VRT_obj_valid(sp)" }, + { "obj.cacheable", BOOL, 0, "VRT_obj_cacheable(sp)" }, { "req.http.", HEADER, 0, NULL }, #if 0 { "req.ttlfactor", FLOAT, 0, "req->ttlfactor" }, @@ -569,7 +573,7 @@ v->fmt = STRING; asprintf(&p, "VRT_GetHdr(sp, \"%s\")", v->name + vh->len); assert(p != NULL); - v->cname = p; + v->rname = p; return (v); } @@ -648,7 +652,7 @@ I(tl); AddRef(tl, tl->t, R_ACL); sbuf_printf(tl->fc, "ip_match(%s, acl_%*.*s)\n", - vp->cname, + vp->rname, tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); NextToken(tl); @@ -657,7 +661,7 @@ case T_NEQ: I(tl); sbuf_printf(tl->fc, "%s %*.*s ", - vp->cname, + vp->rname, tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); NextToken(tl); @@ -682,7 +686,7 @@ switch (tl->t->tok) { case '~': - I(tl); sbuf_printf(tl->fc, "string_match(%s, ", vp->cname); + I(tl); sbuf_printf(tl->fc, "string_match(%s, ", vp->rname); NextToken(tl); ExpectErr(tl, CSTR); sbuf_printf(tl->fc, "%*.*s)\n", @@ -694,7 +698,7 @@ case T_NEQ: I(tl); sbuf_printf(tl->fc, "%sstrcmp(%s, ", - tl->t->tok == T_EQ ? "!" : "", vp->cname); + tl->t->tok == T_EQ ? "!" : "", vp->rname); NextToken(tl); ExpectErr(tl, CSTR); sbuf_printf(tl->fc, "%*.*s)\n", @@ -703,7 +707,7 @@ NextToken(tl); break; default: - I(tl); sbuf_printf(tl->fc, "%s != (void*)0", vp->cname); + I(tl); sbuf_printf(tl->fc, "%s != (void*)0", vp->rname); break; } } @@ -713,7 +717,7 @@ { I(tl); - sbuf_printf(tl->fc, "%s ", vp->cname); + sbuf_printf(tl->fc, "%s ", vp->rname); switch (tl->t->tok) { case T_EQ: case T_NEQ: @@ -764,7 +768,7 @@ { I(tl); - sbuf_printf(tl->fc, "%s\n", vp->cname); + sbuf_printf(tl->fc, "%s\n", vp->rname); } static void @@ -921,19 +925,13 @@ sbuf_printf(tl->fc, "VCL_no_cache(sp);\n"); return; case T_DELIVER: - I(tl); sbuf_printf(tl->fc, "VRT_done(sp, HND_Deliver);\n"); - return; case T_LOOKUP: - I(tl); sbuf_printf(tl->fc, "VRT_done(sp, HND_Lookup);\n"); - return; case T_PASS: - I(tl); sbuf_printf(tl->fc, "VRT_done(sp, HND_Pass);\n"); - return; case T_FETCH: - I(tl); sbuf_printf(tl->fc, "VRT_done(sp, HND_Fetch);\n"); - return; case T_INSERT: - I(tl); sbuf_printf(tl->fc, "VRT_done(sp, HND_Insert);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sp, VRT_H_%*.*s);\n", + at->e - at->b, + at->e - at->b, at->b); return; case T_ERROR: if (tl->t->tok == CNUM) @@ -949,7 +947,7 @@ NextToken(tl); } else sbuf_printf(tl->fc, "(const char *)0);\n"); - I(tl); sbuf_printf(tl->fc, "VRT_done(sp, HND_Error);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sp, VRT_H_error);\n"); return; case T_SWITCH_CONFIG: ExpectErr(tl, ID); @@ -988,7 +986,7 @@ ERRCHK(tl); assert(vp != NULL); I(tl); - sbuf_printf(tl->fc, "%s ", vp->cname); + sbuf_printf(tl->fc, "%s ", vp->rname); NextToken(tl); switch (vp->fmt) { case INT: @@ -1186,20 +1184,22 @@ t_be = tl->t; AddDef(tl, tl->t, R_BACKEND); I(tl); - sbuf_printf(tl->fh, "static struct backend VGC_backend_%*.*s;\n", + sbuf_printf(tl->fh, + "#define VGC_backend_%*.*s (VCL_conf.backend[%d])\n", tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); - sbuf_printf(tl->fc, "static struct backend VGC_backend_%*.*s;\n", + tl->t->e - tl->t->b, tl->t->b, tl->nbackend); + sbuf_printf(tl->fc, "static void\n"); + I(tl); sbuf_printf(tl->fc, + "VGC_init_backend_%*.*s (void)\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); - sbuf_printf(tl->fc, "static void\n"); I(tl); - sbuf_printf(tl->fc, - "VGC_init_backend_%*.*s (struct backend *backend)\n", + sbuf_printf(tl->fc, "{\n"); + I(tl); sbuf_printf(tl->fc, + "\tstruct backend *backend = VGC_backend_%*.*s;\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); - I(tl); - sbuf_printf(tl->fc, "{\n"); + I(tl); sbuf_printf(tl->fc, "\tconst char *p;\n"); NextToken(tl); ExpectErr(tl, '{'); NextToken(tl); @@ -1220,21 +1220,24 @@ ExpectErr(tl, CSTR); t_host = tl->t; host = EncString(tl->t); - I(tl); - sbuf_printf(tl->fc, "\t%s = %*.*s;\n", - vp->cname, + I(tl); sbuf_printf(tl->fc, "\tp = %*.*s;\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); + I(tl); sbuf_printf(tl->fc, "\t"); + sbuf_printf(tl->fc, vp->lname, "p"); + sbuf_printf(tl->fc, ";\n"); NextToken(tl); break; case PORTNAME: ExpectErr(tl, CSTR); t_port = tl->t; port = EncString(tl->t); - sbuf_printf(tl->fc, "\t%s = %*.*s;\n", - vp->cname, + I(tl); sbuf_printf(tl->fc, "\tp = %*.*s;\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); + I(tl); sbuf_printf(tl->fc, "\t"); + sbuf_printf(tl->fc, vp->lname, "p"); + sbuf_printf(tl->fc, ";\n"); NextToken(tl); break; default: @@ -1279,6 +1282,7 @@ I(tl); sbuf_printf(tl->fc, "}\n"); sbuf_printf(tl->fc, "\n"); + tl->nbackend++; } /*--------------------------------------------------------------------*/ @@ -1549,6 +1553,9 @@ "\nstatic void\n" "VGC_Init(void)\n" "{\n\n"); + + sbuf_printf(tl->fc, + "\tVRT_alloc_backends(&VCL_conf);\n"); TAILQ_FOREACH(r, &tl->refs, list) { switch(r->type) { @@ -1558,10 +1565,8 @@ break; case R_BACKEND: sbuf_printf(tl->fc, - "\tVGC_init_backend_%*.*s(&VGC_backend_%*.*s);\n", + "\tVGC_init_backend_%*.*s();\n", r->name->e - r->name->b, - r->name->e - r->name->b, r->name->b, - r->name->e - r->name->b, r->name->e - r->name->b, r->name->b); break; } @@ -1585,7 +1590,7 @@ sbuf_printf(tl->fc, "\t.miss_func = VGC_function_vcl_miss,\n"); sbuf_printf(tl->fc, "\t.fetch_func = VGC_function_vcl_fetch,\n"); sbuf_printf(tl->fc, - "\t.default_backend = &VGC_backend_default,\n"); + "\t.nbackend = %d,\n", tl->nbackend); sbuf_printf(tl->fc, "\t.ref = VGC_ref,\n"); sbuf_printf(tl->fc, @@ -1616,6 +1621,8 @@ tokens.fh = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(tokens.fh != NULL); + sbuf_printf(tokens.fc, "extern struct VCL_conf VCL_conf;\n"); + tokens.b = b; if (e == NULL) e = strchr(b, '\0'); Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-20 09:41:43 UTC (rev 210) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-20 10:31:50 UTC (rev 211) @@ -404,80 +404,8 @@ fputs(" * XXX: *MUST* be rerun.\n", f); fputs(" */\n", f); fputs("\n", f); - fputs("/* XXX: This include is bad. The VCL compiler shouldn't know about it. */\n", f); - fputs("#include \n", f); - fputs("\n", f); fputs("struct sess;\n", f); - fputs("typedef void sesscb_f(struct sess *sp);\n", f); fputs("\n", f); - fputs("#define VCA_ADDRBUFSIZE 32\n", f); - fputs("\n", f); - fputs("struct object { \n", f); - fputs(" unsigned char hash[16];\n", f); - fputs(" unsigned refcnt;\n", f); - fputs(" unsigned valid;\n", f); - fputs(" unsigned cacheable;\n", f); - fputs("\n", f); - fputs(" unsigned busy;\n", f); - fputs(" unsigned len;\n", f); - fputs("\n", f); - fputs(" char *header;\n", f); - fputs("\n", f); - fputs(" TAILQ_HEAD(, storage) store;\n", f); - fputs("};\n", f); - fputs("enum handling {\n", f); - fputs(" HND_Error = (1 << 0),\n", f); - fputs(" HND_Pipe = (1 << 1),\n", f); - fputs(" HND_Pass = (1 << 2),\n", f); - fputs(" HND_Lookup = (1 << 3),\n", f); - fputs(" HND_Fetch = (1 << 4),\n", f); - fputs(" HND_Insert = (1 << 5),\n", f); - fputs(" HND_Deliver = (1 << 6),\n", f); - fputs("};\n", f); - fputs("\n", f); - fputs("struct sess {\n", f); - fputs(" int fd;\n", f); - fputs("\n", f); - fputs(" /* formatted ascii client address */\n", f); - fputs(" char addr[VCA_ADDRBUFSIZE];\n", f); - fputs("\n", f); - fputs(" /* HTTP request */\n", f); - fputs(" struct http *http;\n", f); - fputs("\n", f); - fputs(" enum handling handling;\n", f); - fputs("\n", f); - fputs(" TAILQ_ENTRY(sess) list;\n", f); - fputs("\n", f); - fputs(" sesscb_f *sesscb;\n", f); - fputs("\n", f); - fputs(" struct backend *backend;\n", f); - fputs(" struct object *obj;\n", f); - fputs(" struct VCL_conf *vcl;\n", f); - fputs("\n", f); - fputs(" /* Various internal stuff */\n", f); - fputs(" struct event *rd_e;\n", f); - fputs(" struct sessmem *mem;\n", f); - fputs("};\n", f); - fputs("\n", f); - fputs("struct backend {\n", f); - fputs(" const char *hostname;\n", f); - fputs(" const char *portname;\n", f); - fputs(" struct addrinfo *addr;\n", f); - fputs(" unsigned ip;\n", f); - fputs(" double responsetime;\n", f); - fputs(" double timeout;\n", f); - fputs(" double bandwidth;\n", f); - fputs(" int down;\n", f); - fputs("\n", f); - fputs(" /* internal stuff */\n", f); - fputs(" struct vbe *vbe;\n", f); - fputs("};\n", f); - fputs("\n", f); - fputs("#if 0\n", f); - fputs("int ip_match(unsigned, struct vcl_acl *);\n", f); - fputs("int string_match(const char *, const char *);\n", f); - fputs("#endif\n", f); - fputs("\n", f); fputs("typedef void vcl_init_f(void);\n", f); fputs("typedef int vcl_func_f(struct sess *sp);\n", f); fputs("\n", f); @@ -489,7 +417,8 @@ fputs(" vcl_func_f *hit_func;\n", f); fputs(" vcl_func_f *miss_func;\n", f); fputs(" vcl_func_f *fetch_func;\n", f); - fputs(" struct backend *default_backend;\n", f); + fputs(" struct backend **backend;\n", f); + fputs(" unsigned nbackend;\n", f); fputs(" struct vrt_ref *ref;\n", f); fputs(" unsigned nref;\n", f); fputs(" unsigned busy;\n", f); @@ -502,6 +431,17 @@ fputs(" * XXX: *MUST* be rerun.\n", f); fputs(" */\n", f); fputs("\n", f); + fputs("#define VRT_H_error (1 << 0)\n", f); + fputs("#define VRT_H_pipe (1 << 1)\n", f); + fputs("#define VRT_H_pass (1 << 2)\n", f); + fputs("#define VRT_H_lookup (1 << 3)\n", f); + fputs("#define VRT_H_fetch (1 << 4)\n", f); + fputs("#define VRT_H_insert (1 << 5)\n", f); + fputs("#define VRT_H_deliver (1 << 6)\n", f); + fputs("\n", f); + fputs("struct sess;\n", f); + fputs("struct backend;\n", f); + fputs("\n", f); fputs("struct vrt_ref {\n", f); fputs(" unsigned line;\n", f); fputs(" unsigned pos;\n", f); @@ -527,8 +467,15 @@ fputs("\n", f); fputs("char *VRT_GetHdr(struct sess *, const char *);\n", f); fputs("char *VRT_GetReq(struct sess *);\n", f); - fputs("void VRT_handling(struct sess *sp, enum handling hand);\n", f); + fputs("void VRT_handling(struct sess *sp, unsigned hand);\n", f); + fputs("int VRT_obj_valid(struct sess *);\n", f); + fputs("int VRT_obj_cachable(struct sess *);\n", f); fputs("\n", f); + fputs("void VRT_set_backend_hostname(struct backend *, const char *);\n", f); + fputs("void VRT_set_backend_portname(struct backend *, const char *);\n", f); + fputs("\n", f); + fputs("void VRT_alloc_backends(struct VCL_conf *cp);\n", f); + fputs("\n", f); fputs("#define VRT_done(sp, hand) \\\n", f); fputs(" do { \\\n", f); fputs(" VRT_handling(sp, hand); \\\n", f); From phk at projects.linpro.no Tue Jun 20 11:39:20 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 20 Jun 2006 13:39:20 +0200 (CEST) Subject: r212 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060620113920.D161F1EC22E@projects.linpro.no> Author: phk Date: 2006-06-20 13:39:20 +0200 (Tue, 20 Jun 2006) New Revision: 212 Added: trunk/varnish-cache/include/vcl.h Removed: trunk/varnish-cache/include/vcl_lang.h Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_main.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_vcl.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishd/hash_simple_list.c trunk/varnish-cache/bin/varnishd/rfc2616.c trunk/varnish-cache/bin/varnishd/storage_file.c trunk/varnish-cache/bin/varnishd/storage_malloc.c trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/include/vrt.h trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl Log: Rename vcl_lang.h to vcl.h and include practically nowhere. Remove #include bogohandling in vcl_gen_fixed_token.tcl Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-06-20 11:39:20 UTC (rev 212) @@ -41,7 +41,6 @@ #include "libvarnish.h" #include "shmlog.h" -#include "vcl_lang.h" #include "cache.h" /* A backend connection */ Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-20 11:39:20 UTC (rev 212) @@ -20,7 +20,6 @@ #include "libvarnish.h" #include "shmlog.h" -#include "vcl_lang.h" #include "cache.h" /* Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-20 11:39:20 UTC (rev 212) @@ -19,7 +19,6 @@ #include "libvarnish.h" #include "heritage.h" #include "shmlog.h" -#include "vcl_lang.h" #include "cache.h" #include "cli_event.h" Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-06-20 11:39:20 UTC (rev 212) @@ -19,7 +19,6 @@ #include "libvarnish.h" #include "shmlog.h" -#include "vcl_lang.h" #include "cache.h" Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-06-20 11:39:20 UTC (rev 212) @@ -15,7 +15,6 @@ #include "libvarnish.h" #include "shmlog.h" -#include "vcl_lang.h" #include "cache.h" struct edir { Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-20 11:39:20 UTC (rev 212) @@ -16,7 +16,7 @@ #include "libvarnish.h" #include "shmlog.h" -#include "vcl_lang.h" +#include "vcl.h" #include "cache.h" static TAILQ_HEAD(, sess) shd = TAILQ_HEAD_INITIALIZER(shd); Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-20 11:39:20 UTC (rev 212) @@ -13,7 +13,7 @@ #include "cli.h" #include "cli_priv.h" #include "shmlog.h" -#include "vcl_lang.h" +#include "vcl.h" #include "libvarnish.h" #include "cache.h" Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-20 11:39:20 UTC (rev 212) @@ -17,8 +17,8 @@ #include "cli_priv.h" #include "shmlog.h" #include "vrt.h" +#include "vcl.h" #include "libvarnish.h" -#include "vcl_lang.h" #include "cache.h" /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-06-20 11:39:20 UTC (rev 212) @@ -10,7 +10,6 @@ #include #include -#include #include struct hsl_entry { Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-20 11:39:20 UTC (rev 212) @@ -5,7 +5,6 @@ #include #include #include -#include "vcl_lang.h" #include "cache.h" #include "libvarnish.h" Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-20 11:39:20 UTC (rev 212) @@ -21,7 +21,6 @@ #include #include -#include "vcl_lang.h" #include "libvarnish.h" #include "cache.h" Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-06-20 11:39:20 UTC (rev 212) @@ -9,7 +9,6 @@ #include #include -#include "vcl_lang.h" #include "cache.h" struct sma { Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-20 11:39:20 UTC (rev 212) @@ -25,8 +25,6 @@ #include #include -#include "vcl_lang.h" - #include "mgt.h" #include "heritage.h" #include "cli_event.h" Copied: trunk/varnish-cache/include/vcl.h (from rev 211, trunk/varnish-cache/include/vcl_lang.h) =================================================================== --- trunk/varnish-cache/include/vcl_lang.h 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/include/vcl.h 2006-06-20 11:39:20 UTC (rev 212) @@ -0,0 +1,28 @@ +/* + * $Id$ + * + * Interface to a compiled VCL program. + * + * XXX: When this file is changed, lib/libvcl/vcl_gen_fixed_token.tcl + * XXX: *MUST* be rerun. + */ + +struct sess; + +typedef void vcl_init_f(void); +typedef int vcl_func_f(struct sess *sp); + +struct VCL_conf { + unsigned magic; +#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */ + vcl_init_f *init_func; + vcl_func_f *recv_func; + vcl_func_f *hit_func; + vcl_func_f *miss_func; + vcl_func_f *fetch_func; + struct backend **backend; + unsigned nbackend; + struct vrt_ref *ref; + unsigned nref; + unsigned busy; +}; Deleted: trunk/varnish-cache/include/vcl_lang.h =================================================================== --- trunk/varnish-cache/include/vcl_lang.h 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/include/vcl_lang.h 2006-06-20 11:39:20 UTC (rev 212) @@ -1,26 +0,0 @@ -/* - * Stuff necessary to compile a VCL programs C code - * - * XXX: When this file is changed, lib/libvcl/vcl_gen_fixed_token.tcl - * XXX: *MUST* be rerun. - */ - -struct sess; - -typedef void vcl_init_f(void); -typedef int vcl_func_f(struct sess *sp); - -struct VCL_conf { - unsigned magic; -#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */ - vcl_init_f *init_func; - vcl_func_f *recv_func; - vcl_func_f *hit_func; - vcl_func_f *miss_func; - vcl_func_f *fetch_func; - struct backend **backend; - unsigned nbackend; - struct vrt_ref *ref; - unsigned nref; - unsigned busy; -}; Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/include/vrt.h 2006-06-20 11:39:20 UTC (rev 212) @@ -1,5 +1,6 @@ -/* $Id$ */ /* + * $Id$ + * * Runtime support for compiled VCL programs. * * XXX: When this file is changed, lib/libvcl/vcl_gen_fixed_token.tcl Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-20 11:39:20 UTC (rev 212) @@ -398,8 +398,10 @@ vcl_output_lang_h(FILE *f) { fputs("/*\n", f); - fputs(" * Stuff necessary to compile a VCL programs C code\n", f); + fputs(" * $Id$\n", f); fputs(" *\n", f); + fputs(" * Interface to a compiled VCL program.\n", f); + fputs(" *\n", f); fputs(" * XXX: When this file is changed, lib/libvcl/vcl_gen_fixed_token.tcl\n", f); fputs(" * XXX: *MUST* be rerun.\n", f); fputs(" */\n", f); @@ -423,8 +425,9 @@ fputs(" unsigned nref;\n", f); fputs(" unsigned busy;\n", f); fputs("};\n", f); - fputs("/* $Id$ */\n", f); fputs("/*\n", f); + fputs(" * $Id$ \n", f); + fputs(" *\n", f); fputs(" * Runtime support for compiled VCL programs.\n", f); fputs(" *\n", f); fputs(" * XXX: When this file is changed, lib/libvcl/vcl_gen_fixed_token.tcl\n", f); @@ -441,6 +444,7 @@ fputs("\n", f); fputs("struct sess;\n", f); fputs("struct backend;\n", f); + fputs("struct VCL_conf;\n", f); fputs("\n", f); fputs("struct vrt_ref {\n", f); fputs(" unsigned line;\n", f); Modified: trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-06-20 10:31:50 UTC (rev 211) +++ trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-06-20 11:39:20 UTC (rev 212) @@ -183,16 +183,6 @@ set fi [open $n] while {[gets $fi a] >= 0} { - if {"$a" == "#include "} { - puts "FOO $a" - set fx [open "../../include/http_headers.h"] - while {[gets $fx b] >= 0} { - regsub -all {"} $b {\"} b - puts $fo "\tfputs(\"$b\\n\", f);" - } - close $fx - continue - } regsub -all {\\} $a {\\\\} a puts $fo "\tfputs(\"$a\\n\", f);" } @@ -203,7 +193,7 @@ puts $fo "void" puts $fo "vcl_output_lang_h(FILE *f)" puts $fo "{" -copy_include ../../include/vcl_lang.h +copy_include ../../include/vcl.h copy_include ../../include/vrt.h puts $fo "}" From phk at projects.linpro.no Tue Jun 20 19:31:46 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 20 Jun 2006 21:31:46 +0200 (CEST) Subject: r213 - trunk/varnish-cache/bin/varnishd Message-ID: <20060620193146.63F411EC22D@projects.linpro.no> Author: phk Date: 2006-06-20 21:31:46 +0200 (Tue, 20 Jun 2006) New Revision: 213 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_main.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_shmlog.c trunk/varnish-cache/bin/varnishd/cli_event.c trunk/varnish-cache/bin/varnishd/flint.lnt trunk/varnish-cache/bin/varnishd/storage_file.c trunk/varnish-cache/bin/varnishd/varnishd.c Log: Polish things to silence FlexeLint a bit Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-20 11:39:20 UTC (rev 212) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-20 19:31:46 UTC (rev 213) @@ -60,7 +60,6 @@ /* Storage -----------------------------------------------------------*/ struct sess; -typedef void sesscb_f(struct sess *sp); #define VCA_ADDRBUFSIZE 32 @@ -99,8 +98,6 @@ TAILQ_ENTRY(sess) list; - sesscb_f *sesscb; - struct backend *backend; struct object *obj; struct VCL_conf *vcl; @@ -113,12 +110,14 @@ struct backend { const char *hostname; const char *portname; - struct addrinfo *addr; unsigned ip; +#if 0 + struct addrinfo *addr; double responsetime; double timeout; double bandwidth; int down; +#endif /* internal stuff */ struct vbe *vbe; @@ -138,7 +137,6 @@ /* cache_backend.c */ void VBE_Init(void); int VBE_GetFd(struct backend *bp, void **ptr); -void VBE_Pass(struct sess *sp); void VBE_ClosedFd(void *ptr); void VBE_RecycleFd(void *ptr); Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-06-20 11:39:20 UTC (rev 212) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-06-20 19:31:46 UTC (rev 213) @@ -78,7 +78,7 @@ * be a good thing in that case. */ -void +static void connect_to_backend(struct vbe_conn *vc, struct backend *bp) { struct addrinfo *res, *res0, hint; Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-20 11:39:20 UTC (rev 212) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-20 19:31:46 UTC (rev 213) @@ -139,6 +139,7 @@ i = bp - q; if (i == 0) { } else if (v > i) { + assert(i > 0); memcpy(p, q, i); p += i; st->len += i; Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-20 11:39:20 UTC (rev 212) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-20 19:31:46 UTC (rev 213) @@ -35,7 +35,7 @@ timer_keepalive(int a, short b, void *c) { - printf("%s(%d, %d, %p)\n", __func__, a, b, c); + printf("%s(%d, %d, %p)\n", __func__, a, (int)b, c); printf("Heeellloooo ? Ohh bother...\n"); exit (1); } Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-06-20 11:39:20 UTC (rev 212) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-06-20 19:31:46 UTC (rev 213) @@ -44,7 +44,7 @@ void PipeSession(struct worker *w, struct sess *sp) { - int fd, i, j; + int fd, i; void *fd_token; struct edir e1, e2; @@ -55,13 +55,15 @@ i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); assert(__LINE__ == 0); +#if 0 + { int j; j = 0; -#if 0 i = sp->rcv_len - sp->rcv_ptr; if (i > 0) { j = write(sp->fd, sp->rcv + sp->rcv_ptr, i); assert(j == i); } + } #endif e1.fd = fd; Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-06-20 11:39:20 UTC (rev 212) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-06-20 19:31:46 UTC (rev 213) @@ -23,7 +23,6 @@ void VSLR(enum shmlogtag tag, unsigned id, const char *b, const char *e) { - va_list ap; unsigned char *p, *q; assert(b != NULL); @@ -58,8 +57,6 @@ loghead->ptr = (p + 4 + (e - b)) - logstart; /* XXX: Unlock */ - - va_end(ap); } Modified: trunk/varnish-cache/bin/varnishd/cli_event.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cli_event.c 2006-06-20 11:39:20 UTC (rev 212) +++ trunk/varnish-cache/bin/varnishd/cli_event.c 2006-06-20 19:31:46 UTC (rev 213) @@ -120,7 +120,7 @@ static void excb(struct bufferevent *bev, short what, void *arg) { - printf("%s(%p, %d, %p)\n", __func__, (void*)bev, what, arg); + printf("%s(%p, %d, %p)\n", __func__, (void*)bev, (int)what, arg); } struct cli * Modified: trunk/varnish-cache/bin/varnishd/flint.lnt =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.lnt 2006-06-20 11:39:20 UTC (rev 212) +++ trunk/varnish-cache/bin/varnishd/flint.lnt 2006-06-20 19:31:46 UTC (rev 213) @@ -4,6 +4,7 @@ -printf_code( H, void *, unsigned) -printf_code( ju, long long unsigned) +-printf_code( jx, long long unsigned) -e763 // Redundant declaration for symbol '...' previously declared Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-20 11:39:20 UTC (rev 212) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-20 19:31:46 UTC (rev 213) @@ -48,7 +48,7 @@ struct smf_sc { char *filename; int fd; - int pagesize; + unsigned pagesize; uintmax_t filesize; struct smfhead order; struct smfhead free; @@ -481,7 +481,7 @@ { struct smf *smf; - smf = s->priv; + smf = (struct smf *)(s->priv); free_smf(smf); } Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-20 11:39:20 UTC (rev 212) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-20 19:31:46 UTC (rev 213) @@ -38,7 +38,7 @@ * Generic passthrough for CLI functions */ -void +static void cli_passthrough_cb(unsigned u, const char *r, void *priv) { struct cli *cli = priv; @@ -319,11 +319,13 @@ const char *p, *q; struct stevedore *stp; - q = p = strchr(sflag, ','); + p = strchr(sflag, ','); if (p == NULL) q = p = strchr(sflag, '\0'); else - q++; + q = p + 1; + assert(p != NULL); + assert(q != NULL); if (!cmp_storage(&sma_stevedore, sflag, p)) { stp = &sma_stevedore; } else if (!cmp_storage(&smf_stevedore, sflag, p)) { From phk at projects.linpro.no Tue Jun 20 19:37:46 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 20 Jun 2006 21:37:46 +0200 (CEST) Subject: r214 - trunk/varnish-cache/lib/libvcl Message-ID: <20060620193746.9384D1EC233@projects.linpro.no> Author: phk Date: 2006-06-20 21:37:46 +0200 (Tue, 20 Jun 2006) New Revision: 214 Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c Log: FlexeLint cleanups Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-20 19:31:46 UTC (rev 213) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-20 19:37:46 UTC (rev 214) @@ -300,7 +300,7 @@ /*--------------------------------------------------------------------*/ -char * +static char * EncString(struct token *t) { char *p, *q; @@ -564,6 +564,7 @@ int i; v = calloc(sizeof *v, 1); + assert(v != NULL); i = t->e - t->b; p = malloc(i + 1); assert(p != NULL); @@ -1219,7 +1220,6 @@ case HOSTNAME: ExpectErr(tl, CSTR); t_host = tl->t; - host = EncString(tl->t); I(tl); sbuf_printf(tl->fc, "\tp = %*.*s;\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); @@ -1231,7 +1231,6 @@ case PORTNAME: ExpectErr(tl, CSTR); t_port = tl->t; - port = EncString(tl->t); I(tl); sbuf_printf(tl->fc, "\tp = %*.*s;\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); @@ -1250,13 +1249,14 @@ NextToken(tl); } ExpectErr(tl, '}'); - if (host == NULL) { + if (t_host == NULL) { sbuf_printf(tl->sb, "Backend '%*.*s' has no hostname\n", t_be->e - t_be->b, t_be->e - t_be->b, t_be->b); ErrWhere(tl, tl->t); return; } + host = EncString(t_host); ep = CheckHostPort(host, "80"); if (ep != NULL) { sbuf_printf(tl->sb, @@ -1266,7 +1266,8 @@ ErrWhere(tl, t_host); return; } - if (port != NULL) { + if (t_port != NULL) { + port = EncString(tl->t); ep = CheckHostPort(host, port); if (ep != NULL) { sbuf_printf(tl->sb, From phk at projects.linpro.no Tue Jun 20 19:38:04 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 20 Jun 2006 21:38:04 +0200 (CEST) Subject: r215 - trunk/varnish-cache/lib/libvcl Message-ID: <20060620193804.C95031EC233@projects.linpro.no> Author: phk Date: 2006-06-20 21:38:04 +0200 (Tue, 20 Jun 2006) New Revision: 215 Added: trunk/varnish-cache/lib/libvcl/flint.lnt trunk/varnish-cache/lib/libvcl/flint.sh Log: FlexeLint files Added: trunk/varnish-cache/lib/libvcl/flint.lnt =================================================================== --- trunk/varnish-cache/lib/libvcl/flint.lnt 2006-06-20 19:37:46 UTC (rev 214) +++ trunk/varnish-cache/lib/libvcl/flint.lnt 2006-06-20 19:38:04 UTC (rev 215) @@ -0,0 +1,45 @@ +-passes=3 + +// Review all below this line + +-printf_code( H, void *, unsigned) +-printf_code( ju, long long unsigned) +-printf_code( jx, long long unsigned) + +-e763 // Redundant declaration for symbol '...' previously declared + + +-e737 // Loss of sign in promotion from int to unsigned int +-e715 // Symbol 'arg' (line 43) not referenced +-e818 // Pointer parameter '...' could be declared as pointing to const + +-e534 // Ignoring return value of function +-e767 // macro 'LIST_INIT' was defined differently + +-e506 // Constant value boolean +-e527 // Unreachable code at token 'return' +-e732 // Loss of sign (arg. no. 2) (int to unsigned int) +-e774 // Boolean within 'if' always evaluates to False +-e713 // Loss of precision (assignment) (unsigned long long to long long) +-e574 // Signed-unsigned mix with relational + +-e525 // Negative indentation from line 90 +-e539 // Did not expect positive indentation +-e725 // Expected positive indentation from line 136 +-e734 // Loss of precision (assignment) (31 bits to 8 bits) +-e747 // Significant prototype coercion (arg. no. 2) long +-e712 // Loss of precision (assignment) (long long to + + +-e785 // Too few initializers for aggregate + +-e766 // Header file '../../include/libvarnish.h' not used in module + +-e773 // Expression-like macro 'VCL_FARGS' not parenthesized + +-e788 // enum constant 'HND_Unclass' not used within defaulted switch + +-e716 // while(1) ... +-e641 // Converting enum 'cli_status_e' to int + +-e786 // String concatenation within initializer Added: trunk/varnish-cache/lib/libvcl/flint.sh =================================================================== --- trunk/varnish-cache/lib/libvcl/flint.sh 2006-06-20 19:37:46 UTC (rev 214) +++ trunk/varnish-cache/lib/libvcl/flint.sh 2006-06-20 19:38:04 UTC (rev 215) @@ -0,0 +1,9 @@ +#!/bin/sh + +flexelint \ + -I/usr/include \ + -I. \ + -I../../include \ + -I../../contrib/libevent \ + flint.lnt \ + *.c From phk at projects.linpro.no Tue Jun 20 19:49:28 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 20 Jun 2006 21:49:28 +0200 (CEST) Subject: r216 - trunk/varnish-cache/include Message-ID: <20060620194928.A90CD1EC22E@projects.linpro.no> Author: phk Date: 2006-06-20 21:49:28 +0200 (Tue, 20 Jun 2006) New Revision: 216 Modified: trunk/varnish-cache/include/vrt.h Log: typo Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2006-06-20 19:38:04 UTC (rev 215) +++ trunk/varnish-cache/include/vrt.h 2006-06-20 19:49:28 UTC (rev 216) @@ -46,7 +46,7 @@ char *VRT_GetReq(struct sess *); void VRT_handling(struct sess *sp, unsigned hand); int VRT_obj_valid(struct sess *); -int VRT_obj_cachable(struct sess *); +int VRT_obj_cacheable(struct sess *); void VRT_set_backend_hostname(struct backend *, const char *); void VRT_set_backend_portname(struct backend *, const char *); From phk at projects.linpro.no Wed Jun 21 08:09:02 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 21 Jun 2006 10:09:02 +0200 (CEST) Subject: r217 - in trunk/varnish-cache: include lib/libvarnish lib/libvcl Message-ID: <20060621080902.772581EC236@projects.linpro.no> Author: phk Date: 2006-06-21 10:09:02 +0200 (Wed, 21 Jun 2006) New Revision: 217 Added: trunk/varnish-cache/include/binary_heap.h trunk/varnish-cache/lib/libvarnish/binary_heap.c Modified: trunk/varnish-cache/include/Makefile.am trunk/varnish-cache/lib/libvarnish/Makefile.am trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c Log: Add a binary heap implementation for keeping track of objects expiry time. Modified: trunk/varnish-cache/include/Makefile.am =================================================================== --- trunk/varnish-cache/include/Makefile.am 2006-06-20 19:49:28 UTC (rev 216) +++ trunk/varnish-cache/include/Makefile.am 2006-06-21 08:09:02 UTC (rev 217) @@ -1,6 +1,7 @@ # $Id$ include_HEADERS = \ + binary_heap.h \ compat.h \ hash.h \ libvarnish.h \ Added: trunk/varnish-cache/include/binary_heap.h =================================================================== --- trunk/varnish-cache/include/binary_heap.h 2006-06-20 19:49:28 UTC (rev 216) +++ trunk/varnish-cache/include/binary_heap.h 2006-06-21 08:09:02 UTC (rev 217) @@ -0,0 +1,49 @@ +/* + * $Id$ + * + * Binary Heap API (see: http://en.wikipedia.org/wiki/Binary_heap) + * + * XXX: doesn't scale back the array of pointers when items are deleted. + */ + +/* Public Interface --------------------------------------------------*/ + +struct binheap; + +typedef int binheap_cmp_t(void *priv, void *a, void *b); + /* + * Comparison function. + * Should return true if item 'a' should be closer to the root + * than item 'b' + */ + +typedef void binheap_update_t(void *priv, void *a, unsigned newidx); + /* + * Update function (optional) + * When items move in the tree, this function gets called to + * notify the item of its new index. + * Only needed if deleting non-root items. + */ + +struct binheap *binheap_new(void *priv, binheap_cmp_t, binheap_update_t); + /* + * Create Binary tree + * 'priv' is passed to cmp and update functions. + */ + +void binheap_insert(struct binheap *, void *); + /* + * Insert an item + */ + +void binheap_delete(struct binheap *, unsigned idx); + /* + * Delete an item + * The root item has 'idx' zero + */ + +void *binheap_root(struct binheap *); + /* + * Return the root item + */ + Modified: trunk/varnish-cache/lib/libvarnish/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvarnish/Makefile.am 2006-06-20 19:49:28 UTC (rev 216) +++ trunk/varnish-cache/lib/libvarnish/Makefile.am 2006-06-21 08:09:02 UTC (rev 217) @@ -6,5 +6,6 @@ libvarnish_la_SOURCES = \ argv.c \ + binary_heap.c \ cli.c \ time.c Added: trunk/varnish-cache/lib/libvarnish/binary_heap.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/binary_heap.c 2006-06-20 19:49:28 UTC (rev 216) +++ trunk/varnish-cache/lib/libvarnish/binary_heap.c 2006-06-21 08:09:02 UTC (rev 217) @@ -0,0 +1,260 @@ +/* + * $Id$ + * + * Implementation of a binary heap API + * + * We use a malloc(3)/realloc(3) array to store the pointers using the + * classical FORTRAN strategy. + * + * XXX: the array is not scaled back when items are deleted. + */ + +#include +#include +#include + +#include "binary_heap.h" + +/* Private definitions -----------------------------------------------*/ + +#define MIN_LENGTH 16 + +struct binheap { + unsigned magic; +#define BINHEAP_MAGIC 0xf581581aU /* from /dev/random */ + void *priv; + binheap_cmp_t *cmp; + binheap_update_t *update; + void **array; + unsigned length; + unsigned next; + unsigned granularity; +}; + +#define PARENT(u) (((u) - 1) / 2) +#define CHILD(u,n) ((u) * 2 + (n)) + +/* Implementation ----------------------------------------------------*/ + +static void +binheap_update(struct binheap *bh, unsigned u) +{ + assert(bh->magic == BINHEAP_MAGIC); + assert(u < bh->next); + if (bh->update == NULL) + return; + bh->update(bh->priv, bh->array[u], u); +} + +struct binheap * +binheap_new(void *priv, binheap_cmp_t *cmp_f, binheap_update_t *update_f) +{ + struct binheap *bh; + + bh = calloc(sizeof *bh, 1); + if (bh == NULL) + return (bh); + bh->priv = priv; + bh->cmp = cmp_f; + bh->update = update_f; + bh->next = 0; + bh->length = MIN_LENGTH; + bh->array = calloc(sizeof *bh->array, bh->length); + assert(bh->array != NULL); + bh->granularity = getpagesize() / sizeof *bh->array; + bh->magic = BINHEAP_MAGIC; + return (bh); +} + +static void +binhead_swap(struct binheap *bh, unsigned u, unsigned v) +{ + void *p; + + assert(bh->magic == BINHEAP_MAGIC); + assert(u < bh->next); + assert(v < bh->next); + p = bh->array[u]; + bh->array[u] = bh->array[v]; + bh->array[v] = p; + binheap_update(bh, u); + binheap_update(bh, v); +} + +static unsigned +binheap_trickleup(struct binheap *bh, unsigned u) +{ + unsigned v; + + assert(bh->magic == BINHEAP_MAGIC); + while (u > 0) { + v = PARENT(u); + if (bh->cmp(bh->priv, bh->array[u], bh->array[v])) { + binhead_swap(bh, u, v); + u = v; + } else + break; + } + return (u); +} + +static void +binheap_trickledown(struct binheap *bh, unsigned u) +{ + unsigned v1, v2; + + assert(bh->magic == BINHEAP_MAGIC); + while (1) { + v1 = CHILD(u, 1); + if (v1 >= bh->next) + return; + v2 = CHILD(u, 2); + if (v2 >= bh->next) { + if (!bh->cmp(bh->priv, bh->array[u], bh->array[v1])) + binhead_swap(bh, u, v1); + return; + } + if (bh->cmp(bh->priv, bh->array[v1], bh->array[v2])) { + if (!bh->cmp(bh->priv, bh->array[u], bh->array[v1])) { + binhead_swap(bh, u, v1); + u = v1; + continue; + } + } else { + if (!bh->cmp(bh->priv, bh->array[u], bh->array[v2])) { + binhead_swap(bh, u, v2); + u = v2; + continue; + } + } + return; + } +} + +void +binheap_insert(struct binheap *bh, void *p) +{ + unsigned u; + + assert(bh->magic == BINHEAP_MAGIC); + assert(bh->length >= bh->next); + if (bh->length == bh->next) { + if (bh->length >= bh->granularity * 32) + bh->length += bh->granularity * 32; + else if (bh->length > bh->granularity) + bh->length += bh->granularity; + else + bh->length += bh->length; + bh->array = realloc(bh->array, bh->length * sizeof *bh->array); + assert(bh->array != NULL); + } + u = bh->next++; + bh->array[u] = p; + binheap_update(bh, u); + binheap_trickleup(bh, u); +} + +void * +binheap_root(struct binheap *bh) +{ + + assert(bh->magic == BINHEAP_MAGIC); + if(bh->next == 0) + return (NULL); + return (bh->array[0]); +} + +void +binheap_delete(struct binheap *bh, unsigned idx) +{ + + assert(bh->magic == BINHEAP_MAGIC); + assert(bh->next > 0); + assert(idx < bh->next); + if (idx == --bh->next) + return; + bh->array[idx] = bh->array[bh->next]; + binheap_update(bh, idx); + binheap_trickledown(bh, idx); + /* XXX: free part of array ? */ +} + + +#ifdef TEST_DRIVER +/* Test driver -------------------------------------------------------*/ + +#include + +static int +cmp(void *priv, void *a, void *b) +{ + + return (*(unsigned *)a < *(unsigned *)b); +} + +void +update(void *priv, void *a, unsigned u) +{ + printf("%p is now %u\n", a, u); +} + +static void +dump(struct binheap *bh, const char *what, unsigned ptr) +{ + FILE *f; + unsigned u, *up; + + printf("dump\n"); + f = popen("dot -Tps >> /tmp/_.ps", "w"); + assert(f != NULL); + fprintf(f, "digraph binheap {\n"); + fprintf(f, "size=\"7,10\"\n"); + fprintf(f, "ptr [label=\"%s\"]\n", what); + fprintf(f, "ptr -> node_%u\n", ptr); + for (u = 0; u < bh->next; u++) { + up = bh->array[u]; + fprintf(f, "node_%u [label=\"%u\"];\n", u, *up); + if (u > 0) + fprintf(f, "node_%u -> node_%u\n", PARENT(u), u); + } + fprintf(f, "}\n"); + pclose(f); +} + +#define N 31 +int +main(int argc, char **argv) +{ + struct binheap *bh; + unsigned l[N], u, *up, lu; + + srandomdev(); + u = random(); + printf("Seed %u\n", u); + srandom(u); + system("echo %! > /tmp/_.ps"); + bh = binheap_new(NULL, cmp, update); + for (u = 0; u < N; u++) { + l[u] = random() % 1000; + binheap_insert(bh, &l[u]); + if (1) + dump(bh, "Insert", 0); + } + printf("Inserts done\n"); + lu = 0; + while(1) { + up = binheap_root(bh); + if (up == NULL) + break; + assert(*up >= lu); + lu = *up; + u = random() % bh->next; + binheap_delete(bh, u); + if (1) + dump(bh, "Delete", u); + } + printf("Deletes done\n"); + + return (0); +} +#endif Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-20 19:49:28 UTC (rev 216) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-21 08:09:02 UTC (rev 217) @@ -473,7 +473,7 @@ fputs("char *VRT_GetReq(struct sess *);\n", f); fputs("void VRT_handling(struct sess *sp, unsigned hand);\n", f); fputs("int VRT_obj_valid(struct sess *);\n", f); - fputs("int VRT_obj_cachable(struct sess *);\n", f); + fputs("int VRT_obj_cacheable(struct sess *);\n", f); fputs("\n", f); fputs("void VRT_set_backend_hostname(struct backend *, const char *);\n", f); fputs("void VRT_set_backend_portname(struct backend *, const char *);\n", f); From phk at projects.linpro.no Wed Jun 21 09:58:06 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 21 Jun 2006 11:58:06 +0200 (CEST) Subject: r218 - trunk/varnish-cache/bin/varnishd Message-ID: <20060621095806.C03971EC236@projects.linpro.no> Author: phk Date: 2006-06-21 11:58:06 +0200 (Wed, 21 Jun 2006) New Revision: 218 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: Quench some debugging Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-21 08:09:02 UTC (rev 217) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-21 09:58:06 UTC (rev 218) @@ -278,7 +278,6 @@ TAILQ_REMOVE(&sc->free, sp, status); sp->alloc = 1; TAILQ_INSERT_TAIL(&sc->used, sp, status); - printf("ALLOC %12p %12p %12jx %12jx\n", (void*)sp, (void*)sp->ptr, (uintmax_t)sp->offset, (uintmax_t)sp->size); return (sp); } @@ -295,8 +294,6 @@ sp2->alloc = 1; TAILQ_INSERT_BEFORE(sp, sp2, order); TAILQ_INSERT_TAIL(&sc->used, sp2, status); - printf("SPLIT %12p %12p %12jx %12jx\n", (void*)sp, (void*)sp->ptr, (uintmax_t)sp->offset, (uintmax_t)sp->size); - printf("ALLOC %12p %12p %12jx %12jx\n", (void*)sp2, (void*)sp2->ptr, (uintmax_t)sp2->offset, (uintmax_t)sp2->size); return (sp2); } @@ -311,7 +308,6 @@ struct smf *sp2; struct smf_sc *sc = sp->sc; - printf("FREE %12p %12p %12jx %12jx\n", (void*)sp, (void*)sp->ptr, (uintmax_t)sp->offset, (uintmax_t)sp->size); TAILQ_REMOVE(&sc->used, sp, status); sp->alloc = 0; @@ -324,7 +320,6 @@ TAILQ_REMOVE(&sc->order, sp2, order); TAILQ_REMOVE(&sc->free, sp2, status); free(sp2); - printf("MERGEN %12p %12p %12jx %12jx\n", (void*)sp, (void*)sp->ptr, (uintmax_t)sp->offset, (uintmax_t)sp->size); } sp2 = TAILQ_PREV(sp, smfhead, order); @@ -338,7 +333,6 @@ free(sp); TAILQ_REMOVE(&sc->free, sp2, status); sp = sp2; - printf("MERGEP %12p %12p %12jx %12jx\n", (void*)sp, (void*)sp->ptr, (uintmax_t)sp->offset, (uintmax_t)sp->size); } TAILQ_FOREACH(sp2, &sc->free, status) { @@ -363,7 +357,6 @@ sp = calloc(sizeof *sp, 1); assert(sp != NULL); - printf("NEW %12p %12p %12jx %12jx\n", (void*)sp, (void*)ptr, (uintmax_t)off, (uintmax_t)len); sp->sc = sc; @@ -496,14 +489,16 @@ smf = st->priv; - printf("SEND %12p %12p %12jx %12jx\n", (void*)smf, (void*)smf->ptr, (uintmax_t)smf->offset, (uintmax_t)smf->size); vca_flush(sp); i = sendfile(smf->sc->fd, sp->fd, smf->offset, st->len, NULL, &sent, 0); + if (sent == st->len) + return; printf("sent i=%d sent=%ju size=%ju\n", i, (uintmax_t)sent, (uintmax_t)st->len); + assert(sent == st->len); } /*--------------------------------------------------------------------*/ From phk at projects.linpro.no Wed Jun 21 10:13:53 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 21 Jun 2006 12:13:53 +0200 (CEST) Subject: r219 - trunk/varnish-cache/bin/varnishd Message-ID: <20060621101353.2FD8A1EC22D@projects.linpro.no> Author: phk Date: 2006-06-21 12:13:53 +0200 (Wed, 21 Jun 2006) New Revision: 219 Modified: trunk/varnish-cache/bin/varnishd/heritage.h trunk/varnish-cache/bin/varnishd/varnishd.c Log: Add "-t default_ttl" option. 120 seconds by default. Modified: trunk/varnish-cache/bin/varnishd/heritage.h =================================================================== --- trunk/varnish-cache/bin/varnishd/heritage.h 2006-06-21 09:58:06 UTC (rev 218) +++ trunk/varnish-cache/bin/varnishd/heritage.h 2006-06-21 10:13:53 UTC (rev 219) @@ -30,6 +30,8 @@ /* Storage method */ struct stevedore *stevedore; + + unsigned default_ttl; }; extern struct heritage heritage; Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-21 09:58:06 UTC (rev 218) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-21 10:13:53 UTC (rev 219) @@ -290,6 +290,7 @@ fprintf(stderr, " %-28s # %s\n", "-p number", "TCP listen port"); fprintf(stderr, " %-28s # %s\n", "-s kind[,storageoptions]", "Backend storage specification"); + fprintf(stderr, " %-28s # %s\n", "-t", "Default TTL"); #if 0 -c clusterid at cluster_controller -m memory_limit @@ -394,7 +395,9 @@ VCC_InitCompile(); - while ((o = getopt(argc, argv, "b:df:p:s:")) != -1) + heritage.default_ttl = 120; + + while ((o = getopt(argc, argv, "b:df:p:s:t:")) != -1) switch (o) { case 'b': bflag = optarg; @@ -411,6 +414,9 @@ case 's': sflag = optarg; break; + case 't': + heritage.default_ttl = strtoul(optarg, NULL, 0); + break; default: usage(); } From phk at projects.linpro.no Wed Jun 21 10:21:14 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 21 Jun 2006 12:21:14 +0200 (CEST) Subject: r220 - trunk/varnish-cache/bin/varnishd Message-ID: <20060621102114.EC0751EC233@projects.linpro.no> Author: phk Date: 2006-06-21 12:21:14 +0200 (Wed, 21 Jun 2006) New Revision: 220 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/rfc2616.c Log: Start to respect TTL Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-21 10:13:53 UTC (rev 219) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-21 10:21:14 UTC (rev 220) @@ -71,6 +71,7 @@ unsigned busy; unsigned len; + time_t ttl; char *header; @@ -105,6 +106,7 @@ /* Various internal stuff */ struct event *rd_e; struct sessmem *mem; + time_t t0; }; struct backend { @@ -200,5 +202,4 @@ #endif /* rfc2616.c */ -void RFC2616_Age(struct http *hp, time_t, time_t); - +time_t RFC2616_Ttl(struct http *hp, time_t, time_t); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-21 10:13:53 UTC (rev 219) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-21 10:21:14 UTC (rev 220) @@ -223,7 +223,7 @@ time(&t_resp); http_Dissect(hp, fd, 2); - RFC2616_Age(hp, t_req, t_resp); + sp->obj->ttl = RFC2616_Ttl(hp, t_req, t_resp); switch (http_GetStatus(hp)) { case 200: Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-21 10:13:53 UTC (rev 219) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-21 10:21:14 UTC (rev 220) @@ -47,15 +47,15 @@ MD5Final(key, &ctx); o = hash->lookup(key, w->nobj); sp->obj = o; - if (o == w->nobj) { - VSL(SLT_Debug, 0, "Lookup new %p %s", o, b); - w->nobj = NULL; - VCL_miss_method(sp); - } else { + if (o != w->nobj && o->ttl > sp->t0) { /* XXX: wait while obj->busy */ VSL(SLT_Debug, 0, "Lookup found %p %s", o, b); VCL_hit_method(sp); + return (0); } + VSL(SLT_Debug, 0, "Lookup new %p %s", o, b); + w->nobj = NULL; + VCL_miss_method(sp); return (0); } @@ -97,6 +97,7 @@ AZ(pthread_cond_wait(&shdcnd, &sessmtx)); } TAILQ_REMOVE(&shd, sp, list); + time(&sp->t0); sp->vcl = GetVCL(); AZ(pthread_mutex_unlock(&sessmtx)); Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-21 10:13:53 UTC (rev 219) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-21 10:21:14 UTC (rev 220) @@ -8,6 +8,7 @@ #include "cache.h" #include "libvarnish.h" +#include "heritage.h" /*-------------------------------------------------------------------- * From RFC2616, 13.2.3 Age Calculations @@ -34,13 +35,13 @@ * */ -void -RFC2616_Age(struct http *hp, time_t t_req, time_t t_resp) +time_t +RFC2616_Ttl(struct http *hp, time_t t_req, time_t t_resp) { time_t h_date = 0, h_expires = 0, h_age = 0; time_t apparent_age = 0, corrected_received_age; time_t response_delay, corrected_initial_age; - time_t max_age = -1; + time_t max_age = -1, ttl; char *p; if (http_GetHdrField(hp, "Cache-Control", "max-age", &p)) @@ -67,10 +68,19 @@ h_expires = TIM_parse(p); printf("Date: %d\n", h_date); + printf("Recv: %d\n", t_resp); printf("Expires: %d\n", h_expires); printf("Age: %d\n", h_age); printf("CIAge: %d\n", corrected_initial_age); printf("Max-Age: %d\n", max_age); + ttl = 0; + if (max_age >= 0) + ttl = t_resp + max_age - corrected_initial_age; + if (h_expires && h_expires < ttl) + ttl = h_expires; + if (ttl == 0) + ttl = t_resp + heritage.default_ttl; + printf("TTL: %d (+%d)\n", ttl, ttl - t_resp); - + return (ttl); } From phk at projects.linpro.no Wed Jun 21 10:28:26 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 21 Jun 2006 12:28:26 +0200 (CEST) Subject: r221 - trunk/varnish-cache/bin/varnishd Message-ID: <20060621102826.364F91EC300@projects.linpro.no> Author: phk Date: 2006-06-21 12:28:26 +0200 (Wed, 21 Jun 2006) New Revision: 221 Added: trunk/varnish-cache/bin/varnishd/cache_expire.c Modified: trunk/varnish-cache/bin/varnishd/Makefile.am trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_main.c Log: Add (empty) source file for expiry/pretech code Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-06-21 10:21:14 UTC (rev 220) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-06-21 10:28:26 UTC (rev 221) @@ -7,6 +7,7 @@ varnishd_SOURCES = \ cache_acceptor.c \ cache_backend.c \ + cache_expire.c \ cache_fetch.c \ cache_http.c \ cache_main.c \ Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-21 10:21:14 UTC (rev 220) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-21 10:28:26 UTC (rev 221) @@ -142,6 +142,9 @@ void VBE_ClosedFd(void *ptr); void VBE_RecycleFd(void *ptr); +/* cache_expiry.c */ +void EXP_Init(void); + /* cache_fetch.c */ int FetchSession(struct worker *w, struct sess *sp); Added: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-21 10:21:14 UTC (rev 220) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-21 10:28:26 UTC (rev 221) @@ -0,0 +1,10 @@ +/* + * $Id$ + * + * Expiry of cached objects and execution of prefetcher + */ + +void +EXP_Init(void) +{ +} Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-21 10:21:14 UTC (rev 220) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-21 10:28:26 UTC (rev 221) @@ -114,6 +114,7 @@ CacheInitPool(); VCA_Init(); + EXP_Init(); eb = event_init(); assert(eb != NULL); From phk at projects.linpro.no Thu Jun 22 16:17:11 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 22 Jun 2006 18:17:11 +0200 (CEST) Subject: r222 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060622161711.052061EC233@projects.linpro.no> Author: phk Date: 2006-06-22 18:17:10 +0200 (Thu, 22 Jun 2006) New Revision: 222 Added: trunk/varnish-cache/include/vcl_returns.h Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_vcl.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/include/vcl.h trunk/varnish-cache/include/vrt.h trunk/varnish-cache/lib/libvcl/vcl_compile.c trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl trunk/varnish-cache/lib/libvcl/vcl_token_defs.h Log: Improve the VCL compiler in various ways: Generate the methods and their legal returns with the tcl script. Add consistency checks to make sure methods don't use illegal returns, and also check called subrourtines. Add consistency check to complain about recursive subroutine calls. Add consistency check to complain about unused or undefined subroutines. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-21 10:28:26 UTC (rev 221) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-22 16:17:10 UTC (rev 222) @@ -78,13 +78,7 @@ TAILQ_HEAD(, storage) store; }; -#define HND_Error (1 << 0) -#define HND_Pipe (1 << 1) -#define HND_Pass (1 << 2) -#define HND_Lookup (1 << 3) -#define HND_Fetch (1 << 4) -#define HND_Insert (1 << 5) -#define HND_Deliver (1 << 6) +#include "vcl_returns.h" struct sess { int fd; Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-21 10:28:26 UTC (rev 221) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-22 16:17:10 UTC (rev 222) @@ -109,23 +109,23 @@ for (done = 0; !done; ) { switch(sp->handling) { - case HND_Lookup: + case VCL_RET_LOOKUP: VSL(SLT_Handling, sp->fd, "Lookup"); done = LookupSession(&w, sp); break; - case HND_Fetch: + case VCL_RET_FETCH: done = FetchSession(&w, sp); break; - case HND_Deliver: + case VCL_RET_DELIVER: VSL(SLT_Handling, sp->fd, "Deliver"); done = DeliverSession(&w, sp); break; - case HND_Pipe: + case VCL_RET_PIPE: VSL(SLT_Handling, sp->fd, "Pipe"); PipeSession(&w, sp); done = 1; break; - case HND_Pass: + case VCL_RET_PASS: VSL(SLT_Handling, sp->fd, "Pass"); PassSession(&w, sp); done = 1; Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-21 10:28:26 UTC (rev 221) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-22 16:17:10 UTC (rev 222) @@ -195,13 +195,11 @@ { switch (u) { - case HND_Error: return ("Error"); - case HND_Pass: return ("Pass"); - case HND_Pipe: return ("Pipe"); - case HND_Lookup: return ("Lookup"); - case HND_Fetch: return ("Fetch"); - case HND_Insert: return ("Insert"); - case HND_Deliver: return ("Deliver"); +#define VCL_RET_MAC(a, b, c) case VCL_RET_##b: return(#a); +#define VCL_RET_MAC_E(a, b, c) case VCL_RET_##b: return(#a); +#include "vcl_returns.h" +#undef VCL_RET_MAC +#undef VCL_RET_MAC_E default: return (NULL); } } @@ -226,7 +224,7 @@ "Wrong handling after %s function: 0x%x", func, u); else return; - sp->handling = HND_Error; + sp->handling = VCL_RET_ERROR; } #define VCL_method(func, bitmap) \ @@ -239,7 +237,8 @@ CheckHandling(sp, #func, (bitmap)); \ } -VCL_method(recv, HND_Error|HND_Pass|HND_Pipe|HND_Lookup) -VCL_method(miss, HND_Error|HND_Pass|HND_Pipe|HND_Fetch) -VCL_method(hit, HND_Error|HND_Pass|HND_Pipe|HND_Deliver) -VCL_method(fetch, HND_Error|HND_Pass|HND_Pipe|HND_Insert) +#define VCL_RET_MAC(l,u,b) +#define VCL_MET_MAC(l,u,b) VCL_method(l, b) +#include "vcl_returns.h" +#undef VCL_MET_MAC +#undef VCL_RET_MAC Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-21 10:28:26 UTC (rev 221) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-22 16:17:10 UTC (rev 222) @@ -75,19 +75,7 @@ { assert(!(hand & (hand -1))); /* must be power of two */ - switch (hand) { -#define FOO(a,b) case VRT_H_##a: sp->handling = HND_##b; break; - FOO(error, Error); - FOO(pipe, Pipe); - FOO(pass, Pass); - FOO(lookup, Lookup); - FOO(fetch, Fetch); - FOO(insert, Insert); - FOO(deliver, Deliver); -#undef FOO - default: - assert(hand == 0); - } + sp->handling = hand; } int Modified: trunk/varnish-cache/include/vcl.h =================================================================== --- trunk/varnish-cache/include/vcl.h 2006-06-21 10:28:26 UTC (rev 221) +++ trunk/varnish-cache/include/vcl.h 2006-06-22 16:17:10 UTC (rev 222) @@ -1,10 +1,9 @@ /* * $Id$ * - * Interface to a compiled VCL program. + * NB: This file is machine generated, DO NOT EDIT! * - * XXX: When this file is changed, lib/libvcl/vcl_gen_fixed_token.tcl - * XXX: *MUST* be rerun. + * Edit vcl_gen_fixed_token.tcl instead */ struct sess; @@ -13,16 +12,19 @@ typedef int vcl_func_f(struct sess *sp); struct VCL_conf { - unsigned magic; -#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */ - vcl_init_f *init_func; + unsigned magic; +#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */ + + struct backend **backend; + unsigned nbackend; + struct vrt_ref *ref; + unsigned nref; + unsigned busy; + + vcl_init_f *init_func; + vcl_func_f *recv_func; + vcl_func_f *miss_func; vcl_func_f *hit_func; - vcl_func_f *miss_func; vcl_func_f *fetch_func; - struct backend **backend; - unsigned nbackend; - struct vrt_ref *ref; - unsigned nref; - unsigned busy; }; Added: trunk/varnish-cache/include/vcl_returns.h =================================================================== --- trunk/varnish-cache/include/vcl_returns.h 2006-06-21 10:28:26 UTC (rev 221) +++ trunk/varnish-cache/include/vcl_returns.h 2006-06-22 16:17:10 UTC (rev 222) @@ -0,0 +1,35 @@ +/* + * $Id$ + * + * NB: This file is machine generated, DO NOT EDIT! + * + * Edit vcl_gen_fixed_token.tcl instead + */ + +#ifdef VCL_RET_MAC +#ifdef VCL_RET_MAC_E +VCL_RET_MAC_E(error, ERROR, 0) +#endif +VCL_RET_MAC(lookup, LOOKUP, (1 << 1)) +VCL_RET_MAC(pipe, PIPE, (1 << 2)) +VCL_RET_MAC(pass, PASS, (1 << 3)) +VCL_RET_MAC(fetch, FETCH, (1 << 4)) +VCL_RET_MAC(insert, INSERT, (1 << 5)) +VCL_RET_MAC(deliver, DELIVER, (1 << 6)) +#else +#define VCL_RET_ERROR (1 << 0) +#define VCL_RET_LOOKUP (1 << 1) +#define VCL_RET_PIPE (1 << 2) +#define VCL_RET_PASS (1 << 3) +#define VCL_RET_FETCH (1 << 4) +#define VCL_RET_INSERT (1 << 5) +#define VCL_RET_DELIVER (1 << 6) +#define VCL_RET_MAX 7 +#endif + +#ifdef VCL_MET_MAC +VCL_MET_MAC(recv,RECV,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_LOOKUP)) +VCL_MET_MAC(miss,MISS,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_FETCH)) +VCL_MET_MAC(hit,HIT,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_DELIVER)) +VCL_MET_MAC(fetch,FETCH,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_INSERT)) +#endif Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2006-06-21 10:28:26 UTC (rev 221) +++ trunk/varnish-cache/include/vrt.h 2006-06-22 16:17:10 UTC (rev 222) @@ -7,14 +7,6 @@ * XXX: *MUST* be rerun. */ -#define VRT_H_error (1 << 0) -#define VRT_H_pipe (1 << 1) -#define VRT_H_pass (1 << 2) -#define VRT_H_lookup (1 << 3) -#define VRT_H_fetch (1 << 4) -#define VRT_H_insert (1 << 5) -#define VRT_H_deliver (1 << 6) - struct sess; struct backend; struct VCL_conf; Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-21 10:28:26 UTC (rev 221) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-22 16:17:10 UTC (rev 222) @@ -50,6 +50,7 @@ #include #include "vcl_priv.h" +#include "vcl_returns.h" #include "libvcl.h" @@ -77,6 +78,8 @@ struct sbuf *sb; int err; int nbackend; + TAILQ_HEAD(, proc) procs; + struct proc *curproc; }; enum var_type { @@ -117,6 +120,41 @@ const char *lname; }; +/*-------------------------------------------------------------------- + * Consistency check + */ + +static struct method { + const char *name; + unsigned returns; +} method_tab[] = { +#define VCL_RET_MAC(a,b,c) +#define VCL_MET_MAC(a,b,c) { "vcl_"#a, c }, +#include "vcl_returns.h" +#undef VCL_MET_MAC +#undef VCL_RET_MAC + { NULL, 0U } +}; + +struct proccall { + TAILQ_ENTRY(proccall) list; + struct proc *p; + struct token *t; +}; + +struct proc { + TAILQ_ENTRY(proc) list; + TAILQ_HEAD(,proccall) calls; + struct token *name; + unsigned returns; + unsigned exists; + unsigned called; + unsigned active; + struct token *returnt[VCL_RET_MAX]; +}; + +/*--------------------------------------------------------------------*/ + static struct var be_vars[] = { { "backend.host", HOSTNAME, 0, NULL, "VRT_set_backend_hostname(backend, %s)" }, @@ -152,6 +190,8 @@ static void Compound(struct tokenlist *tl); static void Cond_0(struct tokenlist *tl); +static struct proc *AddProc(struct tokenlist *tl, struct token *t, int def); +static void AddCall(struct tokenlist *tl, struct token *t); /*--------------------------------------------------------------------*/ @@ -925,15 +965,14 @@ I(tl); sbuf_printf(tl->fc, "VCL_no_cache(sp);\n"); return; - case T_DELIVER: - case T_LOOKUP: - case T_PASS: - case T_FETCH: - case T_INSERT: - I(tl); sbuf_printf(tl->fc, "VRT_done(sp, VRT_H_%*.*s);\n", - at->e - at->b, - at->e - at->b, at->b); +#define VCL_RET_MAC(a,b,c) case T_##b: \ + I(tl); \ + sbuf_printf(tl->fc, "VRT_done(sp, VCL_RET_%s);\n", #b); \ + tl->curproc->returns |= VCL_RET_##b; \ + tl->curproc->returnt[c] = at; \ return; +#include "vcl_returns.h" +#undef VCL_RET_MAC case T_ERROR: if (tl->t->tok == CNUM) a = UintVal(tl); @@ -948,7 +987,7 @@ NextToken(tl); } else sbuf_printf(tl->fc, "(const char *)0);\n"); - I(tl); sbuf_printf(tl->fc, "VRT_done(sp, VRT_H_error);\n"); + I(tl); sbuf_printf(tl->fc, "VRT_done(sp, VCL_RET_ERROR);\n"); return; case T_SWITCH_CONFIG: ExpectErr(tl, ID); @@ -960,6 +999,7 @@ return; case T_CALL: ExpectErr(tl, ID); + AddCall(tl, tl->t); AddRef(tl, tl->t, R_FUNC); I(tl); sbuf_printf(tl->fc, "if (VGC_function_%*.*s(sp))\n", @@ -1294,8 +1334,11 @@ NextToken(tl); ExpectErr(tl, ID); + tl->curproc = AddProc(tl, tl->t, 1); + tl->curproc->exists++; AddDef(tl, tl->t, R_FUNC); - sbuf_printf(tl->fh, "static int VGC_function_%*.*s (struct sess *sp);\n", + sbuf_printf(tl->fh, + "static int VGC_function_%*.*s (struct sess *sp);\n", tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); I(tl); sbuf_printf(tl->fc, "static int\n"); @@ -1460,6 +1503,125 @@ AddToken(tl, EOI, p, p); } +/*-------------------------------------------------------------------- + * Consistency check + */ + +static struct proc * +AddProc(struct tokenlist *tl, struct token *t, int def) +{ + struct proc *p; + + TAILQ_FOREACH(p, &tl->procs, list) { + if (p->name->e - p->name->b != t->e - t->b) + continue; + if (!memcmp(p->name->b, t->b, t->e - t->b)) { + if (def) + p->name = t; + return (p); + } + } + p = calloc(sizeof *p, 1); + assert(p != NULL); + p->name = t; + TAILQ_INIT(&p->calls); + TAILQ_INSERT_TAIL(&tl->procs, p, list); + return (p); +} + +static void +AddCall(struct tokenlist *tl, struct token *t) +{ + struct proccall *pc; + struct proc *p; + + p = AddProc(tl, t, 0); + TAILQ_FOREACH(pc, &tl->curproc->calls, list) { + if (pc->p == p) + return; + } + pc = calloc(sizeof *pc, 1); + assert(pc != NULL); + pc->p = p; + pc->t = t; + TAILQ_INSERT_TAIL(&tl->curproc->calls, pc, list); +} + +static int +Consist_Decend(struct tokenlist *tl, struct proc *p, unsigned returns) +{ + unsigned u; + struct proccall *pc; + + if (!p->exists) { + sbuf_printf(tl->sb, "Function %*.*s does not exist\n", + p->name->e - p->name->b, + p->name->e - p->name->b, + p->name->b); + return (1); + } + if (p->active) { + sbuf_printf(tl->sb, "Function recurses on\n"); + ErrWhere(tl, p->name); + return (1); + } + u = p->returns & ~returns; + if (u) { +#define VCL_RET_MAC(a, b, c) \ + if (u & VCL_RET_##b) { \ + sbuf_printf(tl->sb, "Illegal return for method\n"); \ + ErrWhere(tl, p->returnt[c]); \ + } +#include "vcl_returns.h" +#undef VCL_RET_MAC + sbuf_printf(tl->sb, "In function\n"); + ErrWhere(tl, p->name); + return (1); + } + p->active = 1; + TAILQ_FOREACH(pc, &p->calls, list) { + if (Consist_Decend(tl, pc->p, returns)) { + sbuf_printf(tl->sb, "\nCalled from\n"); + ErrWhere(tl, p->name); + sbuf_printf(tl->sb, "at\n"); + ErrWhere(tl, pc->t); + return (1); + } + } + p->active = 0; + p->called++; + return (0); +} + +static int +Consistency(struct tokenlist *tl) +{ + struct proc *p; + struct method *m; + + TAILQ_FOREACH(p, &tl->procs, list) { + for(m = method_tab; m->name != NULL; m++) { + if (IdIs(p->name, m->name)) + break; + } + if (m->name == NULL) + continue; + if (Consist_Decend(tl, p, m->returns)) { + sbuf_printf(tl->sb, + "\nwhich is a %s method\n", m->name); + return (1); + } + } + TAILQ_FOREACH(p, &tl->procs, list) { + if (p->called) + continue; + sbuf_printf(tl->sb, "Function unused\n"); + ErrWhere(tl, p->name); + return (1); + } + return (0); +} + /*--------------------------------------------------------------------*/ static void @@ -1614,6 +1776,7 @@ memset(&tokens, 0, sizeof tokens); TAILQ_INIT(&tokens.tokens); TAILQ_INIT(&tokens.refs); + TAILQ_INIT(&tokens.procs); tokens.sb = sb; tokens.fc = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); @@ -1636,6 +1799,7 @@ Parse(&tokens); if (tokens.err) goto done; + Consistency(&tokens); if (0) CheckRefs(&tokens); if (tokens.err) Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-21 10:28:26 UTC (rev 221) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-22 16:17:10 UTC (rev 222) @@ -1,6 +1,9 @@ /* + * $Id$ + * * NB: This file is machine generated, DO NOT EDIT! - * instead, edit the Tcl script vcl_gen_fixed_token.tcl and run it by hand + * + * Edit vcl_gen_fixed_token.tcl instead */ #include @@ -261,6 +264,11 @@ *q = p + 4; return (T_PROC); } + if (p[0] == 'p' && p[1] == 'i' && p[2] == 'p' && + p[3] == 'e' && !isvar(p[4])) { + *q = p + 4; + return (T_PIPE); + } if (p[0] == 'p' && p[1] == 'a' && p[2] == 's' && p[3] == 's' && !isvar(p[4])) { *q = p + 4; @@ -384,6 +392,7 @@ vcl_tnames[T_NO_CACHE] = "no_cache"; vcl_tnames[T_NO_NEW_CACHE] = "no_new_cache"; vcl_tnames[T_PASS] = "pass"; + vcl_tnames[T_PIPE] = "pipe"; vcl_tnames[T_PROC] = "proc"; vcl_tnames[T_REWRITE] = "rewrite"; vcl_tnames[T_SET] = "set"; @@ -397,13 +406,19 @@ void vcl_output_lang_h(FILE *f) { + fputs("#define VCL_RET_ERROR (1 << 0)\n", f); + fputs("#define VCL_RET_LOOKUP (1 << 1)\n", f); + fputs("#define VCL_RET_PIPE (1 << 2)\n", f); + fputs("#define VCL_RET_PASS (1 << 3)\n", f); + fputs("#define VCL_RET_FETCH (1 << 4)\n", f); + fputs("#define VCL_RET_INSERT (1 << 5)\n", f); + fputs("#define VCL_RET_DELIVER (1 << 6)\n", f); fputs("/*\n", f); fputs(" * $Id$\n", f); fputs(" *\n", f); - fputs(" * Interface to a compiled VCL program.\n", f); + fputs(" * NB: This file is machine generated, DO NOT EDIT!\n", f); fputs(" *\n", f); - fputs(" * XXX: When this file is changed, lib/libvcl/vcl_gen_fixed_token.tcl\n", f); - fputs(" * XXX: *MUST* be rerun.\n", f); + fputs(" * Edit vcl_gen_fixed_token.tcl instead\n", f); fputs(" */\n", f); fputs("\n", f); fputs("struct sess;\n", f); @@ -412,18 +427,21 @@ fputs("typedef int vcl_func_f(struct sess *sp);\n", f); fputs("\n", f); fputs("struct VCL_conf {\n", f); - fputs(" unsigned magic;\n", f); - fputs("#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */\n", f); - fputs(" vcl_init_f *init_func;\n", f); + fputs(" unsigned magic;\n", f); + fputs("#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */\n", f); + fputs("\n", f); + fputs(" struct backend **backend;\n", f); + fputs(" unsigned nbackend;\n", f); + fputs(" struct vrt_ref *ref;\n", f); + fputs(" unsigned nref;\n", f); + fputs(" unsigned busy;\n", f); + fputs("\n", f); + fputs(" vcl_init_f *init_func;\n", f); + fputs("\n", f); fputs(" vcl_func_f *recv_func;\n", f); + fputs(" vcl_func_f *miss_func;\n", f); fputs(" vcl_func_f *hit_func;\n", f); - fputs(" vcl_func_f *miss_func;\n", f); fputs(" vcl_func_f *fetch_func;\n", f); - fputs(" struct backend **backend;\n", f); - fputs(" unsigned nbackend;\n", f); - fputs(" struct vrt_ref *ref;\n", f); - fputs(" unsigned nref;\n", f); - fputs(" unsigned busy;\n", f); fputs("};\n", f); fputs("/*\n", f); fputs(" * $Id$ \n", f); @@ -434,14 +452,6 @@ fputs(" * XXX: *MUST* be rerun.\n", f); fputs(" */\n", f); fputs("\n", f); - fputs("#define VRT_H_error (1 << 0)\n", f); - fputs("#define VRT_H_pipe (1 << 1)\n", f); - fputs("#define VRT_H_pass (1 << 2)\n", f); - fputs("#define VRT_H_lookup (1 << 3)\n", f); - fputs("#define VRT_H_fetch (1 << 4)\n", f); - fputs("#define VRT_H_insert (1 << 5)\n", f); - fputs("#define VRT_H_deliver (1 << 6)\n", f); - fputs("\n", f); fputs("struct sess;\n", f); fputs("struct backend;\n", f); fputs("struct VCL_conf;\n", f); Modified: trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-06-21 10:28:26 UTC (rev 221) +++ trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-06-22 16:17:10 UTC (rev 222) @@ -1,8 +1,32 @@ #!/usr/local/bin/tclsh8.4 # -# Generate a C source file to recognize a set of tokens for the -# Varnish +# Generate various .c and .h files for the VCL compiler and the interfaces +# for it. +# These are the metods which can be called in the VCL program. +# Second element is list of valid return actions. +# +set methods { + {recv {error pass pipe lookup}} + {miss {error pass pipe fetch}} + {hit {error pass pipe deliver}} + {fetch {error pass pipe insert}} +} + +# These are the return actions +# +set returns { + error + lookup + pipe + pass + fetch + insert + deliver +} + +# Language keywords +# set keywords { if else elseif elsif @@ -12,13 +36,6 @@ backend - error - lookup - pass - fetch - insert - deliver - call no_cache no_new_cache @@ -27,6 +44,8 @@ switch_config } +# Non-word tokens +# set magic { {"++" INC} {"--" DEC} @@ -44,25 +63,108 @@ {"/=" DIV} } +# Single char tokens +# set char {{}()*+-/%><=;!&.|~,} +# Other token identifiers +# set extras {ID VAR CNUM CSTR EOI} -set fo [open "vcl_fixed_token.c" w] +#---------------------------------------------------------------------- +# Boilerplate warning for all generated files. -puts $fo {/* - * NB: This file is machine generated, DO NOT EDIT! - * instead, edit the Tcl script vcl_gen_fixed_token.tcl and run it by hand - */ +proc warns {fd} { + + puts $fd "/*" + puts $fd { * $Id$} + puts $fd " *" + puts $fd " * NB: This file is machine generated, DO NOT EDIT!" + puts $fd " *" + puts $fd " * Edit vcl_gen_fixed_token.tcl instead" + puts $fd " */" + puts $fd "" } -set foh [open "vcl_token_defs.h" w] -puts $foh {/* - * NB: This file is machine generated, DO NOT EDIT! - * instead, edit the Tcl script vcl_gen_fixed_token.tcl and run it by hand - */ +#---------------------------------------------------------------------- +# Build the vcl.h #include file + +set fo [open ../../include/vcl.h w] +warns $fo +puts $fo {struct sess; + +typedef void vcl_init_f(void); +typedef int vcl_func_f(struct sess *sp); } +puts $fo "struct VCL_conf {" +puts $fo { unsigned magic; +#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */ + struct backend **backend; + unsigned nbackend; + struct vrt_ref *ref; + unsigned nref; + unsigned busy; + + vcl_init_f *init_func; +} +foreach m $methods { + puts $fo "\tvcl_func_f\t*[lindex $m 0]_func;" +} +puts $fo "};" + +close $fo + +#---------------------------------------------------------------------- +# Build the vcl_returns.h #include file + +set for [open "../../include/vcl_returns.h" w] +warns $for +puts $for "#ifdef VCL_RET_MAC" +set i 0 +foreach k $returns { + if {$k == "error"} { + puts $for "#ifdef VCL_RET_MAC_E" + puts $for "VCL_RET_MAC_E($k, [string toupper $k], $i)" + puts $for "#endif" + } else { + puts $for "VCL_RET_MAC($k, [string toupper $k], (1 << $i))" + } + incr i +} +puts $for "#else" +set i 0 +foreach k $returns { + puts $for "#define VCL_RET_[string toupper $k] (1 << $i)" + incr i +} +puts $for "#define VCL_RET_MAX $i" +puts $for "#endif" +puts $for "" +puts $for "#ifdef VCL_MET_MAC" +foreach m $methods { + puts -nonewline $for "VCL_MET_MAC([lindex $m 0]" + puts -nonewline $for ",[string toupper [lindex $m 0]]" + set l [lindex $m 1] + puts -nonewline $for ",(VCL_RET_[string toupper [lindex $l 0]]" + foreach r [lrange $l 1 end] { + puts -nonewline $for "|VCL_RET_[string toupper $r]" + } + puts -nonewline $for ")" + puts $for ")" +} +puts $for "#endif" +close $for + +#---------------------------------------------------------------------- +# Build the compiler token table and recognizers + +set fo [open "vcl_fixed_token.c" w] +warns $fo + +set foh [open "vcl_token_defs.h" w] +warns $foh + puts $fo "#include " puts $fo "#include " puts $fo "#include \"vcl_priv.h\"" @@ -70,20 +172,24 @@ set tn 128 puts $foh "#define LOW_TOKEN $tn" -foreach k $keywords { - set t T_[string toupper $k] - lappend tokens [list $t $k] - puts $foh "#define $t $tn" + +proc add_token {tok str alpha} { + global tokens tn fixed foh + + lappend tokens [list $tok $str] + puts $foh "#define $tok $tn" incr tn - lappend fixed [list $k $t 1] + lappend fixed [list $str $tok $alpha] } -foreach k $magic { - set t T_[string toupper [lindex $k 1]] - lappend tokens [list $t [lindex $k 0]] - puts $foh "#define $t $tn" - incr tn - lappend fixed [list [lindex $k 0] $t 0] + +proc mk_token {tok str alpha} { + set tok T_[string toupper $tok] + add_token $tok $str $alpha } + +foreach k $keywords { mk_token $k $k 1 } +foreach k $returns { mk_token $k $k 1 } +foreach k $magic { mk_token [lindex $k 1] [lindex $k 0] 0 } foreach k $extras { set t [string toupper $k] lappend tokens [list $t $t] @@ -176,8 +282,10 @@ } puts $fo "}" +#---------------------------------------------------------------------- +# Create the C-code which emits the boilerplate definitions for the +# generated C code output - proc copy_include {n} { global fo @@ -193,6 +301,12 @@ puts $fo "void" puts $fo "vcl_output_lang_h(FILE *f)" puts $fo "{" +set i 0 +foreach k $returns { + puts $fo "\tfputs(\"#define VCL_RET_[string toupper $k] (1 << $i)\\n\", f);" + incr i +} + copy_include ../../include/vcl.h copy_include ../../include/vrt.h Modified: trunk/varnish-cache/lib/libvcl/vcl_token_defs.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-06-21 10:28:26 UTC (rev 221) +++ trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-06-22 16:17:10 UTC (rev 222) @@ -1,6 +1,9 @@ /* + * $Id$ + * * NB: This file is machine generated, DO NOT EDIT! - * instead, edit the Tcl script vcl_gen_fixed_token.tcl and run it by hand + * + * Edit vcl_gen_fixed_token.tcl instead */ #define LOW_TOKEN 128 @@ -13,34 +16,35 @@ #define T_SUB 134 #define T_ACL 135 #define T_BACKEND 136 -#define T_ERROR 137 -#define T_LOOKUP 138 -#define T_PASS 139 -#define T_FETCH 140 -#define T_INSERT 141 -#define T_DELIVER 142 -#define T_CALL 143 -#define T_NO_CACHE 144 -#define T_NO_NEW_CACHE 145 -#define T_SET 146 -#define T_REWRITE 147 -#define T_SWITCH_CONFIG 148 -#define T_INC 149 -#define T_DEC 150 -#define T_CAND 151 -#define T_COR 152 -#define T_LEQ 153 -#define T_EQ 154 -#define T_NEQ 155 -#define T_GEQ 156 -#define T_SHR 157 -#define T_SHL 158 -#define T_INCR 159 -#define T_DECR 160 -#define T_MUL 161 -#define T_DIV 162 -#define ID 163 -#define VAR 164 -#define CNUM 165 -#define CSTR 166 -#define EOI 167 +#define T_CALL 137 +#define T_NO_CACHE 138 +#define T_NO_NEW_CACHE 139 +#define T_SET 140 +#define T_REWRITE 141 +#define T_SWITCH_CONFIG 142 +#define T_ERROR 143 +#define T_LOOKUP 144 +#define T_PIPE 145 +#define T_PASS 146 +#define T_FETCH 147 +#define T_INSERT 148 +#define T_DELIVER 149 +#define T_INC 150 +#define T_DEC 151 +#define T_CAND 152 +#define T_COR 153 +#define T_LEQ 154 +#define T_EQ 155 +#define T_NEQ 156 +#define T_GEQ 157 +#define T_SHR 158 +#define T_SHL 159 +#define T_INCR 160 +#define T_DECR 161 +#define T_MUL 162 +#define T_DIV 163 +#define ID 164 +#define VAR 165 +#define CNUM 166 +#define CSTR 167 +#define EOI 168 From phk at projects.linpro.no Sat Jun 24 19:41:55 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 24 Jun 2006 21:41:55 +0200 (CEST) Subject: r223 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060624194155.DE8691EC232@projects.linpro.no> Author: phk Date: 2006-06-24 21:41:55 +0200 (Sat, 24 Jun 2006) New Revision: 223 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_vcl.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/include/shmlog_tags.h trunk/varnish-cache/include/vrt.h trunk/varnish-cache/lib/libvcl/vcl_compile.c trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c Log: Add more shmemlog tags: one for each VCL method to record the return one for errors one for linking a client session to a backend connection Use them sensibly. Put VCL name of backend into struct backend to improve log messages Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-22 16:17:10 UTC (rev 222) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-24 19:41:55 UTC (rev 223) @@ -104,6 +104,7 @@ }; struct backend { + const char *vcl_name; const char *hostname; const char *portname; unsigned ip; Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-22 16:17:10 UTC (rev 222) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-24 19:41:55 UTC (rev 223) @@ -204,7 +204,7 @@ fd = VBE_GetFd(sp->backend, &fd_token); assert(fd != -1); - VSL(SLT_Handling, sp->fd, "Fetch fd %d", fd); + VSL(SLT_Backend, sp->fd, "%d %s", fd, sp->backend->vcl_name); hp = http_New(); http_BuildSbuf(1, w->sb, sp->http); Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-22 16:17:10 UTC (rev 222) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-24 19:41:55 UTC (rev 223) @@ -110,23 +110,19 @@ for (done = 0; !done; ) { switch(sp->handling) { case VCL_RET_LOOKUP: - VSL(SLT_Handling, sp->fd, "Lookup"); done = LookupSession(&w, sp); break; case VCL_RET_FETCH: done = FetchSession(&w, sp); break; case VCL_RET_DELIVER: - VSL(SLT_Handling, sp->fd, "Deliver"); done = DeliverSession(&w, sp); break; case VCL_RET_PIPE: - VSL(SLT_Handling, sp->fd, "Pipe"); PipeSession(&w, sp); done = 1; break; case VCL_RET_PASS: - VSL(SLT_Handling, sp->fd, "Pass"); PassSession(&w, sp); done = 1; break; Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-22 16:17:10 UTC (rev 222) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-24 19:41:55 UTC (rev 223) @@ -212,15 +212,11 @@ u = sp->handling; n = HandlingName(u); - if (n != NULL) - VSL(SLT_Handling, sp->fd, "%s(): %s", func, n); - else - VSL(SLT_Handling, sp->fd, "%s(): Illegal: 0x%x", func, u); if (u & (u - 1)) - VSL(SLT_Debug, sp->fd, + VSL(SLT_Error, sp->fd, "Illegal handling after %s function: 0x%x", func, u); else if (!(u & bitmap)) - VSL(SLT_Debug, sp->fd, + VSL(SLT_Error, sp->fd, "Wrong handling after %s function: 0x%x", func, u); else return; @@ -235,6 +231,7 @@ sp->handling = 0; \ sp->vcl->func##_func(sp); \ CheckHandling(sp, #func, (bitmap)); \ + VSL(SLT_vcl_##func, sp->fd, "%s", HandlingName(sp->handling)); \ } #define VCL_RET_MAC(l,u,b) Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-22 16:17:10 UTC (rev 222) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-24 19:41:55 UTC (rev 223) @@ -103,6 +103,12 @@ } void +VRT_set_backend_name(struct backend *be, const char *p) +{ + be->vcl_name = p; +} + +void VRT_alloc_backends(struct VCL_conf *cp) { int i; Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-06-22 16:17:10 UTC (rev 222) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-06-24 19:41:55 UTC (rev 223) @@ -7,6 +7,7 @@ */ SLTM(Debug) +SLTM(Error) SLTM(CLI) SLTM(SessionOpen) SLTM(SessionReuse) @@ -16,7 +17,12 @@ SLTM(BackendClose) SLTM(HttpError) SLTM(ClientAddr) -SLTM(Handling) +#define VCL_RET_MAC(l,u,b) +#define VCL_MET_MAC(l,u,b) SLTM(vcl_##l) +#include "vcl_returns.h" +#undef VCL_MET_MAC +#undef VCL_RET_MAC +SLTM(Backend) SLTM(Request) SLTM(Response) SLTM(Status) Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2006-06-22 16:17:10 UTC (rev 222) +++ trunk/varnish-cache/include/vrt.h 2006-06-24 19:41:55 UTC (rev 223) @@ -40,6 +40,7 @@ int VRT_obj_valid(struct sess *); int VRT_obj_cacheable(struct sess *); +void VRT_set_backend_name(struct backend *, const char *); void VRT_set_backend_hostname(struct backend *, const char *); void VRT_set_backend_portname(struct backend *, const char *); Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-22 16:17:10 UTC (rev 222) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-24 19:41:55 UTC (rev 223) @@ -42,8 +42,8 @@ #include #include #include +#include #include -#include #include #include #include @@ -1241,6 +1241,11 @@ tl->t->e - tl->t->b, tl->t->e - tl->t->b, tl->t->b); I(tl); sbuf_printf(tl->fc, "\tconst char *p;\n"); + I(tl); sbuf_printf(tl->fc, "\n"); + I(tl); sbuf_printf(tl->fc, + "\tVRT_set_backend_name(backend, \"%*.*s\");\n", + tl->t->e - tl->t->b, + tl->t->e - tl->t->b, tl->t->b); NextToken(tl); ExpectErr(tl, '{'); NextToken(tl); Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-22 16:17:10 UTC (rev 222) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-24 19:41:55 UTC (rev 223) @@ -485,6 +485,7 @@ fputs("int VRT_obj_valid(struct sess *);\n", f); fputs("int VRT_obj_cacheable(struct sess *);\n", f); fputs("\n", f); + fputs("void VRT_set_backend_name(struct backend *, const char *);\n", f); fputs("void VRT_set_backend_hostname(struct backend *, const char *);\n", f); fputs("void VRT_set_backend_portname(struct backend *, const char *);\n", f); fputs("\n", f); From phk at projects.linpro.no Sat Jun 24 19:50:56 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 24 Jun 2006 21:50:56 +0200 (CEST) Subject: r224 - trunk/varnish-cache/lib/libvcl Message-ID: <20060624195056.3600D1EC22E@projects.linpro.no> Author: phk Date: 2006-06-24 21:50:56 +0200 (Sat, 24 Jun 2006) New Revision: 224 Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c Log: Cave in and use printf format extensions for printing tokens. Both Linux and FreeBSD supports them anyway. Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-24 19:41:55 UTC (rev 223) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-24 19:50:56 UTC (rev 224) @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -202,7 +203,7 @@ if (t->tok == EOI) sbuf_printf(tl->sb, "end of input"); else - sbuf_printf(tl->sb, "'%*.*s'", t->e - t->b, t->e - t->b, t->b); + sbuf_printf(tl->sb, "'%T'", t); } static void @@ -692,19 +693,14 @@ ExpectErr(tl, ID); I(tl); AddRef(tl, tl->t, R_ACL); - sbuf_printf(tl->fc, "ip_match(%s, acl_%*.*s)\n", - vp->rname, - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + sbuf_printf(tl->fc, "ip_match(%s, acl_%T)\n", + vp->rname, tl->t); NextToken(tl); break; case T_EQ: case T_NEQ: I(tl); - sbuf_printf(tl->fc, "%s %*.*s ", - vp->rname, - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + sbuf_printf(tl->fc, "%s %T ", vp->rname, tl->t); NextToken(tl); u = IpVal(tl); sbuf_printf(tl->fc, "%uU /* %u.%u.%u.%u */\n", u, @@ -730,9 +726,7 @@ I(tl); sbuf_printf(tl->fc, "string_match(%s, ", vp->rname); NextToken(tl); ExpectErr(tl, CSTR); - sbuf_printf(tl->fc, "%*.*s)\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + sbuf_printf(tl->fc, "%T)\n", tl->t); NextToken(tl); break; case T_EQ: @@ -742,9 +736,7 @@ tl->t->tok == T_EQ ? "!" : "", vp->rname); NextToken(tl); ExpectErr(tl, CSTR); - sbuf_printf(tl->fc, "%*.*s)\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + sbuf_printf(tl->fc, "%T)\n", tl->t); NextToken(tl); break; default: @@ -766,9 +758,7 @@ case T_GEQ: case '>': case '<': - sbuf_printf(tl->fc, "%*.*s ", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + sbuf_printf(tl->fc, "%T ", tl->t); NextToken(tl); switch(vp->fmt) { case TIME: @@ -776,9 +766,7 @@ break; case INT: ExpectErr(tl, CNUM); - sbuf_printf(tl->fc, "%*.*s ", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + sbuf_printf(tl->fc, "%T ", tl->t); NextToken(tl); break; case SIZE: @@ -981,9 +969,7 @@ I(tl); sbuf_printf(tl->fc, "VRT_error(sp, %u, ", a); if (tl->t->tok == CSTR) { - sbuf_printf(tl->fc, "%*.*s);\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + sbuf_printf(tl->fc, "%T);\n", tl->t); NextToken(tl); } else sbuf_printf(tl->fc, "(const char *)0);\n"); @@ -992,9 +978,7 @@ case T_SWITCH_CONFIG: ExpectErr(tl, ID); I(tl); - sbuf_printf(tl->fc, "VCL_switch_config(\"%*.*s\");\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + sbuf_printf(tl->fc, "VCL_switch_config(\"%T\");\n", tl->t); NextToken(tl); return; case T_CALL: @@ -1002,23 +986,17 @@ AddCall(tl, tl->t); AddRef(tl, tl->t, R_FUNC); I(tl); sbuf_printf(tl->fc, - "if (VGC_function_%*.*s(sp))\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + "if (VGC_function_%T(sp))\n", tl->t); I(tl); sbuf_printf(tl->fc, "\treturn (1);\n"); NextToken(tl); return; case T_REWRITE: ExpectErr(tl, CSTR); I(tl); - sbuf_printf(tl->fc, "VCL_rewrite(%*.*s", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + sbuf_printf(tl->fc, "VCL_rewrite(%T", tl->t); NextToken(tl); ExpectErr(tl, CSTR); - sbuf_printf(tl->fc, ", %*.*s);\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + sbuf_printf(tl->fc, ", %T);\n", tl->t); NextToken(tl); return; case T_SET: @@ -1035,9 +1013,7 @@ case RATE: case TIME: case FLOAT: - sbuf_printf(tl->fc, "%*.*s ", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + sbuf_printf(tl->fc, "%T ", tl->t); a = tl->t->tok; NextToken(tl); if (a == T_MUL || a == T_DIV) @@ -1073,9 +1049,8 @@ case BACKEND: if (tl->t->tok == '=') { NextToken(tl); - sbuf_printf(tl->fc, "= &VGC_backend_%*.*s;\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + sbuf_printf(tl->fc, "= &VGC_backend_%T;\n", + tl->t); NextToken(tl); break; } @@ -1110,13 +1085,9 @@ ExpectErr(tl, ID); AddDef(tl, tl->t, R_ACL); - sbuf_printf(tl->fh, "static struct vcl_acl acl_%*.*s[];\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + sbuf_printf(tl->fh, "static struct vcl_acl acl_%T[];\n", tl->t); I(tl); - sbuf_printf(tl->fc, "static struct vcl_acl acl_%*.*s[] = {\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + sbuf_printf(tl->fc, "static struct vcl_acl acl_%T[] = {\n", tl->t); NextToken(tl); tl->indent += INDENT; @@ -1226,26 +1197,18 @@ AddDef(tl, tl->t, R_BACKEND); I(tl); sbuf_printf(tl->fh, - "#define VGC_backend_%*.*s (VCL_conf.backend[%d])\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b, tl->nbackend); + "#define VGC_backend_%T (VCL_conf.backend[%d])\n", + tl->t, tl->nbackend); sbuf_printf(tl->fc, "static void\n"); - I(tl); sbuf_printf(tl->fc, - "VGC_init_backend_%*.*s (void)\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + I(tl); sbuf_printf(tl->fc, "VGC_init_backend_%T (void)\n", tl->t); I(tl); sbuf_printf(tl->fc, "{\n"); I(tl); sbuf_printf(tl->fc, - "\tstruct backend *backend = VGC_backend_%*.*s;\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + "\tstruct backend *backend = VGC_backend_%T;\n", tl->t); I(tl); sbuf_printf(tl->fc, "\tconst char *p;\n"); I(tl); sbuf_printf(tl->fc, "\n"); I(tl); sbuf_printf(tl->fc, - "\tVRT_set_backend_name(backend, \"%*.*s\");\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + "\tVRT_set_backend_name(backend, \"%T\");\n", tl->t); NextToken(tl); ExpectErr(tl, '{'); NextToken(tl); @@ -1265,9 +1228,7 @@ case HOSTNAME: ExpectErr(tl, CSTR); t_host = tl->t; - I(tl); sbuf_printf(tl->fc, "\tp = %*.*s;\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + I(tl); sbuf_printf(tl->fc, "\tp = %T;\n", tl->t); I(tl); sbuf_printf(tl->fc, "\t"); sbuf_printf(tl->fc, vp->lname, "p"); sbuf_printf(tl->fc, ";\n"); @@ -1276,9 +1237,7 @@ case PORTNAME: ExpectErr(tl, CSTR); t_port = tl->t; - I(tl); sbuf_printf(tl->fc, "\tp = %*.*s;\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + I(tl); sbuf_printf(tl->fc, "\tp = %T;\n", tl->t); I(tl); sbuf_printf(tl->fc, "\t"); sbuf_printf(tl->fc, vp->lname, "p"); sbuf_printf(tl->fc, ";\n"); @@ -1295,19 +1254,14 @@ } ExpectErr(tl, '}'); if (t_host == NULL) { - sbuf_printf(tl->sb, "Backend '%*.*s' has no hostname\n", - t_be->e - t_be->b, - t_be->e - t_be->b, t_be->b); + sbuf_printf(tl->sb, "Backend '%T' has no hostname\n", t_be); ErrWhere(tl, tl->t); return; } host = EncString(t_host); ep = CheckHostPort(host, "80"); if (ep != NULL) { - sbuf_printf(tl->sb, - "Backend '%*.*s': %s\n", - t_be->e - t_be->b, - t_be->e - t_be->b, t_be->b, ep); + sbuf_printf(tl->sb, "Backend '%T': %s\n", t_be, ep); ErrWhere(tl, t_host); return; } @@ -1315,10 +1269,7 @@ port = EncString(tl->t); ep = CheckHostPort(host, port); if (ep != NULL) { - sbuf_printf(tl->sb, - "Backend '%*.*s': %s\n", - t_be->e - t_be->b, - t_be->e - t_be->b, t_be->b, ep); + sbuf_printf(tl->sb, "Backend '%T': %s\n", t_be, ep); ErrWhere(tl, t_port); return; } @@ -1343,13 +1294,10 @@ tl->curproc->exists++; AddDef(tl, tl->t, R_FUNC); sbuf_printf(tl->fh, - "static int VGC_function_%*.*s (struct sess *sp);\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + "static int VGC_function_%T (struct sess *sp);\n", tl->t); I(tl); sbuf_printf(tl->fc, "static int\n"); - I(tl); sbuf_printf(tl->fc, "VGC_function_%*.*s (struct sess *sp)\n", - tl->t->e - tl->t->b, - tl->t->e - tl->t->b, tl->t->b); + I(tl); sbuf_printf(tl->fc, "VGC_function_%T (struct sess *sp)\n", + tl->t); NextToken(tl); L(tl, Compound(tl)); sbuf_printf(tl->fc, "\n"); @@ -1559,10 +1507,7 @@ struct proccall *pc; if (!p->exists) { - sbuf_printf(tl->sb, "Function %*.*s does not exist\n", - p->name->e - p->name->b, - p->name->e - p->name->b, - p->name->b); + sbuf_printf(tl->sb, "Function %T does not exist\n", p->name); return (1); } if (p->active) { @@ -1700,11 +1645,9 @@ } sbuf_printf(tl->fc, - "%*.*s[%3u] = { %4u, %3u, 0, \"%*.*s\" },\n", + "%*.*s[%3u] = { %4u, %3u, 0, \"T\" },\n", INDENT, INDENT, "", - t->cnt, lin, pos + 1, - t->e - t->b, - t->e - t->b, t->b); + t->cnt, lin, pos + 1, t); } sbuf_printf(tl->fc, "};\n"); } @@ -1733,9 +1676,7 @@ break; case R_BACKEND: sbuf_printf(tl->fc, - "\tVGC_init_backend_%*.*s();\n", - r->name->e - r->name->b, - r->name->e - r->name->b, r->name->b); + "\tVGC_init_backend_%T();\n", r->name); break; } } @@ -1883,11 +1824,33 @@ /*--------------------------------------------------------------------*/ +int +VCC_T_render(FILE *f, const struct printf_info *info, const void *const *args) +{ + const struct token *t; + + t = *((const struct token **) (args[0])); + return (fprintf(f, "%*.*s", + t->e - t->b, t->e - t->b, t->b)); +} + +int +VCC_T_arginfo(const struct printf_info *info, size_t n, int *argtypes) +{ + + if (n > 0) + argtypes[0] = PA_POINTER; + return 1; +} + +/*--------------------------------------------------------------------*/ + void VCC_InitCompile(void) { struct var *v; + register_printf_function ('T', VCC_T_render, VCC_T_arginfo); vcl_init_tnames(); for (v = vars; v->name != NULL; v++) v->len = strlen(v->name); From phk at projects.linpro.no Sat Jun 24 20:09:08 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 24 Jun 2006 22:09:08 +0200 (CEST) Subject: r225 - trunk/varnish-cache/lib/libvcl Message-ID: <20060624200908.15B991EC22E@projects.linpro.no> Author: phk Date: 2006-06-24 22:09:07 +0200 (Sat, 24 Jun 2006) New Revision: 225 Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c Log: More printf sanitation: Create to convenience functions for output to the fh and fc sbufs. Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-24 19:50:56 UTC (rev 224) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-24 20:09:07 UTC (rev 225) @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -304,11 +305,6 @@ #define Expect(a, b) _Expect(a, b, __LINE__) #define ExpectErr(a, b) do { _Expect(a, b, __LINE__); ERRCHK(a);} while (0) -#define I(tl) do { \ - sbuf_printf(tl->fc, "/* %-11s */ ", __func__); \ - sbuf_printf(tl->fc, "%*.*s", tl->indent, tl->indent, ""); \ -} while (0) - #define L(tl, foo) do { \ tl->indent += INDENT; \ foo; \ @@ -316,11 +312,36 @@ } while (0) #define C(tl, sep) do { \ - I(tl); \ - sbuf_printf(tl->fc, "VRT_count(sp, %u)%s\n", ++tl->cnt, sep); \ + Fc(tl, 1, "VRT_count(sp, %u)%s\n", ++tl->cnt, sep); \ tl->t->cnt = tl->cnt; \ } while (0) +/*--------------------------------------------------------------------*/ + +static void +Fh(struct tokenlist *tl, int indent, const char *fmt, ...) +{ + va_list ap; + + if (indent) + sbuf_printf(tl->fh, "%*.*s", tl->indent, tl->indent, ""); + va_start(ap, fmt); + sbuf_vprintf(tl->fh, fmt, ap); + va_end(ap); +} + +static void +Fc(struct tokenlist *tl, int indent, const char *fmt, ...) +{ + va_list ap; + + if (indent) + sbuf_printf(tl->fc, "%*.*s", tl->indent, tl->indent, ""); + va_start(ap, fmt); + sbuf_vprintf(tl->fc, fmt, ap); + va_end(ap); +} + /*-------------------------------------------------------------------- * Compare ID token to string, return true of match */ @@ -655,7 +676,7 @@ v = DoubleVal(tl); ExpectErr(tl, ID); sc = TimeUnit(tl); - sbuf_printf(tl->fc, "(%g * %g)", v, sc); + Fc(tl, 0, "(%g * %g)", v, sc); } static void @@ -666,7 +687,7 @@ v = DoubleVal(tl); ExpectErr(tl, ID); sc = SizeUnit(tl); - sbuf_printf(tl->fc, "(%g * %g)", v, sc); + Fc(tl, 0, "(%g * %g)", v, sc); } static void @@ -677,7 +698,7 @@ v = DoubleVal(tl); ExpectErr(tl, ID); sc = RateUnit(tl); - sbuf_printf(tl->fc, "(%g * %g)", v, sc); + Fc(tl, 0, "(%g * %g)", v, sc); } /*--------------------------------------------------------------------*/ @@ -691,19 +712,16 @@ case '~': NextToken(tl); ExpectErr(tl, ID); - I(tl); AddRef(tl, tl->t, R_ACL); - sbuf_printf(tl->fc, "ip_match(%s, acl_%T)\n", - vp->rname, tl->t); + Fc(tl, 1, "ip_match(%s, acl_%T)\n", vp->rname, tl->t); NextToken(tl); break; case T_EQ: case T_NEQ: - I(tl); - sbuf_printf(tl->fc, "%s %T ", vp->rname, tl->t); + Fc(tl, 1, "%s %T ", vp->rname, tl->t); NextToken(tl); u = IpVal(tl); - sbuf_printf(tl->fc, "%uU /* %u.%u.%u.%u */\n", u, + Fc(tl, 0, "%uU /* %u.%u.%u.%u */\n", u, (u >> 24) & 0xff, (u >> 16) & 0xff, (u >> 8) & 0xff, (u) & 0xff); break; @@ -723,24 +741,23 @@ switch (tl->t->tok) { case '~': - I(tl); sbuf_printf(tl->fc, "string_match(%s, ", vp->rname); + Fc(tl, 1, "string_match(%s, ", vp->rname); NextToken(tl); ExpectErr(tl, CSTR); - sbuf_printf(tl->fc, "%T)\n", tl->t); + Fc(tl, 0, "%T)\n", tl->t); NextToken(tl); break; case T_EQ: case T_NEQ: - I(tl); - sbuf_printf(tl->fc, "%sstrcmp(%s, ", + Fc(tl, 1, "%sstrcmp(%s, ", tl->t->tok == T_EQ ? "!" : "", vp->rname); NextToken(tl); ExpectErr(tl, CSTR); - sbuf_printf(tl->fc, "%T)\n", tl->t); + Fc(tl, 0, "%T)\n", tl->t); NextToken(tl); break; default: - I(tl); sbuf_printf(tl->fc, "%s != (void*)0", vp->rname); + Fc(tl, 1, "%s != (void*)0", vp->rname); break; } } @@ -749,8 +766,7 @@ Cond_Int(struct var *vp, struct tokenlist *tl) { - I(tl); - sbuf_printf(tl->fc, "%s ", vp->rname); + Fc(tl, 1, "%s ", vp->rname); switch (tl->t->tok) { case T_EQ: case T_NEQ: @@ -758,7 +774,7 @@ case T_GEQ: case '>': case '<': - sbuf_printf(tl->fc, "%T ", tl->t); + Fc(tl, 0, "%T ", tl->t); NextToken(tl); switch(vp->fmt) { case TIME: @@ -766,7 +782,7 @@ break; case INT: ExpectErr(tl, CNUM); - sbuf_printf(tl->fc, "%T ", tl->t); + Fc(tl, 0, "%T ", tl->t); NextToken(tl); break; case SIZE: @@ -779,7 +795,7 @@ ErrWhere(tl, tl->t); return; } - sbuf_printf(tl->fc, "\n"); + Fc(tl, 0, "\n"); break; default: sbuf_printf(tl->sb, "Illegal condition "); @@ -796,8 +812,7 @@ Cond_Bool(struct var *vp, struct tokenlist *tl) { - I(tl); - sbuf_printf(tl->fc, "%s\n", vp->rname); + Fc(tl, 1, "%s\n", vp->rname); } static void @@ -806,12 +821,12 @@ struct var *vp; C(tl, ","); - I(tl); if (tl->t->tok == '!') { - sbuf_printf(tl->fc, "!"); + Fc(tl, 1, "!(\n"); NextToken(tl); + } else { + Fc(tl, 1, "(\n"); } - sbuf_printf(tl->fc, "(\n"); if (tl->t->tok == '(') { NextToken(tl); Cond_0(tl); @@ -847,36 +862,35 @@ ErrWhere(tl, tl->t); return; } - I(tl); - sbuf_printf(tl->fc, ")\n"); + Fc(tl, 1, ")\n"); } static void Cond_1(struct tokenlist *tl) { - I(tl); sbuf_printf(tl->fc, "(\n"); + Fc(tl, 1, "(\n"); L(tl, Cond_2(tl)); while (tl->t->tok == T_CAND) { NextToken(tl); - I(tl); sbuf_printf(tl->fc, ") && (\n"); + Fc(tl, 1, ") && (\n"); L(tl, Cond_2(tl)); } - I(tl); sbuf_printf(tl->fc, ")\n"); + Fc(tl, 1, ")\n"); } static void Cond_0(struct tokenlist *tl) { - I(tl); sbuf_printf(tl->fc, "(\n"); + Fc(tl, 1, "(\n"); L(tl, Cond_1(tl)); while (tl->t->tok == T_COR) { NextToken(tl); - I(tl); sbuf_printf(tl->fc, ") || (\n"); + Fc(tl, 1, ") || (\n"); L(tl, Cond_1(tl)); } - I(tl); sbuf_printf(tl->fc, ")\n"); + Fc(tl, 1, ")\n"); } static void @@ -885,10 +899,10 @@ ExpectErr(tl, '('); NextToken(tl); - I(tl); sbuf_printf(tl->fc, "(\n"); + Fc(tl, 1, "(\n"); L(tl, Cond_0(tl)); ERRCHK(tl); - I(tl); sbuf_printf(tl->fc, ")\n"); + Fc(tl, 1, ")\n"); ExpectErr(tl, ')'); NextToken(tl); } @@ -900,7 +914,7 @@ { ExpectErr(tl, T_IF); - I(tl); sbuf_printf(tl->fc, "if \n"); + Fc(tl, 1, "if \n"); NextToken(tl); L(tl, Conditional(tl)); ERRCHK(tl); @@ -911,7 +925,7 @@ case T_ELSE: NextToken(tl); if (tl->t->tok != T_IF) { - I(tl); sbuf_printf(tl->fc, "else \n"); + Fc(tl, 1, "else \n"); L(tl, Compound(tl)); ERRCHK(tl); return; @@ -919,7 +933,7 @@ /* FALLTHROUGH */ case T_ELSEIF: case T_ELSIF: - I(tl); sbuf_printf(tl->fc, "else if \n"); + Fc(tl, 1, "else if \n"); NextToken(tl); L(tl, Conditional(tl)); ERRCHK(tl); @@ -946,16 +960,13 @@ NextToken(tl); switch (at->tok) { case T_NO_NEW_CACHE: - I(tl); - sbuf_printf(tl->fc, "VCL_no_new_cache(sp);\n"); + Fc(tl, 1, "VCL_no_new_cache(sp);\n"); return; case T_NO_CACHE: - I(tl); - sbuf_printf(tl->fc, "VCL_no_cache(sp);\n"); + Fc(tl, 1, "VCL_no_cache(sp);\n"); return; #define VCL_RET_MAC(a,b,c) case T_##b: \ - I(tl); \ - sbuf_printf(tl->fc, "VRT_done(sp, VCL_RET_%s);\n", #b); \ + Fc(tl, 1, "VRT_done(sp, VCL_RET_%s);\n", #b); \ tl->curproc->returns |= VCL_RET_##b; \ tl->curproc->returnt[c] = at; \ return; @@ -966,37 +977,33 @@ a = UintVal(tl); else a = 0; - I(tl); - sbuf_printf(tl->fc, "VRT_error(sp, %u, ", a); + Fc(tl, 1, "VRT_error(sp, %u, ", a); if (tl->t->tok == CSTR) { - sbuf_printf(tl->fc, "%T);\n", tl->t); + Fc(tl, 0, "%T);\n", tl->t); NextToken(tl); } else - sbuf_printf(tl->fc, "(const char *)0);\n"); - I(tl); sbuf_printf(tl->fc, "VRT_done(sp, VCL_RET_ERROR);\n"); + Fc(tl, 0, "(const char *)0);\n"); + Fc(tl, 1, "VRT_done(sp, VCL_RET_ERROR);\n"); return; case T_SWITCH_CONFIG: ExpectErr(tl, ID); - I(tl); - sbuf_printf(tl->fc, "VCL_switch_config(\"%T\");\n", tl->t); + Fc(tl, 1, "VCL_switch_config(\"%T\");\n", tl->t); NextToken(tl); return; case T_CALL: ExpectErr(tl, ID); AddCall(tl, tl->t); AddRef(tl, tl->t, R_FUNC); - I(tl); sbuf_printf(tl->fc, - "if (VGC_function_%T(sp))\n", tl->t); - I(tl); sbuf_printf(tl->fc, "\treturn (1);\n"); + Fc(tl, 1, "if (VGC_function_%T(sp))\n", tl->t); + Fc(tl, 1, "\treturn (1);\n"); NextToken(tl); return; case T_REWRITE: ExpectErr(tl, CSTR); - I(tl); - sbuf_printf(tl->fc, "VCL_rewrite(%T", tl->t); + Fc(tl, 1, "VCL_rewrite(%T", tl->t); NextToken(tl); ExpectErr(tl, CSTR); - sbuf_printf(tl->fc, ", %T);\n", tl->t); + Fc(tl, 0, ", %T);\n", tl->t); NextToken(tl); return; case T_SET: @@ -1004,8 +1011,7 @@ vp = FindVar(tl, tl->t, vars); ERRCHK(tl); assert(vp != NULL); - I(tl); - sbuf_printf(tl->fc, "%s ", vp->rname); + Fc(tl, 1, "%s ", vp->rname); NextToken(tl); switch (vp->fmt) { case INT: @@ -1013,11 +1019,11 @@ case RATE: case TIME: case FLOAT: - sbuf_printf(tl->fc, "%T ", tl->t); + Fc(tl, 0, "%T ", tl->t); a = tl->t->tok; NextToken(tl); if (a == T_MUL || a == T_DIV) - sbuf_printf(tl->fc, "%g", DoubleVal(tl)); + Fc(tl, 0, "%g", DoubleVal(tl)); else if (vp->fmt == TIME) TimeVal(tl); else if (vp->fmt == SIZE) @@ -1025,14 +1031,14 @@ else if (vp->fmt == RATE) RateVal(tl); else - sbuf_printf(tl->fc, "%g", DoubleVal(tl)); - sbuf_printf(tl->fc, ";\n"); + Fc(tl, 0, "%g", DoubleVal(tl)); + Fc(tl, 0, ";\n"); break; case IP: if (tl->t->tok == '=') { NextToken(tl); u = IpVal(tl); - sbuf_printf(tl->fc, "= %uU; /* %u.%u.%u.%u */\n", + Fc(tl, 0, "= %uU; /* %u.%u.%u.%u */\n", u, (u >> 24) & 0xff, (u >> 16) & 0xff, @@ -1049,7 +1055,7 @@ case BACKEND: if (tl->t->tok == '=') { NextToken(tl); - sbuf_printf(tl->fc, "= &VGC_backend_%T;\n", + Fc(tl, 0, "= &VGC_backend_%T;\n", tl->t); NextToken(tl); break; @@ -1085,9 +1091,8 @@ ExpectErr(tl, ID); AddDef(tl, tl->t, R_ACL); - sbuf_printf(tl->fh, "static struct vcl_acl acl_%T[];\n", tl->t); - I(tl); - sbuf_printf(tl->fc, "static struct vcl_acl acl_%T[] = {\n", tl->t); + Fh(tl, 0, "static struct vcl_acl acl_%T[];\n", tl->t); + Fc(tl, 1, "static struct vcl_acl acl_%T[] = {\n", tl->t); NextToken(tl); tl->indent += INDENT; @@ -1105,20 +1110,17 @@ m = 32; ExpectErr(tl, ';'); NextToken(tl); - I(tl); - sbuf_printf(tl->fc, "{ %11uU, %3uU }, /* %u.%u.%u.%u/%u */\n", + Fc(tl, 1, "{ %11uU, %3uU }, /* %u.%u.%u.%u/%u */\n", u, m, (u >> 24) & 0xff, (u >> 16) & 0xff, (u >> 8) & 0xff, (u) & 0xff, m); } ExpectErr(tl, '}'); - I(tl); - sbuf_printf(tl->fc, "{ %11uU, %3uU }\n", 0, 0); + Fc(tl, 1, "{ %11uU, %3uU }\n", 0, 0); tl->indent -= INDENT; - I(tl); - sbuf_printf(tl->fc, "};\n\n"); + Fc(tl, 1, "};\n\n"); NextToken(tl); } @@ -1129,7 +1131,7 @@ { ExpectErr(tl, '{'); - I(tl); sbuf_printf(tl->fc, "{\n"); + Fc(tl, 1, "{\n"); tl->indent += INDENT; C(tl, ";"); NextToken(tl); @@ -1145,7 +1147,7 @@ case '}': NextToken(tl); tl->indent -= INDENT; - I(tl); sbuf_printf(tl->fc, "}\n"); + Fc(tl, 1, "}\n"); return; case EOI: sbuf_printf(tl->sb, @@ -1195,20 +1197,15 @@ ExpectErr(tl, ID); t_be = tl->t; AddDef(tl, tl->t, R_BACKEND); - I(tl); - sbuf_printf(tl->fh, - "#define VGC_backend_%T (VCL_conf.backend[%d])\n", + Fh(tl, 1, "#define VGC_backend_%T (VCL_conf.backend[%d])\n", tl->t, tl->nbackend); - sbuf_printf(tl->fc, "static void\n"); - I(tl); sbuf_printf(tl->fc, "VGC_init_backend_%T (void)\n", tl->t); - I(tl); - sbuf_printf(tl->fc, "{\n"); - I(tl); sbuf_printf(tl->fc, - "\tstruct backend *backend = VGC_backend_%T;\n", tl->t); - I(tl); sbuf_printf(tl->fc, "\tconst char *p;\n"); - I(tl); sbuf_printf(tl->fc, "\n"); - I(tl); sbuf_printf(tl->fc, - "\tVRT_set_backend_name(backend, \"%T\");\n", tl->t); + Fc(tl, 0, "static void\n"); + Fc(tl, 1, "VGC_init_backend_%T (void)\n", tl->t); + Fc(tl, 1, "{\n"); + Fc(tl, 1, "\tstruct backend *backend = VGC_backend_%T;\n", tl->t); + Fc(tl, 1, "\tconst char *p;\n"); + Fc(tl, 1, "\n"); + Fc(tl, 1, "\tVRT_set_backend_name(backend, \"%T\");\n", tl->t); NextToken(tl); ExpectErr(tl, '{'); NextToken(tl); @@ -1228,19 +1225,19 @@ case HOSTNAME: ExpectErr(tl, CSTR); t_host = tl->t; - I(tl); sbuf_printf(tl->fc, "\tp = %T;\n", tl->t); - I(tl); sbuf_printf(tl->fc, "\t"); - sbuf_printf(tl->fc, vp->lname, "p"); - sbuf_printf(tl->fc, ";\n"); + Fc(tl, 1, "\tp = %T;\n", tl->t); + Fc(tl, 1, "\t"); + Fc(tl, 0, vp->lname, "p"); + Fc(tl, 0, ";\n"); NextToken(tl); break; case PORTNAME: ExpectErr(tl, CSTR); t_port = tl->t; - I(tl); sbuf_printf(tl->fc, "\tp = %T;\n", tl->t); - I(tl); sbuf_printf(tl->fc, "\t"); - sbuf_printf(tl->fc, vp->lname, "p"); - sbuf_printf(tl->fc, ";\n"); + Fc(tl, 1, "\tp = %T;\n", tl->t); + Fc(tl, 1, "\t"); + Fc(tl, 0, vp->lname, "p"); + Fc(tl, 0, ";\n"); NextToken(tl); break; default: @@ -1276,9 +1273,8 @@ } NextToken(tl); - I(tl); - sbuf_printf(tl->fc, "}\n"); - sbuf_printf(tl->fc, "\n"); + Fc(tl, 1, "}\n"); + Fc(tl, 0, "\n"); tl->nbackend++; } @@ -1293,14 +1289,13 @@ tl->curproc = AddProc(tl, tl->t, 1); tl->curproc->exists++; AddDef(tl, tl->t, R_FUNC); - sbuf_printf(tl->fh, + Fh(tl, 0, "static int VGC_function_%T (struct sess *sp);\n", tl->t); - I(tl); sbuf_printf(tl->fc, "static int\n"); - I(tl); sbuf_printf(tl->fc, "VGC_function_%T (struct sess *sp)\n", - tl->t); + Fc(tl, 1, "static int\n"); + Fc(tl, 1, "VGC_function_%T (struct sess *sp)\n", tl->t); NextToken(tl); L(tl, Compound(tl)); - sbuf_printf(tl->fc, "\n"); + Fc(tl, 0, "\n"); } /*-------------------------------------------------------------------- @@ -1621,11 +1616,11 @@ unsigned lin, pos; const char *p; - sbuf_printf(tl->fh, + Fh(tl, 0, "#define VGC_NREFS %u\n", tl->cnt + 1); - sbuf_printf(tl->fh, + Fh(tl, 0, "static struct vrt_ref VGC_ref[VGC_NREFS];\n"); - sbuf_printf(tl->fc, + Fc(tl, 0, "static struct vrt_ref VGC_ref[VGC_NREFS] = {\n"); lin = 1; pos = 0; @@ -1644,12 +1639,12 @@ pos++; } - sbuf_printf(tl->fc, + Fc(tl, 0, "%*.*s[%3u] = { %4u, %3u, 0, \"T\" },\n", INDENT, INDENT, "", t->cnt, lin, pos + 1, t); } - sbuf_printf(tl->fc, "};\n"); + Fc(tl, 0, "};\n"); } @@ -1660,12 +1655,12 @@ { struct ref *r; - sbuf_printf(tl->fc, + Fc(tl, 0, "\nstatic void\n" "VGC_Init(void)\n" "{\n\n"); - sbuf_printf(tl->fc, + Fc(tl, 0, "\tVRT_alloc_backends(&VCL_conf);\n"); TAILQ_FOREACH(r, &tl->refs, list) { @@ -1675,12 +1670,12 @@ case R_ACL: break; case R_BACKEND: - sbuf_printf(tl->fc, + Fc(tl, 0, "\tVGC_init_backend_%T();\n", r->name); break; } } - sbuf_printf(tl->fc, "}\n"); + Fc(tl, 0, "}\n"); } /*--------------------------------------------------------------------*/ @@ -1689,22 +1684,22 @@ EmitStruct(struct tokenlist *tl) { - sbuf_printf(tl->fc, "\nstruct VCL_conf VCL_conf = {\n"); - sbuf_printf(tl->fc, + Fc(tl, 0, "\nstruct VCL_conf VCL_conf = {\n"); + Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n"); - sbuf_printf(tl->fc, + Fc(tl, 0, "\t.init_func = VGC_Init,\n"); - sbuf_printf(tl->fc, "\t.recv_func = VGC_function_vcl_recv,\n"); - sbuf_printf(tl->fc, "\t.hit_func = VGC_function_vcl_hit,\n"); - sbuf_printf(tl->fc, "\t.miss_func = VGC_function_vcl_miss,\n"); - sbuf_printf(tl->fc, "\t.fetch_func = VGC_function_vcl_fetch,\n"); - sbuf_printf(tl->fc, + Fc(tl, 0, "\t.recv_func = VGC_function_vcl_recv,\n"); + Fc(tl, 0, "\t.hit_func = VGC_function_vcl_hit,\n"); + Fc(tl, 0, "\t.miss_func = VGC_function_vcl_miss,\n"); + Fc(tl, 0, "\t.fetch_func = VGC_function_vcl_fetch,\n"); + Fc(tl, 0, "\t.nbackend = %d,\n", tl->nbackend); - sbuf_printf(tl->fc, + Fc(tl, 0, "\t.ref = VGC_ref,\n"); - sbuf_printf(tl->fc, + Fc(tl, 0, "\t.nref = VGC_NREFS,\n"); - sbuf_printf(tl->fc, "};\n"); + Fc(tl, 0, "};\n"); } /*--------------------------------------------------------------------*/ @@ -1731,7 +1726,7 @@ tokens.fh = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(tokens.fh != NULL); - sbuf_printf(tokens.fc, "extern struct VCL_conf VCL_conf;\n"); + Fh(&tokens, 0, "extern struct VCL_conf VCL_conf;\n"); tokens.b = b; if (e == NULL) From phk at projects.linpro.no Sat Jun 24 20:12:27 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 24 Jun 2006 22:12:27 +0200 (CEST) Subject: r226 - trunk/varnish-cache/lib/libvcl Message-ID: <20060624201227.11FB31EC232@projects.linpro.no> Author: phk Date: 2006-06-24 22:12:26 +0200 (Sat, 24 Jun 2006) New Revision: 226 Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c Log: Collapse multiline Fc and Fh calls where they fit Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-24 20:09:07 UTC (rev 225) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-24 20:12:26 UTC (rev 226) @@ -1055,8 +1055,7 @@ case BACKEND: if (tl->t->tok == '=') { NextToken(tl); - Fc(tl, 0, "= &VGC_backend_%T;\n", - tl->t); + Fc(tl, 0, "= &VGC_backend_%T;\n", tl->t); NextToken(tl); break; } @@ -1289,8 +1288,7 @@ tl->curproc = AddProc(tl, tl->t, 1); tl->curproc->exists++; AddDef(tl, tl->t, R_FUNC); - Fh(tl, 0, - "static int VGC_function_%T (struct sess *sp);\n", tl->t); + Fh(tl, 0, "static int VGC_function_%T (struct sess *sp);\n", tl->t); Fc(tl, 1, "static int\n"); Fc(tl, 1, "VGC_function_%T (struct sess *sp)\n", tl->t); NextToken(tl); @@ -1616,12 +1614,9 @@ unsigned lin, pos; const char *p; - Fh(tl, 0, - "#define VGC_NREFS %u\n", tl->cnt + 1); - Fh(tl, 0, - "static struct vrt_ref VGC_ref[VGC_NREFS];\n"); - Fc(tl, 0, - "static struct vrt_ref VGC_ref[VGC_NREFS] = {\n"); + Fh(tl, 0, "#define VGC_NREFS %u\n", tl->cnt + 1); + Fh(tl, 0, "static struct vrt_ref VGC_ref[VGC_NREFS];\n"); + Fc(tl, 0, "static struct vrt_ref VGC_ref[VGC_NREFS] = {\n"); lin = 1; pos = 0; p = tl->b; @@ -1639,9 +1634,7 @@ pos++; } - Fc(tl, 0, - "%*.*s[%3u] = { %4u, %3u, 0, \"T\" },\n", - INDENT, INDENT, "", + Fc(tl, 0, " [%3u] = { %4u, %3u, 0, \"T\" },\n", t->cnt, lin, pos + 1, t); } Fc(tl, 0, "};\n"); @@ -1655,13 +1648,8 @@ { struct ref *r; - Fc(tl, 0, - "\nstatic void\n" - "VGC_Init(void)\n" - "{\n\n"); - - Fc(tl, 0, - "\tVRT_alloc_backends(&VCL_conf);\n"); + Fc(tl, 0, "\nstatic void\nVGC_Init(void)\n{\n\n"); + Fc(tl, 0, "\tVRT_alloc_backends(&VCL_conf);\n"); TAILQ_FOREACH(r, &tl->refs, list) { switch(r->type) { @@ -1670,8 +1658,7 @@ case R_ACL: break; case R_BACKEND: - Fc(tl, 0, - "\tVGC_init_backend_%T();\n", r->name); + Fc(tl, 0, "\tVGC_init_backend_%T();\n", r->name); break; } } @@ -1685,20 +1672,15 @@ { Fc(tl, 0, "\nstruct VCL_conf VCL_conf = {\n"); - Fc(tl, 0, - "\t.magic = VCL_CONF_MAGIC,\n"); - Fc(tl, 0, - "\t.init_func = VGC_Init,\n"); + Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n"); + Fc(tl, 0, "\t.init_func = VGC_Init,\n"); Fc(tl, 0, "\t.recv_func = VGC_function_vcl_recv,\n"); Fc(tl, 0, "\t.hit_func = VGC_function_vcl_hit,\n"); Fc(tl, 0, "\t.miss_func = VGC_function_vcl_miss,\n"); Fc(tl, 0, "\t.fetch_func = VGC_function_vcl_fetch,\n"); - Fc(tl, 0, - "\t.nbackend = %d,\n", tl->nbackend); - Fc(tl, 0, - "\t.ref = VGC_ref,\n"); - Fc(tl, 0, - "\t.nref = VGC_NREFS,\n"); + Fc(tl, 0, "\t.nbackend = %d,\n", tl->nbackend); + Fc(tl, 0, "\t.ref = VGC_ref,\n"); + Fc(tl, 0, "\t.nref = VGC_NREFS,\n"); Fc(tl, 0, "};\n"); } From phk at projects.linpro.no Sat Jun 24 20:50:17 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 24 Jun 2006 22:50:17 +0200 (CEST) Subject: r227 - trunk/varnish-cache/lib/libvcl Message-ID: <20060624205017.8A09D1EC233@projects.linpro.no> Author: phk Date: 2006-06-24 22:50:17 +0200 (Sat, 24 Jun 2006) New Revision: 227 Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl trunk/varnish-cache/lib/libvcl/vcl_token_defs.h Log: Add a token type "METHOD", we use it for reference counting. Add a reference to the first backend {} we encounter, it is the default. Add a reference to all backends assigned explicitly. Add a reference to all methods. Enable reference check, complain if: backend, function or acl is defined but not used, or used but not defined. Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-24 20:12:26 UTC (rev 226) +++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-06-24 20:50:17 UTC (rev 227) @@ -169,6 +169,7 @@ { "req.request", STRING, 0, "VRT_GetReq(sp)" }, { "obj.valid", BOOL, 0, "VRT_obj_valid(sp)" }, { "obj.cacheable", BOOL, 0, "VRT_obj_cacheable(sp)" }, + { "obj.backend", BACKEND, 0, "VRT_obj_backend(sp)" }, { "req.http.", HEADER, 0, NULL }, #if 0 { "req.ttlfactor", FLOAT, 0, "req->ttlfactor" }, @@ -226,6 +227,8 @@ lin = 1; pos = 0; + if (t->tok == METHOD) + return; for (l = p = tl->b; p < t->b; p++) { if (*p == '\n') { lin++; @@ -316,7 +319,9 @@ tl->t->cnt = tl->cnt; \ } while (0) -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * Printf output to the two sbufs, possibly indented + */ static void Fh(struct tokenlist *tl, int indent, const char *fmt, ...) @@ -343,6 +348,18 @@ } /*-------------------------------------------------------------------- + * Compare token to token + */ + +static int +Teq(struct token *t1, struct token *t2) +{ + if (t1->e - t1->b != t2->e - t2->b) + return (0); + return (!memcmp(t1->b, t2->b, t1->e - t1->b)); +} + +/*-------------------------------------------------------------------- * Compare ID token to string, return true of match */ @@ -424,11 +441,8 @@ TAILQ_FOREACH(r, &tl->refs, list) { if (r->type != type) continue; - if (r->name->e - r->name->b != t->e - t->b) - continue; - if (memcmp(t->b, r->name->b, t->e - t->b)) - continue; - return (r); + if (Teq(r->name, t)) + return (r); } r = calloc(sizeof *r, 1); assert(r != NULL); @@ -446,10 +460,25 @@ } static void +AddRefStr(struct tokenlist *tl, const char *s, enum ref_type type) +{ + struct token *t; + + t = calloc(sizeof *t, 1); + t->b = s; + t->e = strchr(s, '\0'); + t->tok = METHOD; + AddRef(tl, t, type); +} + +static void AddDef(struct tokenlist *tl, struct token *t, enum ref_type type) { + struct ref *r; - FindRef(tl, t, type)->defcnt++; + r = FindRef(tl, t, type); + r->defcnt++; + r->name = t; } /*-------------------------------------------------------------------- @@ -1055,6 +1084,7 @@ case BACKEND: if (tl->t->tok == '=') { NextToken(tl); + AddRef(tl, tl->t, R_BACKEND); Fc(tl, 0, "= &VGC_backend_%T;\n", tl->t); NextToken(tl); break; @@ -1196,6 +1226,8 @@ ExpectErr(tl, ID); t_be = tl->t; AddDef(tl, tl->t, R_BACKEND); + if (tl->nbackend == 0) + AddRef(tl, tl->t, R_BACKEND); Fh(tl, 1, "#define VGC_backend_%T (VCL_conf.backend[%d])\n", tl->t, tl->nbackend); Fc(tl, 0, "static void\n"); @@ -1459,13 +1491,11 @@ struct proc *p; TAILQ_FOREACH(p, &tl->procs, list) { - if (p->name->e - p->name->b != t->e - t->b) + if (!Teq(p->name, t)) continue; - if (!memcmp(p->name->b, t->b, t->e - t->b)) { - if (def) - p->name = t; - return (p); - } + if (def) + p->name = t; + return (p); } p = calloc(sizeof *p, 1); assert(p != NULL); @@ -1567,28 +1597,27 @@ /*--------------------------------------------------------------------*/ -static void +static int CheckRefs(struct tokenlist *tl) { struct ref *r; - const char *bug; + const char *type; + int nerr = 0; TAILQ_FOREACH(r, &tl->refs, list) { - if (r->defcnt == 0) - bug = "Undefined "; - else if (r->refcnt == 0) - bug = "Unreferenced "; - else + if (r->defcnt != 0 && r->refcnt != 0) continue; + nerr++; + switch(r->type) { case R_FUNC: - sbuf_printf(tl->sb, "%s function ", bug); + type = "function"; break; case R_ACL: - sbuf_printf(tl->sb, "%s acl ", bug); + type = "acl"; break; case R_BACKEND: - sbuf_printf(tl->sb, "%s backend ", bug); + type = "backend"; break; default: ErrInternal(tl); @@ -1596,13 +1625,26 @@ ErrToken(tl, r->name); sbuf_printf(tl->sb, " has unknown type %d\n", r->type); - return; + continue; } - ErrToken(tl, r->name); - sbuf_cat(tl->sb, ", first mention is\n"); + if (r->defcnt == 0 && r->name->tok == METHOD) { + sbuf_printf(tl->sb, + "No definition for method %T\n", r->name); + continue; + } + + if (r->defcnt == 0) { + sbuf_printf(tl->sb, + "Undefined %s %T, first reference:\n", + type, r->name); + ErrWhere(tl, r->name); + continue; + } + + sbuf_printf(tl->sb, "Unused %s %T, defined:\n", type, r->name); ErrWhere(tl, r->name); - return; } + return (nerr); } /*--------------------------------------------------------------------*/ @@ -1674,13 +1716,16 @@ Fc(tl, 0, "\nstruct VCL_conf VCL_conf = {\n"); Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n"); Fc(tl, 0, "\t.init_func = VGC_Init,\n"); - Fc(tl, 0, "\t.recv_func = VGC_function_vcl_recv,\n"); - Fc(tl, 0, "\t.hit_func = VGC_function_vcl_hit,\n"); - Fc(tl, 0, "\t.miss_func = VGC_function_vcl_miss,\n"); - Fc(tl, 0, "\t.fetch_func = VGC_function_vcl_fetch,\n"); Fc(tl, 0, "\t.nbackend = %d,\n", tl->nbackend); Fc(tl, 0, "\t.ref = VGC_ref,\n"); Fc(tl, 0, "\t.nref = VGC_NREFS,\n"); +#define VCL_RET_MAC(l,u,b) +#define VCL_MET_MAC(l,u,b) \ + Fc(tl, 0, "\t." #l "_func = VGC_function_vcl_" #l ",\n"); \ + AddRefStr(tl, "vcl_" #l, R_FUNC); +#include "vcl_returns.h" +#undef VCL_MET_MAC +#undef VCL_RET_MAC Fc(tl, 0, "};\n"); } @@ -1723,8 +1768,6 @@ if (tokens.err) goto done; Consistency(&tokens); -if (0) - CheckRefs(&tokens); if (tokens.err) goto done; LocTable(&tokens); @@ -1733,6 +1776,9 @@ EmitStruct(&tokens); + if (CheckRefs(&tokens)) + goto done; + of = strdup("/tmp/vcl.XXXXXXXX"); assert(of != NULL); mktemp(of); Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-24 20:12:26 UTC (rev 226) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-24 20:50:17 UTC (rev 227) @@ -364,6 +364,7 @@ vcl_tnames[CSTR] = "CSTR"; vcl_tnames[EOI] = "EOI"; vcl_tnames[ID] = "ID"; + vcl_tnames[METHOD] = "METHOD"; vcl_tnames[T_ACL] = "acl"; vcl_tnames[T_BACKEND] = "backend"; vcl_tnames[T_CALL] = "call"; Modified: trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-06-24 20:12:26 UTC (rev 226) +++ trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-06-24 20:50:17 UTC (rev 227) @@ -69,7 +69,7 @@ # Other token identifiers # -set extras {ID VAR CNUM CSTR EOI} +set extras {ID VAR CNUM CSTR EOI METHOD} #---------------------------------------------------------------------- # Boilerplate warning for all generated files. Modified: trunk/varnish-cache/lib/libvcl/vcl_token_defs.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-06-24 20:12:26 UTC (rev 226) +++ trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-06-24 20:50:17 UTC (rev 227) @@ -48,3 +48,4 @@ #define CNUM 166 #define CSTR 167 #define EOI 168 +#define METHOD 169 From phk at projects.linpro.no Sat Jun 24 21:07:18 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 24 Jun 2006 23:07:18 +0200 (CEST) Subject: r228 - in trunk/varnish-cache: bin/varnishd include lib/libvcl Message-ID: <20060624210718.E70831EC22E@projects.linpro.no> Author: phk Date: 2006-06-24 23:07:18 +0200 (Sat, 24 Jun 2006) New Revision: 228 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/include/vcl.h trunk/varnish-cache/include/vcl_returns.h trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl trunk/varnish-cache/lib/libvcl/vcl_token_defs.h Log: Add a "timeout" method to VCL, it can return "fetch" or "discard". Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-24 20:50:17 UTC (rev 227) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-24 21:07:18 UTC (rev 228) @@ -107,6 +107,9 @@ " }\n" " insert;\n" "}\n" + "sub vcl_timeout {\n" + " discard;\n" + "}\n" "", bflag); assert(buf != NULL); sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); Modified: trunk/varnish-cache/include/vcl.h =================================================================== --- trunk/varnish-cache/include/vcl.h 2006-06-24 20:50:17 UTC (rev 227) +++ trunk/varnish-cache/include/vcl.h 2006-06-24 21:07:18 UTC (rev 228) @@ -27,4 +27,5 @@ vcl_func_f *miss_func; vcl_func_f *hit_func; vcl_func_f *fetch_func; + vcl_func_f *timeout_func; }; Modified: trunk/varnish-cache/include/vcl_returns.h =================================================================== --- trunk/varnish-cache/include/vcl_returns.h 2006-06-24 20:50:17 UTC (rev 227) +++ trunk/varnish-cache/include/vcl_returns.h 2006-06-24 21:07:18 UTC (rev 228) @@ -16,6 +16,7 @@ VCL_RET_MAC(fetch, FETCH, (1 << 4)) VCL_RET_MAC(insert, INSERT, (1 << 5)) VCL_RET_MAC(deliver, DELIVER, (1 << 6)) +VCL_RET_MAC(discard, DISCARD, (1 << 7)) #else #define VCL_RET_ERROR (1 << 0) #define VCL_RET_LOOKUP (1 << 1) @@ -24,7 +25,8 @@ #define VCL_RET_FETCH (1 << 4) #define VCL_RET_INSERT (1 << 5) #define VCL_RET_DELIVER (1 << 6) -#define VCL_RET_MAX 7 +#define VCL_RET_DISCARD (1 << 7) +#define VCL_RET_MAX 8 #endif #ifdef VCL_MET_MAC @@ -32,4 +34,5 @@ VCL_MET_MAC(miss,MISS,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_FETCH)) VCL_MET_MAC(hit,HIT,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_DELIVER)) VCL_MET_MAC(fetch,FETCH,(VCL_RET_ERROR|VCL_RET_PASS|VCL_RET_PIPE|VCL_RET_INSERT)) +VCL_MET_MAC(timeout,TIMEOUT,(VCL_RET_FETCH|VCL_RET_DISCARD)) #endif Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-24 20:50:17 UTC (rev 227) +++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-06-24 21:07:18 UTC (rev 228) @@ -180,6 +180,12 @@ } return (0); case 'd': + if (p[0] == 'd' && p[1] == 'i' && p[2] == 's' && + p[3] == 'c' && p[4] == 'a' && p[5] == 'r' && + p[6] == 'd' && !isvar(p[7])) { + *q = p + 7; + return (T_DISCARD); + } if (p[0] == 'd' && p[1] == 'e' && p[2] == 'l' && p[3] == 'i' && p[4] == 'v' && p[5] == 'e' && p[6] == 'r' && !isvar(p[7])) { @@ -373,6 +379,7 @@ vcl_tnames[T_DEC] = "--"; vcl_tnames[T_DECR] = "/="; vcl_tnames[T_DELIVER] = "deliver"; + vcl_tnames[T_DISCARD] = "discard"; vcl_tnames[T_DIV] = "/="; vcl_tnames[T_ELSE] = "else"; vcl_tnames[T_ELSEIF] = "elseif"; @@ -414,6 +421,7 @@ fputs("#define VCL_RET_FETCH (1 << 4)\n", f); fputs("#define VCL_RET_INSERT (1 << 5)\n", f); fputs("#define VCL_RET_DELIVER (1 << 6)\n", f); + fputs("#define VCL_RET_DISCARD (1 << 7)\n", f); fputs("/*\n", f); fputs(" * $Id$\n", f); fputs(" *\n", f); @@ -443,6 +451,7 @@ fputs(" vcl_func_f *miss_func;\n", f); fputs(" vcl_func_f *hit_func;\n", f); fputs(" vcl_func_f *fetch_func;\n", f); + fputs(" vcl_func_f *timeout_func;\n", f); fputs("};\n", f); fputs("/*\n", f); fputs(" * $Id$ \n", f); Modified: trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-06-24 20:50:17 UTC (rev 227) +++ trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-06-24 21:07:18 UTC (rev 228) @@ -7,10 +7,11 @@ # Second element is list of valid return actions. # set methods { - {recv {error pass pipe lookup}} - {miss {error pass pipe fetch}} - {hit {error pass pipe deliver}} - {fetch {error pass pipe insert}} + {recv {error pass pipe lookup}} + {miss {error pass pipe fetch}} + {hit {error pass pipe deliver}} + {fetch {error pass pipe insert}} + {timeout {fetch discard}} } # These are the return actions @@ -23,6 +24,7 @@ fetch insert deliver + discard } # Language keywords Modified: trunk/varnish-cache/lib/libvcl/vcl_token_defs.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-06-24 20:50:17 UTC (rev 227) +++ trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-06-24 21:07:18 UTC (rev 228) @@ -29,23 +29,24 @@ #define T_FETCH 147 #define T_INSERT 148 #define T_DELIVER 149 -#define T_INC 150 -#define T_DEC 151 -#define T_CAND 152 -#define T_COR 153 -#define T_LEQ 154 -#define T_EQ 155 -#define T_NEQ 156 -#define T_GEQ 157 -#define T_SHR 158 -#define T_SHL 159 -#define T_INCR 160 -#define T_DECR 161 -#define T_MUL 162 -#define T_DIV 163 -#define ID 164 -#define VAR 165 -#define CNUM 166 -#define CSTR 167 -#define EOI 168 -#define METHOD 169 +#define T_DISCARD 150 +#define T_INC 151 +#define T_DEC 152 +#define T_CAND 153 +#define T_COR 154 +#define T_LEQ 155 +#define T_EQ 156 +#define T_NEQ 157 +#define T_GEQ 158 +#define T_SHR 159 +#define T_SHL 160 +#define T_INCR 161 +#define T_DECR 162 +#define T_MUL 163 +#define T_DIV 164 +#define ID 165 +#define VAR 166 +#define CNUM 167 +#define CSTR 168 +#define EOI 169 +#define METHOD 170 From phk at projects.linpro.no Sat Jun 24 21:09:07 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 24 Jun 2006 23:09:07 +0200 (CEST) Subject: r229 - trunk/varnish-cache/bin/varnishd Message-ID: <20060624210907.E1A351EC233@projects.linpro.no> Author: phk Date: 2006-06-24 23:09:07 +0200 (Sat, 24 Jun 2006) New Revision: 229 Modified: trunk/varnish-cache/bin/varnishd/cache.h Log: Autogenerate prototypes for method calling functions Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-24 21:07:18 UTC (rev 228) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-24 21:09:07 UTC (rev 229) @@ -188,10 +188,13 @@ void RelVCL(struct VCL_conf *vc); struct VCL_conf *GetVCL(void); int CVCL_Load(const char *fn, const char *name); -void VCL_recv_method(struct sess *); -void VCL_hit_method(struct sess *); -void VCL_miss_method(struct sess *); -void VCL_fetch_method(struct sess *); + +#define VCL_RET_MAC(l,u,b) +#define VCL_MET_MAC(l,u,b) void VCL_##l##_method(struct sess *); +#include "vcl_returns.h" +#undef VCL_MET_MAC +#undef VCL_RET_MAC + #ifdef CLI_PRIV_H cli_func_t cli_func_config_list; cli_func_t cli_func_config_load; From phk at projects.linpro.no Sat Jun 24 21:42:27 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 24 Jun 2006 23:42:27 +0200 (CEST) Subject: r230 - trunk/varnish-cache/bin/varnishd Message-ID: <20060624214227.7F6291EC236@projects.linpro.no> Author: phk Date: 2006-06-24 23:42:27 +0200 (Sat, 24 Jun 2006) New Revision: 230 Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c Log: Truncate TTLs in the past to now. Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-24 21:09:07 UTC (rev 229) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-24 21:42:27 UTC (rev 230) @@ -80,7 +80,9 @@ ttl = h_expires; if (ttl == 0) ttl = t_resp + heritage.default_ttl; - printf("TTL: %d (+%d)\n", ttl, ttl - t_resp); + printf("TTL: %d (%+d)\n", ttl, ttl - t_resp); + if (ttl < t_resp) + return (t_resp); return (ttl); } From phk at projects.linpro.no Sat Jun 24 21:54:54 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sat, 24 Jun 2006 23:54:54 +0200 (CEST) Subject: r231 - trunk/varnish-cache/bin/varnishd Message-ID: <20060624215454.6C1B51EC233@projects.linpro.no> Author: phk Date: 2006-06-24 23:54:54 +0200 (Sat, 24 Jun 2006) New Revision: 231 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/rfc2616.c Log: Use ttl=0 as a "invalid TTL" flag. Mark objects with ttl=0 uncachable. Add cacheable objects to the expiry heap Start an expiry thread which polls the root element once per second Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-24 21:42:27 UTC (rev 230) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-24 21:54:54 UTC (rev 231) @@ -138,6 +138,7 @@ void VBE_RecycleFd(void *ptr); /* cache_expiry.c */ +void EXP_Insert(struct object *o); void EXP_Init(void); /* cache_fetch.c */ Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-24 21:42:27 UTC (rev 230) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-24 21:54:54 UTC (rev 231) @@ -4,7 +4,73 @@ * Expiry of cached objects and execution of prefetcher */ +#include +#include +#include +#include + +#include "libvarnish.h" +#include "binary_heap.h" +#include "cache.h" + +static pthread_t exp_thread; +static struct binheap *exp_heap; +static pthread_mutex_t expmtx; + +/*--------------------------------------------------------------------*/ + void +EXP_Insert(struct object *o) +{ + + AZ(pthread_mutex_lock(&expmtx)); + binheap_insert(exp_heap, o); + AZ(pthread_mutex_unlock(&expmtx)); +} + +/*--------------------------------------------------------------------*/ + +static void * +exp_main(void *arg) +{ + struct object *o; + time_t t; + + while (1) { + time(&t); + AZ(pthread_mutex_lock(&expmtx)); + o = binheap_root(exp_heap); + AZ(pthread_mutex_unlock(&expmtx)); + if (o != NULL) { + printf("Root: %p %d (%d)\n", + (void*)o, o->ttl, o->ttl - t); + } + sleep(1); + } + + return ("FOOBAR"); +} + +/*--------------------------------------------------------------------*/ + +static int +object_cmp(void *priv, void *a, void *b) +{ + struct object *aa, *bb; + + aa = a; + bb = b; + return (aa->ttl < bb->ttl); +} + +/*--------------------------------------------------------------------*/ + +void EXP_Init(void) { + + AZ(pthread_create(&exp_thread, NULL, exp_main, NULL)); + AZ(pthread_mutex_init(&expmtx, NULL)); + exp_heap = binheap_new(NULL, object_cmp, NULL); + assert(exp_heap != NULL); } Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-24 21:42:27 UTC (rev 230) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-24 21:54:54 UTC (rev 231) @@ -223,8 +223,6 @@ time(&t_resp); http_Dissect(hp, fd, 2); - sp->obj->ttl = RFC2616_Ttl(hp, t_req, t_resp); - switch (http_GetStatus(hp)) { case 200: case 301: @@ -242,8 +240,16 @@ break; } + sp->obj->ttl = RFC2616_Ttl(hp, t_req, t_resp); + if (sp->obj->ttl == 0) { + sp->obj->cacheable = 0; + } + VCL_fetch_method(sp); + if (sp->obj->cacheable) + EXP_Insert(sp->obj); + if (http_GetHdr(hp, "Content-Length", &b)) cls = fetch_straight(w, sp, fd, hp, b); else if (http_HdrIs(hp, "Transfer-Encoding", "chunked")) @@ -264,5 +270,7 @@ /* XXX: unbusy, and kick other sessions into action */ sp->obj->busy = 0; + /* XXX: if not cachable, destroy */ + return (1); } Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-24 21:42:27 UTC (rev 230) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-24 21:54:54 UTC (rev 231) @@ -82,7 +82,7 @@ ttl = t_resp + heritage.default_ttl; printf("TTL: %d (%+d)\n", ttl, ttl - t_resp); if (ttl < t_resp) - return (t_resp); + return (0); return (ttl); } From phk at projects.linpro.no Sat Jun 24 22:11:55 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 25 Jun 2006 00:11:55 +0200 (CEST) Subject: r232 - trunk/varnish-cache/bin/varnishd Message-ID: <20060624221155.1E24B1EC236@projects.linpro.no> Author: phk Date: 2006-06-25 00:11:55 +0200 (Sun, 25 Jun 2006) New Revision: 232 Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c Log: A little bit more work on the expiry/prefetch thing. Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-24 21:54:54 UTC (rev 231) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-24 22:11:55 UTC (rev 232) @@ -16,6 +16,7 @@ static pthread_t exp_thread; static struct binheap *exp_heap; static pthread_mutex_t expmtx; +static unsigned expearly = 30; /*--------------------------------------------------------------------*/ @@ -40,12 +41,17 @@ time(&t); AZ(pthread_mutex_lock(&expmtx)); o = binheap_root(exp_heap); + if (o == NULL || o->ttl - t > expearly) { + AZ(pthread_mutex_unlock(&expmtx)); + if (o != NULL) + printf("Root: %p %d (%d)\n", + (void*)o, o->ttl, o->ttl - t); + sleep(1); + continue; + } + printf("Root: %p %d (%d)\n", (void*)o, o->ttl, o->ttl - t); + binheap_delete(exp_heap, 0); AZ(pthread_mutex_unlock(&expmtx)); - if (o != NULL) { - printf("Root: %p %d (%d)\n", - (void*)o, o->ttl, o->ttl - t); - } - sleep(1); } return ("FOOBAR"); From phk at projects.linpro.no Mon Jun 26 08:58:08 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 26 Jun 2006 10:58:08 +0200 (CEST) Subject: r233 - trunk/varnish-cache/bin/varnishd Message-ID: <20060626085808.80A721EC30B@projects.linpro.no> Author: phk Date: 2006-06-26 10:58:08 +0200 (Mon, 26 Jun 2006) New Revision: 233 Added: trunk/varnish-cache/bin/varnishd/cache_hash.c Modified: trunk/varnish-cache/bin/varnishd/Makefile.am trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_main.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/hash_simple_list.c Log: Now that we approach the time where objects have to be destroyed again, we need to move the data structures into the right shape. Push hashing into cache_hash.c Add objhead structure to hold the various hits for "Vary:" headers. Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-06-24 22:11:55 UTC (rev 232) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-06-26 08:58:08 UTC (rev 233) @@ -9,6 +9,7 @@ cache_backend.c \ cache_expire.c \ cache_fetch.c \ + cache_hash.c \ cache_http.c \ cache_main.c \ cache_pool.c \ Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-24 22:11:55 UTC (rev 232) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-26 08:58:08 UTC (rev 233) @@ -4,14 +4,22 @@ #include +#include "vcl_returns.h" + +#define VCA_ADDRBUFSIZE 32 /* Sizeof ascii network address */ + struct event_base; struct sbuf; +struct sess; +struct object; +struct objhead; #ifdef EV_TIMEOUT struct worker { struct event_base *eb; struct event e1, e2; struct sbuf *sb; + struct objhead *nobjhead; struct object *nobj; }; #else @@ -21,9 +29,9 @@ /* Hashing -----------------------------------------------------------*/ typedef void hash_init_f(void); -typedef struct object *hash_lookup_f(unsigned char *key, struct object *nobj); -typedef void hash_deref_f(struct object *obj); -typedef void hash_purge_f(struct object *obj); +typedef struct objhead *hash_lookup_f(unsigned char *key, struct objhead *nobj); +typedef void hash_deref_f(struct objhead *obj); +typedef void hash_purge_f(struct objhead *obj); struct hash_slinger { const char *name; @@ -35,8 +43,6 @@ extern struct hash_slinger hsl_slinger; -extern struct hash_slinger *hash; - /* Storage -----------------------------------------------------------*/ struct storage { @@ -57,29 +63,35 @@ */ extern struct stevedore *stevedore; -/* Storage -----------------------------------------------------------*/ +/* -------------------------------------------------------------------*/ -struct sess; - -#define VCA_ADDRBUFSIZE 32 - struct object { unsigned char hash[16]; unsigned refcnt; unsigned valid; unsigned cacheable; + struct objhead *objhead; + pthread_cond_t cv; + unsigned busy; unsigned len; time_t ttl; char *header; + TAILQ_ENTRY(object) list; TAILQ_HEAD(, storage) store; }; -#include "vcl_returns.h" +struct objhead { + unsigned char hash[16]; + unsigned refcnt; + pthread_mutex_t mtx; + TAILQ_HEAD(,object) objects; +}; + struct sess { int fd; @@ -144,6 +156,12 @@ /* cache_fetch.c */ int FetchSession(struct worker *w, struct sess *sp); +/* cache_hash.c */ +struct object *HSH_Lookup(struct worker *w, struct http *h); +void HSH_Unbusy(struct object *o); +void HSH_Unref(struct object *o); +void HSH_Init(void); + /* cache_http.c */ typedef void http_callback_f(void *, int good); struct http; Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-24 22:11:55 UTC (rev 232) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-26 08:58:08 UTC (rev 233) @@ -72,7 +72,9 @@ vca_write_obj(sp, w->sb); +#if 0 hash->deref(sp->obj); +#endif return (0); } @@ -187,7 +189,9 @@ vca_write_obj(sp, w->sb); +#if 0 hash->deref(sp->obj); +#endif return (0); } Added: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-06-24 22:11:55 UTC (rev 232) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-06-26 08:58:08 UTC (rev 233) @@ -0,0 +1,97 @@ +/* + * $Id$ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "libvarnish.h" +#include "cache.h" + +static struct hash_slinger *hash; + + +struct object * +HSH_Lookup(struct worker *w, struct http *h) +{ + struct objhead *oh; + struct object *o; + unsigned char key[16]; + MD5_CTX ctx; + char *b; + + assert(hash != NULL); + /* Make sure we have both a new objhead and object if we need them */ + if (w->nobjhead == NULL) { + w->nobjhead = calloc(sizeof *w->nobjhead, 1); + assert(w->nobjhead != NULL); + TAILQ_INIT(&w->nobjhead->objects); + } + if (w->nobj == NULL) { + w->nobj = calloc(sizeof *w->nobj, 1); + assert(w->nobj != NULL); + w->nobj->busy = 1; + TAILQ_INIT(&w->nobj->store); + pthread_cond_init(&w->nobj->cv, NULL); + } + + assert(http_GetURL(h, &b)); + MD5Init(&ctx); + MD5Update(&ctx, b, strlen(b)); + MD5Final(key, &ctx); + oh = hash->lookup(key, w->nobjhead); + if (oh == w->nobjhead) + w->nobjhead = NULL; + AZ(pthread_mutex_lock(&oh->mtx)); + TAILQ_FOREACH(o, &oh->objects, list) { + o->refcnt++; + if (o->busy) + AZ(pthread_cond_wait(&o->cv, &oh->mtx)); + /* XXX: do Vary: comparison */ + if (1) + break; + o->refcnt--; + } + if (o == NULL) { + o = w->nobj; + w->nobj = NULL; + TAILQ_INSERT_TAIL(&oh->objects, o, list); + } + AZ(pthread_mutex_unlock(&oh->mtx)); + return (o); +} + +void +HSH_Unbusy(struct object *o) +{ + + AZ(pthread_mutex_lock(&o->objhead->mtx)); + o->busy = 0; + AZ(pthread_mutex_unlock(&o->objhead->mtx)); + AZ(pthread_cond_broadcast(&o->cv)); +} + +void +HSH_Unref(struct object *o) +{ + + AZ(pthread_mutex_lock(&o->objhead->mtx)); + o->refcnt--; + AZ(pthread_mutex_unlock(&o->objhead->mtx)); +} + +void +HSH_Init(void) +{ + + hash = &hsl_slinger; + if (hash->init != NULL) + hash->init(); +} Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-24 22:11:55 UTC (rev 232) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-26 08:58:08 UTC (rev 233) @@ -24,7 +24,6 @@ static struct event ev_keepalive; -struct hash_slinger *hash; struct stevedore *stevedore; pthread_mutex_t sessmtx; @@ -115,14 +114,11 @@ VCA_Init(); EXP_Init(); + HSH_Init(); eb = event_init(); assert(eb != NULL); - hash = &hsl_slinger; - if (hash->init != NULL) - hash->init(); - stevedore = heritage.stevedore; if (stevedore->open != NULL) stevedore->open(stevedore); Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-24 22:11:55 UTC (rev 232) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-26 08:58:08 UTC (rev 233) @@ -12,7 +12,6 @@ #include #include #include -#include #include "libvarnish.h" #include "shmlog.h" @@ -29,33 +28,13 @@ LookupSession(struct worker *w, struct sess *sp) { struct object *o; - unsigned char key[16]; - MD5_CTX ctx; - char *b; - /* Make sure worker thread has a fresh object at hand */ - if (w->nobj == NULL) { - w->nobj = calloc(sizeof *w->nobj, 1); - assert(w->nobj != NULL); - w->nobj->busy = 1; - TAILQ_INIT(&w->nobj->store); - } - - assert(http_GetURL(sp->http, &b)); - MD5Init(&ctx); - MD5Update(&ctx, b, strlen(b)); - MD5Final(key, &ctx); - o = hash->lookup(key, w->nobj); + o = HSH_Lookup(w, sp->http); sp->obj = o; - if (o != w->nobj && o->ttl > sp->t0) { - /* XXX: wait while obj->busy */ - VSL(SLT_Debug, 0, "Lookup found %p %s", o, b); + if (o->busy) + VCL_miss_method(sp); + else VCL_hit_method(sp); - return (0); - } - VSL(SLT_Debug, 0, "Lookup new %p %s", o, b); - w->nobj = NULL; - VCL_miss_method(sp); return (0); } Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-06-24 22:11:55 UTC (rev 232) +++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-06-26 08:58:08 UTC (rev 233) @@ -14,7 +14,7 @@ struct hsl_entry { TAILQ_ENTRY(hsl_entry) list; - struct object *obj; + struct objhead *obj; }; static TAILQ_HEAD(, hsl_entry) hsl_head = TAILQ_HEAD_INITIALIZER(hsl_head); @@ -27,8 +27,8 @@ AZ(pthread_mutex_init(&hsl_mutex, NULL)); } -static struct object * -hsl_lookup(unsigned char *key, struct object *nobj) +static struct objhead * +hsl_lookup(unsigned char *key, struct objhead *nobj) { struct hsl_entry *he, *he2; int i; @@ -64,7 +64,7 @@ } static void -hsl_deref(struct object *obj) +hsl_deref(struct objhead *obj) { AZ(pthread_mutex_lock(&hsl_mutex)); @@ -73,7 +73,7 @@ } static void -hsl_purge(struct object *obj) +hsl_purge(struct objhead *obj) { struct hsl_entry *he; From phk at projects.linpro.no Mon Jun 26 14:00:41 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 26 Jun 2006 16:00:41 +0200 (CEST) Subject: r234 - trunk/varnish-cache/bin/varnishd Message-ID: <20060626140041.0F10B1EC2FA@projects.linpro.no> Author: phk Date: 2006-06-26 16:00:40 +0200 (Mon, 26 Jun 2006) New Revision: 234 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/hash_simple_list.c Log: Move a bit more responsibility into the hash-slinger to get a cleaner interface. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-26 08:58:08 UTC (rev 233) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-26 14:00:40 UTC (rev 234) @@ -29,16 +29,14 @@ /* Hashing -----------------------------------------------------------*/ typedef void hash_init_f(void); -typedef struct objhead *hash_lookup_f(unsigned char *key, struct objhead *nobj); +typedef struct objhead *hash_lookup_f(const char *key, struct objhead *nobj); typedef void hash_deref_f(struct objhead *obj); -typedef void hash_purge_f(struct objhead *obj); struct hash_slinger { const char *name; hash_init_f *init; hash_lookup_f *lookup; hash_deref_f *deref; - hash_purge_f *purge; }; extern struct hash_slinger hsl_slinger; @@ -66,14 +64,13 @@ /* -------------------------------------------------------------------*/ struct object { - unsigned char hash[16]; unsigned refcnt; - unsigned valid; - unsigned cacheable; - struct objhead *objhead; pthread_cond_t cv; + unsigned valid; + unsigned cacheable; + unsigned busy; unsigned len; time_t ttl; @@ -85,8 +82,7 @@ }; struct objhead { - unsigned char hash[16]; - unsigned refcnt; + void *hashpriv; pthread_mutex_t mtx; TAILQ_HEAD(,object) objects; Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-06-26 08:58:08 UTC (rev 233) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-06-26 14:00:40 UTC (rev 234) @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -23,8 +22,6 @@ { struct objhead *oh; struct object *o; - unsigned char key[16]; - MD5_CTX ctx; char *b; assert(hash != NULL); @@ -43,10 +40,7 @@ } assert(http_GetURL(h, &b)); - MD5Init(&ctx); - MD5Update(&ctx, b, strlen(b)); - MD5Final(key, &ctx); - oh = hash->lookup(key, w->nobjhead); + oh = hash->lookup(b, w->nobjhead); if (oh == w->nobjhead) w->nobjhead = NULL; AZ(pthread_mutex_lock(&oh->mtx)); Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-06-26 08:58:08 UTC (rev 233) +++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-06-26 14:00:40 UTC (rev 234) @@ -1,5 +1,7 @@ /* * $Id$ + * + * This is the reference hash(/lookup) implementation */ #include @@ -12,14 +14,23 @@ #include #include +/*--------------------------------------------------------------------*/ + struct hsl_entry { TAILQ_ENTRY(hsl_entry) list; + char *key; struct objhead *obj; + unsigned refcnt; }; static TAILQ_HEAD(, hsl_entry) hsl_head = TAILQ_HEAD_INITIALIZER(hsl_head); static pthread_mutex_t hsl_mutex; +/*-------------------------------------------------------------------- + * The ->init method is called during process start and allows + * initialization to happen before the first lookup. + */ + static void hsl_init(void) { @@ -27,23 +38,31 @@ AZ(pthread_mutex_init(&hsl_mutex, NULL)); } +/*-------------------------------------------------------------------- + * Lookup and possibly insert element. + * If nobj != NULL and the lookup does not find key, nobj is inserted. + * If nobj == NULL and the lookup does not find key, NULL is returned. + * A reference to the returned object is held. + */ + static struct objhead * -hsl_lookup(unsigned char *key, struct objhead *nobj) +hsl_lookup(const char *key, struct objhead *nobj) { struct hsl_entry *he, *he2; int i; AZ(pthread_mutex_lock(&hsl_mutex)); TAILQ_FOREACH(he, &hsl_head, list) { - i = memcmp(key, he->obj->hash, sizeof he->obj->hash); + i = strcmp(key, he->key); + if (i < 0) + continue; if (i == 0) { - he->obj->refcnt++; + he->refcnt++; nobj = he->obj; + nobj->hashpriv = he; AZ(pthread_mutex_unlock(&hsl_mutex)); return (nobj); } - if (i < 0) - continue; if (nobj == NULL) { AZ(pthread_mutex_unlock(&hsl_mutex)); return (NULL); @@ -53,8 +72,10 @@ he2 = calloc(sizeof *he2, 1); assert(he2 != NULL); he2->obj = nobj; - nobj->refcnt++; - memcpy(nobj->hash, key, sizeof nobj->hash); + he2->refcnt = 1; + he2->key = strdup(key); + assert(he2->key != NULL); + nobj->hashpriv = he2; if (he != NULL) TAILQ_INSERT_BEFORE(he, he2, list); else @@ -63,37 +84,31 @@ return (nobj); } +/*-------------------------------------------------------------------- + * Dereference and if no references are left, free. + */ + static void hsl_deref(struct objhead *obj) { + struct hsl_entry *he; + assert(obj->hashpriv != NULL); + he = obj->hashpriv; AZ(pthread_mutex_lock(&hsl_mutex)); - obj->refcnt--; + if (--he->refcnt == 0) { + free(he->key); + TAILQ_REMOVE(&hsl_head, he, list); + free(he); + } AZ(pthread_mutex_unlock(&hsl_mutex)); } -static void -hsl_purge(struct objhead *obj) -{ - struct hsl_entry *he; +/*--------------------------------------------------------------------*/ - assert(obj->refcnt > 0); - AZ(pthread_mutex_lock(&hsl_mutex)); - TAILQ_FOREACH(he, &hsl_head, list) { - if (he->obj == obj) { - TAILQ_REMOVE(&hsl_head, he, list); - AZ(pthread_mutex_unlock(&hsl_mutex)); - free(he); - return; - } - } - assert(he != NULL); -} - struct hash_slinger hsl_slinger = { "simple_list", hsl_init, hsl_lookup, hsl_deref, - hsl_purge }; From phk at projects.linpro.no Mon Jun 26 14:33:50 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 26 Jun 2006 16:33:50 +0200 (CEST) Subject: r235 - trunk/varnish-cache/bin/varnishd Message-ID: <20060626143350.1928B1EC304@projects.linpro.no> Author: phk Date: 2006-06-26 16:33:49 +0200 (Mon, 26 Jun 2006) New Revision: 235 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/hash_simple_list.c Log: Start releasing objects when they expire Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-26 14:00:40 UTC (rev 234) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-26 14:33:49 UTC (rev 235) @@ -30,7 +30,7 @@ typedef void hash_init_f(void); typedef struct objhead *hash_lookup_f(const char *key, struct objhead *nobj); -typedef void hash_deref_f(struct objhead *obj); +typedef int hash_deref_f(struct objhead *obj); struct hash_slinger { const char *name; @@ -155,7 +155,7 @@ /* cache_hash.c */ struct object *HSH_Lookup(struct worker *w, struct http *h); void HSH_Unbusy(struct object *o); -void HSH_Unref(struct object *o); +void HSH_Deref(struct object *o); void HSH_Init(void); /* cache_http.c */ Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-26 14:00:40 UTC (rev 234) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-26 14:33:49 UTC (rev 235) @@ -52,6 +52,7 @@ printf("Root: %p %d (%d)\n", (void*)o, o->ttl, o->ttl - t); binheap_delete(exp_heap, 0); AZ(pthread_mutex_unlock(&expmtx)); + HSH_Deref(o); } return ("FOOBAR"); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-26 14:00:40 UTC (rev 234) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-26 14:33:49 UTC (rev 235) @@ -271,10 +271,9 @@ else VBE_RecycleFd(fd_token); - /* XXX: unbusy, and kick other sessions into action */ - sp->obj->busy = 0; + HSH_Unbusy(sp->obj); + if (!sp->obj->cacheable) + HSH_Deref(sp->obj); - /* XXX: if not cachable, destroy */ - return (1); } Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-06-26 14:00:40 UTC (rev 234) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-06-26 14:33:49 UTC (rev 235) @@ -16,7 +16,6 @@ static struct hash_slinger *hash; - struct object * HSH_Lookup(struct worker *w, struct http *h) { @@ -25,18 +24,19 @@ char *b; assert(hash != NULL); - /* Make sure we have both a new objhead and object if we need them */ + /* Precreate an objhead and object in case we need them */ if (w->nobjhead == NULL) { w->nobjhead = calloc(sizeof *w->nobjhead, 1); assert(w->nobjhead != NULL); TAILQ_INIT(&w->nobjhead->objects); + AZ(pthread_mutex_init(&oh->mtx, NULL)); } if (w->nobj == NULL) { w->nobj = calloc(sizeof *w->nobj, 1); assert(w->nobj != NULL); w->nobj->busy = 1; TAILQ_INIT(&w->nobj->store); - pthread_cond_init(&w->nobj->cv, NULL); + AZ(pthread_cond_init(&w->nobj->cv, NULL)); } assert(http_GetURL(h, &b)); @@ -53,11 +53,19 @@ break; o->refcnt--; } - if (o == NULL) { - o = w->nobj; - w->nobj = NULL; - TAILQ_INSERT_TAIL(&oh->objects, o, list); + if (o != NULL) { + AZ(pthread_mutex_unlock(&oh->mtx)); + hash->deref(oh); + return (o); } + + /* Insert (precreated) object in objecthead */ + o = w->nobj; + w->nobj = NULL; + o->refcnt = 1; + o->objhead = oh; + TAILQ_INSERT_TAIL(&oh->objects, o, list); + /* NB: do not deref objhead the new object inherits our reference */ AZ(pthread_mutex_unlock(&oh->mtx)); return (o); } @@ -73,12 +81,39 @@ } void -HSH_Unref(struct object *o) +HSH_Deref(struct object *o) { + struct objhead *oh; + struct storage *st, *stn; - AZ(pthread_mutex_lock(&o->objhead->mtx)); - o->refcnt--; - AZ(pthread_mutex_unlock(&o->objhead->mtx)); + oh = o->objhead; + + /* drop ref on object */ + AZ(pthread_mutex_lock(&oh->mtx)); + if (--o->refcnt == 0) + TAILQ_REMOVE(&oh->objects, o, list); + else + o = NULL; + AZ(pthread_mutex_unlock(&oh->mtx)); + + /* If still referenced, done */ + if (o == NULL) + return; + + AZ(pthread_cond_destroy(&o->cv)); + + TAILQ_FOREACH_SAFE(st, &o->store, list, stn) { + TAILQ_REMOVE(&o->store, st, list); + st->stevedore->free(st); + } + free(o); + + /* Drop our ref on the objhead */ + if (hash->deref(oh)) + return; + assert(TAILQ_EMPTY(&oh->objects)); + AZ(pthread_mutex_destroy(&oh->mtx)); + free(oh); } void Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-26 14:00:40 UTC (rev 234) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-26 14:33:49 UTC (rev 235) @@ -49,6 +49,8 @@ "\r\n", sp->obj->header, sp->obj->len); vca_write_obj(sp, w->sb); + HSH_Deref(sp->obj); + sp->obj = NULL; return (1); } Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-06-26 14:00:40 UTC (rev 234) +++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-06-26 14:33:49 UTC (rev 235) @@ -88,10 +88,11 @@ * Dereference and if no references are left, free. */ -static void +static int hsl_deref(struct objhead *obj) { struct hsl_entry *he; + int ret; assert(obj->hashpriv != NULL); he = obj->hashpriv; @@ -100,8 +101,11 @@ free(he->key); TAILQ_REMOVE(&hsl_head, he, list); free(he); - } + ret = 0; + } else + ret = 1; AZ(pthread_mutex_unlock(&hsl_mutex)); + return (ret); } /*--------------------------------------------------------------------*/ From phk at projects.linpro.no Mon Jun 26 16:19:14 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 26 Jun 2006 18:19:14 +0200 (CEST) Subject: r236 - trunk/varnish-cache/bin/varnishd Message-ID: <20060626161914.825C41EC304@projects.linpro.no> Author: phk Date: 2006-06-26 18:19:14 +0200 (Mon, 26 Jun 2006) New Revision: 236 Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c Log: Make this work again: record the stevedore in the storage object. Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-06-26 14:33:49 UTC (rev 235) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-06-26 16:19:14 UTC (rev 236) @@ -16,7 +16,7 @@ }; static struct storage * -sma_alloc(struct stevedore *st __unused, unsigned size) +sma_alloc(struct stevedore *st, unsigned size) { struct sma *sma; @@ -27,6 +27,7 @@ assert(sma->s.ptr != NULL); sma->s.len = size; sma->s.space = size; + sma->s.stevedore = st; return (&sma->s); } From phk at projects.linpro.no Mon Jun 26 16:30:56 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 26 Jun 2006 18:30:56 +0200 (CEST) Subject: r237 - trunk/varnish-cache/bin/varnishd Message-ID: <20060626163056.47DA51EC304@projects.linpro.no> Author: phk Date: 2006-06-26 18:30:56 +0200 (Mon, 26 Jun 2006) New Revision: 237 Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c Log: typo Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-06-26 16:19:14 UTC (rev 236) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-06-26 16:30:56 UTC (rev 237) @@ -29,7 +29,7 @@ w->nobjhead = calloc(sizeof *w->nobjhead, 1); assert(w->nobjhead != NULL); TAILQ_INIT(&w->nobjhead->objects); - AZ(pthread_mutex_init(&oh->mtx, NULL)); + AZ(pthread_mutex_init(&w->nobjhead->mtx, NULL)); } if (w->nobj == NULL) { w->nobj = calloc(sizeof *w->nobj, 1); From phk at projects.linpro.no Mon Jun 26 16:31:32 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 26 Jun 2006 18:31:32 +0200 (CEST) Subject: r238 - trunk/varnish-cache/bin/varnishd Message-ID: <20060626163132.4BAF01EC304@projects.linpro.no> Author: phk Date: 2006-06-26 18:31:32 +0200 (Mon, 26 Jun 2006) New Revision: 238 Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c Log: Dump numeric handling also, until we figure out trouble. Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-26 16:30:56 UTC (rev 237) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-26 16:31:32 UTC (rev 238) @@ -208,10 +208,8 @@ CheckHandling(struct sess *sp, const char *func, unsigned bitmap) { unsigned u; - const char *n; u = sp->handling; - n = HandlingName(u); if (u & (u - 1)) VSL(SLT_Error, sp->fd, "Illegal handling after %s function: 0x%x", func, u); @@ -231,7 +229,7 @@ sp->handling = 0; \ sp->vcl->func##_func(sp); \ CheckHandling(sp, #func, (bitmap)); \ - VSL(SLT_vcl_##func, sp->fd, "%s", HandlingName(sp->handling)); \ + VSL(SLT_vcl_##func, sp->fd, "0x%x %s", sp->handling, HandlingName(sp->handling)); \ } #define VCL_RET_MAC(l,u,b) From phk at projects.linpro.no Mon Jun 26 17:06:50 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 26 Jun 2006 19:06:50 +0200 (CEST) Subject: r239 - trunk/varnish-cache/bin/varnishd Message-ID: <20060626170650.BABB01EC2FF@projects.linpro.no> Author: phk Date: 2006-06-26 19:06:50 +0200 (Mon, 26 Jun 2006) New Revision: 239 Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c Log: Another little tweak Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-06-26 16:31:32 UTC (rev 238) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-06-26 17:06:50 UTC (rev 239) @@ -25,7 +25,7 @@ sma->s.priv = sma; sma->s.ptr = malloc(size); assert(sma->s.ptr != NULL); - sma->s.len = size; + sma->s.len = 0; sma->s.space = size; sma->s.stevedore = st; return (&sma->s); From phk at projects.linpro.no Mon Jun 26 19:23:24 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 26 Jun 2006 21:23:24 +0200 (CEST) Subject: r240 - trunk/varnish-cache/bin/varnishd Message-ID: <20060626192324.7A3F71EC304@projects.linpro.no> Author: phk Date: 2006-06-26 21:23:24 +0200 (Mon, 26 Jun 2006) New Revision: 240 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c Log: Implement HTTP/1.0 style fetching. Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-26 17:06:50 UTC (rev 239) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-26 19:23:24 UTC (rev 240) @@ -188,12 +188,59 @@ http_BuildSbuf(2, w->sb, hp); vca_write_obj(sp, w->sb); + return (0); +} -#if 0 - hash->deref(sp->obj); -#endif - return (0); +/*--------------------------------------------------------------------*/ + +#include + +static int +fetch_eof(struct worker *w, struct sess *sp, int fd, struct http *hp) +{ + int i; + char *b, *e; + unsigned char *p; + struct storage *st; + unsigned v; + + i = fcntl(fd, F_GETFL); /* XXX ? */ + i &= ~O_NONBLOCK; + i = fcntl(fd, F_SETFL, i); + + p = NULL; + v = 0; + while (1) { + if (v == 0) { + st = stevedore->alloc(stevedore, CHUNK_PREALLOC); + TAILQ_INSERT_TAIL(&sp->obj->store, st, list); + p = st->ptr + st->len; + v = st->space - st->len; + } + if (http_GetTail(hp, v, &b, &e)) { + memcpy(p, b, e - b); + p += e - b; + v -= e - b; + st->len += e - b; + *p = '\0'; + } + i = read(fd, p, v); + assert(i >= 0); + if (i == 0) + break; + p += i; + v -= i; + st->len += i; + } + + if (st != NULL && stevedore->trim != NULL) + stevedore->trim(st, st->len); + + http_BuildSbuf(2, w->sb, hp); + + vca_write_obj(sp, w->sb); + return (1); } /*--------------------------------------------------------------------*/ @@ -258,10 +305,8 @@ cls = fetch_straight(w, sp, fd, hp, b); else if (http_HdrIs(hp, "Transfer-Encoding", "chunked")) cls = fetch_chunked(w, sp, fd, hp); - else { - VSL(SLT_Debug, fd, "No transfer"); - cls = 0; - } + else + cls = fetch_eof(w, sp, fd, hp); if (http_GetHdr(hp, "Connection", &b) && !strcasecmp(b, "close")) cls = 1; From phk at projects.linpro.no Mon Jun 26 19:24:03 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 26 Jun 2006 21:24:03 +0200 (CEST) Subject: r241 - trunk/varnish-cache/bin/varnishd Message-ID: <20060626192403.184211EC304@projects.linpro.no> Author: phk Date: 2006-06-26 21:24:03 +0200 (Mon, 26 Jun 2006) New Revision: 241 Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c Log: A temporary hack to deal with very old Date: headers until we figure out what's going on. Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-26 19:23:24 UTC (rev 240) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-26 19:24:03 UTC (rev 241) @@ -42,6 +42,7 @@ time_t apparent_age = 0, corrected_received_age; time_t response_delay, corrected_initial_age; time_t max_age = -1, ttl; + time_t fudge; char *p; if (http_GetHdrField(hp, "Cache-Control", "max-age", &p)) @@ -50,6 +51,12 @@ if (http_GetHdr(hp, "Date", &p)) h_date = TIM_parse(p); + if (h_date + 3600 < t_resp) { + fudge = t_resp - h_date; + h_date += fudge; + } else + fudge = 0; + if (h_date < t_resp) apparent_age = t_resp - h_date; @@ -64,8 +71,9 @@ response_delay = t_resp - t_req; corrected_initial_age = corrected_received_age + response_delay; - if (http_GetHdr(hp, "Expires", &p)) - h_expires = TIM_parse(p); + if (http_GetHdr(hp, "Expires", &p)) { + h_expires = TIM_parse(p) + fudge; + } printf("Date: %d\n", h_date); printf("Recv: %d\n", t_resp); From phk at projects.linpro.no Mon Jun 26 19:25:09 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 26 Jun 2006 21:25:09 +0200 (CEST) Subject: r242 - trunk/varnish-cache/bin/varnishd Message-ID: <20060626192509.4D6601EC2FF@projects.linpro.no> Author: phk Date: 2006-06-26 21:25:09 +0200 (Mon, 26 Jun 2006) New Revision: 242 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_expire.c Log: Call VCL to decide discard/prefetch for near-expiry objects. Put discard objects on deathrow where they will be culled from in sequence. (prefetch not implemented yet) Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-26 19:24:03 UTC (rev 241) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-26 19:25:09 UTC (rev 242) @@ -78,6 +78,8 @@ char *header; TAILQ_ENTRY(object) list; + TAILQ_ENTRY(object) deathrow; + TAILQ_HEAD(, storage) store; }; Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-26 19:24:03 UTC (rev 241) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-26 19:25:09 UTC (rev 242) @@ -10,13 +10,15 @@ #include #include "libvarnish.h" +#include "shmlog.h" #include "binary_heap.h" #include "cache.h" static pthread_t exp_thread; static struct binheap *exp_heap; -static pthread_mutex_t expmtx; +static pthread_mutex_t exp_mtx; static unsigned expearly = 30; +static TAILQ_HEAD(,object) exp_deathrow = TAILQ_HEAD_INITIALIZER(exp_deathrow); /*--------------------------------------------------------------------*/ @@ -24,35 +26,76 @@ EXP_Insert(struct object *o) { - AZ(pthread_mutex_lock(&expmtx)); + AZ(pthread_mutex_lock(&exp_mtx)); binheap_insert(exp_heap, o); - AZ(pthread_mutex_unlock(&expmtx)); + AZ(pthread_mutex_unlock(&exp_mtx)); } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * This thread monitors deathrow and kills objects when they time out. + */ static void * -exp_main(void *arg) +exp_hangman(void *arg) { struct object *o; time_t t; while (1) { + time (&t); + AZ(pthread_mutex_lock(&exp_mtx)); + o = TAILQ_FIRST(&exp_deathrow); + if (o == NULL || o->ttl >= t) { /* XXX: > or >= ? */ + AZ(pthread_mutex_unlock(&exp_mtx)); + sleep(1); + continue; + } + TAILQ_REMOVE(&exp_deathrow, o, deathrow); + AZ(pthread_mutex_unlock(&exp_mtx)); + HSH_Deref(o); + } +} + +/*-------------------------------------------------------------------- + * This thread monitors the root of the binary heap and whenever an + * object gets close enough, VCL is asked to decide if it should be + * discarded or prefetched. + * If discarded, the object is put on deathrow where exp_hangman() will + * do what needs to be done. + * XXX: If prefetched pass to the pool for pickup. + */ + +static void * +exp_prefetch(void *arg) +{ + struct object *o; + time_t t; + struct sess sp; + + while (1) { time(&t); - AZ(pthread_mutex_lock(&expmtx)); + AZ(pthread_mutex_lock(&exp_mtx)); o = binheap_root(exp_heap); if (o == NULL || o->ttl - t > expearly) { - AZ(pthread_mutex_unlock(&expmtx)); - if (o != NULL) - printf("Root: %p %d (%d)\n", - (void*)o, o->ttl, o->ttl - t); + AZ(pthread_mutex_unlock(&exp_mtx)); sleep(1); continue; } - printf("Root: %p %d (%d)\n", (void*)o, o->ttl, o->ttl - t); binheap_delete(exp_heap, 0); - AZ(pthread_mutex_unlock(&expmtx)); - HSH_Deref(o); + AZ(pthread_mutex_unlock(&exp_mtx)); + + sp.vcl = GetVCL(); + sp.obj = o; + VCL_timeout_method(&sp); + RelVCL(sp.vcl); + + if (sp.handling == VCL_RET_DISCARD) { + AZ(pthread_mutex_lock(&exp_mtx)); + TAILQ_INSERT_TAIL(&exp_deathrow, o, deathrow); + AZ(pthread_mutex_unlock(&exp_mtx)); + continue; + } + assert(sp.handling == VCL_RET_DISCARD); } return ("FOOBAR"); @@ -76,8 +119,9 @@ EXP_Init(void) { - AZ(pthread_create(&exp_thread, NULL, exp_main, NULL)); - AZ(pthread_mutex_init(&expmtx, NULL)); + AZ(pthread_mutex_init(&exp_mtx, NULL)); + AZ(pthread_create(&exp_thread, NULL, exp_prefetch, NULL)); + AZ(pthread_create(&exp_thread, NULL, exp_hangman, NULL)); exp_heap = binheap_new(NULL, object_cmp, NULL); assert(exp_heap != NULL); } From phk at projects.linpro.no Wed Jun 28 09:21:16 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 11:21:16 +0200 (CEST) Subject: r243 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060628092116.35EC01EC233@projects.linpro.no> Author: phk Date: 2006-06-28 11:21:15 +0200 (Wed, 28 Jun 2006) New Revision: 243 Added: trunk/varnish-cache/include/stat_field.h trunk/varnish-cache/include/stats.h Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_shmlog.c trunk/varnish-cache/bin/varnishd/mgt.h trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/include/shmlog.h Log: Add statistics counter support. stat_field.h defines the counter fields with name, type, (printf)format and description. stats.h defines a structure with these fields. shmlog.h makes the structure part of the shared memory logs header. Implent the "stats" CLI word in the management process. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-26 19:25:09 UTC (rev 242) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-28 09:21:15 UTC (rev 243) @@ -200,6 +200,7 @@ assert(__LINE__ == 0); \ } while (0) #endif +extern struct varnish_stats *VSL_stats; /* cache_vcl.c */ void RelVCL(struct VCL_conf *vc); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-26 19:25:09 UTC (rev 242) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-28 09:21:15 UTC (rev 243) @@ -122,6 +122,8 @@ char port[10]; int i; + VSL_stats->cli_conn++; + (void)arg; sm = calloc(sizeof *sm, 1); assert(sm != NULL); /* Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-06-26 19:25:09 UTC (rev 242) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-06-28 09:21:15 UTC (rev 243) @@ -3,15 +3,22 @@ */ #include +#include +#include +#include +#include #include #include #include #include +#include "libvarnish.h" #include "shmlog.h" #include "heritage.h" +struct varnish_stats *VSL_stats; + static struct shmloghead *loghead; static unsigned char *logstart, *logend; @@ -130,4 +137,45 @@ /* XXX check sanity of loghead */ logstart = (unsigned char *)loghead + loghead->start; logend = logstart + loghead->size; + VSL_stats = &loghead->stats; } + +/*--------------------------------------------------------------------*/ + +void +VSL_MgtInit(const char *fn, unsigned size) +{ + struct shmloghead slh; + int i; + + heritage.vsl_fd = open(fn, O_RDWR | O_CREAT, 0600); + if (heritage.vsl_fd < 0) { + fprintf(stderr, "Could not open %s: %s\n", + fn, strerror(errno)); + exit (1); + } + i = read(heritage.vsl_fd, &slh, sizeof slh); + if (i != sizeof slh || slh.magic != SHMLOGHEAD_MAGIC) { + /* XXX more checks */ + + slh.magic = SHMLOGHEAD_MAGIC; + slh.size = size; + slh.ptr = 0; + slh.start = sizeof slh; + AZ(lseek(heritage.vsl_fd, 0, SEEK_SET)); + i = write(heritage.vsl_fd, &slh, sizeof slh); + assert(i == sizeof slh); + AZ(ftruncate(heritage.vsl_fd, sizeof slh + size)); + heritage.vsl_size = slh.size + slh.start; + } else { + heritage.vsl_size = slh.size + slh.start; + } + + /* + * Call VSL_Init so that we get a VSL_stats pointer in the + * management process as well. + */ + VSL_Init(); + memset(VSL_stats, 0, sizeof *VSL_stats); +} + Modified: trunk/varnish-cache/bin/varnishd/mgt.h =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt.h 2006-06-26 19:25:09 UTC (rev 242) +++ trunk/varnish-cache/bin/varnishd/mgt.h 2006-06-28 09:21:15 UTC (rev 243) @@ -18,3 +18,6 @@ extern struct stevedore sma_stevedore; extern struct stevedore smf_stevedore; + +void VSL_MgtInit(const char *fn, unsigned size); +extern struct varnish_stats *VSL_stats; Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-26 19:25:09 UTC (rev 242) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-06-28 09:21:15 UTC (rev 243) @@ -27,6 +27,7 @@ #include "mgt.h" #include "heritage.h" +#include "shmlog.h" #include "cli_event.h" /*--------------------------------------------------------------------*/ @@ -49,7 +50,7 @@ } static void -cli_func_passthrough(struct cli *cli, char **av __unused, void *priv) +m_cli_func_passthrough(struct cli *cli, char **av __unused, void *priv) { cli_suspend(cli); @@ -128,7 +129,7 @@ } static void -cli_func_config_inline(struct cli *cli, char **av, void *priv __unused) +m_cli_func_config_inline(struct cli *cli, char **av, void *priv __unused) { char *vf; struct sbuf *sb; @@ -193,7 +194,7 @@ /*--------------------------------------------------------------------*/ static void -cli_func_server_start(struct cli *cli, char **av __unused, void *priv __unused) +m_cli_func_server_start(struct cli *cli, char **av __unused, void *priv __unused) { mgt_child_start(); @@ -202,7 +203,7 @@ /*--------------------------------------------------------------------*/ static void -cli_func_server_stop(struct cli *cli, char **av __unused, void *priv __unused) +m_cli_func_server_stop(struct cli *cli, char **av __unused, void *priv __unused) { mgt_child_stop(); @@ -211,7 +212,7 @@ /*--------------------------------------------------------------------*/ static void -cli_func_verbose(struct cli *cli, char **av __unused, void *priv) +m_cli_func_verbose(struct cli *cli, char **av __unused, void *priv) { cli->verbose = !cli->verbose; @@ -219,7 +220,7 @@ static void -cli_func_ping(struct cli *cli, char **av, void *priv __unused) +m_cli_func_ping(struct cli *cli, char **av, void *priv __unused) { time_t t; @@ -232,28 +233,41 @@ /*--------------------------------------------------------------------*/ +static void +m_cli_func_stats(struct cli *cli, char **av, void *priv __unused) +{ + + assert (VSL_stats != NULL); +#define MAC_STAT(n,t,f,d) \ + cli_out(cli, "%12ju " d "\n", (VSL_stats->n)); +#include "stat_field.h" +#undef MAC_STAT +} + +/*--------------------------------------------------------------------*/ + static struct cli_proto cli_proto[] = { /* URL manipulation */ - { CLI_URL_QUERY, cli_func_passthrough, NULL }, - { CLI_URL_PURGE, cli_func_passthrough, NULL }, - { CLI_URL_STATUS, cli_func_passthrough, NULL }, + { CLI_URL_QUERY, m_cli_func_passthrough, NULL }, + { CLI_URL_PURGE, m_cli_func_passthrough, NULL }, + { CLI_URL_STATUS, m_cli_func_passthrough, NULL }, { CLI_CONFIG_LOAD, m_cli_func_config_load, NULL }, - { CLI_CONFIG_INLINE, cli_func_config_inline, NULL }, - { CLI_CONFIG_UNLOAD, cli_func_passthrough, NULL }, - { CLI_CONFIG_LIST, cli_func_passthrough, NULL }, - { CLI_CONFIG_USE, cli_func_passthrough, NULL }, - { CLI_SERVER_FREEZE, cli_func_passthrough, NULL }, - { CLI_SERVER_THAW, cli_func_passthrough, NULL }, - { CLI_SERVER_SUSPEND, cli_func_passthrough, NULL }, - { CLI_SERVER_RESUME, cli_func_passthrough, NULL }, - { CLI_SERVER_STOP, cli_func_server_stop, NULL }, - { CLI_SERVER_START, cli_func_server_start, NULL }, + { CLI_CONFIG_INLINE, m_cli_func_config_inline, NULL }, + { CLI_CONFIG_UNLOAD, m_cli_func_passthrough, NULL }, + { CLI_CONFIG_LIST, m_cli_func_passthrough, NULL }, + { CLI_CONFIG_USE, m_cli_func_passthrough, NULL }, + { CLI_SERVER_FREEZE, m_cli_func_passthrough, NULL }, + { CLI_SERVER_THAW, m_cli_func_passthrough, NULL }, + { CLI_SERVER_SUSPEND, m_cli_func_passthrough, NULL }, + { CLI_SERVER_RESUME, m_cli_func_passthrough, NULL }, + { CLI_SERVER_STOP, m_cli_func_server_stop, NULL }, + { CLI_SERVER_START, m_cli_func_server_start, NULL }, { CLI_SERVER_RESTART }, - { CLI_PING, cli_func_ping, NULL }, - { CLI_STATS }, + { CLI_PING, m_cli_func_ping, NULL }, + { CLI_STATS, m_cli_func_stats, NULL }, { CLI_ZERO }, { CLI_HELP, cli_func_help, cli_proto }, - { CLI_VERBOSE, cli_func_verbose, NULL }, + { CLI_VERBOSE, m_cli_func_verbose, NULL }, { CLI_EXIT }, { CLI_QUIT }, { CLI_BYE }, @@ -347,39 +361,6 @@ /*--------------------------------------------------------------------*/ -#include "shmlog.h" - -static void -init_vsl(const char *fn, unsigned size) -{ - struct shmloghead slh; - int i; - - heritage.vsl_fd = open(fn, O_RDWR | O_CREAT, 0600); - if (heritage.vsl_fd < 0) { - fprintf(stderr, "Could not open %s: %s\n", - fn, strerror(errno)); - exit (1); - } - i = read(heritage.vsl_fd, &slh, sizeof slh); - if (i == sizeof slh && slh.magic == SHMLOGHEAD_MAGIC) { - /* XXX more checks */ - heritage.vsl_size = slh.size + slh.start; - return; - } - slh.magic = SHMLOGHEAD_MAGIC; - slh.size = size; - slh.ptr = 0; - slh.start = sizeof slh; - AZ(lseek(heritage.vsl_fd, 0, SEEK_SET)); - i = write(heritage.vsl_fd, &slh, sizeof slh); - assert(i == sizeof slh); - AZ(ftruncate(heritage.vsl_fd, sizeof slh + size)); - heritage.vsl_size = slh.size + slh.start; -} - -/*--------------------------------------------------------------------*/ - /* for development purposes */ #include #include @@ -459,7 +440,7 @@ */ open_tcp(portnumber); - init_vsl(SHMLOG_FILENAME, 1024*1024); + VSL_MgtInit(SHMLOG_FILENAME, 1024*1024); testme(); Modified: trunk/varnish-cache/include/shmlog.h =================================================================== --- trunk/varnish-cache/include/shmlog.h 2006-06-26 19:25:09 UTC (rev 242) +++ trunk/varnish-cache/include/shmlog.h 2006-06-28 09:21:15 UTC (rev 243) @@ -9,21 +9,25 @@ #define SHMLOG_FILENAME "/tmp/_.vsl" +#include "stats.h" + struct shmloghead { #define SHMLOGHEAD_MAGIC 4185512498U /* From /dev/random */ - unsigned magic; + unsigned magic; /* * Byte offset into the file where the fifolog starts * This allows the header to expand later. */ - unsigned start; + unsigned start; /* Length of the fifolog area in bytes */ - unsigned size; + unsigned size; /* Current write position relative to the beginning of start */ - unsigned ptr; + unsigned ptr; + + struct varnish_stats stats; }; /* Added: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2006-06-26 19:25:09 UTC (rev 242) +++ trunk/varnish-cache/include/stat_field.h 2006-06-28 09:21:15 UTC (rev 243) @@ -0,0 +1,11 @@ +/* $Id$ */ + +MAC_STAT(cli_conn, uint64_t, "u", "Client connections accepted") +MAC_STAT(cli_req, uint64_t, "u", "Client requests received") + +MAC_STAT(cache_hit, uint64_t, "u", "Cache hits") +MAC_STAT(cache_miss, uint64_t, "u", "Cache misses") + +MAC_STAT(backend_conn, uint64_t, "u", "Backend connections initiated") +MAC_STAT(backend_req, uint64_t, "u", "Backend requests sent") + Added: trunk/varnish-cache/include/stats.h =================================================================== --- trunk/varnish-cache/include/stats.h 2006-06-26 19:25:09 UTC (rev 242) +++ trunk/varnish-cache/include/stats.h 2006-06-28 09:21:15 UTC (rev 243) @@ -0,0 +1,9 @@ +/* $Id$ */ + +#include + +struct varnish_stats { +#define MAC_STAT(n,t,f,e) t n; +#include "stat_field.h" +#undef MAC_STAT +}; From phk at projects.linpro.no Wed Jun 28 09:39:25 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 11:39:25 +0200 (CEST) Subject: r244 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060628093925.563881EC2FA@projects.linpro.no> Author: phk Date: 2006-06-28 11:39:25 +0200 (Wed, 28 Jun 2006) New Revision: 244 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/include/stat_field.h Log: More stats counters Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-28 09:21:15 UTC (rev 243) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-28 09:39:25 UTC (rev 244) @@ -122,7 +122,7 @@ char port[10]; int i; - VSL_stats->cli_conn++; + VSL_stats->client_conn++; (void)arg; sm = calloc(sizeof *sm, 1); Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-06-28 09:21:15 UTC (rev 243) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-06-28 09:39:25 UTC (rev 244) @@ -239,6 +239,7 @@ vp->nconn++; AZ(pthread_mutex_unlock(&vbemtx)); connect_to_backend(vc, bp); + VSL_stats->backend_conn++; event_set(&vc->ev, vc->fd, EV_READ | EV_PERSIST, vbe_rdf, vc); event_base_set(vbe_evb, &vc->ev); } @@ -270,6 +271,7 @@ struct vbe_conn *vc; int i; + VSL_stats->backend_recycle++; vc = ptr; VSL(SLT_BackendReuse, vc->fd, ""); i = write(vbe_pipe[1], &vc, sizeof vc); Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-28 09:21:15 UTC (rev 243) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-28 09:39:25 UTC (rev 244) @@ -31,10 +31,13 @@ o = HSH_Lookup(w, sp->http); sp->obj = o; - if (o->busy) + if (o->busy) { + VSL_stats->cache_miss++; VCL_miss_method(sp); - else + } else { + VSL_stats->cache_hit++; VCL_hit_method(sp); + } return (0); } @@ -133,6 +136,7 @@ vca_return_session(sp); return; } + VSL_stats->client_req++; AZ(pthread_mutex_lock(&sessmtx)); TAILQ_INSERT_TAIL(&shd, sp, list); AZ(pthread_mutex_unlock(&sessmtx)); Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2006-06-28 09:21:15 UTC (rev 243) +++ trunk/varnish-cache/include/stat_field.h 2006-06-28 09:39:25 UTC (rev 244) @@ -1,11 +1,11 @@ /* $Id$ */ -MAC_STAT(cli_conn, uint64_t, "u", "Client connections accepted") -MAC_STAT(cli_req, uint64_t, "u", "Client requests received") +MAC_STAT(client_conn, uint64_t, "u", "Client connections accepted") +MAC_STAT(client_req, uint64_t, "u", "Client requests received") -MAC_STAT(cache_hit, uint64_t, "u", "Cache hits") -MAC_STAT(cache_miss, uint64_t, "u", "Cache misses") +MAC_STAT(cache_hit, uint64_t, "u", "Cache hits") +MAC_STAT(cache_miss, uint64_t, "u", "Cache misses") -MAC_STAT(backend_conn, uint64_t, "u", "Backend connections initiated") -MAC_STAT(backend_req, uint64_t, "u", "Backend requests sent") +MAC_STAT(backend_conn, uint64_t, "u", "Backend connections initiated") +MAC_STAT(backend_recycle, uint64_t, "u", "Backend connections recyles") From phk at projects.linpro.no Wed Jun 28 10:30:57 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 12:30:57 +0200 (CEST) Subject: r245 - trunk/varnish-cache/bin/varnishd Message-ID: <20060628103057.616591EC22E@projects.linpro.no> Author: phk Date: 2006-06-28 12:30:57 +0200 (Wed, 28 Jun 2006) New Revision: 245 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: Implement ->trim() Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-28 09:39:25 UTC (rev 244) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-28 10:30:57 UTC (rev 245) @@ -347,6 +347,29 @@ } /*-------------------------------------------------------------------- + * Trim the tail of a range. + */ + +static void +trim_smf(struct smf *sp, size_t bytes) +{ + struct smf *sp2; + struct smf_sc *sc = sp->sc; + + assert(bytes > 0); + sp2 = malloc(sizeof *sp2); + assert(sp2 != NULL); + + sp2->size = sp->size - bytes; + sp->size = bytes; + sp2->ptr = sp->ptr + bytes; + sp2->offset = sp->offset + bytes; + TAILQ_INSERT_TAIL(&sc->used, sp2, status); + TAILQ_INSERT_AFTER(&sc->order, sp, sp2, status); + free_smf(sp2); +} + +/*-------------------------------------------------------------------- * Insert a newly created range as busy, then free it to do any collapses */ @@ -463,8 +486,15 @@ static void smf_trim(struct storage *s, size_t size) { + struct smf *smf; + struct smf_sc *sc = s->priv; - /* XXX: implement */ + assert(size > 0); + size += (sc->pagesize - 1); + size &= ~(sc->pagesize - 1); + smf = (struct smf *)(s->priv); + if (smf->size > size) + trim_smf(smf, size); } /*--------------------------------------------------------------------*/ From phk at projects.linpro.no Wed Jun 28 10:31:29 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 12:31:29 +0200 (CEST) Subject: r246 - trunk/varnish-cache/bin/varnishd Message-ID: <20060628103129.9F1761EC325@projects.linpro.no> Author: phk Date: 2006-06-28 12:31:29 +0200 (Wed, 28 Jun 2006) New Revision: 246 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c Log: 304's don't have a body Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-28 10:30:57 UTC (rev 245) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-28 10:31:29 UTC (rev 246) @@ -67,16 +67,7 @@ p += i; cl -= i; } - - http_BuildSbuf(2, w->sb, hp); - - vca_write_obj(sp, w->sb); - -#if 0 - hash->deref(sp->obj); -#endif return (0); - } /*--------------------------------------------------------------------*/ @@ -184,10 +175,6 @@ if (st != NULL && stevedore->trim != NULL) stevedore->trim(st, st->len); - - http_BuildSbuf(2, w->sb, hp); - - vca_write_obj(sp, w->sb); return (0); } @@ -237,9 +224,6 @@ if (st != NULL && stevedore->trim != NULL) stevedore->trim(st, st->len); - http_BuildSbuf(2, w->sb, hp); - - vca_write_obj(sp, w->sb); return (1); } @@ -252,6 +236,7 @@ struct http *hp; char *b; time_t t_req, t_resp; + int body; fd = VBE_GetFd(sp->backend, &fd_token); assert(fd != -1); @@ -282,7 +267,16 @@ sp->obj->valid = 1; sp->obj->cacheable = 1; sp->obj->header = strdup(sbuf_data(w->sb)); + body = 1; break; + case 304: + http_BuildSbuf(3, w->sb, hp); + /* XXX: fill in object from headers */ + sp->obj->valid = 1; + sp->obj->cacheable = 1; + sp->obj->header = strdup(sbuf_data(w->sb)); + body = 0; + break; case 391: sp->obj->valid = 0; sp->obj->cacheable = 0; @@ -301,13 +295,20 @@ if (sp->obj->cacheable) EXP_Insert(sp->obj); - if (http_GetHdr(hp, "Content-Length", &b)) - cls = fetch_straight(w, sp, fd, hp, b); - else if (http_HdrIs(hp, "Transfer-Encoding", "chunked")) - cls = fetch_chunked(w, sp, fd, hp); - else - cls = fetch_eof(w, sp, fd, hp); + if (body) { + if (http_GetHdr(hp, "Content-Length", &b)) + cls = fetch_straight(w, sp, fd, hp, b); + else if (http_HdrIs(hp, "Transfer-Encoding", "chunked")) + cls = fetch_chunked(w, sp, fd, hp); + else + cls = fetch_eof(w, sp, fd, hp); + } else + cls = 0; + http_BuildSbuf(2, w->sb, hp); + + vca_write_obj(sp, w->sb); + if (http_GetHdr(hp, "Connection", &b) && !strcasecmp(b, "close")) cls = 1; From phk at projects.linpro.no Wed Jun 28 11:21:06 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 13:21:06 +0200 (CEST) Subject: r247 - in trunk/varnish-cache: . bin bin/varnishstat Message-ID: <20060628112106.3F3651EC233@projects.linpro.no> Author: phk Date: 2006-06-28 13:21:06 +0200 (Wed, 28 Jun 2006) New Revision: 247 Added: trunk/varnish-cache/bin/varnishstat/ trunk/varnish-cache/bin/varnishstat/Makefile.am trunk/varnish-cache/bin/varnishstat/varnishstat.c Modified: trunk/varnish-cache/bin/Makefile.am trunk/varnish-cache/configure.ac Log: Add varnishstat program Modified: trunk/varnish-cache/bin/Makefile.am =================================================================== --- trunk/varnish-cache/bin/Makefile.am 2006-06-28 10:31:29 UTC (rev 246) +++ trunk/varnish-cache/bin/Makefile.am 2006-06-28 11:21:06 UTC (rev 247) @@ -1,3 +1,3 @@ # $Id$ -SUBDIRS = varnishd varnishlog +SUBDIRS = varnishd varnishlog varnishstat Added: trunk/varnish-cache/bin/varnishstat/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishstat/Makefile.am 2006-06-28 10:31:29 UTC (rev 246) +++ trunk/varnish-cache/bin/varnishstat/Makefile.am 2006-06-28 11:21:06 UTC (rev 247) @@ -0,0 +1,9 @@ +# $Id: Makefile.am 133 2006-04-06 09:38:00Z phk $ + +INCLUDES = -I$(top_srcdir)/include + +bin_PROGRAMS = varnishstat + +varnishlog_SOURCES = varnishstat.c + +# varnishlog_LDADD = $(top_builddir)/lib/libvarnishapi/libvarnishapi.la Added: trunk/varnish-cache/bin/varnishstat/varnishstat.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-06-28 10:31:29 UTC (rev 246) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-06-28 11:21:06 UTC (rev 247) @@ -0,0 +1,62 @@ +/* + * $Id: varnishlog.c 153 2006-04-25 08:17:43Z phk $ + * + * Log tailer for Varnish + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +static struct shmloghead *loghead; + +int +main(int argc, char **argv) +{ + int fd; + int i; + struct shmloghead slh; + struct varnish_stats *VSL_stats; + + fd = open(SHMLOG_FILENAME, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "Cannot open %s: %s\n", + SHMLOG_FILENAME, strerror(errno)); + exit (1); + } + i = read(fd, &slh, sizeof slh); + if (i != sizeof slh) { + fprintf(stderr, "Cannot read %s: %s\n", + SHMLOG_FILENAME, strerror(errno)); + exit (1); + } + if (slh.magic != SHMLOGHEAD_MAGIC) { + fprintf(stderr, "Wrong magic number in file %s\n", + SHMLOG_FILENAME); + exit (1); + } + + loghead = mmap(NULL, slh.size + sizeof slh, + PROT_READ, MAP_HASSEMAPHORE, fd, 0); + if (loghead == MAP_FAILED) { + fprintf(stderr, "Cannot mmap %s: %s\n", + SHMLOG_FILENAME, strerror(errno)); + exit (1); + } + + VSL_stats = &loghead->stats; + +#define MAC_STAT(n,t,f,d) \ + printf("%12ju " d "\n", (VSL_stats->n)); +#include "stat_field.h" +#undef MAC_STAT + + exit (0); + +} Modified: trunk/varnish-cache/configure.ac =================================================================== --- trunk/varnish-cache/configure.ac 2006-06-28 10:31:29 UTC (rev 246) +++ trunk/varnish-cache/configure.ac 2006-06-28 11:21:06 UTC (rev 247) @@ -68,6 +68,7 @@ bin/Makefile bin/varnishd/Makefile bin/varnishlog/Makefile + bin/varnishstat/Makefile contrib/Makefile include/Makefile lib/Makefile From phk at projects.linpro.no Wed Jun 28 11:29:36 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 13:29:36 +0200 (CEST) Subject: r248 - trunk/varnish-cache/bin/varnishstat Message-ID: <20060628112936.39FE01EC325@projects.linpro.no> Author: phk Date: 2006-06-28 13:29:36 +0200 (Wed, 28 Jun 2006) New Revision: 248 Modified: trunk/varnish-cache/bin/varnishstat/Makefile.am trunk/varnish-cache/bin/varnishstat/varnishstat.c Log: Give varnishstat a "-c" option to use curses to continously refresh Modified: trunk/varnish-cache/bin/varnishstat/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishstat/Makefile.am 2006-06-28 11:21:06 UTC (rev 247) +++ trunk/varnish-cache/bin/varnishstat/Makefile.am 2006-06-28 11:29:36 UTC (rev 248) @@ -4,6 +4,7 @@ bin_PROGRAMS = varnishstat -varnishlog_SOURCES = varnishstat.c +varnishstat_SOURCES = varnishstat.c +varnishstat_LDADD = -lcurses # varnishlog_LDADD = $(top_builddir)/lib/libvarnishapi/libvarnishapi.la Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-06-28 11:21:06 UTC (rev 247) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-06-28 11:29:36 UTC (rev 248) @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -20,9 +21,10 @@ main(int argc, char **argv) { int fd; - int i; + int i, c; struct shmloghead slh; struct varnish_stats *VSL_stats; + int c_flag = 0; fd = open(SHMLOG_FILENAME, O_RDONLY); if (fd < 0) { @@ -49,14 +51,41 @@ SHMLOG_FILENAME, strerror(errno)); exit (1); } - + VSL_stats = &loghead->stats; + while ((c = getopt(argc, argv, "c")) != -1) { + switch (c) { + case 'c': + c_flag = 1; + break; + default: + fprintf(stderr, "Usage: varnishstat [-c]\n"); + exit (2); + } + } + + if (c_flag) { + initscr(); + erase(); + + while (1) { + move(0,0); #define MAC_STAT(n,t,f,d) \ - printf("%12ju " d "\n", (VSL_stats->n)); + printw("%12ju " d "\n", (VSL_stats->n)); #include "stat_field.h" #undef MAC_STAT + refresh(); + sleep(1); + } + } else { +#define MAC_STAT(n,t,f,d) \ + printf("%12ju " d "\n", (VSL_stats->n)); +#include "stat_field.h" +#undef MAC_STAT + } + exit (0); } From phk at projects.linpro.no Wed Jun 28 16:04:33 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 18:04:33 +0200 (CEST) Subject: r249 - trunk/varnish-cache/bin/varnishd Message-ID: <20060628160433.8C4371EC325@projects.linpro.no> Author: phk Date: 2006-06-28 18:04:33 +0200 (Wed, 28 Jun 2006) New Revision: 249 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: Fix buglets, include test-driver. Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-28 11:29:36 UTC (rev 248) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-28 16:04:33 UTC (rev 249) @@ -338,7 +338,7 @@ TAILQ_FOREACH(sp2, &sc->free, status) { if (sp->age > sp2->age || (sp->age == sp2->age && sp->offset < sp2->offset)) { - TAILQ_INSERT_BEFORE(sp2, sp, order); + TAILQ_INSERT_BEFORE(sp2, sp, status); break; } } @@ -359,13 +359,14 @@ assert(bytes > 0); sp2 = malloc(sizeof *sp2); assert(sp2 != NULL); + *sp2 = *sp; - sp2->size = sp->size - bytes; + sp2->size -= bytes; sp->size = bytes; - sp2->ptr = sp->ptr + bytes; - sp2->offset = sp->offset + bytes; + sp2->ptr += bytes; + sp2->offset += bytes; TAILQ_INSERT_TAIL(&sc->used, sp2, status); - TAILQ_INSERT_AFTER(&sc->order, sp, sp2, status); + TAILQ_INSERT_AFTER(&sc->order, sp, sp2, order); free_smf(sp2); } @@ -487,14 +488,19 @@ smf_trim(struct storage *s, size_t size) { struct smf *smf; - struct smf_sc *sc = s->priv; + struct smf_sc *sc; + assert(size <= s->space); assert(size > 0); + smf = (struct smf *)(s->priv); + assert(size <= smf->size); + sc = smf->sc; size += (sc->pagesize - 1); size &= ~(sc->pagesize - 1); - smf = (struct smf *)(s->priv); - if (smf->size > size) + if (smf->size > size) { trim_smf(smf, size); + smf->s.space = size; + } } /*--------------------------------------------------------------------*/ @@ -526,9 +532,9 @@ st->len, NULL, &sent, 0); if (sent == st->len) return; - printf("sent i=%d sent=%ju size=%ju\n", - i, (uintmax_t)sent, (uintmax_t)st->len); - assert(sent == st->len); + printf("sent i=%d sent=%ju size=%ju errno=%d\n", + i, (uintmax_t)sent, (uintmax_t)st->len, errno); + vca_close_session(sp, "remote closed"); } /*--------------------------------------------------------------------*/ @@ -542,3 +548,69 @@ smf_free, smf_send }; + +#ifdef INCLUDE_TEST_DRIVER + +void vca_flush(struct sess *sp) {} +void vca_close_session(struct sess *sp, const char *why) {} + +#define N 100 +#define M (128*1024) + +struct storage *s[N]; + +static void +dumpit(void) +{ + struct smf_sc *sc = smf_stevedore.priv; + struct smf *s; + + return (0); + printf("----------------\n"); + printf("Order:\n"); + TAILQ_FOREACH(s, &sc->order, order) { + printf("%10p %12ju %12ju %12ju\n", + s, s->offset, s->size, s->offset + s->size); + } + printf("Used:\n"); + TAILQ_FOREACH(s, &sc->used, status) { + printf("%10p %12ju %12ju %12ju\n", + s, s->offset, s->size, s->offset + s->size); + } + printf("Free:\n"); + TAILQ_FOREACH(s, &sc->free, status) { + printf("%10p %12ju %12ju %12ju\n", + s, s->offset, s->size, s->offset + s->size); + } + printf("================\n"); +} + +int +main(int argc, char **argv) +{ + int i, j; + + setbuf(stdout, NULL); + smf_init(&smf_stevedore, ""); + smf_open(&smf_stevedore); + while (1) { + dumpit(); + i = random() % N; + do + j = random() % M; + while (j == 0); + if (s[i] == NULL) { + s[i] = smf_alloc(&smf_stevedore, j); + printf("A %10p %12d\n", s[i], j); + } else if (j < s[i]->space) { + smf_trim(s[i], j); + printf("T %10p %12d\n", s[i], j); + } else { + smf_free(s[i]); + printf("D %10p\n", s[i]); + s[i] = NULL; + } + } +} + +#endif /* INCLUDE_TEST_DRIVER */ From phk at projects.linpro.no Wed Jun 28 16:14:19 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 18:14:19 +0200 (CEST) Subject: r250 - trunk/varnish-cache/bin/varnishd Message-ID: <20060628161419.B33F41EC325@projects.linpro.no> Author: phk Date: 2006-06-28 18:14:19 +0200 (Wed, 28 Jun 2006) New Revision: 250 Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c Log: Apply correct fd in Shmemlog Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-28 16:04:33 UTC (rev 249) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-28 16:14:19 UTC (rev 250) @@ -36,7 +36,7 @@ VRT_count(struct sess *sp, unsigned u) { - VSL(SLT_VCL, 0, "%u %d.%d", u, + VSL(SLT_VCL, sp->fd, "%u %d.%d", u, sp->vcl->ref[u].line, sp->vcl->ref[u].pos); } From phk at projects.linpro.no Wed Jun 28 16:19:08 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 18:19:08 +0200 (CEST) Subject: r251 - trunk/varnish-cache/include Message-ID: <20060628161908.403EC1EC325@projects.linpro.no> Author: phk Date: 2006-06-28 18:19:08 +0200 (Wed, 28 Jun 2006) New Revision: 251 Modified: trunk/varnish-cache/include/http_headers.h Log: Don't pass If-* headers to backend Modified: trunk/varnish-cache/include/http_headers.h =================================================================== --- trunk/varnish-cache/include/http_headers.h 2006-06-28 16:14:19 UTC (rev 250) +++ trunk/varnish-cache/include/http_headers.h 2006-06-28 16:19:08 UTC (rev 251) @@ -38,11 +38,11 @@ HTTPH("Expires", H_Expires, 2, 0, 0, 0, 0) /* RFC2616 14.21 */ HTTPH("From", H_From, 1, 0, 0, 0, 0) /* RFC2616 14.22 */ HTTPH("Host", H_Host, 1, 0, 0, 0, 0) /* RFC2616 14.23 */ -HTTPH("If-Match", H_If_Match, 1, 0, 0, 0, 0) /* RFC2616 14.24 */ -HTTPH("If-Modified-Since", H_If_Modified_Since, 1, 0, 0, 0, 0) /* RFC2616 14.25 */ -HTTPH("If-None-Match", H_If_None_Match, 1, 0, 0, 0, 0) /* RFC2616 14.26 */ -HTTPH("If-Range", H_If_Range, 1, 0, 0, 0, 0) /* RFC2616 14.27 */ -HTTPH("If-Unmodified-Since", H_If_Unmodifed_Since, 1, 0, 0, 0, 0) /* RFC2616 14.28 */ +HTTPH("If-Match", H_If_Match, 1, 1, 0, 0, 0) /* RFC2616 14.24 */ +HTTPH("If-Modified-Since", H_If_Modified_Since, 1, 1, 0, 0, 0) /* RFC2616 14.25 */ +HTTPH("If-None-Match", H_If_None_Match, 1, 1, 0, 0, 0) /* RFC2616 14.26 */ +HTTPH("If-Range", H_If_Range, 1, 1, 0, 0, 0) /* RFC2616 14.27 */ +HTTPH("If-Unmodified-Since", H_If_Unmodifed_Since, 1, 1, 0, 0, 0) /* RFC2616 14.28 */ HTTPH("Last-Modified", H_Last_Modified, 2, 0, 0, 0, 0) /* RFC2616 14.29 */ HTTPH("Location", H_Location, 2, 0, 0, 0, 0) /* RFC2616 14.30 */ HTTPH("Max-Forwards", H_Max_Forwards, 1, 0, 0, 0, 0) /* RFC2616 14.31 */ From phk at projects.linpro.no Wed Jun 28 16:19:21 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 18:19:21 +0200 (CEST) Subject: r252 - trunk/varnish-cache/include Message-ID: <20060628161921.419461EC325@projects.linpro.no> Author: phk Date: 2006-06-28 18:19:21 +0200 (Wed, 28 Jun 2006) New Revision: 252 Modified: trunk/varnish-cache/include/shmlog_tags.h Log: add tag for generated headers Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-06-28 16:19:08 UTC (rev 251) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-06-28 16:19:21 UTC (rev 252) @@ -29,5 +29,6 @@ SLTM(URL) SLTM(Protocol) SLTM(Header) +SLTM(BldHdr) SLTM(LostHeader) SLTM(VCL) From phk at projects.linpro.no Wed Jun 28 16:20:14 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 18:20:14 +0200 (CEST) Subject: r253 - trunk/varnish-cache/bin/varnishd Message-ID: <20060628162014.5B2201EC233@projects.linpro.no> Author: phk Date: 2006-06-28 18:20:14 +0200 (Wed, 28 Jun 2006) New Revision: 253 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pipe.c Log: Pass fd to shmemlog Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-28 16:19:21 UTC (rev 252) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-28 16:20:14 UTC (rev 253) @@ -174,7 +174,7 @@ int http_GetURL(struct http *hp, char **b); void http_RecvHead(struct http *hp, int fd, struct event_base *eb, http_callback_f *func, void *arg); void http_Dissect(struct http *sp, int fd, int rr); -void http_BuildSbuf(int resp, struct sbuf *sb, struct http *hp); +void http_BuildSbuf(int fd, int resp, struct sbuf *sb, struct http *hp); /* cache_main.c */ extern pthread_mutex_t sessmtx; Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-28 16:19:21 UTC (rev 252) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-28 16:20:14 UTC (rev 253) @@ -243,7 +243,7 @@ VSL(SLT_Backend, sp->fd, "%d %s", fd, sp->backend->vcl_name); hp = http_New(); - http_BuildSbuf(1, w->sb, sp->http); + http_BuildSbuf(fd, 1, w->sb, sp->http); i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); time(&t_req); @@ -262,7 +262,7 @@ switch (http_GetStatus(hp)) { case 200: case 301: - http_BuildSbuf(3, w->sb, hp); + http_BuildSbuf(sp->fd, 3, w->sb, hp); /* XXX: fill in object from headers */ sp->obj->valid = 1; sp->obj->cacheable = 1; @@ -270,7 +270,7 @@ body = 1; break; case 304: - http_BuildSbuf(3, w->sb, hp); + http_BuildSbuf(sp->fd, 3, w->sb, hp); /* XXX: fill in object from headers */ sp->obj->valid = 1; sp->obj->cacheable = 1; @@ -305,7 +305,7 @@ } else cls = 0; - http_BuildSbuf(2, w->sb, hp); + http_BuildSbuf(sp->fd, 2, w->sb, hp); vca_write_obj(sp, w->sb); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-28 16:19:21 UTC (rev 252) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-28 16:20:14 UTC (rev 253) @@ -391,7 +391,7 @@ /*--------------------------------------------------------------------*/ void -http_BuildSbuf(int resp, struct sbuf *sb, struct http *hp) +http_BuildSbuf(int fd, int resp, struct sbuf *sb, struct http *hp) { unsigned u; @@ -419,7 +419,7 @@ if (http_supress(hp->hdr[u], resp)) continue; if (1) - VSL(SLT_Debug, 0, "Build %s", hp->hdr[u]); + VSL(SLT_BldHdr, fd, "%s", hp->hdr[u]); sbuf_cat(sb, hp->hdr[u]); sbuf_cat(sb, "\r\n"); } Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-06-28 16:19:21 UTC (rev 252) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-06-28 16:20:14 UTC (rev 253) @@ -160,7 +160,7 @@ fd = VBE_GetFd(sp->backend, &fd_token); assert(fd != -1); - http_BuildSbuf(1, w->sb, sp->http); + http_BuildSbuf(fd, 1, w->sb, sp->http); i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); @@ -175,7 +175,7 @@ event_base_loop(w->eb, 0); http_Dissect(hp, fd, 2); - http_BuildSbuf(2, w->sb, hp); + http_BuildSbuf(sp->fd, 2, w->sb, hp); vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb)); if (http_GetHdr(hp, "Content-Length", &b)) Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-06-28 16:19:21 UTC (rev 252) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-06-28 16:20:14 UTC (rev 253) @@ -51,7 +51,7 @@ fd = VBE_GetFd(sp->backend, &fd_token); assert(fd != -1); - http_BuildSbuf(0, w->sb, sp->http); /* XXX: 0 ?? */ + http_BuildSbuf(fd, 0, w->sb, sp->http); /* XXX: 0 ?? */ i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); assert(__LINE__ == 0); From phk at projects.linpro.no Wed Jun 28 16:57:07 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 18:57:07 +0200 (CEST) Subject: r254 - trunk/varnish-cache/include Message-ID: <20060628165707.C49E21EC22E@projects.linpro.no> Author: phk Date: 2006-06-28 18:57:07 +0200 (Wed, 28 Jun 2006) New Revision: 254 Modified: trunk/varnish-cache/include/http_headers.h Log: Don't pass Content-Lenght through, we build it ourselves Modified: trunk/varnish-cache/include/http_headers.h =================================================================== --- trunk/varnish-cache/include/http_headers.h 2006-06-28 16:20:14 UTC (rev 253) +++ trunk/varnish-cache/include/http_headers.h 2006-06-28 16:57:07 UTC (rev 254) @@ -27,7 +27,7 @@ HTTPH("Connection", H_Connection, 3, 3, 0, 0, 0) /* RFC2616 14.10 */ HTTPH("Content-Encoding", H_Content_Encoding, 2, 0, 0, 0, 0) /* RFC2616 14.11 */ HTTPH("Content-Langugae", H_Content_Language, 2, 0, 0, 0, 0) /* RFC2616 14.12 */ -HTTPH("Content-Length", H_Content_Length, 2, 0, 0, 0, 0) /* RFC2616 14.13 */ +HTTPH("Content-Length", H_Content_Length, 2, 2, 0, 0, 0) /* RFC2616 14.13 */ HTTPH("Content-Location", H_Content_Location, 2, 0, 0, 0, 0) /* RFC2616 14.14 */ HTTPH("Content-MD5", H_Content_MD5, 2, 0, 0, 0, 0) /* RFC2616 14.15 */ HTTPH("Content-Range", H_Content_Range, 2, 0, 0, 0, 0) /* RFC2616 14.16 */ From phk at projects.linpro.no Wed Jun 28 16:57:50 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 18:57:50 +0200 (CEST) Subject: r255 - trunk/varnish-cache/bin/varnishd Message-ID: <20060628165750.EC6C51EC233@projects.linpro.no> Author: phk Date: 2006-06-28 18:57:50 +0200 (Wed, 28 Jun 2006) New Revision: 255 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c Log: Assert that the lengths of the storage for the object add up. Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-28 16:57:07 UTC (rev 254) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-28 16:57:50 UTC (rev 255) @@ -88,14 +88,17 @@ vca_write_obj(struct sess *sp, struct sbuf *hdr) { struct storage *st; + unsigned u = 0; vca_write(sp, sbuf_data(hdr), sbuf_len(hdr)); TAILQ_FOREACH(st, &sp->obj->store, list) { + u += st->len; if (st->stevedore->send != NULL) st->stevedore->send(st, sp); else vca_write(sp, st->ptr, st->len); } + assert(u == sp->obj->len); vca_flush(sp); } From phk at projects.linpro.no Wed Jun 28 16:58:21 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 18:58:21 +0200 (CEST) Subject: r256 - trunk/varnish-cache/bin/varnishd Message-ID: <20060628165821.D8D1E1EC325@projects.linpro.no> Author: phk Date: 2006-06-28 18:58:21 +0200 (Wed, 28 Jun 2006) New Revision: 256 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Don't finish the sbuf in mode 3 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-28 16:57:50 UTC (rev 255) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-28 16:58:21 UTC (rev 256) @@ -352,7 +352,6 @@ { assert(hp != NULL); - VSL(SLT_Debug, fd, "%s s %p t %p v %p", __func__, hp->s, hp->t, hp->v); assert(hp->t == hp->s || hp->t == hp->v); /* XXX pipelining */ hp->callback = func; hp->arg = arg; @@ -423,7 +422,8 @@ sbuf_cat(sb, hp->hdr[u]); sbuf_cat(sb, "\r\n"); } - if (resp != 3) + if (resp != 3) { sbuf_cat(sb, "\r\n"); - sbuf_finish(sb); + sbuf_finish(sb); + } } From phk at projects.linpro.no Wed Jun 28 16:59:07 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 18:59:07 +0200 (CEST) Subject: r257 - trunk/varnish-cache/bin/varnishd Message-ID: <20060628165907.994791EC233@projects.linpro.no> Author: phk Date: 2006-06-28 18:59:07 +0200 (Wed, 28 Jun 2006) New Revision: 257 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c Log: Construct our own Content-length header, no matter which of the three (straight, chunked, eof) modes we used to fetch the object. Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-28 16:58:21 UTC (rev 256) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-28 16:59:07 UTC (rev 257) @@ -161,6 +161,7 @@ st->len += e - b; v -= e - b; u -= e - b; + sp->obj->len += e - b; } while (v > 0) { i = read(fd, p, v); @@ -169,6 +170,7 @@ v -= i; u -= i; p += i; + sp->obj->len += i; } } } @@ -210,6 +212,7 @@ p += e - b; v -= e - b; st->len += e - b; + sp->obj->len += e - b; *p = '\0'; } i = read(fd, p, v); @@ -219,6 +222,7 @@ p += i; v -= i; st->len += i; + sp->obj->len += i; } if (st != NULL && stevedore->trim != NULL) @@ -262,25 +266,17 @@ switch (http_GetStatus(hp)) { case 200: case 301: - http_BuildSbuf(sp->fd, 3, w->sb, hp); /* XXX: fill in object from headers */ sp->obj->valid = 1; sp->obj->cacheable = 1; - sp->obj->header = strdup(sbuf_data(w->sb)); body = 1; break; case 304: - http_BuildSbuf(sp->fd, 3, w->sb, hp); /* XXX: fill in object from headers */ sp->obj->valid = 1; sp->obj->cacheable = 1; - sp->obj->header = strdup(sbuf_data(w->sb)); body = 0; break; - case 391: - sp->obj->valid = 0; - sp->obj->cacheable = 0; - break; default: break; } @@ -295,6 +291,7 @@ if (sp->obj->cacheable) EXP_Insert(sp->obj); + http_BuildSbuf(sp->fd, 3, w->sb, hp); if (body) { if (http_GetHdr(hp, "Content-Length", &b)) cls = fetch_straight(w, sp, fd, hp, b); @@ -302,11 +299,13 @@ cls = fetch_chunked(w, sp, fd, hp); else cls = fetch_eof(w, sp, fd, hp); + sbuf_printf(w->sb, "Content-Length: %u\r\n", sp->obj->len); } else cls = 0; + sbuf_cat(w->sb, "\r\n"); + sbuf_finish(w->sb); + sp->obj->header = strdup(sbuf_data(w->sb)); - http_BuildSbuf(sp->fd, 2, w->sb, hp); - vca_write_obj(sp, w->sb); if (http_GetHdr(hp, "Connection", &b) && !strcasecmp(b, "close")) From phk at projects.linpro.no Wed Jun 28 17:46:04 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 19:46:04 +0200 (CEST) Subject: r258 - trunk/varnish-cache/bin/varnishd Message-ID: <20060628174604.5FBA11EC325@projects.linpro.no> Author: phk Date: 2006-06-28 19:46:04 +0200 (Wed, 28 Jun 2006) New Revision: 258 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_pool.c Log: Be more consistent about which headers we send back. Start 5 threads in the worker pool. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-28 16:59:07 UTC (rev 257) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-28 17:46:04 UTC (rev 258) @@ -135,7 +135,7 @@ /* cache_acceptor.c */ void vca_write(struct sess *sp, void *ptr, size_t len); -void vca_write_obj(struct sess *sp, struct sbuf *hdr); +void vca_write_obj(struct sess *sp, const char *b, unsigned l); void vca_flush(struct sess *sp); void vca_return_session(struct sess *sp); void vca_close_session(struct sess *sp, const char *why); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-28 16:59:07 UTC (rev 257) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-28 17:46:04 UTC (rev 258) @@ -85,12 +85,14 @@ } void -vca_write_obj(struct sess *sp, struct sbuf *hdr) +vca_write_obj(struct sess *sp, const char *b, unsigned l) { struct storage *st; unsigned u = 0; - vca_write(sp, sbuf_data(hdr), sbuf_len(hdr)); + if (l == 0) + l = strlen(b); + vca_write(sp, b, l); TAILQ_FOREACH(st, &sp->obj->store, list) { u += st->len; if (st->stevedore->send != NULL) Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-28 16:59:07 UTC (rev 257) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-28 17:46:04 UTC (rev 258) @@ -306,7 +306,7 @@ sbuf_finish(w->sb); sp->obj->header = strdup(sbuf_data(w->sb)); - vca_write_obj(sp, w->sb); + vca_write_obj(sp, sp->obj->header, 0); if (http_GetHdr(hp, "Connection", &b) && !strcasecmp(b, "close")) cls = 1; Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-28 16:59:07 UTC (rev 257) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-28 17:46:04 UTC (rev 258) @@ -45,13 +45,8 @@ DeliverSession(struct worker *w, struct sess *sp) { - sbuf_clear(w->sb); - sbuf_printf(w->sb, - "%sServer: Varnish\r\n" - "Content-Length: %u\r\n" - "\r\n", sp->obj->header, sp->obj->len); - vca_write_obj(sp, w->sb); + vca_write_obj(sp, sp->obj->header, 0); HSH_Deref(sp->obj); sp->obj = NULL; return (1); @@ -147,9 +142,11 @@ CacheInitPool(void) { pthread_t tp; + int i; AZ(pthread_cond_init(&shdcnd, NULL)); - AZ(pthread_create(&tp, NULL, CacheWorker, NULL)); + for (i = 0; i < 5; i++) + AZ(pthread_create(&tp, NULL, CacheWorker, NULL)); AZ(pthread_detach(tp)); } From phk at projects.linpro.no Wed Jun 28 20:58:04 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 22:58:04 +0200 (CEST) Subject: r259 - in trunk/varnish-cache: include lib/libvarnishapi Message-ID: <20060628205804.18C1D1EC325@projects.linpro.no> Author: phk Date: 2006-06-28 22:58:03 +0200 (Wed, 28 Jun 2006) New Revision: 259 Modified: trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/lib/libvarnishapi/Makefile.am Log: Add SHMLOG opening and walking functions to libvarnishapi Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2006-06-28 17:46:04 UTC (rev 258) +++ trunk/varnish-cache/include/varnishapi.h 2006-06-28 20:58:03 UTC (rev 259) @@ -7,6 +7,11 @@ #define V_DEAD __attribute__ ((noreturn)) +/* shmlog.c */ +struct shmloghead *VSL_OpenLog(void); +unsigned char *VSL_NextLog(struct shmloghead *lh, unsigned char **pp); + + /* varnish_debug.c */ void vdb_panic(const char *, ...) V_DEAD; Modified: trunk/varnish-cache/lib/libvarnishapi/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/Makefile.am 2006-06-28 17:46:04 UTC (rev 258) +++ trunk/varnish-cache/lib/libvarnishapi/Makefile.am 2006-06-28 20:58:03 UTC (rev 259) @@ -5,6 +5,9 @@ lib_LTLIBRARIES = libvarnishapi.la libvarnishapi_la_SOURCES = \ - varnish_debug.c \ - varnish_log.c \ - varnish_util.c + shmlog.c + + +# varnish_debug.c +# varnish_log.c +# varnish_util.c From phk at projects.linpro.no Wed Jun 28 20:58:36 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 22:58:36 +0200 (CEST) Subject: r260 - trunk/varnish-cache/bin/varnishlog Message-ID: <20060628205836.E38FE1EC325@projects.linpro.no> Author: phk Date: 2006-06-28 22:58:36 +0200 (Wed, 28 Jun 2006) New Revision: 260 Modified: trunk/varnish-cache/bin/varnishlog/Makefile.am trunk/varnish-cache/bin/varnishlog/varnishlog.c Log: Use SHMLOG api in libvarnishapi Modified: trunk/varnish-cache/bin/varnishlog/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishlog/Makefile.am 2006-06-28 20:58:03 UTC (rev 259) +++ trunk/varnish-cache/bin/varnishlog/Makefile.am 2006-06-28 20:58:36 UTC (rev 260) @@ -6,4 +6,4 @@ varnishlog_SOURCES = varnishlog.c -# varnishlog_LDADD = $(top_builddir)/lib/libvarnishapi/libvarnishapi.la +varnishlog_LDADD = $(top_builddir)/lib/libvarnishapi/libvarnishapi.la Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-06-28 20:58:03 UTC (rev 259) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-06-28 20:58:36 UTC (rev 260) @@ -9,10 +9,9 @@ #include #include #include -#include -#include -#include +#include "shmlog.h" +#include "varnishapi.h" /* * It would be simpler to use sparse array initialization and put it @@ -32,69 +31,34 @@ static const char *tagnames[256]; static struct shmloghead *loghead; -static unsigned char *logstart, *logend; int main(int argc, char **argv) { - int fd; int i; unsigned u; - unsigned startup; - struct shmloghead slh; - unsigned char *p; + unsigned char *p, *q; - fd = open(SHMLOG_FILENAME, O_RDONLY); - if (fd < 0) { - fprintf(stderr, "Cannot open %s: %s\n", - SHMLOG_FILENAME, strerror(errno)); - exit (1); - } - i = read(fd, &slh, sizeof slh); - if (i != sizeof slh) { - fprintf(stderr, "Cannot read %s: %s\n", - SHMLOG_FILENAME, strerror(errno)); - exit (1); - } - if (slh.magic != SHMLOGHEAD_MAGIC) { - fprintf(stderr, "Wrong magic number in file %s\n", - SHMLOG_FILENAME); - exit (1); - } - - loghead = mmap(NULL, slh.size + sizeof slh, - PROT_READ, MAP_HASSEMAPHORE, fd, 0); - if (loghead == MAP_FAILED) { - fprintf(stderr, "Cannot mmap %s: %s\n", - SHMLOG_FILENAME, strerror(errno)); - exit (1); - } - logstart = (unsigned char *)loghead + loghead->start; - logend = logstart + loghead->size; - + loghead = VSL_OpenLog(); + for (i = 0; stagnames[i].tag != SLT_ENDMARKER; i++) tagnames[stagnames[i].tag] = stagnames[i].name; - startup = 1; + q = NULL; + while (VSL_NextLog(loghead, &q) != NULL) + ; while (1) { - p = logstart; - while (1) { - if (*p == SLT_WRAPMARKER) - break; - while (*p == SLT_ENDMARKER) { - fflush(stdout); - sleep(1); - startup = 0; - } - u = (p[2] << 8) | p[3]; - if (!startup) { - printf("%02x %3d %4d %-12s <", - p[0], p[1], u, tagnames[p[0]]); - if (p[1] > 0) - fwrite(p + 4, p[1], 1, stdout); - printf(">\n"); - } - p += p[1] + 4; + p = VSL_NextLog(loghead, &q); + if (p == NULL) { + fflush(stdout); + sleep(1); + continue; } + u = (p[2] << 8) | p[3]; + printf("%02x %3d %4d %-12s <", + p[0], p[1], u, tagnames[p[0]]); + if (p[1] > 0) + fwrite(p + 4, p[1], 1, stdout); + printf(">\n"); } } From phk at projects.linpro.no Wed Jun 28 21:03:47 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 23:03:47 +0200 (CEST) Subject: r261 - trunk/varnish-cache/bin/varnishstat Message-ID: <20060628210347.560AF1EC233@projects.linpro.no> Author: phk Date: 2006-06-28 23:03:47 +0200 (Wed, 28 Jun 2006) New Revision: 261 Modified: trunk/varnish-cache/bin/varnishstat/Makefile.am trunk/varnish-cache/bin/varnishstat/varnishstat.c Log: Use shmlog api from libvarnishapi Modified: trunk/varnish-cache/bin/varnishstat/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishstat/Makefile.am 2006-06-28 20:58:36 UTC (rev 260) +++ trunk/varnish-cache/bin/varnishstat/Makefile.am 2006-06-28 21:03:47 UTC (rev 261) @@ -6,5 +6,6 @@ varnishstat_SOURCES = varnishstat.c -varnishstat_LDADD = -lcurses -# varnishlog_LDADD = $(top_builddir)/lib/libvarnishapi/libvarnishapi.la +varnishstat_LDADD = \ + -lcurses \ + $(top_builddir)/lib/libvarnishapi/libvarnishapi.la Modified: trunk/varnish-cache/bin/varnishstat/varnishstat.c =================================================================== --- trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-06-28 20:58:36 UTC (rev 260) +++ trunk/varnish-cache/bin/varnishstat/varnishstat.c 2006-06-28 21:03:47 UTC (rev 261) @@ -9,51 +9,23 @@ #include #include #include -#include #include -#include -#include +#include "shmlog.h" +#include "varnishapi.h" -static struct shmloghead *loghead; - int main(int argc, char **argv) { - int fd; - int i, c; - struct shmloghead slh; + int c; + struct shmloghead *lh; struct varnish_stats *VSL_stats; int c_flag = 0; - fd = open(SHMLOG_FILENAME, O_RDONLY); - if (fd < 0) { - fprintf(stderr, "Cannot open %s: %s\n", - SHMLOG_FILENAME, strerror(errno)); - exit (1); - } - i = read(fd, &slh, sizeof slh); - if (i != sizeof slh) { - fprintf(stderr, "Cannot read %s: %s\n", - SHMLOG_FILENAME, strerror(errno)); - exit (1); - } - if (slh.magic != SHMLOGHEAD_MAGIC) { - fprintf(stderr, "Wrong magic number in file %s\n", - SHMLOG_FILENAME); - exit (1); - } + lh = VSL_OpenLog(); - loghead = mmap(NULL, slh.size + sizeof slh, - PROT_READ, MAP_HASSEMAPHORE, fd, 0); - if (loghead == MAP_FAILED) { - fprintf(stderr, "Cannot mmap %s: %s\n", - SHMLOG_FILENAME, strerror(errno)); - exit (1); - } + VSL_stats = &lh->stats; - VSL_stats = &loghead->stats; - while ((c = getopt(argc, argv, "c")) != -1) { switch (c) { case 'c': From phk at projects.linpro.no Wed Jun 28 21:18:00 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 23:18:00 +0200 (CEST) Subject: r262 - trunk/varnish-cache/bin/varnishlog Message-ID: <20060628211800.643CE1EC325@projects.linpro.no> Author: phk Date: 2006-06-28 23:18:00 +0200 (Wed, 28 Jun 2006) New Revision: 262 Modified: trunk/varnish-cache/bin/varnishlog/Makefile.am trunk/varnish-cache/bin/varnishlog/varnishlog.c Log: Add a -o argument which sorts the log into transactions before output, this is a fair bit easier to chew through than the raw log (the default) Modified: trunk/varnish-cache/bin/varnishlog/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishlog/Makefile.am 2006-06-28 21:03:47 UTC (rev 261) +++ trunk/varnish-cache/bin/varnishlog/Makefile.am 2006-06-28 21:18:00 UTC (rev 262) @@ -6,4 +6,6 @@ varnishlog_SOURCES = varnishlog.c -varnishlog_LDADD = $(top_builddir)/lib/libvarnishapi/libvarnishapi.la +varnishlog_LDADD = \ + $(top_builddir)/lib/libvarnishapi/libvarnishapi.la \ + $(top_builddir)/lib/libsbuf/libsbuf.a Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-06-28 21:03:47 UTC (rev 261) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-06-28 21:18:00 UTC (rev 262) @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include "shmlog.h" #include "varnishapi.h" @@ -30,20 +32,77 @@ static const char *tagnames[256]; -static struct shmloghead *loghead; +/* Ordering-----------------------------------------------------------*/ +static struct sbuf *ob[65536]; + +static void +order(unsigned char *p) +{ + unsigned u; + + u = (p[2] << 8) | p[3]; + if (ob[u] == NULL) { + ob[u] = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); + assert(ob[u] != NULL); + } + sbuf_printf(ob[u], "%02x %3d %4d %-12s", + p[0], p[1], u, tagnames[p[0]]); + if (p[1] > 0) { + sbuf_cat(ob[u], " <"); + sbuf_bcat(ob[u], p + 4, p[1]); + sbuf_cat(ob[u], ">"); + } + sbuf_cat(ob[u], "\n"); + if (u == 0) { + sbuf_finish(ob[u]); + printf("%s", sbuf_data(ob[u])); + sbuf_clear(ob[u]); + return; + } + switch (p[0]) { + case SLT_SessionClose: + case SLT_SessionReuse: + case SLT_BackendClose: + sbuf_finish(ob[u]); + printf("%s\n", sbuf_data(ob[u])); + sbuf_clear(ob[u]); + break; + default: + break; + } +} + + + +/*--------------------------------------------------------------------*/ + + int main(int argc, char **argv) { - int i; + int i, c; unsigned u; unsigned char *p, *q; + int o_flag = 0; + struct shmloghead *loghead; loghead = VSL_OpenLog(); for (i = 0; stagnames[i].tag != SLT_ENDMARKER; i++) tagnames[stagnames[i].tag] = stagnames[i].name; + while ((c = getopt(argc, argv, "o")) != -1) { + switch (c) { + case 'o': + o_flag = 1; + break; + default: + fprintf(stderr, "Usage: varnishlog [-o]\n"); + exit (2); + } + } + q = NULL; while (VSL_NextLog(loghead, &q) != NULL) ; @@ -54,6 +113,10 @@ sleep(1); continue; } + if (o_flag) { + order(p); + continue; + } u = (p[2] << 8) | p[3]; printf("%02x %3d %4d %-12s <", p[0], p[1], u, tagnames[p[0]]); From phk at projects.linpro.no Wed Jun 28 21:33:06 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 23:33:06 +0200 (CEST) Subject: r263 - trunk/varnish-cache/bin/varnishd Message-ID: <20060628213306.60A4E1EC325@projects.linpro.no> Author: phk Date: 2006-06-28 23:33:06 +0200 (Wed, 28 Jun 2006) New Revision: 263 Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c Log: Put a mutex around the shmlog writes, I've seen my first race. Modified: trunk/varnish-cache/bin/varnishd/cache_shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-06-28 21:18:00 UTC (rev 262) +++ trunk/varnish-cache/bin/varnishd/cache_shmlog.c 2006-06-28 21:33:06 UTC (rev 263) @@ -11,6 +11,7 @@ #include #include #include +#include #include "libvarnish.h" #include "shmlog.h" @@ -21,6 +22,7 @@ static struct shmloghead *loghead; static unsigned char *logstart, *logend; +static pthread_mutex_t vsl_mutex; /* * This variant copies a byte-range directly to the log, without @@ -41,7 +43,7 @@ if (e - b > 255) e = b + 255; - /* XXX: Lock */ + AZ(pthread_mutex_lock(&vsl_mutex)); q = NULL; p = logstart + loghead->ptr; assert(p < logend); @@ -63,7 +65,7 @@ loghead->ptr = (p + 4 + (e - b)) - logstart; - /* XXX: Unlock */ + AZ(pthread_mutex_unlock(&vsl_mutex)); } @@ -76,7 +78,7 @@ va_start(ap, fmt); - /* XXX: Lock */ + AZ(pthread_mutex_lock(&vsl_mutex)); q = NULL; p = logstart + loghead->ptr; assert(p < logend); @@ -119,7 +121,7 @@ loghead->ptr = (p + 4 + n) - logstart; - /* XXX: Unlock */ + AZ(pthread_mutex_unlock(&vsl_mutex)); va_end(ap); } @@ -138,6 +140,7 @@ logstart = (unsigned char *)loghead + loghead->start; logend = logstart + loghead->size; VSL_stats = &loghead->stats; + AZ(pthread_mutex_init(&vsl_mutex, NULL)); } /*--------------------------------------------------------------------*/ From phk at projects.linpro.no Wed Jun 28 21:38:20 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 28 Jun 2006 23:38:20 +0200 (CEST) Subject: r264 - trunk/varnish-cache/bin/varnishd Message-ID: <20060628213820.212751EC22E@projects.linpro.no> Author: phk Date: 2006-06-28 23:38:20 +0200 (Wed, 28 Jun 2006) New Revision: 264 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c Log: Convince flexelint that we know what we do with some asserts Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-28 21:33:06 UTC (rev 263) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-28 21:38:20 UTC (rev 264) @@ -200,6 +200,7 @@ p = NULL; v = 0; + st = NULL; while (1) { if (v == 0) { st = stevedore->alloc(stevedore, CHUNK_PREALLOC); @@ -207,6 +208,8 @@ p = st->ptr + st->len; v = st->space - st->len; } + assert(p != NULL); + assert(st != NULL); if (http_GetTail(hp, v, &b, &e)) { memcpy(p, b, e - b); p += e - b; @@ -278,6 +281,7 @@ body = 0; break; default: + body = 0; break; } From phk at projects.linpro.no Thu Jun 29 13:04:55 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 29 Jun 2006 15:04:55 +0200 (CEST) Subject: r265 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060629130455.BE5971EC22E@projects.linpro.no> Author: phk Date: 2006-06-29 15:04:55 +0200 (Thu, 29 Jun 2006) New Revision: 265 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/include/shmlog_tags.h Log: Add a unique transaction-ID to each request, and register it in the shmlog so we can match backend transactions with client transactions. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-28 21:38:20 UTC (rev 264) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-29 13:04:55 UTC (rev 265) @@ -92,6 +92,7 @@ struct sess { int fd; + unsigned xid; /* formatted ascii client address */ char addr[VCA_ADDRBUFSIZE]; @@ -143,7 +144,7 @@ /* cache_backend.c */ void VBE_Init(void); -int VBE_GetFd(struct backend *bp, void **ptr); +int VBE_GetFd(struct backend *bp, void **ptr, unsigned xid); void VBE_ClosedFd(void *ptr); void VBE_RecycleFd(void *ptr); Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-06-28 21:38:20 UTC (rev 264) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2006-06-29 13:04:55 UTC (rev 265) @@ -201,7 +201,7 @@ */ int -VBE_GetFd(struct backend *bp, void **ptr) +VBE_GetFd(struct backend *bp, void **ptr, unsigned xid) { struct vbe *vp; struct vbe_conn *vc; @@ -244,6 +244,7 @@ event_base_set(vbe_evb, &vc->ev); } *ptr = vc; + VSL(SLT_BackendXID, vc->fd, "%u", xid); return (vc->fd); } Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-28 21:38:20 UTC (rev 264) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-29 13:04:55 UTC (rev 265) @@ -245,7 +245,7 @@ time_t t_req, t_resp; int body; - fd = VBE_GetFd(sp->backend, &fd_token); + fd = VBE_GetFd(sp->backend, &fd_token, sp->xid); assert(fd != -1); VSL(SLT_Backend, sp->fd, "%d %s", fd, sp->backend->vcl_name); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-28 21:38:20 UTC (rev 264) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2006-06-29 13:04:55 UTC (rev 265) @@ -335,11 +335,6 @@ } hp->t = ++p; -#if 0 -printf("Head:\n%#H\n", hp->s, hp->t - hp->s); -printf("Tail:\n%#H\n", hp->t, hp->v - hp->t); -#endif - event_del(&hp->ev); if (hp->callback != NULL) hp->callback(hp->arg, 1); Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-06-28 21:38:20 UTC (rev 264) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-06-29 13:04:55 UTC (rev 265) @@ -157,7 +157,7 @@ char *b; int cls; - fd = VBE_GetFd(sp->backend, &fd_token); + fd = VBE_GetFd(sp->backend, &fd_token, sp->xid); assert(fd != -1); http_BuildSbuf(fd, 1, w->sb, sp->http); Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-06-28 21:38:20 UTC (rev 264) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-06-29 13:04:55 UTC (rev 265) @@ -48,7 +48,7 @@ void *fd_token; struct edir e1, e2; - fd = VBE_GetFd(sp->backend, &fd_token); + fd = VBE_GetFd(sp->backend, &fd_token, sp->xid); assert(fd != -1); http_BuildSbuf(fd, 0, w->sb, sp->http); /* XXX: 0 ?? */ Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-28 21:38:20 UTC (rev 264) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-06-29 13:04:55 UTC (rev 265) @@ -21,6 +21,7 @@ static TAILQ_HEAD(, sess) shd = TAILQ_HEAD_INITIALIZER(shd); static pthread_cond_t shdcnd; +static unsigned xids; /*--------------------------------------------------------------------*/ @@ -131,6 +132,13 @@ vca_return_session(sp); return; } + + /* + * No locking necessary, we're serialized in the acceptor thread + */ + sp->xid = xids++; + VSL(SLT_XID, sp->fd, "%u", sp->xid); + VSL_stats->client_req++; AZ(pthread_mutex_lock(&sessmtx)); TAILQ_INSERT_TAIL(&shd, sp, list); @@ -149,4 +157,6 @@ for (i = 0; i < 5; i++) AZ(pthread_create(&tp, NULL, CacheWorker, NULL)); AZ(pthread_detach(tp)); + srandomdev(); + xids = random(); } Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-06-28 21:38:20 UTC (rev 264) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-06-29 13:04:55 UTC (rev 265) @@ -13,6 +13,7 @@ SLTM(SessionReuse) SLTM(SessionClose) SLTM(BackendOpen) +SLTM(BackendXID) SLTM(BackendReuse) SLTM(BackendClose) SLTM(HttpError) @@ -32,3 +33,4 @@ SLTM(BldHdr) SLTM(LostHeader) SLTM(VCL) +SLTM(XID) From phk at projects.linpro.no Thu Jun 29 14:37:15 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 29 Jun 2006 16:37:15 +0200 (CEST) Subject: r266 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060629143715.EC5CE1EC233@projects.linpro.no> Author: phk Date: 2006-06-29 16:37:15 +0200 (Thu, 29 Jun 2006) New Revision: 266 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/include/shmlog_tags.h Log: Tag objects with their origin session xid and log it when we clean up. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-29 13:04:55 UTC (rev 265) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-29 14:37:15 UTC (rev 266) @@ -65,6 +65,7 @@ struct object { unsigned refcnt; + unsigned xid; struct objhead *objhead; pthread_cond_t cv; Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-29 13:04:55 UTC (rev 265) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-29 14:37:15 UTC (rev 266) @@ -52,6 +52,7 @@ } TAILQ_REMOVE(&exp_deathrow, o, deathrow); AZ(pthread_mutex_unlock(&exp_mtx)); + VSL(SLT_ExpKill, 0, "%u", o->xid); HSH_Deref(o); } } @@ -83,6 +84,7 @@ } binheap_delete(exp_heap, 0); AZ(pthread_mutex_unlock(&exp_mtx)); + VSL(SLT_ExpPick, 0, "%u", o->xid); sp.vcl = GetVCL(); sp.obj = o; Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-29 13:04:55 UTC (rev 265) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-29 14:37:15 UTC (rev 266) @@ -245,6 +245,8 @@ time_t t_req, t_resp; int body; + sp->obj->xid = sp->xid; + fd = VBE_GetFd(sp->backend, &fd_token, sp->xid); assert(fd != -1); VSL(SLT_Backend, sp->fd, "%d %s", fd, sp->backend->vcl_name); Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-06-29 13:04:55 UTC (rev 265) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-06-29 14:37:15 UTC (rev 266) @@ -34,3 +34,5 @@ SLTM(LostHeader) SLTM(VCL) SLTM(XID) +SLTM(ExpPick) +SLTM(ExpKill) From phk at projects.linpro.no Thu Jun 29 15:14:15 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 29 Jun 2006 17:14:15 +0200 (CEST) Subject: r267 - in trunk/varnish-cache: bin/varnishd bin/varnishlog include Message-ID: <20060629151415.292751EC22B@projects.linpro.no> Author: phk Date: 2006-06-29 17:14:15 +0200 (Thu, 29 Jun 2006) New Revision: 267 Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishlog/varnishlog.c trunk/varnish-cache/include/shmlog_tags.h Log: Improve shm-logging of VCL activity Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-29 14:37:15 UTC (rev 266) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-29 15:14:15 UTC (rev 267) @@ -227,9 +227,10 @@ { \ \ sp->handling = 0; \ + VSL(SLT_VCL_call, sp->fd, "%s", #func); \ sp->vcl->func##_func(sp); \ CheckHandling(sp, #func, (bitmap)); \ - VSL(SLT_vcl_##func, sp->fd, "0x%x %s", sp->handling, HandlingName(sp->handling)); \ + VSL(SLT_VCL_return, sp->fd, "%s", HandlingName(sp->handling)); \ } #define VCL_RET_MAC(l,u,b) Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-29 14:37:15 UTC (rev 266) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-29 15:14:15 UTC (rev 267) @@ -36,7 +36,7 @@ VRT_count(struct sess *sp, unsigned u) { - VSL(SLT_VCL, sp->fd, "%u %d.%d", u, + VSL(SLT_VCL_trace, sp->fd, "%u %d.%d", u, sp->vcl->ref[u].line, sp->vcl->ref[u].pos); } Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-06-29 14:37:15 UTC (rev 266) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2006-06-29 15:14:15 UTC (rev 267) @@ -46,14 +46,39 @@ ob[u] = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND); assert(ob[u] != NULL); } - sbuf_printf(ob[u], "%02x %3d %4d %-12s", - p[0], p[1], u, tagnames[p[0]]); - if (p[1] > 0) { - sbuf_cat(ob[u], " <"); - sbuf_bcat(ob[u], p + 4, p[1]); - sbuf_cat(ob[u], ">"); + switch (p[0]) { + case SLT_VCL_call: + sbuf_printf(ob[u], "%02x %3d %4d %-12s", + p[0], p[1], u, tagnames[p[0]]); + if (p[1] > 0) { + sbuf_cat(ob[u], " <"); + sbuf_bcat(ob[u], p + 4, p[1]); + } + break; + case SLT_VCL_trace: + if (p[1] > 0) { + sbuf_cat(ob[u], " "); + sbuf_bcat(ob[u], p + 4, p[1]); + } + break; + case SLT_VCL_return: + if (p[1] > 0) { + sbuf_cat(ob[u], " "); + sbuf_bcat(ob[u], p + 4, p[1]); + sbuf_cat(ob[u], ">\n"); + } + break; + default: + sbuf_printf(ob[u], "%02x %3d %4d %-12s", + p[0], p[1], u, tagnames[p[0]]); + if (p[1] > 0) { + sbuf_cat(ob[u], " <"); + sbuf_bcat(ob[u], p + 4, p[1]); + sbuf_cat(ob[u], ">"); + } + sbuf_cat(ob[u], "\n"); + break; } - sbuf_cat(ob[u], "\n"); if (u == 0) { sbuf_finish(ob[u]); printf("%s", sbuf_data(ob[u])); Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-06-29 14:37:15 UTC (rev 266) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-06-29 15:14:15 UTC (rev 267) @@ -18,11 +18,6 @@ SLTM(BackendClose) SLTM(HttpError) SLTM(ClientAddr) -#define VCL_RET_MAC(l,u,b) -#define VCL_MET_MAC(l,u,b) SLTM(vcl_##l) -#include "vcl_returns.h" -#undef VCL_MET_MAC -#undef VCL_RET_MAC SLTM(Backend) SLTM(Request) SLTM(Response) @@ -32,7 +27,9 @@ SLTM(Header) SLTM(BldHdr) SLTM(LostHeader) -SLTM(VCL) +SLTM(VCL_call) +SLTM(VCL_trace) +SLTM(VCL_return) SLTM(XID) SLTM(ExpPick) SLTM(ExpKill) From phk at projects.linpro.no Thu Jun 29 17:09:24 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 29 Jun 2006 19:09:24 +0200 (CEST) Subject: r268 - trunk/varnish-cache/bin/varnishd Message-ID: <20060629170924.E85FD1EC233@projects.linpro.no> Author: phk Date: 2006-06-29 19:09:24 +0200 (Thu, 29 Jun 2006) New Revision: 268 Added: trunk/varnish-cache/bin/varnishd/cache_ban.c Modified: trunk/varnish-cache/bin/varnishd/Makefile.am trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_main.c Log: Add the ability to instantly ban/purge all cached objects matching a given regexp. Modified: trunk/varnish-cache/bin/varnishd/Makefile.am =================================================================== --- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-06-29 15:14:15 UTC (rev 267) +++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-06-29 17:09:24 UTC (rev 268) @@ -7,6 +7,7 @@ varnishd_SOURCES = \ cache_acceptor.c \ cache_backend.c \ + cache_ban.c \ cache_expire.c \ cache_fetch.c \ cache_hash.c \ Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-29 15:14:15 UTC (rev 267) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-29 17:09:24 UTC (rev 268) @@ -9,6 +9,7 @@ #define VCA_ADDRBUFSIZE 32 /* Sizeof ascii network address */ struct event_base; +struct cli; struct sbuf; struct sess; struct object; @@ -69,6 +70,8 @@ struct objhead *objhead; pthread_cond_t cv; + unsigned ban_seq; + unsigned valid; unsigned cacheable; @@ -149,6 +152,12 @@ void VBE_ClosedFd(void *ptr); void VBE_RecycleFd(void *ptr); +/* cache_ban.c */ +void BAN_Init(void); +void cli_func_url_purge(struct cli *cli, char **av, void *priv); +void BAN_NewObj(struct object *o); +int BAN_CheckObject(struct object *o, const char *url); + /* cache_expiry.c */ void EXP_Insert(struct object *o); void EXP_Init(void); Added: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2006-06-29 15:14:15 UTC (rev 267) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2006-06-29 17:09:24 UTC (rev 268) @@ -0,0 +1,87 @@ +/* + * $Id$ + * + * Ban processing + */ + +#include +#include +#include +#include + +#include "shmlog.h" +#include "cli_priv.h" +#include "cache.h" + +struct ban { + TAILQ_ENTRY(ban) list; + unsigned gen; + regex_t regexp; + char *ban; +}; + +static TAILQ_HEAD(,ban) ban_head = TAILQ_HEAD_INITIALIZER(ban_head); +static unsigned ban_next; +static struct ban *ban_start; + +static void +AddBan(const char *regexp) +{ + struct ban *b; + int i; + + b = calloc(sizeof *b, 1); + assert(b != NULL); + + i = regcomp(&b->regexp, regexp, REG_EXTENDED | REG_NOSUB); + if (i) { + char buf[512]; + + regerror(i, &b->regexp, buf, sizeof buf); + VSL(SLT_Debug, 0, "REGEX: <%s>", buf); + } + b->gen = ++ban_next; + b->ban = strdup(regexp); + TAILQ_INSERT_HEAD(&ban_head, b, list); + ban_start = b; +} + +void +BAN_NewObj(struct object *o) +{ + + o->ban_seq = ban_next; +} + +int +BAN_CheckObject(struct object *o, const char *url) +{ + struct ban *b, *b0; + int i; + + b0 = ban_start; + for (b = b0; + b != NULL && b->gen > o->ban_seq; + b = TAILQ_NEXT(b, list)) { + i = regexec(&b->regexp, url, 0, NULL, 0); + if (!i) + return (1); + } + o->ban_seq = b0->gen; + return (0); +} + +void +cli_func_url_purge(struct cli *cli, char **av, void *priv) +{ + + AddBan(av[2]); + cli_out(cli, "PURGE %s\n", av[2]); +} + +void +BAN_Init(void) +{ + + AddBan("a"); +} Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-06-29 15:14:15 UTC (rev 267) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-06-29 17:09:24 UTC (rev 268) @@ -12,6 +12,7 @@ #include #include "libvarnish.h" +#include "shmlog.h" #include "cache.h" static struct hash_slinger *hash; @@ -48,8 +49,13 @@ o->refcnt++; if (o->busy) AZ(pthread_cond_wait(&o->cv, &oh->mtx)); - /* XXX: do Vary: comparison */ - if (1) + /* XXX: check TTL */ + if (o->ttl == 0) { + VSL(SLT_Debug, 0, "Object %p had 0 ttl", o); + } else if (BAN_CheckObject(o, b)) { + o->ttl = 0; + VSL(SLT_Debug, 0, "Object %p was banned", o); + } else break; o->refcnt--; } @@ -67,6 +73,7 @@ TAILQ_INSERT_TAIL(&oh->objects, o, list); /* NB: do not deref objhead the new object inherits our reference */ AZ(pthread_mutex_unlock(&oh->mtx)); + BAN_NewObj(o); return (o); } Modified: trunk/varnish-cache/bin/varnishd/cache_main.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-29 15:14:15 UTC (rev 267) +++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-06-29 17:09:24 UTC (rev 268) @@ -83,6 +83,7 @@ static struct cli_proto cli_proto[] = { { CLI_URL_QUERY, cli_func_url_query }, + { CLI_URL_PURGE, cli_func_url_purge }, { CLI_CONFIG_LOAD, cli_func_config_load }, { CLI_CONFIG_LIST, cli_func_config_list }, { CLI_CONFIG_UNLOAD, cli_func_config_unload }, @@ -115,6 +116,7 @@ VCA_Init(); EXP_Init(); HSH_Init(); + BAN_Init(); eb = event_init(); assert(eb != NULL); From phk at phk.freebsd.dk Thu Jun 29 17:11:53 2006 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Thu, 29 Jun 2006 17:11:53 +0000 Subject: r268 - trunk/varnish-cache/bin/varnishd In-Reply-To: Your message of "Thu, 29 Jun 2006 19:09:24 +0200." <20060629170924.E85FD1EC233@projects.linpro.no> Message-ID: <1153.1151601113@critter.freebsd.dk> In message <20060629170924.E85FD1EC233 at projects.linpro.no>, phk at projects.linpro .no writes: >Author: phk >Date: 2006-06-29 19:09:24 +0200 (Thu, 29 Jun 2006) >New Revision: 268 > >Log: >Add the ability to instantly ban/purge all cached objects matching >a given regexp. The fact that we can do this in 87 lines, and without any locking (provided we're willing to leak the memory for the bans themselves) amazes me to no end... Reclaiming the memory for the bans would require a mutex operation to refcount the bans, for now: live without it. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at projects.linpro.no Thu Jun 29 19:06:44 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 29 Jun 2006 21:06:44 +0200 (CEST) Subject: r269 - trunk/varnish-cache/bin/varnishd Message-ID: <20060629190644.D4DE91EC233@projects.linpro.no> Author: phk Date: 2006-06-29 21:06:44 +0200 (Thu, 29 Jun 2006) New Revision: 269 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_expire.c Log: Track objects heap-position Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-29 17:09:24 UTC (rev 268) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-29 19:06:44 UTC (rev 269) @@ -70,6 +70,7 @@ struct objhead *objhead; pthread_cond_t cv; + unsigned heap_idx; unsigned ban_seq; unsigned valid; Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-29 17:09:24 UTC (rev 268) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-29 19:06:44 UTC (rev 269) @@ -115,6 +115,14 @@ return (aa->ttl < bb->ttl); } +static void +object_update(void *priv, void *p, unsigned u) +{ + struct object *o = p; + + o->heap_idx = u; +} + /*--------------------------------------------------------------------*/ void @@ -124,6 +132,6 @@ AZ(pthread_mutex_init(&exp_mtx, NULL)); AZ(pthread_create(&exp_thread, NULL, exp_prefetch, NULL)); AZ(pthread_create(&exp_thread, NULL, exp_hangman, NULL)); - exp_heap = binheap_new(NULL, object_cmp, NULL); + exp_heap = binheap_new(NULL, object_cmp, object_update); assert(exp_heap != NULL); } From phk at projects.linpro.no Thu Jun 29 19:19:18 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 29 Jun 2006 21:19:18 +0200 (CEST) Subject: r270 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060629191918.469221EC233@projects.linpro.no> Author: phk Date: 2006-06-29 21:19:18 +0200 (Thu, 29 Jun 2006) New Revision: 270 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/include/shmlog_tags.h Log: Log objects banned to shmlog Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-29 19:06:44 UTC (rev 269) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-29 19:19:18 UTC (rev 270) @@ -162,6 +162,7 @@ /* cache_expiry.c */ void EXP_Insert(struct object *o); void EXP_Init(void); +void EXP_TTLchange(struct object *o); /* cache_fetch.c */ int FetchSession(struct worker *w, struct sess *sp); Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-29 19:06:44 UTC (rev 269) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2006-06-29 19:19:18 UTC (rev 270) @@ -31,6 +31,15 @@ AZ(pthread_mutex_unlock(&exp_mtx)); } +void +EXP_TTLchange(struct object *o) +{ + AZ(pthread_mutex_lock(&exp_mtx)); + binheap_delete(exp_heap, o->heap_idx); + binheap_insert(exp_heap, o); + AZ(pthread_mutex_unlock(&exp_mtx)); +} + /*-------------------------------------------------------------------- * This thread monitors deathrow and kills objects when they time out. */ @@ -77,7 +86,7 @@ time(&t); AZ(pthread_mutex_lock(&exp_mtx)); o = binheap_root(exp_heap); - if (o == NULL || o->ttl - t > expearly) { + if (o == NULL || o->ttl > t + expearly) { AZ(pthread_mutex_unlock(&exp_mtx)); sleep(1); continue; Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-06-29 19:06:44 UTC (rev 269) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-06-29 19:19:18 UTC (rev 270) @@ -51,10 +51,11 @@ AZ(pthread_cond_wait(&o->cv, &oh->mtx)); /* XXX: check TTL */ if (o->ttl == 0) { - VSL(SLT_Debug, 0, "Object %p had 0 ttl", o); + /* Object banned but not reaped yet */ } else if (BAN_CheckObject(o, b)) { o->ttl = 0; - VSL(SLT_Debug, 0, "Object %p was banned", o); + VSL(SLT_ExpBan, 0, "%u was banned", o->xid); + EXP_TTLchange(o); } else break; o->refcnt--; Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2006-06-29 19:06:44 UTC (rev 269) +++ trunk/varnish-cache/include/shmlog_tags.h 2006-06-29 19:19:18 UTC (rev 270) @@ -31,5 +31,6 @@ SLTM(VCL_trace) SLTM(VCL_return) SLTM(XID) +SLTM(ExpBan) SLTM(ExpPick) SLTM(ExpKill) From phk at projects.linpro.no Fri Jun 30 09:14:51 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 30 Jun 2006 11:14:51 +0200 (CEST) Subject: r271 - trunk/varnish-cache/bin/varnishd Message-ID: <20060630091451.3EE521EC300@projects.linpro.no> Author: phk Date: 2006-06-30 11:14:51 +0200 (Fri, 30 Jun 2006) New Revision: 271 Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c Log: Delete compiled VCL file after we tried to load it. Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-29 19:19:18 UTC (rev 270) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-30 09:14:51 UTC (rev 271) @@ -3,6 +3,7 @@ */ #include +#include #include #include #include @@ -69,6 +70,7 @@ assert(vcl != NULL); vcl->dlh = dlopen(fn, RTLD_NOW | RTLD_LOCAL); + unlink(fn); if (vcl->dlh == NULL) { fprintf(stderr, "dlopen(%s): %s\n", fn, dlerror()); free(vcl); From phk at projects.linpro.no Fri Jun 30 11:20:10 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 30 Jun 2006 13:20:10 +0200 (CEST) Subject: r272 - trunk/varnish-cache/bin/varnishd Message-ID: <20060630112010.99D0C1EC233@projects.linpro.no> Author: phk Date: 2006-06-30 13:20:10 +0200 (Fri, 30 Jun 2006) New Revision: 272 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c Log: Fix object length double accounting in chunked fetch Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-30 09:14:51 UTC (rev 271) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-30 11:20:10 UTC (rev 272) @@ -161,7 +161,6 @@ st->len += e - b; v -= e - b; u -= e - b; - sp->obj->len += e - b; } while (v > 0) { i = read(fd, p, v); @@ -170,7 +169,6 @@ v -= i; u -= i; p += i; - sp->obj->len += i; } } } From des at linpro.no Fri Jun 30 13:05:24 2006 From: des at linpro.no (Dag-Erling =?iso-8859-1?Q?Sm=F8rgrav?=) Date: Fri, 30 Jun 2006 15:05:24 +0200 Subject: r259 - in trunk/varnish-cache: include lib/libvarnishapi References: <20060628205804.18C1D1EC325@projects.linpro.no> Message-ID: phk at projects.linpro.no writes: > Modified: > trunk/varnish-cache/include/varnishapi.h > trunk/varnish-cache/lib/libvarnishapi/Makefile.am > Log: > Add SHMLOG opening and walking functions to libvarnishapi You forgot to 'svn add shmlog.c'... DES -- Dag-Erling Sm?rgrav Senior Software Developer Linpro AS - www.linpro.no From des at projects.linpro.no Fri Jun 30 13:44:09 2006 From: des at projects.linpro.no (des at projects.linpro.no) Date: Fri, 30 Jun 2006 15:44:09 +0200 (CEST) Subject: r273 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20060630134409.AEFCC1EC30B@projects.linpro.no> Author: des Date: 2006-06-30 15:44:09 +0200 (Fri, 30 Jun 2006) New Revision: 273 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_pass.c trunk/varnish-cache/bin/varnishd/cache_pipe.c trunk/varnish-cache/bin/varnishd/cache_vcl.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishd/hash_simple_list.c trunk/varnish-cache/bin/varnishd/storage_file.c trunk/varnish-cache/bin/varnishd/storage_malloc.c trunk/varnish-cache/include/stats.h Log: Consistently use our own copy of queue.h. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-30 11:20:10 UTC (rev 272) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-30 13:44:09 UTC (rev 273) @@ -2,7 +2,7 @@ * $Id$ */ -#include +#include #include "vcl_returns.h" Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-30 11:20:10 UTC (rev 272) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-06-30 13:44:09 UTC (rev 273) @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-30 11:20:10 UTC (rev 272) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-30 13:44:09 UTC (rev 273) @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include #include Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-06-30 11:20:10 UTC (rev 272) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-06-30 13:44:09 UTC (rev 273) @@ -10,6 +10,7 @@ #include #include #include +#include #include "libvarnish.h" #include "shmlog.h" Modified: trunk/varnish-cache/bin/varnishd/cache_pass.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-06-30 11:20:10 UTC (rev 272) +++ trunk/varnish-cache/bin/varnishd/cache_pass.c 2006-06-30 13:44:09 UTC (rev 273) @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-06-30 11:20:10 UTC (rev 272) +++ trunk/varnish-cache/bin/varnishd/cache_pipe.c 2006-06-30 13:44:09 UTC (rev 273) @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-30 11:20:10 UTC (rev 272) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-06-30 13:44:09 UTC (rev 273) @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include "cli.h" #include "cli_priv.h" Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-30 11:20:10 UTC (rev 272) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2006-06-30 13:44:09 UTC (rev 273) @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include "cli.h" #include "cli_priv.h" Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-06-30 11:20:10 UTC (rev 272) +++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-06-30 13:44:09 UTC (rev 273) @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-30 11:20:10 UTC (rev 272) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2006-06-30 13:44:09 UTC (rev 273) @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-06-30 11:20:10 UTC (rev 272) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2006-06-30 13:44:09 UTC (rev 273) @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include "cache.h" Modified: trunk/varnish-cache/include/stats.h =================================================================== --- trunk/varnish-cache/include/stats.h 2006-06-30 11:20:10 UTC (rev 272) +++ trunk/varnish-cache/include/stats.h 2006-06-30 13:44:09 UTC (rev 273) @@ -1,6 +1,6 @@ /* $Id$ */ -#include +#include struct varnish_stats { #define MAC_STAT(n,t,f,e) t n; From phk at projects.linpro.no Fri Jun 30 20:17:54 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 30 Jun 2006 22:17:54 +0200 (CEST) Subject: r274 - trunk/varnish-cache/bin/varnishd Message-ID: <20060630201754.874A21EC307@projects.linpro.no> Author: phk Date: 2006-06-30 22:17:54 +0200 (Fri, 30 Jun 2006) New Revision: 274 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/rfc2616.c Log: move all policy to rfc2616.c Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2006-06-30 13:44:09 UTC (rev 273) +++ trunk/varnish-cache/bin/varnishd/cache.h 2006-06-30 20:17:54 UTC (rev 274) @@ -105,6 +105,9 @@ /* HTTP request */ struct http *http; + time_t t_req; + time_t t_resp; + unsigned handling; TAILQ_ENTRY(sess) list; @@ -234,4 +237,4 @@ #endif /* rfc2616.c */ -time_t RFC2616_Ttl(struct http *hp, time_t, time_t); +int RFC2616_cache_policy(struct sess *sp, struct http *hp); Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-30 13:44:09 UTC (rev 273) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-06-30 20:17:54 UTC (rev 274) @@ -240,7 +240,6 @@ void *fd_token; struct http *hp; char *b; - time_t t_req, t_resp; int body; sp->obj->xid = sp->xid; @@ -253,7 +252,7 @@ http_BuildSbuf(fd, 1, w->sb, sp->http); i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb)); assert(i == sbuf_len(w->sb)); - time(&t_req); + time(&sp->t_req); /* XXX: copy any contents */ @@ -263,33 +262,11 @@ */ http_RecvHead(hp, fd, w->eb, NULL, NULL); event_base_loop(w->eb, 0); - time(&t_resp); + time(&sp->t_resp); http_Dissect(hp, fd, 2); - switch (http_GetStatus(hp)) { - case 200: - case 301: - /* XXX: fill in object from headers */ - sp->obj->valid = 1; - sp->obj->cacheable = 1; - body = 1; - break; - case 304: - /* XXX: fill in object from headers */ - sp->obj->valid = 1; - sp->obj->cacheable = 1; - body = 0; - break; - default: - body = 0; - break; - } + body = RFC2616_cache_policy(sp, hp); - sp->obj->ttl = RFC2616_Ttl(hp, t_req, t_resp); - if (sp->obj->ttl == 0) { - sp->obj->cacheable = 0; - } - VCL_fetch_method(sp); if (sp->obj->cacheable) Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c =================================================================== --- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-30 13:44:09 UTC (rev 273) +++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-06-30 20:17:54 UTC (rev 274) @@ -9,7 +9,6 @@ #include "cache.h" #include "libvarnish.h" #include "heritage.h" - /*-------------------------------------------------------------------- * From RFC2616, 13.2.3 Age Calculations * @@ -35,7 +34,7 @@ * */ -time_t +static time_t RFC2616_Ttl(struct http *hp, time_t t_req, time_t t_resp) { time_t h_date = 0, h_expires = 0, h_age = 0; @@ -94,3 +93,38 @@ return (ttl); } + +int +RFC2616_cache_policy(struct sess *sp, struct http *hp) +{ + int body = 0; + + /* + * Initial cacheability determination per [RFC2616, 13.4] + * We do not support ranges yet, so 206 is out. + */ + switch (http_GetStatus(hp)) { + case 200: /* OK */ + sp->obj->valid = 1; + case 203: /* Non-Authoritative Information */ + case 300: /* Multiple Choices */ + case 301: /* Moved Permanently */ + case 410: /* Gone */ + sp->obj->cacheable = 1; + body = 1; + break; + default: + sp->obj->cacheable = 0; + body = 0; + break; + } + + sp->obj->ttl = RFC2616_Ttl(hp, sp->t_req, sp->t_resp); + if (sp->obj->ttl == 0) { + sp->obj->cacheable = 0; + } + + return (body); + +} + From phk at projects.linpro.no Fri Jun 30 20:21:15 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 30 Jun 2006 22:21:15 +0200 (CEST) Subject: r275 - trunk/varnish-cache/include Message-ID: <20060630202115.CE18E1EC30B@projects.linpro.no> Author: phk Date: 2006-06-30 22:21:15 +0200 (Fri, 30 Jun 2006) New Revision: 275 Modified: trunk/varnish-cache/include/queue.h Log: Add TAILQ_FOREACH_SAFE() Modified: trunk/varnish-cache/include/queue.h =================================================================== --- trunk/varnish-cache/include/queue.h 2006-06-30 20:17:54 UTC (rev 274) +++ trunk/varnish-cache/include/queue.h 2006-06-30 20:21:15 UTC (rev 275) @@ -496,6 +496,11 @@ (var); \ (var) = ((var)->field.tqe_next)) +#define TAILQ_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = TAILQ_FIRST((head)); \ + (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + #define TAILQ_FOREACH_REVERSE(var, head, headname, field) \ for ((var) = (*(((struct headname *)((head)->tqh_last))->tqh_last)); \ (var); \ From phk at projects.linpro.no Fri Jun 30 20:22:58 2006 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 30 Jun 2006 22:22:58 +0200 (CEST) Subject: r276 - trunk/varnish-cache/lib/libvarnishapi Message-ID: <20060630202258.1E22B1EC30B@projects.linpro.no> Author: phk Date: 2006-06-30 22:22:58 +0200 (Fri, 30 Jun 2006) New Revision: 276 Added: trunk/varnish-cache/lib/libvarnishapi/shmlog.c Log: Forgot to add shmlog.c (reminded by des@) Added: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-06-30 20:21:15 UTC (rev 275) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2006-06-30 20:22:58 UTC (rev 276) @@ -0,0 +1,77 @@ +/* + * $Id: varnishlog.c 153 2006-04-25 08:17:43Z phk $ + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "shmlog.h" +#include "varnishapi.h" + +static unsigned char *logstart, *logend; + +struct shmloghead * +VSL_OpenLog(void) +{ + int fd; + int i; + struct shmloghead slh, *lh; + + fd = open(SHMLOG_FILENAME, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "Cannot open %s: %s\n", + SHMLOG_FILENAME, strerror(errno)); + exit (1); + } + i = read(fd, &slh, sizeof slh); + if (i != sizeof slh) { + fprintf(stderr, "Cannot read %s: %s\n", + SHMLOG_FILENAME, strerror(errno)); + exit (1); + } + if (slh.magic != SHMLOGHEAD_MAGIC) { + fprintf(stderr, "Wrong magic number in file %s\n", + SHMLOG_FILENAME); + exit (1); + } + + lh = mmap(NULL, slh.size + sizeof slh, + PROT_READ, MAP_HASSEMAPHORE, fd, 0); + if (lh == MAP_FAILED) { + fprintf(stderr, "Cannot mmap %s: %s\n", + SHMLOG_FILENAME, strerror(errno)); + exit (1); + } + + logstart = (unsigned char *)lh + lh->start; + logend = logstart + lh->size; + + return (lh); +} + +unsigned char * +VSL_NextLog(struct shmloghead *lh, unsigned char **pp) +{ + unsigned char *p; + + p = *pp; + if (p == NULL) + p = logstart; + while (1) { + if (*p == SLT_WRAPMARKER) { + p = logstart; + continue; + } + if (*p == SLT_ENDMARKER) { + *pp = p; + return (NULL); + } + *pp = p + p[1] + 4; + return (p); + } +}