From bahner at projects.linpro.no Wed Apr 2 09:42:17 2008 From: bahner at projects.linpro.no (bahner at projects.linpro.no) Date: Wed, 2 Apr 2008 11:42:17 +0200 (CEST) Subject: r2618 - trunk/varnish-cache Message-ID: <20080402094217.1F4F71EC0F2@projects.linpro.no> Author: bahner Date: 2008-04-02 11:42:16 +0200 (Wed, 02 Apr 2008) New Revision: 2618 Removed: trunk/varnish-cache/debian/ Log: Debian pakking er flyttet til svn://svn.debian.org/svn/pkg-varnish/packages/varnish/debian From bahner at projects.linpro.no Wed Apr 2 11:51:07 2008 From: bahner at projects.linpro.no (bahner at projects.linpro.no) Date: Wed, 2 Apr 2008 13:51:07 +0200 (CEST) Subject: r2619 - / varnish Message-ID: <20080402115107.6138C1EC0B3@projects.linpro.no> Author: bahner Date: 2008-04-02 13:51:06 +0200 (Wed, 02 Apr 2008) New Revision: 2619 Added: varnish/ varnish/trunk/ Log: temp From bahner at projects.linpro.no Wed Apr 2 11:53:04 2008 From: bahner at projects.linpro.no (bahner at projects.linpro.no) Date: Wed, 2 Apr 2008 13:53:04 +0200 (CEST) Subject: r2620 - / Message-ID: <20080402115304.823D61EC0DD@projects.linpro.no> Author: bahner Date: 2008-04-02 13:53:04 +0200 (Wed, 02 Apr 2008) New Revision: 2620 Removed: varnish/ Log: feil From bahner at projects.linpro.no Wed Apr 2 11:53:49 2008 From: bahner at projects.linpro.no (bahner at projects.linpro.no) Date: Wed, 2 Apr 2008 13:53:49 +0200 (CEST) Subject: r2621 - / Message-ID: <20080402115349.EFF1F1EC0B3@projects.linpro.no> Author: bahner Date: 2008-04-02 13:53:49 +0200 (Wed, 02 Apr 2008) New Revision: 2621 Added: packages/ Log: New folder From bahner at projects.linpro.no Wed Apr 2 11:54:14 2008 From: bahner at projects.linpro.no (bahner at projects.linpro.no) Date: Wed, 2 Apr 2008 13:54:14 +0200 (CEST) Subject: r2622 - packages Message-ID: <20080402115414.ADE2A1EC0DD@projects.linpro.no> Author: bahner Date: 2008-04-02 13:54:14 +0200 (Wed, 02 Apr 2008) New Revision: 2622 Added: packages/varnish/ Log: new folder From bahner at projects.linpro.no Wed Apr 2 11:54:18 2008 From: bahner at projects.linpro.no (bahner at projects.linpro.no) Date: Wed, 2 Apr 2008 13:54:18 +0200 (CEST) Subject: r2623 - packages/varnish Message-ID: <20080402115418.925D81EC0B3@projects.linpro.no> Author: bahner Date: 2008-04-02 13:54:18 +0200 (Wed, 02 Apr 2008) New Revision: 2623 Added: packages/varnish/trunk/ Log: new folder From bahner at projects.linpro.no Wed Apr 2 12:03:05 2008 From: bahner at projects.linpro.no (bahner at projects.linpro.no) Date: Wed, 2 Apr 2008 14:03:05 +0200 (CEST) Subject: r2624 - packages/varnish/trunk Message-ID: <20080402120305.582731EC236@projects.linpro.no> Author: bahner Date: 2008-04-02 14:03:05 +0200 (Wed, 02 Apr 2008) New Revision: 2624 Added: packages/varnish/trunk/debian/ Log: Recover folder Copied: packages/varnish/trunk/debian (from rev 2616, trunk/varnish-cache/debian) From bahner at projects.linpro.no Wed Apr 2 12:15:18 2008 From: bahner at projects.linpro.no (bahner at projects.linpro.no) Date: Wed, 2 Apr 2008 14:15:18 +0200 (CEST) Subject: r2625 - / Message-ID: <20080402121518.02A921EC0DD@projects.linpro.no> Author: bahner Date: 2008-04-02 14:15:17 +0200 (Wed, 02 Apr 2008) New Revision: 2625 Removed: packages/ Log: Didn't work either :( From bahner at projects.linpro.no Thu Apr 3 17:15:59 2008 From: bahner at projects.linpro.no (bahner at projects.linpro.no) Date: Thu, 3 Apr 2008 19:15:59 +0200 (CEST) Subject: r2626 - trunk/varnish-cache Message-ID: <20080403171559.0AA531EC118@projects.linpro.no> Author: bahner Date: 2008-04-03 19:15:58 +0200 (Thu, 03 Apr 2008) New Revision: 2626 Modified: trunk/varnish-cache/ Log: svn-externals debian svn://svn.debian.org:/svn/pkg-varnish/packages/varnish/trunk/debian Property changes on: trunk/varnish-cache ___________________________________________________________________ Name: svn:externals + debian svn://svn.debian.org:/svn/pkg-varnish/packages/varnish/trunk/debian From phk at projects.linpro.no Tue Apr 8 15:33:08 2008 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 8 Apr 2008 17:33:08 +0200 (CEST) Subject: r2627 - trunk/varnish-cache/lib/libvcl Message-ID: <20080408153308.D27551EC10B@projects.linpro.no> Author: phk Date: 2008-04-08 17:33:08 +0200 (Tue, 08 Apr 2008) New Revision: 2627 Modified: trunk/varnish-cache/lib/libvcl/vcc_token.c Log: Wrap long line Modified: trunk/varnish-cache/lib/libvcl/vcc_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_token.c 2008-04-03 17:15:58 UTC (rev 2626) +++ trunk/varnish-cache/lib/libvcl/vcc_token.c 2008-04-08 15:33:08 UTC (rev 2627) @@ -314,7 +314,8 @@ if (q < sp->e) continue; vcc_AddToken(tl, EOI, p, p + 2); - vsb_printf(tl->sb, "Unterminated /* ... */ comment, starting at\n"); + vsb_printf(tl->sb, + "Unterminated /* ... */ comment, starting at\n"); vcc_ErrWhere(tl, tl->t); return; } From phk at projects.linpro.no Tue Apr 8 15:33:31 2008 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 8 Apr 2008 17:33:31 +0200 (CEST) Subject: r2628 - trunk/varnish-cache/bin/varnishd Message-ID: <20080408153331.2492A1EC0E8@projects.linpro.no> Author: phk Date: 2008-04-08 17:33:30 +0200 (Tue, 08 Apr 2008) New Revision: 2628 Modified: trunk/varnish-cache/bin/varnishd/storage_file.c Log: Remove trailing blank from error message Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2008-04-08 15:33:08 UTC (rev 2627) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2008-04-08 15:33:30 UTC (rev 2628) @@ -174,7 +174,7 @@ if (q != NULL) { fprintf(stderr, - "Error: (-sfile) size \"%s\": %s \n", size, q); + "Error: (-sfile) size \"%s\": %s\n", size, q); exit (2); } } From phk at projects.linpro.no Tue Apr 8 15:34:57 2008 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 8 Apr 2008 17:34:57 +0200 (CEST) Subject: r2629 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20080408153457.645EB1EC0E1@projects.linpro.no> Author: phk Date: 2008-04-08 17:34:57 +0200 (Tue, 08 Apr 2008) New Revision: 2629 Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c trunk/varnish-cache/include/stat_field.h Log: Polish up the malloc allocator to make it more useful for test and debugging: Implement an upper limit for allocated memory, specify as: -smalloc,40M Give the malloc allocator its own stats counters so we can see deeply into what it does. Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2008-04-08 15:33:30 UTC (rev 2628) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2008-04-08 15:34:57 UTC (rev 2629) @@ -35,14 +35,20 @@ #include +#include #include +#include #include "shmlog.h" #include "cache.h" #include "stevedore.h" +static size_t sma_max = SIZE_T_MAX; +static MTX sma_mtx; + struct sma { struct storage s; + size_t sz; }; static struct storage * @@ -50,10 +56,24 @@ { struct sma *sma; - VSL_stats->sm_nreq++; + LOCK(&sma_mtx); + VSL_stats->sma_nreq++; + if (VSL_stats->sma_nbytes + size > sma_max) + size = 0; + else { + VSL_stats->sma_nobj++; + VSL_stats->sma_nbytes += size; + VSL_stats->sma_balloc += size; + } + UNLOCK(&sma_mtx); + + if (size == 0) + return (NULL); + sma = calloc(sizeof *sma, 1); if (sma == NULL) return (NULL); + sma->sz = size; sma->s.priv = sma; sma->s.ptr = malloc(size); XXXAN(sma->s.ptr); @@ -62,8 +82,6 @@ sma->s.fd = -1; sma->s.stevedore = st; sma->s.magic = STORAGE_MAGIC; - VSL_stats->sm_nobj++; - VSL_stats->sm_balloc += sma->s.space; return (&sma->s); } @@ -74,8 +92,12 @@ CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC); sma = s->priv; - VSL_stats->sm_nobj--; - VSL_stats->sm_balloc -= sma->s.space; + assert(sma->sz == sma->s.space); + LOCK(&sma_mtx); + VSL_stats->sma_nobj--; + VSL_stats->sma_nbytes -= sma->sz; + VSL_stats->sma_bfree += sma->sz; + UNLOCK(&sma_mtx); free(sma->s.ptr); free(sma); } @@ -88,16 +110,52 @@ CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC); sma = s->priv; + assert(sma->sz == sma->s.space); if ((p = realloc(sma->s.ptr, size)) != NULL) { - VSL_stats->sm_balloc -= sma->s.space; + LOCK(&sma_mtx); + VSL_stats->sma_nbytes -= (sma->sz - size); + VSL_stats->sma_bfree += sma->sz - size; + sma->sz = size; + UNLOCK(&sma_mtx); sma->s.ptr = p; sma->s.space = size; - VSL_stats->sm_balloc += sma->s.space; } } +static void +sma_init(struct stevedore *parent, const char *spec) +{ + const char *e; + uintmax_t u; + + (void)parent; + if (spec != NULL && *spec != '\0') { + e = str2bytes(spec, &u, 0); + if (e != NULL) { + fprintf(stderr, + "Error: (-smalloc) size \"%s\": %s\n", spec, e); + exit(2); + } + if ((u != (uintmax_t)(size_t)u)) { + fprintf(stderr, + "Error: (-smalloc) size \"%s\": too big\n", spec); + exit(2); + } + sma_max = u; + } +} + +static void +sma_open(const struct stevedore *st) +{ + (void)st; + AZ(pthread_mutex_init(&sma_mtx, NULL)); +} + struct stevedore sma_stevedore = { .name = "malloc", + .init = sma_init, + .open = sma_open, .alloc = sma_alloc, .free = sma_free, .trim = sma_trim, Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2008-04-08 15:33:30 UTC (rev 2628) +++ trunk/varnish-cache/include/stat_field.h 2008-04-08 15:34:57 UTC (rev 2629) @@ -97,4 +97,10 @@ MAC_STAT(sm_balloc, uint64_t, 'i', "bytes allocated") MAC_STAT(sm_bfree, uint64_t, 'i', "bytes free") +MAC_STAT(sma_nreq, uint64_t, 'a', "SMA allocator requests") +MAC_STAT(sma_nobj, uint64_t, 'i', "SMA outstanding allocations") +MAC_STAT(sma_nbytes, uint64_t, 'i', "SMA outstanding bytes") +MAC_STAT(sma_balloc, uint64_t, 'i', "SMA bytes allocated") +MAC_STAT(sma_bfree, uint64_t, 'i', "SMA bytes free") + MAC_STAT(backend_req, uint64_t, 'a', "Backend requests made") From anders at fupp.net Sat Apr 12 23:04:03 2008 From: anders at fupp.net (Anders Nordby) Date: Sun, 13 Apr 2008 01:04:03 +0200 Subject: r2614 - trunk/varnish-cache/bin/varnishd In-Reply-To: <20080331114921.12EA01EC0D9@projects.linpro.no> References: <20080331114921.12EA01EC0D9@projects.linpro.no> Message-ID: <20080412230403.GA2412@fupp.net> Hi, I mentioned to des previously that by going from trunk 2605 to 2614, my load and level of context switches per second grew a lot (during peaks), and sent a couple of graphs showing that. I've now tracked the problem down to this particular patch. I can run with the latest trunk (2629), and by removing this changeset, my load and context switches graphs are like before. Could you and/or Arjan please take look at this change again please. Running with or without -s malloc (which I've also been playing with lately) does not seem to make a difference, only applying/removing this changeset does. PS2: I recently heard des' talk on "Light My Fire". Is there any chance it could catch things like this, changes that give a distinct and negative impact on performance? Bye, Anders. On Mon, Mar 31, 2008 at 01:49:21PM +0200, phk at projects.linpro.no wrote: > Author: phk > Date: 2008-03-31 13:49:20 +0200 (Mon, 31 Mar 2008) > New Revision: 2614 > > Modified: > trunk/varnish-cache/bin/varnishd/cache_hash.c > Log: > Fix a memoryleak relating to hash strings, found by: Arjan (noosius). > > > > Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c > =================================================================== > --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2008-03-31 11:47:15 UTC (rev 2613) > +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2008-03-31 11:49:20 UTC (rev 2614) > @@ -383,6 +383,7 @@ > assert(VTAILQ_EMPTY(&oh->objects)); > MTX_DESTROY(&oh->mtx); > VSL_stats->n_objecthead--; > + free(oh->hash); > FREE_OBJ(oh); > } > > > _______________________________________________ > varnish-commit mailing list > varnish-commit at projects.linpro.no > http://projects.linpro.no/mailman/listinfo/varnish-commit -- Anders. From anders at fupp.net Sun Apr 13 20:15:05 2008 From: anders at fupp.net (Anders Nordby) Date: Sun, 13 Apr 2008 22:15:05 +0200 Subject: r2614 - trunk/varnish-cache/bin/varnishd In-Reply-To: <20080412230403.GA2412@fupp.net> References: <20080331114921.12EA01EC0D9@projects.linpro.no> <20080412230403.GA2412@fupp.net> Message-ID: <20080413201505.GA44129@fupp.net> Hi, This evening I got a high peaks for load and context switches in my graphs again, while running with 2629 (and changeset 2614 removed). :-( So I'm not sure how much impact 2614 has, but I'd still like for it to be checked. Checking mutex contests while these peaks are going on I didn't see much, but there sure is a lot of mutex operations going on (catched this in just a few seconds): 27837.73 Debug MTX_LOCK(hcl_lookup,hash_classic.c,150,&hp->mtx) 27651.87 Debug MTX_UNLOCK(SES_Charge,cache_session.c,253,&stat_mtx) 27649.91 Debug MTX_UNLOCK(SES_Charge,cache_session.c,238,&sp->srcaddr->sah- 27643.01 Debug MTX_LOCK(SES_Charge,cache_session.c,235,&sp->srcaddr->sah->m 27561.59 Debug MTX_UNLOCK(HSH_Deref,cache_hash.c,360,&oh->mtx) 27558.65 Debug MTX_LOCK(HSH_Deref,cache_hash.c,351,&oh->mtx) 27555.72 Debug MTX_LOCK(HSH_Lookup,cache_hash.c,196,&oh->mtx) 27499.97 Debug MTX_UNLOCK(WRK_QueueSession,cache_pool.c,294,&qp->mtx) 27499.97 Debug MTX_UNLOCK(wrk_thread,cache_pool.c,243,&qp->mtx) 27423.36 Debug MTX_LOCK(WRK_QueueSession,cache_pool.c,288,&qp->mtx) 27325.80 Debug MTX_LOCK(wrk_thread,cache_pool.c,240,&qp->mtx) 27276.70 Debug MTX_UNLOCK(HSH_Lookup,cache_hash.c,240,&oh->mtx) 27276.70 Debug MTX_UNLOCK(hcl_deref,hash_classic.c,238,&hp->mtx) 27276.70 Debug MTX_UNLOCK(hcl_lookup,hash_classic.c,168,&hp->mtx) 27274.73 Debug MTX_LOCK(hcl_deref,hash_classic.c,233,&hp->mtx) 27144.03 Debug MTX_LOCK(SES_Charge,cache_session.c,245,&stat_mtx) 4813.93 Debug MTX_UNLOCK(EXP_Touch,cache_expire.c,215,&exp_mtx) 4813.93 Debug MTX_TRYLOCK(EXP_Touch,cache_expire.c,206,&exp_mtx) = 0 4588.45 Debug MTX_LOCK(SES_Delete,cache_session.c,353,&ses_mem_mtx) 4588.45 Debug MTX_UNLOCK(SES_Delete,cache_session.c,355,&ses_mem_mtx) 4587.48 Debug MTX_LOCK(ses_relsrcaddr,cache_session.c,201,&ch->mtx) 4587.48 Debug MTX_UNLOCK(ses_relsrcaddr,cache_session.c,207,&ch->mtx) 4304.97 Debug MTX_LOCK(SES_RefSrcAddr,cache_session.c,143,&ch->mtx) 3531.52 Debug MTX_UNLOCK(SES_RefSrcAddr,cache_session.c,158,&ch->mtx) Anything to do about this? A 1-second ktrace from the same time can be downloaded from: http://anders.fupp.net/varnish/varnishd-ktrace-2008-04-13_22-05-27.out.gz I am running FreeBSD 7.0-RELEASE/amd64. The number of hits per second is only around 1200, but I have 3 million objects in the cache now. Bye, Anders. On Sun, Apr 13, 2008 at 01:04:03AM +0200, Anders Nordby wrote: > I mentioned to des previously that by going from trunk 2605 to 2614, my > load and level of context switches per second grew a lot (during peaks), > and sent a couple of graphs showing that. > > I've now tracked the problem down to this particular patch. I can run > with the latest trunk (2629), and by removing this changeset, my load > and context switches graphs are like before. > > Could you and/or Arjan please take look at this change again please. > Running with or without -s malloc (which I've also been playing with > lately) does not seem to make a difference, only applying/removing this > changeset does. > > PS2: I recently heard des' talk on "Light My Fire". Is there any chance > it could catch things like this, changes that give a distinct and > negative impact on performance? > > Bye, > Anders. > > On Mon, Mar 31, 2008 at 01:49:21PM +0200, phk at projects.linpro.no wrote: > > Author: phk > > Date: 2008-03-31 13:49:20 +0200 (Mon, 31 Mar 2008) > > New Revision: 2614 > > > > Modified: > > trunk/varnish-cache/bin/varnishd/cache_hash.c > > Log: > > Fix a memoryleak relating to hash strings, found by: Arjan (noosius). > > > > > > > > Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c > > =================================================================== > > --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2008-03-31 11:47:15 UTC (rev 2613) > > +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2008-03-31 11:49:20 UTC (rev 2614) > > @@ -383,6 +383,7 @@ > > assert(VTAILQ_EMPTY(&oh->objects)); > > MTX_DESTROY(&oh->mtx); > > VSL_stats->n_objecthead--; > > + free(oh->hash); > > FREE_OBJ(oh); > > } > > > > > > _______________________________________________ > > varnish-commit mailing list > > varnish-commit at projects.linpro.no > > http://projects.linpro.no/mailman/listinfo/varnish-commit > > > -- > Anders. > _______________________________________________ > varnish-dev mailing list > varnish-dev at projects.linpro.no > http://projects.linpro.no/mailman/listinfo/varnish-dev -- Anders. From phk at projects.linpro.no Wed Apr 16 07:58:43 2008 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 16 Apr 2008 09:58:43 +0200 (CEST) Subject: r2630 - trunk/varnish-cache/bin/varnishd Message-ID: <20080416075843.B2BE21EC0E8@projects.linpro.no> Author: phk Date: 2008-04-16 09:58:43 +0200 (Wed, 16 Apr 2008) New Revision: 2630 Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c Log: SIZE_MAX is not spelled SIZE_T_MAX in polite company. Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2008-04-08 15:34:57 UTC (rev 2629) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2008-04-16 07:58:43 UTC (rev 2630) @@ -43,7 +43,7 @@ #include "cache.h" #include "stevedore.h" -static size_t sma_max = SIZE_T_MAX; +static size_t sma_max = SIZE_MAX; static MTX sma_mtx; struct sma { From phk at projects.linpro.no Thu Apr 17 20:08:49 2008 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 17 Apr 2008 22:08:49 +0200 (CEST) Subject: r2631 - trunk/varnish-cache/bin/varnishd Message-ID: <20080417200849.3D13D1EC0D6@projects.linpro.no> Author: phk Date: 2008-04-17 22:08:49 +0200 (Thu, 17 Apr 2008) New Revision: 2631 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: Missing handling of sp->restarts and sp->director. Spotted by: "postwait" Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2008-04-16 07:58:43 UTC (rev 2630) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2008-04-17 20:08:49 UTC (rev 2631) @@ -170,6 +170,7 @@ } sp->director = NULL; + sp->restarts = 0; sp->backend = NULL; /* * XXX: we may want to leave this * behind to hint directors ? @@ -205,6 +206,7 @@ AZ(sp->obj); AZ(sp->bereq); sp->director = NULL; + sp->restarts = 0; sp->backend = NULL; /* * XXX: we may want to leave this * behind to hint directors ? @@ -364,6 +366,7 @@ if (sp->handling == VCL_RET_ERROR) sp->step = STP_ERROR; else { + sp->director = NULL; sp->restarts++; sp->step = STP_RECV; } From phk at projects.linpro.no Mon Apr 21 06:33:37 2008 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 21 Apr 2008 08:33:37 +0200 (CEST) Subject: r2632 - trunk/varnish-cache/bin/varnishd Message-ID: <20080421063338.0040F1EC0FF@projects.linpro.no> Author: phk Date: 2008-04-21 08:33:37 +0200 (Mon, 21 Apr 2008) New Revision: 2632 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c trunk/varnish-cache/bin/varnishd/cache_acceptor.h Log: Make it possible for the acceptor to provide a method by which sessions are passed to it. If no method is provided, we fall back to vca_pipes. closes #227 Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2008-04-17 20:08:49 UTC (rev 2631) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2008-04-21 06:33:37 UTC (rev 2632) @@ -270,7 +270,10 @@ AZ(sp->obj); AZ(sp->vcl); assert(sp->fd >= 0); - assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp)); + if (vca_act->pass == NULL) + assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp)); + else + vca_act->pass(sp); } @@ -290,7 +293,8 @@ fprintf(stderr, "No acceptor in program\n"); exit (2); } - AZ(pipe(vca_pipes)); + if (vca_act->pass == NULL) + AZ(pipe(vca_pipes)); vca_act->init(); AZ(pthread_create(&vca_thread_acct, NULL, vca_acct, NULL)); VSL(SLT_Debug, 0, "Acceptor is %s", vca_act->name); Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_acceptor.h 2008-04-17 20:08:49 UTC (rev 2631) +++ trunk/varnish-cache/bin/varnishd/cache_acceptor.h 2008-04-21 06:33:37 UTC (rev 2632) @@ -32,10 +32,12 @@ struct sess; typedef void acceptor_init_f(void); +typedef void acceptor_pass_f(struct sess *); struct acceptor { const char *name; acceptor_init_f *init; + acceptor_pass_f *pass; }; #if defined(HAVE_EPOLL_CTL) From phk at projects.linpro.no Mon Apr 21 07:01:20 2008 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 21 Apr 2008 09:01:20 +0200 (CEST) Subject: r2633 - trunk/varnish-cache/bin/varnishd Message-ID: <20080421070120.B0F391EC0E6@projects.linpro.no> Author: phk Date: 2008-04-21 09:01:20 +0200 (Mon, 21 Apr 2008) New Revision: 2633 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Remove an assert which is no longer true in the case of a non-headered (not-RFC2616 compliant) request. Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2008-04-21 06:33:37 UTC (rev 2632) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2008-04-21 07:01:20 UTC (rev 2633) @@ -372,7 +372,6 @@ hp->nhd = HTTP_HDR_FIRST; hp->conds = 0; r = NULL; /* For FlexeLint */ - assert(p <= t.e); /* http_header_complete() guarantees this */ for (; p < t.e; p = r) { /* XXX: handle continuation lines */ q = strchr(p, '\n'); From ssm at projects.linpro.no Thu Apr 24 06:37:02 2008 From: ssm at projects.linpro.no (ssm at projects.linpro.no) Date: Thu, 24 Apr 2008 08:37:02 +0200 (CEST) Subject: r2634 - trunk/varnish-cache/etc Message-ID: <20080424063702.290831EC106@projects.linpro.no> Author: ssm Date: 2008-04-24 08:37:01 +0200 (Thu, 24 Apr 2008) New Revision: 2634 Modified: trunk/varnish-cache/etc/zope-plone.vcl Log: Example VCL: Cleanup and update for varnish version >= 1.2 Modified: trunk/varnish-cache/etc/zope-plone.vcl =================================================================== --- trunk/varnish-cache/etc/zope-plone.vcl 2008-04-21 07:01:20 UTC (rev 2633) +++ trunk/varnish-cache/etc/zope-plone.vcl 2008-04-24 06:37:01 UTC (rev 2634) @@ -1,13 +1,18 @@ # -# This is a basic VCL configuration file for varnish. See the vcl(7) -# man page for details on VCL syntax and semantics. +# This is an example VCL configuration file for varnish, meant for the +# Plone CMS running within Zope. It defines a "default" backend for +# serving static content from a normal web server, and a "zope" +# backend for requests to the Zope CMS # -# $Id: default.vcl 1424 2007-05-15 19:38:56Z des $ +# See the vcl(7) man page for details on VCL syntax and semantics. # +# $Id$ +# # Default backend definition. Set this to point to your content # server. +# Default backend is the Zope CMS backend default { set backend.host = "127.0.0.1"; set backend.port = "9673"; @@ -19,29 +24,50 @@ } sub vcl_recv { - if (req.request != "GET" && req.request != "HEAD") { - # PURGE request if zope asks nicely - if (req.request == "PURGE") { - if (!client.ip ~ purge) { - error 405 "Not allowed."; - } - lookup; - } - pipe; - } - if (req.http.Expect) { - pipe; - } - if (req.http.Authenticate || req.http.Authorization) { - pass; - } - # We only care about the "__ac.*" cookies, used for authentication - if (req.http.Cookie && req.http.Cookie ~ "__ac(|_(name|password|persistent))=") { - pass; - } - lookup; -} + # Normalize host headers, and do rewriting for the zope sites. Reject + # requests for unknown hosts + if (req.http.host ~ "(www.)?example.com") { + set req.http.host = "example.com"; + set req.url = "/VirtualHostBase/http/example.com:80/example.com/VirtualHostRoot" req.url; + } elsif (req.http.host ~ "(www.)?example.org") { + set req.http.host = "example.org"; + set req.url = "/VirtualHostBase/http/example.org:80/example.org/VirtualHostRoot" req.url; + } else { + error 404 "Unknown virtual host."; + } + + # Handle special requests + if (req.request != "GET" && req.request != "HEAD") { + + # POST - Logins and edits + if (req.request == "POST") { + pass; + } + + # PURGE - The CacheFu product can invalidate updated URLs + if (req.request == "PURGE") { + if (!client.ip ~ purge) { + error 405 "Not allowed."; + } + lookup; + } + } + + # Don't cache authenticated requests + if (req.http.Cookie && req.http.Cookie ~ "__ac(|_(name|password|persistent))=") { + + # Force lookup of specific urls unlikely to need protection + if (req.url ~ "\.(js|css)") { + remove req.http.cookie; + lookup; + } + pass; + } + + # The default vcl_recv is used from here. + } + # Do the PURGE thing sub vcl_hit { if (req.request == "PURGE") { @@ -55,83 +81,11 @@ } } -# Enforce a minimum TTL, since we PURGE changed objects actively from Zope. +# Enforce a minimum TTL, since we can PURGE changed objects actively +# from Zope by using the CacheFu product + sub vcl_fetch { if (obj.ttl < 3600s) { set obj.ttl = 3600s; } } - -# Below is a commented-out copy of the default VCL logic. If you -# redefine any of these subroutines, the built-in logic will be -# appended to your code. - -## Called when a client request is received -# -#sub vcl_recv { -# if (req.request != "GET" && req.request != "HEAD") { -# pipe; -# } -# if (req.http.Expect) { -# pipe; -# } -# if (req.http.Authenticate || req.http.Cookie) { -# pass; -# } -# lookup; -#} -# -## Called when entering pipe mode -# -#sub vcl_pipe { -# pipe; -#} -# -## Called when entering pass mode -# -#sub vcl_pass { -# pass; -#} -# -## Called when entering an object into the cache -# -#sub vcl_hash { -# hash; -#} -# -## Called when the requested object was found in the cache -# -#sub vcl_hit { -# if (!obj.cacheable) { -# pass; -# } -# deliver; -#} -# -## Called when the requested object was not found in the cache -# -#sub vcl_miss { -# fetch; -#} -# -## Called when the requested object has been retrieved from the -## backend, or the request to the backend has failed -# -#sub vcl_fetch { -# if (!obj.valid) { -# error; -# } -# if (!obj.cacheable) { -# pass; -# } -# if (resp.http.Set-Cookie) { -# pass; -# } -# insert; -#} -# -## Called when an object nears its expiry time -# -#sub vcl_timeout { -# discard; -#} Property changes on: trunk/varnish-cache/etc/zope-plone.vcl ___________________________________________________________________ Name: svn:keywords + id From ssm at projects.linpro.no Thu Apr 24 06:46:39 2008 From: ssm at projects.linpro.no (ssm at projects.linpro.no) Date: Thu, 24 Apr 2008 08:46:39 +0200 (CEST) Subject: r2635 - in branches/1.2: . etc Message-ID: <20080424064639.887C51EC0EF@projects.linpro.no> Author: ssm Date: 2008-04-24 08:46:39 +0200 (Thu, 24 Apr 2008) New Revision: 2635 Modified: branches/1.2/ branches/1.2/etc/zope-plone.vcl Log: Merged revisions 2634 via svnmerge from svn+ssh://projects.linpro.no/svn/varnish/trunk/varnish-cache ........ r2634 | ssm | 2008-04-24 08:37:01 +0200 (Thu, 24 Apr 2008) | 2 lines Example VCL: Cleanup and update for varnish version >= 1.2 ........ Property changes on: branches/1.2 ___________________________________________________________________ Name: svnmerge-integrated - /trunk/varnish-cache:1-2101,2104-2107,2115-2120,2122-2130,2133,2151,2153-2154,2157,2161-2162,2166-2168,2173,2175-2176,2180-2184,2186-2193,2206,2208,2210-2215,2220,2222-2232,2243,2246-2255,2270-2271,2288-2291,2296-2297,2299,2304-2305,2307-2309,2311-2312,2315,2317-2319,2321-2327,2329-2330,2337,2357-2359,2361-2364,2366,2374-2382,2384-2386,2404,2414-2415,2426,2432-2434,2444-2445,2453-2461,2467,2492-2505,2520-2524,2545,2563-2579,2581-2583,2586,2588,2600,2608 + /trunk/varnish-cache:1-2101,2104-2107,2115-2120,2122-2130,2133,2151,2153-2154,2157,2161-2162,2166-2168,2173,2175-2176,2180-2184,2186-2193,2206,2208,2210-2215,2220,2222-2232,2243,2246-2255,2270-2271,2288-2291,2296-2297,2299,2304-2305,2307-2309,2311-2312,2315,2317-2319,2321-2327,2329-2330,2337,2357-2359,2361-2364,2366,2374-2382,2384-2386,2404,2414-2415,2426,2432-2434,2444-2445,2453-2461,2467,2492-2505,2520-2524,2545,2563-2579,2581-2583,2586,2588,2600,2608,2634 Modified: branches/1.2/etc/zope-plone.vcl =================================================================== --- branches/1.2/etc/zope-plone.vcl 2008-04-24 06:37:01 UTC (rev 2634) +++ branches/1.2/etc/zope-plone.vcl 2008-04-24 06:46:39 UTC (rev 2635) @@ -1,13 +1,18 @@ # -# This is a basic VCL configuration file for varnish. See the vcl(7) -# man page for details on VCL syntax and semantics. +# This is an example VCL configuration file for varnish, meant for the +# Plone CMS running within Zope. It defines a "default" backend for +# serving static content from a normal web server, and a "zope" +# backend for requests to the Zope CMS # -# $Id: default.vcl 1424 2007-05-15 19:38:56Z des $ +# See the vcl(7) man page for details on VCL syntax and semantics. # +# $Id$ +# # Default backend definition. Set this to point to your content # server. +# Default backend is the Zope CMS backend default { set backend.host = "127.0.0.1"; set backend.port = "9673"; @@ -19,29 +24,50 @@ } sub vcl_recv { - if (req.request != "GET" && req.request != "HEAD") { - # PURGE request if zope asks nicely - if (req.request == "PURGE") { - if (!client.ip ~ purge) { - error 405 "Not allowed."; - } - lookup; - } - pipe; - } - if (req.http.Expect) { - pipe; - } - if (req.http.Authenticate || req.http.Authorization) { - pass; - } - # We only care about the "__ac.*" cookies, used for authentication - if (req.http.Cookie && req.http.Cookie ~ "__ac(|_(name|password|persistent))=") { - pass; - } - lookup; -} + # Normalize host headers, and do rewriting for the zope sites. Reject + # requests for unknown hosts + if (req.http.host ~ "(www.)?example.com") { + set req.http.host = "example.com"; + set req.url = "/VirtualHostBase/http/example.com:80/example.com/VirtualHostRoot" req.url; + } elsif (req.http.host ~ "(www.)?example.org") { + set req.http.host = "example.org"; + set req.url = "/VirtualHostBase/http/example.org:80/example.org/VirtualHostRoot" req.url; + } else { + error 404 "Unknown virtual host."; + } + + # Handle special requests + if (req.request != "GET" && req.request != "HEAD") { + + # POST - Logins and edits + if (req.request == "POST") { + pass; + } + + # PURGE - The CacheFu product can invalidate updated URLs + if (req.request == "PURGE") { + if (!client.ip ~ purge) { + error 405 "Not allowed."; + } + lookup; + } + } + + # Don't cache authenticated requests + if (req.http.Cookie && req.http.Cookie ~ "__ac(|_(name|password|persistent))=") { + + # Force lookup of specific urls unlikely to need protection + if (req.url ~ "\.(js|css)") { + remove req.http.cookie; + lookup; + } + pass; + } + + # The default vcl_recv is used from here. + } + # Do the PURGE thing sub vcl_hit { if (req.request == "PURGE") { @@ -55,83 +81,11 @@ } } -# Enforce a minimum TTL, since we PURGE changed objects actively from Zope. +# Enforce a minimum TTL, since we can PURGE changed objects actively +# from Zope by using the CacheFu product + sub vcl_fetch { if (obj.ttl < 3600s) { set obj.ttl = 3600s; } } - -# Below is a commented-out copy of the default VCL logic. If you -# redefine any of these subroutines, the built-in logic will be -# appended to your code. - -## Called when a client request is received -# -#sub vcl_recv { -# if (req.request != "GET" && req.request != "HEAD") { -# pipe; -# } -# if (req.http.Expect) { -# pipe; -# } -# if (req.http.Authenticate || req.http.Cookie) { -# pass; -# } -# lookup; -#} -# -## Called when entering pipe mode -# -#sub vcl_pipe { -# pipe; -#} -# -## Called when entering pass mode -# -#sub vcl_pass { -# pass; -#} -# -## Called when entering an object into the cache -# -#sub vcl_hash { -# hash; -#} -# -## Called when the requested object was found in the cache -# -#sub vcl_hit { -# if (!obj.cacheable) { -# pass; -# } -# deliver; -#} -# -## Called when the requested object was not found in the cache -# -#sub vcl_miss { -# fetch; -#} -# -## Called when the requested object has been retrieved from the -## backend, or the request to the backend has failed -# -#sub vcl_fetch { -# if (!obj.valid) { -# error; -# } -# if (!obj.cacheable) { -# pass; -# } -# if (resp.http.Set-Cookie) { -# pass; -# } -# insert; -#} -# -## Called when an object nears its expiry time -# -#sub vcl_timeout { -# discard; -#} Property changes on: branches/1.2/etc/zope-plone.vcl ___________________________________________________________________ Name: svn:keywords + id