From phk at projects.linpro.no Thu Oct 1 08:58:06 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 1 Oct 2009 10:58:06 +0200 (CEST) Subject: r4266 - trunk/varnish-cache/bin/varnishd Message-ID: <20091001085806.021F51F7375@projects.linpro.no> Author: phk Date: 2009-10-01 10:58:05 +0200 (Thu, 01 Oct 2009) New Revision: 4266 Modified: trunk/varnish-cache/bin/varnishd/stevedore.c trunk/varnish-cache/bin/varnishd/stevedore.h trunk/varnish-cache/bin/varnishd/storage_file.c trunk/varnish-cache/bin/varnishd/storage_malloc.c trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Give the stevedores alloc method a flag that tells that we are allocating an object structure. Modified: trunk/varnish-cache/bin/varnishd/stevedore.c =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.c 2009-09-30 08:40:01 UTC (rev 4265) +++ trunk/varnish-cache/bin/varnishd/stevedore.c 2009-10-01 08:58:05 UTC (rev 4266) @@ -169,7 +169,7 @@ /* try to allocate from it */ AN(stv->alloc); - st = stv->alloc(stv, size); + st = stv->alloc(stv, size, 0); if (st != NULL) break; Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2009-09-30 08:40:01 UTC (rev 4265) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2009-10-01 08:58:05 UTC (rev 4266) @@ -36,7 +36,7 @@ typedef void storage_init_f(struct stevedore *, int ac, char * const *av); typedef void storage_open_f(const struct stevedore *); -typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); +typedef struct storage *storage_alloc_f(struct stevedore *, size_t size, int isobj); typedef struct object *storage_alloc_obj_f(struct stevedore *, size_t size, double ttl); typedef void storage_trim_f(struct storage *, size_t size); typedef void storage_free_f(struct storage *); Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2009-09-30 08:40:01 UTC (rev 4265) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2009-10-01 08:58:05 UTC (rev 4266) @@ -461,11 +461,12 @@ /*--------------------------------------------------------------------*/ static struct storage * -smf_alloc(struct stevedore *st, size_t size) +smf_alloc(struct stevedore *st, size_t size, int isobj) { struct smf *smf; struct smf_sc *sc = st->priv; + (void)isobj; assert(size > 0); size += (sc->pagesize - 1); size &= ~(sc->pagesize - 1); Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2009-09-30 08:40:01 UTC (rev 4265) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2009-10-01 08:58:05 UTC (rev 4266) @@ -53,10 +53,11 @@ }; static struct storage * -sma_alloc(struct stevedore *st, size_t size) +sma_alloc(struct stevedore *st, size_t size, int isobj) { struct sma *sma; + (void)isobj; Lck_Lock(&sma_mtx); VSL_stats->sma_nreq++; if (VSL_stats->sma_nbytes + size > sma_max) Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-09-30 08:40:01 UTC (rev 4265) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-01 08:58:05 UTC (rev 4266) @@ -1220,13 +1220,14 @@ */ static struct storage * -smp_alloc(struct stevedore *st, size_t size) +smp_alloc(struct stevedore *st, size_t size, int isobj) { struct smp_sc *sc; struct storage *ss; struct smp_seg *sg; size_t sz2; + (void)isobj; CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC); Lck_Lock(&sc->mtx); From phk at projects.linpro.no Thu Oct 1 09:12:09 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 1 Oct 2009 11:12:09 +0200 (CEST) Subject: r4267 - trunk/varnish-cache/bin/varnishd Message-ID: <20091001091209.762C31F7375@projects.linpro.no> Author: phk Date: 2009-10-01 11:12:09 +0200 (Thu, 01 Oct 2009) New Revision: 4267 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/stevedore.c trunk/varnish-cache/bin/varnishd/stevedore.h trunk/varnish-cache/bin/varnishd/storage_file.c trunk/varnish-cache/bin/varnishd/storage_malloc.c trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: OK, pass the objcore instead, we don't have the sess in the stevedore. Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-10-01 08:58:05 UTC (rev 4266) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-10-01 09:12:09 UTC (rev 4267) @@ -64,7 +64,7 @@ assert((uintmax_t)cl == cll); /* Protect against bogusly large values */ while (cl > 0) { - st = STV_alloc(sp, cl); + st = STV_alloc(sp, cl, NULL); VTAILQ_INSERT_TAIL(&sp->obj->store, st, list); sl = st->space; if (sl > cl) @@ -164,7 +164,7 @@ v = u; if (u < params->fetch_chunksize * 1024) v = params->fetch_chunksize * 1024; - st = STV_alloc(sp, v); + st = STV_alloc(sp, v, NULL); VTAILQ_INSERT_TAIL(&sp->obj->store, st, list); } v = st->space - st->len; @@ -247,7 +247,7 @@ if (v == 0) { if (st != NULL && fetchfrag > 0) dump_st(sp, st); - st = STV_alloc(sp, params->fetch_chunksize * 1024LL); + st = STV_alloc(sp, params->fetch_chunksize * 1024LL, NULL); VTAILQ_INSERT_TAIL(&sp->obj->store, st, list); p = st->ptr + st->len; v = st->space - st->len; Modified: trunk/varnish-cache/bin/varnishd/stevedore.c =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.c 2009-10-01 08:58:05 UTC (rev 4266) +++ trunk/varnish-cache/bin/varnishd/stevedore.c 2009-10-01 09:12:09 UTC (rev 4267) @@ -127,7 +127,7 @@ STV_InitObj(sp, o, l); return (o); } - st = STV_alloc(sp, sizeof *o + l); + st = STV_alloc(sp, sizeof *o + l, sp->objcore); XXXAN(st); xxxassert(st->space >= (sizeof *o + l)); @@ -143,7 +143,7 @@ /*********************************************************************/ struct storage * -STV_alloc(struct sess *sp, size_t size) +STV_alloc(struct sess *sp, size_t size, struct objcore *oc) { struct storage *st; struct stevedore *stv = NULL; @@ -169,7 +169,7 @@ /* try to allocate from it */ AN(stv->alloc); - st = stv->alloc(stv, size, 0); + st = stv->alloc(stv, size, oc); if (st != NULL) break; Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2009-10-01 08:58:05 UTC (rev 4266) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2009-10-01 09:12:09 UTC (rev 4267) @@ -33,10 +33,11 @@ struct sess; struct iovec; struct object; +struct objcore; typedef void storage_init_f(struct stevedore *, int ac, char * const *av); typedef void storage_open_f(const struct stevedore *); -typedef struct storage *storage_alloc_f(struct stevedore *, size_t size, int isobj); +typedef struct storage *storage_alloc_f(struct stevedore *, size_t size, struct objcore *); typedef struct object *storage_alloc_obj_f(struct stevedore *, size_t size, double ttl); typedef void storage_trim_f(struct storage *, size_t size); typedef void storage_free_f(struct storage *); @@ -66,7 +67,7 @@ }; struct object *STV_NewObject(struct sess *sp, unsigned len, double ttl); -struct storage *STV_alloc(struct sess *sp, size_t size); +struct storage *STV_alloc(struct sess *sp, size_t size, struct objcore *oc); void STV_trim(struct storage *st, size_t size); void STV_free(struct storage *st); void STV_add(const struct stevedore *stv, int ac, char * const *av); Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2009-10-01 08:58:05 UTC (rev 4266) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2009-10-01 09:12:09 UTC (rev 4267) @@ -461,12 +461,12 @@ /*--------------------------------------------------------------------*/ static struct storage * -smf_alloc(struct stevedore *st, size_t size, int isobj) +smf_alloc(struct stevedore *st, size_t size, struct objcore *oc) { struct smf *smf; struct smf_sc *sc = st->priv; - (void)isobj; + (void)oc; assert(size > 0); size += (sc->pagesize - 1); size &= ~(sc->pagesize - 1); Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_malloc.c 2009-10-01 08:58:05 UTC (rev 4266) +++ trunk/varnish-cache/bin/varnishd/storage_malloc.c 2009-10-01 09:12:09 UTC (rev 4267) @@ -53,11 +53,11 @@ }; static struct storage * -sma_alloc(struct stevedore *st, size_t size, int isobj) +sma_alloc(struct stevedore *st, size_t size, struct objcore *oc) { struct sma *sma; - (void)isobj; + (void)oc; Lck_Lock(&sma_mtx); VSL_stats->sma_nreq++; if (VSL_stats->sma_nbytes + size > sma_max) Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-01 08:58:05 UTC (rev 4266) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-01 09:12:09 UTC (rev 4267) @@ -1220,14 +1220,14 @@ */ static struct storage * -smp_alloc(struct stevedore *st, size_t size, int isobj) +smp_alloc(struct stevedore *st, size_t size, struct objcore *oc) { struct smp_sc *sc; struct storage *ss; struct smp_seg *sg; size_t sz2; - (void)isobj; + (void)oc; CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC); Lck_Lock(&sc->mtx); From phk at projects.linpro.no Thu Oct 1 11:10:32 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 1 Oct 2009 13:10:32 +0200 (CEST) Subject: r4268 - trunk/varnish-cache/bin/varnishd Message-ID: <20091001111032.253B11F72C3@projects.linpro.no> Author: phk Date: 2009-10-01 13:10:31 +0200 (Thu, 01 Oct 2009) New Revision: 4268 Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Elide empty segments when writing the segment list. Take due care to not confuse unloaded segments with empty segments in this respect. Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-01 09:12:09 UTC (rev 4267) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-01 11:10:31 UTC (rev 4268) @@ -98,7 +98,8 @@ struct smp_segment segment; /* Copy of on-disk desc. */ - uint32_t nalloc; /* How many live objects */ + uint32_t nalloc; /* How many alloc objects */ + uint32_t nfilled; /* How many live objects */ uint32_t nfixed; /* How many fixed objects */ /* Only for open segment */ @@ -590,6 +591,7 @@ struct smp_seg *sg; uint64_t length; + Lck_AssertHeld(&sc->mtx); smp_reset_sign(ctx); ss = SIGN_DATA(ctx); length = 0; @@ -598,7 +600,6 @@ assert(sg->offset + sg->length <= sc->mediasize); ss->offset = sg->offset; ss->length = sg->length; -// printf("SAVE_SEG %jx...%jx\n", ss->offset, ss->offset + ss->length); ss++; length += sizeof *ss; } @@ -609,7 +610,20 @@ static void smp_save_segs(struct smp_sc *sc) { + struct smp_seg *sg, *sg2; + Lck_AssertHeld(&sc->mtx); + + /* Elide any empty segments from the list before we write it */ + VTAILQ_FOREACH_SAFE(sg, &sc->segments, list, sg2) { + if (sg->nalloc > 0) + continue; + if (sg == sc->cur_seg) + continue; + VTAILQ_REMOVE(&sc->segments, sg, list); + // XXX: free segment + } + smp_save_seg(sc, &sc->seg1); smp_save_seg(sc, &sc->seg2); } @@ -765,10 +779,16 @@ sg = o->objcore->smp_seg; CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC); + Lck_Lock(&sg->sc->mtx); o->smp_object->ttl = 0; assert(sg->nalloc > 0); + assert(sg->nfixed > 0); + sg->nalloc--; + o->smp_object = NULL; + sg->nfixed--; + Lck_Unlock(&sg->sc->mtx); /* XXX: check if seg is empty, or leave to thread ? */ } @@ -836,6 +856,8 @@ ss = ptr; so = (void*)(sc->ptr + ss->objlist); no = ss->nalloc; + /* Clear the bogus "hold" count */ + sg->nalloc = 0; for (;no > 0; so++,no--) { if (so->ttl < t_now) continue; @@ -946,6 +968,11 @@ sg->lru = LRU_Alloc(); sg->offset = ss->offset; sg->length = ss->length; + /* + * HACK: prevent save_segs from nuking segment until we have + * HACK: loaded it. + */ + sg->nalloc = 1; if (sg1 != NULL) { assert(sg1->offset != sg->offset); if (sg1->offset < sg->offset) @@ -986,6 +1013,7 @@ Lck_AssertHeld(&sc->mtx); ALLOC_OBJ(sg, SMP_SEG_MAGIC); AN(sg); + sg->sc = sc; sg->maxobj = sc->aim_nobj; sg->objs = malloc(sizeof *sg->objs * sg->maxobj); @@ -1032,9 +1060,6 @@ smp_append_sign(sg->ctx, &sg->segment, sizeof sg->segment); smp_sync_sign(sg->ctx); - /* Then add it to the segment list. */ - /* XXX: could be done cheaper with append ? */ - smp_save_segs(sc); /* Set up our allocation point */ sc->cur_seg = sg; @@ -1044,6 +1069,10 @@ SHA256_LEN; memcpy(sc->ptr + sg->next_addr, "HERE", 4); sc->objreserv = sizeof *sg->objs + SHA256_LEN; + + /* Then add it to the segment list. */ + /* XXX: could be done cheaper with append ? */ + smp_save_segs(sc); } /*-------------------------------------------------------------------- @@ -1062,6 +1091,7 @@ /* XXX: if segment is empty, delete instead */ + /* Copy the objects into the segment */ sz = sizeof *sg->objs * sg->nalloc; assert(sg->next_addr + sz <= sg->offset + sg->length); @@ -1109,7 +1139,7 @@ BAN_Deref(&sc->tailban); sc->tailban = NULL; printf("Silo completely loaded\n"); - while (1) + while (1) (void)sleep (1); NEEDLESS_RETURN(NULL); } @@ -1193,24 +1223,22 @@ CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(sp->obj->objstore, STORAGE_MAGIC); CHECK_OBJ_NOTNULL(sp->obj->objstore->stevedore, STEVEDORE_MAGIC); + CHECK_OBJ_NOTNULL(sp->obj->objcore->smp_seg, SMP_SEG_MAGIC); CAST_OBJ_NOTNULL(sc, sp->obj->objstore->priv, SMP_SC_MAGIC); sp->obj->objcore->flags |= OC_F_LRUDONTMOVE; Lck_Lock(&sc->mtx); - sg = sc->cur_seg; + sg = sp->obj->objcore->smp_seg; sc->objreserv += sizeof *so; - if (sg->nalloc >= sg->maxobj) { - smp_close_seg(sc, sc->cur_seg); - smp_new_seg(sc); - fprintf(stderr, "New Segment\n"); - sg = sc->cur_seg; - } - assert(sg->nalloc < sg->maxobj); + assert(sg->nalloc < sg->nfilled); so = &sg->objs[sg->nalloc++]; + sg->nfixed++; memcpy(so->hash, sp->obj->objcore->objhead->digest, DIGEST_LEN); so->ttl = sp->obj->ttl; so->ptr = sp->obj; so->ban = sp->obj->ban_t; + /* XXX: if segment is already closed, write sg->objs */ + sp->obj->smp_object = so; Lck_Unlock(&sc->mtx); } @@ -1229,7 +1257,6 @@ (void)oc; CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC); - Lck_Lock(&sc->mtx); sg = sc->cur_seg; @@ -1267,6 +1294,16 @@ sg->next_addr += size + sizeof *ss; assert(sg->next_addr <= sg->offset + sg->length); memcpy(sc->ptr + sg->next_addr, "HERE", 4); + if (oc != NULL) { + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + if (++sg->nfilled >= sg->maxobj) { + smp_close_seg(sc, sc->cur_seg); + smp_new_seg(sc); + fprintf(stderr, "New Segment\n"); + sg = sc->cur_seg; + } + oc->smp_seg = sg; + } Lck_Unlock(&sc->mtx); /* Grab and fill a storage structure */ From phk at projects.linpro.no Fri Oct 2 20:51:03 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 2 Oct 2009 22:51:03 +0200 (CEST) Subject: r4269 - trunk/varnish-cache/bin/varnishd Message-ID: <20091002205103.B19731F738E@projects.linpro.no> Author: phk Date: 2009-10-02 22:51:02 +0200 (Fri, 02 Oct 2009) New Revision: 4269 Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Nuke this optimization for now. Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-01 11:10:31 UTC (rev 4268) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-02 20:51:02 UTC (rev 4269) @@ -1253,7 +1253,6 @@ struct smp_sc *sc; struct storage *ss; struct smp_seg *sg; - size_t sz2; (void)oc; CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC); @@ -1262,37 +1261,21 @@ AN(sg->next_addr); - /* - * XXX: We do not have a mechanism for deciding which allocations - * XXX: have mandatory size and which can be chopped into smaller - * XXX: allocations. - * XXX: The primary unchoppable allocation is from HSH_NewObject() - * XXX: which needs an object + workspace. - */ - sz2 = sizeof (struct object) + 1024; - if (size < sz2) - sz2 = size; - /* If the segment is full, get a new one right away */ - if (sg->next_addr + sizeof *ss + sz2 + sc->objreserv > + if (sg->next_addr + sizeof *ss + size + sc->objreserv > sg->offset + sg->length) { smp_close_seg(sc, sc->cur_seg); smp_new_seg(sc); sg = sc->cur_seg; } - assert (sg->next_addr + sizeof *ss + sz2 + sc->objreserv <= + assert (sg->next_addr + sizeof *ss + size + sc->objreserv <= sg->offset + sg->length); - if (sg->next_addr + sizeof *ss + size + sc->objreserv > - sg->offset + sg->length) - size = (sg->offset + sg->length) - - (sg->next_addr + sizeof *ss + sc->objreserv); - ss = (void *)(sc->ptr + sg->next_addr); sg->next_addr += size + sizeof *ss; - assert(sg->next_addr <= sg->offset + sg->length); + assert(sg->next_addr + sc->objreserv <= sg->offset + sg->length); memcpy(sc->ptr + sg->next_addr, "HERE", 4); if (oc != NULL) { CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); From phk at projects.linpro.no Sun Oct 4 08:51:36 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 4 Oct 2009 10:51:36 +0200 (CEST) Subject: r4270 - trunk/varnish-cache/bin/varnishd Message-ID: <20091004085136.AD3F01F737A@projects.linpro.no> Author: phk Date: 2009-10-04 10:51:36 +0200 (Sun, 04 Oct 2009) New Revision: 4270 Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Correct a couple of brain-o's in the allocation logic: Reserve space for the objects entry in the index, when the object is allocated. Allow smaller than requested allocations for non-object allocations. Attempt allocation from three segments, then give up. Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-02 20:51:02 UTC (rev 4269) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-04 08:51:36 UTC (rev 4270) @@ -1068,7 +1068,7 @@ sizeof (struct smp_segment) + SHA256_LEN; memcpy(sc->ptr + sg->next_addr, "HERE", 4); - sc->objreserv = sizeof *sg->objs + SHA256_LEN; + sc->objreserv = 0; /* Then add it to the segment list. */ /* XXX: could be done cheaper with append ? */ @@ -1090,11 +1090,10 @@ /* XXX: if segment is empty, delete instead */ - - /* Copy the objects into the segment */ sz = sizeof *sg->objs * sg->nalloc; - assert(sg->next_addr + sz <= sg->offset + sg->length); + /* XXX: Seen by sky 2009-10-02: */ + assert(sg->next_addr + sz <= sg->offset + sg->length); memcpy(sc->ptr + sg->next_addr, sg->objs, sz); @@ -1229,7 +1228,6 @@ sp->obj->objcore->flags |= OC_F_LRUDONTMOVE; Lck_Lock(&sc->mtx); sg = sp->obj->objcore->smp_seg; - sc->objreserv += sizeof *so; assert(sg->nalloc < sg->nfilled); so = &sg->objs[sg->nalloc++]; sg->nfixed++; @@ -1238,7 +1236,10 @@ so->ptr = sp->obj; so->ban = sp->obj->ban_t; /* XXX: if segment is already closed, write sg->objs */ + + /* XXX: Uhm, this should not be the temp version, should it ? */ sp->obj->smp_object = so; + Lck_Unlock(&sc->mtx); } @@ -1247,53 +1248,90 @@ * Allocate a bite */ +static uint64_t +smp_spaceleft(const struct smp_seg *sg) +{ + + assert(sg->next_addr <= (sg->offset + sg->length)); + return ((sg->offset + sg->length) - sg->next_addr); +} + static struct storage * smp_alloc(struct stevedore *st, size_t size, struct objcore *oc) { struct smp_sc *sc; struct storage *ss; struct smp_seg *sg; + uint64_t needed, left, overhead; + void *allocation; + unsigned tries; - (void)oc; CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC); Lck_Lock(&sc->mtx); - sg = sc->cur_seg; - AN(sg->next_addr); + for (tries = 0; tries < 3; tries++) { + sg = sc->cur_seg; + CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC); - /* If the segment is full, get a new one right away */ - if (sg->next_addr + sizeof *ss + size + sc->objreserv > - sg->offset + sg->length) { + overhead = sizeof *ss + sc->objreserv; + needed = overhead + size; + left = smp_spaceleft(sg); + + if (oc != NULL) { + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + /* Objects also need an entry in the index */ + needed += sizeof (struct smp_object); + } else if (needed > left && (overhead + 4096) < left) { + /* + * Non-objects can be trimmed down to fit what we + * have to offer (think: DVD image), but we do not + * want to trim down to trivial sizes. + */ + size = left - overhead; + needed = overhead + size; + assert(needed <= left); + } + + /* If there is space, fine */ + if (needed < left && (oc == NULL || sg->nfilled < sg->maxobj)) + break; + smp_close_seg(sc, sc->cur_seg); smp_new_seg(sc); - sg = sc->cur_seg; } - assert (sg->next_addr + sizeof *ss + size + sc->objreserv <= - sg->offset + sg->length); + if (needed > smp_spaceleft(sg)) + return (NULL); + assert(needed <= smp_spaceleft(sg)); + + /* Grab for storage struct */ ss = (void *)(sc->ptr + sg->next_addr); + sg->next_addr +=sizeof *ss; - sg->next_addr += size + sizeof *ss; - assert(sg->next_addr + sc->objreserv <= sg->offset + sg->length); + /* Grab for allocated space */ + allocation = sc->ptr + sg->next_addr; + sg->next_addr += size; + + /* Paint our marker */ memcpy(sc->ptr + sg->next_addr, "HERE", 4); + if (oc != NULL) { - CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - if (++sg->nfilled >= sg->maxobj) { - smp_close_seg(sc, sc->cur_seg); - smp_new_seg(sc); - fprintf(stderr, "New Segment\n"); - sg = sc->cur_seg; - } + /* Make reservation in the index */ + assert(sg->nfilled < sg->maxobj); + sg->nfilled++; + sc->objreserv += sizeof (struct smp_object); + assert(sc->objreserv <= smp_spaceleft(sg)); oc->smp_seg = sg; } + Lck_Unlock(&sc->mtx); - /* Grab and fill a storage structure */ + /* Fill the storage structure */ memset(ss, 0, sizeof *ss); ss->magic = STORAGE_MAGIC; + ss->ptr = allocation; ss->space = size; - ss->ptr = (void *)(ss + 1); ss->priv = sc; ss->stevedore = st; ss->fd = sc->fd; From phk at projects.linpro.no Sun Oct 4 11:47:09 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 4 Oct 2009 13:47:09 +0200 (CEST) Subject: r4271 - trunk/varnish-cache/bin/varnishtest/tests Message-ID: <20091004114709.D48792855A@projects.linpro.no> Author: phk Date: 2009-10-04 13:47:09 +0200 (Sun, 04 Oct 2009) New Revision: 4271 Modified: trunk/varnish-cache/bin/varnishtest/tests/p00000.vtc Log: Do a start-stop test first, to see basic silo open/close work. Modified: trunk/varnish-cache/bin/varnishtest/tests/p00000.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/p00000.vtc 2009-10-04 08:51:36 UTC (rev 4270) +++ trunk/varnish-cache/bin/varnishtest/tests/p00000.vtc 2009-10-04 11:47:09 UTC (rev 4271) @@ -13,6 +13,10 @@ -arg "-pdiag_bitmap=0x20000" \ -arg "-spersistent,/tmp/__v1/_.per,10m" -vcl+backend { } -start +varnish v1 -stop + +varnish v1 -start + client c1 { txreq -url "/" rxresp From phk at projects.linpro.no Sun Oct 4 11:52:13 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 4 Oct 2009 13:52:13 +0200 (CEST) Subject: r4272 - trunk/varnish-cache/bin/varnishd Message-ID: <20091004115213.24B4D28563@projects.linpro.no> Author: phk Date: 2009-10-04 13:52:12 +0200 (Sun, 04 Oct 2009) New Revision: 4272 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Close the ttl/ban update race for good: Use an index into the segments object index instead of a pointer, to avoid dealing with the temporary buffer used in the active segment. Correctly count the various kinds of states objects can be in. Only check the signature id inside the signature id field. Don't load segments if the object index was not committed. Reuse the temporary object index buffer. Lock updates to ttl and ban, if in the active segment, to close race against segment closing. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-10-04 11:47:09 UTC (rev 4271) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-10-04 11:52:12 UTC (rev 4272) @@ -317,7 +317,7 @@ struct storage *objstore; struct objcore *objcore; - struct smp_object *smp_object; + unsigned smp_index; struct ws ws_o[1]; unsigned char *vary; Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-10-04 11:47:09 UTC (rev 4271) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-10-04 11:52:12 UTC (rev 4272) @@ -473,14 +473,14 @@ if (b == o->ban) { /* not banned */ o->ban = b0; o->ban_t = o->ban->t0; - if (o->smp_object != NULL) + if (o->objcore->smp_seg != NULL) SMP_BANchanged(o, b0->t0); return (0); } else { o->ttl = 0; o->cacheable = 0; o->ban = NULL; - if (o->smp_object != NULL) + if (o->objcore->smp_seg != NULL) SMP_TTLchanged(o); /* BAN also changed, but that is not important any more */ WSP(sp, SLT_ExpBan, "%u was banned", o->xid); Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-10-04 11:47:09 UTC (rev 4271) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-10-04 11:52:12 UTC (rev 4272) @@ -147,7 +147,7 @@ oc->flags |= OC_F_ONLRU; } Lck_Unlock(&exp_mtx); - if (o->smp_object != NULL) + if (o->objcore->smp_seg != NULL) SMP_TTLchanged(o); } @@ -240,7 +240,7 @@ assert(oc->timer_idx != BINHEAP_NOIDX); } Lck_Unlock(&exp_mtx); - if (o->smp_object != NULL) + if (o->objcore->smp_seg != NULL) SMP_TTLchanged(o); } Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-10-04 11:47:09 UTC (rev 4271) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-10-04 11:52:12 UTC (rev 4272) @@ -705,7 +705,7 @@ free(o->vary); ESI_Destroy(o); - if (o->smp_object != NULL) { + if (o->objcore != NULL && o->objcore->smp_seg != NULL) { SMP_FreeObj(o); } else { HSH_Freestore(o); Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-04 11:47:09 UTC (rev 4271) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-04 11:52:12 UTC (rev 4272) @@ -98,13 +98,13 @@ struct smp_segment segment; /* Copy of on-disk desc. */ - uint32_t nalloc; /* How many alloc objects */ - uint32_t nfilled; /* How many live objects */ + uint32_t nobj; /* Number of objects */ + uint32_t nalloc1; /* Allocated objects */ + uint32_t nalloc2; /* Registered objects */ uint32_t nfixed; /* How many fixed objects */ /* Only for open segment */ - uint32_t maxobj; /* Max number of objects */ - struct smp_object *objs; /* objdesc copy */ + struct smp_object *objs; /* objdesc array */ uint64_t next_addr; /* next write address */ struct smp_signctx ctx[1]; @@ -149,6 +149,8 @@ struct lock mtx; + struct smp_object *objbuf; + /* Cleaner metrics */ unsigned min_nseg; @@ -209,7 +211,7 @@ unsigned char sign[SHA256_LEN]; int r = 0; - if (strcmp(ctx->id, ctx->ss->ident)) + if (strncmp(ctx->id, ctx->ss->ident, sizeof ctx->ss->ident)) r = 1; else if (ctx->unique != ctx->ss->unique) r = 2; @@ -227,10 +229,9 @@ r = 4; } if (r) { - fprintf(stderr, "CHK(%p %p %s) = %d\n", - ctx, ctx->ss, ctx->ss->ident, r); - fprintf(stderr, "%p {%s %x %p %ju}\n", - ctx, ctx->id, ctx->unique, ctx->ss, ctx->ss->length); + fprintf(stderr, "CHK(%p %s %p %s) = %d\n", + ctx, ctx->id, ctx->ss, + r > 1 ? ctx->ss->ident : "", r); } return (r); } @@ -616,7 +617,7 @@ /* Elide any empty segments from the list before we write it */ VTAILQ_FOREACH_SAFE(sg, &sc->segments, list, sg2) { - if (sg->nalloc > 0) + if (sg->nobj > 0) continue; if (sg == sc->cur_seg) continue; @@ -643,14 +644,16 @@ sg = oc->smp_seg; CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC); + /* + * XXX: failed checks here should fail gracefully and not assert + */ so = (void*)oc->obj; + xxxassert(so >= sg->objs && so <= sg->objs + sg->nalloc2); + oc->obj = so->ptr; - /* XXX: This check should fail gracefully */ CHECK_OBJ_NOTNULL(oc->obj, OBJECT_MAGIC); - oc->obj->smp_object = so; - AN(oc->flags & OC_F_PERSISTENT); oc->flags &= ~OC_F_PERSISTENT; @@ -773,24 +776,21 @@ { struct smp_seg *sg; - AN(o->smp_object); CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC); AZ(o->objcore->flags & OC_F_PERSISTENT); sg = o->objcore->smp_seg; CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC); Lck_Lock(&sg->sc->mtx); - o->smp_object->ttl = 0; - assert(sg->nalloc > 0); + sg->objs[o->smp_index].ttl = 0; + sg->objs[o->smp_index].ptr = 0; + + assert(sg->nobj > 0); assert(sg->nfixed > 0); + sg->nobj--; + sg->nfixed--; - sg->nalloc--; - o->smp_object = NULL; - - sg->nfixed--; Lck_Unlock(&sg->sc->mtx); - - /* XXX: check if seg is empty, or leave to thread ? */ } void @@ -798,13 +798,19 @@ { struct smp_seg *sg; - AN(o->smp_object); CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC); sg = o->objcore->smp_seg; CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC); CHECK_OBJ_NOTNULL(sg->sc, SMP_SC_MAGIC); - o->smp_object->ban = t; + if (sg == sg->sc->cur_seg) { + /* Lock necessary, we might race close_seg */ + Lck_Lock(&sg->sc->mtx); + sg->objs[o->smp_index].ban = t; + Lck_Unlock(&sg->sc->mtx); + } else { + sg->objs[o->smp_index].ban = t; + } } void @@ -812,15 +818,31 @@ { struct smp_seg *sg; - AN(o->smp_object); CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC); sg = o->objcore->smp_seg; CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC); CHECK_OBJ_NOTNULL(sg->sc, SMP_SC_MAGIC); - o->smp_object->ttl = o->ttl; + if (sg == sg->sc->cur_seg) { + /* Lock necessary, we might race close_seg */ + Lck_Lock(&sg->sc->mtx); + sg->objs[o->smp_index].ttl = o->ttl; + Lck_Unlock(&sg->sc->mtx); + } else { + sg->objs[o->smp_index].ttl = o->ttl; + } } +/*--------------------------------------------------------------------*/ + +static uint64_t +smp_spaceleft(const struct smp_seg *sg) +{ + + assert(sg->next_addr <= (sg->offset + sg->length)); + return ((sg->offset + sg->length) - sg->next_addr); +} + /*-------------------------------------------------------------------- * Load segments * @@ -854,10 +876,14 @@ return; ptr = SIGN_DATA(ctx); ss = ptr; + if (ss->objlist == 0) + return; so = (void*)(sc->ptr + ss->objlist); + sg->objs = so; + sg->nalloc2 = ss->nalloc; no = ss->nalloc; /* Clear the bogus "hold" count */ - sg->nalloc = 0; + sg->nobj = 0; for (;no > 0; so++,no--) { if (so->ttl < t_now) continue; @@ -872,7 +898,7 @@ (void)HSH_Insert(sp); AZ(sp->wrk->nobjcore); EXP_Inject(oc, sg->lru, so->ttl); - sg->nalloc++; + sg->nobj++; } WRK_SumStat(sp->wrk); } @@ -972,7 +998,7 @@ * HACK: prevent save_segs from nuking segment until we have * HACK: loaded it. */ - sg->nalloc = 1; + sg->nobj = 1; if (sg1 != NULL) { assert(sg1->offset != sg->offset); if (sg1->offset < sg->offset) @@ -1015,13 +1041,22 @@ AN(sg); sg->sc = sc; - sg->maxobj = sc->aim_nobj; - sg->objs = malloc(sizeof *sg->objs * sg->maxobj); + if (sc->objbuf == NULL) { + sg->objs = malloc(sizeof *sg->objs * sc->aim_nobj); + } else { + sg->objs = sc->objbuf; + sc->objbuf = NULL; + } + AN(sg->objs); + /* XXX: debugging */ + memset(sg->objs, 0x11, sizeof *sg->objs * sc->aim_nobj); + /* XXX: find where it goes in silo */ sg->offset = sc->free_offset; + // XXX: align */ assert(sg->offset >= sc->ident->stuff[SMP_SPC_STUFF]); assert(sg->offset < sc->mediasize); @@ -1082,33 +1117,34 @@ static void smp_close_seg(struct smp_sc *sc, struct smp_seg *sg) { - size_t sz; - (void)sc; + /* XXX: if segment is empty, delete instead */ + assert(sg = sc->cur_seg); Lck_AssertHeld(&sc->mtx); - /* XXX: if segment is empty, delete instead */ + assert(sg->nalloc1 * sizeof(struct smp_object) == sc->objreserv); + assert(sc->objreserv <= smp_spaceleft(sg)); - /* Copy the objects into the segment */ - sz = sizeof *sg->objs * sg->nalloc; - /* XXX: Seen by sky 2009-10-02: */ - assert(sg->next_addr + sz <= sg->offset + sg->length); - - memcpy(sc->ptr + sg->next_addr, sg->objs, sz); - /* Update the segment header */ sg->segment.objlist = sg->next_addr; - sg->segment.nalloc = sg->nalloc; + sg->segment.nalloc = sg->nalloc1; - /* Write it to silo */ + sc->objbuf = sg->objs; + + sg->objs = (void*)(sc->ptr + sg->next_addr); + sg->next_addr += sc->objreserv; + + memcpy(sg->objs, sc->objbuf, sc->objreserv); + + /* Write segment header to silo */ smp_reset_sign(sg->ctx); memcpy(SIGN_DATA(sg->ctx), &sg->segment, sizeof sg->segment); smp_append_sign(sg->ctx, &sg->segment, sizeof sg->segment); smp_sync_sign(sg->ctx); - sg->next_addr += sizeof *sg->objs * sg->nalloc; sg->length = sg->next_addr - sg->offset; + sg->length |= 15; sg->length++; @@ -1162,6 +1198,8 @@ /* We trust the parent to give us a valid silo, for good measure: */ AZ(smp_valid_silo(sc)); + AZ(mprotect(sc->ptr, 4096, PROT_READ)); + sc->ident = SIGN_DATA(&sc->idn); /* We attempt ban1 first, and if that fails, try ban2 */ @@ -1219,6 +1257,7 @@ struct smp_seg *sg; struct smp_object *so; + CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(sp->obj->objstore, STORAGE_MAGIC); CHECK_OBJ_NOTNULL(sp->obj->objstore->stevedore, STEVEDORE_MAGIC); @@ -1226,36 +1265,27 @@ CAST_OBJ_NOTNULL(sc, sp->obj->objstore->priv, SMP_SC_MAGIC); sp->obj->objcore->flags |= OC_F_LRUDONTMOVE; + Lck_Lock(&sc->mtx); sg = sp->obj->objcore->smp_seg; - assert(sg->nalloc < sg->nfilled); - so = &sg->objs[sg->nalloc++]; + assert(sg->nalloc2 < sg->nalloc1); + + sp->obj->smp_index = sg->nalloc2++; + so = &sg->objs[sp->obj->smp_index]; sg->nfixed++; + assert(sizeof so->hash == DIGEST_LEN); memcpy(so->hash, sp->obj->objcore->objhead->digest, DIGEST_LEN); so->ttl = sp->obj->ttl; so->ptr = sp->obj; so->ban = sp->obj->ban_t; - /* XXX: if segment is already closed, write sg->objs */ - /* XXX: Uhm, this should not be the temp version, should it ? */ - sp->obj->smp_object = so; - Lck_Unlock(&sc->mtx); - } /*-------------------------------------------------------------------- * Allocate a bite */ -static uint64_t -smp_spaceleft(const struct smp_seg *sg) -{ - - assert(sg->next_addr <= (sg->offset + sg->length)); - return ((sg->offset + sg->length) - sg->next_addr); -} - static struct storage * smp_alloc(struct stevedore *st, size_t size, struct objcore *oc) { @@ -1293,7 +1323,7 @@ } /* If there is space, fine */ - if (needed < left && (oc == NULL || sg->nfilled < sg->maxobj)) + if (needed < left && (oc == NULL || sg->nalloc1 < sc->aim_nobj)) break; smp_close_seg(sc, sc->cur_seg); @@ -1318,8 +1348,8 @@ if (oc != NULL) { /* Make reservation in the index */ - assert(sg->nfilled < sg->maxobj); - sg->nfilled++; + assert(sg->nalloc1 < sc->aim_nobj); + sg->nalloc1++; sc->objreserv += sizeof (struct smp_object); assert(sc->objreserv <= smp_spaceleft(sg)); oc->smp_seg = sg; @@ -1335,7 +1365,7 @@ ss->priv = sc; ss->stevedore = st; ss->fd = sc->fd; - ss->where = sg->next_addr + sizeof *ss; + // XXX: wrong: ss->where = sg->next_addr + sizeof *ss; assert((uintmax_t)ss->space == (uintmax_t)size); assert((char*)ss->ptr > (char*)ss); assert((char*)ss->ptr + ss->space <= (char*)sc->ptr + sc->mediasize); From phk at projects.linpro.no Sun Oct 4 12:13:49 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 4 Oct 2009 14:13:49 +0200 (CEST) Subject: r4273 - trunk/varnish-cache/bin/varnishd Message-ID: <20091004121349.E778628563@projects.linpro.no> Author: phk Date: 2009-10-04 14:13:49 +0200 (Sun, 04 Oct 2009) New Revision: 4273 Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Remember to release lock when we fail allocation. Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-04 11:52:12 UTC (rev 4272) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-04 12:13:49 UTC (rev 4273) @@ -1330,8 +1330,10 @@ smp_new_seg(sc); } - if (needed > smp_spaceleft(sg)) + if (needed > smp_spaceleft(sg)) { + Lck_Unlock(&sc->mtx); return (NULL); + } assert(needed <= smp_spaceleft(sg)); From phk at projects.linpro.no Sun Oct 4 12:38:52 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 4 Oct 2009 14:38:52 +0200 (CEST) Subject: r4274 - trunk/varnish-cache/bin/varnishd Message-ID: <20091004123852.CD29C28563@projects.linpro.no> Author: phk Date: 2009-10-04 14:38:52 +0200 (Sun, 04 Oct 2009) New Revision: 4274 Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: count fixed recorded objects as fixed. Use consistent boundary check for end of segment. Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-04 12:13:49 UTC (rev 4273) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-04 12:38:52 UTC (rev 4274) @@ -1273,6 +1273,7 @@ sp->obj->smp_index = sg->nalloc2++; so = &sg->objs[sp->obj->smp_index]; sg->nfixed++; + sg->nobj++; assert(sizeof so->hash == DIGEST_LEN); memcpy(so->hash, sp->obj->objcore->objhead->digest, DIGEST_LEN); so->ttl = sp->obj->ttl; @@ -1323,7 +1324,8 @@ } /* If there is space, fine */ - if (needed < left && (oc == NULL || sg->nalloc1 < sc->aim_nobj)) + if (needed <= left && + (oc == NULL || sg->nalloc1 < sc->aim_nobj)) break; smp_close_seg(sc, sc->cur_seg); From phk at projects.linpro.no Sun Oct 4 12:46:42 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Sun, 4 Oct 2009 14:46:42 +0200 (CEST) Subject: r4275 - trunk/varnish-cache/bin/varnishd Message-ID: <20091004124642.C1577386EE@projects.linpro.no> Author: phk Date: 2009-10-04 14:46:42 +0200 (Sun, 04 Oct 2009) New Revision: 4275 Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Add another band-aid suicide clause when we run out of space in the silo. Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-04 12:38:52 UTC (rev 4274) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-04 12:46:42 UTC (rev 4275) @@ -46,6 +46,7 @@ #include #include +#include "shmlog.h" #include "cache.h" #include "stevedore.h" #include "hash_slinger.h" @@ -1063,7 +1064,18 @@ sg->length = sc->aim_segl; sg->length &= ~7; + if (sg->offset + sg->length > sc->mediasize) { + sc->free_offset = sc->ident->stuff[SMP_SPC_STUFF]; + sg->offset = sc->free_offset; + sg2 = VTAILQ_FIRST(&sc->segments); + if (sg->offset + sg->length > sg2->offset) { + printf("Out of space in persistent silo\n"); + printf("Committing suicide, restart will make space\n"); + exit (0); + } + } + assert(sg->offset + sg->length <= sc->mediasize); sg2 = VTAILQ_FIRST(&sc->segments); From phk at projects.linpro.no Tue Oct 6 08:10:38 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 6 Oct 2009 10:10:38 +0200 (CEST) Subject: r4276 - trunk/varnish-cache/bin/varnishd Message-ID: <20091006081038.6904D1F7397@projects.linpro.no> Author: phk Date: 2009-10-06 10:10:38 +0200 (Tue, 06 Oct 2009) New Revision: 4276 Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Align allocations to 16 bytes, for easier counting in hexdumps Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-04 12:46:42 UTC (rev 4275) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-06 08:10:38 UTC (rev 4276) @@ -1312,6 +1312,12 @@ CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC); Lck_Lock(&sc->mtx); + /* Alignment */ + if (size & 0xf) { + size |= 0xf; + size += 1; + } + for (tries = 0; tries < 3; tries++) { sg = sc->cur_seg; CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC); @@ -1333,6 +1339,7 @@ size = left - overhead; needed = overhead + size; assert(needed <= left); + size &= ~15; } /* If there is space, fine */ From phk at projects.linpro.no Tue Oct 6 08:31:57 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 6 Oct 2009 10:31:57 +0200 (CEST) Subject: r4277 - in trunk/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20091006083157.AAC351F7386@projects.linpro.no> Author: phk Date: 2009-10-06 10:31:57 +0200 (Tue, 06 Oct 2009) New Revision: 4277 Added: trunk/varnish-cache/bin/varnishtest/tests/p00006.vtc Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_vary.c Log: Move the Vary specification into the object worksspace instead of using malloc space. Otherwise -spersistent would resurrect objects without their Vary specifications. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-10-06 08:10:38 UTC (rev 4276) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-10-06 08:31:57 UTC (rev 4277) @@ -652,7 +652,7 @@ void RES_WriteObj(struct sess *sp); /* cache_vary.c */ -void VRY_Create(const struct sess *sp); +struct vsb *VRY_Create(const struct sess *sp, struct http *hp); int VRY_Match(const struct sess *sp, const unsigned char *vary); /* cache_vcl.c */ Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-10-06 08:10:38 UTC (rev 4276) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-10-06 08:31:57 UTC (rev 4277) @@ -420,6 +420,7 @@ struct http *hp, *hp2; char *b; unsigned handling, l; + struct vsb *vary; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC); @@ -515,13 +516,18 @@ if (sp->wrk->cacheable) { CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC); CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC); + vary = VRY_Create(sp, sp->wrk->beresp); } else { AZ(sp->objhead); AZ(sp->objcore); + vary = NULL; } l = http_EstimateWS(sp->wrk->beresp, HTTPH_A_INS); + if (vary != NULL) + l += vsb_len(vary); + /* Space for producing a Content-Length: header */ l += 30; @@ -543,6 +549,15 @@ BAN_NewObj(sp->obj); } + if (vary != NULL) { + sp->obj->vary = + (void *)WS_Alloc(sp->obj->http->ws, vsb_len(vary)); + AN(sp->obj->vary); + memcpy(sp->obj->vary, vsb_data(vary), vsb_len(vary)); + vsb_delete(vary); + vary = NULL; + } + sp->obj->xid = sp->xid; sp->obj->response = sp->err_code; sp->obj->cacheable = sp->wrk->cacheable; @@ -619,7 +634,6 @@ sp->obj->cacheable = 1; if (sp->wrk->cacheable) { - VRY_Create(sp); EXP_Insert(sp->obj); AN(sp->obj->ban); HSH_Unbusy(sp); Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-10-06 08:10:38 UTC (rev 4276) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-10-06 08:31:57 UTC (rev 4277) @@ -701,9 +701,6 @@ DSL(0x40, SLT_Debug, 0, "Object %u workspace min free %u", o->xid, WS_Free(o->ws_o)); - if (o->vary != NULL) - free(o->vary); - ESI_Destroy(o); if (o->objcore != NULL && o->objcore->smp_seg != NULL) { SMP_FreeObj(o); Modified: trunk/varnish-cache/bin/varnishd/cache_vary.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vary.c 2009-10-06 08:10:38 UTC (rev 4276) +++ trunk/varnish-cache/bin/varnishd/cache_vary.c 2009-10-06 08:31:57 UTC (rev 4277) @@ -63,16 +63,16 @@ #include "cache.h" -void -VRY_Create(const struct sess *sp) +struct vsb * +VRY_Create(const struct sess *sp, struct http *hp) { char *v, *p, *q, *h, *e; struct vsb *sb, *sbh; int l; /* No Vary: header, no worries */ - if (!http_GetHdr(sp->obj->http, H_Vary, &v)) - return; + if (!http_GetHdr(hp, H_Vary, &v)) + return (NULL); /* For vary matching string */ sb = vsb_newauto(); @@ -126,16 +126,10 @@ /* Terminate vary matching string */ vsb_printf(sb, "%c", 0); + vsb_delete(sbh); vsb_finish(sb); AZ(vsb_overflowed(sb)); - l = vsb_len(sb); - assert(l >= 0); - sp->obj->vary = malloc(l); - AN(sp->obj->vary); - memcpy(sp->obj->vary, vsb_data(sb), l); - - vsb_delete(sb); - vsb_delete(sbh); + return(sb); } int Added: trunk/varnish-cache/bin/varnishtest/tests/p00006.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/p00006.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/p00006.vtc 2009-10-06 08:31:57 UTC (rev 4277) @@ -0,0 +1,56 @@ +# $Id$ + +test "Check that Vary headers are stored" + +shell "rm -f /tmp/__v1/_.per" + +server s1 { + rxreq + txresp -hdr "Foo: foo1" -hdr "Vary: foo, bar" + rxreq + txresp -hdr "Foo: foo2" -hdr "Vary: foo, bar" +} -start + + +varnish v1 \ + -arg "-spersistent,/tmp/__v1/_.per,10m" \ + -vcl+backend { } -start + +client c1 { + txreq -url "/foo" -hdr "foo: 1" -hdr "bar: 2" + rxresp + expect resp.status == 200 + expect resp.http.X-Varnish == "1001" + expect resp.http.foo == "foo1" + + txreq -url "/foo" -hdr "foo: 2" -hdr "bar: 1" + rxresp + expect resp.status == 200 + expect resp.http.X-Varnish == "1002" + expect resp.http.foo == "foo2" + +} -run + +varnish v1 -expect n_object == 2 +server s1 -wait + +varnish v1 -stop +varnish v1 -start + +varnish v1 -expect n_vampireobject == 2 + +client c1 { + txreq -url "/foo" -hdr "foo: 1" -hdr "bar: 2" + rxresp + expect resp.status == 200 + #expect resp.http.X-Varnish == "1001" + expect resp.http.foo == "foo1" + + txreq -url "/foo" -hdr "foo: 2" -hdr "bar: 1" + rxresp + expect resp.status == 200 + #expect resp.http.X-Varnish == "1002" + expect resp.http.foo == "foo2" + +} -run + From phk at projects.linpro.no Wed Oct 7 07:57:54 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 7 Oct 2009 09:57:54 +0200 (CEST) Subject: r4278 - trunk/varnish-cache/bin/varnishd Message-ID: <20091007075754.6BB261F72A6@projects.linpro.no> Author: phk Date: 2009-10-07 09:57:54 +0200 (Wed, 07 Oct 2009) New Revision: 4278 Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Minor polishing: Only delete empty segments from front of list. Preallocate the temporary object-index buffer at open time. Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-06 08:31:57 UTC (rev 4277) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-07 07:57:54 UTC (rev 4278) @@ -616,16 +616,18 @@ Lck_AssertHeld(&sc->mtx); - /* Elide any empty segments from the list before we write it */ + /* + * Remove empty segments from the front of the list + * before we write the segments to disk. + */ VTAILQ_FOREACH_SAFE(sg, &sc->segments, list, sg2) { if (sg->nobj > 0) - continue; + break; if (sg == sc->cur_seg) continue; VTAILQ_REMOVE(&sc->segments, sg, list); - // XXX: free segment + free(sg); } - smp_save_seg(sc, &sc->seg1); smp_save_seg(sc, &sc->seg2); } @@ -1042,13 +1044,9 @@ AN(sg); sg->sc = sc; - if (sc->objbuf == NULL) { - sg->objs = malloc(sizeof *sg->objs * sc->aim_nobj); - } else { - sg->objs = sc->objbuf; - sc->objbuf = NULL; - } - + AN(sc->objbuf); + sg->objs = sc->objbuf; + sc->objbuf = NULL; AN(sg->objs); /* XXX: debugging */ @@ -1207,6 +1205,9 @@ Lck_New(&sc->mtx); Lck_Lock(&sc->mtx); + sc->objbuf = malloc(sizeof *sc->objbuf * sc->aim_nobj); + AN(sc->objbuf); + /* We trust the parent to give us a valid silo, for good measure: */ AZ(smp_valid_silo(sc)); From phk at projects.linpro.no Wed Oct 7 10:22:28 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 7 Oct 2009 12:22:28 +0200 (CEST) Subject: r4279 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20091007102228.197411F72A6@projects.linpro.no> Author: phk Date: 2009-10-07 12:22:27 +0200 (Wed, 07 Oct 2009) New Revision: 4279 Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c trunk/varnish-cache/include/persistent.h Log: Get rid of some old assumptions and simplify things correspondingly: Move the objindex pointer and length to the segment table, having it in the segment signatures is just a waste of disk-access on startup. Put empty signatures at segment head, objidx and segment tail. Align allocations to 16 bytes boundaries, for cache and hexdump convenience. Discard unsused segments on close. Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-07 07:57:54 UTC (rev 4278) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-10-07 10:22:27 UTC (rev 4279) @@ -65,6 +65,9 @@ #define ASSERT_SILO_THREAD(sc) \ do {assert(pthread_self() == (sc)->thread);} while (0) +#define RDN2(x, y) ((x)&(~((y)-1))) /* if y is powers of two */ +#define RUP2(x, y) (((x)+((y)-1))&(~((y)-1))) /* if y is powers of two */ + /* * Context for a signature. * @@ -94,12 +97,10 @@ VTAILQ_ENTRY(smp_seg) list; /* on smp_sc.smp_segments */ - uint64_t offset; /* coordinates in silo */ - uint64_t length; + struct smp_segptr p; - struct smp_segment segment; /* Copy of on-disk desc. */ - uint32_t nobj; /* Number of objects */ + uint32_t nalloc; /* Allocations */ uint32_t nalloc1; /* Allocated objects */ uint32_t nalloc2; /* Registered objects */ uint32_t nfixed; /* How many fixed objects */ @@ -167,6 +168,12 @@ uint64_t free_reserve; }; +#define CACHE_LINE_ALIGN 16 + +#define C_ALIGN(x) RUP2(x, CACHE_LINE_ALIGN) + +#define SEG_SPACE RUP2(SMP_SIGN_SPACE, CACHE_LINE_ALIGN) + /* * silos is unlocked, it only changes during startup when we are * single-threaded @@ -343,7 +350,7 @@ si->byte_order = 0x12345678; si->size = sizeof *si; si->major_version = 1; - si->minor_version = 1; + si->minor_version = 2; si->unique = sc->unique; si->mediasize = sc->mediasize; si->granularity = sc->granularity; @@ -389,7 +396,7 @@ return (4); if (si->major_version != 1) return (5); - if (si->minor_version != 1) + if (si->minor_version != 2) return (6); if (si->mediasize != sc->mediasize) return (7); @@ -529,6 +536,7 @@ SIZOF(struct smp_object); #undef SIZOF + /* See comments in persistent.h */ assert(sizeof(struct smp_ident) == SMP_IDENT_SIZE); /* Allocate softc */ @@ -598,10 +606,9 @@ ss = SIGN_DATA(ctx); length = 0; VTAILQ_FOREACH(sg, &sc->segments, list) { - assert(sg->offset < sc->mediasize); - assert(sg->offset + sg->length <= sc->mediasize); - ss->offset = sg->offset; - ss->length = sg->length; + assert(sg->p.offset < sc->mediasize); + assert(sg->p.offset + sg->p.length <= sc->mediasize); + *ss = sg->p; ss++; length += sizeof *ss; } @@ -839,11 +846,18 @@ /*--------------------------------------------------------------------*/ static uint64_t +smp_segend(const struct smp_seg *sg) +{ + + return (sg->p.offset + sg->p.length); +} + +static uint64_t smp_spaceleft(const struct smp_seg *sg) { - assert(sg->next_addr <= (sg->offset + sg->length)); - return ((sg->offset + sg->length) - sg->next_addr); + assert(sg->next_addr <= smp_segend(sg)); + return (smp_segend(sg) - sg->next_addr); } /*-------------------------------------------------------------------- @@ -863,8 +877,6 @@ static void smp_load_seg(struct sess *sp, const struct smp_sc *sc, struct smp_seg *sg) { - void *ptr; - struct smp_segment *ss; struct smp_object *so; struct objcore *oc; uint32_t no; @@ -873,18 +885,16 @@ ASSERT_SILO_THREAD(sc); (void)sp; - AN(sg->offset); - smp_def_sign(sc, ctx, sg->offset, "SEGMENT"); + AN(sg->p.offset); + if (sg->p.objlist == 0) + return; + smp_def_sign(sc, ctx, sg->p.offset, "SEGHEAD"); if (smp_chk_sign(ctx)) return; - ptr = SIGN_DATA(ctx); - ss = ptr; - if (ss->objlist == 0) - return; - so = (void*)(sc->ptr + ss->objlist); + so = (void*)(sc->ptr + sg->p.objlist); sg->objs = so; - sg->nalloc2 = ss->nalloc; - no = ss->nalloc; + sg->nalloc2 = sg->p.nalloc; + no = sg->p.nalloc; /* Clear the bogus "hold" count */ sg->nobj = 0; for (;no > 0; so++,no--) { @@ -974,11 +984,11 @@ */ sc->free_offset = sc->ident->stuff[SMP_SPC_STUFF]; while (ss < se) { -//printf("TEST_SEG2 %jx...%jx\n", ss->offset, ss->offset + ss->length); +//printf("TEST_SEG2 %jx...%jx\n", ss->p.offset, ss->p.offset + ss->length); l = ss->offset - sc->free_offset; if (l > sc->free_reserve) break; -//printf("DROP_SEG2 %jx...%jx\n", ss->offset, ss->offset + ss->length); +//printf("DROP_SEG2 %jx...%jx\n", ss->p.offset, ss->p.offset + ss->length); ss++; n++; } @@ -991,30 +1001,29 @@ sg1 = NULL; sg2 = NULL; for(; ss <= se; ss++) { -// printf("LOAD_SEG %jx...%jx\n", ss->offset, ss->offset + ss->length); +// printf("LOAD_SEG %jx...%jx\n", ss->p.offset, ss->p.offset + ss->length); ALLOC_OBJ(sg, SMP_SEG_MAGIC); AN(sg); sg->lru = LRU_Alloc(); - sg->offset = ss->offset; - sg->length = ss->length; + sg->p = *ss; /* * HACK: prevent save_segs from nuking segment until we have * HACK: loaded it. */ sg->nobj = 1; if (sg1 != NULL) { - assert(sg1->offset != sg->offset); - if (sg1->offset < sg->offset) - assert(sg1->offset + sg1->length <= sg->offset); + assert(sg1->p.offset != sg->p.offset); + if (sg1->p.offset < sg->p.offset) + assert(smp_segend(sg1) <= sg->p.offset); else - assert(sg->offset + sg->length <= sg1->offset); + assert(smp_segend(sg) <= sg1->p.offset); } if (sg2 != NULL) { - assert(sg2->offset != sg->offset); - if (sg2->offset < sg->offset) - assert(sg2->offset + sg2->length <= sg->offset); + assert(sg2->p.offset != sg->p.offset); + if (sg2->p.offset < sg->p.offset) + assert(smp_segend(sg2) <= sg->p.offset); else - assert(sg->offset + sg->length <= sg2->offset); + assert(smp_segend(sg) <= sg2->p.offset); } /* XXX: check that they are inside silo */ @@ -1054,19 +1063,19 @@ /* XXX: find where it goes in silo */ - sg->offset = sc->free_offset; + sg->p.offset = sc->free_offset; // XXX: align */ - assert(sg->offset >= sc->ident->stuff[SMP_SPC_STUFF]); - assert(sg->offset < sc->mediasize); + assert(sg->p.offset >= sc->ident->stuff[SMP_SPC_STUFF]); + assert(sg->p.offset < sc->mediasize); - sg->length = sc->aim_segl; - sg->length &= ~7; + sg->p.length = sc->aim_segl; + sg->p.length &= ~7; - if (sg->offset + sg->length > sc->mediasize) { + if (smp_segend(sg) > sc->mediasize) { sc->free_offset = sc->ident->stuff[SMP_SPC_STUFF]; - sg->offset = sc->free_offset; + sg->p.offset = sc->free_offset; sg2 = VTAILQ_FIRST(&sc->segments); - if (sg->offset + sg->length > sg2->offset) { + if (smp_segend(sg) > sg2->p.offset) { printf("Out of space in persistent silo\n"); printf("Committing suicide, restart will make space\n"); exit (0); @@ -1074,50 +1083,36 @@ } - assert(sg->offset + sg->length <= sc->mediasize); + assert(smp_segend(sg) <= sc->mediasize); sg2 = VTAILQ_FIRST(&sc->segments); - if (sg2 != NULL && sg2->offset > sc->free_offset) { -//printf("SG %jx...%jx\n", sg->offset, sg->offset + sg->length); -//printf("SG2 %jx...%jx\n", sg2->offset, sg2->offset + sg2->length); - -//printf("spc %jx %jx\n", sg2->offset + sg2->length - sg->offset, sc->free_reserve); - - if (sg->offset + sg->length > sg2->offset) { + if (sg2 != NULL && sg2->p.offset > sc->free_offset) { + if (smp_segend(sg) > sg2->p.offset) { printf("Out of space in persistent silo\n"); printf("Committing suicide, restart will make space\n"); exit (0); } - assert(sg->offset + sg->length <= sg2->offset); + assert(smp_segend(sg) <= sg2->p.offset); } - sc->free_offset += sg->length; + sc->free_offset += sg->p.length; -//printf("NEW_SEG %jx...%jx\n", sg->offset, sg->offset + sg->length); VTAILQ_INSERT_TAIL(&sc->segments, sg, list); /* Neuter the new segment in case there is an old one there */ - AN(sg->offset); - smp_def_sign(sc, sg->ctx, sg->offset, "SEGMENT"); + AN(sg->p.offset); + smp_def_sign(sc, sg->ctx, sg->p.offset, "SEGHEAD"); smp_reset_sign(sg->ctx); - memcpy(SIGN_DATA(sg->ctx), &sg->segment, sizeof sg->segment); - smp_append_sign(sg->ctx, &sg->segment, sizeof sg->segment); smp_sync_sign(sg->ctx); - /* Set up our allocation point */ sc->cur_seg = sg; - sg->next_addr = sg->offset + - sizeof (struct smp_sign) + - sizeof (struct smp_segment) + + sg->next_addr = sg->p.offset + + sizeof (struct smp_sign) + // XXX use macro SHA256_LEN; memcpy(sc->ptr + sg->next_addr, "HERE", 4); sc->objreserv = 0; - - /* Then add it to the segment list. */ - /* XXX: could be done cheaper with append ? */ - smp_save_segs(sc); } /*-------------------------------------------------------------------- @@ -1127,40 +1122,56 @@ static void smp_close_seg(struct smp_sc *sc, struct smp_seg *sg) { + void *p; + Lck_AssertHeld(&sc->mtx); + /* XXX: if segment is empty, delete instead */ - assert(sg = sc->cur_seg); + assert(sg == sc->cur_seg); + AN(sg->p.offset); - Lck_AssertHeld(&sc->mtx); + sc->cur_seg = NULL; + if (sg->nalloc == 0) { + sc->objbuf = sg->objs; + AN(sc->objbuf); + VTAILQ_REMOVE(&sc->segments, sg, list); + free(sg); + return; + } + assert(sg->nalloc1 * sizeof(struct smp_object) == sc->objreserv); - assert(sc->objreserv <= smp_spaceleft(sg)); + assert(C_ALIGN(sc->objreserv) + 2 * SEG_SPACE <= smp_spaceleft(sg)); + /* Write the OBJIDX */ + smp_def_sign(sc, sg->ctx, sg->next_addr, "OBJIDX"); + smp_reset_sign(sg->ctx); + smp_sync_sign(sg->ctx); + sg->next_addr += SEG_SPACE; + /* Update the segment header */ - sg->segment.objlist = sg->next_addr; - sg->segment.nalloc = sg->nalloc1; + sg->p.objlist = sg->next_addr; + sg->p.nalloc = sg->nalloc1; + p = (void*)(sc->ptr + sg->next_addr); + sg->next_addr += C_ALIGN(sc->objreserv); + + memcpy(p, sg->objs, sc->objreserv); sc->objbuf = sg->objs; + /* XXX: membarrier */ + sg->objs = p; - sg->objs = (void*)(sc->ptr + sg->next_addr); - sg->next_addr += sc->objreserv; - - memcpy(sg->objs, sc->objbuf, sc->objreserv); - - /* Write segment header to silo */ + /* Write the SEGTAIL */ + smp_def_sign(sc, sg->ctx, sg->next_addr, "SEGTAIL"); smp_reset_sign(sg->ctx); - memcpy(SIGN_DATA(sg->ctx), &sg->segment, sizeof sg->segment); - smp_append_sign(sg->ctx, &sg->segment, sizeof sg->segment); smp_sync_sign(sg->ctx); + sg->next_addr += SEG_SPACE; - sg->length = sg->next_addr - sg->offset; + sg->p.length = sg->next_addr - sg->p.offset; - sg->length |= 15; - sg->length++; - /* Save segment list */ smp_save_segs(sc); - sc->free_offset = sg->offset + sg->length; + sc->free_offset = smp_segend(sg); } /*-------------------------------------------------------------------- @@ -1313,25 +1324,26 @@ CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC); Lck_Lock(&sc->mtx); - /* Alignment */ - if (size & 0xf) { - size |= 0xf; - size += 1; - } + size = C_ALIGN(size); for (tries = 0; tries < 3; tries++) { sg = sc->cur_seg; CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC); - overhead = sizeof *ss + sc->objreserv; + overhead = C_ALIGN(sizeof *ss); + overhead += SEG_SPACE * 2; + if (oc == NULL) { + overhead += C_ALIGN(sc->objreserv); + } else { + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + overhead += + C_ALIGN(sizeof(struct smp_object) + sc->objreserv); + } needed = overhead + size; left = smp_spaceleft(sg); - if (oc != NULL) { - CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - /* Objects also need an entry in the index */ - needed += sizeof (struct smp_object); - } else if (needed > left && (overhead + 4096) < left) { + if (oc == NULL && needed > left && (overhead + 4096) < left) { + /* XXX: Also check the bit we cut off isn't silly short */ /* * Non-objects can be trimmed down to fit what we * have to offer (think: DVD image), but we do not @@ -1340,7 +1352,7 @@ size = left - overhead; needed = overhead + size; assert(needed <= left); - size &= ~15; + size &= ~15; /* XXX */ } /* If there is space, fine */ @@ -1352,6 +1364,8 @@ smp_new_seg(sc); } + assert(size == C_ALIGN(size)); + if (needed > smp_spaceleft(sg)) { Lck_Unlock(&sc->mtx); return (NULL); @@ -1361,7 +1375,7 @@ /* Grab for storage struct */ ss = (void *)(sc->ptr + sg->next_addr); - sg->next_addr +=sizeof *ss; + sg->next_addr += C_ALIGN(sizeof *ss); /* Grab for allocated space */ allocation = sc->ptr + sg->next_addr; @@ -1379,6 +1393,7 @@ oc->smp_seg = sg; } + sg->nalloc++; Lck_Unlock(&sc->mtx); /* Fill the storage structure */ Modified: trunk/varnish-cache/include/persistent.h =================================================================== --- trunk/varnish-cache/include/persistent.h 2009-10-07 07:57:54 UTC (rev 4278) +++ trunk/varnish-cache/include/persistent.h 2009-10-07 10:22:27 UTC (rev 4279) @@ -124,19 +124,11 @@ struct smp_segptr { uint64_t offset; uint64_t length; + uint64_t objlist; + uint32_t nalloc; }; /* - * Ban description - */ - -struct smp_ban { - double ttl; - uint16_t length; - uint8_t valid; -}; - -/* * An object descriptor */ @@ -147,12 +139,3 @@ struct object *ptr; uint64_t len; /* XXX: madvise */ }; - -/* - * Segment - */ - -struct smp_segment { - uint64_t objlist; - uint32_t nalloc; -}; From tfheen at projects.linpro.no Thu Oct 8 09:17:03 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 11:17:03 +0200 (CEST) Subject: r4280 - in branches/2.0/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20091008091703.BD54F1F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 11:17:03 +0200 (Thu, 08 Oct 2009) New Revision: 4280 Added: branches/2.0/varnish-cache/bin/varnishtest/tests/e00016.vtc Modified: branches/2.0/varnish-cache/bin/varnishd/cache_vrt.c branches/2.0/varnish-cache/bin/varnishtest/tests/e00015.vtc Log: Merge r4068, r4077, 4081: Prevent turning off ESI in a subrequest r4068: There is probably a bug if an ESI subrequest turns ESI delivery back on -- add reminder so I can circle back and extend the test for it r4077: It turns out that you could turn off ESI processing for subrequest and completely hose the response. Ignore the flag if you are in a subrequest. r4081: I am stupid, missing '/' Modified: branches/2.0/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_vrt.c 2009-10-07 10:22:27 UTC (rev 4279) +++ branches/2.0/varnish-cache/bin/varnishd/cache_vrt.c 2009-10-08 09:17:03 UTC (rev 4280) @@ -502,7 +502,10 @@ VRT_l_req_esi(struct sess *sp, unsigned process_esi) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - sp->disable_esi = !process_esi; + /* only allow you to turn of esi in the main request + else everything gets confused */ + if(sp->esis == 0) + sp->disable_esi = !process_esi; } unsigned Modified: branches/2.0/varnish-cache/bin/varnishtest/tests/e00015.vtc =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/e00015.vtc 2009-10-07 10:22:27 UTC (rev 4279) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/e00015.vtc 2009-10-08 09:17:03 UTC (rev 4280) @@ -2,7 +2,6 @@ test "ESI requests turned off" - server s1 { rxreq txresp -body { Copied: branches/2.0/varnish-cache/bin/varnishtest/tests/e00016.vtc (from rev 4077, trunk/varnish-cache/bin/varnishtest/tests/e00016.vtc) =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/e00016.vtc (rev 0) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/e00016.vtc 2009-10-08 09:17:03 UTC (rev 4280) @@ -0,0 +1,49 @@ +# $Id: e00015.vtc 4068 2009-05-11 08:50:45Z sky $ + +test "ESI request can't be turned off midstream" + +server s1 { + rxreq + txresp -body { + + Before include + + + After include + } + rxreq + expect req.url == "/body" + txresp -body { + + } + rxreq + expect req.url == "/body2" + txresp -body { + included + } + rxreq + expect req.url == "/body3" + txresp -body { + included body3 + } +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set req.esi = true; + if(req.url == "/body") { + set req.esi = false; + } + esi; + } +} -start + +client c1 { + txreq + rxresp + expect resp.bodylen == 105 + expect resp.status == 200 +} + +client c1 -run +varnish v1 -expect esi_errors == 0 From tfheen at projects.linpro.no Thu Oct 8 09:25:07 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 11:25:07 +0200 (CEST) Subject: r4281 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008092507.CAAF31F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 11:25:07 +0200 (Thu, 08 Oct 2009) New Revision: 4281 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c branches/2.0/varnish-cache/bin/varnishd/cache_session.c Log: Merge r4071: Shift the responsibility for washing used sessions. Instead of the acceptor thread doing it when reusing the session, have the worker threads clean it out before putting it on the free list. It could be, and probably was, argued that this is a performance pessimization, but having thought much about it, I can't spot the argument any longer, and certainly moving load off the acceptor thread to the massively parallel worker threads is a good idea. Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c 2009-10-08 09:17:03 UTC (rev 4280) +++ branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c 2009-10-08 09:25:07 UTC (rev 4281) @@ -108,13 +108,17 @@ need_test = 0; } +/*-------------------------------------------------------------------- + * Called once the workerthread gets hold of the session, to do setup + * setup overhead, we don't want to bother the acceptor thread with. + */ + void VCA_Prep(struct sess *sp) { char addr[TCP_ADDRBUFSIZE]; char port[TCP_PORTBUFSIZE]; - TCP_name(sp->sockaddr, sp->sockaddrlen, addr, sizeof addr, port, sizeof port); sp->addr = WS_Dup(sp->ws, addr); Modified: branches/2.0/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_session.c 2009-10-08 09:17:03 UTC (rev 4280) +++ branches/2.0/varnish-cache/bin/varnishd/cache_session.c 2009-10-08 09:25:07 UTC (rev 4281) @@ -267,6 +267,8 @@ sm = malloc(sizeof *sm + u); if (sm == NULL) return (NULL); + /* Don't waste time zeroing the workspace */ + memset(sm, 0, sizeof *sm); sm->magic = SESSMEM_MAGIC; sm->workspace = u; VSL_stats->n_sess_mem++; @@ -274,7 +276,6 @@ CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC); VSL_stats->n_sess++; sp = &sm->sess; - memset(sp, 0, sizeof *sp); sp->magic = SESS_MAGIC; sp->mem = sm; sp->sockaddr = (void*)(&sm->sockaddr[0]); @@ -346,6 +347,7 @@ { struct acct *b = &sp->acct; struct sessmem *sm; + unsigned workspace; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); sm = sp->mem; @@ -365,6 +367,12 @@ VSL_stats->n_sess_mem--; free(sm); } else { + /* Clean and prepare for reuse */ + workspace = sm->workspace; + memset(sm, 0, sizeof *sm); + sm->magic = SESSMEM_MAGIC; + sm->workspace = workspace; + Lck_Lock(&ses_mem_mtx); VTAILQ_INSERT_HEAD(&ses_free_mem[1 - ses_qp], sm, list); Lck_Unlock(&ses_mem_mtx); From tfheen at projects.linpro.no Thu Oct 8 09:38:45 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 11:38:45 +0200 (CEST) Subject: r4282 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008093845.538D91F738D@projects.linpro.no> Author: tfheen Date: 2009-10-08 11:38:45 +0200 (Thu, 08 Oct 2009) New Revision: 4282 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_center.c branches/2.0/varnish-cache/bin/varnishd/cache_vrt_esi.c Log: Merge r4073: Be much more consistent about snapshotting session workspace when doing an esi-include. This should reduce the pressure on session workspace from nested ESI includes considerably. Modified: branches/2.0/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_center.c 2009-10-08 09:25:07 UTC (rev 4281) +++ branches/2.0/varnish-cache/bin/varnishd/cache_center.c 2009-10-08 09:38:45 UTC (rev 4282) @@ -935,6 +935,10 @@ /* XXX: Don't bother with write failures for now */ (void)write(sp->fd, r, strlen(r)); + /* XXX: When we do ESI includes, this is not removed + * XXX: because we use http0 as our basis. Believed + * XXX: safe, but potentially confusing. + */ http_Unset(sp->http, H_Expect); } Modified: branches/2.0/varnish-cache/bin/varnishd/cache_vrt_esi.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_vrt_esi.c 2009-10-08 09:25:07 UTC (rev 4281) +++ branches/2.0/varnish-cache/bin/varnishd/cache_vrt_esi.c 2009-10-08 09:38:45 UTC (rev 4282) @@ -766,8 +766,11 @@ { struct esi_bit *eb; struct object *obj; + char *ws_wm; + struct http http_save; WRW_Reserve(sp->wrk, &sp->fd); + http_save.magic = 0; VTAILQ_FOREACH(eb, &sp->obj->esibits, list) { if (Tlen(eb->verbatim)) { if (sp->http->protover >= 1.1) @@ -789,8 +792,17 @@ sp->esis++; obj = sp->obj; sp->obj = NULL; + + /* Save the master objects HTTP state, we may need it later */ + if (http_save.magic == 0) + http_save = *sp->http; + + /* Reset request to status before we started messing with it */ *sp->http = *sp->http0; - /* XXX: reset sp->ws */ + + /* Take a workspace snapshot */ + ws_wm = WS_Snapshot(sp->ws); + http_SetH(sp->http, HTTP_HDR_URL, eb->include.b); if (eb->host.b != NULL) { http_Unset(sp->http, H_Host); @@ -825,10 +837,17 @@ assert(sp->step == STP_DONE); sp->esis--; sp->obj = obj; + + /* Reset the workspace */ + WS_Reset(sp->ws, ws_wm); + WRW_Reserve(sp->wrk, &sp->fd); if (sp->fd < 0) break; } + /* Restore master objects HTTP state */ + if (http_save.magic) + *sp->http = http_save; if (sp->esis == 0 && sp->http->protover >= 1.1) (void)WRW_Write(sp->wrk, "0\r\n\r\n", -1); if (WRW_FlushRelease(sp->wrk)) From tfheen at projects.linpro.no Thu Oct 8 10:13:28 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 12:13:28 +0200 (CEST) Subject: r4283 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008101328.708501F73A1@projects.linpro.no> Author: tfheen Date: 2009-10-08 12:13:28 +0200 (Thu, 08 Oct 2009) New Revision: 4283 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c branches/2.0/varnish-cache/bin/varnishd/cache_center.c branches/2.0/varnish-cache/bin/varnishd/cache_httpconn.c branches/2.0/varnish-cache/bin/varnishd/mgt_param.c branches/2.0/varnish-cache/bin/varnishd/steps.h Log: Merge r4079, r4080: Make session_linger the default with 50msec timeout. Change the session handling logic, so that we hand any session back to the herder, if the session_linger timer fires. This means that a new session, will get a worker thread, but if nothing happens within session_linger, it disembarks and lets the herder handle the assembly of the request. This actually simplifies the state engine, because the "again" state (now renamed "wait) can also be used both for the first request. Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c 2009-10-08 09:38:45 UTC (rev 4282) +++ branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c 2009-10-08 10:13:28 UTC (rev 4283) @@ -253,6 +253,7 @@ sp->fd = i; sp->id = i; sp->t_open = now; + sp->t_end = now; sp->mylsock = ls; sp->step = STP_FIRST; Modified: branches/2.0/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_center.c 2009-10-08 09:38:45 UTC (rev 4282) +++ branches/2.0/varnish-cache/bin/varnishd/cache_center.c 2009-10-08 10:13:28 UTC (rev 4283) @@ -80,17 +80,15 @@ static unsigned xids; /*-------------------------------------------------------------------- - * AGAIN - * We come here when we just completed a request and already have - * received (part of) the next one. Instead taking the detour - * around the acceptor and then back to a worker, just stay in this - * worker and do what it takes. + * WAIT + * Wait until we have a full request in our htc. */ static int -cnt_again(struct sess *sp) +cnt_wait(struct sess *sp) { int i; + struct pollfd pfd[1]; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); AZ(sp->vcl); @@ -98,12 +96,32 @@ assert(sp->xid == 0); i = HTC_Complete(sp->htc); - while (i == 0) + while (i == 0) { + if (params->session_linger > 0) { + pfd[0].fd = sp->fd; + pfd[0].events = POLLIN; + pfd[0].revents = 0; + i = poll(pfd, 1, params->session_linger); + if (i == 0) { + WSL(sp->wrk, SLT_Debug, sp->fd, "herding"); + VSL_stats->sess_herd++; + sp->wrk = NULL; + vca_return_session(sp); + return (1); + } + } i = HTC_Rx(sp->htc); + } if (i == 1) { sp->step = STP_START; } else { - vca_close_session(sp, "overflow"); + if (i == -2) + vca_close_session(sp, "overflow"); + else if (i == -1 && Tlen(sp->htc->rxbuf) == 0 && + (errno == 0 || errno == ECONNRESET)) + vca_close_session(sp, "EOF"); + else + vca_close_session(sp, "error"); sp->step = STP_DONE; } return (0); @@ -198,7 +216,6 @@ cnt_done(struct sess *sp) { double dh, dp, da; - struct pollfd pfd[1]; int i; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); @@ -267,19 +284,13 @@ } if (Tlen(sp->htc->rxbuf)) { VSL_stats->sess_readahead++; - sp->step = STP_AGAIN; + sp->step = STP_WAIT; return (0); } if (params->session_linger > 0) { - pfd[0].fd = sp->fd; - pfd[0].events = POLLIN; - pfd[0].revents = 0; - i = poll(pfd, 1, params->session_linger); - if (i > 0) { - VSL_stats->sess_linger++; - sp->step = STP_AGAIN; - return (0); - } + VSL_stats->sess_linger++; + sp->step = STP_WAIT; + return (0); } VSL_stats->sess_herd++; SES_Charge(sp); @@ -459,7 +470,6 @@ static int cnt_first(struct sess *sp) { - int i; /* * XXX: If we don't have acceptfilters we are somewhat subject @@ -480,25 +490,8 @@ sp->wrk->lastused = sp->t_open; sp->acct_req.sess++; SES_RefSrcAddr(sp); - do - i = HTC_Rx(sp->htc); - while (i == 0); - switch (i) { - case 1: - sp->step = STP_START; - break; - case -1: - vca_close_session(sp, "error"); - sp->step = STP_DONE; - break; - case -2: - vca_close_session(sp, "blast"); - sp->step = STP_DONE; - break; - default: - WRONG("Illegal return from HTC_Rx"); - } + sp->step = STP_WAIT; return (0); } Modified: branches/2.0/varnish-cache/bin/varnishd/cache_httpconn.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_httpconn.c 2009-10-08 09:38:45 UTC (rev 4282) +++ branches/2.0/varnish-cache/bin/varnishd/cache_httpconn.c 2009-10-08 10:13:28 UTC (rev 4283) @@ -92,6 +92,7 @@ (void)WS_Reserve(htc->ws, (htc->ws->e - htc->ws->s) / 2); htc->rxbuf.b = ws->f; htc->rxbuf.e = ws->f; + *htc->rxbuf.e = '\0'; htc->pipeline.b = NULL; htc->pipeline.e = NULL; } Modified: branches/2.0/varnish-cache/bin/varnishd/mgt_param.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/mgt_param.c 2009-10-08 09:38:45 UTC (rev 4282) +++ branches/2.0/varnish-cache/bin/varnishd/mgt_param.c 2009-10-08 10:13:28 UTC (rev 4283) @@ -712,7 +712,7 @@ "anything for their keep, setting it too low just means that " "more sessions take a detour around the acceptor.", EXPERIMENTAL, - "0", "ms" }, + "50", "ms" }, { "cli_buffer", tweak_uint, &master.cli_buffer, 4096, UINT_MAX, "Size of buffer for CLI input." "\nYou may need to increase this if you have big VCL files " Modified: branches/2.0/varnish-cache/bin/varnishd/steps.h =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/steps.h 2009-10-08 09:38:45 UTC (rev 4282) +++ branches/2.0/varnish-cache/bin/varnishd/steps.h 2009-10-08 10:13:28 UTC (rev 4283) @@ -29,7 +29,7 @@ * $Id$ */ -STEP(again, AGAIN) +STEP(wait, WAIT) STEP(first, FIRST) STEP(recv, RECV) STEP(start, START) From tfheen at projects.linpro.no Thu Oct 8 10:32:43 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 12:32:43 +0200 (CEST) Subject: r4284 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008103243.B31A31F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 12:32:43 +0200 (Thu, 08 Oct 2009) New Revision: 4284 Modified: branches/2.0/varnish-cache/bin/varnishd/cache.h branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_epoll.c Log: Merge r4085: Overhaul of the epoll waiter Kindly submitted by Rog?\195?\169rio Schneider. Fixes: #492 Modified: branches/2.0/varnish-cache/bin/varnishd/cache.h =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache.h 2009-10-08 10:13:28 UTC (rev 4283) +++ branches/2.0/varnish-cache/bin/varnishd/cache.h 2009-10-08 10:32:43 UTC (rev 4284) @@ -48,6 +48,10 @@ #include #include +#if defined(HAVE_EPOLL_CTL) +#include +#endif + #include "vqueue.h" #include "vsb.h" @@ -375,6 +379,10 @@ unsigned ihashptr; unsigned lhashptr; const char **hashptr; + +#if defined(HAVE_EPOLL_CTL) + struct epoll_event ev; +#endif }; Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_epoll.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_epoll.c 2009-10-08 10:13:28 UTC (rev 4283) +++ branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_epoll.c 2009-10-08 10:32:43 UTC (rev 4284) @@ -1,8 +1,10 @@ /*- * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2008 Linpro AS + * Copyright (c) 2006-2009 Linpro AS * All rights reserved. * + * Author: Rogerio Carvalho Schneider + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -36,8 +38,9 @@ #if defined(HAVE_EPOLL_CTL) #include +#include #include -#include +#include #include #include @@ -47,78 +50,148 @@ #include "cache.h" #include "cache_acceptor.h" +#define NEEV 100 + static pthread_t vca_epoll_thread; +static pthread_t vca_epoll_sess_timeout_ticker_thread; static int epfd = -1; static VTAILQ_HEAD(,sess) sesshead = VTAILQ_HEAD_INITIALIZER(sesshead); +int dotimer_pipe[2]; static void -vca_add(int fd, void *data) +vca_modadd(int fd, void *data, short arm) { - struct epoll_event ev = { EPOLLIN | EPOLLPRI, { data } }; - AZ(epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev)); + + assert(fd >= 0); + if (data == vca_pipes || data == dotimer_pipe) { + struct epoll_event ev = { EPOLLIN | EPOLLPRI | EPOLLET, { data } }; + AZ(epoll_ctl(epfd, arm, fd, &ev)); + } else { + struct sess *sp = (struct sess *)data; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + sp->ev.data.ptr = data; +#if defined(EPOLLRDHUP) + sp->ev.events = EPOLLIN | EPOLLPRI | EPOLLONESHOT | EPOLLRDHUP; +#else + sp->ev.events = EPOLLIN | EPOLLPRI | EPOLLONESHOT; +#endif + AZ(epoll_ctl(epfd, arm, fd, &sp->ev)); + } } static void -vca_del(int fd) +vca_cond_modadd(int fd, void *data) { - struct epoll_event ev = { 0, { 0 } }; - AZ(epoll_ctl(epfd, EPOLL_CTL_DEL, fd, &ev)); + struct sess *sp = (struct sess *)data; + + assert(fd >= 0); + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + if (sp->ev.data.ptr) + AZ(epoll_ctl(epfd, EPOLL_CTL_MOD, fd, &sp->ev)); + else { + sp->ev.data.ptr = data; +#if defined(EPOLLRDHUP) + sp->ev.events = EPOLLIN | EPOLLPRI | EPOLLONESHOT | EPOLLRDHUP; +#else + sp->ev.events = EPOLLIN | EPOLLPRI | EPOLLONESHOT; +#endif + AZ(epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &sp->ev)); + } } +static void +vca_eev(const struct epoll_event *ep) +{ + struct sess *ss[NEEV], *sp; + int i, j; + + AN(ep->data.ptr); + if (ep->data.ptr == vca_pipes) { + if (ep->events & EPOLLIN || ep->events & EPOLLPRI) { + j = 0; + i = read(vca_pipes[0], ss, sizeof ss); + if (i == -1 && errno == EAGAIN) + return; + while (i >= sizeof ss[0]) { + CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC); + assert(ss[j]->fd >= 0); + AZ(ss[j]->obj); + VTAILQ_INSERT_TAIL(&sesshead, ss[j], list); + vca_cond_modadd(ss[j]->fd, ss[j]); + j++; + i -= sizeof ss[0]; + } + assert(i == 0); + } + } else { + CAST_OBJ_NOTNULL(sp, ep->data.ptr, SESS_MAGIC); + if (ep->events & EPOLLIN || ep->events & EPOLLPRI) { + i = HTC_Rx(sp->htc); + if (i == 0) { + vca_modadd(sp->fd, sp, EPOLL_CTL_MOD); + return; /* more needed */ + } + VTAILQ_REMOVE(&sesshead, sp, list); + vca_handover(sp, i); + } else if (ep->events & EPOLLERR) { + VTAILQ_REMOVE(&sesshead, sp, list); + vca_close_session(sp, "ERR"); + SES_Delete(sp); + } else if (ep->events & EPOLLHUP) { + VTAILQ_REMOVE(&sesshead, sp, list); + vca_close_session(sp, "HUP"); + SES_Delete(sp); +#if defined(EPOLLRDHUP) + } else if (ep->events & EPOLLRDHUP) { + VTAILQ_REMOVE(&sesshead, sp, list); + vca_close_session(sp, "RHUP"); + SES_Delete(sp); +#endif + } + } +} + +/*--------------------------------------------------------------------*/ + static void * vca_main(void *arg) { - struct epoll_event ev; + struct epoll_event ev[NEEV], *ep; + struct sess *sp; double deadline; - int dotimer = 0; - double last_timeout = 0, tmp_timeout; - struct sess *sp, *sp2; - int i; + int dotimer, i, n; THR_SetName("cache-epoll"); (void)arg; - epfd = epoll_create(16); + epfd = epoll_create(1); assert(epfd >= 0); - vca_add(vca_pipes[0], vca_pipes); + vca_modadd(vca_pipes[0], vca_pipes, EPOLL_CTL_ADD); + vca_modadd(dotimer_pipe[0], dotimer_pipe, EPOLL_CTL_ADD); while (1) { - if ((dotimer = epoll_wait(epfd, &ev, 1, 100)) > 0) { - if (ev.data.ptr == vca_pipes) { - i = read(vca_pipes[0], &sp, sizeof sp); - assert(i == sizeof sp); - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - VTAILQ_INSERT_TAIL(&sesshead, sp, list); - vca_add(sp->fd, sp); - } else { - CAST_OBJ_NOTNULL(sp, ev.data.ptr, SESS_MAGIC); - i = HTC_Rx(sp->htc); - if (i != 0) { - VTAILQ_REMOVE(&sesshead, sp, list); - vca_del(sp->fd); - vca_handover(sp, i); - } - } + dotimer = 0; + n = epoll_wait(epfd, ev, NEEV, -1); + for (ep = ev, i = 0; i < n; i++, ep++) { + if (ep->data.ptr == dotimer_pipe && (ep->events == EPOLLIN || ep->events == EPOLLPRI)) + dotimer = 1; + else + vca_eev(ep); } - tmp_timeout = TIM_mono(); - if ((tmp_timeout - last_timeout) > 60) { - last_timeout = tmp_timeout; - } else { - if (dotimer > 0) - continue; - } + if (!dotimer) + continue; /* check for timeouts */ deadline = TIM_real() - params->sess_timeout; - VTAILQ_FOREACH_SAFE(sp, &sesshead, list, sp2) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + for (;;) { + sp = VTAILQ_FIRST(&sesshead); + if (sp == NULL) + break; if (sp->t_open > deadline) - continue; + break; VTAILQ_REMOVE(&sesshead, sp, list); - vca_del(sp->fd); - TCP_linger(sp->fd, 0); vca_close_session(sp, "timeout"); SES_Delete(sp); } @@ -127,10 +200,44 @@ /*--------------------------------------------------------------------*/ +static void * +vca_sess_timeout_ticker(void *arg) +{ + char ticker = 'R'; + char junk; + + THR_SetName("cache-epoll-sess_timeout_ticker"); + (void)arg; + + while (1) { + /* ticking */ + assert(write(dotimer_pipe[1], &ticker, 1)); + TIM_sleep(100 * 1e-3); + assert(read(dotimer_pipe[0], &junk, 1)); + } +} + +/*--------------------------------------------------------------------*/ + static void vca_epoll_init(void) { + int i; + i = fcntl(vca_pipes[0], F_GETFL); + assert(i != -1); + i |= O_NONBLOCK; + i = fcntl(vca_pipes[0], F_SETFL, i); + assert(i != -1); + + AZ(pipe(dotimer_pipe)); + i = fcntl(dotimer_pipe[0], F_GETFL); + assert(i != -1); + i |= O_NONBLOCK; + i = fcntl(dotimer_pipe[0], F_SETFL, i); + assert(i != -1); + + AZ(pthread_create(&vca_epoll_sess_timeout_ticker_thread, NULL, vca_sess_timeout_ticker, NULL)); AZ(pthread_create(&vca_epoll_thread, NULL, vca_main, NULL)); } From tfheen at projects.linpro.no Thu Oct 8 10:49:10 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 12:49:10 +0200 (CEST) Subject: r4285 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008104910.DBC0E1F738D@projects.linpro.no> Author: tfheen Date: 2009-10-08 12:49:10 +0200 (Thu, 08 Oct 2009) New Revision: 4285 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_ban.c Log: Merge r4087: Try even harder to make ban'ed objects unavailable. Modified: branches/2.0/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_ban.c 2009-10-08 10:32:43 UTC (rev 4284) +++ branches/2.0/varnish-cache/bin/varnishd/cache_ban.c 2009-10-08 10:49:10 UTC (rev 4285) @@ -529,6 +529,7 @@ return (0); } else { o->ttl = 0; + o->cacheable = 0; WSP(sp, SLT_ExpBan, "%u was banned", o->xid); EXP_Rearm(o); o->ban = NULL; From tfheen at projects.linpro.no Thu Oct 8 11:00:26 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 13:00:26 +0200 (CEST) Subject: r4286 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008110026.E571E1F738D@projects.linpro.no> Author: tfheen Date: 2009-10-08 13:00:26 +0200 (Thu, 08 Oct 2009) New Revision: 4286 Modified: branches/2.0/varnish-cache/bin/varnishd/default.vcl Log: Merge r4091: Change Varnish footer in default.vcl Hopefully make it slightly more obvious we are just the provider of the caching software in the default vcl_error. Modified: branches/2.0/varnish-cache/bin/varnishd/default.vcl =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/default.vcl 2009-10-08 10:49:10 UTC (rev 4285) +++ branches/2.0/varnish-cache/bin/varnishd/default.vcl 2009-10-08 11:00:26 UTC (rev 4286) @@ -141,8 +141,9 @@

"} obj.response {"

Guru Meditation:

XID: "} req.xid {"

+
- Varnish + Varnish cache server
From tfheen at projects.linpro.no Thu Oct 8 11:05:04 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 13:05:04 +0200 (CEST) Subject: r4287 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008110504.4AEA11F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 13:05:03 +0200 (Thu, 08 Oct 2009) New Revision: 4287 Modified: branches/2.0/varnish-cache/bin/varnishd/varnishd.1 Log: Merge r4104: Update varnishd.1 Modified: branches/2.0/varnish-cache/bin/varnishd/varnishd.1 =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/varnishd.1 2009-10-08 11:00:26 UTC (rev 4286) +++ branches/2.0/varnish-cache/bin/varnishd/varnishd.1 2009-10-08 11:05:03 UTC (rev 4287) @@ -350,6 +350,38 @@ parameter. .It Cm ping Op Ns Ar timestamp Ping the child process. +.It Cm purge Ar field Ar operator Ar argument Op && Ar field Ar operator Ar argument Op ... +Immediately invalidate all documents matching the purge expression. +See +.Va Purge expressions +for more documentation and examples. +.It Cm purge.hash Ar regex +Immediately invalidate all documents where +.Va obj.hash +matches the +.Va regex . +The default object hash contains the values from +.Va req.url +and either +.Va req.http.host +or +.Va server.ip +depending on the presence of a Host: header in the request sent by the +client. The object hash may be modified further by +.Va VCL. +.It Cm purge.list +Display the purge list. +.Pp +All requests for objects from the cache are matched against items on +the purge list. If an object in the cache is older than a matching +purge list item, it is considered "purged", and will be fetched from +the backend instead. +.Pp +When a purge expression is older than all the objects in the cache, it +is removed from the list. +.It Cm purge.url Ar regexp +Immediately invalidate all documents whose URL matches the specified +regular expression. .It Cm start Start the child process if it is not already running. .It Cm stats @@ -363,8 +395,7 @@ .It Cm stop Stop the child process. .It Cm url.purge Ar regexp -Immediately invalidate all documents whos URL matches the specified -regular expression. +Deprecated, see purge.url instead .It Cm vcl.discard Ar configname Discard the configuration specified by .Ar configname . @@ -583,6 +614,46 @@ The default is .Dv off . .El +.Ss Purge expressions +A purge expression consists of one or more conditions. A condition +consists of a field, a operator, and an argument. Conditions can be +ANDed together with "&&". +.Pp +A field can be any of the variables from VCL, for instance +.Va req.url , +.Va req.http.host +or +.Va obj.set-cookie . +.Pp +Operators are "==" for direct comparision, "~" for a regular +expression match, and ">" or "<" for size comparisons. Prepending an +operator with "!" negates the expression. +.Pp +The argument could be a quoted string, a regexp, or an integer. +Integers can have "KB", "MB", "GB" or "TB" appended for size related +fields. +.Pp +Simple example: All requests where +.Va req.url +exactly matches the string +.Va /news +are purged from the cache. +.Bd -literal -offset 4n +req.url == "/news" +.Ed +.Pp +Example: Purge all documents where the name does not end with ".ogg", +and where the size of the object is greater than 10 megabytes. +.Bd -literal -offset 4n +req.url !~ "\.ogg" && obj.size > 10MB +.Ed +.Pp +Example: Purge all documents where the serving host is "example.com" +or "www.example.com", and where the Set-Cookie header received from +the backend contains "USERID=1663". +.Bd -literal -offset 4n +req.http.host ~ "^(www\.)example.com$" && obj.set-cookie ~ "USERID=1663" +.Ed .Sh SEE ALSO .Xr varnishlog 1 , .Xr varnishhist 1 , @@ -609,4 +680,6 @@ .An Poul-Henning Kamp Aq phk at phk.freebsd.dk in cooperation with Verdens Gang AS and Linpro AS. This manual page was written by -.An Dag-Erling Sm\(/orgrav Aq des at des.no . +.An Dag-Erling Sm\(/orgrav Aq des at des.no +with updates by +.An Stig Sandbeck Mathisen Aq ssm at debian.org From tfheen at projects.linpro.no Thu Oct 8 11:10:41 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 13:10:41 +0200 (CEST) Subject: r4288 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008111041.107B61F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 13:10:40 +0200 (Thu, 08 Oct 2009) New Revision: 4288 Modified: branches/2.0/varnish-cache/bin/varnishd/varnishd.1 Log: Merge r4105: Update documentation for the varnishd management interface Modified: branches/2.0/varnish-cache/bin/varnishd/varnishd.1 =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/varnishd.1 2009-10-08 11:05:03 UTC (rev 4287) +++ branches/2.0/varnish-cache/bin/varnishd/varnishd.1 2009-10-08 11:10:40 UTC (rev 4288) @@ -326,8 +326,12 @@ address and port. The following commands are available: .Bl -tag -width 4n -.It Cm help +.It Cm help Op Ar command Display a list of available commands. +.Pp +If the +.Ar command +is specified, display help for this command. .It Cm param.set Ar param Ar value Set the parameter specified by .Ar param @@ -336,20 +340,24 @@ See .Sx Run-Time Parameters for a list of parameters. -.It Cm param.show Ar param -Display the value of the parameter specified by -.Ar param . -See -.Sx Run-Time Parameters -for a list of parameters. -.It Cm param.show Op Fl l -Display a list of run-time parameters and their values. +.It Xo +.Cm param.show +.Op Fl l +.Op Ar param +.Xc +Display a list if run-time parameters and their values. +.Pp If the .Fl l option is specified, the list includes a brief explanation of each parameter. +.Pp +If a +.Ar param +is specified, display only the value and explanation for this +parameter. .It Cm ping Op Ns Ar timestamp -Ping the child process. +Ping the Varnish cache process, keeping the connection alive. .It Cm purge Ar field Ar operator Ar argument Op && Ar field Ar operator Ar argument Op ... Immediately invalidate all documents matching the purge expression. See @@ -382,20 +390,25 @@ .It Cm purge.url Ar regexp Immediately invalidate all documents whose URL matches the specified regular expression. +.It Cm quit +Close the connection to the varnish admin port. .It Cm start -Start the child process if it is not already running. +Start the Varnish cache process if it is not already running. .It Cm stats -Display server statistics. +Show summary statistics. +.Pp All the numbers presented are totals since server startup; for a better idea of the current situation, use the .Xr varnishstat 1 utility. .It Cm status -Check the status of the child process. +Check the status of the Varnish cache process. .It Cm stop -Stop the child process. +Stop the Varnish cache process. .It Cm url.purge Ar regexp -Deprecated, see purge.url instead +Deprecated, see +.Cm purge.url +instead. .It Cm vcl.discard Ar configname Discard the configuration specified by .Ar configname . From tfheen at projects.linpro.no Thu Oct 8 11:18:31 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 13:18:31 +0200 (CEST) Subject: r4289 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008111831.AEE641F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 13:18:31 +0200 (Thu, 08 Oct 2009) New Revision: 4289 Modified: branches/2.0/varnish-cache/bin/varnishd/varnishd.1 Log: Merge r4106: varnishd.1: fix regular expression examples Modified: branches/2.0/varnish-cache/bin/varnishd/varnishd.1 =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/varnishd.1 2009-10-08 11:10:40 UTC (rev 4288) +++ branches/2.0/varnish-cache/bin/varnishd/varnishd.1 2009-10-08 11:18:31 UTC (rev 4289) @@ -658,14 +658,14 @@ Example: Purge all documents where the name does not end with ".ogg", and where the size of the object is greater than 10 megabytes. .Bd -literal -offset 4n -req.url !~ "\.ogg" && obj.size > 10MB +req.url !~ "\\.ogg$" && obj.size > 10MB .Ed .Pp Example: Purge all documents where the serving host is "example.com" or "www.example.com", and where the Set-Cookie header received from the backend contains "USERID=1663". .Bd -literal -offset 4n -req.http.host ~ "^(www\.)example.com$" && obj.set-cookie ~ "USERID=1663" +req.http.host ~ "^(www\\.)example.com$" && obj.set-cookie ~ "USERID=1663" .Ed .Sh SEE ALSO .Xr varnishlog 1 , From tfheen at projects.linpro.no Thu Oct 8 11:28:22 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 13:28:22 +0200 (CEST) Subject: r4290 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008112822.9E64D1F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 13:28:22 +0200 (Thu, 08 Oct 2009) New Revision: 4290 Modified: branches/2.0/varnish-cache/bin/varnishd/varnishd.1 Log: Merge r4121: Use correct syntax for quotes, emphasis and punctuation in varnishd.1 Modified: branches/2.0/varnish-cache/bin/varnishd/varnishd.1 =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/varnishd.1 2009-10-08 11:18:31 UTC (rev 4289) +++ branches/2.0/varnish-cache/bin/varnishd/varnishd.1 2009-10-08 11:28:22 UTC (rev 4290) @@ -375,15 +375,18 @@ or .Va server.ip depending on the presence of a Host: header in the request sent by the -client. The object hash may be modified further by +client. +The object hash may be modified further by .Va VCL. .It Cm purge.list Display the purge list. .Pp All requests for objects from the cache are matched against items on -the purge list. If an object in the cache is older than a matching -purge list item, it is considered "purged", and will be fetched from -the backend instead. +the purge list. +If an object in the cache is older than a matching purge list item, it +is considered +.Qq purged , +and will be fetched from the backend instead. .Pp When a purge expression is older than all the objects in the cache, it is removed from the list. @@ -628,9 +631,10 @@ .Dv off . .El .Ss Purge expressions -A purge expression consists of one or more conditions. A condition -consists of a field, a operator, and an argument. Conditions can be -ANDed together with "&&". +A purge expression consists of one or more conditions. +A condition consists of a field, an operator, and an argument. +Conditions can be ANDed together with +.Qq && . .Pp A field can be any of the variables from VCL, for instance .Va req.url , @@ -638,13 +642,27 @@ or .Va obj.set-cookie . .Pp -Operators are "==" for direct comparision, "~" for a regular -expression match, and ">" or "<" for size comparisons. Prepending an -operator with "!" negates the expression. +Operators are +.Qq == +for direct comparision, +.Qq ~ +for a regular expression match, and +.Qq > +or +.Qq < +for size comparisons. +Prepending an operator with +.Qq \&! +negates the expression. .Pp The argument could be a quoted string, a regexp, or an integer. -Integers can have "KB", "MB", "GB" or "TB" appended for size related -fields. +Integers can have +.Qq KB , +.Qq MB , +.Qq GB +or +.Qq TB +appended for size related fields. .Pp Simple example: All requests where .Va req.url @@ -655,15 +673,19 @@ req.url == "/news" .Ed .Pp -Example: Purge all documents where the name does not end with ".ogg", +Example: Purge all documents where the name does not end with +.Qq .ogg , and where the size of the object is greater than 10 megabytes. .Bd -literal -offset 4n req.url !~ "\\.ogg$" && obj.size > 10MB .Ed .Pp -Example: Purge all documents where the serving host is "example.com" -or "www.example.com", and where the Set-Cookie header received from -the backend contains "USERID=1663". +Example: Purge all documents where the serving host is +.Qq example.com +or +.Qq www.example.com , +and where the Set-Cookie header received from the backend contains +.Qq USERID=1663 . .Bd -literal -offset 4n req.http.host ~ "^(www\\.)example.com$" && obj.set-cookie ~ "USERID=1663" .Ed From tfheen at projects.linpro.no Thu Oct 8 11:39:00 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 13:39:00 +0200 (CEST) Subject: r4291 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008113900.8BCBB1F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 13:39:00 +0200 (Thu, 08 Oct 2009) New Revision: 4291 Modified: branches/2.0/varnish-cache/bin/varnishd/mgt_child.c Log: Merge r4123: If child fails to start, stop it. If we fail to start the child, set it to state stopping to prevent auto_restart from looping over what is almost guaranteed to be a problem it cannot solve. Modified: branches/2.0/varnish-cache/bin/varnishd/mgt_child.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/mgt_child.c 2009-10-08 11:28:22 UTC (rev 4290) +++ branches/2.0/varnish-cache/bin/varnishd/mgt_child.c 2009-10-08 11:39:00 UTC (rev 4291) @@ -345,7 +345,6 @@ e->callback = child_listener; AZ(vev_add(mgt_evb, e)); ev_listen = e; - AZ(ev_poker); if (params->ping_interval > 0) { e = vev_new(); @@ -362,11 +361,10 @@ if (mgt_push_vcls_and_start(&u, &p)) { REPORT(LOG_ERR, "Pushing vcls failed: %s", p); free(p); - /* Pick up any stuff lingering on stdout/stderr */ - (void)child_listener(NULL, EV_RD); - exit(2); - } - child_state = CH_RUNNING; + child_state = CH_RUNNING; + mgt_stop_child(); + } else + child_state = CH_RUNNING; } /*--------------------------------------------------------------------*/ From tfheen at projects.linpro.no Thu Oct 8 11:44:18 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 13:44:18 +0200 (CEST) Subject: r4292 - branches/2.0/varnish-cache/bin/varnishtop Message-ID: <20091008114418.C607A3818F@projects.linpro.no> Author: tfheen Date: 2009-10-08 13:44:18 +0200 (Thu, 08 Oct 2009) New Revision: 4292 Modified: branches/2.0/varnish-cache/bin/varnishtop/varnishtop.c Log: Merge r4130: Fix once mode in varnishtop -1 mode in varnishtop was broken with the variable length shmlog fields. Fix this. Fixes #514 Modified: branches/2.0/varnish-cache/bin/varnishtop/varnishtop.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishtop/varnishtop.c 2009-10-08 11:39:00 UTC (rev 4291) +++ branches/2.0/varnish-cache/bin/varnishtop/varnishtop.c 2009-10-08 11:44:18 UTC (rev 4292) @@ -267,8 +267,8 @@ VTAILQ_FOREACH_SAFE(tp, &top_head, list, tp2) { if (tp->count <= 1.0) break; - len = tp->rec[1]; - printf("%9.2f %*.*s\n", tp->count, len, len, tp->rec + 4); + len = SHMLOG_LEN(tp->rec); + printf("%9.2f %*.*s\n", tp->count, len, len, tp->rec + SHMLOG_DATA); } } From tfheen at projects.linpro.no Thu Oct 8 11:49:01 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 13:49:01 +0200 (CEST) Subject: r4293 - branches/2.0/varnish-cache/bin/varnishtop Message-ID: <20091008114901.ECA921F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 13:49:01 +0200 (Thu, 08 Oct 2009) New Revision: 4293 Modified: branches/2.0/varnish-cache/bin/varnishtop/varnishtop.c Log: Merge r4131: Add the tag to run once mode Previously, we did not print out the tag in -1 mode, just the value. This has been changed so we now print both. Modified: branches/2.0/varnish-cache/bin/varnishtop/varnishtop.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishtop/varnishtop.c 2009-10-08 11:44:18 UTC (rev 4292) +++ branches/2.0/varnish-cache/bin/varnishtop/varnishtop.c 2009-10-08 11:49:01 UTC (rev 4293) @@ -268,7 +268,7 @@ if (tp->count <= 1.0) break; len = SHMLOG_LEN(tp->rec); - printf("%9.2f %*.*s\n", tp->count, len, len, tp->rec + SHMLOG_DATA); + printf("%9.2f %s %*.*s\n", tp->count, VSL_tags[tp->rec[SHMLOG_TAG]], len, len, tp->rec + SHMLOG_DATA); } } From tfheen at projects.linpro.no Thu Oct 8 12:05:56 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 14:05:56 +0200 (CEST) Subject: r4294 - in branches/2.0/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20091008120556.739771F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 14:05:56 +0200 (Thu, 08 Oct 2009) New Revision: 4294 Added: branches/2.0/varnish-cache/bin/varnishtest/tests/c00025.vtc branches/2.0/varnish-cache/bin/varnishtest/tests/c00026.vtc Modified: branches/2.0/varnish-cache/bin/varnishd/cache_response.c Log: Merge r4146: Add if-none-match support Add if-none-match support to allow people with compliant browsers to save a decent chunk of bandwidth Modified: branches/2.0/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_response.c 2009-10-08 11:49:01 UTC (rev 4293) +++ branches/2.0/varnish-cache/bin/varnishd/cache_response.c 2009-10-08 12:05:56 UTC (rev 4294) @@ -56,8 +56,10 @@ http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Date: %s", lm); http_SetHeader(sp->wrk, sp->fd, sp->http, "Via: 1.1 varnish"); http_PrintfHeader(sp->wrk, sp->fd, sp->http, "X-Varnish: %u", sp->xid); - TIM_format(sp->obj->last_modified, lm); - http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Last-Modified: %s", lm); + if (sp->obj->last_modified) { + TIM_format(sp->obj->last_modified, lm); + http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Last-Modified: %s", lm); + } http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Connection: %s", sp->doclose ? "close" : "keep-alive"); sp->wantbody = 0; @@ -68,17 +70,33 @@ static int res_do_conds(struct sess *sp) { - char *p; + char *p, *e; double ims; + int do_cond = 0; - if (sp->obj->last_modified > 0 && - http_GetHdr(sp->http, H_If_Modified_Since, &p)) { + /* RFC 2616 13.3.4 states we need to match both ETag + and If-Modified-Since if present*/ + + if (http_GetHdr(sp->http, H_If_Modified_Since, &p) ) { + if (!sp->obj->last_modified) + return (0); ims = TIM_parse(p); if (ims > sp->t_req) /* [RFC2616 14.25] */ return (0); if (sp->obj->last_modified > ims) { return (0); } + do_cond = 1; + } + + if (http_GetHdr(sp->http, H_If_None_Match, &p) && + http_GetHdr(sp->obj->http, H_ETag, &e)) { + if (strcmp(p,e) != 0) + return (0); + do_cond = 1; + } + + if (do_cond == 1) { res_do_304(sp); return (1); } Copied: branches/2.0/varnish-cache/bin/varnishtest/tests/c00025.vtc (from rev 4146, trunk/varnish-cache/bin/varnishtest/tests/c00025.vtc) =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/c00025.vtc (rev 0) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/c00025.vtc 2009-10-08 12:05:56 UTC (rev 4294) @@ -0,0 +1,33 @@ +# $Id$ + +test "Test If-None-Match" + +server s1 { + rxreq + expect req.url == "/foo" + txresp -hdr "ETag: 123456789" \ + -body "11111\n" +} -start + +varnish v1 -vcl+backend { } -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.http.content-length == 6 + + txreq -url "/foo" \ + -hdr "If-None-Match: 12345678" + rxresp + expect resp.status == 200 + + txreq -url "/foo" \ + -hdr "If-None-Match: 123456789" + rxresp + expect resp.status == 304 +} + +client c1 -run + +client c1 -run Copied: branches/2.0/varnish-cache/bin/varnishtest/tests/c00026.vtc (from rev 4146, trunk/varnish-cache/bin/varnishtest/tests/c00026.vtc) =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/c00026.vtc (rev 0) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/c00026.vtc 2009-10-08 12:05:56 UTC (rev 4294) @@ -0,0 +1,53 @@ +# $Id$ + +test "Test Combination of If-None-Match and If-Modified-Since" + +server s1 { + rxreq + expect req.url == "/foo" + txresp -hdr "ETag: 123456789" \ + -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -body "11111\n" +} -start + +varnish v1 -vcl+backend { } -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.http.content-length == 6 + + txreq -url "/foo" \ + -hdr "If-None-Match: 123456789" + rxresp + expect resp.status == 304 + + txreq -url "/foo" \ + -hdr "If-Modified-Since: Thu, 26 Jun 2008 12:00:01 GMT" + rxresp + expect resp.status == 304 + + txreq -url "/foo" \ + -hdr "If-Modified-Since: Thu, 26 Jun 2008 12:00:00 GMT" \ + -hdr "If-None-Match: 123456789" + rxresp + expect resp.status == 200 + + txreq -url "/foo" \ + -hdr "If-Modified-Since: Thu, 26 Jun 2008 12:00:01 GMT" \ + -hdr "If-None-Match: 12345678" + rxresp + expect resp.status == 200 + + txreq -url "/foo" \ + -hdr "If-Modified-Since: Thu, 26 Jun 2008 12:00:01 GMT" \ + -hdr "If-None-Match: 123456789" + rxresp + expect resp.status == 304 + +} + +client c1 -run + +client c1 -run From tfheen at projects.linpro.no Thu Oct 8 12:26:18 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 14:26:18 +0200 (CEST) Subject: r4295 - in branches/2.0/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20091008122618.D2F2F1F73A2@projects.linpro.no> Author: tfheen Date: 2009-10-08 14:26:18 +0200 (Thu, 08 Oct 2009) New Revision: 4295 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_response.c branches/2.0/varnish-cache/bin/varnishtest/tests/c00008.vtc Log: Merge r4150: Emit more headers when hitting 304 When hitting 304, ETag, Content-Location, Expires, Cache-control and Vary should all be sent to the client. See http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5 for some more details Thanks to Rog?\195?\169rio Schneider for the patch. Fixes #529 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_response.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_response.c 2009-10-08 12:05:56 UTC (rev 4294) +++ branches/2.0/varnish-cache/bin/varnishd/cache_response.c 2009-10-08 12:26:18 UTC (rev 4295) @@ -46,6 +46,7 @@ res_do_304(struct sess *sp) { char lm[64]; + char *p; WSP(sp, SLT_Length, "%u", 0); @@ -60,6 +61,19 @@ TIM_format(sp->obj->last_modified, lm); http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Last-Modified: %s", lm); } + + /* http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5 */ + if (http_GetHdr(sp->obj->http, H_Cache_Control, &p)) + http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Cache-Control: %s", p); + if (http_GetHdr(sp->obj->http, H_Content_Location, &p)) + http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Content-Location: %s", p); + if (http_GetHdr(sp->obj->http, H_ETag, &p)) + http_PrintfHeader(sp->wrk, sp->fd, sp->http, "ETag: %s", p); + if (http_GetHdr(sp->obj->http, H_Expires, &p)) + http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Expires: %s", p); + if (http_GetHdr(sp->obj->http, H_Vary, &p)) + http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Vary: %s", p); + http_PrintfHeader(sp->wrk, sp->fd, sp->http, "Connection: %s", sp->doclose ? "close" : "keep-alive"); sp->wantbody = 0; Modified: branches/2.0/varnish-cache/bin/varnishtest/tests/c00008.vtc =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/c00008.vtc 2009-10-08 12:05:56 UTC (rev 4294) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/c00008.vtc 2009-10-08 12:26:18 UTC (rev 4295) @@ -6,6 +6,7 @@ rxreq expect req.url == "/foo" txresp -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \ + -hdr "ETag: foo" \ -body "11111\n" } -start @@ -15,22 +16,26 @@ txreq -url "/foo" rxresp expect resp.status == 200 + expect resp.http.etag == "foo" expect resp.http.content-length == 6 txreq -url "/foo" \ -hdr "If-Modified-Since: Thu, 26 Jun 2008 12:00:00 GMT" rxresp expect resp.status == 200 + expect resp.http.etag == "foo" txreq -url "/foo" \ -hdr "If-Modified-Since: Thu, 26 Jun 2008 12:00:01 GMT" rxresp expect resp.status == 304 + expect resp.http.etag == "foo" txreq -url "/foo" \ -hdr "If-Modified-Since: Thu, 26 Jun 2008 12:00:02 GMT" rxresp expect resp.status == 304 + expect resp.http.etag == "foo" } client c1 -run From tfheen at projects.linpro.no Thu Oct 8 12:47:16 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 14:47:16 +0200 (CEST) Subject: r4296 - branches/2.0/varnish-cache/bin/varnishtest Message-ID: <20091008124716.BD23B1F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 14:47:16 +0200 (Thu, 08 Oct 2009) New Revision: 4296 Modified: branches/2.0/varnish-cache/bin/varnishtest/vtc_varnish.c Log: Merge r4151: Add #ifdef for WCOREDUMP AIX doesn't have WCOREDUMP, so compilation failed there. Add #ifdef. Partially addresses #531. Modified: branches/2.0/varnish-cache/bin/varnishtest/vtc_varnish.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/vtc_varnish.c 2009-10-08 12:26:18 UTC (rev 4295) +++ branches/2.0/varnish-cache/bin/varnishtest/vtc_varnish.c 2009-10-08 12:47:16 UTC (rev 4296) @@ -326,9 +326,14 @@ vtc_log(v->vl, 2, "R %d Status: %04x", r, status); if (WIFEXITED(status) && WEXITSTATUS(status) == 2) return; +#ifdef WCOREDUMP vtc_log(v->vl, 0, "Bad exit code: %04x sig %x exit %x core %x", status, WTERMSIG(status), WEXITSTATUS(status), WCOREDUMP(status)); +#else + vtc_log(v->vl, 0, "Bad exit code: %04x sig %x exit %x", + status, WTERMSIG(status), WEXITSTATUS(status)); +#endif } /********************************************************************** From tfheen at projects.linpro.no Thu Oct 8 12:57:04 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 14:57:04 +0200 (CEST) Subject: r4297 - branches/2.0/varnish-cache/bin/varnishreplay Message-ID: <20091008125704.78EA01F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 14:57:04 +0200 (Thu, 08 Oct 2009) New Revision: 4297 Modified: branches/2.0/varnish-cache/bin/varnishreplay/varnishreplay.c Log: Merge r4152: Rename struct thread to struct replay_thread in varnishreplay AIX has a "struct thread" in pthread.h, which conflicts with our struct thread. Rename ours to replay_thread. Fixes #531 Thanks to demik for patch Modified: branches/2.0/varnish-cache/bin/varnishreplay/varnishreplay.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishreplay/varnishreplay.c 2009-10-08 12:47:16 UTC (rev 4296) +++ branches/2.0/varnish-cache/bin/varnishreplay/varnishreplay.c 2009-10-08 12:57:04 UTC (rev 4297) @@ -190,7 +190,7 @@ pthread_mutex_unlock(&log_mutex); } -struct thread { +struct replay_thread { pthread_t thread_id; struct mailbox mbox; @@ -212,14 +212,14 @@ char temp[2048]; }; -static struct thread **threads; +static struct replay_thread **threads; static size_t nthreads; /* * Clear thread state */ static void -thread_clear(struct thread *thr) +thread_clear(struct replay_thread *thr) { thr->method = thr->proto = thr->url = NULL; @@ -236,17 +236,17 @@ thr->sock = -1; } -#define THREAD_FAIL ((struct thread *)-1) +#define THREAD_FAIL ((struct replay_thread *)-1) static pthread_attr_t thread_attr; -static struct thread * +static struct replay_thread * thread_get(int fd, void *(*thread_main)(void *)) { assert(fd != 0); if (fd >= nthreads) { - struct thread **newthreads = threads; + struct replay_thread **newthreads = threads; size_t newnthreads = nthreads; while (fd >= newnthreads) @@ -309,7 +309,7 @@ * Allocate from thread arena */ static void * -thread_alloc(struct thread *thr, size_t len) +thread_alloc(struct replay_thread *thr, size_t len) { void *ptr; @@ -325,7 +325,7 @@ * trimmed. */ static char * -trimline(struct thread *thr, const char *str) +trimline(struct replay_thread *thr, const char *str) { size_t len; char *p; @@ -355,7 +355,7 @@ * A line is terminated by \r\n */ static int -read_line(struct thread *thr) +read_line(struct replay_thread *thr) { int i, len; @@ -389,7 +389,7 @@ * the number of bytes read. */ static int -read_block(struct thread *thr, int len) +read_block(struct replay_thread *thr, int len) { int n, r, tot; @@ -412,7 +412,7 @@ /* Receive the response after sending a request. */ static int -receive_response(struct thread *thr) +receive_response(struct replay_thread *thr) { const char *next; int line_len; @@ -496,7 +496,7 @@ { struct iovec iov[6]; char space[1] = " ", crlf[2] = "\r\n"; - struct thread *thr = arg; + struct replay_thread *thr = arg; struct message *msg; enum shmlogtag tag; size_t len; @@ -642,7 +642,7 @@ gen_traffic(void *priv, enum shmlogtag tag, unsigned fd, unsigned len, unsigned spec, const char *ptr) { - struct thread *thr; + struct replay_thread *thr; const char *end; struct message *msg; From tfheen at projects.linpro.no Thu Oct 8 13:07:00 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 15:07:00 +0200 (CEST) Subject: r4298 - branches/2.0/varnish-cache/lib/libvarnish Message-ID: <20091008130700.AB1F81F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 15:07:00 +0200 (Thu, 08 Oct 2009) New Revision: 4298 Modified: branches/2.0/varnish-cache/lib/libvarnish/tcp.c Log: Merge r4171: Implement the linux equivalent of BSD accept filter using TCP_DEFER_ACCEPT Modified: branches/2.0/varnish-cache/lib/libvarnish/tcp.c =================================================================== --- branches/2.0/varnish-cache/lib/libvarnish/tcp.c 2009-10-08 12:57:04 UTC (rev 4297) +++ branches/2.0/varnish-cache/lib/libvarnish/tcp.c 2009-10-08 13:07:00 UTC (rev 4298) @@ -36,6 +36,10 @@ #include +#ifdef __linux +#include +#endif + #include #include #ifdef HAVE_SYS_FILIO_H @@ -113,6 +117,10 @@ printf("Acceptfilter(%d, httpready): %d %s\n", sock, i, strerror(errno)); return (i); +#elif defined(__linux) + int defer = 1; + setsockopt(sock, SOL_TCP,TCP_DEFER_ACCEPT,(char *) &defer, sizeof(int)); + return (0); #else (void)sock; return (0); From tfheen at projects.linpro.no Thu Oct 8 13:13:14 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 15:13:14 +0200 (CEST) Subject: r4299 - branches/2.0/varnish-cache/bin/varnishtop Message-ID: <20091008131314.D8BDC1F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 15:13:14 +0200 (Thu, 08 Oct 2009) New Revision: 4299 Modified: branches/2.0/varnish-cache/bin/varnishtop/varnishtop.c Log: Merge r4173: varnishtop: handle shm_reclen > 255 varnishtop used to have a hard-coded max record length of 255 bytes. This broke with the change that allows larger record length. Use a dynamic buffer instead. Modified: branches/2.0/varnish-cache/bin/varnishtop/varnishtop.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishtop/varnishtop.c 2009-10-08 13:07:00 UTC (rev 4298) +++ branches/2.0/varnish-cache/bin/varnishtop/varnishtop.c 2009-10-08 13:13:14 UTC (rev 4299) @@ -54,7 +54,8 @@ #include "varnishapi.h" struct top { - unsigned char rec[4 + 255]; + unsigned char rec[4]; + unsigned char *rec_data; unsigned clen; unsigned hash; VTAILQ_ENTRY(top) list; @@ -99,7 +100,7 @@ continue; if (tp->clen != q - p) continue; - if (memcmp(p + SHMLOG_DATA, tp->rec + SHMLOG_DATA, + if (memcmp(p + SHMLOG_DATA, tp->rec_data, q - (p + SHMLOG_DATA))) continue; tp->count += 1.0; @@ -109,12 +110,15 @@ ntop++; tp = calloc(sizeof *tp, 1); assert(tp != NULL); + tp->rec_data = calloc(l, 1); + assert(tp->rec_data != NULL); tp->hash = u; tp->count = 1.0; tp->clen = q - p; VTAILQ_INSERT_TAIL(&top_head, tp, list); } - memcpy(tp->rec, p, SHMLOG_DATA + l); + memcpy(tp->rec, p, SHMLOG_DATA - 1); + memcpy(tp->rec_data, p + SHMLOG_DATA, l); while (1) { tp2 = VTAILQ_PREV(tp, tophead, list); if (tp2 == NULL || tp2->count >= tp->count) @@ -157,12 +161,13 @@ mvprintw(l, 0, "%9.2f %-*.*s %*.*s\n", tp->count, maxfieldlen, maxfieldlen, VSL_tags[tp->rec[SHMLOG_TAG]], - len, len, tp->rec + SHMLOG_DATA); + len, len, tp->rec_data); t = tp->count; } tp->count *= .999; if (tp->count * 10 < t || l > LINES * 10) { VTAILQ_REMOVE(&top_head, tp, list); + free(tp->rec_data); free(tp); ntop--; } From tfheen at projects.linpro.no Thu Oct 8 13:17:00 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 15:17:00 +0200 (CEST) Subject: r4300 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008131700.E0ECF1F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 15:17:00 +0200 (Thu, 08 Oct 2009) New Revision: 4300 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_lck.c Log: Merge r4171: In Lck__Trylock we probably do want to try locks -- and check return value not errno Modified: branches/2.0/varnish-cache/bin/varnishd/cache_lck.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_lck.c 2009-10-08 13:13:14 UTC (rev 4299) +++ branches/2.0/varnish-cache/bin/varnishd/cache_lck.c 2009-10-08 13:17:00 UTC (rev 4300) @@ -111,8 +111,8 @@ int r; CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC); - r = pthread_mutex_lock(&ilck->mtx); - assert(r == 0 || errno == EBUSY); + r = pthread_mutex_trylock(&ilck->mtx); + assert(r == 0 || r == EBUSY); if (params->diag_bitmap & 0x8) VSL(SLT_Debug, 0, "MTX_TRYLOCK(%s,%s,%d,%s) = %d", p, f, l, ilck->w); From tfheen at projects.linpro.no Thu Oct 8 13:23:37 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 15:23:37 +0200 (CEST) Subject: r4301 - in branches/2.0/varnish-cache: bin/varnishd include lib/libvarnish Message-ID: <20091008132337.2933F1F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 15:23:36 +0200 (Thu, 08 Oct 2009) New Revision: 4301 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_ban.c branches/2.0/varnish-cache/include/vsb.h branches/2.0/varnish-cache/lib/libvarnish/cli_common.c branches/2.0/varnish-cache/lib/libvarnish/vsb.c Log: Merge r4176: Give vsb_quote() an optional length paramter (Pass -1 for strlen). Add a complementary vsb_unquote() function. Modified: branches/2.0/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_ban.c 2009-10-08 13:17:00 UTC (rev 4300) +++ branches/2.0/varnish-cache/bin/varnishd/cache_ban.c 2009-10-08 13:23:36 UTC (rev 4301) @@ -366,7 +366,7 @@ sb = vsb_newauto(); XXXAN(sb); vsb_printf(sb, "%s %s ", a1, a2); - vsb_quote(sb, a3, 0); + vsb_quote(sb, a3, -1, 0); vsb_finish(sb); AZ(vsb_overflowed(sb)); bt->test = strdup(vsb_data(sb)); Modified: branches/2.0/varnish-cache/include/vsb.h =================================================================== --- branches/2.0/varnish-cache/include/vsb.h 2009-10-08 13:17:00 UTC (rev 4300) +++ branches/2.0/varnish-cache/include/vsb.h 2009-10-08 13:23:36 UTC (rev 4301) @@ -77,7 +77,8 @@ int vsb_len(struct vsb *); int vsb_done(const struct vsb *); void vsb_delete(struct vsb *); -void vsb_quote(struct vsb *s, const char *p, int how); +void vsb_quote(struct vsb *s, const char *p, int len, int how); +const char *vsb_unquote(struct vsb *s, const char *p, int len, int how); #ifdef __cplusplus }; #endif Modified: branches/2.0/varnish-cache/lib/libvarnish/cli_common.c =================================================================== --- branches/2.0/varnish-cache/lib/libvarnish/cli_common.c 2009-10-08 13:17:00 UTC (rev 4300) +++ branches/2.0/varnish-cache/lib/libvarnish/cli_common.c 2009-10-08 13:23:36 UTC (rev 4301) @@ -73,7 +73,7 @@ cli_quote(struct cli *cli, const char *s) { - vsb_quote(cli->sb, s, 0); + vsb_quote(cli->sb, s, -1, 0); } void Modified: branches/2.0/varnish-cache/lib/libvarnish/vsb.c =================================================================== --- branches/2.0/varnish-cache/lib/libvarnish/vsb.c 2009-10-08 13:17:00 UTC (rev 4300) +++ branches/2.0/varnish-cache/lib/libvarnish/vsb.c 2009-10-08 13:23:36 UTC (rev 4301) @@ -481,25 +481,27 @@ * Quote a string */ void -vsb_quote(struct vsb *s, const char *p, int how) +vsb_quote(struct vsb *s, const char *p, int len, int how) { const char *q; int quote = 0; (void)how; /* For future enhancements */ + if (len == -1) + len = strlen(p); - for (q = p; *q != '\0'; q++) { + for (q = p; q < p + len; q++) { if (!isgraph(*q) || *q == '"') { quote++; break; } } if (!quote) { - (void)vsb_cat(s, p); + (void)vsb_bcat(s, p, len); return; } (void)vsb_putc(s, '"'); - for (q = p; *q != '\0'; q++) { + for (q = p; q < p + len; q++) { switch (*q) { case ' ': (void)vsb_putc(s, *q); @@ -522,9 +524,66 @@ if (isgraph(*q)) (void)vsb_putc(s, *q); else - (void)vsb_printf(s, "\\%o", *q); + (void)vsb_printf(s, "\\%o", *q & 0xff); break; } } (void)vsb_putc(s, '"'); } + +/* + * Unquote a string + */ +const char * +vsb_unquote(struct vsb *s, const char *p, int len, int how) +{ + const char *q; + char *r; + unsigned long u; + char c; + + (void)how; /* For future enhancements */ + + if (len == -1) + len = strlen(p); + + for (q = p; q < p + len; q++) { + if (*q != '\\') { + (void)vsb_bcat(s, q, 1); + continue; + } + if (++q >= p + len) + return ("Incomplete '\\'-sequence at end of string"); + + switch(*q) { + case 'n': + (void)vsb_bcat(s, "\n", 1); + continue; + case 'r': + (void)vsb_bcat(s, "\r", 1); + continue; + case 't': + (void)vsb_bcat(s, "\t", 1); + continue; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + errno = 0; + u = strtoul(q, &r, 8); + if (errno != 0 || (u & ~0xff)) + return ("\\ooo sequence out of range"); + c = (char)u; + (void)vsb_bcat(s, &c, 1); + q = r - 1; + continue; + default: + (void)vsb_bcat(s, q, 1); + } + } + return (NULL); +} From tfheen at projects.linpro.no Thu Oct 8 13:34:03 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 15:34:03 +0200 (CEST) Subject: r4302 - in branches/2.0/varnish-cache: bin/varnishd include Message-ID: <20091008133403.8CEEC1F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 15:34:03 +0200 (Thu, 08 Oct 2009) New Revision: 4302 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_backend.c branches/2.0/varnish-cache/include/stat_field.h Log: Merge r4177: Add a stats counter for backend connections which we came too late to recycle Modified: branches/2.0/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_backend.c 2009-10-08 13:23:36 UTC (rev 4301) +++ branches/2.0/varnish-cache/bin/varnishd/cache_backend.c 2009-10-08 13:34:03 UTC (rev 4302) @@ -314,11 +314,11 @@ if (VBE_CheckFd(vc->fd)) { /* XXX locking of stats */ VSL_stats->backend_reuse += 1; - VSL_stats->backend_conn++; WSP(sp, SLT_Backend, "%d %s %s", vc->fd, sp->director->vcl_name, bp->vcl_name); return (vc); } + VSL_stats->backend_toolate++; sp->vbe = vc; VBE_ClosedFd(sp); } Modified: branches/2.0/varnish-cache/include/stat_field.h =================================================================== --- branches/2.0/varnish-cache/include/stat_field.h 2009-10-08 13:23:36 UTC (rev 4301) +++ branches/2.0/varnish-cache/include/stat_field.h 2009-10-08 13:34:03 UTC (rev 4302) @@ -37,14 +37,14 @@ MAC_STAT(cache_hitpass, uint64_t, 'a', "Cache hits for pass") MAC_STAT(cache_miss, uint64_t, 'a', "Cache misses") -MAC_STAT(backend_conn, uint64_t, 'a', "Backend connections success") -MAC_STAT(backend_unhealthy, uint64_t, 'a', - "Backend connections not attempted") -MAC_STAT(backend_busy, uint64_t, 'a', "Backend connections too many") -MAC_STAT(backend_fail, uint64_t, 'a', "Backend connections failures") -MAC_STAT(backend_reuse, uint64_t, 'a', "Backend connections reuses") -MAC_STAT(backend_recycle, uint64_t, 'a', "Backend connections recycles") -MAC_STAT(backend_unused, uint64_t, 'a', "Backend connections unused") +MAC_STAT(backend_conn, uint64_t, 'a', "Backend conn. success") +MAC_STAT(backend_unhealthy, uint64_t, 'a', "Backend conn. not attempted") +MAC_STAT(backend_busy, uint64_t, 'a', "Backend conn. too many") +MAC_STAT(backend_fail, uint64_t, 'a', "Backend conn. failures") +MAC_STAT(backend_reuse, uint64_t, 'a', "Backend conn. reuses") +MAC_STAT(backend_toolate, uint64_t, 'a', "Backend conn. was closed") +MAC_STAT(backend_recycle, uint64_t, 'a', "Backend conn. recycles") +MAC_STAT(backend_unused, uint64_t, 'a', "Backend conn. unused") MAC_STAT(n_srcaddr, uint64_t, 'i', "N struct srcaddr") MAC_STAT(n_srcaddr_act, uint64_t, 'i', "N active struct srcaddr") From tfheen at projects.linpro.no Thu Oct 8 13:58:25 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 15:58:25 +0200 (CEST) Subject: r4303 - in branches/2.0/varnish-cache: bin/varnishd bin/varnishtest/tests include lib/libvcl Message-ID: <20091008135825.E829837EE7@projects.linpro.no> Author: tfheen Date: 2009-10-08 15:58:25 +0200 (Thu, 08 Oct 2009) New Revision: 4303 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c branches/2.0/varnish-cache/bin/varnishd/cache_backend_poll.c branches/2.0/varnish-cache/bin/varnishtest/tests/r00306.vtc branches/2.0/varnish-cache/bin/varnishtest/tests/s00002.vtc branches/2.0/varnish-cache/bin/varnishtest/tests/v00014.vtc branches/2.0/varnish-cache/include/vrt.h branches/2.0/varnish-cache/lib/libvcl/vcc_backend.c branches/2.0/varnish-cache/lib/libvcl/vcc_fixed_token.c Log: Merge r4184, r4185: Add a ".initial" property to backend probe specifications. r4184: An explanatory comment. r4185: Add a ".initial" property to backend probe specifications. This is the number of good probes we pretend to have already seen when we start up, in order to speed up getting healthy backends. The default value is one less than the .threshold, so the backend will be set healthy if it manages to respond correctly to the very first probe we send to it. (A bit of this commit leaked in during r4184) Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c 2009-10-08 13:34:03 UTC (rev 4302) +++ branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c 2009-10-08 13:58:25 UTC (rev 4303) @@ -68,6 +68,13 @@ pthread_t VCA_thread; static struct timeval tv_sndtimeo; static struct timeval tv_rcvtimeo; + +/* + * We want to get out of any kind of touble-hit TCP connections as fast + * as absolutely possible, so we set them LINGER enabled with zero timeout, + * so that even if there are outstanding write data on the socket, a close(2) + * will return immediately. + */ static const struct linger linger = { .l_onoff = 1, }; Modified: branches/2.0/varnish-cache/bin/varnishd/cache_backend_poll.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_backend_poll.c 2009-10-08 13:34:03 UTC (rev 4302) +++ branches/2.0/varnish-cache/bin/varnishd/cache_backend_poll.c 2009-10-08 13:58:25 UTC (rev 4303) @@ -241,18 +241,82 @@ } /*-------------------------------------------------------------------- - * One thread per backend to be poked. + * Record pokings... */ -static void * -vbp_wrk_poll_backend(void *priv) +static void +vbp_start_poke(struct vbp_target *vt) { - struct vbp_target *vt; + CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC); + +#define BITMAP(n, c, t, b) vt->n <<= 1; +#include "cache_backend_poll.h" +#undef BITMAP + + vt->last = 0; + vt->resp_buf[0] = '\0'; +} + +static void +vbp_has_poked(struct vbp_target *vt) +{ unsigned i, j; uint64_t u; const char *logmsg; char bits[10]; + CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC); + + /* Calculate exponential average */ + if (vt->happy & 1) { + if (vt->rate < AVG_RATE) + vt->rate += 1.0; + vt->avg += (vt->last - vt->avg) / vt->rate; + } + + i = 0; +#define BITMAP(n, c, t, b) bits[i++] = (vt->n & 1) ? c : '-'; +#include "cache_backend_poll.h" +#undef BITMAP + bits[i] = '\0'; + + u = vt->happy; + for (i = j = 0; i < vt->probe.window; i++) { + if (u & 1) + j++; + u >>= 1; + } + vt->good = j; + + if (vt->good >= vt->probe.threshold) { + if (vt->backend->healthy) + logmsg = "Still healthy"; + else + logmsg = "Back healthy"; + vt->backend->healthy = 1; + } else { + if (vt->backend->healthy) + logmsg = "Went sick"; + else + logmsg = "Still sick"; + vt->backend->healthy = 0; + } + VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s", + vt->backend->vcl_name, logmsg, bits, + vt->good, vt->probe.threshold, vt->probe.window, + vt->last, vt->avg, vt->resp_buf); +} + +/*-------------------------------------------------------------------- + * One thread per backend to be poked. + */ + +static void * +vbp_wrk_poll_backend(void *priv) +{ + struct vbp_target *vt; + unsigned u; + THR_SetName("backend poll"); CAST_OBJ_NOTNULL(vt, priv, VBP_TARGET_MAGIC); @@ -272,59 +336,25 @@ if (vt->probe.threshold == 0) vt->probe.threshold = 3; + if (vt->probe.threshold == ~0U) + vt->probe.initial = vt->probe.threshold - 1; + + if (vt->probe.initial > vt->probe.threshold) + vt->probe.initial = vt->probe.threshold; + printf("Probe(\"%s\", %g, %g)\n", - vt->req, - vt->probe.timeout, - vt->probe.interval); + vt->req, vt->probe.timeout, vt->probe.interval); - /*lint -e{525} indent */ + for (u = 0; u < vt->probe.initial; u++) { + vbp_start_poke(vt); + vt->happy |= 1; + vbp_has_poked(vt); + } + while (!vt->stop) { -#define BITMAP(n, c, t, b) vt->n <<= 1; -#include "cache_backend_poll.h" -#undef BITMAP - vt->last = 0; - vt->resp_buf[0] = '\0'; + vbp_start_poke(vt); vbp_poke(vt); - - /* Calculate exponential average */ - if (vt->happy & 1) { - if (vt->rate < AVG_RATE) - vt->rate += 1.0; - vt->avg += (vt->last - vt->avg) / vt->rate; - } - - i = 0; -#define BITMAP(n, c, t, b) bits[i++] = (vt->n & 1) ? c : '-'; -#include "cache_backend_poll.h" -#undef BITMAP - bits[i] = '\0'; - - u = vt->happy; - for (i = j = 0; i < vt->probe.window; i++) { - if (u & 1) - j++; - u >>= 1; - } - vt->good = j; - - if (vt->good >= vt->probe.threshold) { - if (vt->backend->healthy) - logmsg = "Still healthy"; - else - logmsg = "Back healthy"; - vt->backend->healthy = 1; - } else { - if (vt->backend->healthy) - logmsg = "Went sick"; - else - logmsg = "Still sick"; - vt->backend->healthy = 0; - } - VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s", - vt->backend->vcl_name, logmsg, bits, - vt->good, vt->probe.threshold, vt->probe.window, - vt->last, vt->avg, vt->resp_buf); - + vbp_has_poked(vt); if (!vt->stop) TIM_sleep(vt->probe.interval); } Modified: branches/2.0/varnish-cache/bin/varnishtest/tests/r00306.vtc =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/r00306.vtc 2009-10-08 13:34:03 UTC (rev 4302) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/r00306.vtc 2009-10-08 13:58:25 UTC (rev 4303) @@ -27,6 +27,7 @@ .host = "127.0.0.1"; .port = "9180"; .probe = { .url = "/"; + .initial = 0; } } director foo random { Modified: branches/2.0/varnish-cache/bin/varnishtest/tests/s00002.vtc =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/s00002.vtc 2009-10-08 13:34:03 UTC (rev 4302) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/s00002.vtc 2009-10-08 13:58:25 UTC (rev 4303) @@ -18,6 +18,7 @@ .interval = 1s; .window = 2; .threshold = 1; + .initial = 0; } } sub vcl_fetch { Modified: branches/2.0/varnish-cache/bin/varnishtest/tests/v00014.vtc =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/v00014.vtc 2009-10-08 13:34:03 UTC (rev 4302) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/v00014.vtc 2009-10-08 13:58:25 UTC (rev 4303) @@ -20,6 +20,7 @@ .interval = 1s; .window = 3; .threshold = 2; + .initial = 0; } } Modified: branches/2.0/varnish-cache/include/vrt.h =================================================================== --- branches/2.0/varnish-cache/include/vrt.h 2009-10-08 13:34:03 UTC (rev 4302) +++ branches/2.0/varnish-cache/include/vrt.h 2009-10-08 13:58:25 UTC (rev 4303) @@ -54,6 +54,7 @@ double interval; unsigned window; unsigned threshold; + unsigned initial; }; /* Modified: branches/2.0/varnish-cache/lib/libvcl/vcc_backend.c =================================================================== --- branches/2.0/varnish-cache/lib/libvcl/vcc_backend.c 2009-10-08 13:34:03 UTC (rev 4302) +++ branches/2.0/varnish-cache/lib/libvcl/vcc_backend.c 2009-10-08 13:58:25 UTC (rev 4303) @@ -340,7 +340,8 @@ struct fld_spec *fs; struct token *t_field; struct token *t_did = NULL, *t_window = NULL, *t_threshold = NULL; - unsigned window, threshold; + struct token *t_initial = NULL; + unsigned window, threshold, initial; fs = vcc_FldSpec(tl, "?url", @@ -349,6 +350,7 @@ "?interval", "?window", "?threshold", + "?initial", NULL); ExpectErr(tl, '{'); @@ -356,6 +358,7 @@ window = 0; threshold = 0; + initial = 0; Fb(tl, 0, "\t.probe = {\n"); while (tl->t->tok != '}') { @@ -396,6 +399,11 @@ window = vcc_UintVal(tl); vcc_NextToken(tl); ERRCHK(tl); + } else if (vcc_IdIs(t_field, "initial")) { + t_initial = tl->t; + initial = vcc_UintVal(tl); + vcc_NextToken(tl); + ERRCHK(tl); } else if (vcc_IdIs(t_field, "threshold")) { t_threshold = tl->t; threshold = vcc_UintVal(tl); @@ -441,8 +449,12 @@ vcc_ErrWhere(tl, t_window); } Fb(tl, 0, "\t\t.window = %u,\n", window); - Fb(tl, 0, "\t\t.threshold = %u\n", threshold); + Fb(tl, 0, "\t\t.threshold = %u,\n", threshold); } + if (t_initial != NULL) + Fb(tl, 0, "\t\t.initial = %u,\n", initial); + else + Fb(tl, 0, "\t\t.initial = ~0U,\n", initial); Fb(tl, 0, "\t},\n"); ExpectErr(tl, '}'); vcc_NextToken(tl); Modified: branches/2.0/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- branches/2.0/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-10-08 13:34:03 UTC (rev 4302) +++ branches/2.0/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-10-08 13:58:25 UTC (rev 4303) @@ -247,12 +247,12 @@ vsb_cat(sb, "\nstruct vrt_backend_probe {\n\tconst char\t*url;\n"); vsb_cat(sb, "\tconst char\t*request;\n\tdouble\t\ttimeout;\n"); vsb_cat(sb, "\tdouble\t\tinterval;\n\tunsigned\twindow;\n"); - vsb_cat(sb, "\tunsigned\tthreshold;\n};\n\n/*\n"); - vsb_cat(sb, " * A backend is a host+port somewhere on the network\n"); - vsb_cat(sb, " */\nstruct vrt_backend {\n\tconst char\t\t\t*vcl_name"); - vsb_cat(sb, ";\n\tconst char\t\t\t*ident;\n\n"); - vsb_cat(sb, "\tconst char\t\t\t*hosthdr;\n\n"); - vsb_cat(sb, "\tconst unsigned char\t\t*ipv4_sockaddr;\n"); + vsb_cat(sb, "\tunsigned\tthreshold;\n\tunsigned\tinitial;\n"); + vsb_cat(sb, "};\n\n/*\n * A backend is a host+port somewhere on the"); + vsb_cat(sb, " network\n */\nstruct vrt_backend {\n"); + vsb_cat(sb, "\tconst char\t\t\t*vcl_name;\n\tconst char\t\t\t*ident"); + vsb_cat(sb, ";\n\n\tconst char\t\t\t*hosthdr;\n"); + vsb_cat(sb, "\n\tconst unsigned char\t\t*ipv4_sockaddr;\n"); vsb_cat(sb, "\tconst unsigned char\t\t*ipv6_sockaddr;\n"); vsb_cat(sb, "\n\tdouble\t\t\t\tconnect_timeout;\n"); vsb_cat(sb, "\tdouble\t\t\t\tfirst_byte_timeout;\n"); @@ -324,9 +324,9 @@ /* ../../include/vrt_obj.h */ - vsb_cat(sb, "/*\n * $Id$\n"); - vsb_cat(sb, " *\n * NB: This file is machine generated, DO NOT EDI"); - vsb_cat(sb, "T!\n *\n * Edit vcc_gen_obj.tcl instead\n"); + vsb_cat(sb, "/*\n * $Id: vcc_gen_obj.tcl 4082 2009-05-19 07:14:00Z "); + vsb_cat(sb, "sky $\n *\n * NB: This file is machine generated, DO "); + vsb_cat(sb, "NOT EDIT!\n *\n * Edit vcc_gen_obj.tcl instead\n"); vsb_cat(sb, " */\n\nstruct sockaddr * VRT_r_client_ip(const struct "); vsb_cat(sb, "sess *);\nstruct sockaddr * VRT_r_server_ip(struct ses"); vsb_cat(sb, "s *);\nconst char * VRT_r_server_hostname(struct sess "); From tfheen at projects.linpro.no Thu Oct 8 14:04:08 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 16:04:08 +0200 (CEST) Subject: r4304 - branches/2.0/varnish-cache/lib/libvcl Message-ID: <20091008140408.0991E37EE7@projects.linpro.no> Author: tfheen Date: 2009-10-08 16:04:07 +0200 (Thu, 08 Oct 2009) New Revision: 4304 Modified: branches/2.0/varnish-cache/lib/libvcl/vcc_fixed_token.c branches/2.0/varnish-cache/lib/libvcl/vcc_parse.c Log: Merge r4186: Make string comparisons against other than string literals possible. Modified: branches/2.0/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- branches/2.0/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-10-08 13:58:25 UTC (rev 4303) +++ branches/2.0/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-10-08 14:04:07 UTC (rev 4304) @@ -235,8 +235,8 @@ vsb_cat(sb, " * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWI"); vsb_cat(sb, "SE) ARISING IN ANY WAY\n * OUT OF THE USE OF THIS SOFT"); vsb_cat(sb, "WARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n"); - vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id: vrt.h 3724 2009-02-10 14:"); - vsb_cat(sb, "58:17Z tfheen $\n *\n * Runtime support for compiled V"); + vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id: vrt.h 4303 2009-10-08 13:"); + vsb_cat(sb, "58:25Z tfheen $\n *\n * Runtime support for compiled V"); vsb_cat(sb, "CL programs.\n *\n * XXX: When this file is changed, l"); vsb_cat(sb, "ib/libvcl/vcc_gen_fixed_token.tcl\n"); vsb_cat(sb, " * XXX: *MUST* be rerun.\n */\n"); Modified: branches/2.0/varnish-cache/lib/libvcl/vcc_parse.c =================================================================== --- branches/2.0/varnish-cache/lib/libvcl/vcc_parse.c 2009-10-08 13:58:25 UTC (rev 4303) +++ branches/2.0/varnish-cache/lib/libvcl/vcc_parse.c 2009-10-08 14:04:07 UTC (rev 4304) @@ -263,10 +263,11 @@ Fb(tl, 1, "%sVRT_strcmp(%s, ", tl->t->tok == T_EQ ? "!" : "", vp->rname); vcc_NextToken(tl); - ExpectErr(tl, CSTR); - EncToken(tl->fb, tl->t); + if (!vcc_StringVal(tl)) { + vcc_ExpectedStringval(tl); + break; + } Fb(tl, 0, ")\n"); - vcc_NextToken(tl); break; default: Fb(tl, 1, "%s != (void*)0\n", vp->rname); From tfheen at projects.linpro.no Thu Oct 8 14:09:55 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 16:09:55 +0200 (CEST) Subject: r4305 - branches/2.0/varnish-cache/bin/varnishtest/tests Message-ID: <20091008140955.1303F37EE7@projects.linpro.no> Author: tfheen Date: 2009-10-08 16:09:54 +0200 (Thu, 08 Oct 2009) New Revision: 4305 Added: branches/2.0/varnish-cache/bin/varnishtest/tests/v00024.vtc Log: Merge r4187: New test for string comparisons. Copied: branches/2.0/varnish-cache/bin/varnishtest/tests/v00024.vtc (from rev 4187, trunk/varnish-cache/bin/varnishtest/tests/v00024.vtc) =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/v00024.vtc (rev 0) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/v00024.vtc 2009-10-08 14:09:54 UTC (rev 4305) @@ -0,0 +1,27 @@ +# $Id$ + +test "Test that headers can be compared" + +server s1 { + rxreq + expect req.url == "/foo" + txresp -status 200 -body "1" +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.http.etag == req.http.if-none-match) { + error 400 "FOO"; + } + } +} -start + +client c1 { + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.bodylen == 1 + txreq -url "/foo" -hdr "etag: foo" -hdr "if-none-match: foo" + rxresp + expect resp.status == 400 +} -run From tfheen at projects.linpro.no Thu Oct 8 14:16:41 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 16:16:41 +0200 (CEST) Subject: r4306 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008141641.E41E937EE7@projects.linpro.no> Author: tfheen Date: 2009-10-08 16:16:41 +0200 (Thu, 08 Oct 2009) New Revision: 4306 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_cli.c Log: Merge r4192: Log that the worker stops in the shmlog Modified: branches/2.0/varnish-cache/bin/varnishd/cache_cli.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_cli.c 2009-10-08 14:09:54 UTC (rev 4305) +++ branches/2.0/varnish-cache/bin/varnishd/cache_cli.c 2009-10-08 14:16:41 UTC (rev 4306) @@ -158,8 +158,7 @@ continue; assert(i == 1); if (pfd[0].revents & POLLHUP) { - fprintf(stderr, - "EOF on CLI connection, exiting\n"); + VSL(SLT_CLI, 0, "EOF on CLI connection, worker stops"); exit(0); } i = VLU_Fd(heritage.cli_in, vlu); From tfheen at projects.linpro.no Thu Oct 8 14:22:18 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 16:22:18 +0200 (CEST) Subject: r4307 - branches/2.0/varnish-cache/bin/varnishtest Message-ID: <20091008142218.2E35D28116@projects.linpro.no> Author: tfheen Date: 2009-10-08 16:22:17 +0200 (Thu, 08 Oct 2009) New Revision: 4307 Modified: branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c Log: Merge r4193: Fail gracefully on Rx error Modified: branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c 2009-10-08 14:16:41 UTC (rev 4306) +++ branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c 2009-10-08 14:22:17 UTC (rev 4307) @@ -326,7 +326,9 @@ i = read(hp->fd, hp->rxbuf + hp->prxbuf, n); if (i == 0) return (i); - assert(i > 0); + if (i <= 0) + vtc_log(hp->vl, 0, "HTTP rx failed (%s)", + strerror(errno)); hp->prxbuf += i; hp->rxbuf[hp->prxbuf] = '\0'; n -= i; From tfheen at projects.linpro.no Thu Oct 8 14:29:34 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 16:29:34 +0200 (CEST) Subject: r4308 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008142934.A3E3C1F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 16:29:34 +0200 (Thu, 08 Oct 2009) New Revision: 4308 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_expire.c branches/2.0/varnish-cache/bin/varnishd/cache_vcl.c Log: Merge r4195: Avoid the 10 second sleep while waiting for the VCL to appear. Modified: branches/2.0/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_expire.c 2009-10-08 14:22:17 UTC (rev 4307) +++ branches/2.0/varnish-cache/bin/varnishd/cache_expire.c 2009-10-08 14:29:34 UTC (rev 4308) @@ -282,7 +282,6 @@ double t; (void)priv; - AZ(sleep(10)); /* XXX: Takes time for VCL to arrive */ VCL_Get(&sp->vcl); t = TIM_real(); while (1) { Modified: branches/2.0/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_vcl.c 2009-10-08 14:22:17 UTC (rev 4307) +++ branches/2.0/varnish-cache/bin/varnishd/cache_vcl.c 2009-10-08 14:29:34 UTC (rev 4308) @@ -83,7 +83,13 @@ void VCL_Get(struct VCL_conf **vcc) { + static int once; + while (!once && vcl_active == NULL) { + sleep(1); + } + once = 1; + Lck_Lock(&vcl_mtx); AN(vcl_active); *vcc = vcl_active->conf; From tfheen at projects.linpro.no Thu Oct 8 14:34:44 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 16:34:44 +0200 (CEST) Subject: r4309 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008143444.6DEE528117@projects.linpro.no> Author: tfheen Date: 2009-10-08 16:34:44 +0200 (Thu, 08 Oct 2009) New Revision: 4309 Modified: branches/2.0/varnish-cache/bin/varnishd/shmlog.c Log: Merge r4200: Be more paranoid about shmlog magics, Don't explode manager if it gets overwritten. Modified: branches/2.0/varnish-cache/bin/varnishd/shmlog.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/shmlog.c 2009-10-08 14:29:34 UTC (rev 4308) +++ branches/2.0/varnish-cache/bin/varnishd/shmlog.c 2009-10-08 14:34:44 UTC (rev 4309) @@ -71,16 +71,19 @@ vsl_wrap(void) { + assert(loghead->magic == SHMLOGHEAD_MAGIC); *logstart = SLT_ENDMARKER; logstart[loghead->ptr] = SLT_WRAPMARKER; loghead->ptr = 0; VSL_stats->shm_cycles++; + assert(loghead->magic == SHMLOGHEAD_MAGIC); } static void vsl_hdr(enum shmlogtag tag, unsigned char *p, unsigned len, unsigned id) { + assert(loghead->magic == SHMLOGHEAD_MAGIC); assert(len < 0x10000); assert(id < 0x10000); p[__SHMLOG_LEN_HIGH] = (len >> 8) & 0xff; @@ -280,13 +283,18 @@ void VSL_Panic(int *len, char **ptr) { + static char a[1] = { '\0' }; AN(len); AN(ptr); - assert(loghead->magic == SHMLOGHEAD_MAGIC); - assert(loghead->hdrsize == sizeof *loghead); - *len = sizeof(loghead->panicstr); - *ptr = loghead->panicstr; + if (loghead->magic == SHMLOGHEAD_MAGIC) { + assert(loghead->hdrsize == sizeof *loghead); + *len = sizeof(loghead->panicstr); + *ptr = loghead->panicstr; + } else { + *len = 0; + *ptr = a; + } } /*--------------------------------------------------------------------*/ From tfheen at projects.linpro.no Thu Oct 8 14:39:24 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 16:39:24 +0200 (CEST) Subject: r4310 - in branches/2.0/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20091008143924.1BF0028117@projects.linpro.no> Author: tfheen Date: 2009-10-08 16:39:23 +0200 (Thu, 08 Oct 2009) New Revision: 4310 Added: branches/2.0/varnish-cache/bin/varnishtest/tests/c00028.vtc Modified: branches/2.0/varnish-cache/bin/varnishd/cache_center.c Log: Merge r4204: Prevent infinite restart recursion it was possible to have infinite recursion between vcl_error returning restart and cnt_recv calling vcl_error when you had too many restarts Modified: branches/2.0/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_center.c 2009-10-08 14:34:44 UTC (rev 4309) +++ branches/2.0/varnish-cache/bin/varnishd/cache_center.c 2009-10-08 14:39:23 UTC (rev 4310) @@ -349,13 +349,15 @@ http_StatusMessage(sp->err_code)); VCL_error_method(sp); - if (sp->handling == VCL_RET_RESTART) { + if (sp->handling == VCL_RET_RESTART && sp->restarts < params->max_restarts) { HSH_Drop(sp); sp->director = NULL; sp->restarts++; sp->step = STP_RECV; return (0); - } + } else if (sp->handling == VCL_RET_RESTART) + sp->handling = VCL_RET_DELIVER; + /* We always close when we take this path */ sp->doclose = "error"; Copied: branches/2.0/varnish-cache/bin/varnishtest/tests/c00028.vtc (from rev 4204, trunk/varnish-cache/bin/varnishtest/tests/c00028.vtc) =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/c00028.vtc (rev 0) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/c00028.vtc 2009-10-08 14:39:23 UTC (rev 4310) @@ -0,0 +1,24 @@ +# $Id$ + +test "Test that we can't recurse restarts forever" + +varnish v1 -vcl { + backend bad { + .host = "127.0.0.1"; + .port = "9090"; + } + + sub vcl_recv { + set req.backend = bad; + } + sub vcl_error { + restart; + } + } -start + +client c1 { + txreq -url "/" + rxresp + expect resp.status == 503 +} -run + From tfheen at projects.linpro.no Thu Oct 8 14:55:41 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 16:55:41 +0200 (CEST) Subject: r4311 - branches/2.0/varnish-cache/man Message-ID: <20091008145541.4B17138629@projects.linpro.no> Author: tfheen Date: 2009-10-08 16:55:41 +0200 (Thu, 08 Oct 2009) New Revision: 4311 Modified: branches/2.0/varnish-cache/man/vcl.7so Log: Merge r4207: Add obj.hits to the vcl(7) manual Modified: branches/2.0/varnish-cache/man/vcl.7so =================================================================== --- branches/2.0/varnish-cache/man/vcl.7so 2009-10-08 14:39:23 UTC (rev 4310) +++ branches/2.0/varnish-cache/man/vcl.7so 2009-10-08 14:55:41 UTC (rev 4311) @@ -570,6 +570,9 @@ .It Va obj.lastuse The approximate time elapsed since the object was last requests, in seconds. +.It Va obj.hits +The approximate number of times the object has been delivered. A value of 0 +indicates a cache miss. .El .Pp The following variables are available while determining the hash key From tfheen at projects.linpro.no Thu Oct 8 15:01:02 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 17:01:02 +0200 (CEST) Subject: r4312 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008150102.F0D181F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 17:01:02 +0200 (Thu, 08 Oct 2009) New Revision: 4312 Modified: branches/2.0/varnish-cache/bin/varnishd/mgt_child.c branches/2.0/varnish-cache/bin/varnishd/varnishd.c Log: Merge r4210: Factor getdtablesize() out of close loop. Modified: branches/2.0/varnish-cache/bin/varnishd/mgt_child.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/mgt_child.c 2009-10-08 14:55:41 UTC (rev 4311) +++ branches/2.0/varnish-cache/bin/varnishd/mgt_child.c 2009-10-08 15:01:02 UTC (rev 4312) @@ -236,7 +236,7 @@ unsigned u; char *p; struct vev *e; - int i, cp[2]; + int i, j, cp[2]; if (child_state != CH_STOPPED && child_state != CH_DIED) return; @@ -304,7 +304,8 @@ /* Close anything we shouldn't know about */ closelog(); printf("Closed fds:"); - for (i = STDERR_FILENO + 1; i < getdtablesize(); i++) { + j = getdtablesize(); + for (i = STDERR_FILENO + 1; i < j; i++) { if (vbit_test(fd_map, i)) continue; if (close(i) == 0) Modified: branches/2.0/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/varnishd.c 2009-10-08 14:55:41 UTC (rev 4311) +++ branches/2.0/varnish-cache/bin/varnishd/varnishd.c 2009-10-08 15:01:02 UTC (rev 4312) @@ -339,7 +339,8 @@ pipes[1][1] = 1; /* close the rest */ - for (i = 5; i < getdtablesize(); i++) + j = getdtablesize(); + for (i = 5; i < j; i++) (void)close(i); pfd[0].fd = pipes[0][0]; From tfheen at projects.linpro.no Thu Oct 8 15:12:41 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 17:12:41 +0200 (CEST) Subject: r4313 - in branches/2.0/varnish-cache: bin/varnishtest bin/varnishtest/tests include lib/libvarnish Message-ID: <20091008151241.B0D4D1F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 17:12:41 +0200 (Thu, 08 Oct 2009) New Revision: 4313 Modified: branches/2.0/varnish-cache/bin/varnishtest/tests/c00001.vtc branches/2.0/varnish-cache/bin/varnishtest/vtc.c branches/2.0/varnish-cache/include/libvarnish.h branches/2.0/varnish-cache/lib/libvarnish/argv.c Log: Merge r4219, r4220: Fix backslash parsing in varnishtest r4219: Expose the good string backslash implementation from argv.c and replace a half-baked one in vtc.c with it. r4220: Add a newline to feed the new backslash handler Modified: branches/2.0/varnish-cache/bin/varnishtest/tests/c00001.vtc =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/c00001.vtc 2009-10-08 15:01:02 UTC (rev 4312) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/c00001.vtc 2009-10-08 15:12:41 UTC (rev 4313) @@ -36,7 +36,7 @@ expect resp.http.snafu2 == "_frap_" expect resp.http.snafu3 == "_f\\rap_" expect resp.http.snafu4 == "_f&rap_" - expect resp.http.snafu5 == "_barffra\p_" - # NB: have to escape the \\ in the next line + # NB: have to escape the \\ in the next two lines + expect resp.http.snafu5 == "_barffra\\p_" expect resp.http.snafu6 == "_f&rap\\_" } -run Modified: branches/2.0/varnish-cache/bin/varnishtest/vtc.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/vtc.c 2009-10-08 15:01:02 UTC (rev 4312) +++ branches/2.0/varnish-cache/bin/varnishtest/vtc.c 2009-10-08 15:12:41 UTC (rev 4313) @@ -130,19 +130,9 @@ for (; *p != '\0'; p++) { if (*p == '"') break; - - if (*p == '\\' && p[1] == 'n') { - *q++ = '\n'; - p++; - } else if (*p == '\\' && p[1] == 'r') { - *q++ = '\r'; - p++; - } else if (*p == '\\' && p[1] == '\\') { - *q++ = '\\'; - p++; - } else if (*p == '\\' && p[1] == '"') { - *q++ = '"'; - p++; + if (*p == '\\') { + p += BackSlash(p, q) - 1; + q++; } else { if (*p == '\n') fprintf(stderr, Modified: branches/2.0/varnish-cache/include/libvarnish.h =================================================================== --- branches/2.0/varnish-cache/include/libvarnish.h 2009-10-08 15:01:02 UTC (rev 4312) +++ branches/2.0/varnish-cache/include/libvarnish.h 2009-10-08 15:12:41 UTC (rev 4313) @@ -42,6 +42,8 @@ /* from libvarnish/argv.c */ void FreeArgv(char **argv); char **ParseArgv(const char *s, int flag); +char *BackSlashDecode(const char *s, const char *e); +int BackSlash(const char *s, char *res); #define ARGV_COMMENT (1 << 0) #define ARGV_COMMA (1 << 1) Modified: branches/2.0/varnish-cache/lib/libvarnish/argv.c =================================================================== --- branches/2.0/varnish-cache/lib/libvarnish/argv.c 2009-10-08 15:01:02 UTC (rev 4312) +++ branches/2.0/varnish-cache/lib/libvarnish/argv.c 2009-10-08 15:12:41 UTC (rev 4313) @@ -43,11 +43,12 @@ #include #include #include +#include #include #include "libvarnish.h" -static int +int BackSlash(const char *s, char *res) { int r; @@ -103,13 +104,16 @@ return (r); } -static char * +char * BackSlashDecode(const char *s, const char *e) { const char *q; char *p, *r; int i; + if (e == NULL) + e = strchr(s, '\0'); + assert(e != NULL); p = calloc((e - s) + 1, 1); if (p == NULL) return (p); From tfheen at projects.linpro.no Thu Oct 8 15:19:34 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 17:19:34 +0200 (CEST) Subject: r4314 - in branches/2.0/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20091008151934.E905D1F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 17:19:34 +0200 (Thu, 08 Oct 2009) New Revision: 4314 Added: branches/2.0/varnish-cache/bin/varnishtest/tests/r00549.vtc Modified: branches/2.0/varnish-cache/bin/varnishd/cache_http.c Log: Merge r4221: Be much more paranoid about control-characters in backend responses. Fixes #549 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_http.c 2009-10-08 15:12:41 UTC (rev 4313) +++ branches/2.0/varnish-cache/bin/varnishd/cache_http.c 2009-10-08 15:19:34 UTC (rev 4314) @@ -390,7 +390,7 @@ http_splitline(struct worker *w, int fd, struct http *hp, const struct http_conn *htc, int h1, int h2, int h3) { - char *p; + char *p, *q; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); @@ -403,40 +403,47 @@ continue; /* First field cannot contain SP, CRLF or CTL */ - hp->hd[h1].b = p; - for (; !vct_issp(*p); p++) + q = p; + for (; !vct_issp(*p); p++) { if (vct_isctl(*p)) return (400); + } + hp->hd[h1].b = q; hp->hd[h1].e = p; /* Skip SP */ - for (; vct_issp(*p); p++) - ; + for (; vct_issp(*p); p++) { + if (vct_isctl(*p)) + return (400); + } - /* Second field cannot contain LWS */ - hp->hd[h2].b = p; - for (; !vct_islws(*p); p++) - ; + /* Second field cannot contain LWS or CTL */ + q = p; + for (; !vct_islws(*p); p++) { + if (vct_isctl(*p)) + return (400); + } + hp->hd[h2].b = q; hp->hd[h2].e = p; if (!Tlen(hp->hd[h2])) return (400); /* Skip SP */ - for (; vct_issp(*p); p++) - ; + for (; vct_issp(*p); p++) { + if (vct_isctl(*p)) + return (400); + } /* Third field is optional and cannot contain CTL */ + q = p; if (!vct_iscrlf(*p)) { - hp->hd[h3].b = p; for (; !vct_iscrlf(*p); p++) if (vct_isctl(*p)) return (400); - hp->hd[h3].e = p; - } else { - hp->hd[h3].b = p; - hp->hd[h3].e = p; } + hp->hd[h3].b = q; + hp->hd[h3].e = p; /* Skip CRLF */ p += vct_skipcrlf(p); Copied: branches/2.0/varnish-cache/bin/varnishtest/tests/r00549.vtc (from rev 4221, trunk/varnish-cache/bin/varnishtest/tests/r00549.vtc) =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/r00549.vtc (rev 0) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/r00549.vtc 2009-10-08 15:19:34 UTC (rev 4314) @@ -0,0 +1,15 @@ +# $Id$ + +# Regression test for bad backend reply with ctrl char. + +server s1 { + rxreq + send "HTTP/1.1 200 OK\013\r\n\r\nTest" +} -start + +varnish v1 -vcl+backend {} -start + +client c1 { + txreq + rxresp +} -run From tfheen at projects.linpro.no Thu Oct 8 15:33:25 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 17:33:25 +0200 (CEST) Subject: r4315 - in branches/2.0/varnish-cache: bin/varnishd include Message-ID: <20091008153325.5B1F11F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 17:33:25 +0200 (Thu, 08 Oct 2009) New Revision: 4315 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_fetch.c branches/2.0/varnish-cache/include/stat_field.h Log: Merge r4231: Add counters Add a family of 9 counters to tell us how fetching from the backends progress. Modified: branches/2.0/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_fetch.c 2009-10-08 15:19:34 UTC (rev 4314) +++ branches/2.0/varnish-cache/bin/varnishd/cache_fetch.c 2009-10-08 15:33:25 UTC (rev 4315) @@ -407,14 +407,17 @@ cls = 0; mklen = 0; if (is_head) { - /* nothing */ + VSL_stats->fetch_head++; } else if (http_GetHdr(hp, H_Content_Length, &b)) { + VSL_stats->fetch_length++; cls = fetch_straight(sp, htc, b); mklen = 1; } else if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) { + VSL_stats->fetch_chunked++; cls = fetch_chunked(sp, htc); mklen = 1; } else if (http_GetHdr(hp, H_Transfer_Encoding, &b)) { + VSL_stats->fetch_bad++; /* XXX: AUGH! */ WSL(sp->wrk, SLT_Debug, vc->fd, "Invalid Transfer-Encoding"); VBE_ClosedFd(sp); @@ -427,6 +430,7 @@ */ mklen = 1; } else if (http_HdrIs(hp, H_Connection, "close")) { + VSL_stats->fetch_close++; /* * If we have connection closed, it is safe to read what * comes in any case. @@ -434,19 +438,23 @@ cls = fetch_eof(sp, htc); mklen = 1; } else if (hp->protover < 1.1) { + VSL_stats->fetch_oldhttp++; /* * With no Connection header, assume EOF */ cls = fetch_eof(sp, htc); mklen = 1; } else { + VSL_stats->fetch_zero++; /* * Assume zero length + * XXX: ??? */ mklen = 1; } if (cls < 0) { + VSL_stats->fetch_failed++; /* XXX: Wouldn't this store automatically be released ? */ while (!VTAILQ_EMPTY(&sp->obj->store)) { st = VTAILQ_FIRST(&sp->obj->store); Modified: branches/2.0/varnish-cache/include/stat_field.h =================================================================== --- branches/2.0/varnish-cache/include/stat_field.h 2009-10-08 15:19:34 UTC (rev 4314) +++ branches/2.0/varnish-cache/include/stat_field.h 2009-10-08 15:33:25 UTC (rev 4315) @@ -46,6 +46,17 @@ MAC_STAT(backend_recycle, uint64_t, 'a', "Backend conn. recycles") MAC_STAT(backend_unused, uint64_t, 'a', "Backend conn. unused") + +MAC_STAT(fetch_head, uint64_t, 'a', "Fetch head") +MAC_STAT(fetch_length, uint64_t, 'a', "Fetch with Length") +MAC_STAT(fetch_chunked, uint64_t, 'a', "Fetch chunked") +MAC_STAT(fetch_eof, uint64_t, 'a', "Fetch EOF") +MAC_STAT(fetch_bad, uint64_t, 'a', "Fetch had bad headers") +MAC_STAT(fetch_close, uint64_t, 'a', "Fetch wanted close") +MAC_STAT(fetch_oldhttp, uint64_t, 'a', "Fetch pre HTTP/1.1 closed") +MAC_STAT(fetch_zero, uint64_t, 'a', "Fetch zero len") +MAC_STAT(fetch_failed, uint64_t, 'a', "Fetch failed") + MAC_STAT(n_srcaddr, uint64_t, 'i', "N struct srcaddr") MAC_STAT(n_srcaddr_act, uint64_t, 'i', "N active struct srcaddr") MAC_STAT(n_sess_mem, uint64_t, 'i', "N struct sess_mem") From tfheen at projects.linpro.no Thu Oct 8 15:41:57 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 17:41:57 +0200 (CEST) Subject: r4316 - in branches/2.0/varnish-cache: include man Message-ID: <20091008154157.9E0F01F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 17:41:57 +0200 (Thu, 08 Oct 2009) New Revision: 4316 Modified: branches/2.0/varnish-cache/include/shmlog_tags.h branches/2.0/varnish-cache/man/vcl.7so Log: Merge r4238: Document max_connections somewhat Modified: branches/2.0/varnish-cache/include/shmlog_tags.h =================================================================== --- branches/2.0/varnish-cache/include/shmlog_tags.h 2009-10-08 15:33:25 UTC (rev 4315) +++ branches/2.0/varnish-cache/include/shmlog_tags.h 2009-10-08 15:41:57 UTC (rev 4316) @@ -56,6 +56,8 @@ SLTM(Backend) SLTM(Length) +SLTM(FetchError) + SLTM(RxRequest) SLTM(RxResponse) SLTM(RxStatus) Modified: branches/2.0/varnish-cache/man/vcl.7so =================================================================== --- branches/2.0/varnish-cache/man/vcl.7so 2009-10-08 15:33:25 UTC (rev 4315) +++ branches/2.0/varnish-cache/man/vcl.7so 2009-10-08 15:41:57 UTC (rev 4316) @@ -93,6 +93,11 @@ } .Ed .Pp +To avoid overloading backend servers, +.Fa .max_connections +can be set to limit the maximum number of concurrent backend connections. +.Ed +.Pp The timeout parameters can be overridden in the backend declaration. The timeout parameters are .Fa .connect_timeout From tfheen at projects.linpro.no Thu Oct 8 15:55:18 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 17:55:18 +0200 (CEST) Subject: r4317 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008155518.90E861F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 17:55:18 +0200 (Thu, 08 Oct 2009) New Revision: 4317 Modified: branches/2.0/varnish-cache/bin/varnishd/mgt_param.c Log: Merge r4239: Turn purge_dups on by default Modified: branches/2.0/varnish-cache/bin/varnishd/mgt_param.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/mgt_param.c 2009-10-08 15:41:57 UTC (rev 4316) +++ branches/2.0/varnish-cache/bin/varnishd/mgt_param.c 2009-10-08 15:55:18 UTC (rev 4317) @@ -765,7 +765,7 @@ { "purge_dups", tweak_bool, &master.purge_dups, 0, 0, "Detect and eliminate duplicate purges.\n", 0, - "off", "bool" }, + "on", "bool" }, { NULL, NULL, NULL } }; From tfheen at projects.linpro.no Thu Oct 8 16:01:45 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 18:01:45 +0200 (CEST) Subject: r4318 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008160145.C684C1F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 18:01:45 +0200 (Thu, 08 Oct 2009) New Revision: 4318 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_center.c Log: Merge r4240: Handle return(error) gracefully If we come into vcl_error{} with a non-3-digit error status, for instance because vcl_recv does "return(error);", force the status to 501. Fixes: #558 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_center.c 2009-10-08 15:55:18 UTC (rev 4317) +++ branches/2.0/varnish-cache/bin/varnishd/cache_center.c 2009-10-08 16:01:45 UTC (rev 4318) @@ -335,6 +335,9 @@ CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); h = sp->obj->http; + if (sp->err_code < 100 || sp->err_code > 999) + sp->err_code = 501; + http_PutProtocol(w, sp->fd, h, "HTTP/1.1"); http_PutStatus(w, sp->fd, h, sp->err_code); TIM_format(TIM_real(), date); From tfheen at projects.linpro.no Thu Oct 8 16:18:33 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 18:18:33 +0200 (CEST) Subject: r4319 - branches/2.0/varnish-cache/bin/varnishtest/tests Message-ID: <20091008161833.517E31F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 18:18:33 +0200 (Thu, 08 Oct 2009) New Revision: 4319 Added: branches/2.0/varnish-cache/bin/varnishtest/tests/r00558.vtc Log: Merge r4241: Regression test case for 558 Copied: branches/2.0/varnish-cache/bin/varnishtest/tests/r00558.vtc (from rev 4241, trunk/varnish-cache/bin/varnishtest/tests/r00558.vtc) =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/r00558.vtc (rev 0) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/r00558.vtc 2009-10-08 16:18:33 UTC (rev 4319) @@ -0,0 +1,19 @@ +# $Id$ + +test "error from vcl_recv{} has no numeric code" + + +server s1 { +} + +varnish v1 -vcl+backend { + sub vcl_recv { + return (error); + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 501 +} -run From tfheen at projects.linpro.no Thu Oct 8 16:20:46 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 8 Oct 2009 18:20:46 +0200 (CEST) Subject: r4320 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091008162046.B11D91F736F@projects.linpro.no> Author: tfheen Date: 2009-10-08 18:20:46 +0200 (Thu, 08 Oct 2009) New Revision: 4320 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_vrt_esi.c Log: Merge r4242: Fix a typo. Submitted by: "Jos I. Boumans" Modified: branches/2.0/varnish-cache/bin/varnishd/cache_vrt_esi.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_vrt_esi.c 2009-10-08 16:18:33 UTC (rev 4319) +++ branches/2.0/varnish-cache/bin/varnishd/cache_vrt_esi.c 2009-10-08 16:20:46 UTC (rev 4320) @@ -369,7 +369,7 @@ continue; if (Tlen(val) == 0) { esi_error(ew, tag.b, Tlen(tag), - "ESI esi:include src attribute withou value"); + "ESI esi:include src attribute without value"); continue; } From des at projects.linpro.no Fri Oct 9 09:03:40 2009 From: des at projects.linpro.no (des at projects.linpro.no) Date: Fri, 9 Oct 2009 11:03:40 +0200 (CEST) Subject: r4321 - trunk/varnish-cache/etc Message-ID: <20091009090340.27B021F7338@projects.linpro.no> Author: des Date: 2009-10-09 11:03:39 +0200 (Fri, 09 Oct 2009) New Revision: 4321 Modified: trunk/varnish-cache/etc/Makefile.am Log: Fix some formatting nits in default.vcl. Modified: trunk/varnish-cache/etc/Makefile.am =================================================================== --- trunk/varnish-cache/etc/Makefile.am 2009-10-08 16:20:46 UTC (rev 4320) +++ trunk/varnish-cache/etc/Makefile.am 2009-10-09 09:03:39 UTC (rev 4321) @@ -13,13 +13,13 @@ server.\n\ \n\ backend default {\n\ - .host = \"127.0.0.1\";\n\ - .port = \"8080\";\n\ + .host = \"127.0.0.1\";\n\ + .port = \"8080\";\n\ }\n\ \n\ Below is a commented-out copy of the default VCL logic. If you\n\ redefine any of these subroutines, the built-in logic will be\n\ appended to your code.\n" > tmp.vcl @sed -n '/vcl_recv/,$$p' $(top_srcdir)/bin/varnishd/default.vcl >> tmp.vcl - @sed 's/^\(.*\)$$/#\1/' tmp.vcl > default.vcl + @sed 's/^\(.*\)$$/# \1/' tmp.vcl > default.vcl @rm tmp.vcl From edward at projects.linpro.no Mon Oct 12 10:13:49 2009 From: edward at projects.linpro.no (edward at projects.linpro.no) Date: Mon, 12 Oct 2009 12:13:49 +0200 (CEST) Subject: r4322 - trunk/varnish-tools/security.vcl/vcl/modules Message-ID: <20091012101349.D9BD81F737A@projects.linpro.no> Author: edward Date: 2009-10-12 12:13:49 +0200 (Mon, 12 Oct 2009) New Revision: 4322 Modified: trunk/varnish-tools/security.vcl/vcl/modules/php.vcl Log: added a check for "_PHPLIB[" which is activly exploitet Modified: trunk/varnish-tools/security.vcl/vcl/modules/php.vcl =================================================================== --- trunk/varnish-tools/security.vcl/vcl/modules/php.vcl 2009-10-09 09:03:39 UTC (rev 4321) +++ trunk/varnish-tools/security.vcl/vcl/modules/php.vcl 2009-10-12 10:13:49 UTC (rev 4322) @@ -89,6 +89,13 @@ call sec_php_sev1; } + if (req.url ~ "_PHPLIB\[") { + set req.http.X-SEC-RuleName = "Manipulation of Predefined Variable _PHPLIB"; + set req.http.X-SEC-RuleId = "13"; + set req.http.X-SEC-RuleInfo = "Manipulation of Predefined Variable: _PHPLIB"; + call sec_php_sev1; + } + # One could make one long regexp with common php statements. For now: # Generic check for code execution From tfheen at projects.linpro.no Tue Oct 13 08:36:10 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Tue, 13 Oct 2009 10:36:10 +0200 (CEST) Subject: r4323 - in branches/2.0/varnish-cache: bin/varnishd lib/libvcl Message-ID: <20091013083610.7EB211F739D@projects.linpro.no> Author: tfheen Date: 2009-10-13 10:36:10 +0200 (Tue, 13 Oct 2009) New Revision: 4323 Modified: branches/2.0/varnish-cache/bin/varnishd/varnishd.c branches/2.0/varnish-cache/lib/libvcl/vcc_acl.c Log: Merge r4237: Further Flexelint polishing Modified: branches/2.0/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/varnishd.c 2009-10-12 10:13:49 UTC (rev 4322) +++ branches/2.0/varnish-cache/bin/varnishd/varnishd.c 2009-10-13 08:36:10 UTC (rev 4323) @@ -445,15 +445,12 @@ VTAILQ_FOREACH(s, &symbols, list) { if (s->a > pp) continue; - if (s0 != NULL && s->a < s0->a) - continue; - s0 = s; + if (s0 == NULL || s->a < s0->a) + s0 = s; } if (s0 == NULL) return (-1); - vsb_printf(vsb, "%p", ptr); - if (s0 != NULL) - vsb_printf(vsb, ": %s+%jx", s0->n, (uintmax_t)pp - s0->a); + vsb_printf(vsb, "%p: %s+%jx", ptr, s0->n, (uintmax_t)pp - s0->a); return (0); } @@ -465,40 +462,43 @@ uintptr_t a; struct symbols *s; - strcpy(buf, "nm -an "); - strcat(buf, a0); + p = NULL; + asprintf(&p, "nm -an %s", a0); + if (p == NULL) + return; fi = popen(buf, "r"); - if (fi != NULL) { - while (fgets(buf, sizeof buf, fi)) { - if (buf[0] == ' ') - continue; - p = NULL; - a = strtoul(buf, &p, 16); - if (p == NULL) - continue; - if (a == 0) - continue; - if (*p++ != ' ') - continue; - p++; - if (*p++ != ' ') - continue; - if (*p <= ' ') - continue; - e = strchr(p, '\0'); - AN(e); - while (e > p && isspace(e[-1])) - e--; - *e = '\0'; - s = malloc(sizeof *s + strlen(p) + 1); - AN(s); - s->a = a; - s->n = (void*)(s + 1); - strcpy(s->n, p); - VTAILQ_INSERT_TAIL(&symbols, s, list); - } - pclose(fi); + free(p); + if (fi == NULL) + return; + while (fgets(buf, sizeof buf, fi)) { + if (buf[0] == ' ') + continue; + p = NULL; + a = strtoul(buf, &p, 16); + if (p == NULL) + continue; + if (a == 0) + continue; + if (*p++ != ' ') + continue; + p++; + if (*p++ != ' ') + continue; + if (*p <= ' ') + continue; + e = strchr(p, '\0'); + AN(e); + while (e > p && isspace(e[-1])) + e--; + *e = '\0'; + s = malloc(sizeof *s + strlen(p) + 1); + AN(s); + s->a = a; + s->n = (void*)(s + 1); + strcpy(s->n, p); + VTAILQ_INSERT_TAIL(&symbols, s, list); } + (void)pclose(fi); } /*--------------------------------------------------------------------*/ Modified: branches/2.0/varnish-cache/lib/libvcl/vcc_acl.c =================================================================== --- branches/2.0/varnish-cache/lib/libvcl/vcc_acl.c 2009-10-12 10:13:49 UTC (rev 4322) +++ branches/2.0/varnish-cache/lib/libvcl/vcc_acl.c 2009-10-13 08:36:10 UTC (rev 4323) @@ -250,14 +250,18 @@ { unsigned char b[4]; int i, j, k; + unsigned u; const char *p; memset(b, 0, sizeof b); p = ae->t_addr->dec; for (i = 0; i < 4; i++) { - j = sscanf(p, "%hhu%n", &b[i], &k); + j = sscanf(p, "%u%n", &u, &k); if (j != 1) return (0); + if (u & ~0xff) + return (0); + b[i] = (unsigned char)u; if (p[k] == '\0') break; if (p[k] != '.') From tfheen at projects.linpro.no Tue Oct 13 08:43:42 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Tue, 13 Oct 2009 10:43:42 +0200 (CEST) Subject: r4324 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091013084342.D0A451F739D@projects.linpro.no> Author: tfheen Date: 2009-10-13 10:43:42 +0200 (Tue, 13 Oct 2009) New Revision: 4324 Modified: branches/2.0/varnish-cache/bin/varnishd/varnishd.c Log: Merge r4261: Unbreak the symbol-hack Modified: branches/2.0/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/varnishd.c 2009-10-13 08:36:10 UTC (rev 4323) +++ branches/2.0/varnish-cache/bin/varnishd/varnishd.c 2009-10-13 08:43:42 UTC (rev 4324) @@ -445,7 +445,7 @@ VTAILQ_FOREACH(s, &symbols, list) { if (s->a > pp) continue; - if (s0 == NULL || s->a < s0->a) + if (s0 == NULL || s->a > s0->a) s0 = s; } if (s0 == NULL) @@ -466,7 +466,7 @@ asprintf(&p, "nm -an %s", a0); if (p == NULL) return; - fi = popen(buf, "r"); + fi = popen(p, "r"); free(p); if (fi == NULL) return; From tfheen at projects.linpro.no Tue Oct 13 09:05:12 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Tue, 13 Oct 2009 11:05:12 +0200 (CEST) Subject: r4325 - in branches/2.0/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20091013090512.794541F739D@projects.linpro.no> Author: tfheen Date: 2009-10-13 11:05:12 +0200 (Tue, 13 Oct 2009) New Revision: 4325 Added: branches/2.0/varnish-cache/bin/varnishtest/tests/r00561.vtc Removed: branches/2.0/varnish-cache/bin/varnishtest/tests/r00354.vtc Modified: branches/2.0/varnish-cache/bin/varnishd/cache_center.c branches/2.0/varnish-cache/bin/varnishd/cache_http.c Log: Merge r4263: Reject garbled requests If we cannot even make sense of the request, don't bother with attempting a reply. Fixes #561 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_center.c 2009-10-13 08:43:42 UTC (rev 4324) +++ branches/2.0/varnish-cache/bin/varnishd/cache_center.c 2009-10-13 09:05:12 UTC (rev 4325) @@ -905,6 +905,13 @@ http_Setup(sp->http, sp->ws); done = http_DissectRequest(sp); + /* If we could not even parse the request, just close */ + if (done < 0) { + sp->step = STP_DONE; + vca_close_session(sp, "junk"); + return (0); + } + /* Catch request snapshot */ sp->ws_req = WS_Snapshot(sp->ws); Modified: branches/2.0/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_http.c 2009-10-13 08:43:42 UTC (rev 4324) +++ branches/2.0/varnish-cache/bin/varnishd/cache_http.c 2009-10-13 09:05:12 UTC (rev 4325) @@ -406,7 +406,7 @@ q = p; for (; !vct_issp(*p); p++) { if (vct_isctl(*p)) - return (400); + return (-1); } hp->hd[h1].b = q; hp->hd[h1].e = p; @@ -414,14 +414,14 @@ /* Skip SP */ for (; vct_issp(*p); p++) { if (vct_isctl(*p)) - return (400); + return (-1); } /* Second field cannot contain LWS or CTL */ q = p; for (; !vct_islws(*p); p++) { if (vct_isctl(*p)) - return (400); + return (-1); } hp->hd[h2].b = q; hp->hd[h2].e = p; @@ -432,7 +432,7 @@ /* Skip SP */ for (; vct_issp(*p); p++) { if (vct_isctl(*p)) - return (400); + return (-1); } /* Third field is optional and cannot contain CTL */ @@ -440,7 +440,7 @@ if (!vct_iscrlf(*p)) { for (; !vct_iscrlf(*p); p++) if (vct_isctl(*p)) - return (400); + return (-1); } hp->hd[h3].b = q; hp->hd[h3].e = p; Deleted: branches/2.0/varnish-cache/bin/varnishtest/tests/r00354.vtc =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/r00354.vtc 2009-10-13 08:43:42 UTC (rev 4324) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/r00354.vtc 2009-10-13 09:05:12 UTC (rev 4325) @@ -1,16 +0,0 @@ -# $Id$ - -test "#354 Segfault in strcmp in http_DissectRequest()" - -server s1 { - rxreq - txresp -} - -varnish v1 -vcl+backend {} -start - -client c1 { - send "FOO\r\n\r\n" - rxresp - expect resp.status == 400 -} -run Copied: branches/2.0/varnish-cache/bin/varnishtest/tests/r00561.vtc (from rev 4263, trunk/varnish-cache/bin/varnishtest/tests/r00561.vtc) =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/r00561.vtc (rev 0) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/r00561.vtc 2009-10-13 09:05:12 UTC (rev 4325) @@ -0,0 +1,25 @@ +# $Id$ + +test "Junk request should not go to vcl_error" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_error { + return (restart); + } +} -start + +client c1 { + send "sljdslf\r\n\r\n" + delay .1 +} -run + +client c1 { + txreq + rxresp +} -run + From tfheen at projects.linpro.no Tue Oct 13 09:23:25 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Tue, 13 Oct 2009 11:23:25 +0200 (CEST) Subject: r4326 - in branches/2.0/varnish-cache: bin/varnishd include Message-ID: <20091013092325.CBD741F73A9@projects.linpro.no> Author: tfheen Date: 2009-10-13 11:23:25 +0200 (Tue, 13 Oct 2009) New Revision: 4326 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_fetch.c branches/2.0/varnish-cache/include/shmlog_tags.h Log: Merge r4232: Add a new SHMLOG tag FetchError and record why a backend fetch croaked. Modified: branches/2.0/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_fetch.c 2009-10-13 09:05:12 UTC (rev 4325) +++ branches/2.0/varnish-cache/bin/varnishd/cache_fetch.c 2009-10-13 09:23:25 UTC (rev 4326) @@ -70,8 +70,11 @@ while (cl > 0) { i = HTC_Read(htc, p, cl); - if (i <= 0) + if (i <= 0) { + WSP(sp, SLT_FetchError, + "straight read_error: %d", errno); return (-1); + } p += i; cl -= i; } @@ -110,8 +113,11 @@ /* If we didn't succeed, add to buffer, try again */ if (q == NULL || q == buf || *q != '\n') { - if (bp >= be) + if (bp >= be) { + WSP(sp, SLT_FetchError, + "chunked hex-line too long"); return (-1); + } /* * The semantics we need here is "read until you have * received at least one character, but feel free to @@ -128,8 +134,11 @@ * at a time. */ i = HTC_Read(htc, bp, 1); - if (i <= 0) + if (i <= 0) { + WSP(sp, SLT_FetchError, + "chunked read_error: %d", errno); return (-1); + } bp += i; continue; } @@ -241,8 +250,11 @@ AN(p); AN(st); i = HTC_Read(htc, p, v); - if (i < 0) + if (i < 0) { + WSP(sp, SLT_FetchError, + "eof read_error: %d", errno); return (-1); + } if (i == 0) break; p += i; Modified: branches/2.0/varnish-cache/include/shmlog_tags.h =================================================================== --- branches/2.0/varnish-cache/include/shmlog_tags.h 2009-10-13 09:05:12 UTC (rev 4325) +++ branches/2.0/varnish-cache/include/shmlog_tags.h 2009-10-13 09:23:25 UTC (rev 4326) @@ -56,8 +56,6 @@ SLTM(Backend) SLTM(Length) -SLTM(FetchError) - SLTM(RxRequest) SLTM(RxResponse) SLTM(RxStatus) @@ -103,3 +101,5 @@ SLTM(Hash) SLTM(Backend_health) + +SLTM(FetchError) From tfheen at projects.linpro.no Tue Oct 13 09:32:22 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Tue, 13 Oct 2009 11:32:22 +0200 (CEST) Subject: r4327 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091013093222.DEEC61F73A9@projects.linpro.no> Author: tfheen Date: 2009-10-13 11:32:22 +0200 (Tue, 13 Oct 2009) New Revision: 4327 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_fetch.c Log: Merge r4233: Add SHMLOG records for more backend fetch failures Modified: branches/2.0/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_fetch.c 2009-10-13 09:23:25 UTC (rev 4326) +++ branches/2.0/varnish-cache/bin/varnishd/cache_fetch.c 2009-10-13 09:32:22 UTC (rev 4327) @@ -350,8 +350,10 @@ /* Set up obj's workspace */ WS_Assert(sp->obj->ws_o); VBE_GetFd(sp); - if (sp->vbe == NULL) + if (sp->vbe == NULL) { + WSP(sp, SLT_FetchError, "no backend connection"); return (__LINE__); + } vc = sp->vbe; /* Inherit the backend timeouts from the selected backend */ SES_InheritBackendTimeouts(sp); @@ -371,6 +373,7 @@ /* Deal with any message-body the request might have */ i = FetchReqBody(sp); if (WRW_FlushRelease(w) || i > 0) { + WSP(sp, SLT_FetchError, "backend write error: %d", errno); VBE_ClosedFd(sp); /* XXX: other cleanup ? */ return (__LINE__); @@ -386,17 +389,19 @@ TCP_set_read_timeout(vc->fd, sp->first_byte_timeout); do { i = HTC_Rx(htc); + if (i < 0) { + WSP(sp, SLT_FetchError, + "http read error: %d", errno); + VBE_ClosedFd(sp); + /* XXX: other cleanup ? */ + return (__LINE__); + } TCP_set_read_timeout(vc->fd, sp->between_bytes_timeout); } while (i == 0); - if (i < 0) { - VBE_ClosedFd(sp); - /* XXX: other cleanup ? */ - return (__LINE__); - } - if (http_DissectResponse(sp->wrk, htc, hp)) { + WSP(sp, SLT_FetchError, "http format error"); VBE_ClosedFd(sp); /* XXX: other cleanup ? */ return (__LINE__); @@ -531,5 +536,3 @@ CLI_AddFuncs(DEBUG_CLI, debug_cmds); } - - From kristian at projects.linpro.no Tue Oct 13 14:10:44 2009 From: kristian at projects.linpro.no (kristian at projects.linpro.no) Date: Tue, 13 Oct 2009 16:10:44 +0200 (CEST) Subject: r4328 - in trunk/varnish-cache: bin/varnishd bin/varnishtest/tests lib/libvcl Message-ID: <20091013141044.60A391F72BB@projects.linpro.no> Author: kristian Date: 2009-10-13 16:10:43 +0200 (Tue, 13 Oct 2009) New Revision: 4328 Added: trunk/varnish-cache/bin/varnishtest/tests/c00029.vtc Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/heritage.h trunk/varnish-cache/bin/varnishd/mgt_param.c trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c trunk/varnish-cache/lib/libvcl/vcc_obj.c Log: Add saintmode_threshold When saintmode_threshold items have been added to the trouble-list, the backend will be considerd sick until one has timed out. Setting the threshold to 0 disables saintmode and any lock contention associated with it. Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-10-13 09:32:22 UTC (rev 4327) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-10-13 14:10:43 UTC (rev 4328) @@ -231,7 +231,12 @@ /* * It evaluates if a backend is healthy _for_a_specific_object_. * That means that it relies on sp->objhead. This is mainly for saint-mode, - * but also takes backend->healthy into account. + * but also takes backend->healthy into account. If + * params->saintmode_threshold is 0, this is basically just a test of + * backend->healthy. + * + * The threshold has to be evaluated _after_ the timeout check, otherwise + * items would never time out once the threshold is reached. */ unsigned int backend_is_healthy(const struct sess *sp, struct backend *backend) @@ -239,6 +244,7 @@ struct trouble *tr; struct trouble *tr2; struct trouble *old = NULL; + unsigned i = 0; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(backend, BACKEND_MAGIC); @@ -246,6 +252,10 @@ if (!backend->healthy) return 0; + /* Saintmode is disabled */ + if (params->saintmode_threshold == 0) + return 1; + /* No need to test if we don't have an object head to test against. * FIXME: Should check the magic too, but probably not assert? */ @@ -265,6 +275,15 @@ Lck_Unlock(&backend->mtx); return 0; } + + /* If the threshold is at 1, a single entry on the list + * will disable the backend. Since 0 is disable, ++i + * instead of i++ to allow this behavior. + */ + if (++i >= params->saintmode_threshold) { + Lck_Unlock(&backend->mtx); + return 0; + } } Lck_Unlock(&backend->mtx); Modified: trunk/varnish-cache/bin/varnishd/heritage.h =================================================================== --- trunk/varnish-cache/bin/varnishd/heritage.h 2009-10-13 09:32:22 UTC (rev 4327) +++ trunk/varnish-cache/bin/varnishd/heritage.h 2009-10-13 14:10:43 UTC (rev 4328) @@ -199,6 +199,9 @@ /* How long time does the ban lurker sleep */ double ban_lurker_sleep; + + /* Max size of the saintmode list. 0 == no saint mode. */ + unsigned saintmode_threshold; }; extern volatile struct params *params; Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-10-13 09:32:22 UTC (rev 4327) +++ trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-10-13 14:10:43 UTC (rev 4328) @@ -765,6 +765,12 @@ "A value of zero disables the ban lurker.", 0, "0.0", "s" }, + { "saintmode_threshold", tweak_uint, &master.saintmode_threshold, 0, UINT_MAX, + "The maximum number of objects held off by saint mode before no further " + "will be made to the backend until one times out. A value of 0 disables " + "saintmode.", + EXPERIMENTAL, + "10", "objects" }, { NULL, NULL, NULL } }; Added: trunk/varnish-cache/bin/varnishtest/tests/c00029.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/c00029.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/c00029.vtc 2009-10-13 14:10:43 UTC (rev 4328) @@ -0,0 +1,101 @@ +# $Id$ + +test "Test that saintmode_threshold correctly marks a backend as sick" + +server s1 { + rxreq + txresp + + rxreq + txresp + + rxreq + txresp + + rxreq + txresp + + rxreq + txresp -hdr "X-Saint: yes" + + rxreq + txresp -hdr "X-Saint: yes" + + rxreq + txresp -hdr "X-Saint: yes" + + rxreq + txresp -hdr "X-Saint: yes" +} -start + +varnish v1 -arg "-p saintmode_threshold=2" -vcl+backend { + sub vcl_recv { + set req.grace = 1h; + } + + sub vcl_fetch { + if (beresp.http.X-Saint == "yes") { + set beresp.saintmode = 20s; + restart; + } + set beresp.grace = 1h; + set beresp.ttl = 1s; + } + sub vcl_deliver { + set resp.http.X-Restarts = req.restarts; + } + } -start + +client c1 { + txreq -url "/one" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "0" + + txreq -url "/two" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "0" + + txreq -url "/three" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "0" + + txreq -url "/four" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "0" + + delay 2 + + txreq -url "/one" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "1" + + txreq -url "/two" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "1" + + txreq -url "/three" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "0" + + txreq -url "/four" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "0" + +} -run + Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-10-13 09:32:22 UTC (rev 4327) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-10-13 14:10:43 UTC (rev 4328) @@ -1,5 +1,5 @@ /* - * $Id: vcc_gen_fixed_token.tcl 4208 2009-08-24 14:51:26Z kristian $ + * $Id$ * * NB: This file is machine generated, DO NOT EDIT! * @@ -159,10 +159,9 @@ /* ../../include/vcl.h */ - vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4208 2009-08-24 14"); - vsb_cat(sb, ":51:26Z kristian $\n *\n * NB: This file is machine g"); - vsb_cat(sb, "enerated, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fi"); - vsb_cat(sb, "xed_token.tcl instead\n */\n\nstruct sess;\n"); + vsb_cat(sb, "/*\n * $Id$\n *\n * NB: This file is machine generate"); + vsb_cat(sb, "d, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_tok"); + vsb_cat(sb, "en.tcl instead\n */\n\nstruct sess;\n"); vsb_cat(sb, "struct cli;\n\ntypedef void vcl_init_f(struct cli *);\n"); vsb_cat(sb, "typedef void vcl_fini_f(struct cli *);\n"); vsb_cat(sb, "typedef int vcl_func_f(struct sess *sp);\n"); @@ -228,24 +227,25 @@ vsb_cat(sb, " * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWI"); vsb_cat(sb, "SE) ARISING IN ANY WAY\n * OUT OF THE USE OF THIS SOFT"); vsb_cat(sb, "WARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n"); - vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id: vrt.h 4235 2009-09-11 13:"); - vsb_cat(sb, "06:15Z phk $\n *\n * Runtime support for compiled VCL "); - vsb_cat(sb, "programs.\n *\n * XXX: When this file is changed, lib/"); - vsb_cat(sb, "libvcl/vcc_gen_fixed_token.tcl\n"); - vsb_cat(sb, " * XXX: *MUST* be rerun.\n */\n"); - vsb_cat(sb, "\nstruct sess;\nstruct vsb;\nstruct cli;\n"); - vsb_cat(sb, "struct director;\nstruct VCL_conf;\n"); - vsb_cat(sb, "struct sockaddr;\n\n/*\n * A backend probe specificati"); - vsb_cat(sb, "on\n */\n\nextern const void *vrt_magic_string_end;\n"); - vsb_cat(sb, "\nstruct vrt_backend_probe {\n\tconst char\t*url;\n"); - vsb_cat(sb, "\tconst char\t*request;\n\tdouble\t\ttimeout;\n"); - vsb_cat(sb, "\tdouble\t\tinterval;\n\tunsigned\twindow;\n"); - vsb_cat(sb, "\tunsigned\tthreshold;\n\tunsigned\tinitial;\n"); - vsb_cat(sb, "};\n\n/*\n * A backend is a host+port somewhere on the"); - vsb_cat(sb, " network\n */\nstruct vrt_backend {\n"); - vsb_cat(sb, "\tconst char\t\t\t*vcl_name;\n\tconst char\t\t\t*ident"); - vsb_cat(sb, ";\n\n\tconst char\t\t\t*hosthdr;\n"); - vsb_cat(sb, "\n\tconst unsigned char\t\t*ipv4_sockaddr;\n"); + vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id$\n *\n"); + vsb_cat(sb, " * Runtime support for compiled VCL programs.\n"); + vsb_cat(sb, " *\n * XXX: When this file is changed, lib/libvcl/vcc_"); + vsb_cat(sb, "gen_fixed_token.tcl\n * XXX: *MUST* be rerun.\n"); + vsb_cat(sb, " */\n\nstruct sess;\nstruct vsb;\n"); + vsb_cat(sb, "struct cli;\nstruct director;\n"); + vsb_cat(sb, "struct VCL_conf;\nstruct sockaddr;\n"); + vsb_cat(sb, "\n/*\n * A backend probe specification\n"); + vsb_cat(sb, " */\n\nextern const void * const vrt_magic_string_end;"); + vsb_cat(sb, "\n\nstruct vrt_backend_probe {\n"); + vsb_cat(sb, "\tconst char\t*url;\n\tconst char\t*request;\n"); + vsb_cat(sb, "\tdouble\t\ttimeout;\n\tdouble\t\tinterval;\n"); + vsb_cat(sb, "\tunsigned\twindow;\n\tunsigned\tthreshold;\n"); + vsb_cat(sb, "\tunsigned\tinitial;\n};\n\n/*\n"); + vsb_cat(sb, " * A backend is a host+port somewhere on the network\n"); + vsb_cat(sb, " */\nstruct vrt_backend {\n\tconst char\t\t\t*vcl_name"); + vsb_cat(sb, ";\n\tconst char\t\t\t*ident;\n\n"); + vsb_cat(sb, "\tconst char\t\t\t*hosthdr;\n\n"); + vsb_cat(sb, "\tconst unsigned char\t\t*ipv4_sockaddr;\n"); vsb_cat(sb, "\tconst unsigned char\t\t*ipv6_sockaddr;\n"); vsb_cat(sb, "\n\tdouble\t\t\t\tconnect_timeout;\n"); vsb_cat(sb, "\tdouble\t\t\t\tfirst_byte_timeout;\n"); @@ -317,27 +317,25 @@ /* ../../include/vrt_obj.h */ - vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4208 2009-08-24 14"); - vsb_cat(sb, ":51:26Z kristian $\n *\n * NB: This file is machine g"); - vsb_cat(sb, "enerated, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fi"); - vsb_cat(sb, "xed_token.tcl instead\n */\n\nstruct sockaddr * VRT_r_"); - vsb_cat(sb, "client_ip(const struct sess *);\n"); - vsb_cat(sb, "struct sockaddr * VRT_r_server_ip(struct sess *);\n"); - vsb_cat(sb, "const char * VRT_r_server_hostname(struct sess *);\n"); - vsb_cat(sb, "const char * VRT_r_server_identity(struct sess *);\n"); - vsb_cat(sb, "int VRT_r_server_port(struct sess *);\n"); - vsb_cat(sb, "const char * VRT_r_req_request(const struct sess *);\n"); - vsb_cat(sb, "void VRT_l_req_request(const struct sess *, const char"); - vsb_cat(sb, " *, ...);\nconst char * VRT_r_req_url(const struct ses"); - vsb_cat(sb, "s *);\nvoid VRT_l_req_url(const struct sess *, const c"); - vsb_cat(sb, "har *, ...);\nconst char * VRT_r_req_proto(const struc"); - vsb_cat(sb, "t sess *);\nvoid VRT_l_req_proto(const struct sess *, "); - vsb_cat(sb, "const char *, ...);\nvoid VRT_l_req_hash(struct sess *"); - vsb_cat(sb, ", const char *);\nstruct director * VRT_r_req_backend("); - vsb_cat(sb, "struct sess *);\nvoid VRT_l_req_backend(struct sess *,"); - vsb_cat(sb, " struct director *);\nint VRT_r_req_restarts(const str"); - vsb_cat(sb, "uct sess *);\ndouble VRT_r_req_grace(struct sess *);\n"); - vsb_cat(sb, "void VRT_l_req_grace(struct sess *, double);\n"); + vsb_cat(sb, "/*\n * $Id$\n *\n * NB: This file is machine generate"); + vsb_cat(sb, "d, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_tok"); + vsb_cat(sb, "en.tcl instead\n */\n\nstruct sockaddr * VRT_r_client_"); + vsb_cat(sb, "ip(const struct sess *);\nstruct sockaddr * VRT_r_serv"); + vsb_cat(sb, "er_ip(struct sess *);\nconst char * VRT_r_server_hostn"); + vsb_cat(sb, "ame(struct sess *);\nconst char * VRT_r_server_identit"); + vsb_cat(sb, "y(struct sess *);\nint VRT_r_server_port(struct sess *"); + vsb_cat(sb, ");\nconst char * VRT_r_req_request(const struct sess *"); + vsb_cat(sb, ");\nvoid VRT_l_req_request(const struct sess *, const "); + vsb_cat(sb, "char *, ...);\nconst char * VRT_r_req_url(const struct"); + vsb_cat(sb, " sess *);\nvoid VRT_l_req_url(const struct sess *, con"); + vsb_cat(sb, "st char *, ...);\nconst char * VRT_r_req_proto(const s"); + vsb_cat(sb, "truct sess *);\nvoid VRT_l_req_proto(const struct sess"); + vsb_cat(sb, " *, const char *, ...);\nvoid VRT_l_req_hash(struct se"); + vsb_cat(sb, "ss *, const char *);\nstruct director * VRT_r_req_back"); + vsb_cat(sb, "end(struct sess *);\nvoid VRT_l_req_backend(struct ses"); + vsb_cat(sb, "s *, struct director *);\nint VRT_r_req_restarts(const"); + vsb_cat(sb, " struct sess *);\ndouble VRT_r_req_grace(struct sess *"); + vsb_cat(sb, ");\nvoid VRT_l_req_grace(struct sess *, double);\n"); vsb_cat(sb, "const char * VRT_r_req_xid(struct sess *);\n"); vsb_cat(sb, "unsigned VRT_r_req_esi(struct sess *);\n"); vsb_cat(sb, "void VRT_l_req_esi(struct sess *, unsigned);\n"); Modified: trunk/varnish-cache/lib/libvcl/vcc_obj.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_obj.c 2009-10-13 09:32:22 UTC (rev 4327) +++ trunk/varnish-cache/lib/libvcl/vcc_obj.c 2009-10-13 14:10:43 UTC (rev 4328) @@ -1,5 +1,5 @@ /* - * $Id: vcc_gen_fixed_token.tcl 4208 2009-08-24 14:51:26Z kristian $ + * $Id$ * * NB: This file is machine generated, DO NOT EDIT! * From tfheen at projects.linpro.no Tue Oct 13 14:29:51 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Tue, 13 Oct 2009 16:29:51 +0200 (CEST) Subject: r4329 - trunk/varnish-cache/bin/varnishd Message-ID: <20091013142951.3D9E71F72BB@projects.linpro.no> Author: tfheen Date: 2009-10-13 16:29:50 +0200 (Tue, 13 Oct 2009) New Revision: 4329 Modified: trunk/varnish-cache/bin/varnishd/varnishd.1 Log: s/http_workspace/sess_workspace/ Fixes #557 Modified: trunk/varnish-cache/bin/varnishd/varnishd.1 =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.1 2009-10-13 14:10:43 UTC (rev 4328) +++ trunk/varnish-cache/bin/varnishd/varnishd.1 2009-10-13 14:29:50 UTC (rev 4329) @@ -520,7 +520,7 @@ are specified, the latter should be specified last. .Pp The default is "nogroup". -.It Va http_workspace +.It Va sess_workspace The size of the per-session workspace for HTTP protocol data. For performance reasons, this space is preallocated, so any change to this parameter will only apply to new client sessions. From tfheen at projects.linpro.no Tue Oct 13 15:26:18 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Tue, 13 Oct 2009 17:26:18 +0200 (CEST) Subject: r4330 - in trunk/varnish-cache: . doc Message-ID: <20091013152618.6546D1F72BB@projects.linpro.no> Author: tfheen Date: 2009-10-13 17:26:18 +0200 (Tue, 13 Oct 2009) New Revision: 4330 Modified: trunk/varnish-cache/configure.ac trunk/varnish-cache/doc/Makefile.am Log: Skip building documentation when we lack xsltproc Previously, trying to build from SVN without xsltproc installed would get you errors like: --xinclude -o changes-2.0.1.html changes-2.0.1.xml --xinclude:No such file or directory which is quite confusing. We now rather just skip the docs completely if we can't find any xsltproc. If we actually do need to rebuild the docs (which should only happen when running make dist), we error out with a useful error message. Modified: trunk/varnish-cache/configure.ac =================================================================== --- trunk/varnish-cache/configure.ac 2009-10-13 14:29:50 UTC (rev 4329) +++ trunk/varnish-cache/configure.ac 2009-10-13 15:26:18 UTC (rev 4330) @@ -24,7 +24,11 @@ AC_PROG_INSTALL AC_PROG_LIBTOOL AC_PROG_MAKE_SET -AC_CHECK_PROGS(XSLTPROC, [xsltproc]) +AC_CHECK_PROGS(XSLTPROC, [xsltproc], "no") +if test "x$XSLTPROC" = "xno"; then + AC_MSG_WARN([xsltproc not found ? not building documentation]) +fi +AM_CONDITIONAL(HAVE_XSLTPROC,[test "x$XSLTPROC" != "xno"]) # Checks for libraries. save_LIBS="${LIBS}" Modified: trunk/varnish-cache/doc/Makefile.am =================================================================== --- trunk/varnish-cache/doc/Makefile.am 2009-10-13 14:29:50 UTC (rev 4329) +++ trunk/varnish-cache/doc/Makefile.am 2009-10-13 15:26:18 UTC (rev 4330) @@ -17,17 +17,26 @@ changes-1.0.3-1.0.4.xml \ ${CHANGELOGS:.html=.xml} +if HAVE_XSLTPROC all: ${CHANGELOGS} +endif EXTRA_DIST = ${CHANGELOGS} ${XML} \ changes.css changes-html.xsl \ getting-started.html CLEANFILES = ${CHANGELOGS} - SUFFIXES = .xml .html .xml.html: +if HAVE_XSLTPROC ${XSLTPROC} --xinclude -o $@ $< +else + @echo "========================================" + @echo "You need xsltproc installed to make dist" + @echo "========================================" + @false +endif ${CHANGELOGS}: changes-html.xsl + From tfheen at projects.linpro.no Tue Oct 13 15:43:55 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Tue, 13 Oct 2009 17:43:55 +0200 (CEST) Subject: r4331 - trunk/varnish-cache/man Message-ID: <20091013154355.7BFE81F72BB@projects.linpro.no> Author: tfheen Date: 2009-10-13 17:43:55 +0200 (Tue, 13 Oct 2009) New Revision: 4331 Modified: trunk/varnish-cache/man/vcl.7so Log: Document string concatenation Fixes #420 Modified: trunk/varnish-cache/man/vcl.7so =================================================================== --- trunk/varnish-cache/man/vcl.7so 2009-10-13 15:26:18 UTC (rev 4330) +++ trunk/varnish-cache/man/vcl.7so 2009-10-13 15:43:55 UTC (rev 4331) @@ -60,6 +60,9 @@ in strings in VCL, which use the (%xx) escape mechanism just like URLs, so it can be freely used in regular expressions without doubling. .Pp +Strings are concatenated by just putting them one after each other +without any operator in between. +.Pp Assignments are introduced with the .Cm set keyword. From tfheen at projects.linpro.no Thu Oct 15 15:42:11 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Thu, 15 Oct 2009 17:42:11 +0200 (CEST) Subject: r4332 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091015154211.D2BBF38507@projects.linpro.no> Author: tfheen Date: 2009-10-15 17:42:11 +0200 (Thu, 15 Oct 2009) New Revision: 4332 Modified: branches/2.0/varnish-cache/bin/varnishd/cache.h branches/2.0/varnish-cache/bin/varnishd/cache_ban.c branches/2.0/varnish-cache/bin/varnishd/cache_hash.c Log: Merge r4205: First part of ban list lurker Do the simple part of ban list lurker: link the objcores off the bans the reference. Modified: branches/2.0/varnish-cache/bin/varnishd/cache.h =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache.h 2009-10-13 15:43:55 UTC (rev 4331) +++ branches/2.0/varnish-cache/bin/varnishd/cache.h 2009-10-15 15:42:11 UTC (rev 4332) @@ -285,6 +285,7 @@ struct http http[1]; VTAILQ_ENTRY(object) list; + VTAILQ_ENTRY(object) ban_list; VTAILQ_HEAD(, storage) store; Modified: branches/2.0/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_ban.c 2009-10-13 15:43:55 UTC (rev 4331) +++ branches/2.0/varnish-cache/bin/varnishd/cache_ban.c 2009-10-15 15:42:11 UTC (rev 4332) @@ -86,6 +86,7 @@ int flags; #define BAN_F_GONE (1 << 0) VTAILQ_HEAD(,ban_test) tests; + VTAILQ_HEAD(,object) obj; }; static VTAILQ_HEAD(banhead,ban) ban_head = VTAILQ_HEAD_INITIALIZER(ban_head); @@ -103,6 +104,7 @@ if (b == NULL) return (b); VTAILQ_INIT(&b->tests); + VTAILQ_INIT(&b->obj); return (b); } @@ -146,6 +148,9 @@ struct ban_test *bt; CHECK_OBJ_NOTNULL(b, BAN_MAGIC); + AZ(b->refcount); + assert(VTAILQ_EMPTY(&b->obj)); + while (!VTAILQ_EMPTY(&b->tests)) { bt = VTAILQ_FIRST(&b->tests); VTAILQ_REMOVE(&b->tests, bt, list); @@ -439,6 +444,7 @@ Lck_Lock(&ban_mtx); o->ban = ban_start; ban_start->refcount++; + VTAILQ_INSERT_TAIL(&ban_start->obj, o, ban_list); Lck_Unlock(&ban_mtx); } @@ -470,6 +476,7 @@ CHECK_OBJ_NOTNULL(o->ban, BAN_MAGIC); Lck_Lock(&ban_mtx); o->ban->refcount--; + VTAILQ_REMOVE(&o->ban->obj, o, ban_list); o->ban = NULL; /* Attempt to purge last ban entry */ @@ -518,8 +525,11 @@ Lck_Lock(&ban_mtx); o->ban->refcount--; - if (b == o->ban) /* not banned */ + VTAILQ_REMOVE(&o->ban->obj, o, ban_list); + if (b == o->ban) { /* not banned */ + VTAILQ_INSERT_TAIL(&b0->obj, o, ban_list); b0->refcount++; + } VSL_stats->n_purge_obj_test++; VSL_stats->n_purge_re_test += tests; Lck_Unlock(&ban_mtx); Modified: branches/2.0/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_hash.c 2009-10-13 15:43:55 UTC (rev 4331) +++ branches/2.0/varnish-cache/bin/varnishd/cache_hash.c 2009-10-15 15:42:11 UTC (rev 4332) @@ -460,7 +460,8 @@ if (r != 0) return; - BAN_DestroyObj(o); + if (oh != NULL) + BAN_DestroyObj(o); DSL(0x40, SLT_Debug, 0, "Object %u workspace min free %u", o->xid, WS_Free(o->ws_o)); From tfheen at projects.linpro.no Tue Oct 20 07:11:22 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Tue, 20 Oct 2009 09:11:22 +0200 (CEST) Subject: r4333 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20091020071122.A291E1F735B@projects.linpro.no> Author: tfheen Date: 2009-10-20 09:11:22 +0200 (Tue, 20 Oct 2009) New Revision: 4333 Modified: branches/2.0/varnish-cache/bin/varnishd/mgt_param.c Log: Change srcaddr_ttl default to 0s srcaddr_{ttl,hash} was never really used for anything, so turn it off to gain a small bit of performance. It is gone in trunk, so no corresponding change there we can merge. Modified: branches/2.0/varnish-cache/bin/varnishd/mgt_param.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/mgt_param.c 2009-10-15 15:42:11 UTC (rev 4332) +++ branches/2.0/varnish-cache/bin/varnishd/mgt_param.c 2009-10-20 07:11:22 UTC (rev 4333) @@ -585,7 +585,7 @@ "Lifetime of srcaddr entries.\n" "Zero will disable srcaddr accounting entirely.", EXPERIMENTAL, - "30", "seconds" }, + "0", "seconds" }, { "backend_http11", tweak_bool, &master.backend_http11, 0, 0, "Force all backend requests to be HTTP/1.1.\n" "By default we copy the protocol version from the " From tfheen at projects.linpro.no Tue Oct 20 13:55:55 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Tue, 20 Oct 2009 15:55:55 +0200 (CEST) Subject: r4334 - in trunk/varnish-cache: . bin/varnishd bin/varnishlog include lib/libvarnish lib/libvarnishapi lib/libvcl Message-ID: <20091020135555.5F4DC1F73CE@projects.linpro.no> Author: tfheen Date: 2009-10-20 15:55:55 +0200 (Tue, 20 Oct 2009) New Revision: 4334 Added: trunk/varnish-cache/include/vre.h trunk/varnish-cache/lib/libvarnish/vre.c Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/bin/varnishd/cache_ban.h trunk/varnish-cache/bin/varnishd/cache_vrt_re.c trunk/varnish-cache/bin/varnishlog/varnishlog.c trunk/varnish-cache/configure.ac trunk/varnish-cache/include/vrt.h trunk/varnish-cache/lib/libvarnish/Makefile.am trunk/varnish-cache/lib/libvarnishapi/shmlog.c trunk/varnish-cache/lib/libvcl/vcc_compile.h trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c trunk/varnish-cache/lib/libvcl/vcc_parse.c trunk/varnish-cache/lib/libvcl/vcc_string.c Log: Switch regex engine to PCRE We no longer use POSIX style regexes, but rather PCRE regexes. This introduces a hard dependency on libpcre. Test suite passes, but only lightly tested. The regex functions are in its own source file to make it easy to change the decision if somebody wants to use non-PCRE for some reason. Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-10-20 07:11:22 UTC (rev 4333) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-10-20 13:55:55 UTC (rev 4334) @@ -50,13 +50,13 @@ #include #include #include -#include #include "shmlog.h" #include "cli.h" #include "cli_priv.h" #include "cache.h" #include "hash_slinger.h" +#include "vre.h" #include "cache_ban.h" @@ -116,7 +116,7 @@ bt = VTAILQ_FIRST(&b->tests); VTAILQ_REMOVE(&b->tests, bt, list); if (bt->flags & BAN_T_REGEXP) - regfree(&bt->re); + VRE_free(&bt->re); if (bt->dst != NULL) free(bt->dst); if (bt->src != NULL) @@ -137,10 +137,13 @@ if (p == NULL) return(!(bt->flags & BAN_T_NOT)); - if (bt->flags & BAN_T_REGEXP) - i = regexec(&bt->re, p, 0, NULL, 0); - else + if (bt->flags & BAN_T_REGEXP) { + i = VRE_exec(bt->re, p, strlen(p), 0, 0, NULL, 0); + if (i >= 0) + i = 0; + } else { i = strcmp(bt->dst, p); + } if (bt->flags & BAN_T_NOT) return (!i); return (i); @@ -199,15 +202,13 @@ static int ban_parse_regexp(struct cli *cli, struct ban_test *bt, const char *a3) { - int i; - char buf[512]; + const char *error; + int erroroffset; - i = regcomp(&bt->re, a3, REG_EXTENDED | REG_ICASE | REG_NOSUB); - if (i) { - (void)regerror(i, &bt->re, buf, sizeof buf); - regfree(&bt->re); - VSL(SLT_Debug, 0, "REGEX: <%s>", buf); - cli_out(cli, "%s", buf); + bt->re = VRE_compile(a3, 0, &error, &erroroffset); + if (bt->re == NULL) { + VSL(SLT_Debug, 0, "REGEX: <%s>", error); + cli_out(cli, "%s", error); cli_result(cli, CLIS_PARAM); return (-1); } Modified: trunk/varnish-cache/bin/varnishd/cache_ban.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.h 2009-10-20 07:11:22 UTC (rev 4333) +++ trunk/varnish-cache/bin/varnishd/cache_ban.h 2009-10-20 13:55:55 UTC (rev 4334) @@ -44,7 +44,7 @@ int flags; #define BAN_T_REGEXP (1 << 0) #define BAN_T_NOT (1 << 1) - regex_t re; + vre_t *re; char *dst; char *src; }; Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_re.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt_re.c 2009-10-20 07:11:22 UTC (rev 4333) +++ trunk/varnish-cache/bin/varnishd/cache_vrt_re.c 2009-10-20 13:55:55 UTC (rev 4334) @@ -40,47 +40,47 @@ #include #include #include -#include #include "shmlog.h" #include "vrt.h" +#include "vre.h" #include "vcl.h" #include "cache.h" void -VRT_re_init(void **rep, const char *re, int sub) +VRT_re_init(void **rep, const char *re) { - regex_t *t; + vre_t *t; + const char *error; + int erroroffset; - t = calloc(sizeof *t, 1); - XXXAN(t); /* This was already check-compiled by the VCL compiler */ - AZ(regcomp(t, re, REG_EXTENDED | REG_ICASE | (sub ? 0 : REG_NOSUB))); + t = VRE_compile(re, 0, &error, &erroroffset); + AN(t); *rep = t; } void VRT_re_fini(void *rep) { - if (rep != NULL) - regfree(rep); + VRE_free((vre_t**)&rep); } int VRT_re_match(const char *s, void *re) { - regex_t *t; + vre_t *t; int i; if (s == NULL) return (0); AN(re); t = re; - i = regexec(t, s, 0, NULL, 0); - if (i == 0) + i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0); + if (i >= 0) return (1); - assert(i == REG_NOMATCH); + assert(i == VRE_ERROR_NOMATCH); return (0); } @@ -88,8 +88,8 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re, const char *sub) { - regmatch_t pm[10]; - regex_t *t; + int ovector[30]; + vre_t *t; int i, l; txt res; char *b0; @@ -100,10 +100,11 @@ if (str == NULL) return (""); t = re; - i = regexec(t, str, 10, pm, 0); + memset(&ovector, 0, sizeof(ovector)); + i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30); /* If it didn't match, we can return the original string */ - if (i == REG_NOMATCH) + if (i == VRE_ERROR_NOMATCH) return(str); u = WS_Reserve(sp->http->ws, 0); @@ -112,8 +113,7 @@ do { /* Copy prefix to match */ - Tadd(&res, str, pm[0].rm_so); - + Tadd(&res, str, ovector[0]); for (s = sub ; *s != '\0'; s++ ) { if (*s != '\\' || s[1] == '\0') { if (res.b < res.e) @@ -123,19 +123,20 @@ s++; if (isdigit(*s)) { x = *s - '0'; - l = pm[x].rm_eo - pm[x].rm_so; - Tadd(&res, str + pm[x].rm_so, l); + l = ovector[2*x+1] - ovector[2*x]; + Tadd(&res, str + ovector[2*x], l); continue; } else { if (res.b < res.e) *res.b++ = *s; } } - str += pm[0].rm_eo; + str += ovector[1]; if (!all) break; - i = regexec(t, str, 10, pm, 0); - } while (i != REG_NOMATCH); + memset(&ovector, 0, sizeof(ovector)); + i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30); + } while (i != VRE_ERROR_NOMATCH); /* Copy suffix to match */ l = strlen(str) + 1; Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c =================================================================== --- trunk/varnish-cache/bin/varnishlog/varnishlog.c 2009-10-20 07:11:22 UTC (rev 4333) +++ trunk/varnish-cache/bin/varnishlog/varnishlog.c 2009-10-20 13:55:55 UTC (rev 4334) @@ -36,7 +36,6 @@ #include #include -#include #include #include #include @@ -51,6 +50,7 @@ #include "libvarnish.h" #include "shmlog.h" +#include "vre.h" #include "varnishapi.h" static int b_flag, c_flag; @@ -80,7 +80,7 @@ #define F_MATCH (1 << 1) static int match_tag = -1; -static regex_t match_re; +static vre_t *match_re; static void h_order_finish(int fd) @@ -132,7 +132,7 @@ assert(ob[fd] != NULL); } if (tag == match_tag && - !regexec(&match_re, ptr, 0, NULL, 0)) + VRE_exec(match_re, ptr, len, 0, 0, NULL, 0) > 0) flg[fd] |= F_MATCH; if ((tag == SLT_BackendOpen || tag == SLT_SessionOpen || @@ -198,6 +198,8 @@ do_order(struct VSL_data *vd, int argc, char **argv) { int i; + const char *error; + int erroroffset; if (argc == 2) { match_tag = name2tag(argv[0]); @@ -205,11 +207,9 @@ fprintf(stderr, "Tag \"%s\" unknown\n", argv[0]); exit(2); } - i = regcomp(&match_re, argv[1], REG_EXTENDED | REG_NOSUB); - if (i) { - char buf[BUFSIZ]; - regerror(i, &match_re, buf, sizeof buf); - fprintf(stderr, "%s\n", buf); + match_re = VRE_compile(argv[1], 0, &error, &erroroffset); + if (match_re == NULL) { + fprintf(stderr, "Invalid regex: %s\n", error); exit(2); } } Modified: trunk/varnish-cache/configure.ac =================================================================== --- trunk/varnish-cache/configure.ac 2009-10-20 07:11:22 UTC (rev 4333) +++ trunk/varnish-cache/configure.ac 2009-10-20 13:55:55 UTC (rev 4334) @@ -75,6 +75,37 @@ AC_CHECK_LIBM AC_SUBST(LIBM) +PKG_PROG_PKG_CONFIG +if test -n $PKG_CONFIG; then + PKG_CHECK_MODULES([PCRE], [libpcre]) +else + AC_CHECK_PROG(PCRE_CONFIG, pcre-config, pcre-config) + AC_ARG_WITH(pcre-config, + AS_HELP_STRING([--with-pcre-config=PATH], + [Location of PCRE pcre-config (auto)]), + [pcre_config="$withval"], + [pcre_config=""]) + + if test "x$pcre_config" != "x" ; then + AC_MSG_CHECKING(for $pcre_config) + + if test -f $pcre_config ; then + PCRE_CONFIG=$pcre_config + AC_MSG_RESULT(yes) + else + AC_MSG_RESULT(no - searching PATH) + fi + fi + if test "x$PCRE_CONFIG" = "x"; then + AC_CHECK_PROGS(PCRE_CONFIG, pcre-config) + fi + PCRE_CFLAGS=`$PCRE_CONFIG --cflags` + PCRE_LIBS=`$PCRE_CONFIG --libs` +fi +AC_SUBST(PCRE_CFLAGS) +AC_SUBST(PCRE_LIBS) + + # Checks for header files. AC_HEADER_STDC AC_HEADER_SYS_WAIT Copied: trunk/varnish-cache/include/vre.h (from rev 4331, trunk/varnish-cache/bin/varnishd/cache_ban.h) =================================================================== --- trunk/varnish-cache/include/vre.h (rev 0) +++ trunk/varnish-cache/include/vre.h 2009-10-20 13:55:55 UTC (rev 4334) @@ -0,0 +1,45 @@ +/*- + * Copyright (c) 2009 Redpill Linpro AS + * All rights reserved. + * + * Author: Tollef Fog Heen + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id$ + * + * Regular expression support + * + */ + +struct vre; +typedef struct vre vre_t; + +/* This maps to PCRE error codes */ +#define VRE_ERROR_NOMATCH (-1) + +/* And those to PCRE options */ +#define VRE_CASELESS 0x00000001 + +vre_t *VRE_compile(const char *, int, const char **, int *); +int VRE_exec(const vre_t *, const char *, int, int, int, int *, int); +void VRE_free(vre_t **); Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2009-10-20 07:11:22 UTC (rev 4333) +++ trunk/varnish-cache/include/vrt.h 2009-10-20 13:55:55 UTC (rev 4334) @@ -136,7 +136,7 @@ void VRT_acl_log(const struct sess *, const char *msg); /* Regexp related */ -void VRT_re_init(void **, const char *, int sub); +void VRT_re_init(void **, const char *); void VRT_re_fini(void *); int VRT_re_match(const char *, void *re); const char *VRT_regsub(const struct sess *sp, int all, const char *, Modified: trunk/varnish-cache/lib/libvarnish/Makefile.am =================================================================== --- trunk/varnish-cache/lib/libvarnish/Makefile.am 2009-10-20 07:11:22 UTC (rev 4333) +++ trunk/varnish-cache/lib/libvarnish/Makefile.am 2009-10-20 13:55:55 UTC (rev 4334) @@ -1,6 +1,6 @@ # $Id$ -INCLUDES = -I$(top_srcdir)/include +INCLUDES = -I$(top_srcdir)/include @PCRE_CFLAGS@ lib_LTLIBRARIES = libvarnish.la @@ -23,12 +23,13 @@ vev.c \ vlu.c \ vpf.c \ + vre.c \ vsb.c \ vsha256.c \ vss.c \ vtmpfile.c -libvarnish_la_LIBADD = ${RT_LIBS} ${NET_LIBS} ${LIBM} +libvarnish_la_LIBADD = ${RT_LIBS} ${NET_LIBS} ${LIBM} @PCRE_LIBS@ DISTCLEANFILES = svn_version.c svn_version.c: FORCE Copied: trunk/varnish-cache/lib/libvarnish/vre.c (from rev 4331, trunk/varnish-cache/bin/varnishd/cache_ban.h) =================================================================== --- trunk/varnish-cache/lib/libvarnish/vre.c (rev 0) +++ trunk/varnish-cache/lib/libvarnish/vre.c 2009-10-20 13:55:55 UTC (rev 4334) @@ -0,0 +1,78 @@ +/*- + * Copyright (c) 2006-2009 Redpill Linpro AS + * All rights reserved. + * + * Author: Tollef Fog Heen + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +#include "libvarnish.h" +#include "miniobj.h" +#include "vre.h" + +struct vre { + unsigned magic; +#define VRE_MAGIC 0xe83097dc + pcre *re; +}; + +vre_t *VRE_compile(const char *pattern, int options, + const char **errptr, int *erroffset) { + vre_t *v; + *errptr = NULL; *erroffset = 0; + + ALLOC_OBJ(v, VRE_MAGIC); + AN(v); + v->re = pcre_compile(pattern, options, errptr, erroffset, NULL); + if (v->re == NULL) { + VRE_free(&v); + return NULL; + } + return v; +} + +int VRE_exec(const vre_t *code, const char *subject, int length, + int startoffset, int options, int *ovector, int ovecsize) { + CHECK_OBJ_NOTNULL(code, VRE_MAGIC); + int ov[30]; + if (ovector == NULL) { + ovector = ov; + ovecsize = sizeof(ov)/sizeof(ov[0]); + } + + return pcre_exec(code->re, NULL, subject, length, + startoffset, options, ovector, ovecsize); +} + +void VRE_free(vre_t **vv) { + + vre_t *v = *vv; + + *vv = NULL; + CHECK_OBJ(v, VRE_MAGIC); + pcre_free(v->re); + v->magic = 0; + FREE_OBJ(v); +} Modified: trunk/varnish-cache/lib/libvarnishapi/shmlog.c =================================================================== --- trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2009-10-20 07:11:22 UTC (rev 4333) +++ trunk/varnish-cache/lib/libvarnishapi/shmlog.c 2009-10-20 13:55:55 UTC (rev 4334) @@ -40,13 +40,13 @@ #include #include #include -#include #include #include #include #include #include "shmlog.h" +#include "vre.h" #include "miniobj.h" #include "varnishapi.h" @@ -84,8 +84,8 @@ #define M_SELECT (1 << 3) int regflags; - regex_t *regincl; - regex_t *regexcl; + vre_t *regincl; + vre_t *regexcl; unsigned long skip; unsigned long keep; @@ -170,7 +170,7 @@ assert(VSL_S_BACKEND == M_BACKEND); vd = calloc(sizeof *vd, 1); assert(vd != NULL); - vd->regflags = REG_EXTENDED | REG_NOSUB; + vd->regflags = 0; vd->magic = VSL_MAGIC; vd->fd = -1; return (vd); @@ -272,7 +272,6 @@ VSL_NextLog(struct VSL_data *vd, unsigned char **pp) { unsigned char *p; - regmatch_t rm; unsigned u, l; int i; @@ -315,19 +314,19 @@ if (vd->c_opt && !(vd->map[u] & M_CLIENT)) continue; if (vd->regincl != NULL) { - rm.rm_so = 0; - rm.rm_eo = l; - i = regexec(vd->regincl, - (char *)p + SHMLOG_DATA, 1, &rm, 0); - if (i == REG_NOMATCH) + i = VRE_exec(vd->regincl, + (char *)p + SHMLOG_DATA, + SHMLOG_LEN(p) - SHMLOG_DATA, /* Length */ + 0, 0, NULL, 0); + if (i == VRE_ERROR_NOMATCH) continue; } if (vd->regexcl != NULL) { - rm.rm_so = 0; - rm.rm_eo = l; - i = regexec(vd->regexcl, - (char *)p + SHMLOG_DATA, 1, &rm, 0); - if (i != REG_NOMATCH) + i = VRE_exec(vd->regincl, + (char *)p + SHMLOG_DATA, + SHMLOG_LEN(p) - SHMLOG_DATA, /* Length */ + 0, 0, NULL, 0); + if (i != VRE_ERROR_NOMATCH) continue; } *pp = p; @@ -412,9 +411,9 @@ static int vsl_IX_arg(struct VSL_data *vd, const char *opt, int arg) { - int i; - regex_t **rp; - char buf[BUFSIZ]; + vre_t **rp; + const char *error; + int erroroffset; CHECK_OBJ_NOTNULL(vd, VSL_MAGIC); if (arg == 'I') @@ -425,17 +424,11 @@ fprintf(stderr, "Option %c can only be given once", arg); return (-1); } - *rp = calloc(sizeof(regex_t), 1); + *rp = VRE_compile(opt, vd->regflags, &error, &erroroffset); if (*rp == NULL) { - perror("malloc"); + fprintf(stderr, "Illegal regex: %s\n", error); return (-1); } - i = regcomp(*rp, opt, vd->regflags); - if (i) { - regerror(i, *rp, buf, sizeof buf); - fprintf(stderr, "%s", buf); - return (-1); - } return (1); } @@ -547,7 +540,7 @@ case 'i': case 'x': return (vsl_ix_arg(vd, opt, arg)); case 'r': return (vsl_r_arg(vd, opt)); case 'I': case 'X': return (vsl_IX_arg(vd, opt, arg)); - case 'C': vd->regflags = REG_ICASE; return (1); + case 'C': vd->regflags = VRE_CASELESS; return (1); case 's': return (vsl_s_arg(vd, opt)); case 'k': return (vsl_k_arg(vd, opt)); default: Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.h 2009-10-20 07:11:22 UTC (rev 4333) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.h 2009-10-20 13:55:55 UTC (rev 4334) @@ -198,7 +198,7 @@ double vcc_DoubleVal(struct tokenlist *tl); /* vcc_string.c */ -char *vcc_regexp(struct tokenlist *tl, int sub); +char *vcc_regexp(struct tokenlist *tl); int vcc_StringVal(struct tokenlist *tl); void vcc_ExpectedStringval(struct tokenlist *tl); Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-10-20 07:11:22 UTC (rev 4333) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-10-20 13:55:55 UTC (rev 4334) @@ -276,7 +276,7 @@ vsb_cat(sb, "#define VRT_ACL_MAXADDR\t\t16\t/* max(IPv4, IPv6) */\n"); vsb_cat(sb, "\nvoid VRT_acl_log(const struct sess *, const char *ms"); vsb_cat(sb, "g);\n\n/* Regexp related */\nvoid VRT_re_init(void **,"); - vsb_cat(sb, " const char *, int sub);\nvoid VRT_re_fini(void *);\n"); + vsb_cat(sb, " const char *);\nvoid VRT_re_fini(void *);\n"); vsb_cat(sb, "int VRT_re_match(const char *, void *re);\n"); vsb_cat(sb, "const char *VRT_regsub(const struct sess *sp, int all,"); vsb_cat(sb, " const char *,\n void *, const char *);\n"); Modified: trunk/varnish-cache/lib/libvcl/vcc_parse.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_parse.c 2009-10-20 07:11:22 UTC (rev 4333) +++ trunk/varnish-cache/lib/libvcl/vcc_parse.c 2009-10-20 13:55:55 UTC (rev 4334) @@ -225,7 +225,7 @@ tl->t->tok == '~' ? "" : "!"); vcc_NextToken(tl); ExpectErr(tl, CSTR); - p = vcc_regexp(tl, 0); + p = vcc_regexp(tl); ERRCHK(tl); vcc_NextToken(tl); Fb(tl, 1, "%s, %s)\n", vp->rname, p); Modified: trunk/varnish-cache/lib/libvcl/vcc_string.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_string.c 2009-10-20 07:11:22 UTC (rev 4333) +++ trunk/varnish-cache/lib/libvcl/vcc_string.c 2009-10-20 13:55:55 UTC (rev 4334) @@ -34,7 +34,6 @@ #include #include -#include #include "vsb.h" @@ -43,30 +42,30 @@ #include "libvarnish.h" #include "vrt.h" +#include "vre.h" /*--------------------------------------------------------------------*/ char * -vcc_regexp(struct tokenlist *tl, int sub) +vcc_regexp(struct tokenlist *tl) { char buf[BUFSIZ], *p; - regex_t t; - int i; + vre_t *t; + const char *error; + int erroroffset; Expect(tl, CSTR); if (tl->err) return (NULL); memset(&t, 0, sizeof t); - i = regcomp(&t, tl->t->dec, REG_EXTENDED | (sub ? 0 : REG_NOSUB)); - if (i != 0) { - (void)regerror(i, &t, buf, sizeof buf); + t = VRE_compile(tl->t->dec, 0, &error, &erroroffset); + if (t == NULL) { vsb_printf(tl->sb, - "Regexp compilation error:\n\n%s\n\n", buf); + "Regexp compilation error:\n\n%s\n\n", error); vcc_ErrWhere(tl, tl->t); - regfree(&t); return (NULL); } - regfree(&t); + VRE_free(&t); sprintf(buf, "VGC_re_%u", tl->recnt++); p = TlAlloc(tl, strlen(buf) + 1); strcpy(p, buf); @@ -74,7 +73,7 @@ Fh(tl, 0, "static void *%s;\n", buf); Fi(tl, 0, "\tVRT_re_init(&%s, ",buf); EncToken(tl->fi, tl->t); - Fi(tl, 0, ", %d);\n", sub); + Fi(tl, 0, ");\n"); Ff(tl, 0, "\tVRT_re_fini(%s);\n", buf); return (p); } @@ -108,7 +107,7 @@ Expect(tl, CSTR); if (tl->err) return (0); - p = vcc_regexp(tl, 1); + p = vcc_regexp(tl); vcc_NextToken(tl); Fb(tl, 0, ", %s, ", p); From kristian at projects.linpro.no Wed Oct 21 08:24:24 2009 From: kristian at projects.linpro.no (kristian at projects.linpro.no) Date: Wed, 21 Oct 2009 10:24:24 +0200 (CEST) Subject: r4335 - trunk/varnish-cache/bin/varnishd Message-ID: <20091021082424.7C5411F72BB@projects.linpro.no> Author: kristian Date: 2009-10-21 10:24:24 +0200 (Wed, 21 Oct 2009) New Revision: 4335 Modified: trunk/varnish-cache/bin/varnishd/varnishd.1 Log: Document saintmode_threshold parameter Modified: trunk/varnish-cache/bin/varnishd/varnishd.1 =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.1 2009-10-20 13:55:55 UTC (rev 4334) +++ trunk/varnish-cache/bin/varnishd/varnishd.1 2009-10-21 08:24:24 UTC (rev 4335) @@ -555,6 +555,12 @@ The time to wait before dropping an idle pipe mode connection. .Pp The default is 60 seconds. +.It Va saintmode_threshold +The number of items allowed on the saintmode list for a backend before it +is considered sick. A value of 0 disables saintmode and any associated +overhead. +.Pp +The default is 10 objects .It Va sendfile_threshold The size threshold beyond which documents are sent to the client using .Xr sendfile 2 From kristian at projects.linpro.no Wed Oct 21 11:36:28 2009 From: kristian at projects.linpro.no (kristian at projects.linpro.no) Date: Wed, 21 Oct 2009 13:36:28 +0200 (CEST) Subject: r4336 - in trunk/varnish-cache: bin/varnishd bin/varnishtest/tests include lib/libvcl man Message-ID: <20091021113628.7AA721F72BB@projects.linpro.no> Author: kristian Date: 2009-10-21 13:36:28 +0200 (Wed, 21 Oct 2009) New Revision: 4336 Added: trunk/varnish-cache/bin/varnishtest/tests/c00030.vtc Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c trunk/varnish-cache/bin/varnishd/cache_backend.h trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c trunk/varnish-cache/include/vrt.h trunk/varnish-cache/lib/libvcl/vcc_backend.c trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c trunk/varnish-cache/man/vcl.7so Log: Add saintmode_threshold to backend definitions Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-10-21 08:24:24 UTC (rev 4335) +++ trunk/varnish-cache/bin/varnishd/cache_backend.c 2009-10-21 11:36:28 UTC (rev 4336) @@ -35,6 +35,7 @@ #include "svnid.h" SVNID("$Id$") +#include #include #include #include @@ -245,15 +246,23 @@ struct trouble *tr2; struct trouble *old = NULL; unsigned i = 0; - + unsigned int threshold; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(backend, BACKEND_MAGIC); if (!backend->healthy) return 0; + /* VRT/VCC sets threshold to UINT_MAX to mark that it's not + * specified by VCL (thus use param). + */ + if (backend->saintmode_threshold == UINT_MAX) + threshold = params->saintmode_threshold; + else + threshold = backend->saintmode_threshold; + /* Saintmode is disabled */ - if (params->saintmode_threshold == 0) + if (threshold == 0) return 1; /* No need to test if we don't have an object head to test against. @@ -280,7 +289,7 @@ * will disable the backend. Since 0 is disable, ++i * instead of i++ to allow this behavior. */ - if (++i >= params->saintmode_threshold) { + if (++i >=threshold) { Lck_Unlock(&backend->mtx); return 0; } Modified: trunk/varnish-cache/bin/varnishd/cache_backend.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend.h 2009-10-21 08:24:24 UTC (rev 4335) +++ trunk/varnish-cache/bin/varnishd/cache_backend.h 2009-10-21 11:36:28 UTC (rev 4336) @@ -137,6 +137,7 @@ struct vbp_target *probe; unsigned healthy; VTAILQ_HEAD(, trouble) troublelist; + unsigned saintmode_threshold; }; /* cache_backend.c */ Modified: trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c 2009-10-21 08:24:24 UTC (rev 4335) +++ trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c 2009-10-21 11:36:28 UTC (rev 4336) @@ -240,6 +240,7 @@ b->first_byte_timeout = vb->first_byte_timeout; b->between_bytes_timeout = vb->between_bytes_timeout; b->max_conn = vb->max_connections; + b->saintmode_threshold = vb->saintmode_threshold; /* * Copy over the sockaddrs Added: trunk/varnish-cache/bin/varnishtest/tests/c00030.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/c00030.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/c00030.vtc 2009-10-21 11:36:28 UTC (rev 4336) @@ -0,0 +1,108 @@ +# $Id$ + +test "Test that saintmode_threshold in VCL" + +server s1 { + rxreq + txresp + + rxreq + txresp + + rxreq + txresp + + rxreq + txresp + + rxreq + txresp -hdr "X-Saint: yes" + + rxreq + txresp -hdr "X-Saint: yes" + + rxreq + txresp -hdr "X-Saint: yes" + + rxreq + txresp -hdr "X-Saint: yes" +} -start + +varnish v1 -arg "-p saintmode_threshold=10" -vcl { + backend foo { + .host = "127.0.0.1"; + .port = "9080"; + .saintmode_threshold = 2; + } + + sub vcl_recv { + set req.backend = foo; + set req.grace = 1h; + } + + sub vcl_fetch { + if (beresp.http.X-Saint == "yes") { + set beresp.saintmode = 20s; + restart; + } + set beresp.grace = 1h; + set beresp.ttl = 1s; + } + sub vcl_deliver { + set resp.http.X-Restarts = req.restarts; + } + } -start + +client c1 { + txreq -url "/one" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "0" + + txreq -url "/two" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "0" + + txreq -url "/three" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "0" + + txreq -url "/four" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "0" + + delay 2 + + txreq -url "/one" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "1" + + txreq -url "/two" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "1" + + txreq -url "/three" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "0" + + txreq -url "/four" + rxresp + expect resp.status == 200 + expect resp.http.X-Saint != "yes" + expect resp.http.X-Restarts == "0" + +} -run + Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2009-10-21 08:24:24 UTC (rev 4335) +++ trunk/varnish-cache/include/vrt.h 2009-10-21 11:36:28 UTC (rev 4336) @@ -73,6 +73,7 @@ double first_byte_timeout; double between_bytes_timeout; unsigned max_connections; + unsigned saintmode_threshold; struct vrt_backend_probe probe; }; Modified: trunk/varnish-cache/lib/libvcl/vcc_backend.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_backend.c 2009-10-21 08:24:24 UTC (rev 4335) +++ trunk/varnish-cache/lib/libvcl/vcc_backend.c 2009-10-21 11:36:28 UTC (rev 4336) @@ -56,6 +56,7 @@ #include #include +#include #include #include #include @@ -476,6 +477,7 @@ struct token *t_host = NULL; struct token *t_port = NULL; struct token *t_hosthdr = NULL; + unsigned saint = UINT_MAX; const char *ep; struct fld_spec *fs; struct vsb *vsb; @@ -490,6 +492,7 @@ "?between_bytes_timeout", "?probe", "?max_connections", + "?saintmode_threshold", NULL); t_first = tl->t; @@ -573,6 +576,22 @@ ExpectErr(tl, ';'); vcc_NextToken(tl); Fb(tl, 0, "\t.max_connections = %u,\n", u); + } else if (vcc_IdIs(t_field, "saintmode_threshold")) { + u = vcc_UintVal(tl); + /* UINT_MAX == magic number to mark as unset, so + * not allowed here. + */ + if (u == UINT_MAX) { + vsb_printf(tl->sb, "Value outside allowed range: "); + vcc_ErrToken(tl, tl->t); + vsb_printf(tl->sb, " at\n"); + vcc_ErrWhere(tl, tl->t); + } + vcc_NextToken(tl); + ERRCHK(tl); + saint = u; + ExpectErr(tl, ';'); + vcc_NextToken(tl); } else if (vcc_IdIs(t_field, "probe")) { vcc_ParseProbe(tl); ERRCHK(tl); @@ -623,6 +642,8 @@ EncToken(tl->fb, t_host); Fb(tl, 0, ",\n"); + Fb(tl, 0, "\t.saintmode_threshold = %d,\n",saint); + /* Close the struct */ Fb(tl, 0, "};\n"); Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-10-21 08:24:24 UTC (rev 4335) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-10-21 11:36:28 UTC (rev 4336) @@ -251,6 +251,7 @@ vsb_cat(sb, "\tdouble\t\t\t\tfirst_byte_timeout;\n"); vsb_cat(sb, "\tdouble\t\t\t\tbetween_bytes_timeout;\n"); vsb_cat(sb, "\tunsigned\t\t\tmax_connections;\n"); + vsb_cat(sb, "\tunsigned\t\t\tsaintmode_threshold;\n"); vsb_cat(sb, "\tstruct vrt_backend_probe\tprobe;\n"); vsb_cat(sb, "};\n\n/*\n * A director with a predictable reply\n"); vsb_cat(sb, " */\n\nstruct vrt_dir_simple {\n"); Modified: trunk/varnish-cache/man/vcl.7so =================================================================== --- trunk/varnish-cache/man/vcl.7so 2009-10-21 08:24:24 UTC (rev 4335) +++ trunk/varnish-cache/man/vcl.7so 2009-10-21 11:36:28 UTC (rev 4336) @@ -120,6 +120,14 @@ .between_bytes_timeout = 2s; } .Ed +.Pp +To stop using a backend after number of items have been added to it's +saintmode list +.Fa .saintmode_threshold +can be set to the maximum list size. Setting a value of 0 disables +saintmode checking entirely for that backend. The value in the backend +declaration overrides the parameter. + .Ss Directors Directors choose from different backends based on health status and a per-director algorithm. From kristian at projects.linpro.no Wed Oct 21 12:35:23 2009 From: kristian at projects.linpro.no (kristian at projects.linpro.no) Date: Wed, 21 Oct 2009 14:35:23 +0200 (CEST) Subject: r4337 - trunk/varnish-cache/man Message-ID: <20091021123523.5776E1F7317@projects.linpro.no> Author: kristian Date: 2009-10-21 14:35:23 +0200 (Wed, 21 Oct 2009) New Revision: 4337 Modified: trunk/varnish-cache/man/vcl.7so Log: Minor language nitpick for saintmode_threshold Modified: trunk/varnish-cache/man/vcl.7so =================================================================== --- trunk/varnish-cache/man/vcl.7so 2009-10-21 11:36:28 UTC (rev 4336) +++ trunk/varnish-cache/man/vcl.7so 2009-10-21 12:35:23 UTC (rev 4337) @@ -121,8 +121,8 @@ } .Ed .Pp -To stop using a backend after number of items have been added to it's -saintmode list +To mark a backend as unhealthy after number of items have been added to +it's saintmode list .Fa .saintmode_threshold can be set to the maximum list size. Setting a value of 0 disables saintmode checking entirely for that backend. The value in the backend From tfheen at projects.linpro.no Fri Oct 23 07:58:41 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Fri, 23 Oct 2009 09:58:41 +0200 (CEST) Subject: r4338 - trunk/varnish-cache/bin/varnishd Message-ID: <20091023075841.DEC8538C5E@projects.linpro.no> Author: tfheen Date: 2009-10-23 09:58:41 +0200 (Fri, 23 Oct 2009) New Revision: 4338 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Send stderr to /dev/null when running nm Previously, running varnishd -V would sometimes say: nm: 'varnishd': No such file which is wrong. Silence this message Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2009-10-21 12:35:23 UTC (rev 4337) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2009-10-23 07:58:41 UTC (rev 4338) @@ -436,7 +436,7 @@ struct symbols *s; p = NULL; - asprintf(&p, "nm -an %s", a0); + asprintf(&p, "nm -an %s 2>/dev/null", a0); if (p == NULL) return; fi = popen(p, "r"); From tfheen at projects.linpro.no Fri Oct 23 09:18:41 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Fri, 23 Oct 2009 11:18:41 +0200 (CEST) Subject: r4339 - trunk/varnish-cache Message-ID: <20091023091841.99BA038C5E@projects.linpro.no> Author: tfheen Date: 2009-10-23 11:18:41 +0200 (Fri, 23 Oct 2009) New Revision: 4339 Modified: trunk/varnish-cache/configure.ac Log: Forbid PKG tokens in configure output Now that we need PCRE to build, we also do need pkg-config at autoreconf time, so make autoconf complain as early as possible to not confuse people. Modified: trunk/varnish-cache/configure.ac =================================================================== --- trunk/varnish-cache/configure.ac 2009-10-23 07:58:41 UTC (rev 4338) +++ trunk/varnish-cache/configure.ac 2009-10-23 09:18:41 UTC (rev 4339) @@ -75,6 +75,7 @@ AC_CHECK_LIBM AC_SUBST(LIBM) +m4_pattern_forbid([^_?PKG_[A-Z_]+$],[pkg.m4 missing, please install pkg-config]) PKG_PROG_PKG_CONFIG if test -n $PKG_CONFIG; then PKG_CHECK_MODULES([PCRE], [libpcre])