From phk at projects.linpro.no Tue Sep 1 10:39:49 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 1 Sep 2009 12:39:49 +0200 (CEST) Subject: r4218 - trunk/varnish-cache/bin/varnishd Message-ID: <20090901103949.9AFAB1F72C0@projects.linpro.no> Author: phk Date: 2009-09-01 12:39:49 +0200 (Tue, 01 Sep 2009) New Revision: 4218 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/stevedore.c trunk/varnish-cache/bin/varnishd/stevedore.h trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Give each -spersistent segment its own lru list, and prevent the expiry thread from moving objcores from one list to another. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-08-31 09:10:02 UTC (rev 4217) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-09-01 10:39:49 UTC (rev 4218) @@ -283,11 +283,12 @@ struct object *obj; struct objhead *objhead; double timer_when; - unsigned char flags; + unsigned flags; #define OC_F_ONLRU (1<<0) #define OC_F_BUSY (1<<1) #define OC_F_PASS (1<<2) #define OC_F_PERSISTENT (1<<3) +#define OC_F_LRUDONTMOVE (1<<4) unsigned timer_idx; VTAILQ_ENTRY(objcore) list; VLIST_ENTRY(objcore) lru_list; Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-08-31 09:10:02 UTC (rev 4217) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-09-01 10:39:49 UTC (rev 4218) @@ -163,21 +163,33 @@ int EXP_Touch(const struct object *o) { - int retval = 0; + int retval; struct objcore *oc; struct lru *lru; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); oc = o->objcore; if (oc == NULL) - return (retval); + return (0); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); /* We must have an objhead, otherwise we have no business on a LRU */ CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); + + /* + * For -spersistent we don't move objects on the lru list. Each + * segment has its own LRU list, and the order on it is not material + * for anything. The code below would move the objects to the + * LRU list of the currently open segment, which would prevent + * the cleaner from doing its job. + */ + if (oc->flags & OC_F_LRUDONTMOVE) + return (0); + if (o->objstore == NULL) /* XXX ?? */ - return (retval); + return (0); lru = STV_lru(o->objstore); CHECK_OBJ_NOTNULL(lru, LRU_MAGIC); + retval = 0; if (Lck_Trylock(&exp_mtx)) return (retval); if (oc->flags & OC_F_ONLRU) { /* XXX ?? */ Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-31 09:10:02 UTC (rev 4217) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-09-01 10:39:49 UTC (rev 4218) @@ -132,6 +132,7 @@ CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(sp->obj->objstore, STORAGE_MAGIC); CHECK_OBJ_NOTNULL(sp->obj->objstore->stevedore, STEVEDORE_MAGIC); + AN(ObjIsBusy(sp->obj)); if (sp->obj->objstore->stevedore->object != NULL) sp->obj->objstore->stevedore->object(sp); } @@ -372,7 +373,10 @@ fprintf(stderr, ">\n"); } -/**********************************************************************/ +/********************************************************************** + * Insert an object which magically appears out of nowhere or, more likely, + * comes off some persistent storage device. + */ struct objcore * HSH_Insert(const struct sess *sp) Modified: trunk/varnish-cache/bin/varnishd/stevedore.c =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.c 2009-08-31 09:10:02 UTC (rev 4217) +++ trunk/varnish-cache/bin/varnishd/stevedore.c 2009-09-01 10:39:49 UTC (rev 4218) @@ -43,7 +43,16 @@ static const struct stevedore * volatile stv_next; -static struct lru * +/********************************************************************* + * NB! Dirty trick alert: + * + * We use a captive objcore as tail senteniel for LRU lists, but to + * make sure it does not get into play by accident, we do _not_ + * initialize its magic with OBJCORE_MAGIC. + * + */ + +struct lru * LRU_Alloc(void) { struct lru *l; @@ -55,6 +64,8 @@ return (l); } +/*********************************************************************/ + struct storage * STV_alloc(struct sess *sp, size_t size) { Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2009-08-31 09:10:02 UTC (rev 4217) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2009-09-01 10:39:49 UTC (rev 4218) @@ -71,6 +71,7 @@ void STV_close(void); struct lru *STV_lru(const struct storage *st); +struct lru *LRU_Alloc(void); int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx); uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity, const char *ctx); Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-31 09:10:02 UTC (rev 4217) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-09-01 10:39:49 UTC (rev 4218) @@ -89,6 +89,7 @@ #define SMP_SEG_MAGIC 0x45c61895 struct smp_sc *sc; + struct lru *lru; VTAILQ_ENTRY(smp_seg) list; /* on smp_sc.smp_segments */ @@ -276,12 +277,11 @@ { int i; -#if 1 + /* XXX: round to pages */ i = msync(ctx->ss, ctx->ss->length + SHA256_LEN, MS_SYNC); - if (i) -fprintf(stderr, "SyncSign(%p %s) = %d %s\n", - ctx->ss, ctx->id, i, strerror(errno)); -#endif + if (i && 0) + fprintf(stderr, "SyncSign(%p %s) = %d %s\n", + ctx->ss, ctx->id, i, strerror(errno)); } /*-------------------------------------------------------------------- @@ -805,7 +805,6 @@ smp_load_seg(struct sess *sp, const struct smp_sc *sc, struct smp_seg *sg) { void *ptr; - uint64_t length; struct smp_segment *ss; struct smp_object *so; struct objcore *oc; @@ -820,7 +819,6 @@ if (smp_chk_sign(ctx)) return; ptr = SIGN_DATA(ctx); - length = ctx->ss->length; ss = ptr; so = (void*)(sc->ptr + ss->objlist); no = ss->nalloc; @@ -829,7 +827,7 @@ continue; HSH_Prealloc(sp); oc = sp->wrk->nobjcore; - oc->flags |= OC_F_PERSISTENT; + oc->flags |= OC_F_PERSISTENT | OC_F_LRUDONTMOVE; oc->flags &= ~OC_F_BUSY; oc->obj = (void*)so; oc->smp_seg = sg; @@ -837,7 +835,7 @@ memcpy(sp->wrk->nobjhead->digest, so->hash, SHA256_LEN); (void)HSH_Insert(sp); AZ(sp->wrk->nobjcore); - EXP_Inject(oc, sc->parent->lru, so->ttl); + EXP_Inject(oc, sg->lru, so->ttl); sg->nalloc++; } WRK_SumStat(sp->wrk); @@ -865,6 +863,7 @@ for(; length > 0; length -= sizeof *ss, ss ++) { ALLOC_OBJ(sg, SMP_SEG_MAGIC); AN(sg); + sg->lru = LRU_Alloc(); sg->offset = ss->offset; sg->length = ss->length; /* XXX: check that they are inside silo */ @@ -1081,6 +1080,7 @@ CHECK_OBJ_NOTNULL(sp->obj->objstore->stevedore, STEVEDORE_MAGIC); CAST_OBJ_NOTNULL(sc, sp->obj->objstore->priv, SMP_SC_MAGIC); + sp->obj->objcore->flags |= OC_F_LRUDONTMOVE; Lck_Lock(&sc->mtx); sg = sc->cur_seg; sc->objreserv += sizeof *so; From phk at projects.linpro.no Wed Sep 2 08:40:35 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 2 Sep 2009 10:40:35 +0200 (CEST) Subject: r4219 - in trunk/varnish-cache: bin/varnishtest include lib/libvarnish Message-ID: <20090902084035.1C12C1F72A5@projects.linpro.no> Author: phk Date: 2009-09-02 10:40:34 +0200 (Wed, 02 Sep 2009) New Revision: 4219 Modified: trunk/varnish-cache/bin/varnishtest/vtc.c trunk/varnish-cache/include/libvarnish.h trunk/varnish-cache/lib/libvarnish/argv.c Log: Expose the good string backslash implementation from argv.c and replace a half-baked one in vtc.c with it. Modified: trunk/varnish-cache/bin/varnishtest/vtc.c =================================================================== --- trunk/varnish-cache/bin/varnishtest/vtc.c 2009-09-01 10:39:49 UTC (rev 4218) +++ trunk/varnish-cache/bin/varnishtest/vtc.c 2009-09-02 08:40:34 UTC (rev 4219) @@ -133,19 +133,9 @@ for (; *p != '\0'; p++) { if (*p == '"') break; - - if (*p == '\\' && p[1] == 'n') { - *q++ = '\n'; - p++; - } else if (*p == '\\' && p[1] == 'r') { - *q++ = '\r'; - p++; - } else if (*p == '\\' && p[1] == '\\') { - *q++ = '\\'; - p++; - } else if (*p == '\\' && p[1] == '"') { - *q++ = '"'; - p++; + if (*p == '\\') { + p += BackSlash(p, q) - 1; + q++; } else { if (*p == '\n') fprintf(stderr, Modified: trunk/varnish-cache/include/libvarnish.h =================================================================== --- trunk/varnish-cache/include/libvarnish.h 2009-09-01 10:39:49 UTC (rev 4218) +++ trunk/varnish-cache/include/libvarnish.h 2009-09-02 08:40:34 UTC (rev 4219) @@ -42,6 +42,8 @@ /* from libvarnish/argv.c */ void FreeArgv(char **argv); char **ParseArgv(const char *s, int flag); +char *BackSlashDecode(const char *s, const char *e); +int BackSlash(const char *s, char *res); #define ARGV_COMMENT (1 << 0) #define ARGV_COMMA (1 << 1) Modified: trunk/varnish-cache/lib/libvarnish/argv.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/argv.c 2009-09-01 10:39:49 UTC (rev 4218) +++ trunk/varnish-cache/lib/libvarnish/argv.c 2009-09-02 08:40:34 UTC (rev 4219) @@ -44,11 +44,12 @@ #include #include #include +#include #include #include "libvarnish.h" -static int +int BackSlash(const char *s, char *res) { int r; @@ -104,13 +105,16 @@ return (r); } -static char * +char * BackSlashDecode(const char *s, const char *e) { const char *q; char *p, *r; int i; + if (e == NULL) + e = strchr(s, '\0'); + assert(e != NULL); p = calloc((e - s) + 1, 1); if (p == NULL) return (p); From phk at projects.linpro.no Wed Sep 2 09:07:47 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 2 Sep 2009 11:07:47 +0200 (CEST) Subject: r4220 - trunk/varnish-cache/bin/varnishtest/tests Message-ID: <20090902090747.2E28D1F72B5@projects.linpro.no> Author: phk Date: 2009-09-02 11:07:46 +0200 (Wed, 02 Sep 2009) New Revision: 4220 Modified: trunk/varnish-cache/bin/varnishtest/tests/c00001.vtc Log: Add a newline to feed the new backslash handler Modified: trunk/varnish-cache/bin/varnishtest/tests/c00001.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/c00001.vtc 2009-09-02 08:40:34 UTC (rev 4219) +++ trunk/varnish-cache/bin/varnishtest/tests/c00001.vtc 2009-09-02 09:07:46 UTC (rev 4220) @@ -36,7 +36,7 @@ expect resp.http.snafu2 == "_frap_" expect resp.http.snafu3 == "_f\\rap_" expect resp.http.snafu4 == "_f&rap_" - expect resp.http.snafu5 == "_barffra\p_" - # NB: have to escape the \\ in the next line + # NB: have to escape the \\ in the next two lines + expect resp.http.snafu5 == "_barffra\\p_" expect resp.http.snafu6 == "_f&rap\\_" } -run From phk at projects.linpro.no Wed Sep 2 09:08:24 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 2 Sep 2009 11:08:24 +0200 (CEST) Subject: r4221 - in trunk/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20090902090824.876301F72C9@projects.linpro.no> Author: phk Date: 2009-09-02 11:08:24 +0200 (Wed, 02 Sep 2009) New Revision: 4221 Added: trunk/varnish-cache/bin/varnishtest/tests/r00549.vtc Modified: trunk/varnish-cache/bin/varnishd/cache_http.c Log: Be much more paranoid about control-characters in backend responses. Fixes #549 Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2009-09-02 09:07:46 UTC (rev 4220) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2009-09-02 09:08:24 UTC (rev 4221) @@ -390,7 +390,7 @@ http_splitline(struct worker *w, int fd, struct http *hp, const struct http_conn *htc, int h1, int h2, int h3) { - char *p; + char *p, *q; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); @@ -403,40 +403,47 @@ continue; /* First field cannot contain SP, CRLF or CTL */ - hp->hd[h1].b = p; - for (; !vct_issp(*p); p++) + q = p; + for (; !vct_issp(*p); p++) { if (vct_isctl(*p)) return (400); + } + hp->hd[h1].b = q; hp->hd[h1].e = p; /* Skip SP */ - for (; vct_issp(*p); p++) - ; + for (; vct_issp(*p); p++) { + if (vct_isctl(*p)) + return (400); + } - /* Second field cannot contain LWS */ - hp->hd[h2].b = p; - for (; !vct_islws(*p); p++) - ; + /* Second field cannot contain LWS or CTL */ + q = p; + for (; !vct_islws(*p); p++) { + if (vct_isctl(*p)) + return (400); + } + hp->hd[h2].b = q; hp->hd[h2].e = p; if (!Tlen(hp->hd[h2])) return (400); /* Skip SP */ - for (; vct_issp(*p); p++) - ; + for (; vct_issp(*p); p++) { + if (vct_isctl(*p)) + return (400); + } /* Third field is optional and cannot contain CTL */ + q = p; if (!vct_iscrlf(*p)) { - hp->hd[h3].b = p; for (; !vct_iscrlf(*p); p++) if (vct_isctl(*p)) return (400); - hp->hd[h3].e = p; - } else { - hp->hd[h3].b = p; - hp->hd[h3].e = p; } + hp->hd[h3].b = q; + hp->hd[h3].e = p; /* Skip CRLF */ p += vct_skipcrlf(p); Added: trunk/varnish-cache/bin/varnishtest/tests/r00549.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/r00549.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/r00549.vtc 2009-09-02 09:08:24 UTC (rev 4221) @@ -0,0 +1,15 @@ +# $Id$ + +# Regression test for bad backend reply with ctrl char. + +server s1 { + rxreq + send "HTTP/1.1 200 OK\013\r\n\r\nTest" +} -start + +varnish v1 -vcl+backend {} -start + +client c1 { + txreq + rxresp +} -run From phk at projects.linpro.no Wed Sep 2 15:43:34 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 2 Sep 2009 17:43:34 +0200 (CEST) Subject: r4222 - trunk/varnish-cache/bin/varnishd Message-ID: <20090902154334.0BF5A1F72CC@projects.linpro.no> Author: phk Date: 2009-09-02 17:43:33 +0200 (Wed, 02 Sep 2009) New Revision: 4222 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/hash_slinger.h Log: Try to simplify the variable structure around the decision to populate a busy cache entry or to ditch it. Ditch it as soon as we can, so we don't hold other clients on a busy objcore longer than necessary. Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-02 09:08:24 UTC (rev 4221) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-02 15:43:33 UTC (rev 4222) @@ -333,7 +333,8 @@ w = sp->wrk; if (sp->obj == NULL) { HSH_Prealloc(sp); - sp->obj = HSH_NewObject(sp, 1); + sp->wrk->cacheable = 0; + sp->obj = HSH_NewObject(sp); sp->obj->xid = sp->xid; sp->obj->entered = sp->t_req; } else { @@ -411,7 +412,7 @@ static int cnt_fetch(struct sess *sp) { - int i, transient; + int i; struct http *hp, *hp2; char *b; unsigned handling; @@ -436,10 +437,12 @@ *sp->wrk->beresp1 = *sp->wrk->beresp; if (i) { - if (sp->objhead) { + if (sp->objcore != NULL) { CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC); CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC); HSH_DerefObjCore(sp); + AZ(sp->objhead); + AZ(sp->objcore); } AZ(sp->obj); sp->wrk->bereq = NULL; @@ -475,7 +478,7 @@ sp->wrk->age = 0; sp->wrk->ttl = RFC2616_Ttl(sp); - if (sp->wrk->ttl == 0.) + if (sp->objcore == NULL) sp->wrk->cacheable = 0; sp->wrk->do_esi = 0; @@ -490,23 +493,28 @@ */ handling = sp->handling; - if (sp->objhead == NULL) + if (sp->objcore == NULL) { /* This is a pass from vcl_recv */ - transient = 1; - else if (sp->handling == VCL_RET_PASS) - /* A pass from vcl_fetch is not transient */ - transient = 0; - else if (sp->handling == VCL_RET_DELIVER) - /* Regular object */ - transient = 0; - else - transient = 1; + AZ(sp->objhead); + sp->wrk->cacheable = 0; + } else if (!sp->wrk->cacheable) { + if (sp->objhead != NULL) + HSH_DerefObjCore(sp); + } + if (sp->wrk->cacheable) { + CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC); + CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC); + } else { + AZ(sp->objhead); + AZ(sp->objcore); + } + /* * XXX: If we have a Length: header, we should allocate the body * XXX: also. */ - sp->obj = HSH_NewObject(sp, transient); + sp->obj = HSH_NewObject(sp); if (sp->objhead != NULL) { CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC); @@ -558,7 +566,7 @@ return (0); } - if (!transient) + if (sp->wrk->cacheable) HSH_Object(sp); if (sp->wrk->do_esi) @@ -594,7 +602,7 @@ } sp->obj->cacheable = 1; - if (!transient) { + if (sp->wrk->cacheable) { VRY_Create(sp); EXP_Insert(sp->obj); AN(sp->obj->ban); Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-09-02 09:08:24 UTC (rev 4221) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-09-02 15:43:33 UTC (rev 4222) @@ -82,13 +82,13 @@ } struct object * -HSH_NewObject(struct sess *sp, int transient) +HSH_NewObject(struct sess *sp) { struct object *o; struct storage *st; void *p; - if (transient) { + if (!sp->wrk->cacheable) { p = malloc(sizeof *o + params->obj_workspace); XXXAN(p); o = p; Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-09-02 09:08:24 UTC (rev 4221) +++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-09-02 15:43:33 UTC (rev 4222) @@ -50,7 +50,7 @@ }; /* cache_hash.c */ -struct object *HSH_NewObject(struct sess *sp, int transient); +struct object *HSH_NewObject(struct sess *sp); void HSH_Object(const struct sess *sp); void HSH_Prealloc(const struct sess *sp); void HSH_Cleanup(struct worker *w); From phk at projects.linpro.no Wed Sep 2 16:43:19 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 2 Sep 2009 18:43:19 +0200 (CEST) Subject: r4223 - trunk/varnish-cache/bin/varnishd Message-ID: <20090902164319.0AE331F7333@projects.linpro.no> Author: phk Date: 2009-09-02 18:43:18 +0200 (Wed, 02 Sep 2009) New Revision: 4223 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: A clarifying comment. Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-02 15:43:33 UTC (rev 4222) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-02 16:43:18 UTC (rev 4223) @@ -502,6 +502,12 @@ HSH_DerefObjCore(sp); } + /* + * At this point we are either committed to flesh out the busy + * object we have in the hash or we have let go of it if we ever + * had one. + */ + if (sp->wrk->cacheable) { CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC); CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC); From phk at projects.linpro.no Wed Sep 2 18:05:15 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 2 Sep 2009 20:05:15 +0200 (CEST) Subject: r4224 - trunk/varnish-cache/bin/varnishd Message-ID: <20090902180515.BE9D81F7338@projects.linpro.no> Author: phk Date: 2009-09-02 20:05:15 +0200 (Wed, 02 Sep 2009) New Revision: 4224 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/flint.lnt trunk/varnish-cache/bin/varnishd/hash_slinger.h trunk/varnish-cache/bin/varnishd/mgt_param.c Log: Originally, we did not have the bereq.* and beresp.* variable sets, and instead (ab)used the obj.* set. This allowed, amongst other things, the headers of cached objects to be munged by subsequent cache-hits. This was just as bad an idea as it sounds. The downside was, that we had to reserve object workspace for the potentially created headers, and this really bites, memory wise, if your content is many small objects. This commit allows, and sets obj_workspace to zero, allocating only as much space as is necessary for the actual headers after vcl_fetch() has returned. The downside is, without extra workspace, you can no longer change the content of obj.* string variables after the vcl_fetch() which instantiated the object. For now we still allow the obj_workspace to be set to reserve such space, but in all likelyhood, this ability will disappear in the future. Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-09-02 16:43:18 UTC (rev 4223) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-09-02 18:05:15 UTC (rev 4224) @@ -154,6 +154,7 @@ txt hd[HTTP_HDR_MAX]; unsigned char hdf[HTTP_HDR_MAX]; #define HDF_FILTER (1 << 0) /* Filtered by Connection */ +#define HDF_COPY (1 << 1) /* Copy this field */ unsigned nhd; }; @@ -512,6 +513,7 @@ /* cache_http.c */ const char *http_StatusMessage(unsigned); +unsigned http_EstimateWS(struct http *fm, unsigned how); void HTTP_Init(void); void http_ClrHeader(struct http *to); unsigned http_Write(struct worker *w, const struct http *hp, int resp); Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-02 16:43:18 UTC (rev 4223) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-02 18:05:15 UTC (rev 4224) @@ -334,7 +334,7 @@ if (sp->obj == NULL) { HSH_Prealloc(sp); sp->wrk->cacheable = 0; - sp->obj = HSH_NewObject(sp); + sp->obj = HSH_NewObject(sp, 0); sp->obj->xid = sp->xid; sp->obj->entered = sp->t_req; } else { @@ -415,7 +415,7 @@ int i; struct http *hp, *hp2; char *b; - unsigned handling; + unsigned handling, l; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC); @@ -516,11 +516,16 @@ AZ(sp->objcore); } + l = http_EstimateWS(sp->wrk->beresp, HTTPH_A_INS); + + /* Space for producing a Content-Length: header */ + l += 30; + /* * XXX: If we have a Length: header, we should allocate the body * XXX: also. */ - sp->obj = HSH_NewObject(sp); + sp->obj = HSH_NewObject(sp, l); if (sp->objhead != NULL) { CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC); Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-09-02 16:43:18 UTC (rev 4223) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-09-02 18:05:15 UTC (rev 4224) @@ -82,22 +82,27 @@ } struct object * -HSH_NewObject(struct sess *sp) +HSH_NewObject(struct sess *sp, unsigned l) { struct object *o; struct storage *st; void *p; + if (l == 0) + l = 1024; + if (params->obj_workspace > 0 && params->obj_workspace > l) + l = params->obj_workspace; + if (!sp->wrk->cacheable) { - p = malloc(sizeof *o + params->obj_workspace); + p = malloc(sizeof *o + l); XXXAN(p); o = p; p = o + 1; memset(o, 0, sizeof *o); o->magic = OBJECT_MAGIC; - WS_Init(o->ws_o, "obj", p, params->obj_workspace); + WS_Init(o->ws_o, "obj", p, l); } else { - st = STV_alloc(sp, params->obj_workspace); + st = STV_alloc(sp, sizeof *o + l); XXXAN(st); assert(st->space > sizeof *o); o = (void *)st->ptr; /* XXX: align ? */ Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2009-09-02 16:43:18 UTC (rev 4223) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2009-09-02 18:05:15 UTC (rev 4224) @@ -624,6 +624,34 @@ } } +/*-------------------------------------------------------------------- + * Estimate how much workspace we need to Filter this header according + * to 'how'. + */ + +unsigned +http_EstimateWS(struct http *fm, unsigned how) +{ + unsigned u, l; + + l = 0; + CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC); + for (u = 0; u < fm->nhd; u++) { + if (fm->hd[u].b == NULL) + continue; + if (fm->hdf[u] & HDF_FILTER) + continue; +#define HTTPH(a, b, c, d, e, f, g) \ + if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \ + continue; +#include "http_headers.h" +#undef HTTPH + l += Tlen(fm->hd[u]) + 1; + // fm->hdf[u] |= HDF_COPY; + } + return (l); +} + /*--------------------------------------------------------------------*/ void Modified: trunk/varnish-cache/bin/varnishd/flint.lnt =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.lnt 2009-09-02 16:43:18 UTC (rev 4223) +++ trunk/varnish-cache/bin/varnishd/flint.lnt 2009-09-02 18:05:15 UTC (rev 4224) @@ -131,7 +131,9 @@ // cache_center.c -efunc(525, CNT_Session) // Negative indentation from line -efunc(525, http_FilterFields) // Negative indentation from line +-efunc(525, http_EstimateWS) // Negative indentation from line -efunc(539, http_FilterFields) // Positive indentation from line +-efunc(539, http_EstimateWS) // Positive indentation from line // cache_vcl.c -efunc(525, vcl_handlingname) // Negative indentation from line Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-09-02 16:43:18 UTC (rev 4223) +++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-09-02 18:05:15 UTC (rev 4224) @@ -50,7 +50,7 @@ }; /* cache_hash.c */ -struct object *HSH_NewObject(struct sess *sp); +struct object *HSH_NewObject(struct sess *sp, unsigned len); void HSH_Object(const struct sess *sp); void HSH_Prealloc(const struct sess *sp); void HSH_Cleanup(struct worker *w); Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-09-02 16:43:18 UTC (rev 4223) +++ trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-09-02 18:05:15 UTC (rev 4224) @@ -492,14 +492,14 @@ "Minimum is 1024 bytes.", DELAYED_EFFECT, "16384", "bytes" }, - { "obj_workspace", tweak_uint, &master.obj_workspace, 1024, UINT_MAX, + { "obj_workspace", tweak_uint, &master.obj_workspace, 0, UINT_MAX, "Bytes of HTTP protocol workspace allocated for objects. " "This space must be big enough for the entire HTTP protocol " "header and any edits done to it in the VCL code while it " "is cached.\n" "Minimum is 1024 bytes.", DELAYED_EFFECT, - "8192", "bytes" }, + "0", "bytes" }, { "shm_workspace", tweak_uint, &master.shm_workspace, 4096, UINT_MAX, "Bytes of shmlog workspace allocated for worker threads. " "If too big, it wastes some ram, if too small it causes " From phk at projects.linpro.no Wed Sep 2 20:09:29 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 2 Sep 2009 22:09:29 +0200 (CEST) Subject: r4225 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20090902200929.B41611F72C6@projects.linpro.no> Author: phk Date: 2009-09-02 22:09:29 +0200 (Wed, 02 Sep 2009) New Revision: 4225 Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/include/stat_field.h Log: Add a counter for objhead allocations on the off-chance that they are what is leaking. Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-09-02 18:05:15 UTC (rev 4224) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-09-02 20:09:29 UTC (rev 4225) @@ -158,6 +158,7 @@ ALLOC_OBJ(oc, OBJCORE_MAGIC); XXXAN(oc); w->nobjcore = oc; + w->stats.n_objectcore++; oc->flags |= OC_F_BUSY; } CHECK_OBJ_NOTNULL(w->nobjcore, OBJCORE_MAGIC); @@ -664,6 +665,7 @@ Lck_Unlock(&oh->mtx); assert(oh->refcnt > 0); FREE_OBJ(oc); + sp->wrk->stats.n_objectcore--; if (hash->deref(oh)) return; HSH_DeleteObjHead(sp->wrk, oh); @@ -766,6 +768,7 @@ } AN(oh); FREE_OBJ(oc); + w->stats.n_objectcore--; /* Drop our ref on the objhead */ assert(oh->refcnt > 0); if (hash->deref(oh)) Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2009-09-02 18:05:15 UTC (rev 4224) +++ trunk/varnish-cache/include/stat_field.h 2009-09-02 20:09:29 UTC (rev 4225) @@ -52,6 +52,7 @@ MAC_STAT(n_sess_mem, uint64_t, 0, 'i', "N struct sess_mem") MAC_STAT(n_sess, uint64_t, 0, 'i', "N struct sess") MAC_STAT(n_object, uint64_t, 1, 'i', "N struct object") +MAC_STAT(n_objectcore, uint64_t, 1, 'i', "N struct objectcore") MAC_STAT(n_objecthead, uint64_t, 1, 'i', "N struct objecthead") MAC_STAT(n_smf, uint64_t, 0, 'i', "N struct smf") MAC_STAT(n_smf_frag, uint64_t, 0, 'i', "N small free smf") From phk at projects.linpro.no Thu Sep 3 10:48:16 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 3 Sep 2009 12:48:16 +0200 (CEST) Subject: r4226 - in trunk/varnish-cache: bin/varnishd bin/varnishtest/tests include Message-ID: <20090903104816.406D21F72A5@projects.linpro.no> Author: phk Date: 2009-09-03 12:48:15 +0200 (Thu, 03 Sep 2009) New Revision: 4226 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/storage_persistent.c trunk/varnish-cache/bin/varnishtest/tests/p00004.vtc trunk/varnish-cache/include/stat_field.h Log: Account for unresurrected persistent objects in their own stats counter to fix the discrepancy in the object/objcore count they would otherwise represent. XXX: Not quite sure if they should also be counted as a subset of n_object to give people an easy way to access "total objects available in cache". Will decide this later. Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-02 20:09:29 UTC (rev 4225) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-03 10:48:15 UTC (rev 4226) @@ -504,7 +504,7 @@ /* * At this point we are either committed to flesh out the busy - * object we have in the hash or we have let go of it if we ever + * object we have in the hash or we have let go of it, if we ever * had one. */ Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-09-02 20:09:29 UTC (rev 4225) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-09-03 10:48:15 UTC (rev 4226) @@ -306,6 +306,7 @@ sp->objhead = oc->objhead; sp->objcore = oc; HSH_DerefObjCore(sp); + sp->wrk->stats.n_vampireobject--; } } NEEDLESS_RETURN(NULL); Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-09-02 20:09:29 UTC (rev 4225) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-09-03 10:48:15 UTC (rev 4226) @@ -183,6 +183,7 @@ if (w->nobjcore != NULL) { FREE_OBJ(w->nobjcore); + w->stats.n_objectcore--; w->nobjcore = NULL; } if (w->nobjhead != NULL) { @@ -419,7 +420,7 @@ /* NB: do not deref objhead the new object inherits our reference */ oc->objhead = oh; Lck_Unlock(&oh->mtx); - sp->wrk->stats.n_object++; + sp->wrk->stats.n_vampireobject++; return (oc); } @@ -661,7 +662,6 @@ Lck_Lock(&oh->mtx); VTAILQ_REMOVE(&oh->objcs, oc, list); - sp->wrk->stats.n_object--; Lck_Unlock(&oh->mtx); assert(oh->refcnt > 0); FREE_OBJ(oc); Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-09-02 20:09:29 UTC (rev 4225) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-09-03 10:48:15 UTC (rev 4226) @@ -635,6 +635,8 @@ oc->obj->ban = oc->ban; sg->nfixed++; + sp->wrk->stats.n_object++; + sp->wrk->stats.n_vampireobject--; } /*-------------------------------------------------------------------- Modified: trunk/varnish-cache/bin/varnishtest/tests/p00004.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/p00004.vtc 2009-09-02 20:09:29 UTC (rev 4225) +++ trunk/varnish-cache/bin/varnishtest/tests/p00004.vtc 2009-09-03 10:48:15 UTC (rev 4226) @@ -40,7 +40,8 @@ varnish v1 -start varnish v1 -cliok "debug.xid 2000" -varnish v1 -expect n_object == 2 +varnish v1 -expect n_vampireobject == 2 +varnish v1 -expect n_object == 0 client c1 { txreq -url "/foo" @@ -50,7 +51,8 @@ expect resp.http.foo == "foo" } -run -varnish v1 -expect n_object == 2 +varnish v1 -expect n_vampireobject == 1 +varnish v1 -expect n_object == 1 client c1 { txreq -url "/bar" @@ -61,3 +63,4 @@ } -run varnish v1 -expect n_object == 2 +varnish v1 -expect n_vampireobject == 0 Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2009-09-02 20:09:29 UTC (rev 4225) +++ trunk/varnish-cache/include/stat_field.h 2009-09-03 10:48:15 UTC (rev 4226) @@ -52,6 +52,7 @@ MAC_STAT(n_sess_mem, uint64_t, 0, 'i', "N struct sess_mem") MAC_STAT(n_sess, uint64_t, 0, 'i', "N struct sess") MAC_STAT(n_object, uint64_t, 1, 'i', "N struct object") +MAC_STAT(n_vampireobject, uint64_t, 1, 'i', "N unresurrected objects") MAC_STAT(n_objectcore, uint64_t, 1, 'i', "N struct objectcore") MAC_STAT(n_objecthead, uint64_t, 1, 'i', "N struct objecthead") MAC_STAT(n_smf, uint64_t, 0, 'i', "N struct smf") From phk at projects.linpro.no Fri Sep 4 09:07:27 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 4 Sep 2009 11:07:27 +0200 (CEST) Subject: r4227 - trunk/varnish-cache/bin/varnishd Message-ID: <20090904090727.C46F21F72DA@projects.linpro.no> Author: phk Date: 2009-09-04 11:07:27 +0200 (Fri, 04 Sep 2009) New Revision: 4227 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/hash_slinger.h trunk/varnish-cache/bin/varnishd/stevedore.c trunk/varnish-cache/bin/varnishd/stevedore.h Log: Rename and move HSH_NewObject() to STV_NewObject(), this is a more appropriate location for this code. Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-03 10:48:15 UTC (rev 4226) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-04 09:07:27 UTC (rev 4227) @@ -77,6 +77,7 @@ #include "cli_priv.h" #include "cache.h" #include "hash_slinger.h" +#include "stevedore.h" static unsigned xids; @@ -334,7 +335,7 @@ if (sp->obj == NULL) { HSH_Prealloc(sp); sp->wrk->cacheable = 0; - sp->obj = HSH_NewObject(sp, 0); + sp->obj = STV_NewObject(sp, 0); sp->obj->xid = sp->xid; sp->obj->entered = sp->t_req; } else { @@ -525,7 +526,7 @@ * XXX: If we have a Length: header, we should allocate the body * XXX: also. */ - sp->obj = HSH_NewObject(sp, l); + sp->obj = STV_NewObject(sp, l); if (sp->objhead != NULL) { CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC); Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-09-03 10:48:15 UTC (rev 4226) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-09-04 09:07:27 UTC (rev 4227) @@ -81,51 +81,6 @@ return (g); } -struct object * -HSH_NewObject(struct sess *sp, unsigned l) -{ - struct object *o; - struct storage *st; - void *p; - - if (l == 0) - l = 1024; - if (params->obj_workspace > 0 && params->obj_workspace > l) - l = params->obj_workspace; - - if (!sp->wrk->cacheable) { - p = malloc(sizeof *o + l); - XXXAN(p); - o = p; - p = o + 1; - memset(o, 0, sizeof *o); - o->magic = OBJECT_MAGIC; - WS_Init(o->ws_o, "obj", p, l); - } else { - st = STV_alloc(sp, sizeof *o + l); - XXXAN(st); - assert(st->space > sizeof *o); - o = (void *)st->ptr; /* XXX: align ? */ - st->len = sizeof *o; - memset(o, 0, sizeof *o); - o->magic = OBJECT_MAGIC; - o->objstore = st; - WS_Init(o->ws_o, "obj", - st->ptr + st->len, st->space - st->len); - st->len = st->space; - } - WS_Assert(o->ws_o); - http_Setup(o->http, o->ws_o); - o->http->magic = HTTP_MAGIC; - o->refcnt = 1; - o->grace = NAN; - o->entered = NAN; - VTAILQ_INIT(&o->store); - VTAILQ_INIT(&o->esibits); - sp->wrk->stats.n_object++; - return (o); -} - /* * XXX: this should vector through stevedore.c instead of calling the * XXX: member function directly. Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h =================================================================== --- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-09-03 10:48:15 UTC (rev 4226) +++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-09-04 09:07:27 UTC (rev 4227) @@ -50,7 +50,6 @@ }; /* cache_hash.c */ -struct object *HSH_NewObject(struct sess *sp, unsigned len); void HSH_Object(const struct sess *sp); void HSH_Prealloc(const struct sess *sp); void HSH_Cleanup(struct worker *w); Modified: trunk/varnish-cache/bin/varnishd/stevedore.c =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.c 2009-09-03 10:48:15 UTC (rev 4226) +++ trunk/varnish-cache/bin/varnishd/stevedore.c 2009-09-04 09:07:27 UTC (rev 4227) @@ -34,6 +34,7 @@ #include #include #include +#include #include "cache.h" #include "stevedore.h" @@ -66,6 +67,53 @@ /*********************************************************************/ +struct object * +STV_NewObject(struct sess *sp, unsigned l) +{ + struct object *o; + struct storage *st; + void *p; + + if (l == 0) + l = 1024; + if (params->obj_workspace > 0 && params->obj_workspace > l) + l = params->obj_workspace; + + if (!sp->wrk->cacheable) { + p = malloc(sizeof *o + l); + XXXAN(p); + o = p; + p = o + 1; + memset(o, 0, sizeof *o); + o->magic = OBJECT_MAGIC; + WS_Init(o->ws_o, "obj", p, l); + } else { + st = STV_alloc(sp, sizeof *o + l); + XXXAN(st); + assert(st->space > sizeof *o); + o = (void *)st->ptr; /* XXX: align ? */ + st->len = sizeof *o; + memset(o, 0, sizeof *o); + o->magic = OBJECT_MAGIC; + o->objstore = st; + WS_Init(o->ws_o, "obj", + st->ptr + st->len, st->space - st->len); + st->len = st->space; + } + WS_Assert(o->ws_o); + http_Setup(o->http, o->ws_o); + o->http->magic = HTTP_MAGIC; + o->refcnt = 1; + o->grace = NAN; + o->entered = NAN; + VTAILQ_INIT(&o->store); + VTAILQ_INIT(&o->esibits); + sp->wrk->stats.n_object++; + return (o); +} + +/*********************************************************************/ + struct storage * STV_alloc(struct sess *sp, size_t size) { Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2009-09-03 10:48:15 UTC (rev 4226) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2009-09-04 09:07:27 UTC (rev 4227) @@ -63,6 +63,7 @@ VTAILQ_ENTRY(stevedore) list; }; +struct object *STV_NewObject(struct sess *sp, unsigned len); struct storage *STV_alloc(struct sess *sp, size_t size); void STV_trim(struct storage *st, size_t size); void STV_free(struct storage *st); From phk at projects.linpro.no Fri Sep 4 10:08:24 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 4 Sep 2009 12:08:24 +0200 (CEST) Subject: r4228 - trunk/varnish-cache/bin/varnishd Message-ID: <20090904100824.444B61F7236@projects.linpro.no> Author: phk Date: 2009-09-04 12:08:23 +0200 (Fri, 04 Sep 2009) New Revision: 4228 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/stevedore.c trunk/varnish-cache/bin/varnishd/stevedore.h Log: Polish the object/storage allocation code a bit, in preparation for some significant changes to the policy. Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-04 09:07:27 UTC (rev 4227) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-04 10:08:23 UTC (rev 4228) @@ -335,7 +335,7 @@ if (sp->obj == NULL) { HSH_Prealloc(sp); sp->wrk->cacheable = 0; - sp->obj = STV_NewObject(sp, 0); + sp->obj = STV_NewObject(sp, 0, 0); sp->obj->xid = sp->xid; sp->obj->entered = sp->t_req; } else { @@ -526,8 +526,9 @@ * XXX: If we have a Length: header, we should allocate the body * XXX: also. */ - sp->obj = STV_NewObject(sp, l); + sp->obj = STV_NewObject(sp, l, sp->wrk->ttl); + if (sp->objhead != NULL) { CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC); CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC); Modified: trunk/varnish-cache/bin/varnishd/stevedore.c =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.c 2009-09-04 09:07:27 UTC (rev 4227) +++ trunk/varnish-cache/bin/varnishd/stevedore.c 2009-09-04 10:08:23 UTC (rev 4228) @@ -65,42 +65,60 @@ return (l); } +/********************************************************************* + * XXX: trust pointer writes to be atomic + */ + +static struct stevedore * +stv_pick_stevedore(void) +{ + struct stevedore *stv; + + /* pick a stevedore and bump the head along */ + stv = VTAILQ_NEXT(stv_next, list); + if (stv == NULL) + stv = VTAILQ_FIRST(&stevedores); + AN(stv); + AN(stv->name); + stv_next = stv; + return (stv); +} + + /*********************************************************************/ struct object * -STV_NewObject(struct sess *sp, unsigned l) +STV_NewObject(struct sess *sp, unsigned l, double ttl) { struct object *o; - struct storage *st; - void *p; + struct storage *st = NULL; + (void)ttl; if (l == 0) l = 1024; if (params->obj_workspace > 0 && params->obj_workspace > l) l = params->obj_workspace; if (!sp->wrk->cacheable) { - p = malloc(sizeof *o + l); - XXXAN(p); - o = p; - p = o + 1; - memset(o, 0, sizeof *o); - o->magic = OBJECT_MAGIC; - WS_Init(o->ws_o, "obj", p, l); + o = malloc(sizeof *o + l); + XXXAN(o); } else { st = STV_alloc(sp, sizeof *o + l); XXXAN(st); - assert(st->space > sizeof *o); + xxxassert(st->space >= (sizeof *o + l)); + + st->len = st->space; + l = st->space - sizeof *o; + o = (void *)st->ptr; /* XXX: align ? */ - st->len = sizeof *o; - memset(o, 0, sizeof *o); - o->magic = OBJECT_MAGIC; - o->objstore = st; - WS_Init(o->ws_o, "obj", - st->ptr + st->len, st->space - st->len); - st->len = st->space; } + memset(o, 0, sizeof *o); + o->objstore = st; + o->magic = OBJECT_MAGIC; + + WS_Init(o->ws_o, "obj", (o + 1), l); WS_Assert(o->ws_o); + http_Setup(o->http, o->ws_o); o->http->magic = HTTP_MAGIC; o->refcnt = 1; @@ -135,13 +153,7 @@ for (;;) { if (stv == NULL) { - /* pick a stevedore and bump the head along */ - stv = VTAILQ_NEXT(stv_next, list); - if (stv == NULL) - stv = VTAILQ_FIRST(&stevedores); - AN(stv); - AN(stv->name); - stv_next = stv; + stv = stv_pick_stevedore(); fail = 0; } Modified: trunk/varnish-cache/bin/varnishd/stevedore.h =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.h 2009-09-04 09:07:27 UTC (rev 4227) +++ trunk/varnish-cache/bin/varnishd/stevedore.h 2009-09-04 10:08:23 UTC (rev 4228) @@ -37,6 +37,7 @@ typedef void storage_init_f(struct stevedore *, int ac, char * const *av); typedef void storage_open_f(const struct stevedore *); typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); +typedef struct object *storage_alloc_obj_f(struct stevedore *, size_t size, double ttl); typedef void storage_trim_f(struct storage *, size_t size); typedef void storage_free_f(struct storage *); typedef void storage_object_f(const struct sess *sp); @@ -50,6 +51,7 @@ storage_init_f *init; /* called by mgt process */ storage_open_f *open; /* called by cache process */ storage_alloc_f *alloc; + storage_alloc_obj_f *allocobj; storage_trim_f *trim; storage_free_f *free; storage_object_f *object; @@ -63,7 +65,7 @@ VTAILQ_ENTRY(stevedore) list; }; -struct object *STV_NewObject(struct sess *sp, unsigned len); +struct object *STV_NewObject(struct sess *sp, unsigned len, double ttl); struct storage *STV_alloc(struct sess *sp, size_t size); void STV_trim(struct storage *st, size_t size); void STV_free(struct storage *st); From phk at projects.linpro.no Fri Sep 4 10:53:52 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 4 Sep 2009 12:53:52 +0200 (CEST) Subject: r4229 - trunk/varnish-cache/bin/varnishd Message-ID: <20090904105352.116C41F7236@projects.linpro.no> Author: phk Date: 2009-09-04 12:53:51 +0200 (Fri, 04 Sep 2009) New Revision: 4229 Modified: trunk/varnish-cache/bin/varnishd/stevedore.c Log: Factor the actual initialization of an object into a separate function. Modified: trunk/varnish-cache/bin/varnishd/stevedore.c =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.c 2009-09-04 10:08:23 UTC (rev 4228) +++ trunk/varnish-cache/bin/varnishd/stevedore.c 2009-09-04 10:53:51 UTC (rev 4229) @@ -87,6 +87,28 @@ /*********************************************************************/ +static void +STV_InitObj(struct sess *sp, struct object *o, unsigned wsl) +{ + + memset(o, 0, sizeof *o); + o->magic = OBJECT_MAGIC; + + WS_Init(o->ws_o, "obj", (o + 1), wsl); + WS_Assert(o->ws_o); + + http_Setup(o->http, o->ws_o); + o->http->magic = HTTP_MAGIC; + o->refcnt = 1; + o->grace = NAN; + o->entered = NAN; + VTAILQ_INIT(&o->store); + VTAILQ_INIT(&o->esibits); + sp->wrk->stats.n_object++; +} + +/*********************************************************************/ + struct object * STV_NewObject(struct sess *sp, unsigned l, double ttl) { @@ -102,31 +124,19 @@ if (!sp->wrk->cacheable) { o = malloc(sizeof *o + l); XXXAN(o); - } else { - st = STV_alloc(sp, sizeof *o + l); - XXXAN(st); - xxxassert(st->space >= (sizeof *o + l)); + STV_InitObj(sp, o, l); + return (o); + } + st = STV_alloc(sp, sizeof *o + l); + XXXAN(st); + xxxassert(st->space >= (sizeof *o + l)); - st->len = st->space; - l = st->space - sizeof *o; + st->len = st->space; - o = (void *)st->ptr; /* XXX: align ? */ - } - memset(o, 0, sizeof *o); + o = (void *)st->ptr; /* XXX: align ? */ + + STV_InitObj(sp, o, st->space - sizeof *o); o->objstore = st; - o->magic = OBJECT_MAGIC; - - WS_Init(o->ws_o, "obj", (o + 1), l); - WS_Assert(o->ws_o); - - http_Setup(o->http, o->ws_o); - o->http->magic = HTTP_MAGIC; - o->refcnt = 1; - o->grace = NAN; - o->entered = NAN; - VTAILQ_INIT(&o->store); - VTAILQ_INIT(&o->esibits); - sp->wrk->stats.n_object++; return (o); } From phk at projects.linpro.no Tue Sep 8 11:32:10 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 8 Sep 2009 13:32:10 +0200 (CEST) Subject: r4230 - trunk/varnish-cache/bin/varnishd Message-ID: <20090908113210.BA3491F72B2@projects.linpro.no> Author: phk Date: 2009-09-08 13:32:10 +0200 (Tue, 08 Sep 2009) New Revision: 4230 Modified: trunk/varnish-cache/bin/varnishd/flint.sh Log: Drop the diff, it's more annoying than helpful. Modified: trunk/varnish-cache/bin/varnishd/flint.sh =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.sh 2009-09-04 10:53:51 UTC (rev 4229) +++ trunk/varnish-cache/bin/varnishd/flint.sh 2009-09-08 11:32:10 UTC (rev 4230) @@ -1,6 +1,5 @@ #!/bin/sh -T=/tmp/_$$ flexelint \ -I/usr/include \ -I. \ @@ -10,11 +9,4 @@ flint.lnt \ *.c \ ../../lib/libvarnish/*.c \ - ../../lib/libvcl/*.c \ - > $T -if [ -f _flint.old ] ; then - diff -u _flint.old $T -fi -cat $T -cp $T _flint.old -rm $T + ../../lib/libvcl/*.c From phk at projects.linpro.no Tue Sep 8 11:35:40 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 8 Sep 2009 13:35:40 +0200 (CEST) Subject: r4231 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20090908113540.23AE61F72E6@projects.linpro.no> Author: phk Date: 2009-09-08 13:35:39 +0200 (Tue, 08 Sep 2009) New Revision: 4231 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/include/stat_field.h Log: Add a family of 9 counters to tell us how fetching from the backends progress. Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-09-08 11:32:10 UTC (rev 4230) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-09-08 11:35:39 UTC (rev 4231) @@ -428,14 +428,17 @@ cls = 0; mklen = 0; if (is_head) { - /* nothing */ + sp->wrk->stats.fetch_head++; } else if (http_GetHdr(hp, H_Content_Length, &b)) { + sp->wrk->stats.fetch_length++; cls = fetch_straight(sp, sp->wrk->htc, b); mklen = 1; } else if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) { + sp->wrk->stats.fetch_chunked++; cls = fetch_chunked(sp, sp->wrk->htc); mklen = 1; } else if (http_GetHdr(hp, H_Transfer_Encoding, &b)) { + sp->wrk->stats.fetch_bad++; /* XXX: AUGH! */ WSL(sp->wrk, SLT_Debug, vc->fd, "Invalid Transfer-Encoding"); VBE_ClosedFd(sp); @@ -448,6 +451,7 @@ */ mklen = 1; } else if (http_HdrIs(hp, H_Connection, "close")) { + sp->wrk->stats.fetch_close++; /* * If we have connection closed, it is safe to read what * comes in any case. @@ -455,19 +459,23 @@ cls = fetch_eof(sp, sp->wrk->htc); mklen = 1; } else if (hp->protover < 1.1) { + sp->wrk->stats.fetch_oldhttp++; /* * With no Connection header, assume EOF */ cls = fetch_eof(sp, sp->wrk->htc); mklen = 1; } else { + sp->wrk->stats.fetch_zero++; /* * Assume zero length + * XXX: ??? */ mklen = 1; } if (cls < 0) { + sp->wrk->stats.fetch_failed++; /* XXX: Wouldn't this store automatically be released ? */ while (!VTAILQ_EMPTY(&sp->obj->store)) { st = VTAILQ_FIRST(&sp->obj->store); Modified: trunk/varnish-cache/include/stat_field.h =================================================================== --- trunk/varnish-cache/include/stat_field.h 2009-09-08 11:32:10 UTC (rev 4230) +++ trunk/varnish-cache/include/stat_field.h 2009-09-08 11:35:39 UTC (rev 4231) @@ -49,6 +49,17 @@ MAC_STAT(backend_recycle, uint64_t, 0, 'a', "Backend conn. recycles") MAC_STAT(backend_unused, uint64_t, 0, 'a', "Backend conn. unused") +MAC_STAT(fetch_head, uint64_t, 1, 'a', "Fetch head") +MAC_STAT(fetch_length, uint64_t, 1, 'a', "Fetch with Length") +MAC_STAT(fetch_chunked, uint64_t, 1, 'a', "Fetch chunked") +MAC_STAT(fetch_eof, uint64_t, 1, 'a', "Fetch EOF") +MAC_STAT(fetch_bad, uint64_t, 1, 'a', "Fetch had bad headers") +MAC_STAT(fetch_close, uint64_t, 1, 'a', "Fetch wanted close") +MAC_STAT(fetch_oldhttp, uint64_t, 1, 'a', "Fetch pre HTTP/1.1 closed") +MAC_STAT(fetch_zero, uint64_t, 1, 'a', "Fetch zero len") +MAC_STAT(fetch_failed, uint64_t, 1, 'a', "Fetch failed") + + MAC_STAT(n_sess_mem, uint64_t, 0, 'i', "N struct sess_mem") MAC_STAT(n_sess, uint64_t, 0, 'i', "N struct sess") MAC_STAT(n_object, uint64_t, 1, 'i', "N struct object") From phk at projects.linpro.no Tue Sep 8 12:07:02 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 8 Sep 2009 14:07:02 +0200 (CEST) Subject: r4232 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20090908120702.AA94C1F72B2@projects.linpro.no> Author: phk Date: 2009-09-08 14:07:02 +0200 (Tue, 08 Sep 2009) New Revision: 4232 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/include/shmlog_tags.h Log: Add a new SHMLOG tag FetchError and record why a backend fetch croaked. Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-09-08 11:35:39 UTC (rev 4231) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-09-08 12:07:02 UTC (rev 4232) @@ -73,8 +73,11 @@ while (sl > 0) { i = HTC_Read(htc, p, sl); - if (i <= 0) + if (i <= 0) { + WSP(sp, SLT_FetchError, + "straight read_error: %d", errno); return (-1); + } p += i; st->len += i; sp->obj->len += i; @@ -117,8 +120,11 @@ /* If we didn't succeed, add to buffer, try again */ if (q == NULL || q == buf || *q != '\n') { - if (bp >= be) + if (bp >= be) { + WSP(sp, SLT_FetchError, + "chunked hex-line too long"); return (-1); + } /* * The semantics we need here is "read until you have * received at least one character, but feel free to @@ -135,8 +141,11 @@ * at a time. */ i = HTC_Read(htc, bp, 1); - if (i <= 0) + if (i <= 0) { + WSP(sp, SLT_FetchError, + "chunked read_error: %d", errno); return (-1); + } bp += i; continue; } @@ -248,8 +257,11 @@ AN(p); AN(st); i = HTC_Read(htc, p, v); - if (i < 0) + if (i < 0) { + WSP(sp, SLT_FetchError, + "eof read_error: %d", errno); return (-1); + } if (i == 0) break; p += i; Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2009-09-08 11:35:39 UTC (rev 4231) +++ trunk/varnish-cache/include/shmlog_tags.h 2009-09-08 12:07:02 UTC (rev 4232) @@ -55,6 +55,8 @@ SLTM(Backend) SLTM(Length) +SLTM(FetchError) + SLTM(RxRequest) SLTM(RxResponse) SLTM(RxStatus) From phk at projects.linpro.no Tue Sep 8 12:24:40 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 8 Sep 2009 14:24:40 +0200 (CEST) Subject: r4233 - trunk/varnish-cache/bin/varnishd Message-ID: <20090908122440.5B0BA1F731D@projects.linpro.no> Author: phk Date: 2009-09-08 14:24:40 +0200 (Tue, 08 Sep 2009) New Revision: 4233 Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c Log: Add SHMLOG records for more backend fetch failures Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-09-08 12:07:02 UTC (rev 4232) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-09-08 12:24:40 UTC (rev 4233) @@ -351,8 +351,10 @@ hp = sp->wrk->bereq; VBE_GetFd(sp); - if (sp->vbe == NULL) + if (sp->vbe == NULL) { + WSP(sp, SLT_FetchError, "no backend connection"); return (__LINE__); + } vc = sp->vbe; /* Inherit the backend timeouts from the selected backend */ SES_InheritBackendTimeouts(sp); @@ -372,6 +374,7 @@ /* Deal with any message-body the request might have */ i = FetchReqBody(sp); if (WRW_FlushRelease(w) || i > 0) { + WSP(sp, SLT_FetchError, "backend write error: %d", errno); VBE_ClosedFd(sp); /* XXX: other cleanup ? */ return (__LINE__); @@ -389,19 +392,21 @@ TCP_set_read_timeout(vc->fd, sp->first_byte_timeout); do { i = HTC_Rx(sp->wrk->htc); + if (i < 0) { + WSP(sp, SLT_FetchError, + "http read error: %d", errno); + VBE_ClosedFd(sp); + /* XXX: other cleanup ? */ + return (__LINE__); + } TCP_set_read_timeout(vc->fd, sp->between_bytes_timeout); } while (i == 0); - if (i < 0) { - VBE_ClosedFd(sp); - /* XXX: other cleanup ? */ - return (__LINE__); - } - hp = sp->wrk->beresp; if (http_DissectResponse(sp->wrk, sp->wrk->htc, hp)) { + WSP(sp, SLT_FetchError, "http format error"); VBE_ClosedFd(sp); /* XXX: other cleanup ? */ return (__LINE__); @@ -552,5 +557,3 @@ CLI_AddFuncs(DEBUG_CLI, debug_cmds); } - - From phk at projects.linpro.no Tue Sep 8 18:19:52 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 8 Sep 2009 20:19:52 +0200 (CEST) Subject: r4234 - in trunk/varnish-cache: bin/varnishd lib/libvarnish lib/libvcl Message-ID: <20090908181952.CFB2A1F72B2@projects.linpro.no> Author: phk Date: 2009-09-08 20:19:52 +0200 (Tue, 08 Sep 2009) New Revision: 4234 Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c trunk/varnish-cache/bin/varnishd/storage_file.c trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/lib/libvarnish/argv.c trunk/varnish-cache/lib/libvarnish/cli_common.c trunk/varnish-cache/lib/libvcl/vcc_compile.c trunk/varnish-cache/lib/libvcl/vcc_dir_random.c trunk/varnish-cache/lib/libvcl/vcc_dir_round_robin.c Log: Remove some unnecessary #includes Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-09-08 12:24:40 UTC (rev 4233) +++ trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-09-08 18:19:52 UTC (rev 4234) @@ -38,7 +38,6 @@ #include #include #include -#include #include #include #include Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2009-09-08 12:24:40 UTC (rev 4233) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2009-09-08 18:19:52 UTC (rev 4234) @@ -36,11 +36,9 @@ #include #include -#include #include #include -#include #include #include #include Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2009-09-08 12:24:40 UTC (rev 4233) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2009-09-08 18:19:52 UTC (rev 4234) @@ -38,8 +38,6 @@ #include #include -#include -#include #include #include #include Modified: trunk/varnish-cache/lib/libvarnish/argv.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/argv.c 2009-09-08 12:24:40 UTC (rev 4233) +++ trunk/varnish-cache/lib/libvarnish/argv.c 2009-09-08 18:19:52 UTC (rev 4234) @@ -45,7 +45,6 @@ #include #include #include -#include #include "libvarnish.h" Modified: trunk/varnish-cache/lib/libvarnish/cli_common.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/cli_common.c 2009-09-08 12:24:40 UTC (rev 4233) +++ trunk/varnish-cache/lib/libvarnish/cli_common.c 2009-09-08 18:19:52 UTC (rev 4234) @@ -35,12 +35,10 @@ #include #include -#include #include #include #include #include -#include #include #include #include Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_compile.c 2009-09-08 12:24:40 UTC (rev 4233) +++ trunk/varnish-cache/lib/libvcl/vcc_compile.c 2009-09-08 18:19:52 UTC (rev 4234) @@ -67,7 +67,6 @@ #include #include -#include #include #include #include Modified: trunk/varnish-cache/lib/libvcl/vcc_dir_random.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_dir_random.c 2009-09-08 12:24:40 UTC (rev 4233) +++ trunk/varnish-cache/lib/libvcl/vcc_dir_random.c 2009-09-08 18:19:52 UTC (rev 4234) @@ -35,7 +35,6 @@ #include #include -#include #include #include #include Modified: trunk/varnish-cache/lib/libvcl/vcc_dir_round_robin.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_dir_round_robin.c 2009-09-08 12:24:40 UTC (rev 4233) +++ trunk/varnish-cache/lib/libvcl/vcc_dir_round_robin.c 2009-09-08 18:19:52 UTC (rev 4234) @@ -34,7 +34,6 @@ #include #include -#include #include #include #include From phk at projects.linpro.no Fri Sep 11 13:06:16 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Fri, 11 Sep 2009 15:06:16 +0200 (CEST) Subject: r4235 - in trunk/varnish-cache: bin/varnishd include lib/libvarnish lib/libvcl Message-ID: <20090911130616.638E828533@projects.linpro.no> Author: phk Date: 2009-09-11 15:06:15 +0200 (Fri, 11 Sep 2009) New Revision: 4235 Modified: trunk/varnish-cache/bin/varnishd/cache.h trunk/varnish-cache/bin/varnishd/cache_ban.c trunk/varnish-cache/bin/varnishd/cache_expire.c trunk/varnish-cache/bin/varnishd/cache_fetch.c trunk/varnish-cache/bin/varnishd/cache_hash.c trunk/varnish-cache/bin/varnishd/cache_http.c trunk/varnish-cache/bin/varnishd/cache_pool.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishd/flint.lnt trunk/varnish-cache/bin/varnishd/flint.sh trunk/varnish-cache/bin/varnishd/mgt_cli.c trunk/varnish-cache/bin/varnishd/mgt_param.c trunk/varnish-cache/bin/varnishd/shmlog.c trunk/varnish-cache/bin/varnishd/stevedore.c trunk/varnish-cache/bin/varnishd/storage_file.c trunk/varnish-cache/bin/varnishd/storage_persistent.c trunk/varnish-cache/include/vrt.h trunk/varnish-cache/include/vsb.h trunk/varnish-cache/lib/libvarnish/cli.c trunk/varnish-cache/lib/libvarnish/cli_common.c trunk/varnish-cache/lib/libvarnish/vev.c trunk/varnish-cache/lib/libvarnish/vlu.c trunk/varnish-cache/lib/libvcl/vcc_acl.c trunk/varnish-cache/lib/libvcl/vcc_backend.c trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c trunk/varnish-cache/lib/libvcl/vcc_obj.c Log: Various FlexeLint silencing Modified: trunk/varnish-cache/bin/varnishd/cache.h =================================================================== --- trunk/varnish-cache/bin/varnishd/cache.h 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/cache.h 2009-09-11 13:06:15 UTC (rev 4235) @@ -503,7 +503,7 @@ void EXP_Init(void); void EXP_Rearm(const struct object *o); int EXP_Touch(const struct object *o); -int EXP_NukeOne(struct sess *sp, const struct lru *lru); +int EXP_NukeOne(const struct sess *sp, const struct lru *lru); /* cache_fetch.c */ int FetchHdr(struct sess *sp); @@ -513,7 +513,7 @@ /* cache_http.c */ const char *http_StatusMessage(unsigned); -unsigned http_EstimateWS(struct http *fm, unsigned how); +unsigned http_EstimateWS(const struct http *fm, unsigned how); void HTTP_Init(void); void http_ClrHeader(struct http *to); unsigned http_Write(struct worker *w, const struct http *hp, int resp); Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -222,7 +222,7 @@ l = strlen(a1); assert(l < 127); - bt->src = malloc(l + 3); + bt->src = malloc(l + 3L); XXXAN(bt->src); bt->src[0] = (char)l + 1; memcpy(bt->src + 1, a1, l); Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/cache_expire.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -319,7 +319,7 @@ */ int -EXP_NukeOne(struct sess *sp, const struct lru *lru) +EXP_NukeOne(const struct sess *sp, const struct lru *lru) { struct objcore *oc; Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -247,7 +247,7 @@ if (v == 0) { if (st != NULL && fetchfrag > 0) dump_st(sp, st); - st = STV_alloc(sp, params->fetch_chunksize * 1024); + st = STV_alloc(sp, params->fetch_chunksize * 1024LL); VTAILQ_INSERT_TAIL(&sp->obj->store, st, list); p = st->ptr + st->len; v = st->space - st->len; Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -211,7 +211,7 @@ sp->lhashptr = 1; /* space for NUL */ sp->ihashptr = 0; sp->nhashptr = nhashcount * 2; - p = WS_Alloc(sp->http->ws, sizeof(const char *) * (sp->nhashptr + 1)); + p = WS_Alloc(sp->http->ws, sizeof(const char *) * (sp->nhashptr + 1L)); XXXAN(p); /* Align pointer properly (?) */ u = (uintptr_t)p; Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -630,7 +630,7 @@ */ unsigned -http_EstimateWS(struct http *fm, unsigned how) +http_EstimateWS(const struct http *fm, unsigned how) { unsigned u, l; @@ -723,7 +723,7 @@ p = WS_Alloc(hp->ws, l + 1); if (p != NULL) { WSLH(w, fd, hp, u); - memcpy(p, hp->hd[u].b, l + 1); + memcpy(p, hp->hd[u].b, l + 1L); hp->hd[u].b = p; hp->hd[u].e = p + l; } else { @@ -781,7 +781,7 @@ to->hd[field].e = NULL; to->hdf[field] = 0; } else { - memcpy(p, string, l + 1); + memcpy(p, string, l + 1L); to->hd[field].b = p; to->hd[field].e = p + l; to->hdf[field] = 0; Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -327,7 +327,7 @@ owq = wq; wq = pwq; for (u = nwq; u < pools; u++) { - wq[u] = calloc(sizeof *wq[u], 1); + wq[u] = calloc(sizeof *wq[0], 1); XXXAN(wq[u]); wq[u]->magic = WQ_MAGIC; Lck_New(&wq[u]->mtx); Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -975,7 +975,7 @@ /*--------------------------------------------------------------------*/ void -VRT_purge_string(struct sess *sp, char *str, ...) +VRT_purge_string(struct sess *sp, const char *str, ...) { char *p, *a1, *a2, *a3; char **av; Modified: trunk/varnish-cache/bin/varnishd/flint.lnt =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.lnt 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/flint.lnt 2009-09-11 13:06:15 UTC (rev 4235) @@ -1,6 +1,13 @@ -d__flexelint_v9__=1 +// FLINT Bug20090910_838 +-efunc(838, VRT_purge) +-efunc(838, VRT_purge_string) + +// FLINT Bug Bug20090908_766 +-efile(766, cache_vcl.c) + //-sem (pthread_mutex_lock, thread_lock) -sem (pthread_mutex_trylock, thread_lock) -sem (VBE_DropRefLocked, thread_unlock) @@ -42,6 +49,7 @@ -esym(458, heritage) -esym(458, name_key) -esym(528, svnid) + ////////////// -passes=3 @@ -142,22 +150,9 @@ // Review all below this line /////////////////////////////////////////////// --e732 // Loss of sign (arg. no. 2) (int to unsigned --e737 // [45] Loss of sign in promotion from int to unsigned --e713 // Loss of precision (assignment) (unsigned long long to long long) --e574 // Signed-unsigned mix with relational --e712 // Loss of precision (assignment) (long long to --e747 // Significant prototype coercion (arg. no. 2) long --e776 // Possible truncation of addition - -/* - --e767 // Macro redef (system queue.h vs ours ) - --e506 // Constant value boolean --e818 // Pointer parameter '...' could be declared as pointing to const --e774 // Boolean within 'if' always evaluates to False --e534 // Ignoring return value of function --e557 // unrecog format - -*/ +-e732 // 183 Loss of sign (___) (___ to ___) +-e737 // 143 Loss of sign in promotion from ___ to ___ +-e713 // 42 Loss of precision (___) (___ to ___) +-e574 // 48 Signed-unsigned mix with relational +-e712 // 96 Loss of precision (___) (___ to ___) +-e747 // 297 Significant prototype coercion (___) ___ to ___ Modified: trunk/varnish-cache/bin/varnishd/flint.sh =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.sh 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/flint.sh 2009-09-11 13:06:15 UTC (rev 4235) @@ -1,12 +1,11 @@ #!/bin/sh flexelint \ - -I/usr/include \ + flint.lnt \ -I. \ -I../../include \ -I../.. \ -DVARNISH_STATE_DIR=\"foo\" \ - flint.lnt \ *.c \ ../../lib/libvarnish/*.c \ ../../lib/libvcl/*.c Modified: trunk/varnish-cache/bin/varnishd/mgt_cli.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_cli.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/mgt_cli.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -346,7 +346,7 @@ { int i; - for (i = 0; i + 2 < sizeof cp->challenge; i++) + for (i = 0; i + 2L < sizeof cp->challenge; i++) cp->challenge[i] = (random() % 26) + 'a'; cp->challenge[i++] = '\n'; cp->challenge[i] = '\0'; Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -758,7 +758,7 @@ 0, "on", "bool" }, { "ban_lurker_sleep", tweak_timeout_double, - &master.ban_lurker_sleep, 0.0, UINT_MAX, + &master.ban_lurker_sleep, 0, UINT_MAX, "How long time does the ban lurker thread sleeps between " "successfull attempts to push the last item up the purge " " list. It always sleeps a second when nothing can be done.\n" @@ -919,7 +919,7 @@ margin = strlen(pp->name) + 1; n++; } - parspec = realloc(parspec, (nparspec + n + 1) * sizeof *parspec); + parspec = realloc(parspec, (1L + nparspec + n) * sizeof *parspec); XXXAN(parspec); for (pp = ps; pp->name != NULL; pp++) parspec[nparspec++] = pp; Modified: trunk/varnish-cache/bin/varnishd/shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/shmlog.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/shmlog.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -166,7 +166,7 @@ p = logstart + loghead->ptr; /* +1 for the NUL */ - n = vsnprintf((char *)(p + SHMLOG_DATA), mlen + 1, fmt, ap); + n = vsnprintf((char *)(p + SHMLOG_DATA), mlen + 1L, fmt, ap); if (n > mlen) n = mlen; /* we truncate long fields */ vsl_hdr(tag, p, n, id); @@ -266,7 +266,7 @@ p = w->wlp; /* +1 for the NUL */ - n = vsnprintf((char *)(p + SHMLOG_DATA), mlen + 1, fmt, ap); + n = vsnprintf((char *)(p + SHMLOG_DATA), mlen + 1L, fmt, ap); if (n > mlen) n = mlen; /* we truncate long fields */ vsl_hdr(tag, p, n, id); Modified: trunk/varnish-cache/bin/varnishd/stevedore.c =================================================================== --- trunk/varnish-cache/bin/varnishd/stevedore.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/stevedore.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -113,7 +113,7 @@ STV_NewObject(struct sess *sp, unsigned l, double ttl) { struct object *o; - struct storage *st = NULL; + struct storage *st; (void)ttl; if (l == 0) Modified: trunk/varnish-cache/bin/varnishd/storage_file.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_file.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/storage_file.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -412,7 +412,7 @@ if (*fail < (uintmax_t)sc->pagesize * MINPAGES) return; - if (sz > 0 && sz < *fail && sz < SIZE_MAX) { + if (sz > 0 && sz < *fail && sz < SSIZE_MAX) { p = mmap(NULL, sz, PROT_READ|PROT_WRITE, MAP_NOCORE | MAP_NOSYNC | MAP_SHARED, sc->fd, off); if (p != MAP_FAILED) { @@ -427,8 +427,8 @@ *fail = sz; h = sz / 2; - if (h > SIZE_MAX) - h = SIZE_MAX; + if (h > SSIZE_MAX) + h = SSIZE_MAX; h -= (h % sc->pagesize); smf_open_chunk(sc, h, off, fail, sum); Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -997,7 +997,7 @@ sc->tailban = NULL; printf("Silo completely loaded\n"); while (1) - sleep (1); + (void)sleep (1); NEEDLESS_RETURN(NULL); } Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/include/vrt.h 2009-09-11 13:06:15 UTC (rev 4235) @@ -144,7 +144,7 @@ void VRT_panic(struct sess *sp, const char *, ...); void VRT_purge(struct sess *sp, char *, ...); -void VRT_purge_string(struct sess *sp, char *, ...); +void VRT_purge_string(struct sess *sp, const char *, ...); void VRT_count(const struct sess *, unsigned); int VRT_rewrite(const char *, const char *); Modified: trunk/varnish-cache/include/vsb.h =================================================================== --- trunk/varnish-cache/include/vsb.h 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/include/vsb.h 2009-09-11 13:06:15 UTC (rev 4235) @@ -37,8 +37,8 @@ unsigned s_magic; char *s_buf; /* storage buffer */ void *s_unused; /* binary compatibility. */ - int s_size; /* size of storage buffer */ - int s_len; /* current length of string */ + size_t s_size; /* size of storage buffer */ + size_t s_len; /* current length of string */ #define VSB_FIXEDLEN 0x00000000 /* fixed length buffer (default) */ #define VSB_AUTOEXTEND 0x00000001 /* automatically extend buffer */ #define VSB_USRFLAGMSK 0x0000ffff /* mask of flags the user may specify */ Modified: trunk/varnish-cache/lib/libvarnish/cli.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/cli.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/lib/libvarnish/cli.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -150,7 +150,7 @@ for(c = c2; c != NULL && c->request != NULL; c++) i2++; - c = malloc(sizeof(*c) * (i1 + i2 + 1)); + c = malloc(sizeof(*c) * (1L + i1 + i2)); if (c == NULL) return (c); if (c1 != NULL) Modified: trunk/varnish-cache/lib/libvarnish/cli_common.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/cli_common.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/lib/libvarnish/cli_common.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -174,7 +174,7 @@ j = sscanf(res, "%u %u\n", &u, &v); assert(j == 2); *status = u; - p = malloc(v + 1); + p = malloc(v + 1L); assert(p != NULL); i = read_tmo(fd, p, v + 1, tmo); if (i < 0) { Modified: trunk/varnish-cache/lib/libvarnish/vev.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/vev.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/lib/libvarnish/vev.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -151,7 +151,7 @@ if (sig < vev_nsig) return (0); - os = calloc(sizeof *os, (sig + 1)); + os = calloc(sizeof *os, (sig + 1L)); if (os == NULL) return (ENOMEM); Modified: trunk/varnish-cache/lib/libvarnish/vlu.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/vlu.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/lib/libvarnish/vlu.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -64,7 +64,7 @@ l->priv = priv; l->bufl = bufsize - 1; l->telnet = -1; - l->buf = malloc(l->bufl + 1); + l->buf = malloc(l->bufl + 1L); if (l->buf == NULL) { FREE_OBJ(l); l = NULL; Modified: trunk/varnish-cache/lib/libvcl/vcc_acl.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_acl.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/lib/libvcl/vcc_acl.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -40,7 +40,6 @@ #include #include #include -#include #include "vsb.h" #include "vrt.h" Modified: trunk/varnish-cache/lib/libvcl/vcc_backend.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_backend.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/lib/libvcl/vcc_backend.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -137,7 +137,7 @@ "Please specify which exact address " "you want to use, we found these:\n", PF(t_host), multiple); - for (res = res0; res; res = res->ai_next) { + for (res = res0; res != NULL; res = res->ai_next) { error = getnameinfo(res->ai_addr, res->ai_addrlen, hbuf, sizeof hbuf, NULL, 0, NI_NUMERICHOST); Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -1,5 +1,5 @@ /* - * $Id$ + * $Id: vcc_gen_fixed_token.tcl 4208 2009-08-24 14:51:26Z kristian $ * * NB: This file is machine generated, DO NOT EDIT! * @@ -159,9 +159,10 @@ /* ../../include/vcl.h */ - vsb_cat(sb, "/*\n * $Id$\n *\n * NB: This file is machine generate"); - vsb_cat(sb, "d, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_tok"); - vsb_cat(sb, "en.tcl instead\n */\n\nstruct sess;\n"); + vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4208 2009-08-24 14"); + vsb_cat(sb, ":51:26Z kristian $\n *\n * NB: This file is machine g"); + vsb_cat(sb, "enerated, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fi"); + vsb_cat(sb, "xed_token.tcl instead\n */\n\nstruct sess;\n"); vsb_cat(sb, "struct cli;\n\ntypedef void vcl_init_f(struct cli *);\n"); vsb_cat(sb, "typedef void vcl_fini_f(struct cli *);\n"); vsb_cat(sb, "typedef int vcl_func_f(struct sess *sp);\n"); @@ -227,15 +228,15 @@ vsb_cat(sb, " * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWI"); vsb_cat(sb, "SE) ARISING IN ANY WAY\n * OUT OF THE USE OF THIS SOFT"); vsb_cat(sb, "WARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n"); - vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id$\n *\n"); - vsb_cat(sb, " * Runtime support for compiled VCL programs.\n"); - vsb_cat(sb, " *\n * XXX: When this file is changed, lib/libvcl/vcc_"); - vsb_cat(sb, "gen_fixed_token.tcl\n * XXX: *MUST* be rerun.\n"); - vsb_cat(sb, " */\n\nstruct sess;\nstruct vsb;\n"); - vsb_cat(sb, "struct cli;\nstruct director;\n"); - vsb_cat(sb, "struct VCL_conf;\nstruct sockaddr;\n"); - vsb_cat(sb, "\n/*\n * A backend probe specification\n"); - vsb_cat(sb, " */\n\nextern void *vrt_magic_string_end;\n"); + vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id: vrt.h 4185 2009-08-17 11:"); + vsb_cat(sb, "53:01Z phk $\n *\n * Runtime support for compiled VCL "); + vsb_cat(sb, "programs.\n *\n * XXX: When this file is changed, lib/"); + vsb_cat(sb, "libvcl/vcc_gen_fixed_token.tcl\n"); + vsb_cat(sb, " * XXX: *MUST* be rerun.\n */\n"); + vsb_cat(sb, "\nstruct sess;\nstruct vsb;\nstruct cli;\n"); + vsb_cat(sb, "struct director;\nstruct VCL_conf;\n"); + vsb_cat(sb, "struct sockaddr;\n\n/*\n * A backend probe specificati"); + vsb_cat(sb, "on\n */\n\nextern void *vrt_magic_string_end;\n"); vsb_cat(sb, "\nstruct vrt_backend_probe {\n\tconst char\t*url;\n"); vsb_cat(sb, "\tconst char\t*request;\n\tdouble\t\ttimeout;\n"); vsb_cat(sb, "\tdouble\t\tinterval;\n\tunsigned\twindow;\n"); @@ -281,9 +282,9 @@ vsb_cat(sb, " const char *,\n void *, const char *);\n"); vsb_cat(sb, "\nvoid VRT_panic(struct sess *sp, const char *, ...);\n"); vsb_cat(sb, "void VRT_purge(struct sess *sp, char *, ...);\n"); - vsb_cat(sb, "void VRT_purge_string(struct sess *sp, char *, ...);\n"); - vsb_cat(sb, "\nvoid VRT_count(const struct sess *, unsigned);\n"); - vsb_cat(sb, "int VRT_rewrite(const char *, const char *);\n"); + vsb_cat(sb, "void VRT_purge_string(struct sess *sp, const char *, ."); + vsb_cat(sb, "..);\n\nvoid VRT_count(const struct sess *, unsigned);"); + vsb_cat(sb, "\nint VRT_rewrite(const char *, const char *);\n"); vsb_cat(sb, "void VRT_error(struct sess *, unsigned, const char *);"); vsb_cat(sb, "\nint VRT_switch_config(const char *);\n"); vsb_cat(sb, "\nenum gethdr_e { HDR_REQ, HDR_RESP, HDR_OBJ, HDR_BERE"); @@ -316,25 +317,27 @@ /* ../../include/vrt_obj.h */ - vsb_cat(sb, "/*\n * $Id$\n *\n * NB: This file is machine generate"); - vsb_cat(sb, "d, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fixed_tok"); - vsb_cat(sb, "en.tcl instead\n */\n\nstruct sockaddr * VRT_r_client_"); - vsb_cat(sb, "ip(const struct sess *);\nstruct sockaddr * VRT_r_serv"); - vsb_cat(sb, "er_ip(struct sess *);\nconst char * VRT_r_server_hostn"); - vsb_cat(sb, "ame(struct sess *);\nconst char * VRT_r_server_identit"); - vsb_cat(sb, "y(struct sess *);\nint VRT_r_server_port(struct sess *"); - vsb_cat(sb, ");\nconst char * VRT_r_req_request(const struct sess *"); - vsb_cat(sb, ");\nvoid VRT_l_req_request(const struct sess *, const "); - vsb_cat(sb, "char *, ...);\nconst char * VRT_r_req_url(const struct"); - vsb_cat(sb, " sess *);\nvoid VRT_l_req_url(const struct sess *, con"); - vsb_cat(sb, "st char *, ...);\nconst char * VRT_r_req_proto(const s"); - vsb_cat(sb, "truct sess *);\nvoid VRT_l_req_proto(const struct sess"); - vsb_cat(sb, " *, const char *, ...);\nvoid VRT_l_req_hash(struct se"); - vsb_cat(sb, "ss *, const char *);\nstruct director * VRT_r_req_back"); - vsb_cat(sb, "end(struct sess *);\nvoid VRT_l_req_backend(struct ses"); - vsb_cat(sb, "s *, struct director *);\nint VRT_r_req_restarts(const"); - vsb_cat(sb, " struct sess *);\ndouble VRT_r_req_grace(struct sess *"); - vsb_cat(sb, ");\nvoid VRT_l_req_grace(struct sess *, double);\n"); + vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 4208 2009-08-24 14"); + vsb_cat(sb, ":51:26Z kristian $\n *\n * NB: This file is machine g"); + vsb_cat(sb, "enerated, DO NOT EDIT!\n *\n * Edit and run vcc_gen_fi"); + vsb_cat(sb, "xed_token.tcl instead\n */\n\nstruct sockaddr * VRT_r_"); + vsb_cat(sb, "client_ip(const struct sess *);\n"); + vsb_cat(sb, "struct sockaddr * VRT_r_server_ip(struct sess *);\n"); + vsb_cat(sb, "const char * VRT_r_server_hostname(struct sess *);\n"); + vsb_cat(sb, "const char * VRT_r_server_identity(struct sess *);\n"); + vsb_cat(sb, "int VRT_r_server_port(struct sess *);\n"); + vsb_cat(sb, "const char * VRT_r_req_request(const struct sess *);\n"); + vsb_cat(sb, "void VRT_l_req_request(const struct sess *, const char"); + vsb_cat(sb, " *, ...);\nconst char * VRT_r_req_url(const struct ses"); + vsb_cat(sb, "s *);\nvoid VRT_l_req_url(const struct sess *, const c"); + vsb_cat(sb, "har *, ...);\nconst char * VRT_r_req_proto(const struc"); + vsb_cat(sb, "t sess *);\nvoid VRT_l_req_proto(const struct sess *, "); + vsb_cat(sb, "const char *, ...);\nvoid VRT_l_req_hash(struct sess *"); + vsb_cat(sb, ", const char *);\nstruct director * VRT_r_req_backend("); + vsb_cat(sb, "struct sess *);\nvoid VRT_l_req_backend(struct sess *,"); + vsb_cat(sb, " struct director *);\nint VRT_r_req_restarts(const str"); + vsb_cat(sb, "uct sess *);\ndouble VRT_r_req_grace(struct sess *);\n"); + vsb_cat(sb, "void VRT_l_req_grace(struct sess *, double);\n"); vsb_cat(sb, "const char * VRT_r_req_xid(struct sess *);\n"); vsb_cat(sb, "unsigned VRT_r_req_esi(struct sess *);\n"); vsb_cat(sb, "void VRT_l_req_esi(struct sess *, unsigned);\n"); Modified: trunk/varnish-cache/lib/libvcl/vcc_obj.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_obj.c 2009-09-08 18:19:52 UTC (rev 4234) +++ trunk/varnish-cache/lib/libvcl/vcc_obj.c 2009-09-11 13:06:15 UTC (rev 4235) @@ -1,5 +1,5 @@ /* - * $Id$ + * $Id: vcc_gen_fixed_token.tcl 4208 2009-08-24 14:51:26Z kristian $ * * NB: This file is machine generated, DO NOT EDIT! * From phk at projects.linpro.no Mon Sep 14 08:47:04 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Mon, 14 Sep 2009 10:47:04 +0200 (CEST) Subject: r4236 - in trunk/varnish-cache: bin/varnishd include lib/libvarnish lib/libvcl Message-ID: <20090914084704.3A1CE38000@projects.linpro.no> Author: phk Date: 2009-09-14 10:47:03 +0200 (Mon, 14 Sep 2009) New Revision: 4236 Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c trunk/varnish-cache/bin/varnishd/cache_vrt.c trunk/varnish-cache/bin/varnishd/flint.lnt trunk/varnish-cache/include/cli_priv.h trunk/varnish-cache/include/shmlog_tags.h trunk/varnish-cache/include/varnishapi.h trunk/varnish-cache/include/vcl.h trunk/varnish-cache/include/vcl_returns.h trunk/varnish-cache/include/vct.h trunk/varnish-cache/include/vrt.h trunk/varnish-cache/include/vsb.h trunk/varnish-cache/lib/libvarnish/cli_common.c trunk/varnish-cache/lib/libvarnish/vct.c trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl trunk/varnish-cache/lib/libvcl/vcc_token_defs.h Log: Another FlexeLint washing of the source. Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2009-09-14 08:47:03 UTC (rev 4236) @@ -315,11 +315,11 @@ sp->handling = 0; \ sp->cur_method = VCL_MET_ ## upper; \ WSP(sp, SLT_VCL_call, "%s", #func); \ - sp->vcl->func##_func(sp); \ + (void)sp->vcl->func##_func(sp); \ WSP(sp, SLT_VCL_return, "%s", VCC_Return_Name(sp->handling)); \ sp->cur_method = 0; \ - assert((1 << sp->handling) & bitmap); \ - assert(!((1 << sp->handling) & ~bitmap)); \ + assert((1U << sp->handling) & bitmap); \ + assert(!((1U << sp->handling) & ~bitmap)); \ } #include "vcl_returns.h" Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/bin/varnishd/cache_vrt.c 2009-09-14 08:47:03 UTC (rev 4236) @@ -57,7 +57,7 @@ /*XXX: sort of a hack, improve the Tcl code in the compiler to avoid */ /*lint -save -esym(818,sp) */ -void *vrt_magic_string_end = &vrt_magic_string_end; +const void * const vrt_magic_string_end = &vrt_magic_string_end; static char vrt_hostname[255] = ""; /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/bin/varnishd/flint.lnt =================================================================== --- trunk/varnish-cache/bin/varnishd/flint.lnt 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/bin/varnishd/flint.lnt 2009-09-14 08:47:03 UTC (rev 4236) @@ -1,13 +1,68 @@ -d__flexelint_v9__=1 +-libdir(../../include) +-libdir(../../lib/) + +// -efile(765, ../../lib/) +// -efile(765, ../../include/) +// -efile(759, ../../lib/) +// -efile(759, ../../include/) + +-esym(755, vct_*) +-esym(759, vev_*) +-esym(765, vev_*) +-esym(714, vev_*) +-esym(759, vsb_*) +-esym(765, vsb_*) +-esym(714, vsb_*) +-esym(759, vpf_*) +-esym(765, vpf_*) +-esym(714, vpf_*) +-esym(759, H_*) +-esym(765, H_*) +-esym(757, VSL_*) +-esym(765, VLU_*) +-esym(759, VLU_*) +-esym(714, VLU_*) +-esym(765, VSS_*) +-esym(759, VSS_*) +-esym(755, VSL_*) +-esym(765, VSL_*) +-esym(759, VSL_*) +-esym(765, CLI_*) +-esym(759, CLI_*) +-esym(755, CLI_*) + +-esym(755, VLIST_*) +-esym(755, VSLIST_*) +-esym(755, VSTAILQ_*) +-esym(755, VTAILQ_*) + +// Stuff used outside varnishd +-esym(759, BackSlash) +-esym(765, BackSlash) +-esym(759, BackSlashDecode) +-esym(765, BackSlashDecode) +-esym(714, VSS_open) +-esym(757, base64*) + + +// Stuff used by compiled VCL +-esym(757, VRT_*) +-esym(759, VRT_*) +-esym(765, VRT_*) +-esym(714, VRT_*) +-esym(755, VRT_*) +-esym(765, vrt_magic_string_end) +-esym(759, vrt_magic_string_end) +-esym(768, vrt_ref::*) +-esym(768, VCL_conf::*) + // FLINT Bug20090910_838 -efunc(838, VRT_purge) -efunc(838, VRT_purge_string) -// FLINT Bug Bug20090908_766 --efile(766, cache_vcl.c) - //-sem (pthread_mutex_lock, thread_lock) -sem (pthread_mutex_trylock, thread_lock) -sem (VBE_DropRefLocked, thread_unlock) @@ -25,8 +80,8 @@ ////////////// -efunc(1791, pdiff) // return last on line ////////////// --efile(451, "sys/*.h") // No include guard --efile(451, "machine/*.h") // No include guard +-efile(451, "sys/\*.h") // No include guard +-efile(451, "machine/\*.h") // No include guard -efile(451, "vcl_returns.h") // No include guard -efile(451, "cache_backend_poll.h") // No include guard -efile(451, "steps.h") // No include guard Modified: trunk/varnish-cache/include/cli_priv.h =================================================================== --- trunk/varnish-cache/include/cli_priv.h 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/include/cli_priv.h 2009-09-14 08:47:03 UTC (rev 4236) @@ -55,7 +55,6 @@ /* The implementation must provide these functions */ void cli_out(struct cli *cli, const char *fmt, ...); void cli_quote(struct cli *cli, const char *str); -void cli_param(struct cli *cli); void cli_result(struct cli *cli, unsigned r); /* From libvarnish/cli.c */ Modified: trunk/varnish-cache/include/shmlog_tags.h =================================================================== --- trunk/varnish-cache/include/shmlog_tags.h 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/include/shmlog_tags.h 2009-09-14 08:47:03 UTC (rev 4236) @@ -49,9 +49,7 @@ SLTM(BackendXID) SLTM(BackendReuse) SLTM(BackendClose) -SLTM(HttpError) SLTM(HttpGarbage) -SLTM(ClientAddr) SLTM(Backend) SLTM(Length) @@ -86,17 +84,13 @@ SLTM(VCL_trace) SLTM(VCL_return) SLTM(VCL_error) -SLTM(VCL_info) SLTM(ReqStart) SLTM(Hit) SLTM(HitPass) SLTM(ExpBan) -SLTM(ExpPick) SLTM(ExpKill) SLTM(WorkThread) -SLTM(Terminate) -SLTM(ESItrace) SLTM(ESI_xmlerror) SLTM(Hash) Modified: trunk/varnish-cache/include/varnishapi.h =================================================================== --- trunk/varnish-cache/include/varnishapi.h 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/include/varnishapi.h 2009-09-14 08:47:03 UTC (rev 4236) @@ -34,8 +34,6 @@ #include "shmlog.h" -#define V_DEAD __attribute__ ((noreturn)) - /* base64.c */ void base64_init(void); int base64_decode(char *d, unsigned dlen, const char *s); Modified: trunk/varnish-cache/include/vcl.h =================================================================== --- trunk/varnish-cache/include/vcl.h 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/include/vcl.h 2009-09-14 08:47:03 UTC (rev 4236) @@ -14,15 +14,15 @@ typedef int vcl_func_f(struct sess *sp); /* VCL Methods */ -#define VCL_MET_RECV (1 << 0) -#define VCL_MET_PIPE (1 << 1) -#define VCL_MET_PASS (1 << 2) -#define VCL_MET_HASH (1 << 3) -#define VCL_MET_MISS (1 << 4) -#define VCL_MET_HIT (1 << 5) -#define VCL_MET_FETCH (1 << 6) -#define VCL_MET_DELIVER (1 << 7) -#define VCL_MET_ERROR (1 << 8) +#define VCL_MET_RECV (1U << 0) +#define VCL_MET_PIPE (1U << 1) +#define VCL_MET_PASS (1U << 2) +#define VCL_MET_HASH (1U << 3) +#define VCL_MET_MISS (1U << 4) +#define VCL_MET_HIT (1U << 5) +#define VCL_MET_FETCH (1U << 6) +#define VCL_MET_DELIVER (1U << 7) +#define VCL_MET_ERROR (1U << 8) #define VCL_MET_MAX 9 Modified: trunk/varnish-cache/include/vcl_returns.h =================================================================== --- trunk/varnish-cache/include/vcl_returns.h 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/include/vcl_returns.h 2009-09-14 08:47:03 UTC (rev 4236) @@ -19,47 +19,47 @@ #ifdef VCL_MET_MAC VCL_MET_MAC(recv,RECV, - ((1 << VCL_RET_ERROR) - | (1 << VCL_RET_PASS) - | (1 << VCL_RET_PIPE) - | (1 << VCL_RET_LOOKUP) + ((1U << VCL_RET_ERROR) + | (1U << VCL_RET_PASS) + | (1U << VCL_RET_PIPE) + | (1U << VCL_RET_LOOKUP) )) VCL_MET_MAC(pipe,PIPE, - ((1 << VCL_RET_ERROR) - | (1 << VCL_RET_PIPE) + ((1U << VCL_RET_ERROR) + | (1U << VCL_RET_PIPE) )) VCL_MET_MAC(pass,PASS, - ((1 << VCL_RET_ERROR) - | (1 << VCL_RET_RESTART) - | (1 << VCL_RET_PASS) + ((1U << VCL_RET_ERROR) + | (1U << VCL_RET_RESTART) + | (1U << VCL_RET_PASS) )) VCL_MET_MAC(hash,HASH, - ((1 << VCL_RET_HASH) + ((1U << VCL_RET_HASH) )) VCL_MET_MAC(miss,MISS, - ((1 << VCL_RET_ERROR) - | (1 << VCL_RET_RESTART) - | (1 << VCL_RET_PASS) - | (1 << VCL_RET_FETCH) + ((1U << VCL_RET_ERROR) + | (1U << VCL_RET_RESTART) + | (1U << VCL_RET_PASS) + | (1U << VCL_RET_FETCH) )) VCL_MET_MAC(hit,HIT, - ((1 << VCL_RET_ERROR) - | (1 << VCL_RET_RESTART) - | (1 << VCL_RET_PASS) - | (1 << VCL_RET_DELIVER) + ((1U << VCL_RET_ERROR) + | (1U << VCL_RET_RESTART) + | (1U << VCL_RET_PASS) + | (1U << VCL_RET_DELIVER) )) VCL_MET_MAC(fetch,FETCH, - ((1 << VCL_RET_ERROR) - | (1 << VCL_RET_RESTART) - | (1 << VCL_RET_PASS) - | (1 << VCL_RET_DELIVER) + ((1U << VCL_RET_ERROR) + | (1U << VCL_RET_RESTART) + | (1U << VCL_RET_PASS) + | (1U << VCL_RET_DELIVER) )) VCL_MET_MAC(deliver,DELIVER, - ((1 << VCL_RET_RESTART) - | (1 << VCL_RET_DELIVER) + ((1U << VCL_RET_RESTART) + | (1U << VCL_RET_DELIVER) )) VCL_MET_MAC(error,ERROR, - ((1 << VCL_RET_RESTART) - | (1 << VCL_RET_DELIVER) + ((1U << VCL_RET_RESTART) + | (1U << VCL_RET_DELIVER) )) #endif Modified: trunk/varnish-cache/include/vct.h =================================================================== --- trunk/varnish-cache/include/vct.h 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/include/vct.h 2009-09-14 08:47:03 UTC (rev 4236) @@ -40,7 +40,7 @@ #define VCT_DIGIT (1<<5) #define VCT_HEX (1<<6) -extern unsigned char vct_typtab[256]; +extern const unsigned char vct_typtab[256]; static inline int vct_is(unsigned char x, unsigned char y) Modified: trunk/varnish-cache/include/vrt.h =================================================================== --- trunk/varnish-cache/include/vrt.h 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/include/vrt.h 2009-09-14 08:47:03 UTC (rev 4236) @@ -45,7 +45,7 @@ * A backend probe specification */ -extern void *vrt_magic_string_end; +extern const void * const vrt_magic_string_end; struct vrt_backend_probe { const char *url; Modified: trunk/varnish-cache/include/vsb.h =================================================================== --- trunk/varnish-cache/include/vsb.h 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/include/vsb.h 2009-09-14 08:47:03 UTC (rev 4236) @@ -36,7 +36,6 @@ struct vsb { unsigned s_magic; char *s_buf; /* storage buffer */ - void *s_unused; /* binary compatibility. */ size_t s_size; /* size of storage buffer */ size_t s_len; /* current length of string */ #define VSB_FIXEDLEN 0x00000000 /* fixed length buffer (default) */ Modified: trunk/varnish-cache/lib/libvarnish/cli_common.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/cli_common.c 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/lib/libvarnish/cli_common.c 2009-09-14 08:47:03 UTC (rev 4236) @@ -85,15 +85,6 @@ printf("CLI result = %u\n", res); } -void -cli_param(struct cli *cli) -{ - - cli_result(cli, CLIS_PARAM); - cli_out(cli, - "Parameter error, use \"help [command]\" for more info.\n"); -} - int cli_writeres(int fd, const struct cli *cli) { Modified: trunk/varnish-cache/lib/libvarnish/vct.c =================================================================== --- trunk/varnish-cache/lib/libvarnish/vct.c 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/lib/libvarnish/vct.c 2009-09-14 08:47:03 UTC (rev 4236) @@ -40,7 +40,7 @@ #define VCT_UPALPHA VCT_ALPHA #define VCT_LOALPHA VCT_ALPHA -unsigned char vct_typtab[256] = { +unsigned const char vct_typtab[256] = { [0x00] = VCT_CTL, [0x01] = VCT_CTL, [0x02] = VCT_CTL, Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-09-14 08:47:03 UTC (rev 4236) @@ -166,15 +166,15 @@ vsb_cat(sb, "struct cli;\n\ntypedef void vcl_init_f(struct cli *);\n"); vsb_cat(sb, "typedef void vcl_fini_f(struct cli *);\n"); vsb_cat(sb, "typedef int vcl_func_f(struct sess *sp);\n"); - vsb_cat(sb, "\n/* VCL Methods */\n#define VCL_MET_RECV\t\t(1 << 0)\n"); - vsb_cat(sb, "#define VCL_MET_PIPE\t\t(1 << 1)\n"); - vsb_cat(sb, "#define VCL_MET_PASS\t\t(1 << 2)\n"); - vsb_cat(sb, "#define VCL_MET_HASH\t\t(1 << 3)\n"); - vsb_cat(sb, "#define VCL_MET_MISS\t\t(1 << 4)\n"); - vsb_cat(sb, "#define VCL_MET_HIT\t\t(1 << 5)\n"); - vsb_cat(sb, "#define VCL_MET_FETCH\t\t(1 << 6)\n"); - vsb_cat(sb, "#define VCL_MET_DELIVER\t\t(1 << 7)\n"); - vsb_cat(sb, "#define VCL_MET_ERROR\t\t(1 << 8)\n"); + vsb_cat(sb, "\n/* VCL Methods */\n#define VCL_MET_RECV\t\t(1U << 0)"); + vsb_cat(sb, "\n#define VCL_MET_PIPE\t\t(1U << 1)\n"); + vsb_cat(sb, "#define VCL_MET_PASS\t\t(1U << 2)\n"); + vsb_cat(sb, "#define VCL_MET_HASH\t\t(1U << 3)\n"); + vsb_cat(sb, "#define VCL_MET_MISS\t\t(1U << 4)\n"); + vsb_cat(sb, "#define VCL_MET_HIT\t\t(1U << 5)\n"); + vsb_cat(sb, "#define VCL_MET_FETCH\t\t(1U << 6)\n"); + vsb_cat(sb, "#define VCL_MET_DELIVER\t\t(1U << 7)\n"); + vsb_cat(sb, "#define VCL_MET_ERROR\t\t(1U << 8)\n"); vsb_cat(sb, "\n#define VCL_MET_MAX\t\t9\n\n/* VCL Returns */\n"); vsb_cat(sb, "#define VCL_RET_DELIVER\t\t0\n#define VCL_RET_ERROR\t\t"); vsb_cat(sb, "1\n#define VCL_RET_FETCH\t\t2\n"); @@ -228,15 +228,15 @@ vsb_cat(sb, " * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWI"); vsb_cat(sb, "SE) ARISING IN ANY WAY\n * OUT OF THE USE OF THIS SOFT"); vsb_cat(sb, "WARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n"); - vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id: vrt.h 4185 2009-08-17 11:"); - vsb_cat(sb, "53:01Z phk $\n *\n * Runtime support for compiled VCL "); + vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id: vrt.h 4235 2009-09-11 13:"); + vsb_cat(sb, "06:15Z phk $\n *\n * Runtime support for compiled VCL "); vsb_cat(sb, "programs.\n *\n * XXX: When this file is changed, lib/"); vsb_cat(sb, "libvcl/vcc_gen_fixed_token.tcl\n"); vsb_cat(sb, " * XXX: *MUST* be rerun.\n */\n"); vsb_cat(sb, "\nstruct sess;\nstruct vsb;\nstruct cli;\n"); vsb_cat(sb, "struct director;\nstruct VCL_conf;\n"); vsb_cat(sb, "struct sockaddr;\n\n/*\n * A backend probe specificati"); - vsb_cat(sb, "on\n */\n\nextern void *vrt_magic_string_end;\n"); + vsb_cat(sb, "on\n */\n\nextern const void *vrt_magic_string_end;\n"); vsb_cat(sb, "\nstruct vrt_backend_probe {\n\tconst char\t*url;\n"); vsb_cat(sb, "\tconst char\t*request;\n\tdouble\t\ttimeout;\n"); vsb_cat(sb, "\tdouble\t\tinterval;\n\tunsigned\twindow;\n"); Modified: trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl 2009-09-14 08:47:03 UTC (rev 4236) @@ -554,7 +554,7 @@ } else { set sp "" } - puts $fo "#define VCL_MET_[string toupper [lindex $m 0]]\t${sp}(1 << $u)" + puts $fo "#define VCL_MET_[string toupper [lindex $m 0]]\t${sp}(1U << $u)" incr u } @@ -620,9 +620,9 @@ puts -nonewline $for ",[string toupper [lindex $m 0]]" set l [lindex $m 1] puts $for "," - puts $for " ((1 << VCL_RET_[string toupper [lindex $l 0]])" + puts $for " ((1U << VCL_RET_[string toupper [lindex $l 0]])" foreach r [lrange $l 1 end] { - puts $for " | (1 << VCL_RET_[string toupper $r])" + puts $for " | (1U << VCL_RET_[string toupper $r])" } puts $for "))" incr u @@ -647,9 +647,7 @@ puts $fo "#include \"vsb.h\"" set tn 128 -puts $foh "#define LOW_TOKEN $tn" - proc add_token {tok str alpha} { global tokens tn fixed foh Modified: trunk/varnish-cache/lib/libvcl/vcc_token_defs.h =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_token_defs.h 2009-09-11 13:06:15 UTC (rev 4235) +++ trunk/varnish-cache/lib/libvcl/vcc_token_defs.h 2009-09-14 08:47:03 UTC (rev 4236) @@ -6,7 +6,6 @@ * Edit and run vcc_gen_fixed_token.tcl instead */ -#define LOW_TOKEN 128 #define T_INCLUDE 128 #define T_IF 129 #define T_ELSE 130 From phk at projects.linpro.no Tue Sep 15 07:44:47 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 15 Sep 2009 09:44:47 +0200 (CEST) Subject: r4237 - in trunk/varnish-cache: bin/varnishd lib/libvcl Message-ID: <20090915074447.51F5B1F72AD@projects.linpro.no> Author: phk Date: 2009-09-15 09:44:47 +0200 (Tue, 15 Sep 2009) New Revision: 4237 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c trunk/varnish-cache/lib/libvcl/vcc_acl.c Log: Further Flexelint polishing Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2009-09-14 08:47:03 UTC (rev 4236) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2009-09-15 07:44:47 UTC (rev 4237) @@ -418,15 +418,12 @@ VTAILQ_FOREACH(s, &symbols, list) { if (s->a > pp) continue; - if (s0 != NULL && s->a < s0->a) - continue; - s0 = s; + if (s0 == NULL || s->a < s0->a) + s0 = s; } if (s0 == NULL) return (-1); - vsb_printf(vsb, "%p", ptr); - if (s0 != NULL) - vsb_printf(vsb, ": %s+%jx", s0->n, (uintmax_t)pp - s0->a); + vsb_printf(vsb, "%p: %s+%jx", ptr, s0->n, (uintmax_t)pp - s0->a); return (0); } @@ -438,40 +435,43 @@ uintptr_t a; struct symbols *s; - strcpy(buf, "nm -an "); - strcat(buf, a0); + p = NULL; + asprintf(&p, "nm -an %s", a0); + if (p == NULL) + return; fi = popen(buf, "r"); - if (fi != NULL) { - while (fgets(buf, sizeof buf, fi)) { - if (buf[0] == ' ') - continue; - p = NULL; - a = strtoul(buf, &p, 16); - if (p == NULL) - continue; - if (a == 0) - continue; - if (*p++ != ' ') - continue; - p++; - if (*p++ != ' ') - continue; - if (*p <= ' ') - continue; - e = strchr(p, '\0'); - AN(e); - while (e > p && isspace(e[-1])) - e--; - *e = '\0'; - s = malloc(sizeof *s + strlen(p) + 1); - AN(s); - s->a = a; - s->n = (void*)(s + 1); - strcpy(s->n, p); - VTAILQ_INSERT_TAIL(&symbols, s, list); - } - (void)pclose(fi); + free(p); + if (fi == NULL) + return; + while (fgets(buf, sizeof buf, fi)) { + if (buf[0] == ' ') + continue; + p = NULL; + a = strtoul(buf, &p, 16); + if (p == NULL) + continue; + if (a == 0) + continue; + if (*p++ != ' ') + continue; + p++; + if (*p++ != ' ') + continue; + if (*p <= ' ') + continue; + e = strchr(p, '\0'); + AN(e); + while (e > p && isspace(e[-1])) + e--; + *e = '\0'; + s = malloc(sizeof *s + strlen(p) + 1); + AN(s); + s->a = a; + s->n = (void*)(s + 1); + strcpy(s->n, p); + VTAILQ_INSERT_TAIL(&symbols, s, list); } + (void)pclose(fi); } /*--------------------------------------------------------------------*/ Modified: trunk/varnish-cache/lib/libvcl/vcc_acl.c =================================================================== --- trunk/varnish-cache/lib/libvcl/vcc_acl.c 2009-09-14 08:47:03 UTC (rev 4236) +++ trunk/varnish-cache/lib/libvcl/vcc_acl.c 2009-09-15 07:44:47 UTC (rev 4237) @@ -250,14 +250,18 @@ { unsigned char b[4]; int i, j, k; + unsigned u; const char *p; memset(b, 0, sizeof b); p = ae->t_addr->dec; for (i = 0; i < 4; i++) { - j = sscanf(p, "%hhu%n", &b[i], &k); + j = sscanf(p, "%u%n", &u, &k); if (j != 1) return (0); + if (u & ~0xff) + return (0); + b[i] = (unsigned char)u; if (p[k] == '\0') break; if (p[k] != '.') From tfheen at projects.linpro.no Mon Sep 21 15:35:04 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 21 Sep 2009 17:35:04 +0200 (CEST) Subject: r4238 - trunk/varnish-cache/man Message-ID: <20090921153504.47B361F7365@projects.linpro.no> Author: tfheen Date: 2009-09-21 17:35:03 +0200 (Mon, 21 Sep 2009) New Revision: 4238 Modified: trunk/varnish-cache/man/vcl.7so Log: Document max_connections somewhat Modified: trunk/varnish-cache/man/vcl.7so =================================================================== --- trunk/varnish-cache/man/vcl.7so 2009-09-15 07:44:47 UTC (rev 4237) +++ trunk/varnish-cache/man/vcl.7so 2009-09-21 15:35:03 UTC (rev 4238) @@ -93,6 +93,11 @@ } .Ed .Pp +To avoid overloading backend servers, +.Fa .max_connections +can be set to limit the maximum number of concurrent backend connections. +.Ed +.Pp The timeout parameters can be overridden in the backend declaration. The timeout parameters are .Fa .connect_timeout From tfheen at projects.linpro.no Tue Sep 22 14:36:05 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Tue, 22 Sep 2009 16:36:05 +0200 (CEST) Subject: r4239 - trunk/varnish-cache/bin/varnishd Message-ID: <20090922143605.E282F1F7358@projects.linpro.no> Author: tfheen Date: 2009-09-22 16:36:05 +0200 (Tue, 22 Sep 2009) New Revision: 4239 Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c Log: Turn purge_dups on by default Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c =================================================================== --- trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-09-21 15:35:03 UTC (rev 4238) +++ trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-09-22 14:36:05 UTC (rev 4239) @@ -751,7 +751,7 @@ { "purge_dups", tweak_bool, &master.purge_dups, 0, 0, "Detect and eliminate duplicate purges.\n", 0, - "off", "bool" }, + "on", "bool" }, { "cli_banner", tweak_bool, &master.cli_banner, 0, 0, "Emit CLI banner on connect.\n" "Set to off for compatibility with pre 2.1 versions.\n", From phk at projects.linpro.no Thu Sep 24 14:21:37 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 24 Sep 2009 16:21:37 +0200 (CEST) Subject: r4240 - trunk/varnish-cache/bin/varnishd Message-ID: <20090924142137.5F1FF1F7352@projects.linpro.no> Author: phk Date: 2009-09-24 16:21:37 +0200 (Thu, 24 Sep 2009) New Revision: 4240 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c Log: If we come into vcl_error{} with a non-3-digit error status, for instance because vcl_recv does "return(error);", force the status to 501. Fixes: #558 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-22 14:36:05 UTC (rev 4239) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-24 14:21:37 UTC (rev 4240) @@ -344,6 +344,9 @@ CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC); h = sp->obj->http; + if (sp->err_code < 100 || sp->err_code > 999) + sp->err_code = 501; + http_PutProtocol(w, sp->fd, h, "HTTP/1.1"); http_PutStatus(w, sp->fd, h, sp->err_code); TIM_format(TIM_real(), date); From phk at projects.linpro.no Thu Sep 24 14:21:54 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 24 Sep 2009 16:21:54 +0200 (CEST) Subject: r4241 - trunk/varnish-cache/bin/varnishtest/tests Message-ID: <20090924142154.85CC61F7374@projects.linpro.no> Author: phk Date: 2009-09-24 16:21:54 +0200 (Thu, 24 Sep 2009) New Revision: 4241 Added: trunk/varnish-cache/bin/varnishtest/tests/r00558.vtc Log: Regression test case for 558 Added: trunk/varnish-cache/bin/varnishtest/tests/r00558.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/r00558.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/r00558.vtc 2009-09-24 14:21:54 UTC (rev 4241) @@ -0,0 +1,19 @@ +# $Id$ + +test "error from vcl_recv{} has no numeric code" + + +server s1 { +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + return (error); + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 501 +} -run From phk at projects.linpro.no Thu Sep 24 14:22:21 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Thu, 24 Sep 2009 16:22:21 +0200 (CEST) Subject: r4242 - trunk/varnish-cache/bin/varnishd Message-ID: <20090924142221.D5AEB1F736E@projects.linpro.no> Author: phk Date: 2009-09-24 16:22:21 +0200 (Thu, 24 Sep 2009) New Revision: 4242 Modified: trunk/varnish-cache/bin/varnishd/cache_esi.c Log: Fix a typo. Submitted by: "Jos I. Boumans" Modified: trunk/varnish-cache/bin/varnishd/cache_esi.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_esi.c 2009-09-24 14:21:54 UTC (rev 4241) +++ trunk/varnish-cache/bin/varnishd/cache_esi.c 2009-09-24 14:22:21 UTC (rev 4242) @@ -370,7 +370,7 @@ continue; if (Tlen(val) == 0) { esi_error(ew, tag.b, Tlen(tag), - "ESI esi:include src attribute withou value"); + "ESI esi:include src attribute without value"); continue; } From tfheen at projects.linpro.no Mon Sep 28 08:43:38 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 10:43:38 +0200 (CEST) Subject: r4243 - in branches/2.0/varnish-cache: . bin/varnishd lib/libvcl Message-ID: <20090928084338.A086F1F733A@projects.linpro.no> Author: tfheen Date: 2009-09-28 10:43:38 +0200 (Mon, 28 Sep 2009) New Revision: 4243 Modified: branches/2.0/varnish-cache/bin/varnishd/cache.h branches/2.0/varnish-cache/configure.ac branches/2.0/varnish-cache/lib/libvcl/vcc_fixed_token.c Log: Merge r4030: Make HTTP_HDR_MAX_VAL a configure option Thanks to "whocares" in the bts. Fixes #455 Modified: branches/2.0/varnish-cache/bin/varnishd/cache.h =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache.h 2009-09-24 14:22:21 UTC (rev 4242) +++ branches/2.0/varnish-cache/bin/varnishd/cache.h 2009-09-28 08:43:38 UTC (rev 4243) @@ -58,8 +58,6 @@ #include "heritage.h" #include "miniobj.h" -#define HTTP_HDR_MAX_VAL 32 - enum { /* Fields from the first line of HTTP proto */ HTTP_HDR_REQ, Modified: branches/2.0/varnish-cache/configure.ac =================================================================== --- branches/2.0/varnish-cache/configure.ac 2009-09-24 14:22:21 UTC (rev 4242) +++ branches/2.0/varnish-cache/configure.ac 2009-09-28 08:43:38 UTC (rev 4243) @@ -360,6 +360,15 @@ fi AC_DEFINE_UNQUOTED([VCC_CC],"$VCC_CC",[C compiler command line for VCL code]) +# Define HTTP_HDR_MAX_VAL +AC_ARG_WITH(max-header-fields, + AS_HELP_STRING([--with-max-header-fields=NUM], + [How many header fields to support (default=32)]), + [], + [with_max_header_fields=32]) + +AC_DEFINE_UNQUOTED(HTTP_HDR_MAX_VAL, $with_max_header_fields, [Define maximum number of header fields supported by varnish ]) + # Use jemalloc on Linux JEMALLOC_SUBDIR= JEMALLOC_LDADD= Modified: branches/2.0/varnish-cache/lib/libvcl/vcc_fixed_token.c =================================================================== --- branches/2.0/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-09-24 14:22:21 UTC (rev 4242) +++ branches/2.0/varnish-cache/lib/libvcl/vcc_fixed_token.c 2009-09-28 08:43:38 UTC (rev 4243) @@ -324,9 +324,9 @@ /* ../../include/vrt_obj.h */ - vsb_cat(sb, "/*\n * $Id: vcc_gen_obj.tcl 3992 2009-03-23 12:50:04Z "); - vsb_cat(sb, "tfheen $\n *\n * NB: This file is machine generated, "); - vsb_cat(sb, "DO NOT EDIT!\n *\n * Edit vcc_gen_obj.tcl instead\n"); + vsb_cat(sb, "/*\n * $Id$\n"); + vsb_cat(sb, " *\n * NB: This file is machine generated, DO NOT EDI"); + vsb_cat(sb, "T!\n *\n * Edit vcc_gen_obj.tcl instead\n"); vsb_cat(sb, " */\n\nstruct sockaddr * VRT_r_client_ip(const struct "); vsb_cat(sb, "sess *);\nstruct sockaddr * VRT_r_server_ip(struct ses"); vsb_cat(sb, "s *);\nconst char * VRT_r_server_hostname(struct sess "); From tfheen at projects.linpro.no Mon Sep 28 08:53:10 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 10:53:10 +0200 (CEST) Subject: r4244 - branches/2.0/varnish-cache/bin/varnishtest/tests Message-ID: <20090928085310.6A4281F733A@projects.linpro.no> Author: tfheen Date: 2009-09-28 10:53:10 +0200 (Mon, 28 Sep 2009) New Revision: 4244 Modified: branches/2.0/varnish-cache/bin/varnishtest/tests/e00013.vtc Log: Merge r4035: Make sure the esi parser kicks in. Modified: branches/2.0/varnish-cache/bin/varnishtest/tests/e00013.vtc =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/e00013.vtc 2009-09-28 08:43:38 UTC (rev 4243) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/e00013.vtc 2009-09-28 08:53:10 UTC (rev 4244) @@ -6,7 +6,7 @@ rxreq expect req.url == "/foo" txresp -hdr "Connection: close" - send { } + send { } } -start varnish v1 -vcl+backend { From tfheen at projects.linpro.no Mon Sep 28 09:05:05 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 11:05:05 +0200 (CEST) Subject: r4245 - in branches/2.0/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20090928090505.EC9CA1F733A@projects.linpro.no> Author: tfheen Date: 2009-09-28 11:05:05 +0200 (Mon, 28 Sep 2009) New Revision: 4245 Added: branches/2.0/varnish-cache/bin/varnishtest/tests/r00494.vtc Modified: branches/2.0/varnish-cache/bin/varnishd/cache_http.c Log: Merge r4036: Implement HTTP continuation lines. Fixes #494 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_http.c 2009-09-28 08:53:10 UTC (rev 4244) +++ branches/2.0/varnish-cache/bin/varnishd/cache_http.c 2009-09-28 09:05:05 UTC (rev 4245) @@ -332,12 +332,29 @@ hp->conds = 0; r = NULL; /* For FlexeLint */ for (; p < t.e; p = r) { - /* XXX: handle continuation lines */ - q = strchr(p, '\n'); - assert(q != NULL); - r = q + 1; - if (q > p && q[-1] == '\r') - q--; + + /* Find end of next header */ + q = r = p; + while (r < t.e) { + if (!vct_iscrlf(*r)) { + r++; + continue; + } + q = r; + assert(r < t.e); + r += vct_skipcrlf(r); + if (r >= t.e) + break; + /* If line does not continue: got it. */ + if (!vct_issp(*r)) + break; + + /* Clear line continuation LWS to spaces */ + while (vct_islws(*q)) + *q++ = ' '; + } + + /* Empty header = end of headers */ if (p == q) break; Copied: branches/2.0/varnish-cache/bin/varnishtest/tests/r00494.vtc (from rev 4036, trunk/varnish-cache/bin/varnishtest/tests/r00494.vtc) =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/r00494.vtc (rev 0) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/r00494.vtc 2009-09-28 09:05:05 UTC (rev 4245) @@ -0,0 +1,27 @@ +# $Id$ + +test "HTTP continuation lines" + +#NB: careful about spaces and tabs in this test. + +server s1 { + rxreq + txresp -hdr {Foo: bar, + barf: fail} -body "xxx" +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set obj.http.bar = obj.http.foo; + remove obj.http.foo; + } +} -start + +client c1 { + txreq + rxresp + expect resp.http.bar == "bar, barf: fail" + expect resp.http.barf == resp.http.barf + expect resp.http.foo == resp.http.foo +} -run + From tfheen at projects.linpro.no Mon Sep 28 09:15:21 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 11:15:21 +0200 (CEST) Subject: r4246 - branches/2.0/varnish-cache/bin/varnishtest Message-ID: <20090928091521.8F0121F733A@projects.linpro.no> Author: tfheen Date: 2009-09-28 11:15:21 +0200 (Mon, 28 Sep 2009) New Revision: 4246 Modified: branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c Log: Merge r4044: Don't overrun rxbuf (varnishtest) Modified: branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c 2009-09-28 09:05:05 UTC (rev 4245) +++ branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c 2009-09-28 09:15:21 UTC (rev 4246) @@ -322,7 +322,7 @@ pfd[0].revents = 0; i = poll(pfd, 1, hp->timeout); assert(i > 0); - assert(hp->prxbuf < hp->nrxbuf); + assert(hp->prxbuf + n < hp->nrxbuf); i = read(hp->fd, hp->rxbuf + hp->prxbuf, n); if (i == 0) return (i); From tfheen at projects.linpro.no Mon Sep 28 09:24:38 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 11:24:38 +0200 (CEST) Subject: r4247 - branches/2.0/varnish-cache/bin/varnishtest Message-ID: <20090928092438.EF6491F733A@projects.linpro.no> Author: tfheen Date: 2009-09-28 11:24:38 +0200 (Mon, 28 Sep 2009) New Revision: 4247 Modified: branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c Log: Merge r4045: And allocate a larger rxbuf while at it. (varnishtest) Modified: branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c 2009-09-28 09:15:21 UTC (rev 4246) +++ branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c 2009-09-28 09:24:38 UTC (rev 4247) @@ -781,7 +781,7 @@ hp->vl = vl; hp->client = client; hp->timeout = 3000; - hp->nrxbuf = 8192; + hp->nrxbuf = 64*1024; hp->vsb = vsb_newauto(); hp->rxbuf = malloc(hp->nrxbuf); /* XXX */ AN(hp->rxbuf); From tfheen at projects.linpro.no Mon Sep 28 09:46:59 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 11:46:59 +0200 (CEST) Subject: r4248 - in branches/2.0/varnish-cache: bin/varnishd bin/varnishtest bin/varnishtest/tests include lib/libvarnish Message-ID: <20090928094659.7CF941F736B@projects.linpro.no> Author: tfheen Date: 2009-09-28 11:46:59 +0200 (Mon, 28 Sep 2009) New Revision: 4248 Added: branches/2.0/varnish-cache/bin/varnishtest/tests/b00029.vtc Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_epoll.c branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_kqueue.c branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_poll.c branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_ports.c branches/2.0/varnish-cache/bin/varnishd/cache_center.c branches/2.0/varnish-cache/bin/varnishd/cache_pipe.c branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c branches/2.0/varnish-cache/include/libvarnish.h branches/2.0/varnish-cache/lib/libvarnish/tcp.c Log: Merge: r4046, r4047, r4183: Change the way we close client sessions. r4046: Change the way we close client sessions. Previously we always used SO_LINGER to make sure that all queued data got transmitted, no matter under which circumstances we closed the client connection. Change this so that SO_LINGER is only activated for orderly connection closure (ie: "Connection: close" from client or error handling), in all other cases (usually the client connecting on us, abandon any data queued for transmission. This _may_ reduce the tendency of worker threads to get hung up on network failures a little bit. r4047: r4046 forgot to reset SO_LINGER for pipe handling which basically broke pipehandling. Fixes #505 r4183: Disable SO_LINGER when we time out a connection due to sess_timeout, so that we do not RST connections that have still not transmitted their data. Since we were able to get the writev(2) to detach the socket, we should not end up sleeping in the close(2) either. We still RST the socket for all error conditions. Ideally I would still like to RST connections that have no outstanding data after their sess_timeout, in order to avoid the 2*RTT+misc timeouts delays associated with loosing a TCP socket for a client that have gone to meet some other IP#. In particular with load-balancers, this allows the load balancer to declare the session dead right away, and reuse it for something more productive. Unfortunately, this lacks OS support in all presently released OS'es: you cannot ask if a socket is done transmitting what you asked it to. FreeBSD-8.0 will have experimental support for this (FIONWRITE) and I will revisit it in that context. Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c 2009-09-28 09:24:38 UTC (rev 4247) +++ branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c 2009-09-28 09:46:59 UTC (rev 4248) @@ -68,7 +68,9 @@ static pthread_t vca_thread_acct; static struct timeval tv_sndtimeo; static struct timeval tv_rcvtimeo; -static struct linger linger; +static const struct linger linger = { + .l_onoff = 1, +}; static unsigned char need_sndtimeo, need_rcvtimeo, need_linger, need_test; Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_epoll.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_epoll.c 2009-09-28 09:24:38 UTC (rev 4247) +++ branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_epoll.c 2009-09-28 09:46:59 UTC (rev 4248) @@ -118,6 +118,7 @@ continue; VTAILQ_REMOVE(&sesshead, sp, list); vca_del(sp->fd); + TCP_linger(sp->fd, 0); vca_close_session(sp, "timeout"); SES_Delete(sp); } Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_kqueue.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_kqueue.c 2009-09-28 09:24:38 UTC (rev 4247) +++ branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_kqueue.c 2009-09-28 09:46:59 UTC (rev 4248) @@ -197,6 +197,7 @@ if (sp->t_open > deadline) break; VTAILQ_REMOVE(&sesshead, sp, list); + TCP_linger(sp->fd, 0); vca_close_session(sp, "timeout"); SES_Delete(sp); } Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_poll.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_poll.c 2009-09-28 09:24:38 UTC (rev 4247) +++ branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_poll.c 2009-09-28 09:46:59 UTC (rev 4248) @@ -146,6 +146,7 @@ continue; VTAILQ_REMOVE(&sesshead, sp, list); vca_unpoll(fd); + TCP_linger(sp->fd, 0); vca_close_session(sp, "timeout"); SES_Delete(sp); } Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_ports.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_ports.c 2009-09-28 09:24:38 UTC (rev 4247) +++ branches/2.0/varnish-cache/bin/varnishd/cache_acceptor_ports.c 2009-09-28 09:46:59 UTC (rev 4248) @@ -138,6 +138,7 @@ VTAILQ_REMOVE(&sesshead, sp, list); if(sp->fd != -1) vca_del(sp->fd); + TCP_linger(sp->fd, 0); vca_close_session(sp, "timeout"); SES_Delete(sp); } Modified: branches/2.0/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_center.c 2009-09-28 09:24:38 UTC (rev 4247) +++ branches/2.0/varnish-cache/bin/varnishd/cache_center.c 2009-09-28 09:46:59 UTC (rev 4248) @@ -240,8 +240,14 @@ sp->t_req = NAN; - if (sp->fd >= 0 && sp->doclose != NULL) + if (sp->fd >= 0 && sp->doclose != NULL) { + /* + * This is an orderly close of the connection; ditch linger + * before we close, to get queued data transmitted. + */ + TCP_linger(sp->fd, 0); vca_close_session(sp, sp->doclose); + } if (sp->fd < 0) { SES_Charge(sp); VSL_stats->sess_closed++; Modified: branches/2.0/varnish-cache/bin/varnishd/cache_pipe.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_pipe.c 2009-09-28 09:24:38 UTC (rev 4247) +++ branches/2.0/varnish-cache/bin/varnishd/cache_pipe.c 2009-09-28 09:46:59 UTC (rev 4248) @@ -102,8 +102,12 @@ sp->t_resp = TIM_real(); memset(fds, 0, sizeof fds); + + TCP_linger(vc->fd, 0); fds[0].fd = vc->fd; fds[0].events = POLLIN | POLLERR; + + TCP_linger(sp->fd, 0); fds[1].fd = sp->fd; fds[1].events = POLLIN | POLLERR; Copied: branches/2.0/varnish-cache/bin/varnishtest/tests/b00029.vtc (from rev 4046, trunk/varnish-cache/bin/varnishtest/tests/b00029.vtc) =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/b00029.vtc (rev 0) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/b00029.vtc 2009-09-28 09:46:59 UTC (rev 4248) @@ -0,0 +1,18 @@ +# $Id$ + +test "Test orderly connection closure" + + +server s1 { + rxreq + txresp -bodylen 130000 +} -start + +varnish v1 -vcl+backend { } -start + +client c1 { + txreq -hdr "Connection: close" + delay 3 + rxresp + expect resp.bodylen == 130000 +} -run Modified: branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c 2009-09-28 09:24:38 UTC (rev 4247) +++ branches/2.0/varnish-cache/bin/varnishtest/vtc_http.c 2009-09-28 09:46:59 UTC (rev 4248) @@ -781,7 +781,7 @@ hp->vl = vl; hp->client = client; hp->timeout = 3000; - hp->nrxbuf = 64*1024; + hp->nrxbuf = 640*1024; hp->vsb = vsb_newauto(); hp->rxbuf = malloc(hp->nrxbuf); /* XXX */ AN(hp->rxbuf); Modified: branches/2.0/varnish-cache/include/libvarnish.h =================================================================== --- branches/2.0/varnish-cache/include/libvarnish.h 2009-09-28 09:24:38 UTC (rev 4247) +++ branches/2.0/varnish-cache/include/libvarnish.h 2009-09-28 09:46:59 UTC (rev 4248) @@ -65,6 +65,7 @@ int TCP_filter_http(int sock); void TCP_blocking(int sock); void TCP_nonblocking(int sock); +void TCP_linger(int sock, int linger); #ifdef SOL_SOCKET void TCP_name(const struct sockaddr *addr, unsigned l, char *abuf, unsigned alen, char *pbuf, unsigned plen); Modified: branches/2.0/varnish-cache/lib/libvarnish/tcp.c =================================================================== --- branches/2.0/varnish-cache/lib/libvarnish/tcp.c 2009-09-28 09:24:38 UTC (rev 4247) +++ branches/2.0/varnish-cache/lib/libvarnish/tcp.c 2009-09-28 09:46:59 UTC (rev 4248) @@ -222,3 +222,17 @@ AZ(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout)); #endif } + +/*-------------------------------------------------------------------- + * Set or reset SO_LINGER flag + */ + +void +TCP_linger(int sock, int linger) +{ + struct linger lin; + + memset(&lin, 0, sizeof lin); + lin.l_onoff = linger; + AZ(setsockopt(sock, SOL_SOCKET, SO_LINGER, &lin, sizeof lin)); +} From tfheen at projects.linpro.no Mon Sep 28 09:56:02 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 11:56:02 +0200 (CEST) Subject: r4249 - in branches/2.0/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20090928095602.B26E71F7383@projects.linpro.no> Author: tfheen Date: 2009-09-28 11:56:02 +0200 (Mon, 28 Sep 2009) New Revision: 4249 Added: branches/2.0/varnish-cache/bin/varnishtest/tests/r00502.vtc Modified: branches/2.0/varnish-cache/bin/varnishd/cache_vrt.c Log: Merge r4048: Fix a parse error in VCL:purge() string version. Fixes #502 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_vrt.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_vrt.c 2009-09-28 09:46:59 UTC (rev 4248) +++ branches/2.0/varnish-cache/bin/varnishd/cache_vrt.c 2009-09-28 09:56:02 UTC (rev 4249) @@ -130,7 +130,9 @@ return (p); } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * XXX: Optimize the single element case ? + */ /*lint -e{818} ap,hp could be const */ static char * @@ -862,20 +864,25 @@ } b = BAN_New(); good = 0; - for (i = 1; ; i += 3) { - a1 = av[i]; + for (i = 1; ;) { + a1 = av[i++]; if (a1 == NULL) break; good = 0; - a2 = av[i + 1]; + a2 = av[i++]; if (a2 == NULL) break; - a3 = av[i + 2]; + a3 = av[i++]; if (a3 == NULL) break; if (BAN_AddTest(NULL, b, a1, a2, a3)) break; good = 1; + if (av[i] == NULL) + break; + good = 0; + if (strcmp(av[i++], "&&")) + break; } if (!good) /* XXX: report error how ? */ Copied: branches/2.0/varnish-cache/bin/varnishtest/tests/r00502.vtc (from rev 4048, trunk/varnish-cache/bin/varnishtest/tests/r00502.vtc) =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/r00502.vtc (rev 0) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/r00502.vtc 2009-09-28 09:56:02 UTC (rev 4249) @@ -0,0 +1,30 @@ +# $Id$ + +test "multi element purge" + +server s1 { + rxreq + txresp -hdr "foo: bar1" -body "1" + rxreq + txresp -hdr "foo: bar2" -body "22" +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + purge("req.url == / && obj.http.foo ~ bar1"); + } +} -start + +client c1 { + txreq + rxresp + expect resp.http.foo == "bar1" + txreq + rxresp + expect resp.http.foo == "bar2" + txreq + rxresp + expect resp.http.foo == "bar2" +} -run + +varnish v1 -cliok purge.list From tfheen at projects.linpro.no Mon Sep 28 10:13:48 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 12:13:48 +0200 (CEST) Subject: r4250 - in branches/2.0/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20090928101348.EF4B71F736B@projects.linpro.no> Author: tfheen Date: 2009-09-28 12:13:48 +0200 (Mon, 28 Sep 2009) New Revision: 4250 Added: branches/2.0/varnish-cache/bin/varnishtest/tests/r00506.vtc Modified: branches/2.0/varnish-cache/bin/varnishd/cache_http.c Log: Merge r4052, r4053: Be more paranoid about backend responses r4052: Be more paranoid about backend responses, a response of: HTTP/1.1 1000\n\r\n\r would panic us trying to find a suitable message for 1000. Now we 503 the response instead. Fixes #506 r4053: Regression test for #506 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_http.c 2009-09-28 09:56:02 UTC (rev 4249) +++ branches/2.0/varnish-cache/bin/varnishd/cache_http.c 2009-09-28 10:13:48 UTC (rev 4250) @@ -503,25 +503,35 @@ http_DissectResponse(struct worker *w, const struct http_conn *htc, struct http *hp) { - int i; + int i = 0; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); hp->logtag = HTTP_Rx; - i = http_splitline(w, htc->fd, hp, htc, - HTTP_HDR_PROTO, HTTP_HDR_STATUS, HTTP_HDR_RESPONSE); + if (http_splitline(w, htc->fd, hp, htc, + HTTP_HDR_PROTO, HTTP_HDR_STATUS, HTTP_HDR_RESPONSE)) + i = 503; - if (i != 0 || memcmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.", 7)) + if (i == 0 && memcmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.", 7)) + i = 503; + + if (i == 0 && Tlen(hp->hd[HTTP_HDR_STATUS]) != 3) + i = 503; + + if (i == 0) { + hp->status = strtoul(hp->hd[HTTP_HDR_STATUS].b, NULL, 10); + if (hp->status < 100 || hp->status > 999) + i = 503; + } + + if (i != 0) { WSLR(w, SLT_HttpGarbage, htc->fd, htc->rxbuf); - if (i != 0) { - if (hp->status == 0) - hp->status = i; + hp->status = i; } else { - hp->status = - strtoul(hp->hd[HTTP_HDR_STATUS].b, NULL /* XXX */, 10); + http_ProtoVer(hp); } - http_ProtoVer(hp); + if (hp->hd[HTTP_HDR_RESPONSE].b == NULL || !Tlen(hp->hd[HTTP_HDR_RESPONSE])) { /* Backend didn't send a response string, use the standard */ Copied: branches/2.0/varnish-cache/bin/varnishtest/tests/r00506.vtc (from rev 4053, trunk/varnish-cache/bin/varnishtest/tests/r00506.vtc) =================================================================== --- branches/2.0/varnish-cache/bin/varnishtest/tests/r00506.vtc (rev 0) +++ branches/2.0/varnish-cache/bin/varnishtest/tests/r00506.vtc 2009-09-28 10:13:48 UTC (rev 4250) @@ -0,0 +1,18 @@ +# $Id$ + +test "Illegal HTTP status from backend" + +server s1 { + rxreq + send "HTTP/1.1 1000\n\nFoo" +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + } +} -start + +client c1 { + txreq + rxresp +} -run From tfheen at projects.linpro.no Mon Sep 28 10:23:25 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 12:23:25 +0200 (CEST) Subject: r4251 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20090928102325.BB1731F7383@projects.linpro.no> Author: tfheen Date: 2009-09-28 12:23:25 +0200 (Mon, 28 Sep 2009) New Revision: 4251 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_panic.c Log: Merge r4054: stylistic nit Modified: branches/2.0/varnish-cache/bin/varnishd/cache_panic.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_panic.c 2009-09-28 10:13:48 UTC (rev 4250) +++ branches/2.0/varnish-cache/bin/varnishd/cache_panic.c 2009-09-28 10:23:25 UTC (rev 4251) @@ -63,7 +63,7 @@ vsb_printf(vsp, "%*sws = %p { %s\n", indent, "", ws, ws->overflow ? "overflow" : ""); vsb_printf(vsp, "%*sid = \"%s\",\n", indent + 2, "", ws->id); - vsb_printf(vsp, "%*s{s,f,r,e} = {%p,", indent + 2, "", ws->s); + vsb_printf(vsp, "%*s{s,f,r,e} = {%p", indent + 2, "", ws->s); if (ws->f > ws->s) vsb_printf(vsp, ",+%d", ws->f - ws->s); else From tfheen at projects.linpro.no Mon Sep 28 10:34:47 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 12:34:47 +0200 (CEST) Subject: r4252 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20090928103447.BF6151F7383@projects.linpro.no> Author: tfheen Date: 2009-09-28 12:34:47 +0200 (Mon, 28 Sep 2009) New Revision: 4252 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_center.c branches/2.0/varnish-cache/bin/varnishd/cache_hash.c Log: Merge r4055: Check that object has a ban before we unbusy it. Modified: branches/2.0/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_center.c 2009-09-28 10:23:25 UTC (rev 4251) +++ branches/2.0/varnish-cache/bin/varnishd/cache_center.c 2009-09-28 10:34:47 UTC (rev 4252) @@ -445,6 +445,7 @@ if (sp->obj->objhead != NULL) { VRY_Create(sp); EXP_Insert(sp->obj); + AN(sp->obj->ban); HSH_Unbusy(sp); } sp->acct_req.fetch++; Modified: branches/2.0/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_hash.c 2009-09-28 10:23:25 UTC (rev 4251) +++ branches/2.0/varnish-cache/bin/varnishd/cache_hash.c 2009-09-28 10:34:47 UTC (rev 4252) @@ -389,6 +389,7 @@ o = sp->obj; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); assert(o->busy); + AN(o->ban); assert(o->refcnt > 0); if (o->ws_o->overflow) VSL_stats->n_objoverflow++; From tfheen at projects.linpro.no Mon Sep 28 11:04:32 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 13:04:32 +0200 (CEST) Subject: r4253 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20090928110432.CE6942854A@projects.linpro.no> Author: tfheen Date: 2009-09-28 13:04:32 +0200 (Mon, 28 Sep 2009) New Revision: 4253 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_hash.c Log: Partially revert r4055/r4252: Not suitable for 2.0 The object will not have a ban yet in the 2.0 branch, so revert the cache_hash part of r4252. Modified: branches/2.0/varnish-cache/bin/varnishd/cache_hash.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_hash.c 2009-09-28 10:34:47 UTC (rev 4252) +++ branches/2.0/varnish-cache/bin/varnishd/cache_hash.c 2009-09-28 11:04:32 UTC (rev 4253) @@ -389,7 +389,6 @@ o = sp->obj; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); assert(o->busy); - AN(o->ban); assert(o->refcnt > 0); if (o->ws_o->overflow) VSL_stats->n_objoverflow++; From tfheen at projects.linpro.no Mon Sep 28 11:11:42 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 13:11:42 +0200 (CEST) Subject: r4254 - branches/2.0/varnish-cache/lib/libvarnishcompat Message-ID: <20090928111142.7C0B737F3F@projects.linpro.no> Author: tfheen Date: 2009-09-28 13:11:42 +0200 (Mon, 28 Sep 2009) New Revision: 4254 Modified: branches/2.0/varnish-cache/lib/libvarnishcompat/Makefile.am Log: Merge r4056: Add a compat version of backtrace()/backtrace_symbols() written by Maxim Sobolev Modified: branches/2.0/varnish-cache/lib/libvarnishcompat/Makefile.am =================================================================== --- branches/2.0/varnish-cache/lib/libvarnishcompat/Makefile.am 2009-09-28 11:04:32 UTC (rev 4253) +++ branches/2.0/varnish-cache/lib/libvarnishcompat/Makefile.am 2009-09-28 11:11:42 UTC (rev 4254) @@ -9,6 +9,7 @@ libvarnishcompat_la_SOURCES = \ asprintf.c \ daemon.c \ + execinfo.c \ vasprintf.c \ setproctitle.c \ srandomdev.c \ From tfheen at projects.linpro.no Mon Sep 28 11:46:03 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 13:46:03 +0200 (CEST) Subject: r4255 - branches/2.0/varnish-cache Message-ID: <20090928114603.9A0BB28546@projects.linpro.no> Author: tfheen Date: 2009-09-28 13:46:03 +0200 (Mon, 28 Sep 2009) New Revision: 4255 Modified: branches/2.0/varnish-cache/configure.ac Log: Merge r4057: Look for execinfo.h header Modified: branches/2.0/varnish-cache/configure.ac =================================================================== --- branches/2.0/varnish-cache/configure.ac 2009-09-28 11:11:42 UTC (rev 4254) +++ branches/2.0/varnish-cache/configure.ac 2009-09-28 11:46:03 UTC (rev 4255) @@ -81,6 +81,7 @@ AC_CHECK_HEADERS([sys/statvfs.h]) AC_CHECK_HEADERS([sys/vfs.h]) AC_CHECK_HEADERS([endian.h]) +AC_CHECK_HEADERS([execinfo.h]) AC_CHECK_HEADERS([netinet/in.h]) AC_CHECK_HEADERS([pthread_np.h]) AC_CHECK_HEADERS([stddef.h]) From tfheen at projects.linpro.no Mon Sep 28 12:32:39 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 14:32:39 +0200 (CEST) Subject: r4256 - in branches/2.0/varnish-cache: . bin/varnishd include include/compat lib/libvarnishcompat Message-ID: <20090928123239.55FEA37F3F@projects.linpro.no> Author: tfheen Date: 2009-09-28 14:32:39 +0200 (Mon, 28 Sep 2009) New Revision: 4256 Added: branches/2.0/varnish-cache/include/compat/execinfo.h branches/2.0/varnish-cache/lib/libvarnishcompat/execinfo.c Modified: branches/2.0/varnish-cache/bin/varnishd/cache_panic.c branches/2.0/varnish-cache/bin/varnishd/common.h branches/2.0/varnish-cache/bin/varnishd/varnishd.c branches/2.0/varnish-cache/configure.ac branches/2.0/varnish-cache/include/Makefile.am Log: Merge 4058-4065: Add a stack backtrace to the panic message. Note that to use this for full effect, varnishd should be installed unstripped. r4058: Also remember compat execinfo.h file. r4059: Tell autocrap about compat/execinfo.h r4060: Attempt to add a stack backtrace to the panic message. r4061: Spit out more information on panic r4062: Autocrap the dladdr() functions existence. r4063: More autocrappery. r4064: Comment out unneeded function. r4065: Pass a void* to Symbol_Lookup() to avoid stdint requirement on common.h Modified: branches/2.0/varnish-cache/bin/varnishd/cache_panic.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_panic.c 2009-09-28 11:46:03 UTC (rev 4255) +++ branches/2.0/varnish-cache/bin/varnishd/cache_panic.c 2009-09-28 12:32:39 UTC (rev 4256) @@ -37,6 +37,11 @@ #include #include +#ifndef HAVE_EXECINFO_H +#include "compat/execinfo.h" +#else +#include +#endif #include "cache.h" #include "cache_backend.h" #include "vcl.h" @@ -131,21 +136,21 @@ /*--------------------------------------------------------------------*/ static void -pan_http(const struct http *h) +pan_http(const char *id, const struct http *h, int indent) { int i; - vsb_printf(vsp, " http = {\n"); - pan_ws(h->ws, 6); - if (h->nhd > HTTP_HDR_FIRST) { - vsb_printf(vsp, " hd = {\n"); - for (i = HTTP_HDR_FIRST; i < h->nhd; ++i) - vsb_printf(vsp, " \"%.*s\",\n", - (int)(h->hd[i].e - h->hd[i].b), - h->hd[i].b); - vsb_printf(vsp, " },\n"); + vsb_printf(vsp, "%*shttp[%s] = {\n", indent, "", id); + vsb_printf(vsp, "%*sws = %p[%s]\n", indent + 2, "", + h->ws, h->ws ? h->ws->id : ""); + for (i = 0; i < h->nhd; ++i) { + if (h->hd[i].b == NULL && h->hd[i].e == NULL) + continue; + vsb_printf(vsp, "%*s\"%.*s\",\n", indent + 4, "", + (int)(h->hd[i].e - h->hd[i].b), + h->hd[i].b); } - vsb_printf(vsp, " },\n"); + vsb_printf(vsp, "%*s},\n", indent, ""); } @@ -159,7 +164,7 @@ vsb_printf(vsp, " obj = %p {\n", o); vsb_printf(vsp, " refcnt = %u, xid = %u,\n", o->refcnt, o->xid); pan_ws(o->ws_o, 4); - pan_http(o->http); + pan_http("obj", o->http, 4); vsb_printf(vsp, " len = %u,\n", o->len); vsb_printf(vsp, " store = {\n"); VTAILQ_FOREACH(st, &o->store, list) @@ -190,8 +195,7 @@ pan_wrk(const struct worker *wrk) { - vsb_printf(vsp, " worker = %p {\n", wrk); - vsb_printf(vsp, " },\n"); + vsb_printf(vsp, " worker = %p\n", wrk); } /*--------------------------------------------------------------------*/ @@ -229,7 +233,11 @@ " err_code = %d, err_reason = %s,\n", sp->err_code, sp->err_reason ? sp->err_reason : "(null)"); + vsb_printf(vsp, " restarts = %d, esis = %d\n", + sp->restarts, sp->esis); + pan_ws(sp->ws, 2); + pan_http("req", sp->http, 2); if (sp->wrk != NULL) pan_wrk(sp->wrk); @@ -249,6 +257,28 @@ /*--------------------------------------------------------------------*/ static void +pan_backtrace(void) +{ + void *array[10]; + size_t size; + size_t i; + + size = backtrace (array, 10); + vsb_printf(vsp, "Backtrace:\n"); + for (i = 0; i < size; i++) { + vsb_printf (vsp, " "); + if (Symbol_Lookup(vsp, array[i]) < 0) { + char **strings; + strings = backtrace_symbols(&array[i], 1); + vsb_printf(vsp, "%p: %s", array[i], strings[0]); + } + vsb_printf (vsp, "\n"); + } +} + +/*--------------------------------------------------------------------*/ + +static void pan_ic(const char *func, const char *file, int line, const char *cond, int err, int xxx) { @@ -264,7 +294,7 @@ break; case 2: vsb_printf(vsp, - "Panic from VCL:\n%s\n", cond); + "Panic from VCL:\n %s\n", cond); break; case 1: vsb_printf(vsp, @@ -276,16 +306,19 @@ case 0: vsb_printf(vsp, "Assert error in %s(), %s line %d:\n" - " Condition(%s) not true.", + " Condition(%s) not true.\n", func, file, line, cond); break; } if (err) - vsb_printf(vsp, " errno = %d (%s)", err, strerror(err)); + vsb_printf(vsp, "errno = %d (%s)\n", err, strerror(err)); q = THR_GetName(); if (q != NULL) - vsb_printf(vsp, " thread = (%s)", q); + vsb_printf(vsp, "thread = (%s)\n", q); + + pan_backtrace(); + if (!(params->diag_bitmap & 0x2000)) { sp = THR_GetSession(); if (sp != NULL) Modified: branches/2.0/varnish-cache/bin/varnishd/common.h =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/common.h 2009-09-28 11:46:03 UTC (rev 4255) +++ branches/2.0/varnish-cache/bin/varnishd/common.h 2009-09-28 12:32:39 UTC (rev 4256) @@ -42,6 +42,10 @@ void VSL_MgtInit(const char *fn, unsigned size); extern struct varnish_stats *VSL_stats; +/* varnishd.c */ +struct vsb; +int Symbol_Lookup(struct vsb *vsb, void *ptr); + #define TRUST_ME(ptr) ((void*)(uintptr_t)(ptr)) /* Really belongs in mgt.h, but storage_file chokes on both */ Modified: branches/2.0/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/varnishd.c 2009-09-28 11:46:03 UTC (rev 4255) +++ branches/2.0/varnish-cache/bin/varnishd/varnishd.c 2009-09-28 12:32:39 UTC (rev 4256) @@ -414,6 +414,92 @@ exit (2); } +/*-------------------------------------------------------------------- + * All praise POSIX! Thanks to our glorious standards there are no + * standard way to get a back-trace of the stack, and even if we hack + * that together from spit and pieces of string, there is no way no + * standard way to translate a pointer to a symbol, which returns anything + * usable. (See for instance FreeBSD PR-134391). + * + * Attempt to run nm(1) on our binary during startup, hoping it will + * give us a usable list of symbols. + */ + +struct symbols { + uintptr_t a; + char *n; + VTAILQ_ENTRY(symbols) list; +}; + +static VTAILQ_HEAD(,symbols) symbols = VTAILQ_HEAD_INITIALIZER(symbols); + +int +Symbol_Lookup(struct vsb *vsb, void *ptr) +{ + struct symbols *s, *s0; + uintptr_t pp; + + pp = (uintptr_t)ptr; + s0 = NULL; + VTAILQ_FOREACH(s, &symbols, list) { + if (s->a > pp) + continue; + if (s0 != NULL && s->a < s0->a) + continue; + s0 = s; + } + if (s0 == NULL) + return (-1); + vsb_printf(vsb, "%p", ptr); + if (s0 != NULL) + vsb_printf(vsb, ": %s+%jx", s0->n, (uintmax_t)pp - s0->a); + return (0); +} + +static void +Symbol_hack(const char *a0) +{ + char buf[BUFSIZ], *p, *e; + FILE *fi; + uintptr_t a; + struct symbols *s; + + strcpy(buf, "nm -an "); + strcat(buf, a0); + fi = popen(buf, "r"); + if (fi != NULL) { + while (fgets(buf, sizeof buf, fi)) { + if (buf[0] == ' ') + continue; + p = NULL; + a = strtoul(buf, &p, 16); + if (p == NULL) + continue; + if (a == 0) + continue; + if (*p++ != ' ') + continue; + p++; + if (*p++ != ' ') + continue; + if (*p <= ' ') + continue; + e = strchr(p, '\0'); + AN(e); + while (e > p && isspace(e[-1])) + e--; + *e = '\0'; + s = malloc(sizeof *s + strlen(p) + 1); + AN(s); + s->a = a; + s->n = (void*)(s + 1); + strcpy(s->n, p); + VTAILQ_INSERT_TAIL(&symbols, s, list); + } + pclose(fi); + } +} + /*--------------------------------------------------------------------*/ int @@ -443,6 +529,11 @@ setbuf(stdout, NULL); setbuf(stderr, NULL); + Symbol_hack(argv[0]); + + /* for ASSERT_MGT() */ + mgt_pid = getpid(); + /* * Run in UTC timezone, on the off-chance that this operating * system does not have a timegm() function, and translates Modified: branches/2.0/varnish-cache/configure.ac =================================================================== --- branches/2.0/varnish-cache/configure.ac 2009-09-28 11:46:03 UTC (rev 4255) +++ branches/2.0/varnish-cache/configure.ac 2009-09-28 12:32:39 UTC (rev 4256) @@ -103,6 +103,7 @@ AC_FUNC_VPRINTF AC_CHECK_FUNCS([strerror]) AC_FUNC_STRERROR_R +AC_CHECK_FUNCS([dladdr]) AC_CHECK_FUNCS([socket]) AC_CHECK_FUNCS([strptime]) AC_CHECK_FUNCS([fmtcheck]) Modified: branches/2.0/varnish-cache/include/Makefile.am =================================================================== --- branches/2.0/varnish-cache/include/Makefile.am 2009-09-28 11:46:03 UTC (rev 4255) +++ branches/2.0/varnish-cache/include/Makefile.am 2009-09-28 12:32:39 UTC (rev 4256) @@ -14,6 +14,7 @@ cli_priv.h \ compat/asprintf.h \ compat/daemon.h \ + compat/execinfo.h \ compat/setproctitle.h \ compat/srandomdev.h \ compat/strlcat.h \ Copied: branches/2.0/varnish-cache/include/compat/execinfo.h (from rev 4058, trunk/varnish-cache/include/compat/execinfo.h) =================================================================== --- branches/2.0/varnish-cache/include/compat/execinfo.h (rev 0) +++ branches/2.0/varnish-cache/include/compat/execinfo.h 2009-09-28 12:32:39 UTC (rev 4256) @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2003 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: execinfo.h,v 1.2 2004/07/19 05:20:29 sobomax Exp $ + */ + +#ifndef COMPAT_EXECINFO_H_INCLUDED +#define COMPAT_EXECINFO_H_INCLUDED + +#ifdef __cplusplus +extern "C" { +#endif + +int backtrace(void **, int); +char ** backtrace_symbols(void *const *, int); +void backtrace_symbols_fd(void *const *, int, int); + +#ifdef __cplusplus +} +#endif + +#endif /* COMPAT_EXECINFO_H_INCLUDED */ Added: branches/2.0/varnish-cache/lib/libvarnishcompat/execinfo.c =================================================================== --- branches/2.0/varnish-cache/lib/libvarnishcompat/execinfo.c (rev 0) +++ branches/2.0/varnish-cache/lib/libvarnishcompat/execinfo.c 2009-09-28 12:32:39 UTC (rev 4256) @@ -0,0 +1,451 @@ +/* + * Copyright (c) 2003 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: execinfo.c,v 1.3 2004/07/19 05:21:09 sobomax Exp $ + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "execinfo.h" + +void *getreturnaddr(int); +void *getframeaddr(int); + +#define D10(x) ceil(log10(((x) == 0) ? 2 : ((x) + 1))) + +inline static void * +realloc_safe(void *ptr, size_t size) +{ + void *nptr; + + nptr = realloc(ptr, size); + if (nptr == NULL) + free(ptr); + return nptr; +} + +int +backtrace(void **buffer, int size) +{ + int i; + + for (i = 1; getframeaddr(i + 1) != NULL && i != size + 1; i++) { + buffer[i - 1] = getreturnaddr(i); + if (buffer[i - 1] == NULL) + break; + } + + return i - 1; +} + +char ** +backtrace_symbols(void *const *buffer, int size) +{ + size_t clen, alen; + int i; + char **rval; + + clen = size * sizeof(char *); + rval = malloc(clen); + if (rval == NULL) + return NULL; + for (i = 0; i < size; i++) { + +#ifdef HAVE_DLADDR + { + Dl_info info; + int offset; + + if (dladdr(buffer[i], &info) != 0) { + if (info.dli_sname == NULL) + info.dli_sname = "???"; + if (info.dli_saddr == NULL) + info.dli_saddr = buffer[i]; + offset = (const char*)buffer[i] - (const char*)info.dli_saddr; + /* "0x01234567 at filename" */ + alen = 2 + /* "0x" */ + (sizeof(void *) * 2) + /* "01234567" */ + 2 + /* " <" */ + strlen(info.dli_sname) + /* "function" */ + 1 + /* "+" */ + 10 + /* "offset */ + 5 + /* "> at " */ + strlen(info.dli_fname) + /* "filename" */ + 1; /* "\0" */ + rval = realloc_safe(rval, clen + alen); + if (rval == NULL) + return NULL; + snprintf((char *) rval + clen, alen, "%p <%s+%d> at %s", + buffer[i], info.dli_sname, offset, info.dli_fname); + rval[i] = (char *) clen; + clen += alen; + continue; + } + } +#endif + alen = 2 + /* "0x" */ + (sizeof(void *) * 2) + /* "01234567" */ + 1; /* "\0" */ + rval = realloc_safe(rval, clen + alen); + if (rval == NULL) + return NULL; + snprintf((char *) rval + clen, alen, "%p", buffer[i]); + rval[i] = (char *) clen; + clen += alen; + } + + for (i = 0; i < size; i++) + rval[i] += (long) rval; + + return rval; +} + +#if 0 +void +backtrace_symbols_fd(void *const *buffer, int size, int fd) +{ + int i, len, offset; + char *buf; + Dl_info info; + + for (i = 0; i < size; i++) { + if (dladdr(buffer[i], &info) != 0) { + if (info.dli_sname == NULL) + info.dli_sname = "???"; + if (info.dli_saddr == NULL) + info.dli_saddr = buffer[i]; + offset = (const char *)buffer[i] - (const char *)info.dli_saddr; + /* "0x01234567 at filename" */ + len = 2 + /* "0x" */ + (sizeof(void *) * 2) + /* "01234567" */ + 2 + /* " <" */ + strlen(info.dli_sname) + /* "function" */ + 1 + /* "+" */ + D10(offset) + /* "offset */ + 5 + /* "> at " */ + strlen(info.dli_fname) + /* "filename" */ + 2; /* "\n\0" */ + buf = alloca(len); + if (buf == NULL) + return; + snprintf(buf, len, "%p <%s+%d> at %s\n", + buffer[i], info.dli_sname, offset, info.dli_fname); + } else { + len = 2 + /* "0x" */ + (sizeof(void *) * 2) + /* "01234567" */ + 2; /* "\n\0" */ + buf = alloca(len); + if (buf == NULL) + return; + snprintf(buf, len, "%p\n", buffer[i]); + } + write(fd, buf, len - 1); + } +} +#endif + +void * +getreturnaddr(int level) +{ + + switch(level) { + case 0: return __builtin_return_address(1); + case 1: return __builtin_return_address(2); + case 2: return __builtin_return_address(3); + case 3: return __builtin_return_address(4); + case 4: return __builtin_return_address(5); + case 5: return __builtin_return_address(6); + case 6: return __builtin_return_address(7); + case 7: return __builtin_return_address(8); + case 8: return __builtin_return_address(9); + case 9: return __builtin_return_address(10); + case 10: return __builtin_return_address(11); + case 11: return __builtin_return_address(12); + case 12: return __builtin_return_address(13); + case 13: return __builtin_return_address(14); + case 14: return __builtin_return_address(15); + case 15: return __builtin_return_address(16); + case 16: return __builtin_return_address(17); + case 17: return __builtin_return_address(18); + case 18: return __builtin_return_address(19); + case 19: return __builtin_return_address(20); + case 20: return __builtin_return_address(21); + case 21: return __builtin_return_address(22); + case 22: return __builtin_return_address(23); + case 23: return __builtin_return_address(24); + case 24: return __builtin_return_address(25); + case 25: return __builtin_return_address(26); + case 26: return __builtin_return_address(27); + case 27: return __builtin_return_address(28); + case 28: return __builtin_return_address(29); + case 29: return __builtin_return_address(30); + case 30: return __builtin_return_address(31); + case 31: return __builtin_return_address(32); + case 32: return __builtin_return_address(33); + case 33: return __builtin_return_address(34); + case 34: return __builtin_return_address(35); + case 35: return __builtin_return_address(36); + case 36: return __builtin_return_address(37); + case 37: return __builtin_return_address(38); + case 38: return __builtin_return_address(39); + case 39: return __builtin_return_address(40); + case 40: return __builtin_return_address(41); + case 41: return __builtin_return_address(42); + case 42: return __builtin_return_address(43); + case 43: return __builtin_return_address(44); + case 44: return __builtin_return_address(45); + case 45: return __builtin_return_address(46); + case 46: return __builtin_return_address(47); + case 47: return __builtin_return_address(48); + case 48: return __builtin_return_address(49); + case 49: return __builtin_return_address(50); + case 50: return __builtin_return_address(51); + case 51: return __builtin_return_address(52); + case 52: return __builtin_return_address(53); + case 53: return __builtin_return_address(54); + case 54: return __builtin_return_address(55); + case 55: return __builtin_return_address(56); + case 56: return __builtin_return_address(57); + case 57: return __builtin_return_address(58); + case 58: return __builtin_return_address(59); + case 59: return __builtin_return_address(60); + case 60: return __builtin_return_address(61); + case 61: return __builtin_return_address(62); + case 62: return __builtin_return_address(63); + case 63: return __builtin_return_address(64); + case 64: return __builtin_return_address(65); + case 65: return __builtin_return_address(66); + case 66: return __builtin_return_address(67); + case 67: return __builtin_return_address(68); + case 68: return __builtin_return_address(69); + case 69: return __builtin_return_address(70); + case 70: return __builtin_return_address(71); + case 71: return __builtin_return_address(72); + case 72: return __builtin_return_address(73); + case 73: return __builtin_return_address(74); + case 74: return __builtin_return_address(75); + case 75: return __builtin_return_address(76); + case 76: return __builtin_return_address(77); + case 77: return __builtin_return_address(78); + case 78: return __builtin_return_address(79); + case 79: return __builtin_return_address(80); + case 80: return __builtin_return_address(81); + case 81: return __builtin_return_address(82); + case 82: return __builtin_return_address(83); + case 83: return __builtin_return_address(84); + case 84: return __builtin_return_address(85); + case 85: return __builtin_return_address(86); + case 86: return __builtin_return_address(87); + case 87: return __builtin_return_address(88); + case 88: return __builtin_return_address(89); + case 89: return __builtin_return_address(90); + case 90: return __builtin_return_address(91); + case 91: return __builtin_return_address(92); + case 92: return __builtin_return_address(93); + case 93: return __builtin_return_address(94); + case 94: return __builtin_return_address(95); + case 95: return __builtin_return_address(96); + case 96: return __builtin_return_address(97); + case 97: return __builtin_return_address(98); + case 98: return __builtin_return_address(99); + case 99: return __builtin_return_address(100); + case 100: return __builtin_return_address(101); + case 101: return __builtin_return_address(102); + case 102: return __builtin_return_address(103); + case 103: return __builtin_return_address(104); + case 104: return __builtin_return_address(105); + case 105: return __builtin_return_address(106); + case 106: return __builtin_return_address(107); + case 107: return __builtin_return_address(108); + case 108: return __builtin_return_address(109); + case 109: return __builtin_return_address(110); + case 110: return __builtin_return_address(111); + case 111: return __builtin_return_address(112); + case 112: return __builtin_return_address(113); + case 113: return __builtin_return_address(114); + case 114: return __builtin_return_address(115); + case 115: return __builtin_return_address(116); + case 116: return __builtin_return_address(117); + case 117: return __builtin_return_address(118); + case 118: return __builtin_return_address(119); + case 119: return __builtin_return_address(120); + case 120: return __builtin_return_address(121); + case 121: return __builtin_return_address(122); + case 122: return __builtin_return_address(123); + case 123: return __builtin_return_address(124); + case 124: return __builtin_return_address(125); + case 125: return __builtin_return_address(126); + case 126: return __builtin_return_address(127); + case 127: return __builtin_return_address(128); + default: return NULL; + } +} + +void * +getframeaddr(int level) +{ + + switch(level) { + case 0: return __builtin_frame_address(1); + case 1: return __builtin_frame_address(2); + case 2: return __builtin_frame_address(3); + case 3: return __builtin_frame_address(4); + case 4: return __builtin_frame_address(5); + case 5: return __builtin_frame_address(6); + case 6: return __builtin_frame_address(7); + case 7: return __builtin_frame_address(8); + case 8: return __builtin_frame_address(9); + case 9: return __builtin_frame_address(10); + case 10: return __builtin_frame_address(11); + case 11: return __builtin_frame_address(12); + case 12: return __builtin_frame_address(13); + case 13: return __builtin_frame_address(14); + case 14: return __builtin_frame_address(15); + case 15: return __builtin_frame_address(16); + case 16: return __builtin_frame_address(17); + case 17: return __builtin_frame_address(18); + case 18: return __builtin_frame_address(19); + case 19: return __builtin_frame_address(20); + case 20: return __builtin_frame_address(21); + case 21: return __builtin_frame_address(22); + case 22: return __builtin_frame_address(23); + case 23: return __builtin_frame_address(24); + case 24: return __builtin_frame_address(25); + case 25: return __builtin_frame_address(26); + case 26: return __builtin_frame_address(27); + case 27: return __builtin_frame_address(28); + case 28: return __builtin_frame_address(29); + case 29: return __builtin_frame_address(30); + case 30: return __builtin_frame_address(31); + case 31: return __builtin_frame_address(32); + case 32: return __builtin_frame_address(33); + case 33: return __builtin_frame_address(34); + case 34: return __builtin_frame_address(35); + case 35: return __builtin_frame_address(36); + case 36: return __builtin_frame_address(37); + case 37: return __builtin_frame_address(38); + case 38: return __builtin_frame_address(39); + case 39: return __builtin_frame_address(40); + case 40: return __builtin_frame_address(41); + case 41: return __builtin_frame_address(42); + case 42: return __builtin_frame_address(43); + case 43: return __builtin_frame_address(44); + case 44: return __builtin_frame_address(45); + case 45: return __builtin_frame_address(46); + case 46: return __builtin_frame_address(47); + case 47: return __builtin_frame_address(48); + case 48: return __builtin_frame_address(49); + case 49: return __builtin_frame_address(50); + case 50: return __builtin_frame_address(51); + case 51: return __builtin_frame_address(52); + case 52: return __builtin_frame_address(53); + case 53: return __builtin_frame_address(54); + case 54: return __builtin_frame_address(55); + case 55: return __builtin_frame_address(56); + case 56: return __builtin_frame_address(57); + case 57: return __builtin_frame_address(58); + case 58: return __builtin_frame_address(59); + case 59: return __builtin_frame_address(60); + case 60: return __builtin_frame_address(61); + case 61: return __builtin_frame_address(62); + case 62: return __builtin_frame_address(63); + case 63: return __builtin_frame_address(64); + case 64: return __builtin_frame_address(65); + case 65: return __builtin_frame_address(66); + case 66: return __builtin_frame_address(67); + case 67: return __builtin_frame_address(68); + case 68: return __builtin_frame_address(69); + case 69: return __builtin_frame_address(70); + case 70: return __builtin_frame_address(71); + case 71: return __builtin_frame_address(72); + case 72: return __builtin_frame_address(73); + case 73: return __builtin_frame_address(74); + case 74: return __builtin_frame_address(75); + case 75: return __builtin_frame_address(76); + case 76: return __builtin_frame_address(77); + case 77: return __builtin_frame_address(78); + case 78: return __builtin_frame_address(79); + case 79: return __builtin_frame_address(80); + case 80: return __builtin_frame_address(81); + case 81: return __builtin_frame_address(82); + case 82: return __builtin_frame_address(83); + case 83: return __builtin_frame_address(84); + case 84: return __builtin_frame_address(85); + case 85: return __builtin_frame_address(86); + case 86: return __builtin_frame_address(87); + case 87: return __builtin_frame_address(88); + case 88: return __builtin_frame_address(89); + case 89: return __builtin_frame_address(90); + case 90: return __builtin_frame_address(91); + case 91: return __builtin_frame_address(92); + case 92: return __builtin_frame_address(93); + case 93: return __builtin_frame_address(94); + case 94: return __builtin_frame_address(95); + case 95: return __builtin_frame_address(96); + case 96: return __builtin_frame_address(97); + case 97: return __builtin_frame_address(98); + case 98: return __builtin_frame_address(99); + case 99: return __builtin_frame_address(100); + case 100: return __builtin_frame_address(101); + case 101: return __builtin_frame_address(102); + case 102: return __builtin_frame_address(103); + case 103: return __builtin_frame_address(104); + case 104: return __builtin_frame_address(105); + case 105: return __builtin_frame_address(106); + case 106: return __builtin_frame_address(107); + case 107: return __builtin_frame_address(108); + case 108: return __builtin_frame_address(109); + case 109: return __builtin_frame_address(110); + case 110: return __builtin_frame_address(111); + case 111: return __builtin_frame_address(112); + case 112: return __builtin_frame_address(113); + case 113: return __builtin_frame_address(114); + case 114: return __builtin_frame_address(115); + case 115: return __builtin_frame_address(116); + case 116: return __builtin_frame_address(117); + case 117: return __builtin_frame_address(118); + case 118: return __builtin_frame_address(119); + case 119: return __builtin_frame_address(120); + case 120: return __builtin_frame_address(121); + case 121: return __builtin_frame_address(122); + case 122: return __builtin_frame_address(123); + case 123: return __builtin_frame_address(124); + case 124: return __builtin_frame_address(125); + case 125: return __builtin_frame_address(126); + case 126: return __builtin_frame_address(127); + case 127: return __builtin_frame_address(128); + default: return NULL; + } +} From tfheen at projects.linpro.no Mon Sep 28 14:00:24 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 16:00:24 +0200 (CEST) Subject: r4257 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20090928140024.22973386F1@projects.linpro.no> Author: tfheen Date: 2009-09-28 16:00:23 +0200 (Mon, 28 Sep 2009) New Revision: 4257 Modified: branches/2.0/varnish-cache/bin/varnishd/cache.h branches/2.0/varnish-cache/bin/varnishd/cache_expire.c branches/2.0/varnish-cache/bin/varnishd/cache_pool.c Log: Merge r3905: Small facility for background threads Add a miniature facility for starting a background thread with its own session and worker strutures all set up and ready. Use this for the cache-timeout thread, instead of home-rolling it. Modified: branches/2.0/varnish-cache/bin/varnishd/cache.h =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache.h 2009-09-28 12:32:39 UTC (rev 4256) +++ branches/2.0/varnish-cache/bin/varnishd/cache.h 2009-09-28 14:00:23 UTC (rev 4257) @@ -543,6 +543,9 @@ void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len); #endif /* SENDFILE_WORKS */ +typedef void *bgthread_t(struct sess *); +void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func); + /* cache_session.c [SES] */ void SES_Init(void); struct sess *SES_New(const struct sockaddr *addr, unsigned len); Modified: branches/2.0/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_expire.c 2009-09-28 12:32:39 UTC (rev 4256) +++ branches/2.0/varnish-cache/bin/varnishd/cache_expire.c 2009-09-28 14:00:23 UTC (rev 4257) @@ -275,25 +275,12 @@ */ static void * -exp_timer(void *arg) +exp_timer(struct sess *sp) { - struct worker ww; struct objexp *oe; struct object *o; double t; - struct sess *sp; - unsigned char logbuf[1024]; /* XXX size ? */ - THR_SetName("cache-timeout"); - (void)arg; - - sp = SES_New(NULL, 0); - XXXAN(sp); - sp->wrk = &ww; - ww.magic = WORKER_MAGIC; - ww.wlp = ww.wlb = logbuf; - ww.wle = logbuf + sizeof logbuf; - AZ(sleep(10)); /* XXX: Takes time for VCL to arrive */ VCL_Get(&sp->vcl); t = TIM_real(); @@ -303,7 +290,7 @@ CHECK_OBJ_ORNULL(oe, OBJEXP_MAGIC); if (oe == NULL || oe->timer_when > t) { /* XXX: > or >= ? */ Lck_Unlock(&exp_mtx); - WSL_Flush(&ww, 0); + WSL_Flush(sp->wrk, 0); AZ(sleep(1)); VCL_Refresh(&sp->vcl); t = TIM_real(); @@ -327,7 +314,7 @@ assert(oe->on_lru); Lck_Unlock(&exp_mtx); - WSL(&ww, SLT_ExpPick, 0, "%u %s", o->xid, oe->timer_what); + WSL(sp->wrk, SLT_ExpPick, 0, "%u %s", o->xid, oe->timer_what); if (oe->timer_what == tmr_prefetch) { o->prefetch = 0.0; @@ -335,7 +322,7 @@ VCL_prefetch_method(sp); sp->obj = NULL; if (sp->handling == VCL_RET_FETCH) { - WSL(&ww, SLT_Debug, 0, "Attempt Prefetch %u", + WSL(sp->wrk, SLT_Debug, 0, "Attempt Prefetch %u", o->xid); } Lck_Lock(&exp_mtx); @@ -351,7 +338,7 @@ sp->obj = NULL; assert(sp->handling == VCL_RET_DISCARD); - WSL(&ww, SLT_ExpKill, 0, + WSL(sp->wrk, SLT_ExpKill, 0, "%u %d", o->xid, (int)(o->ttl - t)); Lck_Lock(&exp_mtx); assert(oe->timer_idx == BINHEAP_NOIDX); @@ -481,5 +468,5 @@ Lck_New(&exp_mtx); exp_heap = binheap_new(NULL, object_cmp, object_update); XXXAN(exp_heap); - AZ(pthread_create(&exp_thread, NULL, exp_timer, NULL)); + WRK_BgThread(&exp_thread, "cache-timeout", exp_timer); } Modified: branches/2.0/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_pool.c 2009-09-28 12:32:39 UTC (rev 4256) +++ branches/2.0/varnish-cache/bin/varnishd/cache_pool.c 2009-09-28 14:00:23 UTC (rev 4257) @@ -618,6 +618,56 @@ } } +/*-------------------------------------------------------------------- + * Create and starte a back-ground thread which as its own worker and + * session data structures; + */ + +struct bgthread { + unsigned magic; +#define BGTHREAD_MAGIC 0x23b5152b + const char *name; + bgthread_t *func; +}; + +static void * +wrk_bgthread(void *arg) +{ + struct bgthread *bt; + struct worker ww; + struct sess *sp; + unsigned char logbuf[1024]; /* XXX: size ? */ + + CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC); + THR_SetName(bt->name); + sp = SES_New(NULL, 0); + XXXAN(sp); + memset(&ww, 0, sizeof ww); + sp->wrk = &ww; + ww.magic = WORKER_MAGIC; + ww.wlp = ww.wlb = logbuf; + ww.wle = logbuf + sizeof logbuf; + + (void)bt->func(sp); + + WRONG("BgThread terminated"); + + return (NULL); +} + +void +WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func) +{ + struct bgthread *bt; + + ALLOC_OBJ(bt, BGTHREAD_MAGIC); + AN(bt); + + bt->name = name; + bt->func = func; + AZ(pthread_create(thr, NULL, wrk_bgthread, bt)); +} + /*--------------------------------------------------------------------*/ void From tfheen at projects.linpro.no Mon Sep 28 14:05:51 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 16:05:51 +0200 (CEST) Subject: r4258 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20090928140551.8F99F386F1@projects.linpro.no> Author: tfheen Date: 2009-09-28 16:05:51 +0200 (Mon, 28 Sep 2009) New Revision: 4258 Modified: branches/2.0/varnish-cache/bin/varnishd/cache.h branches/2.0/varnish-cache/bin/varnishd/cache_expire.c branches/2.0/varnish-cache/bin/varnishd/cache_pool.c Log: Merge r3906: Give WRK_BgThread() a "void *priv" argument also. Modified: branches/2.0/varnish-cache/bin/varnishd/cache.h =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache.h 2009-09-28 14:00:23 UTC (rev 4257) +++ branches/2.0/varnish-cache/bin/varnishd/cache.h 2009-09-28 14:05:51 UTC (rev 4258) @@ -543,8 +543,8 @@ void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len); #endif /* SENDFILE_WORKS */ -typedef void *bgthread_t(struct sess *); -void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func); +typedef void *bgthread_t(struct sess *, void *priv); +void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv); /* cache_session.c [SES] */ void SES_Init(void); Modified: branches/2.0/varnish-cache/bin/varnishd/cache_expire.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_expire.c 2009-09-28 14:00:23 UTC (rev 4257) +++ branches/2.0/varnish-cache/bin/varnishd/cache_expire.c 2009-09-28 14:05:51 UTC (rev 4258) @@ -275,12 +275,13 @@ */ static void * -exp_timer(struct sess *sp) +exp_timer(struct sess *sp, void *priv) { struct objexp *oe; struct object *o; double t; + (void)priv; AZ(sleep(10)); /* XXX: Takes time for VCL to arrive */ VCL_Get(&sp->vcl); t = TIM_real(); @@ -468,5 +469,5 @@ Lck_New(&exp_mtx); exp_heap = binheap_new(NULL, object_cmp, object_update); XXXAN(exp_heap); - WRK_BgThread(&exp_thread, "cache-timeout", exp_timer); + WRK_BgThread(&exp_thread, "cache-timeout", exp_timer, NULL); } Modified: branches/2.0/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_pool.c 2009-09-28 14:00:23 UTC (rev 4257) +++ branches/2.0/varnish-cache/bin/varnishd/cache_pool.c 2009-09-28 14:05:51 UTC (rev 4258) @@ -628,6 +628,7 @@ #define BGTHREAD_MAGIC 0x23b5152b const char *name; bgthread_t *func; + void *priv; }; static void * @@ -648,7 +649,7 @@ ww.wlp = ww.wlb = logbuf; ww.wle = logbuf + sizeof logbuf; - (void)bt->func(sp); + (void)bt->func(sp, bt->priv); WRONG("BgThread terminated"); @@ -656,7 +657,7 @@ } void -WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func) +WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv) { struct bgthread *bt; @@ -665,6 +666,7 @@ bt->name = name; bt->func = func; + bt->priv = priv; AZ(pthread_create(thr, NULL, wrk_bgthread, bt)); } From tfheen at projects.linpro.no Mon Sep 28 14:11:08 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 16:11:08 +0200 (CEST) Subject: r4259 - branches/2.0/varnish-cache/bin/varnishd Message-ID: <20090928141108.C339528535@projects.linpro.no> Author: tfheen Date: 2009-09-28 16:11:08 +0200 (Mon, 28 Sep 2009) New Revision: 4259 Modified: branches/2.0/varnish-cache/bin/varnishd/cache.h branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c branches/2.0/varnish-cache/bin/varnishd/cache_pool.c branches/2.0/varnish-cache/bin/varnishd/cache_session.c branches/2.0/varnish-cache/bin/varnishd/cache_ws.c Log: Merge r4069: Factor out session allocation for background threads Fix an inconsequential oversight in session management, and prevent it from happening again: We optimize session allocation, SES_New(), to minimize the amount of locking the VCA_thread participates in with flip-flop lists of free sessions: Allocate from one list, free to the other, flip lists only when necessary. This scheme only works if nobody but VCA_thread calls SES_New(). Background worker threads need dummy sessions and thus called into SES_New() as well. These calls all happen during startup, so they do in fact not mess up the locking, but made mockery of a number of very stern comments. Add SES_Alloc() for such "other uses" and enforce VCA_thread monopoly with an assert. Modified: branches/2.0/varnish-cache/bin/varnishd/cache.h =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache.h 2009-09-28 14:05:51 UTC (rev 4258) +++ branches/2.0/varnish-cache/bin/varnishd/cache.h 2009-09-28 14:11:08 UTC (rev 4259) @@ -397,6 +397,7 @@ void vca_close_session(struct sess *sp, const char *why); void VCA_Prep(struct sess *sp); void VCA_Init(void); +extern pthread_t VCA_thread; /* cache_backend.c */ @@ -549,6 +550,7 @@ /* cache_session.c [SES] */ void SES_Init(void); struct sess *SES_New(const struct sockaddr *addr, unsigned len); +struct sess *SES_Alloc(const struct sockaddr *addr, unsigned len); void SES_Delete(struct sess *sp); void SES_RefSrcAddr(struct sess *sp); void SES_Charge(struct sess *sp); Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c 2009-09-28 14:05:51 UTC (rev 4258) +++ branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c 2009-09-28 14:11:08 UTC (rev 4259) @@ -65,7 +65,7 @@ static struct acceptor const *vca_act; -static pthread_t vca_thread_acct; +pthread_t VCA_thread; static struct timeval tv_sndtimeo; static struct timeval tv_rcvtimeo; static const struct linger linger = { @@ -332,7 +332,7 @@ if (vca_act->pass == NULL) AZ(pipe(vca_pipes)); vca_act->init(); - AZ(pthread_create(&vca_thread_acct, NULL, vca_acct, NULL)); + AZ(pthread_create(&VCA_thread, NULL, vca_acct, NULL)); VSL(SLT_Debug, 0, "Acceptor is %s", vca_act->name); } Modified: branches/2.0/varnish-cache/bin/varnishd/cache_pool.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_pool.c 2009-09-28 14:05:51 UTC (rev 4258) +++ branches/2.0/varnish-cache/bin/varnishd/cache_pool.c 2009-09-28 14:11:08 UTC (rev 4259) @@ -641,7 +641,7 @@ CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC); THR_SetName(bt->name); - sp = SES_New(NULL, 0); + sp = SES_Alloc(NULL, 0); XXXAN(sp); memset(&ww, 0, sizeof ww); sp->wrk = &ww; Modified: branches/2.0/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_session.c 2009-09-28 14:05:51 UTC (rev 4258) +++ branches/2.0/varnish-cache/bin/varnishd/cache_session.c 2009-09-28 14:11:08 UTC (rev 4259) @@ -249,35 +249,14 @@ /*--------------------------------------------------------------------*/ -struct sess * -SES_New(const struct sockaddr *addr, unsigned len) +static struct sess * +ses_setup(struct sessmem *sm, const struct sockaddr *addr, unsigned len) { - struct sessmem *sm; struct sess *sp; volatile unsigned u; - /* - * One of the two queues is unlocked because only one - * thread ever gets here to empty it. - */ - assert(ses_qp <= 1); - sm = VTAILQ_FIRST(&ses_free_mem[ses_qp]); if (sm == NULL) { /* - * If that queue is empty, flip queues holding the lock - * and try the new unlocked queue. - */ - Lck_Lock(&ses_mem_mtx); - ses_qp = 1 - ses_qp; - Lck_Unlock(&ses_mem_mtx); - sm = VTAILQ_FIRST(&ses_free_mem[ses_qp]); - } - if (sm != NULL) { - VTAILQ_REMOVE(&ses_free_mem[ses_qp], sm, list); - } else { - /* - * If that fails, alloc new one. - * * It is not necessary to lock mem_workspace, but we * need to cache it locally, to make sure we get a * consistent view of it. @@ -323,6 +302,43 @@ return (sp); } +/*-------------------------------------------------------------------- + * Try to recycle an existing session. + */ + +struct sess * +SES_New(const struct sockaddr *addr, unsigned len) +{ + struct sessmem *sm; + + assert(pthread_self() == VCA_thread); + assert(ses_qp <= 1); + sm = VTAILQ_FIRST(&ses_free_mem[ses_qp]); + if (sm == NULL) { + /* + * If that queue is empty, flip queues holding the lock + * and try the new unlocked queue. + */ + Lck_Lock(&ses_mem_mtx); + ses_qp = 1 - ses_qp; + Lck_Unlock(&ses_mem_mtx); + sm = VTAILQ_FIRST(&ses_free_mem[ses_qp]); + } + if (sm != NULL) + VTAILQ_REMOVE(&ses_free_mem[ses_qp], sm, list); + return (ses_setup(sm, addr, len)); +} + +/*--------------------------------------------------------------------*/ + +struct sess * +SES_Alloc(const struct sockaddr *addr, unsigned len) +{ + return (ses_setup(NULL, addr, len)); +} + +/*--------------------------------------------------------------------*/ + void SES_Delete(struct sess *sp) { Modified: branches/2.0/varnish-cache/bin/varnishd/cache_ws.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_ws.c 2009-09-28 14:05:51 UTC (rev 4258) +++ branches/2.0/varnish-cache/bin/varnishd/cache_ws.c 2009-09-28 14:11:08 UTC (rev 4259) @@ -82,6 +82,10 @@ WS_Assert(ws); } +/* + * Reset a WS to start or a given pointer, likely from WS_Snapshot + */ + void WS_Reset(struct ws *ws, char *p) { From tfheen at projects.linpro.no Mon Sep 28 14:20:13 2009 From: tfheen at projects.linpro.no (tfheen at projects.linpro.no) Date: Mon, 28 Sep 2009 16:20:13 +0200 (CEST) Subject: r4260 - in branches/2.0/varnish-cache: bin/varnishd include Message-ID: <20090928142013.3105E28546@projects.linpro.no> Author: tfheen Date: 2009-09-28 16:20:12 +0200 (Mon, 28 Sep 2009) New Revision: 4260 Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c branches/2.0/varnish-cache/bin/varnishd/cache_session.c branches/2.0/varnish-cache/bin/varnishd/heritage.h branches/2.0/varnish-cache/bin/varnishd/mgt_param.c branches/2.0/varnish-cache/include/stat_field.h Log: Merge r4070: Limit max number of sessions Put an upper limit on how many sessions we will allocate before we just drop new connections summarily. The parameter is session_max, default is 100k and the stats variable client_drop counts how many sessions were dropped. This is mostly an anti-DoS measure and your feedback and experience with it is most welcome. Modified: branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c 2009-09-28 14:11:08 UTC (rev 4259) +++ branches/2.0/varnish-cache/bin/varnishd/cache_acceptor.c 2009-09-28 14:20:12 UTC (rev 4260) @@ -241,8 +241,11 @@ continue; } sp = SES_New(addr, l); - XXXAN(sp); - + if (sp == NULL) { + AZ(close(i)); + VSL_stats->client_drop++; + continue; + } sp->fd = i; sp->id = i; sp->t_open = now; Modified: branches/2.0/varnish-cache/bin/varnishd/cache_session.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/cache_session.c 2009-09-28 14:11:08 UTC (rev 4259) +++ branches/2.0/varnish-cache/bin/varnishd/cache_session.c 2009-09-28 14:20:12 UTC (rev 4260) @@ -256,6 +256,8 @@ volatile unsigned u; if (sm == NULL) { + if (VSL_stats->n_sess_mem >= params->max_sess) + return (NULL); /* * It is not necessary to lock mem_workspace, but we * need to cache it locally, to make sure we get a Modified: branches/2.0/varnish-cache/bin/varnishd/heritage.h =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/heritage.h 2009-09-28 14:11:08 UTC (rev 4259) +++ branches/2.0/varnish-cache/bin/varnishd/heritage.h 2009-09-28 14:20:12 UTC (rev 4260) @@ -82,6 +82,9 @@ /* TTL used for synthesized error pages */ unsigned err_ttl; + /* Maximum concurrent sessions */ + unsigned max_sess; + /* Worker threads and pool */ unsigned wthread_min; unsigned wthread_max; Modified: branches/2.0/varnish-cache/bin/varnishd/mgt_param.c =================================================================== --- branches/2.0/varnish-cache/bin/varnishd/mgt_param.c 2009-09-28 14:11:08 UTC (rev 4259) +++ branches/2.0/varnish-cache/bin/varnishd/mgt_param.c 2009-09-28 14:20:12 UTC (rev 4260) @@ -692,6 +692,15 @@ "have both IPv4 and IPv6 addresses.", 0, "off", "bool" }, + { "session_max", tweak_uint, + &master.max_sess, 1000, UINT_MAX, + "Maximum number of sessions we will allocate " + "before just dropping connections.\n" + "This is mostly an anti-DoS measure, and setting it plenty " + "high should not hurt, as long as you have the memory for " + "it.\n", + 0, + "100000", "sessions" }, { "session_linger", tweak_uint, &master.session_linger,0, UINT_MAX, "How long time the workerthread lingers on the session " Modified: branches/2.0/varnish-cache/include/stat_field.h =================================================================== --- branches/2.0/varnish-cache/include/stat_field.h 2009-09-28 14:11:08 UTC (rev 4259) +++ branches/2.0/varnish-cache/include/stat_field.h 2009-09-28 14:20:12 UTC (rev 4260) @@ -30,6 +30,7 @@ */ MAC_STAT(client_conn, uint64_t, 'a', "Client connections accepted") +MAC_STAT(client_drop, uint64_t, 'a', "Connection dropped, no sess") MAC_STAT(client_req, uint64_t, 'a', "Client requests received") MAC_STAT(cache_hit, uint64_t, 'a', "Cache hits") From phk at projects.linpro.no Tue Sep 29 12:16:03 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 29 Sep 2009 14:16:03 +0200 (CEST) Subject: r4261 - trunk/varnish-cache/bin/varnishd Message-ID: <20090929121603.F001D1F736B@projects.linpro.no> Author: phk Date: 2009-09-29 14:16:03 +0200 (Tue, 29 Sep 2009) New Revision: 4261 Modified: trunk/varnish-cache/bin/varnishd/varnishd.c Log: Unbreak the symbol-hack Modified: trunk/varnish-cache/bin/varnishd/varnishd.c =================================================================== --- trunk/varnish-cache/bin/varnishd/varnishd.c 2009-09-28 14:20:12 UTC (rev 4260) +++ trunk/varnish-cache/bin/varnishd/varnishd.c 2009-09-29 12:16:03 UTC (rev 4261) @@ -418,7 +418,7 @@ VTAILQ_FOREACH(s, &symbols, list) { if (s->a > pp) continue; - if (s0 == NULL || s->a < s0->a) + if (s0 == NULL || s->a > s0->a) s0 = s; } if (s0 == NULL) @@ -439,7 +439,7 @@ asprintf(&p, "nm -an %s", a0); if (p == NULL) return; - fi = popen(buf, "r"); + fi = popen(p, "r"); free(p); if (fi == NULL) return; From phk at projects.linpro.no Tue Sep 29 12:25:21 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 29 Sep 2009 14:25:21 +0200 (CEST) Subject: r4262 - trunk/varnish-cache/bin/varnishd Message-ID: <20090929122521.D62001F736D@projects.linpro.no> Author: phk Date: 2009-09-29 14:25:21 +0200 (Tue, 29 Sep 2009) New Revision: 4262 Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c Log: Add the free_reserve and wrap-around code. We still do not have the cleaner, so when we run out of space we simply exit the worker process with a message, the restart will blow away some segments and thus make some space for us. Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c =================================================================== --- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-09-29 12:16:03 UTC (rev 4261) +++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-09-29 12:25:21 UTC (rev 4262) @@ -131,6 +131,8 @@ struct smp_seghead segments; struct smp_seg *cur_seg; + uint64_t free_offset; + uint64_t objreserv; pthread_t thread; @@ -157,6 +159,8 @@ uint64_t min_segl; uint64_t aim_segl; uint64_t max_segl; + + uint64_t free_reserve; }; /* @@ -492,6 +496,13 @@ sc->aim_nobj = sc->max_segl / 4000; fprintf(stderr, "aim_nobj = %u\n", sc->aim_nobj); + + /* + * How much space in the free reserve pool ? + */ + sc->free_reserve = sc->aim_segl * 10; + + fprintf(stderr, "free_reserve = %ju\n", sc->free_reserve); } /*-------------------------------------------------------------------- @@ -587,6 +598,7 @@ assert(sg->offset + sg->length <= sc->mediasize); ss->offset = sg->offset; ss->length = sg->length; +// printf("SAVE_SEG %jx...%jx\n", ss->offset, ss->offset + ss->length); ss++; length += sizeof *ss; } @@ -850,30 +862,115 @@ static int smp_open_segs(struct smp_sc *sc, struct smp_signctx *ctx) { - uint64_t length; - struct smp_segptr *ss; - struct smp_seg *sg; - int i; + uint64_t length, l; + struct smp_segptr *ss, *se; + struct smp_seg *sg, *sg1, *sg2; + int i, n = 0; ASSERT_CLI(); i = smp_chk_sign(ctx); if (i) return (i); + ss = SIGN_DATA(ctx); length = ctx->ss->length; - for(; length > 0; length -= sizeof *ss, ss ++) { + if (length == 0) { + /* No segments */ + sc->free_offset = sc->ident->stuff[SMP_SPC_STUFF]; + return (0); + } + se = ss + length / sizeof *ss; + se--; + assert(ss <= se); + + /* + * Locate the free reserve, there are only two basic cases, + * but once we start dropping segments, things gets more complicated. + */ + + sc->free_offset = se->offset + se->length; + l = sc->mediasize - sc->free_offset; + if (se->offset > ss->offset && l >= sc->free_reserve) { + /* + * [__xxxxyyyyzzzz___] + * Plenty of space at tail, do nothing. + */ +//printf("TRS: %jx @ %jx\n", l, sc->free_offset); + } else if (ss->offset > se->offset) { + /* + * [zzzz____xxxxyyyy_] + * (make) space between ends + * We might nuke the entire tail end without getting + * enough space, in which case we fall through to the + * last check. + */ + while (ss < se && ss->offset > se->offset) { +//printf("TEST_SEG1 %jx...%jx\n", ss->offset, ss->offset + ss->length); + l = ss->offset - (se->offset + se->length); + if (l > sc->free_reserve) + break; +//printf("DROP_SEG1 %jx...%jx\n", ss->offset, ss->offset + ss->length); + ss++; + n++; + } + } + + if (l < sc->free_reserve) { + /* + * [__xxxxyyyyzzzz___] + * (make) space at front + */ + sc->free_offset = sc->ident->stuff[SMP_SPC_STUFF]; + while (ss < se) { +//printf("TEST_SEG2 %jx...%jx\n", ss->offset, ss->offset + ss->length); + l = ss->offset - sc->free_offset; + if (l > sc->free_reserve) + break; +//printf("DROP_SEG2 %jx...%jx\n", ss->offset, ss->offset + ss->length); + ss++; + n++; + } + } + + assert (l >= sc->free_reserve); + +//printf("FRS: %jx @ %jx\n", l, sc->free_offset); + + sg1 = NULL; + sg2 = NULL; + for(; ss <= se; ss++) { +// printf("LOAD_SEG %jx...%jx\n", ss->offset, ss->offset + ss->length); ALLOC_OBJ(sg, SMP_SEG_MAGIC); AN(sg); sg->lru = LRU_Alloc(); sg->offset = ss->offset; sg->length = ss->length; + if (sg1 != NULL) { + assert(sg1->offset != sg->offset); + if (sg1->offset < sg->offset) + assert(sg1->offset + sg1->length <= sg->offset); + else + assert(sg->offset + sg->length <= sg1->offset); + } + if (sg2 != NULL) { + assert(sg2->offset != sg->offset); + if (sg2->offset < sg->offset) + assert(sg2->offset + sg2->length <= sg->offset); + else + assert(sg->offset + sg->length <= sg2->offset); + } + /* XXX: check that they are inside silo */ /* XXX: check that they don't overlap */ /* XXX: check that they are serial */ sg->sc = sc; VTAILQ_INSERT_TAIL(&sc->segments, sg, list); + sg2 = sg; + if (sg1 == NULL) + sg1 = sg; } + printf("Dropped %d segments to make free_reserve\n", n); return (0); } @@ -896,19 +993,35 @@ /* XXX: find where it goes in silo */ - sg2 = VTAILQ_LAST(&sc->segments, smp_seghead); - if (sg2 == NULL) { - sg->offset = sc->ident->stuff[SMP_SPC_STUFF]; - assert(sc->ident->stuff[SMP_SPC_STUFF] < sc->mediasize); - } else { - sg->offset = sg2->offset + sg2->length; - assert(sg->offset < sc->mediasize); - } + sg->offset = sc->free_offset; + assert(sg->offset >= sc->ident->stuff[SMP_SPC_STUFF]); + assert(sg->offset < sc->mediasize); + sg->length = sc->aim_segl; sg->length &= ~7; + assert(sg->offset + sg->length <= sc->mediasize); + sg2 = VTAILQ_FIRST(&sc->segments); + if (sg2 != NULL && sg2->offset > sc->free_offset) { +//printf("SG %jx...%jx\n", sg->offset, sg->offset + sg->length); +//printf("SG2 %jx...%jx\n", sg2->offset, sg2->offset + sg2->length); + +//printf("spc %jx %jx\n", sg2->offset + sg2->length - sg->offset, sc->free_reserve); + + if (sg->offset + sg->length > sg2->offset) { + printf("Out of space in persistent silo\n"); + printf("Committing suicide, restart will make space\n"); + exit (0); + } + assert(sg->offset + sg->length <= sg2->offset); + } + + sc->free_offset += sg->length; + +//printf("NEW_SEG %jx...%jx\n", sg->offset, sg->offset + sg->length); + VTAILQ_INSERT_TAIL(&sc->segments, sg, list); /* Neuter the new segment in case there is an old one there */ @@ -972,7 +1085,7 @@ /* Save segment list */ smp_save_segs(sc); - + sc->free_offset = sg->offset + sg->length; } /*-------------------------------------------------------------------- From phk at projects.linpro.no Tue Sep 29 17:07:35 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Tue, 29 Sep 2009 19:07:35 +0200 (CEST) Subject: r4263 - in trunk/varnish-cache/bin: varnishd varnishtest/tests Message-ID: <20090929170735.D699B1F736B@projects.linpro.no> Author: phk Date: 2009-09-29 19:07:35 +0200 (Tue, 29 Sep 2009) New Revision: 4263 Added: trunk/varnish-cache/bin/varnishtest/tests/r00561.vtc Removed: trunk/varnish-cache/bin/varnishtest/tests/r00354.vtc Modified: trunk/varnish-cache/bin/varnishd/cache_center.c trunk/varnish-cache/bin/varnishd/cache_http.c Log: If we cannot even make sense of the request, don't bother with attempting a reply. Fixes #561 Modified: trunk/varnish-cache/bin/varnishd/cache_center.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-29 12:25:21 UTC (rev 4262) +++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-09-29 17:07:35 UTC (rev 4263) @@ -1093,6 +1093,13 @@ http_Setup(sp->http, sp->ws); done = http_DissectRequest(sp); + /* If we could not even parse the request, just close */ + if (done < 0) { + sp->step = STP_DONE; + vca_close_session(sp, "junk"); + return (0); + } + /* Catch request snapshot */ sp->ws_req = WS_Snapshot(sp->ws); Modified: trunk/varnish-cache/bin/varnishd/cache_http.c =================================================================== --- trunk/varnish-cache/bin/varnishd/cache_http.c 2009-09-29 12:25:21 UTC (rev 4262) +++ trunk/varnish-cache/bin/varnishd/cache_http.c 2009-09-29 17:07:35 UTC (rev 4263) @@ -406,7 +406,7 @@ q = p; for (; !vct_issp(*p); p++) { if (vct_isctl(*p)) - return (400); + return (-1); } hp->hd[h1].b = q; hp->hd[h1].e = p; @@ -414,14 +414,14 @@ /* Skip SP */ for (; vct_issp(*p); p++) { if (vct_isctl(*p)) - return (400); + return (-1); } /* Second field cannot contain LWS or CTL */ q = p; for (; !vct_islws(*p); p++) { if (vct_isctl(*p)) - return (400); + return (-1); } hp->hd[h2].b = q; hp->hd[h2].e = p; @@ -432,7 +432,7 @@ /* Skip SP */ for (; vct_issp(*p); p++) { if (vct_isctl(*p)) - return (400); + return (-1); } /* Third field is optional and cannot contain CTL */ @@ -440,7 +440,7 @@ if (!vct_iscrlf(*p)) { for (; !vct_iscrlf(*p); p++) if (vct_isctl(*p)) - return (400); + return (-1); } hp->hd[h3].b = q; hp->hd[h3].e = p; Deleted: trunk/varnish-cache/bin/varnishtest/tests/r00354.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/r00354.vtc 2009-09-29 12:25:21 UTC (rev 4262) +++ trunk/varnish-cache/bin/varnishtest/tests/r00354.vtc 2009-09-29 17:07:35 UTC (rev 4263) @@ -1,16 +0,0 @@ -# $Id$ - -test "#354 Segfault in strcmp in http_DissectRequest()" - -server s1 { - rxreq - txresp -} - -varnish v1 -vcl+backend {} -start - -client c1 { - send "FOO\r\n\r\n" - rxresp - expect resp.status == 400 -} -run Added: trunk/varnish-cache/bin/varnishtest/tests/r00561.vtc =================================================================== --- trunk/varnish-cache/bin/varnishtest/tests/r00561.vtc (rev 0) +++ trunk/varnish-cache/bin/varnishtest/tests/r00561.vtc 2009-09-29 17:07:35 UTC (rev 4263) @@ -0,0 +1,25 @@ +# $Id$ + +test "Junk request should not go to vcl_error" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_error { + return (restart); + } +} -start + +client c1 { + send "sljdslf\r\n\r\n" + delay .1 +} -run + +client c1 { + txreq + rxresp +} -run + From phk at projects.linpro.no Wed Sep 30 08:38:08 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 30 Sep 2009 10:38:08 +0200 (CEST) Subject: r4264 - in trunk/varnish-cache: bin/varnishd include Message-ID: <20090930083808.0FA501F7355@projects.linpro.no> Author: phk Date: 2009-09-30 10:38:07 +0200 (Wed, 30 Sep 2009) New Revision: 4264 Modified: trunk/varnish-cache/bin/varnishd/shmlog.c trunk/varnish-cache/include/shmlog.h Log: Ohh, this tiny men with their mighty machines: Increase the ID field in the shmlog records from 16 to 32 bits to cater for 64k+ connections. Both varnishd and libvarnishapi needs to be recompiled, but apps should not need to. Modified: trunk/varnish-cache/bin/varnishd/shmlog.c =================================================================== --- trunk/varnish-cache/bin/varnishd/shmlog.c 2009-09-29 17:07:35 UTC (rev 4263) +++ trunk/varnish-cache/bin/varnishd/shmlog.c 2009-09-30 08:38:07 UTC (rev 4264) @@ -86,10 +86,11 @@ assert(loghead->magic == SHMLOGHEAD_MAGIC); assert(len < 0x10000); - assert(id < 0x10000); p[__SHMLOG_LEN_HIGH] = (len >> 8) & 0xff; p[__SHMLOG_LEN_LOW] = len & 0xff; - p[__SHMLOG_ID_HIGH] = (id >> 8) & 0xff; + p[__SHMLOG_ID_HIGH] = (id >> 24) & 0xff; + p[__SHMLOG_ID_MEDHIGH] = (id >> 16) & 0xff; + p[__SHMLOG_ID_MEDLOW] = (id >> 8) & 0xff; p[__SHMLOG_ID_LOW] = id & 0xff; p[SHMLOG_DATA + len] = '\0'; p[SHMLOG_NEXTTAG + len] = SLT_ENDMARKER; Modified: trunk/varnish-cache/include/shmlog.h =================================================================== --- trunk/varnish-cache/include/shmlog.h 2009-09-29 17:07:35 UTC (rev 4263) +++ trunk/varnish-cache/include/shmlog.h 2009-09-30 08:38:07 UTC (rev 4264) @@ -73,7 +73,7 @@ * * 1 byte field type (enum shmlogtag) * 2 bytes length of contents - * 2 bytes record identifier + * 4 bytes record identifier * n bytes field contents (isgraph(c) || isspace(c)) allowed. */ @@ -81,12 +81,18 @@ #define __SHMLOG_LEN_HIGH 1 #define __SHMLOG_LEN_LOW 2 #define __SHMLOG_ID_HIGH 3 -#define __SHMLOG_ID_LOW 4 -#define SHMLOG_DATA 5 -#define SHMLOG_NEXTTAG 6 /* ... + len */ +#define __SHMLOG_ID_MEDHIGH 4 +#define __SHMLOG_ID_MEDLOW 5 +#define __SHMLOG_ID_LOW 6 +#define SHMLOG_DATA 7 +#define SHMLOG_NEXTTAG 8 /* ... + len */ #define SHMLOG_LEN(p) (((p)[__SHMLOG_LEN_HIGH] << 8) | (p)[__SHMLOG_LEN_LOW]) -#define SHMLOG_ID(p) (((p)[__SHMLOG_ID_HIGH] << 8) | (p)[__SHMLOG_ID_LOW]) +#define SHMLOG_ID(p) ( \ + ((p)[__SHMLOG_ID_HIGH] << 24) | \ + ((p)[__SHMLOG_ID_MEDHIGH] << 16) | \ + ((p)[__SHMLOG_ID_MEDLOW] << 8) | \ + (p)[__SHMLOG_ID_LOW]) /* * The identifiers in shmlogtag are "SLT_" + XML tag. A script may be run From phk at projects.linpro.no Wed Sep 30 08:40:01 2009 From: phk at projects.linpro.no (phk at projects.linpro.no) Date: Wed, 30 Sep 2009 10:40:01 +0200 (CEST) Subject: r4265 - trunk/varnish-cache/include Message-ID: <20090930084001.B68201F7371@projects.linpro.no> Author: phk Date: 2009-09-30 10:40:01 +0200 (Wed, 30 Sep 2009) New Revision: 4265 Modified: trunk/varnish-cache/include/shmlog.h Log: Bump the shmlog magic to make sure we catch incompatible formats. Modified: trunk/varnish-cache/include/shmlog.h =================================================================== --- trunk/varnish-cache/include/shmlog.h 2009-09-30 08:38:07 UTC (rev 4264) +++ trunk/varnish-cache/include/shmlog.h 2009-09-30 08:40:01 UTC (rev 4265) @@ -43,7 +43,7 @@ #include "stats.h" struct shmloghead { -#define SHMLOGHEAD_MAGIC 4185512498U /* From /dev/random */ +#define SHMLOGHEAD_MAGIC 4185512499U /* From /dev/random */ unsigned magic; unsigned hdrsize;