From phk at varnish-cache.org  Thu Sep  1 07:57:56 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 01 Sep 2011 09:57:56 +0200
Subject: [master] 0d079cc Reset the "built vary spec" (also) if we came back
	from the waiting list, otherwise it might turn into garbage.
Message-ID: <E1Qz29s-0000Cm-Kj@project.varnish-software.com>

commit 0d079cc0576fa5bb9c84f34131c61ebb51dcc89c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 07:55:46 2011 +0000

    Reset the "built vary spec" (also) if we came back from the waiting
    list, otherwise it might turn into garbage.
    
    Fixes	#994
    Fixes	#1001

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index df9941c..0f69861 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -1080,10 +1080,12 @@ cnt_lookup(struct sess *sp)
 		AZ(sp->vary_l);
 		AZ(sp->vary_e);
 		(void)WS_Reserve(sp->ws, 0);
-		sp->vary_b = (void*)sp->ws->f;
-		sp->vary_e = (void*)sp->ws->r;
-		sp->vary_b[2] = '\0';
+	} else {
+		AN(sp->ws->r);
 	}
+	sp->vary_b = (void*)sp->ws->f;
+	sp->vary_e = (void*)sp->ws->r;
+	sp->vary_b[2] = '\0';
 
 	oc = HSH_Lookup(sp, &oh);
 



From phk at varnish-cache.org  Thu Sep  1 08:39:23 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 01 Sep 2011 10:39:23 +0200
Subject: [master] 69e5bd5 Also remove DIAGNISTICS around the prototype for
	MCF_DumpRst()
Message-ID: <E1Qz2nz-0001u2-TV@project.varnish-software.com>

commit 69e5bd536a561f56cf52926328716c000e35a5d3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 08:37:08 2011 +0000

    Also remove DIAGNISTICS around the prototype for MCF_DumpRst()

diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
index 30b90ba..485de26 100644
--- a/bin/varnishd/mgt.h
+++ b/bin/varnishd/mgt.h
@@ -66,9 +66,7 @@ void mgt_cli_close_all(void);
 void MCF_ParamSync(void);
 void MCF_ParamInit(struct cli *);
 void MCF_ParamSet(struct cli *, const char *param, const char *val);
-#ifdef DIAGNOSTICS
 void MCF_DumpRst(void);
-#endif
 
 /* mgt_sandbox.c */
 void mgt_sandbox(void);



From phk at varnish-cache.org  Thu Sep  1 08:39:24 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 01 Sep 2011 10:39:24 +0200
Subject: [master] d600ab8 Move acct_tmp to worker instead of session and
	report # gunziped bytes rather than storage size when we gunzip
	for delivery.
Message-ID: <E1Qz2o0-0001u7-KY@project.varnish-software.com>

commit d600ab8745c72f08272250cdb86dd4254f3feacd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 08:38:33 2011 +0000

    Move acct_tmp to worker instead of session and report # gunziped
    bytes rather than storage size when we gunzip for delivery.
    
    Fixes	#992

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 0509d99..43a0c41 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -353,6 +353,8 @@ struct worker {
 #define RES_ESI_CHILD		(1<<5)
 #define RES_GUNZIP		(1<<6)
 
+	/* Temporary accounting */
+	struct acct		acct_tmp;
 };
 
 /* Work Request for worker thread ------------------------------------*/
@@ -601,7 +603,6 @@ struct sess {
 	struct sessmem		*mem;
 
 	struct workreq		workreq;
-	struct acct		acct_tmp;
 	struct acct		acct_req;
 	struct acct		acct_ses;
 
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 0f69861..a4e5051 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -869,7 +869,7 @@ cnt_fetchbody(struct sess *sp)
 		AN(sp->obj->objcore->ban);
 		HSH_Unbusy(sp);
 	}
-	sp->acct_tmp.fetch++;
+	sp->wrk->acct_tmp.fetch++;
 	sp->step = STP_PREPRESP;
 	return (0);
 }
@@ -926,7 +926,7 @@ cnt_streambody(struct sess *sp)
 	} else {
 		sp->doclose = "Stream error";
 	}
-	sp->acct_tmp.fetch++;
+	sp->wrk->acct_tmp.fetch++;
 	sp->director = NULL;
 	sp->restarts = 0;
 
@@ -968,7 +968,7 @@ cnt_first(struct sess *sp)
 	HTC_Init(sp->htc, sp->ws, sp->fd, params->http_req_size,
 	    params->http_req_hdr_len);
 	sp->wrk->lastused = sp->t_open;
-	sp->acct_tmp.sess++;
+	sp->wrk->acct_tmp.sess++;
 
 	sp->step = STP_WAIT;
 	return (0);
@@ -1272,7 +1272,7 @@ cnt_pass(struct sess *sp)
 		return (0);
 	}
 	assert(sp->handling == VCL_RET_PASS);
-	sp->acct_tmp.pass++;
+	sp->wrk->acct_tmp.pass++;
 	sp->sendbody = 1;
 	sp->step = STP_FETCH;
 	return (0);
@@ -1310,7 +1310,7 @@ cnt_pipe(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
-	sp->acct_tmp.pipe++;
+	sp->wrk->acct_tmp.pipe++;
 	WS_Reset(sp->wrk->ws, NULL);
 	http_Setup(sp->wrk->bereq, sp->wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_PIPE);
@@ -1459,7 +1459,7 @@ cnt_start(struct sess *sp)
 	sp->wrk->stats.client_req++;
 	sp->t_req = TIM_real();
 	sp->wrk->lastused = sp->t_req;
-	sp->acct_tmp.req++;
+	sp->wrk->acct_tmp.req++;
 
 	/* Assign XID and log */
 	sp->xid = ++xids;				/* XXX not locked */
@@ -1614,6 +1614,9 @@ CNT_Session(struct sess *sp)
 	AZ(w->is_gunzip);
 	AZ(w->do_gunzip);
 	AZ(w->do_esi);
+#define ACCT(foo)	AZ(w->acct_tmp.foo);
+#include "acct_fields.h"
+#undef ACCT
 	assert(WRW_IsReleased(w));
 }
 
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 40a8bdf..bed3104 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -378,6 +378,7 @@ VGZ_WrwGunzip(const struct sess *sp, struct vgz *vg, const void *ibuf,
 			return (-1);
 		}
 		if (obufl == *obufp || i == VGZ_STUCK) {
+			sp->wrk->acct_tmp.bodybytes += *obufp;
 			(void)WRW_Write(sp->wrk, obuf, *obufp);
 			(void)WRW_Flush(sp->wrk);
 			*obufp = 0;
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 17d1525..c592909 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -175,7 +175,6 @@ res_WriteGunzipObj(struct sess *sp)
 		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
 		u += st->len;
 
-		sp->acct_tmp.bodybytes += st->len;	/* XXX ? */
 		VSC_C_main->n_objwrite++;
 
 		i = VGZ_WrwGunzip(sp, vg,
@@ -195,7 +194,7 @@ res_WriteGunzipObj(struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 static void
-res_WriteDirObj(struct sess *sp, ssize_t low, ssize_t high)
+res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high)
 {
 	ssize_t u = 0;
 	size_t ptr, off, len;
@@ -227,7 +226,7 @@ res_WriteDirObj(struct sess *sp, ssize_t low, ssize_t high)
 
 		ptr += len;
 
-		sp->acct_tmp.bodybytes += len;
+		sp->wrk->acct_tmp.bodybytes += len;
 #ifdef SENDFILE_WORKS
 		/*
 		 * XXX: the overhead of setting up sendfile is not
@@ -292,7 +291,7 @@ RES_WriteObj(struct sess *sp)
 	 * Send HTTP protocol header, unless interior ESI object
 	 */
 	if (!(sp->wrk->res_mode & RES_ESI_CHILD))
-		sp->acct_tmp.hdrbytes +=
+		sp->wrk->acct_tmp.hdrbytes +=
 		    http_Write(sp->wrk, sp->wrk->resp, 1);
 
 	if (!sp->wantbody)
@@ -348,7 +347,7 @@ RES_StreamStart(struct sess *sp)
 		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
 		    "Content-Length: %s", sp->wrk->h_content_length);
 
-	sp->acct_tmp.hdrbytes +=
+	sp->wrk->acct_tmp.hdrbytes +=
 	    http_Write(sp->wrk, sp->wrk->resp, 1);
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index af422a6..1ebca44 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -80,7 +80,7 @@ static struct lock		stat_mtx;
 void
 SES_Charge(struct sess *sp)
 {
-	struct acct *a = &sp->acct_tmp;
+	struct acct *a = &sp->wrk->acct_tmp;
 
 #define ACCT(foo)				\
 	sp->wrk->stats.s_##foo += a->foo;	\



From phk at varnish-cache.org  Thu Sep  1 09:59:34 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 01 Sep 2011 11:59:34 +0200
Subject: [master] c14ced8 Take another stab at #1001/#994
Message-ID: <E1Qz43a-0002k4-M6@project.varnish-software.com>

commit c14ced83338ed506318c98d167b28787d24669d8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 09:59:22 2011 +0000

    Take another stab at #1001/#994

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 43a0c41..f9989fb 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -899,6 +899,7 @@ void RES_StreamPoll(const struct sess *sp);
 /* cache_vary.c */
 struct vsb *VRY_Create(const struct sess *sp, const struct http *hp);
 int VRY_Match(struct sess *sp, const uint8_t *vary);
+void VRY_Validate(const uint8_t *vary);
 
 /* cache_vcl.c */
 void VCL_Init(void);
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index a4e5051..6bb8324 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -802,6 +802,7 @@ cnt_fetchbody(struct sess *sp)
 		    (void *)WS_Alloc(sp->obj->http->ws, varyl);
 		AN(sp->obj->vary);
 		memcpy(sp->obj->vary, VSB_data(vary), varyl);
+		VRY_Validate(sp->obj->vary);
 		VSB_delete(vary);
 	}
 
@@ -1107,10 +1108,14 @@ cnt_lookup(struct sess *sp)
 	if (oc->flags & OC_F_BUSY) {
 		sp->wrk->stats.cache_miss++;
 
-		if (sp->vary_l != NULL)
+		if (sp->vary_l != NULL) {
+			assert(oc->busyobj->vary == sp->vary_b);
+			VRY_Validate(oc->busyobj->vary);
 			WS_ReleaseP(sp->ws, (void*)sp->vary_l);
-		else
-			WS_Release(sp->ws, 0);
+		} else {
+			AZ(oc->busyobj->vary);
+			WS_Release(sp->ws, 0); 
+		}
 		sp->vary_b = NULL;
 		sp->vary_l = NULL;
 		sp->vary_e = NULL;
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index b070b38..328964a 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -461,7 +461,12 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	AN(oc->flags & OC_F_BUSY);
 	oc->refcnt = 1;
 
-	w->nbusyobj->vary = sp->vary_b;
+	/* XXX: clear w->nbusyobj before use */
+	VRY_Validate(sp->vary_b);
+	if (sp->vary_l != NULL)
+		w->nbusyobj->vary = sp->vary_b;
+	else
+		w->nbusyobj->vary = NULL;
 	oc->busyobj = w->nbusyobj;
 	w->nbusyobj = NULL;
 
diff --git a/bin/varnishd/cache_vary.c b/bin/varnishd/cache_vary.c
index 73d6354..b7a6b01 100644
--- a/bin/varnishd/cache_vary.c
+++ b/bin/varnishd/cache_vary.c
@@ -247,3 +247,13 @@ VRY_Match(struct sess *sp, const uint8_t *vary)
 		sp->vary_l = vsp + 3;
 	return (retval);
 }
+
+void
+VRY_Validate(const uint8_t *vary)
+{
+
+	while (vary[2] != 0) {
+		assert(strlen((const char*)vary+3) == vary[2]);
+		vary += vry_len(vary);
+	}
+}



From kristian at varnish-cache.org  Thu Sep  1 13:20:47 2011
From: kristian at varnish-cache.org (=?UTF-8?Q?Kristian_Lyngst=C3=B8l?=)
Date: Thu, 01 Sep 2011 15:20:47 +0200
Subject: [master] 1bddcfa Document hit_for_pass in the tutorial
Message-ID: <E1Qz7CJ-0002CF-O7@project.varnish-software.com>

commit 1bddcfa742f5a48808429681b5df21520fadd0b4
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Thu Sep 1 15:20:36 2011 +0200

    Document hit_for_pass in the tutorial

diff --git a/doc/sphinx/tutorial/vcl.rst b/doc/sphinx/tutorial/vcl.rst
index f52c1ed..3ed78c5 100644
--- a/doc/sphinx/tutorial/vcl.rst
+++ b/doc/sphinx/tutorial/vcl.rst
@@ -58,7 +58,15 @@ The most common actions to return are these:
 *pass*
  When you return pass the request and subsequent response will be passed to
  and from the backend server. It won't be cached. pass can be returned from
- both vcl_recv and vcl_fetch.
+ vcl_recv
+
+*hit_for_pass*
+  Similar to pass, but accessible from vcl_fetch. Unlike pass, hit_for_pass
+  will create a hitforpass object in the cache. This has the side-effect of
+  caching the decision not to cache. This is to allow would-be uncachable
+  requests to be passed to the backend at the same time. The same logic is
+  not necessary in vcl_recv because this happens before any potential
+  queueing for an object takes place.
 
 *lookup*
   When you return lookup from vcl_recv you tell Varnish to deliver content 



From phk at varnish-cache.org  Thu Sep  1 15:20:24 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 01 Sep 2011 17:20:24 +0200
Subject: [master] cb07c41 A bit of spit and polish inspired by DocWilcos
	varnishtest patch.
Message-ID: <E1Qz944-0000EO-Pw@project.varnish-software.com>

commit cb07c41dfa1a32ddefaece2850386618820e19a6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 15:19:33 2011 +0000

    A bit of spit and polish inspired by DocWilcos varnishtest patch.

diff --git a/bin/varnishlog/flint.lnt b/bin/varnishlog/flint.lnt
index 149eec1..3a0395c 100644
--- a/bin/varnishlog/flint.lnt
+++ b/bin/varnishlog/flint.lnt
@@ -1,4 +1,5 @@
-
+-efile(451, "../../include/vsc_all.h")
+-e835   // A zero has been given as ___ argument to operator '___'  (<<)
 -e712	// 14      Info    712     Loss of precision (___) (___ to ___)
 -e747	// 16      Info    747     Significant prototype coercion (___) ___ to ___
 
@@ -17,3 +18,4 @@
 -e788	// enum constant '___' not used within defaulted switch
 -e641	// Converting enum '___' to '___'
 
+-esym(785,VSL_tags);	// Sparse array
diff --git a/bin/varnishlog/flint.sh b/bin/varnishlog/flint.sh
index a94710d..dc9e9ba 100755
--- a/bin/varnishlog/flint.sh
+++ b/bin/varnishlog/flint.sh
@@ -1,11 +1,28 @@
 #!/bin/sh
 
+if [ "x$1" = "x-ok" -a -f _.fl ] ; then
+	echo "Saved as reference"
+	mv _.fl _.fl.old
+	exit 0
+fi
+
 flexelint \
-	-I/usr/include \
+	../flint.lnt \
+	flint.lnt \
 	-I. \
 	-I../../include \
 	-I../.. \
-	../flint.lnt \
-	flint.lnt \
+	-I/usr/local/include \
+	-DVARNISH_STATE_DIR=\"foo\" \
 	*.c \
-	../../lib/libvarnishapi/*.c
+	../../lib/libvarnishapi/*.c \
+	2>&1 | tee _.fl
+
+if [ -f _.fl.old ] ; then
+	diff -u _.fl.old _.fl
+fi
+
+if [ "x$1" = "x-ok" ] ; then
+	echo "Saved as reference"
+	mv _.fl _.fl.old
+fi
diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index ed29865..c7749cb 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -46,7 +46,6 @@
 
 #include "libvarnish.h"
 #include "vsl.h"
-#include "vre.h"
 #include "varnishapi.h"
 
 static int	b_flag, c_flag;
@@ -60,7 +59,7 @@ static uint64_t       bitmap[65536];
 #define F_INVCL		(1 << 0)
 
 static void
-h_order_finish(int fd, struct VSM_data *vd)
+h_order_finish(int fd, const struct VSM_data *vd)
 {
 
 	AZ(VSB_finish(ob[fd]));
@@ -72,7 +71,7 @@ h_order_finish(int fd, struct VSM_data *vd)
 }
 
 static void
-clean_order(struct VSM_data *vd)
+clean_order(const struct VSM_data *vd)
 {
 	unsigned u;
 
@@ -321,7 +320,8 @@ main(int argc, char * const *argv)
 			w_arg = optarg;
 			break;
 		case 'm':
-			m_flag = 1; /* fall through */
+			m_flag = 1;
+			/* FALLTHROUGH */
 		default:
 			if (VSL_Arg(vd, c, optarg) > 0)
 				break;
diff --git a/include/varnishapi.h b/include/varnishapi.h
index 0e97755..dcdd74f 100644
--- a/include/varnishapi.h
+++ b/include/varnishapi.h
@@ -109,7 +109,7 @@ int VSM_ReOpen(struct VSM_data *vd, int diag);
 	 *	-1 failure to reopen.
 	 */
 
-unsigned VSM_Seq(struct VSM_data *vd);
+unsigned VSM_Seq(const struct VSM_data *vd);
 	/*
 	 * Return the allocation sequence number
 	 */
@@ -260,6 +260,7 @@ void VSL_NonBlocking(const struct VSM_data *vd, int nb);
 int VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv);
 int VSL_NextLog(const struct VSM_data *lh, uint32_t **pp, uint64_t *bitmap);
 int VSL_Matched(const struct VSM_data *vd, uint64_t bitmap);
+int VSL_Name2Tag(const char *name, int l);
 extern const char *VSL_tags[256];
 
 #endif
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index d9624b9..fc18cb3 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -330,7 +330,7 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 		CHECK_OBJ_NOTNULL(sha, VSM_CHUNK_MAGIC);
 		if (strcmp(sha->class, VSC_CLASS))
 			continue;
-
+		/*lint -save -e525 -e539 */
 #define VSC_F(a,b,c,d,e)
 #define VSC_DONE(a,b,c)
 #define VSC_DO(U,l,t)						\
@@ -343,6 +343,7 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 #undef VSC_F
 #undef VSC_DO
 #undef VSC_DONE
+		/*lint -restore */
 		break;
 	}
 	return (i);
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index f4e633b..6791888 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -206,7 +206,7 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 }
 
 int
-VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *mb)
+VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 {
 	struct vsl *vsl;
 	uint32_t *p;
@@ -268,7 +268,7 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *mb)
 			if (i != VRE_ERROR_NOMATCH)
 				continue;
 		}
-		if (mb != NULL) {
+		if (bits != NULL) {
 			struct vsl_re_match *vrm;
 			int j = 0;
 			VTAILQ_FOREACH(vrm, &vsl->matchers, next) {
@@ -276,7 +276,7 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *mb)
 					i = VRE_exec(vrm->re, VSL_DATA(p),
 						     VSL_LEN(p), 0, 0, NULL, 0);
 					if (i >= 0)
-						*mb |= 1 << j;
+						*bits |= (uintmax_t)1 << j;
 				}
 				j++;
 			}
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 49930c8..21cd8e9 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -41,7 +41,6 @@
 #include <unistd.h>
 
 #include "vas.h"
-#include "vin.h"
 #include "vre.h"
 #include "vbm.h"
 #include "miniobj.h"
@@ -50,6 +49,34 @@
 #include "vsm_api.h"
 #include "vsl_api.h"
 
+/*--------------------------------------------------------------------
+ * Look up a tag
+ *   0..255	tag number
+ *   -1		no tag matches
+ *   -2		multiple tags match
+ */
+
+int
+VSL_Name2Tag(const char *name, int l)
+{
+	int i, n;
+
+	if (l == -1)
+		l = strlen(name);
+	n = -1;
+	for (i = 0; i < 256; i++) {
+		if (VSL_tags[i] != NULL &&
+		    !strncasecmp(name, VSL_tags[i], l)) {
+			if (n == -1)
+				n = i;
+			else
+				n = -2;
+		}
+	}
+	return (n);
+}
+
+
 /*--------------------------------------------------------------------*/
 
 static int
@@ -99,8 +126,8 @@ vsl_IX_arg(const struct VSM_data *vd, const char *opt, int arg)
 static int
 vsl_ix_arg(const struct VSM_data *vd, const char *opt, int arg)
 {
-	int i, j, l;
-	const char *b, *e, *p, *q;
+	int i, l;
+	const char *b, *e;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	/* If first option is 'i', set all bits for supression */
@@ -120,24 +147,17 @@ vsl_ix_arg(const struct VSM_data *vd, const char *opt, int arg)
 			e++;
 		while (isspace(b[l - 1]))
 			l--;
-		for (i = 0; i < 256; i++) {
-			if (VSL_tags[i] == NULL)
-				continue;
-			p = VSL_tags[i];
-			q = b;
-			for (j = 0; j < l; j++)
-				if (tolower(*q++) != tolower(*p++))
-					break;
-			if (j != l || *p != '\0')
-				continue;
-
+		i = VSL_Name2Tag(b, l);
+		if (i >= 0) {
 			if (arg == 'x')
 				vbit_set(vd->vsl->vbm_supress, i);
 			else
 				vbit_clr(vd->vsl->vbm_supress, i);
-			break;
-		}
-		if (i == 256) {
+		} else if (i == -2) {
+			fprintf(stderr,
+			    "\"%*.*s\" matches multiple tags\n", l, l, b);
+			return (-1);
+		} else {
 			fprintf(stderr,
 			    "Could not match \"%*.*s\" to any tag\n", l, l, b);
 			return (-1);
@@ -148,19 +168,6 @@ vsl_ix_arg(const struct VSM_data *vd, const char *opt, int arg)
 
 /*--------------------------------------------------------------------*/
 
-static int
-name2tag(const char *n)
-{
-	int i;
-
-	for (i = 0; i < 256; i++) {
-		if (VSL_tags[i] == NULL)
-			continue;
-		if (!strcasecmp(n, VSL_tags[i]))
-			return (i);
-	}
-	return (-1);
-}
 
 static int
 vsl_m_arg(const struct VSM_data *vd, const char *opt)
@@ -171,8 +178,6 @@ vsl_m_arg(const struct VSM_data *vd, const char *opt)
 	int erroroffset;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	ALLOC_OBJ(m, VSL_RE_MATCH_MAGIC);
-	AN(m);
 
 	if (!strchr(opt, ':')) {
 		fprintf(stderr, "No : found in -o option %s\n", opt);
@@ -185,10 +190,13 @@ vsl_m_arg(const struct VSM_data *vd, const char *opt)
 	*regex = '\0';
 	regex++;
 
-	m->tag = name2tag(o);
-	if (m->tag == -1) {
+	ALLOC_OBJ(m, VSL_RE_MATCH_MAGIC);
+	AN(m);
+	m->tag = VSL_Name2Tag(o, -1);
+	if (m->tag < 0) {
 		fprintf(stderr, "Illegal tag %s specified\n", o);
 		free(o);
+		FREE_OBJ(m);
 		return (-1);
 	}
 	/* Get tag, regex */
@@ -196,6 +204,7 @@ vsl_m_arg(const struct VSM_data *vd, const char *opt)
 	if (m->re == NULL) {
 		fprintf(stderr, "Illegal regex: %s\n", error);
 		free(o);
+		FREE_OBJ(m);
 		return (-1);
 	}
 	vd->vsl->num_matchers++;
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index efb7592..888a0d7 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -224,7 +224,7 @@ VSM_Open(struct VSM_data *vd, int diag)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	AZ(vd->VSM_head);
 	if (!vd->n_opt)
-		VSM_n_Arg(vd, "");
+		(void)VSM_n_Arg(vd, "");
 	return (vsm_open(vd, diag));
 }
 
@@ -331,7 +331,7 @@ VSM_iter0(struct VSM_data *vd)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	vd->alloc_seq = vd->VSM_head->alloc_seq;
 	while (vd->alloc_seq == 0) {
-		usleep(50000);
+		(void)usleep(50000);
 		vd->alloc_seq = vd->VSM_head->alloc_seq;
 	}
 	CHECK_OBJ_NOTNULL(&vd->VSM_head->head, VSM_CHUNK_MAGIC);
@@ -358,7 +358,7 @@ VSM_itern(const struct VSM_data *vd, struct VSM_chunk **pp)
 
 /*--------------------------------------------------------------------*/
 unsigned
-VSM_Seq(struct VSM_data *vd)
+VSM_Seq(const struct VSM_data *vd)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);



From phk at varnish-cache.org  Thu Sep  1 15:20:25 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 01 Sep 2011 17:20:25 +0200
Subject: [master] 33774e3 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1Qz945-0000EU-0Q@project.varnish-software.com>

commit 33774e3dc2d661598839db95444cd759b2ebdd72
Merge: cb07c41 1bddcfa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 15:20:22 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From apj at varnish-cache.org  Mon Sep  5 10:13:03 2011
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Mon, 05 Sep 2011 12:13:03 +0200
Subject: [master] e000786 Ignore invalid HTTP headers
Message-ID: <E1R0WAp-0006tu-Op@project.varnish-software.com>

commit e0007862dee9352d63bc2da613304f4652810b9d
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Sep 5 12:05:24 2011 +0200

    Ignore invalid HTTP headers

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 9ebb096..25c63ee 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -250,7 +250,7 @@ static int
 collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
     const char *ptr, unsigned len)
 {
-	const char *end, *next;
+	const char *end, *next, *split;
 
 	assert(spec & VSL_S_BACKEND);
 	end = ptr + len;
@@ -330,18 +330,19 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		break;
 
 	case SLT_TxHeader:
+		split = strchr(ptr, ':');
 		if (!lp->active)
 			break;
+		if (split == NULL)
+			break;
 		if (isprefix(ptr, "authorization:", end, &next) &&
 		    isprefix(next, "basic", end, &next)) {
 			lp->df_u = trimline(next, end);
 		} else {
 			struct hdr *h;
-			const char *split;
 			size_t l;
 			h = malloc(sizeof(struct hdr));
 			AN(h);
-			split = strchr(ptr, ':');
 			AN(split);
 			l = strlen(split);
 			h->key = trimline(ptr, split-1);
@@ -369,7 +370,7 @@ static int
 collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
     const char *ptr, unsigned len)
 {
-	const char *end, *next;
+	const char *end, *next, *split;
 	long l;
 	time_t t;
 
@@ -437,8 +438,11 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 
 	case SLT_TxHeader:
 	case SLT_RxHeader:
+		split = strchr(ptr, ':');
 		if (!lp->active)
 			break;
+		if (split == NULL)
+			break;
 		if (tag == SLT_RxHeader &&
 		    isprefix(ptr, "authorization:", end, &next) &&
 		    isprefix(next, "basic", end, &next)) {
@@ -446,10 +450,8 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 			lp->df_u = trimline(next, end);
 		} else {
 			struct hdr *h;
-			const char *split;
 			h = malloc(sizeof(struct hdr));
 			AN(h);
-			split = strchr(ptr, ':');
 			AN(split);
 			h->key = trimline(ptr, split);
 			h->value = trimline(split+1, end);



From apj at varnish-cache.org  Mon Sep  5 11:06:12 2011
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Mon, 05 Sep 2011 13:06:12 +0200
Subject: [master] dce01d1 Small optimization: don't process header when
	inactive
Message-ID: <E1R0X0G-00012O-Mb@project.varnish-software.com>

commit dce01d1ec45c5e0bc57e7e4a2a9ea7ec097a07a2
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Sep 5 13:04:37 2011 +0200

    Small optimization: don't process header when inactive

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 25c63ee..28f6f21 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -330,9 +330,9 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		break;
 
 	case SLT_TxHeader:
-		split = strchr(ptr, ':');
 		if (!lp->active)
 			break;
+		split = strchr(ptr, ':');
 		if (split == NULL)
 			break;
 		if (isprefix(ptr, "authorization:", end, &next) &&
@@ -438,9 +438,9 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 
 	case SLT_TxHeader:
 	case SLT_RxHeader:
-		split = strchr(ptr, ':');
 		if (!lp->active)
 			break;
+		split = strchr(ptr, ':');
 		if (split == NULL)
 			break;
 		if (tag == SLT_RxHeader &&



From apj at varnish-cache.org  Mon Sep  5 18:11:46 2011
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Mon, 05 Sep 2011 20:11:46 +0200
Subject: [master] db1a21f Add documentation of string syntax, rollback,
	panic and synthetic keywords.
Message-ID: <E1R0de6-0007gT-Pg@project.varnish-software.com>

commit db1a21ffba8f34776c0c971de80e29c7fd0766d6
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Sep 5 20:09:59 2011 +0200

    Add documentation of string syntax, rollback, panic and synthetic keywords.
    
    Fixes: #991

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 4f005d0..82e20f4 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -39,6 +39,12 @@ In addition to the C-like assignment (=), comparison (==, !=) and
 boolean (!, && and \|\|) operators, VCL supports both regular
 expression and ACL matching using the ~ and the !~ operators.
 
+Basic strings are enclosed in " ... " and uses URL-style %-escapes.
+
+Long strings are enclosed in {" ... "} and do not have an escape
+character. They may contain any character including ", newline and
+other control characters except for the NUL (0x00) character.
+
 Unlike C and Perl, the backslash (\) character has no special meaning
 in strings in VCL, so it can be freely used in regular expressions
 without doubling.
@@ -55,6 +61,15 @@ You can use the *set* keyword to arbitrary HTTP headers. You can
 remove headers with the *remove* or *unset* keywords, which are
 synonym.
 
+You can use the *rollback* keyword to revert any changes to req at
+any time.
+
+The *synthetic* keyword is used to produce a synthetic response
+body in vcl_error. It takes a single string as argument.
+
+You can force a crash of the client process with the *panic* keyword.
+*panic* takes a string as argument.
+
 The ``return(action)`` keyword terminates the subroutine. *action* can be,
 depending on context one of
 



From phk at varnish-cache.org  Tue Sep  6 08:15:55 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 06 Sep 2011 10:15:55 +0200
Subject: [master] 8548216 I have not been able to reproduce this exact
	circumstance, but the fix is readily obvious,
	so:  Tighten requirement for range delivery.
Message-ID: <E1R0qp1-0007a7-Ee@project.varnish-software.com>

commit 8548216821a2710953f2063be75490bb0d9d7eba
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 6 08:15:06 2011 +0000

    I have not been able to reproduce this exact circumstance, but the
    fix is readily obvious, so:  Tighten requirement for range delivery.
    
    Fixes	#1007

diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index c592909..73338f7 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -276,9 +276,13 @@ RES_WriteObj(struct sess *sp)
 	 */
 	low = 0;
 	high = sp->obj->len - 1;
-	if (!(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
-	    params->http_range_support && sp->obj->response == 200 &&
-	    sp->wantbody && http_GetHdr(sp->http, H_Range, &r))
+	if (
+	    sp->wantbody &&
+	    (sp->wrk->res_mode & RES_LEN) &&
+	    !(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
+	    params->http_range_support &&
+	    sp->obj->response == 200 &&
+	    http_GetHdr(sp->http, H_Range, &r))
 		res_dorange(sp, r, &low, &high);
 
 	/*



From tfheen at varnish-cache.org  Tue Sep  6 12:56:01 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 06 Sep 2011 14:56:01 +0200
Subject: [master] daa446e Unlink -s file with autogenerated names
Message-ID: <E1R0vC5-0007Nu-QZ@project.varnish-software.com>

commit daa446eb29a1d90b452dc8ccf31bf28b6c4c516b
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 6 13:49:59 2011 +0200

    Unlink -s file with autogenerated names
    
    If just -s file or -s file,/path/to/directory is given, the file
    stevedore will generate a name.  Make sure to unlink this to not leak
    disk space over time.
    
    Fixes: #1008

diff --git a/bin/varnishd/stevedore_utils.c b/bin/varnishd/stevedore_utils.c
index a38588d..bd4e368 100644
--- a/bin/varnishd/stevedore_utils.c
+++ b/bin/varnishd/stevedore_utils.c
@@ -108,6 +108,7 @@ STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx)
 		if (fd < 0)
 			ARGV_ERR("(%s) \"%s\" mkstemp(%s) failed (%s)\n",
 			    ctx, fn, buf, strerror(errno));
+		AZ(unlink(buf));
 		*fnp = strdup(buf);
 		AN(*fnp);
 		retval = 2;



From tfheen at varnish-cache.org  Tue Sep  6 12:56:04 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 06 Sep 2011 14:56:04 +0200
Subject: [master] 5facfd3 Make the default -s file size 100MB rather than 50%
	of free disk space
Message-ID: <E1R0vC8-0007Nz-Nn@project.varnish-software.com>

commit 5facfd3ce2639309a7a02a9e21cea738cc43f12d
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 6 13:51:04 2011 +0200

    Make the default -s file size 100MB rather than 50% of free disk space

diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index ea78526..7687179 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -116,7 +116,7 @@ smf_initfile(struct stevedore *st, struct smf_sc *sc, const char *size)
 	/* XXX: force block allocation here or in open ? */
 }
 
-static const char default_size[] = "50%";
+static const char default_size[] = "100M";
 static const char default_filename[] = ".";
 
 static void



From tfheen at varnish-cache.org  Tue Sep  6 13:10:05 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 06 Sep 2011 15:10:05 +0200
Subject: [master] 4633d00 Fix up default format string in man page and code
	comments
Message-ID: <E1R0vPh-0008Cx-Mz@project.varnish-software.com>

commit 4633d008fac58813b06bdc421dd31787851b6c50
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 6 15:09:14 2011 +0200

    Fix up default format string in man page and code comments
    
    We're actually using %s, not %>s, since we don't support >.
    
    Thanks to Kai for patch
    Fixes: #999

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 70e915f..7c35c3e 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -31,7 +31,7 @@
  * Obtain log data from the shared memory log, order it by session ID, and
  * display it in Apache / NCSA combined log format:
  *
- *	%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"
+ *	%h %l %u %t "%r" %s %b "%{Referer}i" "%{User-agent}i"
  *
  * where the fields are defined as follows:
  *
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 65f08dd..a4bac0a 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -52,7 +52,7 @@ The following options are available:
 -F format   Specify the log format used. If no format is specified the  
    	    default log format is used. Currently it is:
 
-            %h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"
+            %h %l %u %t "%r" %s %b "%{Referer}i" "%{User-agent}i"
 
 	    Supported formatters are:
 



From tfheen at varnish-cache.org  Tue Sep  6 13:10:03 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 06 Sep 2011 15:10:03 +0200
Subject: [master] efaa48a Fix handling vs Varnish:handling in varnishncsa
Message-ID: <E1R0vPf-0008AQ-RB@project.varnish-software.com>

commit efaa48a908806f9d0a0969f840393e5b77771036
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 6 15:07:03 2011 +0200

    Fix handling vs Varnish:handling in varnishncsa
    
    Thanks to Kai for patch
    Fixes: #999

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 28f6f21..70e915f 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -716,7 +716,7 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 					VSB_cat(os, (lp->df_hitmiss ? lp->df_hitmiss : "-"));
 					p = tmp;
 					break;
-				} else if (strcmp(fname, "handling") == 0) {
+				} else if (strcmp(fname, "Varnish:handling") == 0) {
 					VSB_cat(os, (lp->df_handling ? lp->df_handling : "-"));
 					p = tmp;
 					break;



From apj at varnish-cache.org  Tue Sep  6 17:38:13 2011
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Tue, 06 Sep 2011 19:38:13 +0200
Subject: [master] b09f55e Fix string documentation for 3.0
Message-ID: <E1R0zbB-0002hV-7K@project.varnish-software.com>

commit b09f55ec50e4fcfaa5ee122a39b2a4c31ae3e9cf
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Sep 6 19:30:40 2011 +0200

    Fix string documentation for 3.0

diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst
index ee57df4..d00e9b0 100644
--- a/doc/sphinx/installation/upgrade.rst
+++ b/doc/sphinx/installation/upgrade.rst
@@ -13,6 +13,10 @@ string concatenation operator
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 String concatenation did not have an operator previously, but this has now been changed to ``+``.
 
+no more %-escapes in strings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+To simplify strings, the %-encoding has been removed. If you need non-printable characters, you need to use inline C.
+
 ``log`` moved to the std vmod
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 82e20f4..7359e6f 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -39,11 +39,11 @@ In addition to the C-like assignment (=), comparison (==, !=) and
 boolean (!, && and \|\|) operators, VCL supports both regular
 expression and ACL matching using the ~ and the !~ operators.
 
-Basic strings are enclosed in " ... " and uses URL-style %-escapes.
+Basic strings are enclosed in " ... ", and may not contain newlines.
 
-Long strings are enclosed in {" ... "} and do not have an escape
-character. They may contain any character including ", newline and
-other control characters except for the NUL (0x00) character.
+Long strings are enclosed in {" ... "}. They may contain any
+character including ", newline and other control characters except
+for the NUL (0x00) character.
 
 Unlike C and Perl, the backslash (\) character has no special meaning
 in strings in VCL, so it can be freely used in regular expressions
diff --git a/lib/libvcl/vcc_token.c b/lib/libvcl/vcc_token.c
index 5a1b00c..c5766c0 100644
--- a/lib/libvcl/vcc_token.c
+++ b/lib/libvcl/vcc_token.c
@@ -492,7 +492,7 @@ vcc_Lexer(struct vcc *tl, struct source *sp)
 			continue;
 		}
 
-		/* Match strings, with \\ and \" escapes */
+		/* Match strings */
 		if (*p == '"') {
 			for (q = p + 1; q < sp->e; q++) {
 				if (*q == '"') {



From apj at varnish-cache.org  Tue Sep  6 17:38:13 2011
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Tue, 06 Sep 2011 19:38:13 +0200
Subject: [master] d56069e Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1R0zbB-0002ha-DJ@project.varnish-software.com>

commit d56069e8ef221310d75455feb9b03483c9caf63b
Merge: b09f55e 4633d00
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Sep 6 19:33:47 2011 +0200

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From phk at varnish-cache.org  Wed Sep  7 11:05:51 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 07 Sep 2011 13:05:51 +0200
Subject: [master] c397697 Add text representation to Fetch_Body VSL message.
Message-ID: <E1R1Fx1-00044M-N4@project.varnish-software.com>

commit c397697d40e90e7848b04009891a264802b8b8d8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 11:03:35 2011 +0000

    Add text representation to Fetch_Body VSL message.

diff --git a/bin/varnishd/body_status.h b/bin/varnishd/body_status.h
new file mode 100644
index 0000000..33803f9
--- /dev/null
+++ b/bin/varnishd/body_status.h
@@ -0,0 +1,38 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Various ways to handle the body coming from the backend.
+ */
+
+/*lint -save -e525 -e539 */
+BODYSTATUS(NONE, none)
+BODYSTATUS(ZERO, zero)
+BODYSTATUS(ERROR, error)
+BODYSTATUS(CHUNKED, chunked)
+BODYSTATUS(LENGTH, length)
+BODYSTATUS(EOF, eof)
+/*lint -restore */
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index f9989fb..09c94f3 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -66,14 +66,23 @@
 #include "vsl.h"
 
 enum body_status {
-	BS_NONE,
-	BS_ZERO,
-	BS_ERROR,
-	BS_CHUNKED,
-	BS_LENGTH,
-	BS_EOF
+#define BODYSTATUS(U,l)	BS_##U,
+#include "body_status.h"
+#undef BODYSTATUS
 };
 
+static inline const char *
+body_status(enum body_status e)
+{
+	switch(e) {
+#define BODYSTATUS(U,l)	case BS_##U: return (#l);
+#include "body_status.h"
+#undef BODYSTATUS
+	default:
+		return ("?");
+	}
+}
+
 /*
  * NB: HDR_STATUS is only used in cache_http.c, everybody else uses the
  * http->status integer field.
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 657392f..24463f5 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -543,8 +543,9 @@ FetchBody(struct sess *sp)
 	 */
 	AZ(vfp_nop_end(sp));
 
-	WSL(sp->wrk, SLT_Fetch_Body, sp->vbc->fd, "%u %d %u",
-	    sp->wrk->body_status, cls, mklen);
+	WSL(sp->wrk, SLT_Fetch_Body, sp->vbc->fd, "%u(%s) %d %u",
+	    sp->wrk->body_status, body_status(sp->wrk->body_status),
+	    cls, mklen);
 
 	if (sp->wrk->body_status == BS_ERROR) {
 		VDI_CloseFd(sp);
diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index 1e88f1b..f2d2a6b 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -86,6 +86,7 @@
 -efile(451, "sys/\*.h")	// No include guard
 -efile(451, "machine/\*.h")	// No include guard
 -efile(451, "vcl_returns.h")	// No include guard
+-efile(451, "body_status.h")	// No include guard
 -efile(451, "locks.h")	// No include guard
 -efile(451, "cache_backend_poll.h")	// No include guard
 -efile(451, "steps.h")	// No include guard



From phk at varnish-cache.org  Wed Sep  7 12:08:22 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 07 Sep 2011 14:08:22 +0200
Subject: [master] bac89d8 Improve the FetchError VSL messages.
Message-ID: <E1R1GvW-0008RY-FZ@project.varnish-software.com>

commit bac89d865448f9525e2a82a9ace0c8d708e0c144
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 12:08:02 2011 +0000

    Improve the FetchError VSL messages.
    
    Fixes	#1005

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 09c94f3..1c1952d 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -203,6 +203,7 @@ struct http_conn {
 	struct ws		*ws;
 	txt			rxbuf;
 	txt			pipeline;
+	const char		*error;
 };
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 24463f5..36df887 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -88,8 +88,10 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 
 	while (bytes > 0) {
 		st = FetchStorage(sp, 0);
-		if (st == NULL)
+		if (st == NULL) {
+			htc->error = "Could not get storage";
 			return (-1);
+		}
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
@@ -202,7 +204,7 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 	assert(sp->wrk->body_status == BS_LENGTH);
 	cl = fetch_number(b, 10);
 	if (cl < 0) {
-		WSP(sp, SLT_FetchError, "straight length syntax");
+		WSP(sp, SLT_FetchError, "straight length field bogus");
 		return (-1);
 	}
 	/*
@@ -217,7 +219,7 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 	i = sp->wrk->vfp->bytes(sp, htc, cl);
 	if (i <= 0) {
 		WSP(sp, SLT_FetchError, "straight read_error: %d %d (%s)",
-		    i, errno, strerror(errno));
+		    i, errno, htc->error);
 		return (-1);
 	}
 	return (0);
@@ -230,7 +232,7 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 		if (i != 1) {					\
 			WSP(sp, SLT_FetchError,			\
 			    "chunked read_error: %d (%s)",	\
-			    errno, strerror(errno));		\
+			    errno, htc->error);			\
 			return (-1);				\
 		}						\
 	} while (0)
@@ -315,7 +317,7 @@ fetch_eof(struct sess *sp, struct http_conn *htc)
 	i = sp->wrk->vfp->bytes(sp, htc, SSIZE_MAX);
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "eof read_error: %d (%s)",
-		    errno, strerror(errno));
+		    errno, htc->error);
 		return (-1);
 	}
 	return (0);
@@ -396,7 +398,7 @@ FetchHdr(struct sess *sp)
 		AN(sp->objcore->flags & OC_F_BUSY);
 	}
 
-	hp = sp->wrk->bereq;
+	hp = w->bereq;
 
 	sp->vbc = VDI_GetFd(NULL, sp);
 	if (sp->vbc == NULL) {
@@ -437,16 +439,16 @@ FetchHdr(struct sess *sp)
 
 	/* Receive response */
 
-	HTC_Init(sp->wrk->htc, sp->wrk->ws, vc->fd, params->http_resp_size,
+	HTC_Init(w->htc, w->ws, vc->fd, params->http_resp_size,
 	    params->http_resp_hdr_len);
 
 	VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout);
 
-	i = HTC_Rx(sp->wrk->htc);
+	i = HTC_Rx(w->htc);
 
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "http first read error: %d %d (%s)",
-		    i, errno, strerror(errno));
+		    i, errno, w->htc->error);
 		VDI_CloseFd(sp);
 		/* XXX: other cleanup ? */
 		/* Retryable if we never received anything */
@@ -456,20 +458,20 @@ FetchHdr(struct sess *sp)
 	VTCP_set_read_timeout(vc->fd, vc->between_bytes_timeout);
 
 	while (i == 0) {
-		i = HTC_Rx(sp->wrk->htc);
+		i = HTC_Rx(w->htc);
 		if (i < 0) {
 			WSP(sp, SLT_FetchError,
 			    "http first read error: %d %d (%s)",
-			    i, errno, strerror(errno));
+			    i, errno, w->htc->error);
 			VDI_CloseFd(sp);
 			/* XXX: other cleanup ? */
 			return (-1);
 		}
 	}
 
-	hp = sp->wrk->beresp;
+	hp = w->beresp;
 
-	if (http_DissectResponse(sp->wrk, sp->wrk->htc, hp)) {
+	if (http_DissectResponse(w, w->htc, hp)) {
 		WSP(sp, SLT_FetchError, "http format error");
 		VDI_CloseFd(sp);
 		/* XXX: other cleanup ? */
@@ -485,22 +487,24 @@ FetchBody(struct sess *sp)
 {
 	int cls;
 	struct storage *st;
+	struct worker *w;
 	int mklen;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+	w = sp->wrk;
 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->obj->http, HTTP_MAGIC);
 
-	if (sp->wrk->vfp == NULL)
-		sp->wrk->vfp = &vfp_nop;
+	if (w->vfp == NULL)
+		w->vfp = &vfp_nop;
 
 	AN(sp->director);
 	AssertObjCorePassOrBusy(sp->obj->objcore);
 
-	AZ(sp->wrk->vgz_rx);
+	AZ(w->vgz_rx);
 	AZ(VTAILQ_FIRST(&sp->obj->store));
-	switch (sp->wrk->body_status) {
+	switch (w->body_status) {
 	case BS_NONE:
 		cls = 0;
 		mklen = 0;
@@ -510,20 +514,20 @@ FetchBody(struct sess *sp)
 		mklen = 1;
 		break;
 	case BS_LENGTH:
-		cls = fetch_straight(sp, sp->wrk->htc,
-		    sp->wrk->h_content_length);
+		cls = fetch_straight(sp, w->htc,
+		    w->h_content_length);
 		mklen = 1;
-		XXXAZ(sp->wrk->vfp->end(sp));
+		XXXAZ(w->vfp->end(sp));
 		break;
 	case BS_CHUNKED:
-		cls = fetch_chunked(sp, sp->wrk->htc);
+		cls = fetch_chunked(sp, w->htc);
 		mklen = 1;
-		XXXAZ(sp->wrk->vfp->end(sp));
+		XXXAZ(w->vfp->end(sp));
 		break;
 	case BS_EOF:
-		cls = fetch_eof(sp, sp->wrk->htc);
+		cls = fetch_eof(sp, w->htc);
 		mklen = 1;
-		XXXAZ(sp->wrk->vfp->end(sp));
+		XXXAZ(w->vfp->end(sp));
 		break;
 	case BS_ERROR:
 		cls = 1;
@@ -534,26 +538,26 @@ FetchBody(struct sess *sp)
 		mklen = 0;
 		INCOMPL();
 	}
-	AZ(sp->wrk->vgz_rx);
+	AZ(w->vgz_rx);
 
 	/*
 	 * It is OK for ->end to just leave the last storage segment
-	 * sitting on sp->wrk->storage, we will always call vfp_nop_end()
+	 * sitting on w->storage, we will always call vfp_nop_end()
 	 * to get it trimmed or thrown out if empty.
 	 */
 	AZ(vfp_nop_end(sp));
 
-	WSL(sp->wrk, SLT_Fetch_Body, sp->vbc->fd, "%u(%s) %d %u",
-	    sp->wrk->body_status, body_status(sp->wrk->body_status),
+	WSL(w, SLT_Fetch_Body, sp->vbc->fd, "%u(%s) cls %d mklen %u",
+	    w->body_status, body_status(w->body_status),
 	    cls, mklen);
 
-	if (sp->wrk->body_status == BS_ERROR) {
+	if (w->body_status == BS_ERROR) {
 		VDI_CloseFd(sp);
 		return (__LINE__);
 	}
 
 	if (cls < 0) {
-		sp->wrk->stats.fetch_failed++;
+		w->stats.fetch_failed++;
 		/* XXX: Wouldn't this store automatically be released ? */
 		while (!VTAILQ_EMPTY(&sp->obj->store)) {
 			st = VTAILQ_FIRST(&sp->obj->store);
@@ -565,10 +569,10 @@ FetchBody(struct sess *sp)
 		return (__LINE__);
 	}
 
-	if (cls == 0 && sp->wrk->do_close)
+	if (cls == 0 && w->do_close)
 		cls = 1;
 
-	WSL(sp->wrk, SLT_Length, sp->vbc->fd, "%u", sp->obj->len);
+	WSL(w, SLT_Length, sp->vbc->fd, "%u", sp->obj->len);
 
 	{
 	/* Sanity check fetch methods accounting */
@@ -586,7 +590,7 @@ FetchBody(struct sess *sp)
 
 	if (mklen > 0) {
 		http_Unset(sp->obj->http, H_Content_Length);
-		http_PrintfHeader(sp->wrk, sp->fd, sp->obj->http,
+		http_PrintfHeader(w, sp->fd, sp->obj->http,
 		    "Content-Length: %jd", (intmax_t)sp->obj->len);
 	}
 
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index bed3104..675718a 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -469,8 +469,10 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			bytes -= w;
 		}
 
-		if (VGZ_ObufStorage(sp, vg))
+		if (VGZ_ObufStorage(sp, vg)) {
+			htc->error = "Could not get storage";
 			return (-1);
+		}
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		assert(i == VGZ_OK || i == VGZ_END);
 		sp->obj->len += dl;
@@ -479,6 +481,8 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	}
 	if (i == Z_OK || i == Z_STREAM_END)
 		return (1);
+	htc->error = "See other message";
+	WSP(sp, SLT_FetchError, "Gunzip trouble (%d)", i);
 	return (-1);
 }
 
@@ -540,8 +544,10 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
-		if (VGZ_ObufStorage(sp, vg))
+		if (VGZ_ObufStorage(sp, vg)) {
+			htc->error = "Could not get storage";
 			return (-1);
+		}
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
 		sp->obj->len += dl;
@@ -612,8 +618,10 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	AZ(vg->vz.avail_in);
 	while (bytes > 0) {
 		st = FetchStorage(sp, 0);
-		if (st == NULL)
+		if (st == NULL) {
+			htc->error = "Could not get storage";
 			return (-1);
+		}
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
@@ -631,10 +639,11 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			VGZ_Obuf(vg, obuf, sizeof obuf);
 			i = VGZ_Gunzip(vg, &dp, &dl);
 			if (i == VGZ_END && !VGZ_IbufEmpty(vg)) {
-				WSP(sp, SLT_FetchError, "Junk after gzip data");
+				htc->error = "Junk after gzip data";
 				return (-1);
 			}
 			if (i != VGZ_OK && i != VGZ_END) {
+				htc->error = "See other message";
 				WSP(sp, SLT_FetchError,
 				    "Invalid Gzip data: %s", vg->vz.msg);
 				return (-1);
@@ -643,6 +652,7 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	}
 	if (i == VGZ_OK || i == VGZ_END)
 		return (1);
+	htc->error = "See other message";
 	WSP(sp, SLT_FetchError, "Gunzip trouble (%d)", i);
 	return (-1);
 }
@@ -665,5 +675,3 @@ struct vfp vfp_testgzip = {
         .bytes  =       vfp_testgzip_bytes,
         .end    =       vfp_testgzip_end,
 };
-
-
diff --git a/bin/varnishd/cache_httpconn.c b/bin/varnishd/cache_httpconn.c
index 3840ed6..9d6d926 100644
--- a/bin/varnishd/cache_httpconn.c
+++ b/bin/varnishd/cache_httpconn.c
@@ -88,6 +88,7 @@ HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned maxbytes,
 	htc->fd = fd;
 	htc->maxbytes = maxbytes;
 	htc->maxhdr = maxhdr;
+	htc->error = "No error recorded";
 
 	(void)WS_Reserve(htc->ws, htc->maxbytes);
 	htc->rxbuf.b = ws->f;
@@ -109,6 +110,7 @@ HTC_Reinit(struct http_conn *htc)
 	unsigned l;
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	htc->error = "No error recorded";
 	(void)WS_Reserve(htc->ws, htc->maxbytes);
 	htc->rxbuf.b = htc->ws->f;
 	htc->rxbuf.e = htc->ws->f;
@@ -201,7 +203,10 @@ HTC_Read(struct http_conn *htc, void *d, size_t len)
 	if (len == 0)
 		return (l);
 	i = read(htc->fd, p, len);
-	if (i < 0)
+	if (i < 0) {
+		htc->error = strerror(errno);
 		return (i);
+	} else if (i == 0)
+		htc->error = "Remote closed connection";
 	return (i + l);
 }



From phk at varnish-cache.org  Wed Sep  7 19:31:28 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 07 Sep 2011 21:31:28 +0200
Subject: [master] b3690c6 Make it possible for VCL initializtion to fail
	gracefully,
	particularly so VMOD loading can emit sensible diagnostics.
Message-ID: <E1R1NqK-0005d3-NZ@project.varnish-software.com>

commit b3690c63fdbb06caf6b33aa55b8ca83a1b8455d2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 19:30:48 2011 +0000

    Make it possible for VCL initializtion to fail gracefully, particularly
    so VMOD loading can emit sensible diagnostics.

diff --git a/bin/varnishd/cache_vcl.c b/bin/varnishd/cache_vcl.c
index 49f3fbb..ac712fb 100644
--- a/bin/varnishd/cache_vcl.c
+++ b/bin/varnishd/cache_vcl.c
@@ -170,10 +170,15 @@ VCL_Load(const char *fn, const char *name, struct cli *cli)
 		FREE_OBJ(vcl);
 		return (1);
 	}
+	if (vcl->conf->init_vcl(cli)) {
+		VCLI_Out(cli, "VCL \"%s\" Failed to initialize", name);
+		(void)dlclose(vcl->dlh);
+		FREE_OBJ(vcl);
+		return (1);
+	}
 	REPLACE(vcl->name, name);
-	VTAILQ_INSERT_TAIL(&vcl_head, vcl, list);
 	VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name);
-	vcl->conf->init_vcl(cli);
+	VTAILQ_INSERT_TAIL(&vcl_head, vcl, list);
 	(void)vcl->conf->init_func(NULL);
 	Lck_Lock(&vcl_mtx);
 	if (vcl_active == NULL)
diff --git a/bin/varnishd/cache_vrt_vmod.c b/bin/varnishd/cache_vrt_vmod.c
index 4a4d230..84a2bb7 100644
--- a/bin/varnishd/cache_vrt_vmod.c
+++ b/bin/varnishd/cache_vrt_vmod.c
@@ -60,12 +60,12 @@ struct vmod {
 
 static VTAILQ_HEAD(,vmod)	vmods = VTAILQ_HEAD_INITIALIZER(vmods);
 
-void
-VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, const char *path)
+int
+VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
+    const char *path, struct cli *cli)
 {
 	struct vmod *v;
-	void *x;
-	const int *i;
+	void *x, *y, *z;
 
 	ASSERT_CLI();
 
@@ -76,31 +76,46 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, const char *path)
 		ALLOC_OBJ(v, VMOD_MAGIC);
 		AN(v);
 
-		VTAILQ_INSERT_TAIL(&vmods, v, list);
-		VSC_C_main->vmods++;
-
-		REPLACE(v->nm, nm);
-		REPLACE(v->path, path);
-
-		v->hdl = dlopen(v->path, RTLD_NOW | RTLD_LOCAL);
-		if (! v->hdl) {
-			char buf[1024];
-			sprintf(buf, "dlopen failed (child process lacks permission?): %.512s", dlerror());
-			VAS_Fail(__func__, __FILE__, __LINE__, buf, 0, 0);
+		v->hdl = dlopen(path, RTLD_NOW | RTLD_LOCAL);
+		if (v->hdl == NULL) {
+			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
+			VCLI_Out(cli, "dlopen() failed: %s\n", dlerror());
+			VCLI_Out(cli, "Check child process permissions.\n"); 
+			FREE_OBJ(v);
+			return (1);
 		}
 
 		x = dlsym(v->hdl, "Vmod_Name");
+		y = dlsym(v->hdl, "Vmod_Len");
+		z = dlsym(v->hdl, "Vmod_Func");
+		if (x == NULL || y == NULL || z == NULL) {
+			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
+			VCLI_Out(cli, "VMOD symbols not found\n");
+			VCLI_Out(cli, "Check relative pathnames.\n");
+			(void)dlclose(v->hdl);
+			FREE_OBJ(v);
+			return (1);
+		}
 		AN(x);
-		/* XXX: check that name is correct */
+		AN(y);
+		AN(z);
+		if (strcmp(x, nm)) {
+			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
+			VCLI_Out(cli, "File contain wrong VMOD (\"%s\")\n", x);
+			VCLI_Out(cli, "Check relative pathnames ?.\n");
+			(void)dlclose(v->hdl);
+			FREE_OBJ(v);
+			return (1);
+		}
 
-		x = dlsym(v->hdl, "Vmod_Len");
-		AN(x);
-		i = x;
-		v->funclen = *i;
+		v->funclen = *(const int *)y;
+		v->funcs = z;
 
-		x = dlsym(v->hdl, "Vmod_Func");
-		AN(x);
-		v->funcs = x;
+		REPLACE(v->nm, nm);
+		REPLACE(v->path, path);
+
+		VSC_C_main->vmods++;
+		VTAILQ_INSERT_TAIL(&vmods, v, list);
 	}
 
 	assert(len == v->funclen);
@@ -108,6 +123,7 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, const char *path)
 	v->ref++;
 
 	*hdl = v;
+	return (0);
 }
 
 void
diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index 2664931..ebdf57b 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -385,7 +385,7 @@ start_child(struct cli *cli)
 	mgt_cli_start_child(child_cli_in, child_VCLI_Out);
 	child_pid = pid;
 	if (mgt_push_vcls_and_start(&u, &p)) {
-		REPORT(LOG_ERR, "Pushing vcls failed: %s", p);
+		REPORT(LOG_ERR, "Pushing vcls failed:\n%s", p);
 		free(p);
 		child_state = CH_RUNNING;
 		mgt_stop_child();
diff --git a/include/vrt.h b/include/vrt.h
index cfafe75..9f0228f 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -183,8 +183,8 @@ void VRT_init_dir(struct cli *, struct director **, const char *name,
 void VRT_fini_dir(struct cli *, struct director *);
 
 /* VMOD/Modules related */
-void VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
-    const char *path);
+int VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
+    const char *path, struct cli *cli);
 void VRT_Vmod_Fini(void **hdl);
 
 struct vmod_priv;
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 5407a51..fde8bb2 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -712,7 +712,7 @@ fo.write("""
 struct sess;
 struct cli;
 
-typedef void vcl_init_f(struct cli *);
+typedef int vcl_init_f(struct cli *);
 typedef void vcl_fini_f(struct cli *);
 typedef int vcl_func_f(struct sess *sp);
 """)
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index b0019ca..c2a52d4 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -310,9 +310,10 @@ static void
 EmitInitFunc(const struct vcc *tl)
 {
 
-	Fc(tl, 0, "\nstatic void\nVGC_Init(struct cli *cli)\n{\n\n");
+	Fc(tl, 0, "\nstatic int\nVGC_Init(struct cli *cli)\n{\n\n");
 	AZ(VSB_finish(tl->fi));
 	VSB_cat(tl->fc, VSB_data(tl->fi));
+	Fc(tl, 0, "\treturn(0);\n");
 	Fc(tl, 0, "}\n");
 }
 
@@ -321,7 +322,7 @@ EmitFiniFunc(const struct vcc *tl)
 {
 	unsigned u;
 
-	Fc(tl, 0, "\nstatic void\nVGC_Fini(struct cli *cli)\n{\n\n");
+	Fc(tl, 0, "\nstatic int\nVGC_Fini(struct cli *cli)\n{\n\n");
 
 	/*
 	 * We do this here, so we are sure they happen before any
@@ -332,6 +333,7 @@ EmitFiniFunc(const struct vcc *tl)
 
 	AZ(VSB_finish(tl->ff));
 	VSB_cat(tl->fc, VSB_data(tl->ff));
+	Fc(tl, 0, "\treturn(0);\n");
 	Fc(tl, 0, "}\n");
 }
 
diff --git a/lib/libvcl/vcc_vmod.c b/lib/libvcl/vcc_vmod.c
index 35cc702..f74f874 100644
--- a/lib/libvcl/vcc_vmod.c
+++ b/lib/libvcl/vcc_vmod.c
@@ -102,13 +102,15 @@ vcc_ParseImport(struct vcc *tl)
 
 	Fh(tl, 0, "static void *VGC_vmod_%.*s;\n", PF(mod));
 
-	Fi(tl, 0, "\tVRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod));
+	Fi(tl, 0, "\tif (VRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod));
 	Fi(tl, 0, "\t    &Vmod_Func_%.*s,\n", PF(mod));
 	Fi(tl, 0, "\t    sizeof(Vmod_Func_%.*s),\n", PF(mod));
 	Fi(tl, 0, "\t    \"%.*s\",\n", PF(mod));
 	Fi(tl, 0, "\t    ");
 	EncString(tl->fi, fn, NULL, 0);
-	Fi(tl, 0, ");\n");
+	Fi(tl, 0, ",\n\t    ");
+	Fi(tl, 0, "cli))\n");
+	Fi(tl, 0, "\t\treturn(1);\n");
 
 	SkipToken(tl, ';');
 



From phk at varnish-cache.org  Wed Sep  7 19:54:47 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 07 Sep 2011 21:54:47 +0200
Subject: [master] eea97db Prototype adjustment for last commit
Message-ID: <E1R1OCt-0007Os-2B@project.varnish-software.com>

commit eea97dbea388be4d52a5919b0ec938bae4d0e6e3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 19:54:35 2011 +0000

    Prototype adjustment for last commit

diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index c2a52d4..f4a85a0 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -322,7 +322,7 @@ EmitFiniFunc(const struct vcc *tl)
 {
 	unsigned u;
 
-	Fc(tl, 0, "\nstatic int\nVGC_Fini(struct cli *cli)\n{\n\n");
+	Fc(tl, 0, "\nstatic void\nVGC_Fini(struct cli *cli)\n{\n\n");
 
 	/*
 	 * We do this here, so we are sure they happen before any
@@ -333,7 +333,6 @@ EmitFiniFunc(const struct vcc *tl)
 
 	AZ(VSB_finish(tl->ff));
 	VSB_cat(tl->fc, VSB_data(tl->ff));
-	Fc(tl, 0, "\treturn(0);\n");
 	Fc(tl, 0, "}\n");
 }
 



From phk at varnish-cache.org  Wed Sep  7 20:16:47 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 07 Sep 2011 22:16:47 +0200
Subject: [master] 3302916 Quench a pointless debug printf
Message-ID: <E1R1OYB-0000K8-Bq@project.varnish-software.com>

commit 3302916f320dd64ad5a866b03687306ceb441936
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 20:13:07 2011 +0000

    Quench a pointless debug printf

diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index 7687179..45dfac4 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -104,13 +104,10 @@ struct smf_sc {
 /*--------------------------------------------------------------------*/
 
 static void
-smf_initfile(struct stevedore *st, struct smf_sc *sc, const char *size)
+smf_initfile(struct smf_sc *sc, const char *size)
 {
 	sc->filesize = STV_FileSize(sc->fd, size, &sc->pagesize, "-sfile");
 
-	printf("SMF.%s: filename: %s size %ju MB.\n",
-	      st->ident, sc->filename, sc->filesize / (1024 * 1024));
-
 	AZ(ftruncate(sc->fd, (off_t)sc->filesize));
 
 	/* XXX: force block allocation here or in open ? */
@@ -162,7 +159,7 @@ smf_init(struct stevedore *parent, int ac, char * const *av)
 	(void)STV_GetFile(fn, &sc->fd, &sc->filename, "-sfile");
 
 	mgt_child_inherit(sc->fd, "storage_file");
-	smf_initfile(parent, sc, size);
+	smf_initfile(sc, size);
 }
 
 /*--------------------------------------------------------------------



From phk at varnish-cache.org  Wed Sep  7 20:16:49 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 07 Sep 2011 22:16:49 +0200
Subject: [master] 8b4c13c Don't leave pid file in weird error case.
Message-ID: <E1R1OYD-0000KD-Ak@project.varnish-software.com>

commit 8b4c13c4f3c4aa73d2466445459d4c950b6689d6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 20:16:30 2011 +0000

    Don't leave pid file in weird error case.

diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index ebdf57b..9d6966a 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -595,8 +595,10 @@ MGT_Run(void)
 		REPORT0(LOG_ERR, "No VCL loaded yet");
 	else if (!d_flag) {
 		start_child(NULL);
-		if (child_state == CH_STOPPED)
-			exit(2);
+		if (child_state == CH_STOPPED) {
+			exit_status = 2;
+			return;
+		}
 	}
 
 	i = vev_schedule(mgt_evb);



From kristian at varnish-cache.org  Thu Sep  8 07:49:25 2011
From: kristian at varnish-cache.org (=?UTF-8?Q?Kristian_Lyngst=C3=B8l?=)
Date: Thu, 08 Sep 2011 09:49:25 +0200
Subject: [master] 20732d8 Add return(restart) in vcl_error to the dot-graph
Message-ID: <E1R1ZMT-0008Tl-2u@project.varnish-software.com>

commit 20732d87437fd0daa32d5abbbc85c38f08a8991a
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Thu Sep 8 09:48:50 2011 +0200

    Add return(restart) in vcl_error to the dot-graph
    
    Thanks to scoof for pointing this out.

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 6bb8324..39a2104 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -413,6 +413,8 @@ DOT	]
 DOT	ERROR -> vcl_error
 DOT	vcl_error-> prepresp [label=deliver]
 DOT }
+DOT vcl_error-> rsterr [label="restart",color=purple]
+DOT rsterr [label="RESTART",shape=plaintext]
  */
 
 static int



From phk at varnish-cache.org  Thu Sep  8 21:50:44 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 08 Sep 2011 23:50:44 +0200
Subject: [master] 3a08c3d Add Vmod_Id handle which VMOD can identify
	themselves to VRT with.
Message-ID: <E1R1mUe-0000YO-Hk@project.varnish-software.com>

commit 3a08c3d95cace24133756e5e204ef73545921171
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 8 11:53:05 2011 +0000

    Add Vmod_Id handle which VMOD can identify themselves to VRT with.

diff --git a/bin/varnishd/cache_vrt_vmod.c b/bin/varnishd/cache_vrt_vmod.c
index 84a2bb7..bd2e172 100644
--- a/bin/varnishd/cache_vrt_vmod.c
+++ b/bin/varnishd/cache_vrt_vmod.c
@@ -56,6 +56,7 @@ struct vmod {
 	void			*hdl;
 	const void		*funcs;
 	int			funclen;
+	const void		*idptr;
 };
 
 static VTAILQ_HEAD(,vmod)	vmods = VTAILQ_HEAD_INITIALIZER(vmods);
@@ -65,12 +66,12 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
     const char *path, struct cli *cli)
 {
 	struct vmod *v;
-	void *x, *y, *z;
+	void *x, *y, *z, *w;
 
 	ASSERT_CLI();
 
 	VTAILQ_FOREACH(v, &vmods, list)
-		if (!strcmp(v->nm, nm))
+		if (!strcmp(v->nm, nm))	// Also path, len ?
 			break;
 	if (v == NULL) {
 		ALLOC_OBJ(v, VMOD_MAGIC);
@@ -88,7 +89,8 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 		x = dlsym(v->hdl, "Vmod_Name");
 		y = dlsym(v->hdl, "Vmod_Len");
 		z = dlsym(v->hdl, "Vmod_Func");
-		if (x == NULL || y == NULL || z == NULL) {
+		w = dlsym(v->hdl, "Vmod_Id");
+		if (x == NULL || y == NULL || z == NULL || w == NULL) {
 			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
 			VCLI_Out(cli, "VMOD symbols not found\n");
 			VCLI_Out(cli, "Check relative pathnames.\n");
@@ -99,6 +101,7 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 		AN(x);
 		AN(y);
 		AN(z);
+		AN(w);
 		if (strcmp(x, nm)) {
 			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
 			VCLI_Out(cli, "File contain wrong VMOD (\"%s\")\n", x);
@@ -116,6 +119,7 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 
 		VSC_C_main->vmods++;
 		VTAILQ_INSERT_TAIL(&vmods, v, list);
+		v->idptr = w;
 	}
 
 	assert(len == v->funclen);
diff --git a/lib/libvmod_std/vmod.py b/lib/libvmod_std/vmod.py
index 83f8eca..1b0f1c0 100755
--- a/lib/libvmod_std/vmod.py
+++ b/lib/libvmod_std/vmod.py
@@ -308,5 +308,9 @@ fc.write("\n");
 fc.write('const char * const Vmod_Spec[] = {\n' + slist + '\t0\n};\n')
 
 fc.write('const char Vmod_Varnish_ABI[] = VMOD_ABI_Version;\n')
+
+fh.write('extern const void * const Vmod_Id;\n')
+fc.write('const void * const Vmod_Id = &Vmod_Id;\n')
+
 fc.write("\n")
 



From phk at varnish-cache.org  Thu Sep  8 21:50:48 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 08 Sep 2011 23:50:48 +0200
Subject: [master] 4213531 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1R1mUi-0000YT-Qi@project.varnish-software.com>

commit 4213531e8043bf382f9070e0bd6fac2aa4c8d547
Merge: 3a08c3d 20732d8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 8 21:50:34 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From apj at varnish-cache.org  Fri Sep  9 11:36:16 2011
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Fri, 09 Sep 2011 13:36:16 +0200
Subject: [master] 8156c29 Restore old nuke_limit, as 10 seems to be too low
Message-ID: <E1R1zNY-0001jl-IO@project.varnish-software.com>

commit 8156c297082a43396324f062bdc97aba2a9af2ec
Author: Andreas Plesner Jacobsen <apj at marvin.nerd.dk>
Date:   Fri Sep 9 13:33:38 2011 +0200

    Restore old nuke_limit, as 10 seems to be too low
    
    Fixes #1012

diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index de93679..33101bb 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -622,7 +622,7 @@ static const struct parspec input_parspec[] = {
 		"Maximum number of objects we attempt to nuke in order"
 		"to make space for a object body.",
 		EXPERIMENTAL,
-		"10", "allocations" },
+		"50", "allocations" },
 	{ "fetch_chunksize",
 		tweak_uint, &master.fetch_chunksize, 4, UINT_MAX / 1024.,
 		"The default chunksize used by fetcher. "



From apj at varnish-cache.org  Fri Sep  9 12:56:30 2011
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Fri, 09 Sep 2011 14:56:30 +0200
Subject: [master] 76a050f Add .expected_response to probe documentation. Fix
	error message for .expected_response.
Message-ID: <E1R20dC-00067K-P7@project.varnish-software.com>

commit 76a050fd10b6ab423be673cce5ee12fda049356f
Author: Andreas Plesner Jacobsen <apj at marvin.nerd.dk>
Date:   Fri Sep 9 14:54:24 2011 +0200

    Add .expected_response to probe documentation.
    Fix error message for .expected_response.
    
    Fixes #997

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 7359e6f..0e39187 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -284,7 +284,8 @@ req.backend.healthy .window is how many of the latest polls we
 examine, while .threshold is how many of those must have succeeded for
 us to consider the backend healthy.  .initial is how many of the
 probes are considered good when Varnish starts - defaults to the same
-amount as the threshold.
+amount as the threshold. .expected_response is the expected backend
+HTTP response code.
 
 A backend with a probe can be defined like this, together with the
 backend or director:::
@@ -310,6 +311,7 @@ Or it can be defined separately and then referenced:::
      .window = 8;
      .threshold = 3;
      .initial = 3;
+     .expected_response = 200;
   }	
 
   backend www {
diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c
index 1b8f8a0..f79447f 100644
--- a/lib/libvcl/vcc_backend.c
+++ b/lib/libvcl/vcc_backend.c
@@ -305,8 +305,9 @@ vcc_ParseProbeSpec(struct vcc *tl)
 			status = vcc_UintVal(tl);
 			if (status < 100 || status > 999) {
 				VSB_printf(tl->sb,
-				    "Must specify .status with exactly three "
-				    " digits (100 <= x <= 999)\n");
+				    "Must specify .expected_response with "
+				    "exactly three digits "
+				    "(100 <= x <= 999)\n");
 				vcc_ErrWhere(tl, tl->t);
 				return;
 			}



From apj at varnish-cache.org  Fri Sep  9 13:31:42 2011
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Fri, 09 Sep 2011 15:31:42 +0200
Subject: [master] ce9dd3b Add type to connect_timeout
Message-ID: <E1R21BG-0000YY-Iu@project.varnish-software.com>

commit ce9dd3bb3b887731a203781e6d23b85010497191
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Fri Sep 9 15:30:55 2011 +0200

    Add type to connect_timeout
    
    References #971

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 0e39187..aae16bc 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -232,7 +232,7 @@ random or round-robin director or using .list::
           .list = {
                   .host_header = "www.example.com";
                   .port = "80";
-                  .connect_timeout = 0.4;
+                  .connect_timeout = 0.4s;
                   "192.168.15.0"/24;
                   "192.168.16.128"/25;
           }



From apj at varnish-cache.org  Fri Sep  9 15:44:12 2011
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Fri, 09 Sep 2011 17:44:12 +0200
Subject: [master] f7c36f9 Tabulate probe properties and add all missing ones
Message-ID: <E1R23FU-0008VR-U6@project.varnish-software.com>

commit f7c36f94d4be6f1b92c93d9c723267700a8c5b97
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Fri Sep 9 17:43:31 2011 +0200

    Tabulate probe properties and add all missing ones
    
    Really fixes #997

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index aae16bc..517f4fb 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -280,12 +280,36 @@ Backend probes
 
 Backends can be probed to see whether they should be considered
 healthy or not.  The return status can also be checked by using
-req.backend.healthy .window is how many of the latest polls we
-examine, while .threshold is how many of those must have succeeded for
-us to consider the backend healthy.  .initial is how many of the
-probes are considered good when Varnish starts - defaults to the same
-amount as the threshold. .expected_response is the expected backend
-HTTP response code.
+req.backend.healthy.
+
+Probes take the following parameters:
+
+.url
+  Specify a URL to request from the backend.
+  Defaults to "/".
+.request
+  Specify a full HTTP request using multiple strings.
+  .request will have \r\n automatically inserted after every string.
+  If specified, .request will take precedence over .url.
+.window
+  How many of the latest polls we examine to determine backend health.
+  Defaults to 8.
+.threshold 
+  How many of the polls in .window must have succeeded for us to consider
+  the backend healthy.
+  Defaults to 3.
+.initial
+  How many of the probes are considered good when Varnish starts.
+  Defaults to the same amount as the threshold.
+.expected_response
+  The expected backend HTTP response code.
+  Defaults to 200.
+.interval
+  Defines how often the probe should check the backend.
+  Default is every 5 seconds.
+.timeout
+  How fast each probe times out.
+  Default is 2 seconds.
 
 A backend with a probe can be defined like this, together with the
 backend or director:::



From apj at varnish-cache.org  Sat Sep 10 13:18:24 2011
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Sat, 10 Sep 2011 15:18:24 +0200
Subject: [master] ef0387e Fix escapes,
	and workaround man-page formatting issues
Message-ID: <E1R2NRw-0003N2-FB@project.varnish-software.com>

commit ef0387ea8c123a757e36cf6ef60985440e56a37d
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Sat Sep 10 15:16:37 2011 +0200

    Fix escapes, and workaround man-page formatting issues

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 517f4fb..e3169ea 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -288,8 +288,8 @@ Probes take the following parameters:
   Specify a URL to request from the backend.
   Defaults to "/".
 .request
-  Specify a full HTTP request using multiple strings.
-  .request will have \r\n automatically inserted after every string.
+  Specify a full HTTP request using multiple strings. .request will
+  have \\r\\n automatically inserted after every string.
   If specified, .request will take precedence over .url.
 .window
   How many of the latest polls we examine to determine backend health.



From apj at varnish-cache.org  Sun Sep 11 10:54:29 2011
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Sun, 11 Sep 2011 12:54:29 +0200
Subject: [master] e13b98c Add undocumented variables
Message-ID: <E1R2hgD-00058J-61@project.varnish-software.com>

commit e13b98cc8deff63a6d1470eb0f795d200e83929f
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Sun Sep 11 12:52:01 2011 +0200

    Add undocumented variables

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index e3169ea..9fd526c 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -687,6 +687,9 @@ The following variables are available while processing a request:
 client.ip
   The client's IP address.
 
+client.identity
+  Identification of the client, used to load balance in the client director.
+
 server.hostname
   The host name of the server.
 
@@ -733,6 +736,21 @@ req.hash_ignore_busy
 req.can_gzip
   Does the client accept the gzip transfer encoding.
 
+req.restarts
+  A count of how many times this request has been restarted.
+
+req.esi
+  True if the request is an ESI request.
+
+req.esi_level
+  A count of how many levels of ESI requests we're currently at.
+
+req.grace
+  Set to a period to enable grace.
+
+req.xid
+  Unique ID of this request.
+
 The following variables are available while preparing a backend
 request (either for a cache miss or for pass or pipe mode):
 
@@ -792,6 +810,25 @@ beresp.response
 beresp.ttl
   The object's remaining time to live, in seconds. beresp.ttl is writable.
 
+beresp.grace
+  Set to a period to enable grace.
+
+beresp.saintmode
+  Set to a period to enable saint mode.
+
+beresp.backend.name
+  Name of the backend this response was fetched from.
+
+beresp.backend.ip
+  IP of the backend this response was fetched from.
+
+beresp.backend.port
+  Port of the backend this response was fetched from.
+
+beresp.storage
+  Set to force Varnish to save this object to a particular storage
+  backend.
+
 After the object is entered into the cache, the following (mostly
 read-only) variables are available when the object has been located in
 cache, typically in vcl_hit and vcl_deliver.
@@ -816,6 +853,12 @@ obj.hits
   The approximate number of times the object has been delivered. A value 
   of 0 indicates a cache miss.
 
+obj.grace
+  The object's grace period in seconds. obj.grace is writable.
+
+obj.http.header
+  The corresponding HTTP header.
+
 The following variables are available while determining the hash key
 of an object:
 
@@ -1000,8 +1043,8 @@ and Per Buer.
 COPYRIGHT
 =========
 
-This document is licensed under the same licence as Varnish
-itself. See LICENCE for details.
+This document is licensed under the same license as Varnish
+itself. See LICENSE for details.
 
 * Copyright (c) 2006 Verdens Gang AS
 * Copyright (c) 2006-2011 Varnish Software AS



From phk at varnish-cache.org  Mon Sep 12 07:36:56 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Sep 2011 09:36:56 +0200
Subject: [master] c717971 Always call fetch-processor begin,
	also if C-L header is bogus.
Message-ID: <E1R314a-0001Ci-H8@project.varnish-software.com>

commit c71797102fae2ea0775ceabba96431990394939f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 12 07:36:12 2011 +0000

    Always call fetch-processor begin, also if C-L header is bogus.
    
    Fixes	#1014

diff --git a/bin/varnishtest/tests/r01014.vtc b/bin/varnishtest/tests/r01014.vtc
new file mode 100644
index 0000000..53a4473
--- /dev/null
+++ b/bin/varnishtest/tests/r01014.vtc
@@ -0,0 +1,13 @@
+varnishtest "bug 1014, Invalid C-L header with gzip"
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Content-Encoding: gzip" -hdr "Content-Length:"
+} -start
+
+varnish v1 -vcl+backend { } -start
+
+client c1 {
+	txreq
+	rxresp
+} -run



From phk at varnish-cache.org  Mon Sep 12 07:36:59 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Sep 2011 09:36:59 +0200
Subject: [master] 87ae60f Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1R314d-0001Cr-Hy@project.varnish-software.com>

commit 87ae60f887d7b487be548898b64ac8183b797187
Merge: c717971 e13b98c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 12 07:36:51 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From phk at varnish-cache.org  Mon Sep 12 10:13:54 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Sep 2011 12:13:54 +0200
Subject: [master] b16790a Avoid 32 bit variables in stats aggregation.
Message-ID: <E1R33WU-0002xt-Ap@project.varnish-software.com>

commit b16790a74f5e5ffb34d2f27027778399a90a3f71
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 12 10:13:35 2011 +0000

    Avoid 32 bit variables in stats aggregation.
    
    Fixes	#993

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 1c1952d..e2faf30 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -218,7 +218,7 @@ struct acct {
 /*--------------------------------------------------------------------*/
 
 #define L0(n)
-#define L1(n)			int n;
+#define L1(n)			uint64_t n;
 #define VSC_F(n, t, l, f, e)	L##l(n)
 #define VSC_DO_MAIN
 struct dstat {



From phk at varnish-cache.org  Mon Sep 12 10:14:59 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Sep 2011 12:14:59 +0200
Subject: [master] f336ffe Commit the actual fix for #1014 and not just the
	test-case.
Message-ID: <E1R33XX-0003vg-9U@project.varnish-software.com>

commit f336ffe66fe7f0a565539cf83e46aa93ba96f5ef
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 12 10:14:27 2011 +0000

    Commit the actual fix for #1014 and not just the test-case.
    
    Fixes	#1014

diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 36df887..73edaa1 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -202,20 +202,15 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 	ssize_t cl;
 
 	assert(sp->wrk->body_status == BS_LENGTH);
+
 	cl = fetch_number(b, 10);
+	sp->wrk->vfp->begin(sp, cl > 0 ? cl : 0);
 	if (cl < 0) {
 		WSP(sp, SLT_FetchError, "straight length field bogus");
 		return (-1);
-	}
-	/*
-	 * XXX: we shouldn't need this if we have cl==0
-	 * XXX: but we must also conditionalize the vfp->end()
-	 */
-	sp->wrk->vfp->begin(sp, cl);
-	if (cl == 0)
+	} else if (cl == 0)
 		return (0);
 
-
 	i = sp->wrk->vfp->bytes(sp, htc, cl);
 	if (i <= 0) {
 		WSP(sp, SLT_FetchError, "straight read_error: %d %d (%s)",
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 675718a..b32b5cf 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -600,6 +600,7 @@ vfp_testgzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
 	sp->wrk->vgz_rx = VGZ_NewUngzip(sp, "u F -");
+	CHECK_OBJ_NOTNULL(sp->wrk->vgz_rx, VGZ_MAGIC);
 }
 
 static int __match_proto__()



From martin at varnish-cache.org  Mon Sep 12 10:33:15 2011
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Mon, 12 Sep 2011 12:33:15 +0200
Subject: [master] 6447d64 Updating configure line for ppc64,
	as --disabled-jemalloc has changed to --without-jemalloc
Message-ID: <E1R33pD-0001Pu-Gf@project.varnish-software.com>

commit 6447d64122b337c6c6ef662a398eac6babc1dd80
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Sep 12 12:31:13 2011 +0200

    Updating configure line for ppc64, as --disabled-jemalloc has changed
    to --without-jemalloc
    
    Patch from P?l-Eivind Johnsen
    
    Fixes: #1011

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index ca282e2..0d66765 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -87,7 +87,7 @@ cp bin/varnishd/default.vcl etc/zope-plone.vcl examples
 # Remove "--disable static" if you want to build static libraries 
 # jemalloc is not compatible with Red Hat's ppc64 RHEL kernel :-(
 %ifarch ppc64 ppc
-	%configure --disable-static --localstatedir=/var/lib --disable-jemalloc
+	%configure --disable-static --localstatedir=/var/lib --without-jemalloc
 %else
 	%configure --disable-static --localstatedir=/var/lib
 %endif



From phk at varnish-cache.org  Wed Sep 14 07:44:40 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 14 Sep 2011 09:44:40 +0200
Subject: [master] 083d19a Dump the VSL into the varnishtest output.
Message-ID: <E1R3k9A-0005w9-NC@project.varnish-software.com>

commit 083d19a60d7df575333736302f8bd91cd7b50f3e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 14 07:44:17 2011 +0000

    Dump the VSL into the varnishtest output.
    
    Based on code from:	DocWilco

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 2b6e3d4..c53482e 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -47,7 +47,6 @@
 #include "libvarnish.h"
 #include "varnishapi.h"
 #include "vcli.h"
-#include "cli_common.h"
 #include "vss.h"
 #include "vsb.h"
 
@@ -58,7 +57,6 @@ struct varnish {
 #define VARNISH_MAGIC		0x208cd8e3
 	char			*name;
 	struct vtclog		*vl;
-	struct vtclog		*vl1;
 	VTAILQ_ENTRY(varnish)	list;
 
 	struct vsb		*storage;
@@ -68,16 +66,22 @@ struct varnish {
 	pid_t			pid;
 
 	pthread_t		tp;
+	pthread_t		tp_vsl;
 
 	int			cli_fd;
 	int			vcl_nbr;
 	char			*workdir;
 
-	struct VSM_data		*vd;
+	struct VSM_data		*vd;		/* vsc use */
+
+	unsigned		vsl_tag_count[256];
+	unsigned		vsl_sleep;
 };
 
 #define NONSENSE	"%XJEIFLH|)Xspa8P"
 
+#define VSL_SLEEP_USEC	(50*1000)
+
 static VTAILQ_HEAD(, varnish)	varnishes =
     VTAILQ_HEAD_INITIALIZER(varnishes);
 
@@ -116,6 +120,57 @@ varnish_ask_cli(const struct varnish *v, const char *cmd, char **repl)
 }
 
 /**********************************************************************
+ * Varnishlog gatherer + thread
+ */
+
+static int
+h_addlog(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
+    unsigned spec, const char *ptr, uint64_t bitmap)
+{
+	struct varnish *v;
+	int type;
+
+	(void) bitmap;
+
+	type = (spec & VSL_S_CLIENT) ? 'c' :
+	    (spec & VSL_S_BACKEND) ? 'b' : '-';
+	CAST_OBJ_NOTNULL(v, priv, VARNISH_MAGIC);
+
+	v->vsl_tag_count[tag]++;
+
+	vtc_log(v->vl, 4, "vsl| %5u %-12s %c %.*s", fd,
+	    VSL_tags[tag], type, len, ptr);
+	v->vsl_sleep = 100;
+	return (0);
+}
+
+static void *
+varnishlog_thread(void *priv)
+{
+	struct varnish *v;
+	struct VSM_data	*vsl;
+
+	CAST_OBJ_NOTNULL(v, priv, VARNISH_MAGIC);
+	vsl = VSM_New();
+	VSL_Setup(vsl);
+	(void)VSL_Arg(vsl, 'n', v->workdir);
+	VSL_NonBlocking(vsl, 1);
+	while (v->pid  && VSL_Open(vsl, 0) != 0) {
+		assert(usleep(VSL_SLEEP_USEC) == 0 || errno == EINTR);
+	}
+	while (v->pid) {
+		if (VSL_Dispatch(vsl, h_addlog, v) < 0) {
+			assert(usleep(v->vsl_sleep) == 0 || errno == EINTR);
+			v->vsl_sleep += v->vsl_sleep;
+			if (v->vsl_sleep > VSL_SLEEP_USEC)
+				v->vsl_sleep = VSL_SLEEP_USEC;
+		}
+	}
+	VSM_Delete(vsl);
+	return (NULL);
+}
+
+/**********************************************************************
  * Allocate and initialize a varnish
  */
 
@@ -145,9 +200,6 @@ varnish_new(const char *name)
 	    v->workdir, v->workdir, random(), v->workdir);
 	AZ(system(buf));
 
-	v->vl1 = vtc_logopen(name);
-	AN(v->vl1);
-
 	if (*v->name != 'v')
 		vtc_log(v->vl, 0, "Varnish name must start with 'v'");
 
@@ -218,7 +270,7 @@ varnish_thread(void *priv)
 		if (i <= 0)
 			break;
 		buf[i] = '\0';
-		vtc_dump(v->vl1, 3, "debug", buf, -1);
+		vtc_dump(v->vl, 3, "debug", buf, -1);
 	}
 	return (NULL);
 }
@@ -294,6 +346,7 @@ varnish_launch(struct varnish *v)
 	v->fds[2] = v->fds[3] = -1;
 	VSB_delete(vsb);
 	AZ(pthread_create(&v->tp, NULL, varnish_thread, v));
+	AZ(pthread_create(&v->tp_vsl, NULL, varnishlog_thread, v));
 
 	/* Wait for the varnish to call home */
 	fd[0].fd = v->cli_fd;
@@ -458,7 +511,9 @@ varnish_wait(struct varnish *v)
 	AZ(pthread_join(v->tp, &p));
 	AZ(close(v->fds[0]));
 	r = wait4(v->pid, &status, 0, NULL);
+	v->pid = 0;
 	vtc_log(v->vl, 2, "R %d Status: %04x", r, status);
+	AZ(pthread_join(v->tp_vsl, &p));
 	if (WIFEXITED(status) && WEXITSTATUS(status) == 0)
 		return;
 #ifdef WCOREDUMP



From phk at varnish-cache.org  Wed Sep 14 07:50:51 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 14 Sep 2011 09:50:51 +0200
Subject: [master] 0bda0cb Minor flexelint'ing
Message-ID: <E1R3kF9-0006W9-D5@project.varnish-software.com>

commit 0bda0cb32712fa873d771ffc6f1ed593fd640de6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 14 07:50:40 2011 +0000

    Minor flexelint'ing

diff --git a/bin/varnishtest/flint.lnt b/bin/varnishtest/flint.lnt
index f50040c..8f3d880 100644
--- a/bin/varnishtest/flint.lnt
+++ b/bin/varnishtest/flint.lnt
@@ -22,3 +22,4 @@
 -e737	// [45]  Loss of sign in promotion from int to unsigned
 -e713	// Loss of precision (assignment) (unsigned long long to long long)
 -e574	// Signed-unsigned mix with relational
+-e835	//  A zero has been given as ___ argument to operator '___'  (<<)
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 5b14edc..a034fef 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -60,7 +60,6 @@ volatile sig_atomic_t	vtc_error;	/* Error encountered */
 int			vtc_stop;	/* Stops current test without error */
 pthread_t		vtc_thread;
 static struct vtclog	*vltop;
-int			in_tree = 0;	/* Are we running in-tree */
 
 /**********************************************************************
  * Macro facility
@@ -238,18 +237,6 @@ extmacro_def(const char *name, const char *fmt, ...)
 	}
 }
 
-const char *
-extmacro_get(const char *name)
-{
-	struct extmacro *m;
-
-	VTAILQ_FOREACH(m, &extmacro_list, list)
-		if (!strcmp(name, m->name))
-			return (m->val);
-
-	return (NULL);
-}
-
 /**********************************************************************
  * Execute a file
  */
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 16d577f..eb9a02a 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -85,4 +85,3 @@ void macro_def(struct vtclog *vl, const char *instance, const char *name,
 struct vsb *macro_expand(struct vtclog *vl, const char *text);
 
 void extmacro_def(const char *name, const char *fmt, ...);
-const char *extmacro_get(const char *name);
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 758113b..9c0f175 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -586,7 +586,7 @@ cmd_http_gunzip_body(CMD_ARGS)
  */
 
 static void
-gzip_body(struct http *hp, const char *txt, char **body, int *bodylen)
+gzip_body(const struct http *hp, const char *txt, char **body, int *bodylen)
 {
 	int l, i;
 	z_stream vz;



From phk at varnish-cache.org  Sat Sep 17 09:32:52 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 11:32:52 +0200
Subject: [master] b52b3a6 Start separating out the worker thread from the
	worker pool stuff.
Message-ID: <E1R4rGW-0006rV-7s@project.varnish-software.com>

commit b52b3a688e2d13c57034a53bdd09565209e81c78
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 09:32:18 2011 +0000

    Start separating out the worker thread from the worker pool stuff.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 0e0bc83..288f1f1 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -48,6 +48,7 @@ varnishd_SOURCES = \
 	cache_vrt_re.c \
 	cache_vrt_var.c \
 	cache_vrt_vmod.c \
+	cache_wrk.c \
 	cache_wrw.c \
 	cache_ws.c \
 	hash_classic.c \
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index e2faf30..1abd6db 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -840,8 +840,11 @@ void PipeSession(struct sess *sp);
 
 /* cache_pool.c */
 void WRK_Init(void);
+void WRK2_Init(void);
 int WRK_QueueSession(struct sess *sp);
 void WRK_SumStat(struct worker *w);
+int WRK_TrySumStat(struct worker *w);
+void WRK_thread_real(void *priv, struct worker *w);
 
 #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
 int WRW_Error(const struct worker *w);
@@ -932,6 +935,10 @@ void ESI_DeliverChild(const struct sess *);
 /* cache_vrt_vmod.c */
 void VMOD_Init(void);
 
+/* cache_wrk.c */
+
+void *WRK_thread(void *priv);
+
 /* cache_ws.c */
 
 void WS_Init(struct ws *ws, const char *id, void *space, unsigned len);
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 0875e07..8286162 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -82,73 +82,16 @@ static unsigned			nthr_max;
 
 static pthread_cond_t		herder_cond;
 static struct lock		herder_mtx;
-static struct lock		wstat_mtx;
 
 /*--------------------------------------------------------------------*/
 
-static void
-wrk_sumstat(struct worker *w)
-{
-
-	Lck_AssertHeld(&wstat_mtx);
-#define L0(n)
-#define L1(n) (VSC_C_main->n += w->stats.n)
-#define VSC_DO_MAIN
-#define VSC_F(n, t, l, f, d) L##l(n);
-#include "vsc_fields.h"
-#undef VSC_F
-#undef VSC_DO_MAIN
-#undef L0
-#undef L1
-	memset(&w->stats, 0, sizeof w->stats);
-}
-
 void
-WRK_SumStat(struct worker *w)
-{
-
-	Lck_Lock(&wstat_mtx);
-	wrk_sumstat(w);
-	Lck_Unlock(&wstat_mtx);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void *
-wrk_thread_real(struct wq *qp, unsigned shm_workspace, unsigned sess_workspace,
-    uint16_t nhttp, unsigned http_space, unsigned siov)
+WRK_thread_real(void *priv, struct worker *w)
 {
-	struct worker *w, ww;
-	uint32_t wlog[shm_workspace / 4];
-	/* XXX: can we trust these to be properly aligned ? */
-	unsigned char ws[sess_workspace];
-	unsigned char http0[http_space];
-	unsigned char http1[http_space];
-	unsigned char http2[http_space];
-	struct iovec iov[siov];
-	struct SHA256Context sha256;
+	struct wq *qp;
 	int stats_clean;
 
-	THR_SetName("cache-worker");
-	w = &ww;
-	memset(w, 0, sizeof *w);
-	w->magic = WORKER_MAGIC;
-	w->lastused = NAN;
-	w->wlb = w->wlp = wlog;
-	w->wle = wlog + (sizeof wlog) / 4;
-	w->sha256ctx = &sha256;
-	w->bereq = HTTP_create(http0, nhttp);
-	w->beresp = HTTP_create(http1, nhttp);
-	w->resp = HTTP_create(http2, nhttp);
-	w->wrw.iov = iov;
-	w->wrw.siov = siov;
-	w->wrw.ciov = siov;
-	AZ(pthread_cond_init(&w->cond, NULL));
-
-	WS_Init(w->ws, "wrk", ws, sess_workspace);
-
-	VSL(SLT_WorkThread, 0, "%p start", w);
-
+	CAST_OBJ_NOTNULL(qp, priv, WQ_MAGIC);
 	Lck_Lock(&qp->mtx);
 	qp->nthr++;
 	stats_clean = 1;
@@ -195,43 +138,11 @@ wrk_thread_real(struct wq *qp, unsigned shm_workspace, unsigned sess_workspace,
 			if (w->vcl != NULL)
 				VCL_Rel(&w->vcl);
 		}
-		if (!Lck_Trylock(&wstat_mtx)) {
-			wrk_sumstat(w);
-			Lck_Unlock(&wstat_mtx);
-			stats_clean = 1;
-		}
+		stats_clean = WRK_TrySumStat(w);
 		Lck_Lock(&qp->mtx);
 	}
 	qp->nthr--;
 	Lck_Unlock(&qp->mtx);
-
-	VSL(SLT_WorkThread, 0, "%p end", w);
-	if (w->vcl != NULL)
-		VCL_Rel(&w->vcl);
-	AZ(pthread_cond_destroy(&w->cond));
-	HSH_Cleanup(w);
-	WRK_SumStat(w);
-	return (NULL);
-}
-
-static void *
-wrk_thread(void *priv)
-{
-	struct wq *qp;
-	uint16_t nhttp;
-	unsigned siov;
-
-	CAST_OBJ_NOTNULL(qp, priv, WQ_MAGIC);
-	assert(params->http_max_hdr <= 65535);
-	/* We need to snapshot these two for consistency */
-	nhttp = (uint16_t)params->http_max_hdr;
-	siov = nhttp * 2;
-	if (siov > IOV_MAX)
-		siov = IOV_MAX;
-	return (wrk_thread_real(qp,
-	    params->shm_workspace,
-	    params->wthread_workspace,
-	    nhttp, HTTP_estimate(nhttp), siov));
 }
 
 /*--------------------------------------------------------------------
@@ -481,7 +392,7 @@ wrk_breed_flock(struct wq *qp, const pthread_attr_t *tp_attr)
 	    qp->lqueue > qp->last_lqueue)) {	/* not getting better since last */
 		if (qp->nthr >= nthr_max) {
 			VSC_C_main->n_wrk_max++;
-		} else if (pthread_create(&tp, tp_attr, wrk_thread, qp)) {
+		} else if (pthread_create(&tp, tp_attr, WRK_thread, qp)) {
 			VSL(SLT_Debug, 0, "Create worker thread failed %d %s",
 			    errno, strerror(errno));
 			VSC_C_main->n_wrk_failed++;
@@ -600,13 +511,12 @@ WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv)
 /*--------------------------------------------------------------------*/
 
 void
-WRK_Init(void)
+WRK2_Init(void)
 {
 	pthread_t tp;
 
 	AZ(pthread_cond_init(&herder_cond, NULL));
 	Lck_New(&herder_mtx, lck_herder);
-	Lck_New(&wstat_mtx, lck_wstat);
 
 	wrk_addpools(params->wthread_pools);
 	AZ(pthread_create(&tp, NULL, wrk_herdtimer_thread, NULL));
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
new file mode 100644
index 0000000..5b78c19
--- /dev/null
+++ b/bin/varnishd/cache_wrk.c
@@ -0,0 +1,160 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Worker thread stuff unrealted to the worker thread pools.
+ */
+
+#include "config.h"
+
+#include <sys/types.h>
+
+#include <errno.h>
+#include <stdio.h>
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "vcl.h"
+#include "cli_priv.h"
+#include "cache.h"
+#include "stevedore.h"
+#include "hash_slinger.h"
+#include "vsha256.h"
+
+static struct lock		wstat_mtx;
+
+/*--------------------------------------------------------------------*/
+
+static void
+wrk_sumstat(struct worker *w)
+{
+
+	Lck_AssertHeld(&wstat_mtx);
+#define L0(n)
+#define L1(n) (VSC_C_main->n += w->stats.n)
+#define VSC_DO_MAIN
+#define VSC_F(n, t, l, f, d) L##l(n);
+#include "vsc_fields.h"
+#undef VSC_F
+#undef VSC_DO_MAIN
+#undef L0
+#undef L1
+	memset(&w->stats, 0, sizeof w->stats);
+}
+
+void
+WRK_SumStat(struct worker *w)
+{
+
+	Lck_Lock(&wstat_mtx);
+	wrk_sumstat(w);
+	Lck_Unlock(&wstat_mtx);
+}
+
+int
+WRK_TrySumStat(struct worker *w)
+{
+	if (Lck_Trylock(&wstat_mtx)) 
+		return (0);
+	wrk_sumstat(w);
+	Lck_Unlock(&wstat_mtx);
+	return (1);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
+wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
+    uint16_t nhttp, unsigned http_space, unsigned siov)
+{
+	struct worker *w, ww;
+	uint32_t wlog[shm_workspace / 4];
+	/* XXX: can we trust these to be properly aligned ? */
+	unsigned char ws[sess_workspace];
+	unsigned char http0[http_space];
+	unsigned char http1[http_space];
+	unsigned char http2[http_space];
+	struct iovec iov[siov];
+	struct SHA256Context sha256;
+
+	THR_SetName("cache-worker");
+	w = &ww;
+	memset(w, 0, sizeof *w);
+	w->magic = WORKER_MAGIC;
+	w->lastused = NAN;
+	w->wlb = w->wlp = wlog;
+	w->wle = wlog + (sizeof wlog) / 4;
+	w->sha256ctx = &sha256;
+	w->bereq = HTTP_create(http0, nhttp);
+	w->beresp = HTTP_create(http1, nhttp);
+	w->resp = HTTP_create(http2, nhttp);
+	w->wrw.iov = iov;
+	w->wrw.siov = siov;
+	w->wrw.ciov = siov;
+	AZ(pthread_cond_init(&w->cond, NULL));
+
+	WS_Init(w->ws, "wrk", ws, sess_workspace);
+
+	VSL(SLT_WorkThread, 0, "%p start", w);
+
+	WRK_thread_real(priv, w);
+
+	VSL(SLT_WorkThread, 0, "%p end", w);
+	if (w->vcl != NULL)
+		VCL_Rel(&w->vcl);
+	AZ(pthread_cond_destroy(&w->cond));
+	HSH_Cleanup(w);
+	WRK_SumStat(w);
+	return (NULL);
+}
+
+void *
+WRK_thread(void *priv)
+{
+	uint16_t nhttp;
+	unsigned siov;
+
+	assert(params->http_max_hdr <= 65535);
+	/* We need to snapshot these two for consistency */
+	nhttp = (uint16_t)params->http_max_hdr;
+	siov = nhttp * 2;
+	if (siov > IOV_MAX)
+		siov = IOV_MAX;
+	return (wrk_thread_real(priv,
+	    params->shm_workspace,
+	    params->wthread_workspace,
+	    nhttp, HTTP_estimate(nhttp), siov));
+}
+
+void
+WRK_Init(void)
+{
+	Lck_New(&wstat_mtx, lck_wstat);
+	WRK2_Init();
+}



From phk at varnish-cache.org  Sat Sep 17 09:40:42 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 11:40:42 +0200
Subject: [master] 2a754e9 Rename the pool initializer to Pool_Init()
Message-ID: <E1R4rO6-0001JU-Ji@project.varnish-software.com>

commit 2a754e9eb282cdf5bf65fd77f4b2479a383b6f5b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 09:40:20 2011 +0000

    Rename the pool initializer to Pool_Init()

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 1abd6db..5b4b575 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -839,8 +839,7 @@ void PAN_Init(void);
 void PipeSession(struct sess *sp);
 
 /* cache_pool.c */
-void WRK_Init(void);
-void WRK2_Init(void);
+void Pool_Init(void);
 int WRK_QueueSession(struct sess *sp);
 void WRK_SumStat(struct worker *w);
 int WRK_TrySumStat(struct worker *w);
@@ -937,6 +936,7 @@ void VMOD_Init(void);
 
 /* cache_wrk.c */
 
+void WRK_Init(void);
 void *WRK_thread(void *priv);
 
 /* cache_ws.c */
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index f03604d..4312393 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -116,6 +116,7 @@ child_main(void)
 	VBE_Init();
 	VBP_Init();
 	WRK_Init();
+	Pool_Init();
 
 	EXP_Init();
 	HSH_Init();
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 8286162..b7f1f41 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -511,7 +511,7 @@ WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv)
 /*--------------------------------------------------------------------*/
 
 void
-WRK2_Init(void)
+Pool_Init(void)
 {
 	pthread_t tp;
 
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 5b78c19..9b3bc2b 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -156,5 +156,4 @@ void
 WRK_Init(void)
 {
 	Lck_New(&wstat_mtx, lck_wstat);
-	WRK2_Init();
 }



From phk at varnish-cache.org  Sat Sep 17 09:54:27 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 11:54:27 +0200
Subject: [master] c9d9539 Further Pool/Wrk separation.
Message-ID: <E1R4rbP-0000mN-CA@project.varnish-software.com>

commit c9d9539defa9769e688a5af0864fadcdf63ae7e9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 09:54:05 2011 +0000

    Further Pool/Wrk separation.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 5b4b575..a6b1786 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -840,10 +840,8 @@ void PipeSession(struct sess *sp);
 
 /* cache_pool.c */
 void Pool_Init(void);
-int WRK_QueueSession(struct sess *sp);
-void WRK_SumStat(struct worker *w);
-int WRK_TrySumStat(struct worker *w);
-void WRK_thread_real(void *priv, struct worker *w);
+int Pool_QueueSession(struct sess *sp);
+void Pool_Work_Thread(void *priv, struct worker *w);
 
 #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
 int WRW_Error(const struct worker *w);
@@ -858,9 +856,6 @@ unsigned WRW_WriteH(struct worker *w, const txt *hh, const char *suf);
 void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 #endif  /* SENDFILE_WORKS */
 
-typedef void *bgthread_t(struct sess *, void *priv);
-void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func,
-    void *priv);
 
 /* cache_session.c [SES] */
 void SES_Init(void);
@@ -937,7 +932,12 @@ void VMOD_Init(void);
 /* cache_wrk.c */
 
 void WRK_Init(void);
+int WRK_TrySumStat(struct worker *w);
+void WRK_SumStat(struct worker *w);
 void *WRK_thread(void *priv);
+typedef void *bgthread_t(struct sess *, void *priv);
+void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func,
+    void *priv);
 
 /* cache_ws.c */
 
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 2b3f87e..0692d46 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -319,7 +319,7 @@ vca_acct(void *arg)
 			sp->sockaddrlen = l;
 
 			sp->step = STP_FIRST;
-			if (WRK_QueueSession(sp)) {
+			if (Pool_QueueSession(sp)) {
 				VSC_C_main->client_drop++;
 				pace += params->acceptor_sleep_incr;
 			} else {
@@ -347,7 +347,7 @@ vca_handover(struct sess *sp, int status)
 		break;
 	case 1:
 		sp->step = STP_START;
-		if (WRK_QueueSession(sp))
+		if (Pool_QueueSession(sp))
 			VSC_C_main->client_drop_late++;
 		break;
 	default:
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 328964a..16e5cf0 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -504,7 +504,7 @@ hsh_rush(struct objhead *oh)
 		AZ(sp->wrk);
 		VTAILQ_REMOVE(&wl->list, sp, list);
 		DSL(0x20, SLT_Debug, sp->id, "off waiting list");
-		if (WRK_QueueSession(sp)) {
+		if (Pool_QueueSession(sp)) {
 			/*
 			 * We could not schedule the session, leave the
 			 * rest on the busy list.
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index b7f1f41..a80cd9b 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -56,7 +56,6 @@
 #include "cache.h"
 #include "stevedore.h"
 #include "hash_slinger.h"
-#include "vsha256.h"
 
 VTAILQ_HEAD(workerhead, worker);
 
@@ -86,7 +85,7 @@ static struct lock		herder_mtx;
 /*--------------------------------------------------------------------*/
 
 void
-WRK_thread_real(void *priv, struct worker *w)
+Pool_Work_Thread(void *priv, struct worker *w)
 {
 	struct wq *qp;
 	int stats_clean;
@@ -217,7 +216,7 @@ wrk_do_cnt_sess(struct worker *w, void *priv)
 /*--------------------------------------------------------------------*/
 
 int
-WRK_QueueSession(struct sess *sp)
+Pool_QueueSession(struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk);
@@ -456,58 +455,6 @@ wrk_herder_thread(void *priv)
 	NEEDLESS_RETURN(NULL);
 }
 
-/*--------------------------------------------------------------------
- * Create and starte a back-ground thread which as its own worker and
- * session data structures;
- */
-
-struct bgthread {
-	unsigned	magic;
-#define BGTHREAD_MAGIC	0x23b5152b
-	const char	*name;
-	bgthread_t	*func;
-	void		*priv;
-};
-
-static void *
-wrk_bgthread(void *arg)
-{
-	struct bgthread *bt;
-	struct worker ww;
-	struct sess *sp;
-	uint32_t logbuf[1024];	/* XXX:  size ? */
-
-	CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC);
-	THR_SetName(bt->name);
-	sp = SES_Alloc();
-	XXXAN(sp);
-	memset(&ww, 0, sizeof ww);
-	sp->wrk = &ww;
-	ww.magic = WORKER_MAGIC;
-	ww.wlp = ww.wlb = logbuf;
-	ww.wle = logbuf + (sizeof logbuf) / 4;
-
-	(void)bt->func(sp, bt->priv);
-
-	WRONG("BgThread terminated");
-
-	NEEDLESS_RETURN(NULL);
-}
-
-void
-WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv)
-{
-	struct bgthread *bt;
-
-	ALLOC_OBJ(bt, BGTHREAD_MAGIC);
-	AN(bt);
-
-	bt->name = name;
-	bt->func = func;
-	bt->priv = priv;
-	AZ(pthread_create(thr, NULL, wrk_bgthread, bt));
-}
-
 /*--------------------------------------------------------------------*/
 
 void
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 9b3bc2b..66ba04b 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -43,7 +43,6 @@
 #include "vcl.h"
 #include "cli_priv.h"
 #include "cache.h"
-#include "stevedore.h"
 #include "hash_slinger.h"
 #include "vsha256.h"
 
@@ -87,6 +86,58 @@ WRK_TrySumStat(struct worker *w)
 	return (1);
 }
 
+/*--------------------------------------------------------------------
+ * Create and starte a back-ground thread which as its own worker and
+ * session data structures;
+ */
+
+struct bgthread {
+	unsigned	magic;
+#define BGTHREAD_MAGIC	0x23b5152b
+	const char	*name;
+	bgthread_t	*func;
+	void		*priv;
+};
+
+static void *
+wrk_bgthread(void *arg)
+{
+	struct bgthread *bt;
+	struct worker ww;
+	struct sess *sp;
+	uint32_t logbuf[1024];	/* XXX:  size ? */
+
+	CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC);
+	THR_SetName(bt->name);
+	sp = SES_Alloc();
+	XXXAN(sp);
+	memset(&ww, 0, sizeof ww);
+	sp->wrk = &ww;
+	ww.magic = WORKER_MAGIC;
+	ww.wlp = ww.wlb = logbuf;
+	ww.wle = logbuf + (sizeof logbuf) / 4;
+
+	(void)bt->func(sp, bt->priv);
+
+	WRONG("BgThread terminated");
+
+	NEEDLESS_RETURN(NULL);
+}
+
+void
+WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv)
+{
+	struct bgthread *bt;
+
+	ALLOC_OBJ(bt, BGTHREAD_MAGIC);
+	AN(bt);
+
+	bt->name = name;
+	bt->func = func;
+	bt->priv = priv;
+	AZ(pthread_create(thr, NULL, wrk_bgthread, bt));
+}
+
 /*--------------------------------------------------------------------*/
 
 static void *
@@ -123,7 +174,7 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
 
 	VSL(SLT_WorkThread, 0, "%p start", w);
 
-	WRK_thread_real(priv, w);
+	Pool_Work_Thread(priv, w);
 
 	VSL(SLT_WorkThread, 0, "%p end", w);
 	if (w->vcl != NULL)



From phk at varnish-cache.org  Sat Sep 17 10:12:02 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 12:12:02 +0200
Subject: [master] 83753e2 Change the calling convention for waiters,
	to make it OO
Message-ID: <E1R4rsQ-00037h-Tu@project.varnish-software.com>

commit 83753e28342e4925e52ebf0b8e36ad9600c1e9a3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 10:11:40 2011 +0000

    Change the calling convention for waiters, to make it OO

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 0692d46..2afc975 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -62,6 +62,8 @@ static struct waiter * const vca_waiters[] = {
 
 static struct waiter const *vca_act;
 
+static void *waiter_priv;
+
 pthread_t		VCA_thread;
 static struct timeval	tv_sndtimeo;
 static struct timeval	tv_rcvtimeo;
@@ -384,10 +386,7 @@ vca_return_session(struct sess *sp)
 	 */
 	if (VTCP_nonblocking(sp->fd))
 		vca_close_session(sp, "remote closed");
-	else if (vca_act->pass == NULL)
-		assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
-	else
-		vca_act->pass(sp);
+	vca_act->pass(waiter_priv, sp);
 }
 
 /*--------------------------------------------------------------------*/
@@ -405,10 +404,11 @@ ccf_start(struct cli *cli, const char * const *av, void *priv)
 
 	AN(vca_act);
 	AN(vca_act->name);
+	AN(vca_act->init);
+	AN(vca_act->pass);
 
-	if (vca_act->pass == NULL)
-		AZ(pipe(vca_pipes));
-	vca_act->init();
+	AZ(pipe(vca_pipes)); 	/* XXX */
+	waiter_priv = vca_act->init();
 	AZ(pthread_create(&VCA_thread, NULL, vca_acct, NULL));
 	VSL(SLT_Debug, 0, "Acceptor is %s", vca_act->name);
 }
diff --git a/bin/varnishd/cache_waiter.h b/bin/varnishd/cache_waiter.h
index 2e0953b..9d6eee9 100644
--- a/bin/varnishd/cache_waiter.h
+++ b/bin/varnishd/cache_waiter.h
@@ -30,8 +30,8 @@
 
 struct sess;
 
-typedef void waiter_init_f(void);
-typedef void waiter_pass_f(struct sess *);
+typedef void* waiter_init_f(void);
+typedef void waiter_pass_f(void *priv, struct sess *);
 
 extern int vca_pipes[2];
 
diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index fe64961..5726b03 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -222,6 +222,16 @@ vca_sess_timeout_ticker(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void
+vca_epoll_pass(void *priv, hhstruct sess *sp)
+{
+
+	(void)priv;
+	assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
 vca_epoll_init(void)
 {
 	int i;
@@ -242,11 +252,15 @@ vca_epoll_init(void)
 	AZ(pthread_create(&vca_epoll_timeout_thread,
 	    NULL, vca_sess_timeout_ticker, NULL));
 	AZ(pthread_create(&vca_epoll_thread, NULL, vca_main, NULL));
+	return(NULL);
 }
 
+/*--------------------------------------------------------------------*/
+
 struct waiter waiter_epoll = {
 	.name =		"epoll",
 	.init =		vca_epoll_init,
+	.pass =		vca_epoll_pass,
 };
 
 #endif /* defined(HAVE_EPOLL_CTL) */
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index ea0e2fc..12e7bd4 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -204,6 +204,16 @@ vca_kqueue_main(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void
+vca_kqueue_pass(void *priv, struct sess *sp)
+{
+
+	(void)priv;
+	assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
 vca_kqueue_init(void)
 {
 	int i;
@@ -215,11 +225,15 @@ vca_kqueue_init(void)
 	assert(i != -1);
 
 	AZ(pthread_create(&vca_kqueue_thread, NULL, vca_kqueue_main, NULL));
+	return (NULL);
 }
 
+/*--------------------------------------------------------------------*/
+
 struct waiter waiter_kqueue = {
 	.name =		"kqueue",
 	.init =		vca_kqueue_init,
+	.pass =		vca_kqueue_pass,
 };
 
 #endif /* defined(HAVE_KQUEUE) */
diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index 42b54dc..528a880 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -191,16 +191,29 @@ vca_main(void *arg)
 }
 
 /*--------------------------------------------------------------------*/
-
 static void
+vca_poll_pass(void *priv, struct sess *sp)
+{
+
+	(void)priv;
+	assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
 vca_poll_init(void)
 {
 
 	vca_pollspace(256);
 	AZ(pthread_create(&vca_poll_thread, NULL, vca_main, NULL));
+	return (NULL);
 }
 
+/*--------------------------------------------------------------------*/
+
 struct waiter waiter_poll = {
 	.name =		"poll",
 	.init =		vca_poll_init,
+	.pass =		vca_poll_pass,
 };
diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index 9ee3aea..f1cc55c 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -243,10 +243,14 @@ vca_main(void *arg)
 	}
 }
 
+/*--------------------------------------------------------------------*/
+
 static void
-vca_ports_pass(struct sess *sp)
+vca_ports_pass(void *priv, struct sess *sp)
 {
 	int r;
+
+	(void)priv;
 	while((r = port_send(solaris_dport, 0, sp)) == -1 &&
 		errno == EAGAIN);
 	AZ(r);
@@ -254,13 +258,16 @@ vca_ports_pass(struct sess *sp)
 
 /*--------------------------------------------------------------------*/
 
-static void
+static void *
 vca_ports_init(void)
 {
 
 	AZ(pthread_create(&vca_ports_thread, NULL, vca_main, NULL));
+	return (NULL);
 }
 
+/*--------------------------------------------------------------------*/
+
 struct waiter waiter_ports = {
 	.name =		"ports",
 	.init =		vca_ports_init,



From phk at varnish-cache.org  Sat Sep 17 10:32:15 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 12:32:15 +0200
Subject: [master] 0e9e4d1 Use VTLA VWP for poll waiter, turn it into an object.
Message-ID: <E1R4sBz-00046h-7R@project.varnish-software.com>

commit 0e9e4d1bf784e9cc875ca5928ed926667710f101
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 10:31:36 2011 +0000

    Use VTLA VWP for poll waiter, turn it into an object.

diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index 528a880..44b4033 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -42,23 +42,29 @@
 
 #define NEEV	128
 
-static pthread_t vca_poll_thread;
-static struct pollfd *pollfd;
-static unsigned npoll, hpoll;
-
-static VTAILQ_HEAD(,sess) sesshead = VTAILQ_HEAD_INITIALIZER(sesshead);
+struct vwp {
+	unsigned		magic;
+#define VWP_MAGIC		0x4b2cc735
+	int			pipes[2];
+	pthread_t		poll_thread;
+	struct pollfd		*pollfd;
+	unsigned		npoll;
+	unsigned		hpoll;
+
+	VTAILQ_HEAD(,sess)	sesshead;
+};
 
 /*--------------------------------------------------------------------*/
 
 static void
-vca_pollspace(unsigned fd)
+vwp_pollspace(struct vwp *vwp, unsigned fd)
 {
-	struct pollfd *newpollfd = pollfd;
+	struct pollfd *newpollfd = vwp->pollfd;
 	unsigned newnpoll;
 
-	if (fd < npoll)
+	if (fd < vwp->npoll)
 		return;
-	newnpoll = npoll;
+	newnpoll = vwp->npoll;
 	if (newnpoll == 0)
 		newnpoll = 1;
 	while (fd >= newnpoll)
@@ -66,123 +72,125 @@ vca_pollspace(unsigned fd)
 	VSL(SLT_Debug, 0, "Acceptor poll space increased to %u", newnpoll);
 	newpollfd = realloc(newpollfd, newnpoll * sizeof *newpollfd);
 	XXXAN(newpollfd);
-	memset(newpollfd + npoll, 0, (newnpoll - npoll) * sizeof *newpollfd);
-	pollfd = newpollfd;
-	while (npoll < newnpoll)
-		pollfd[npoll++].fd = -1;
-	assert(fd < npoll);
+	memset(newpollfd + vwp->npoll, 0,
+	    (newnpoll - vwp->npoll) * sizeof *newpollfd);
+	vwp->pollfd = newpollfd;
+	while (vwp->npoll < newnpoll)
+		vwp->pollfd[vwp->npoll++].fd = -1;
+	assert(fd < vwp->npoll);
 }
 
 /*--------------------------------------------------------------------*/
 
 static void
-vca_poll(int fd)
+vwp_poll(struct vwp *vwp, int fd)
 {
 
 	assert(fd >= 0);
-	vca_pollspace((unsigned)fd);
-	assert(fd < npoll);
+	vwp_pollspace(vwp, (unsigned)fd);
+	assert(fd < vwp->npoll);
 
-	if (hpoll < fd)
-		hpoll = fd;
+	if (vwp->hpoll < fd)
+		vwp->hpoll = fd;
 
-	assert(pollfd[fd].fd == -1);
-	assert(pollfd[fd].events == 0);
-	assert(pollfd[fd].revents == 0);
+	assert(vwp->pollfd[fd].fd == -1);
+	assert(vwp->pollfd[fd].events == 0);
+	assert(vwp->pollfd[fd].revents == 0);
 
-	pollfd[fd].fd = fd;
-	pollfd[fd].events = POLLIN;
+	vwp->pollfd[fd].fd = fd;
+	vwp->pollfd[fd].events = POLLIN;
 }
 
 static void
-vca_unpoll(int fd)
+vwp_unpoll(struct vwp *vwp, int fd)
 {
 
 	assert(fd >= 0);
-	assert(fd < npoll);
-	vca_pollspace((unsigned)fd);
+	assert(fd < vwp->npoll);
+	vwp_pollspace(vwp, (unsigned)fd);
 
-	assert(pollfd[fd].fd == fd);
-	assert(pollfd[fd].events == POLLIN);
-	assert(pollfd[fd].revents == 0);
+	assert(vwp->pollfd[fd].fd == fd);
+	assert(vwp->pollfd[fd].events == POLLIN);
+	assert(vwp->pollfd[fd].revents == 0);
 
-	pollfd[fd].fd = -1;
-	pollfd[fd].events = 0;
+	vwp->pollfd[fd].fd = -1;
+	vwp->pollfd[fd].events = 0;
 }
 
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_main(void *arg)
+vwp_main(void *priv)
 {
 	int v;
+	struct vwp *vwp;
 	struct sess *ss[NEEV], *sp, *sp2;
 	double deadline;
 	int i, j, fd;
 
+	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
 	THR_SetName("cache-poll");
-	(void)arg;
 
-	vca_poll(vca_pipes[0]);
+	vwp_poll(vwp, vwp->pipes[0]);
 
 	while (1) {
-		assert(hpoll < npoll);
-		while (hpoll > 0 && pollfd[hpoll].fd == -1)
-			hpoll--;
-		assert(vca_pipes[0] <= hpoll);
-		assert(pollfd[vca_pipes[0]].fd == vca_pipes[0]);
-		assert(pollfd[vca_pipes[1]].fd == -1);
-		v = poll(pollfd, hpoll + 1, 100);
+		assert(vwp->hpoll < vwp->npoll);
+		while (vwp->hpoll > 0 && vwp->pollfd[vwp->hpoll].fd == -1)
+			vwp->hpoll--;
+		assert(vwp->pipes[0] <= vwp->hpoll);
+		assert(vwp->pollfd[vwp->pipes[0]].fd == vwp->pipes[0]);
+		assert(vwp->pollfd[vwp->pipes[1]].fd == -1);
+		v = poll(vwp->pollfd, vwp->hpoll + 1, 100);
 		assert(v >= 0);
 		deadline = TIM_real() - params->sess_timeout;
-		VTAILQ_FOREACH_SAFE(sp, &sesshead, list, sp2) {
+		VTAILQ_FOREACH_SAFE(sp, &vwp->sesshead, list, sp2) {
 			if (v == 0)
 				break;
 			CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 			fd = sp->fd;
 			assert(fd >= 0);
-			assert(fd <= hpoll);
-			assert(fd < npoll);
-			assert(pollfd[fd].fd == fd);
-			if (pollfd[fd].revents) {
+			assert(fd <= vwp->hpoll);
+			assert(fd < vwp->npoll);
+			assert(vwp->pollfd[fd].fd == fd);
+			if (vwp->pollfd[fd].revents) {
 				v--;
 				i = HTC_Rx(sp->htc);
-				if (pollfd[fd].revents != POLLIN)
+				if (vwp->pollfd[fd].revents != POLLIN)
 					VSL(SLT_Debug, fd, "Poll: %x / %d",
-					    pollfd[fd].revents, i);
-				pollfd[fd].revents = 0;
-				VTAILQ_REMOVE(&sesshead, sp, list);
+					    vwp->pollfd[fd].revents, i);
+				vwp->pollfd[fd].revents = 0;
+				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
 				if (i == 0) {
 					/* Mov to front of list for speed */
-					VTAILQ_INSERT_HEAD(&sesshead, sp, list);
+					VTAILQ_INSERT_HEAD(&vwp->sesshead, sp, list);
 				} else {
-					vca_unpoll(fd);
+					vwp_unpoll(vwp, fd);
 					vca_handover(sp, i);
 				}
 			} else if (sp->t_open <= deadline) {
-				VTAILQ_REMOVE(&sesshead, sp, list);
-				vca_unpoll(fd);
+				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
+				vwp_unpoll(vwp, fd);
 				// XXX: not yet (void)VTCP_linger(sp->fd, 0);
 				vca_close_session(sp, "timeout");
 				SES_Delete(sp);
 			}
 		}
-		if (v && pollfd[vca_pipes[0]].revents) {
+		if (v && vwp->pollfd[vwp->pipes[0]].revents) {
 
-			if (pollfd[vca_pipes[0]].revents != POLLIN)
+			if (vwp->pollfd[vwp->pipes[0]].revents != POLLIN)
 				VSL(SLT_Debug, 0, "pipe.revents= 0x%x",
-				    pollfd[vca_pipes[0]].revents);
-			assert(pollfd[vca_pipes[0]].revents == POLLIN);
-			pollfd[vca_pipes[0]].revents = 0;
+				    vwp->pollfd[vwp->pipes[0]].revents);
+			assert(vwp->pollfd[vwp->pipes[0]].revents == POLLIN);
+			vwp->pollfd[vwp->pipes[0]].revents = 0;
 			v--;
-			i = read(vca_pipes[0], ss, sizeof ss);
+			i = read(vwp->pipes[0], ss, sizeof ss);
 			assert(i >= 0);
 			assert(((unsigned)i % sizeof ss[0]) == 0);
 			for (j = 0; j * sizeof ss[0] < i; j++) {
 				CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
 				assert(ss[j]->fd >= 0);
-				VTAILQ_INSERT_TAIL(&sesshead, ss[j], list);
-				vca_poll(ss[j]->fd);
+				VTAILQ_INSERT_TAIL(&vwp->sesshead, ss[j], list);
+				vwp_poll(vwp, ss[j]->fd);
 			}
 		}
 		assert(v == 0);
@@ -191,29 +199,37 @@ vca_main(void *arg)
 }
 
 /*--------------------------------------------------------------------*/
+
 static void
-vca_poll_pass(void *priv, struct sess *sp)
+vwp_poll_pass(void *priv, struct sess *sp)
 {
+	struct vwp *vwp;
 
-	(void)priv;
-	assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
+	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
+
+	assert(sizeof sp == write(vwp->pipes[1], &sp, sizeof sp));
 }
 
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_poll_init(void)
+vwp_poll_init(void)
 {
-
-	vca_pollspace(256);
-	AZ(pthread_create(&vca_poll_thread, NULL, vca_main, NULL));
-	return (NULL);
+	struct vwp *vwp;
+
+	ALLOC_OBJ(vwp, VWP_MAGIC);
+	AN(vwp);
+	VTAILQ_INIT(&vwp->sesshead);
+	AZ(pipe(vwp->pipes));
+	vwp_pollspace(vwp, 256);
+	AZ(pthread_create(&vwp->poll_thread, NULL, vwp_main, vwp));
+	return (vwp);
 }
 
 /*--------------------------------------------------------------------*/
 
 struct waiter waiter_poll = {
 	.name =		"poll",
-	.init =		vca_poll_init,
-	.pass =		vca_poll_pass,
+	.init =		vwp_poll_init,
+	.pass =		vwp_poll_pass,
 };



From phk at varnish-cache.org  Sat Sep 17 10:46:52 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 12:46:52 +0200
Subject: [master] 8bebce8 Use VTLA VWK for Kqueue waiter and make it an object
Message-ID: <E1R4sQ8-0004tf-DK@project.varnish-software.com>

commit 8bebce803ffe25049fa4c5694dcc4b642f5155f6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 10:46:30 2011 +0000

    Use VTLA VWK for Kqueue waiter and make it an object

diff --git a/bin/varnishd/cache_waiter.h b/bin/varnishd/cache_waiter.h
index 9d6eee9..9bf139c 100644
--- a/bin/varnishd/cache_waiter.h
+++ b/bin/varnishd/cache_waiter.h
@@ -31,7 +31,7 @@
 struct sess;
 
 typedef void* waiter_init_f(void);
-typedef void waiter_pass_f(void *priv, struct sess *);
+typedef void waiter_pass_f(void *priv, const struct sess *);
 
 extern int vca_pipes[2];
 
diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index 5726b03..c405bfc 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -222,7 +222,7 @@ vca_sess_timeout_ticker(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void
-vca_epoll_pass(void *priv, hhstruct sess *sp)
+vca_epoll_pass(void *priv, const struct sess *sp)
 {
 
 	(void)priv;
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index 12e7bd4..cbea248 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -47,64 +47,64 @@
 #include "cache.h"
 #include "cache_waiter.h"
 
-
-/*--------------------------------------------------------------------*/
-
-
-static pthread_t vca_kqueue_thread;
-static int kq = -1;
-
-
-static VTAILQ_HEAD(,sess) sesshead = VTAILQ_HEAD_INITIALIZER(sesshead);
-
 #define NKEV	100
 
-static struct kevent ki[NKEV];
-static unsigned nki;
+struct vwk {
+	unsigned		magic;
+#define VWK_MAGIC		0x1cc2acc2
+	pthread_t 		thread;
+	int			pipes[2];
+	int 			kq;
+	struct kevent		ki[NKEV];
+	unsigned		nki;
+	VTAILQ_HEAD(,sess)	sesshead;
+};
+
+/*--------------------------------------------------------------------*/
 
 static void
-vca_kq_flush(void)
+vwk_kq_flush(struct vwk *vwk)
 {
 	int i;
 
-	if (nki == 0)
+	if (vwk->nki == 0)
 		return;
-	i = kevent(kq, ki, nki, NULL, 0, NULL);
+	i = kevent(vwk->kq, vwk->ki, vwk->nki, NULL, 0, NULL);
 	assert(i == 0);
-	nki = 0;
+	vwk->nki = 0;
 }
 
 static void
-vca_kq_sess(struct sess *sp, short arm)
+vwk_kq_sess(struct vwk *vwk, struct sess *sp, short arm)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	assert(sp->fd >= 0);
 	DSL(0x04, SLT_Debug, sp->fd, "KQ: EV_SET sp %p arm %x", sp, arm);
-	EV_SET(&ki[nki], sp->fd, EVFILT_READ, arm, 0, 0, sp);
-	if (++nki == NKEV)
-		vca_kq_flush();
+	EV_SET(&vwk->ki[vwk->nki], sp->fd, EVFILT_READ, arm, 0, 0, sp);
+	if (++vwk->nki == NKEV)
+		vwk_kq_flush(vwk);
 }
 
 static void
-vca_kev(const struct kevent *kp)
+vwk_kev(struct vwk *vwk, const struct kevent *kp)
 {
 	int i, j;
 	struct sess *sp;
 	struct sess *ss[NKEV];
 
 	AN(kp->udata);
-	if (kp->udata == vca_pipes) {
+	if (kp->udata == vwk->pipes) {
 		j = 0;
-		i = read(vca_pipes[0], ss, sizeof ss);
+		i = read(vwk->pipes[0], ss, sizeof ss);
 		if (i == -1 && errno == EAGAIN)
 			return;
 		while (i >= sizeof ss[0]) {
 			CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
 			assert(ss[j]->fd >= 0);
 			AZ(ss[j]->obj);
-			VTAILQ_INSERT_TAIL(&sesshead, ss[j], list);
-			vca_kq_sess(ss[j], EV_ADD | EV_ONESHOT);
+			VTAILQ_INSERT_TAIL(&vwk->sesshead, ss[j], list);
+			vwk_kq_sess(vwk, ss[j], EV_ADD | EV_ONESHOT);
 			j++;
 			i -= sizeof ss[0];
 		}
@@ -121,14 +121,14 @@ vca_kev(const struct kevent *kp)
 	if (kp->data > 0) {
 		i = HTC_Rx(sp->htc);
 		if (i == 0) {
-			vca_kq_sess(sp, EV_ADD | EV_ONESHOT);
+			vwk_kq_sess(vwk, sp, EV_ADD | EV_ONESHOT);
 			return;	/* more needed */
 		}
-		VTAILQ_REMOVE(&sesshead, sp, list);
+		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
 		vca_handover(sp, i);
 		return;
 	} else if (kp->flags & EV_EOF) {
-		VTAILQ_REMOVE(&sesshead, sp, list);
+		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
 		vca_close_session(sp, "EOF");
 		SES_Delete(sp);
 		return;
@@ -142,39 +142,40 @@ vca_kev(const struct kevent *kp)
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_kqueue_main(void *arg)
+vwk_thread(void *priv)
 {
+	struct vwk *vwk;
 	struct kevent ke[NKEV], *kp;
 	int j, n, dotimer;
 	double deadline;
 	struct sess *sp;
 
+	CAST_OBJ_NOTNULL(vwk, priv, VWK_MAGIC);
 	THR_SetName("cache-kqueue");
-	(void)arg;
 
-	kq = kqueue();
-	assert(kq >= 0);
+	vwk->kq = kqueue();
+	assert(vwk->kq >= 0);
 
 	j = 0;
 	EV_SET(&ke[j], 0, EVFILT_TIMER, EV_ADD, 0, 100, NULL);
 	j++;
-	EV_SET(&ke[j], vca_pipes[0], EVFILT_READ, EV_ADD, 0, 0, vca_pipes);
+	EV_SET(&ke[j], vwk->pipes[0], EVFILT_READ, EV_ADD, 0, 0, vwk->pipes);
 	j++;
-	AZ(kevent(kq, ke, j, NULL, 0, NULL));
+	AZ(kevent(vwk->kq, ke, j, NULL, 0, NULL));
 
-	nki = 0;
+	vwk->nki = 0;
 	while (1) {
 		dotimer = 0;
-		n = kevent(kq, ki, nki, ke, NKEV, NULL);
+		n = kevent(vwk->kq, vwk->ki, vwk->nki, ke, NKEV, NULL);
 		assert(n >= 1 && n <= NKEV);
-		nki = 0;
+		vwk->nki = 0;
 		for (kp = ke, j = 0; j < n; j++, kp++) {
 			if (kp->filter == EVFILT_TIMER) {
 				dotimer = 1;
 				continue;
 			}
 			assert(kp->filter == EVFILT_READ);
-			vca_kev(kp);
+			vwk_kev(vwk, kp);
 		}
 		if (!dotimer)
 			continue;
@@ -185,15 +186,15 @@ vca_kqueue_main(void *arg)
 		 * the kevent(2) at the top of this loop, the kernel
 		 * would not know we meant "the old fd of this number".
 		 */
-		vca_kq_flush();
+		vwk_kq_flush(vwk);
 		deadline = TIM_real() - params->sess_timeout;
 		for (;;) {
-			sp = VTAILQ_FIRST(&sesshead);
+			sp = VTAILQ_FIRST(&vwk->sesshead);
 			if (sp == NULL)
 				break;
 			if (sp->t_open > deadline)
 				break;
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			VTAILQ_REMOVE(&vwk->sesshead, sp, list);
 			// XXX: not yet (void)VTCP_linger(sp->fd, 0);
 			vca_close_session(sp, "timeout");
 			SES_Delete(sp);
@@ -204,11 +205,12 @@ vca_kqueue_main(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void
-vca_kqueue_pass(void *priv, struct sess *sp)
+vca_kqueue_pass(void *priv, const struct sess *sp)
 {
+	struct vwk *vwk;
 
-	(void)priv;
-	assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
+	CAST_OBJ_NOTNULL(vwk, priv, VWK_MAGIC);
+	assert(sizeof sp == write(vwk->pipes[1], &sp, sizeof sp));
 }
 
 /*--------------------------------------------------------------------*/
@@ -217,15 +219,22 @@ static void *
 vca_kqueue_init(void)
 {
 	int i;
+	struct vwk *vwk;
+
+	ALLOC_OBJ(vwk, VWK_MAGIC);
+	AN(vwk);
+
+	VTAILQ_INIT(&vwk->sesshead);
+	AZ(pipe(vwk->pipes));
 
-	i = fcntl(vca_pipes[0], F_GETFL);
+	i = fcntl(vwk->pipes[0], F_GETFL);
 	assert(i != -1);
 	i |= O_NONBLOCK;
-	i = fcntl(vca_pipes[0], F_SETFL, i);
+	i = fcntl(vwk->pipes[0], F_SETFL, i);
 	assert(i != -1);
 
-	AZ(pthread_create(&vca_kqueue_thread, NULL, vca_kqueue_main, NULL));
-	return (NULL);
+	AZ(pthread_create(&vwk->thread, NULL, vwk_thread, vwk));
+	return (vwk);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index 44b4033..c51776d 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -201,7 +201,7 @@ vwp_main(void *priv)
 /*--------------------------------------------------------------------*/
 
 static void
-vwp_poll_pass(void *priv, struct sess *sp)
+vwp_poll_pass(void *priv, const struct sess *sp)
 {
 	struct vwp *vwp;
 
diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index f1cc55c..6a587cc 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -246,7 +246,7 @@ vca_main(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void
-vca_ports_pass(void *priv, struct sess *sp)
+vca_ports_pass(void *priv, const struct sess *sp)
 {
 	int r;
 



From phk at varnish-cache.org  Sat Sep 17 11:02:45 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 13:02:45 +0200
Subject: [master] 685d9b9 Missed renaming these two bits.
Message-ID: <E1R4sfV-0002VN-BB@project.varnish-software.com>

commit 685d9b962246d7dbad53d5076b6aa547809c2b06
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 11:02:31 2011 +0000

    Missed renaming these two bits.

diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index cbea248..8febec7 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -205,7 +205,7 @@ vwk_thread(void *priv)
 /*--------------------------------------------------------------------*/
 
 static void
-vca_kqueue_pass(void *priv, const struct sess *sp)
+vwk_pass(void *priv, const struct sess *sp)
 {
 	struct vwk *vwk;
 
@@ -216,7 +216,7 @@ vca_kqueue_pass(void *priv, const struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_kqueue_init(void)
+vwk_init(void)
 {
 	int i;
 	struct vwk *vwk;
@@ -241,8 +241,8 @@ vca_kqueue_init(void)
 
 struct waiter waiter_kqueue = {
 	.name =		"kqueue",
-	.init =		vca_kqueue_init,
-	.pass =		vca_kqueue_pass,
+	.init =		vwk_init,
+	.pass =		vwk_pass,
 };
 
 #endif /* defined(HAVE_KQUEUE) */



From phk at varnish-cache.org  Sat Sep 17 11:11:20 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 13:11:20 +0200
Subject: [master] b190a57 Allocate VTLA VWE to the Epoll Waiter and make it an
	object
Message-ID: <E1R4sno-00060T-9N@project.varnish-software.com>

commit b190a57e1a3ca8f60ec7c726a80a257147b8e891
Author: Poul-Henning Kamp <phk at varnish-cache.org>
Date:   Sat Sep 17 13:10:36 2011 +0200

    Allocate VTLA VWE to the Epoll Waiter and make it an object

diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index c405bfc..05c8bc2 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -53,15 +53,21 @@
 
 #define NEEV	100
 
-static pthread_t vca_epoll_thread;
-static pthread_t vca_epoll_timeout_thread;;
-static int epfd = -1;
+struct vwe {
+	unsigned		magic;
+#define VWE_MAGIC		0x6bd73424
 
-static VTAILQ_HEAD(,sess) sesshead = VTAILQ_HEAD_INITIALIZER(sesshead);
-int dotimer_pipe[2];
+	pthread_t		epoll_thread;
+	pthread_t		timer_thread;
+	int			epfd;
+
+	VTAILQ_HEAD(,sess)	sesshead;
+	int			pipes[2];
+	int			timer_pipes[2];
+};
 
 static void
-vca_modadd(int fd, void *data, short arm)
+vwe_modadd(struct vwe *vwe, int fd, void *data, short arm)
 {
 
 	/* XXX: EPOLLET (edge triggered) can cause rather Bad Things to
@@ -69,55 +75,55 @@ vca_modadd(int fd, void *data, short arm)
 	 * XXX: will hang. See #644.
 	 */
 	assert(fd >= 0);
-	if (data == vca_pipes || data == dotimer_pipe) {
+	if (data == vwe->pipes || data == vwe->timer_pipes) {
 		struct epoll_event ev = {
 		    EPOLLIN | EPOLLPRI , { data }
 		};
-		AZ(epoll_ctl(epfd, arm, fd, &ev));
+		AZ(epoll_ctl(vwe->epfd, arm, fd, &ev));
 	} else {
 		struct sess *sp = (struct sess *)data;
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 		sp->ev.data.ptr = data;
 		sp->ev.events = EPOLLIN | EPOLLPRI | EPOLLONESHOT | EPOLLRDHUP;
-		AZ(epoll_ctl(epfd, arm, fd, &sp->ev));
+		AZ(epoll_ctl(vwe->epfd, arm, fd, &sp->ev));
 	}
 }
 
 static void
-vca_cond_modadd(int fd, void *data)
+vwe_cond_modadd(struct vwe *vwe, int fd, void *data)
 {
 	struct sess *sp = (struct sess *)data;
 
 	assert(fd >= 0);
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	if (sp->ev.data.ptr)
-		AZ(epoll_ctl(epfd, EPOLL_CTL_MOD, fd, &sp->ev));
+		AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_MOD, fd, &sp->ev));
 	else {
 		sp->ev.data.ptr = data;
 		sp->ev.events = EPOLLIN | EPOLLPRI | EPOLLONESHOT | EPOLLRDHUP;
-		AZ(epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &sp->ev));
+		AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_ADD, fd, &sp->ev));
 	}
 }
 
 static void
-vca_eev(const struct epoll_event *ep)
+vwe_eev(struct vwe *vwe, const struct epoll_event *ep)
 {
 	struct sess *ss[NEEV], *sp;
 	int i, j;
 
 	AN(ep->data.ptr);
-	if (ep->data.ptr == vca_pipes) {
+	if (ep->data.ptr == vwe->pipes) {
 		if (ep->events & EPOLLIN || ep->events & EPOLLPRI) {
 			j = 0;
-			i = read(vca_pipes[0], ss, sizeof ss);
+			i = read(vwe->pipes[0], ss, sizeof ss);
 			if (i == -1 && errno == EAGAIN)
 				return;
 			while (i >= sizeof ss[0]) {
 				CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
 				assert(ss[j]->fd >= 0);
 				AZ(ss[j]->obj);
-				VTAILQ_INSERT_TAIL(&sesshead, ss[j], list);
-				vca_cond_modadd(ss[j]->fd, ss[j]);
+				VTAILQ_INSERT_TAIL(&vwe->sesshead, ss[j], list);
+				vwe_cond_modadd(vwe, ss[j]->fd, ss[j]);
 				j++;
 				i -= sizeof ss[0];
 			}
@@ -128,21 +134,21 @@ vca_eev(const struct epoll_event *ep)
 		if (ep->events & EPOLLIN || ep->events & EPOLLPRI) {
 			i = HTC_Rx(sp->htc);
 			if (i == 0) {
-				vca_modadd(sp->fd, sp, EPOLL_CTL_MOD);
+				vwe_modadd(vwe, sp->fd, sp, EPOLL_CTL_MOD);
 				return;	/* more needed */
 			}
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
 			vca_handover(sp, i);
 		} else if (ep->events & EPOLLERR) {
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
 			vca_close_session(sp, "ERR");
 			SES_Delete(sp);
 		} else if (ep->events & EPOLLHUP) {
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
 			vca_close_session(sp, "HUP");
 			SES_Delete(sp);
 		} else if (ep->events & EPOLLRDHUP) {
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
 			vca_close_session(sp, "RHUP");
 			SES_Delete(sp);
 		}
@@ -152,34 +158,36 @@ vca_eev(const struct epoll_event *ep)
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_main(void *arg)
+vwe_thread(void *priv)
 {
 	struct epoll_event ev[NEEV], *ep;
 	struct sess *sp;
 	char junk;
 	double deadline;
 	int dotimer, i, n;
+	struct vwe *vwe;
+
+	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
 
 	THR_SetName("cache-epoll");
-	(void)arg;
 
-	epfd = epoll_create(1);
-	assert(epfd >= 0);
+	vwe->epfd = epoll_create(1);
+	assert(vwe->epfd >= 0);
 
-	vca_modadd(vca_pipes[0], vca_pipes, EPOLL_CTL_ADD);
-	vca_modadd(dotimer_pipe[0], dotimer_pipe, EPOLL_CTL_ADD);
+	vwe_modadd(vwe, vwe->pipes[0], vwe->pipes, EPOLL_CTL_ADD);
+	vwe_modadd(vwe, vwe->timer_pipes[0], vwe->timer_pipes, EPOLL_CTL_ADD);
 
 	while (1) {
 		dotimer = 0;
-		n = epoll_wait(epfd, ev, NEEV, -1);
+		n = epoll_wait(vwe->epfd, ev, NEEV, -1);
 		for (ep = ev, i = 0; i < n; i++, ep++) {
-			if (ep->data.ptr == dotimer_pipe &&
+			if (ep->data.ptr == vwe->timer_pipes &&
 			    (ep->events == EPOLLIN || ep->events == EPOLLPRI))
 			{
-				assert(read(dotimer_pipe[0], &junk, 1));
+				assert(read(vwe->timer_pipes[0], &junk, 1));
 				dotimer = 1;
 			} else
-				vca_eev(ep);
+				vwe_eev(vwe, ep);
 		}
 		if (!dotimer)
 			continue;
@@ -187,12 +195,12 @@ vca_main(void *arg)
 		/* check for timeouts */
 		deadline = TIM_real() - params->sess_timeout;
 		for (;;) {
-			sp = VTAILQ_FIRST(&sesshead);
+			sp = VTAILQ_FIRST(&vwe->sesshead);
 			if (sp == NULL)
 				break;
 			if (sp->t_open > deadline)
 				break;
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
 			// XXX: not yet VTCP_linger(sp->fd, 0);
 			vca_close_session(sp, "timeout");
 			SES_Delete(sp);
@@ -204,16 +212,17 @@ vca_main(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_sess_timeout_ticker(void *arg)
+vwe_sess_timeout_ticker(void *priv)
 {
 	char ticker = 'R';
+	struct vwe *vwe;
 
+	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
 	THR_SetName("cache-epoll-sess_timeout_ticker");
-	(void)arg;
 
 	while (1) {
 		/* ticking */
-		assert(write(dotimer_pipe[1], &ticker, 1));
+		assert(write(vwe->timer_pipes[1], &ticker, 1));
 		TIM_sleep(100 * 1e-3);
 	}
 	return NULL;
@@ -222,45 +231,52 @@ vca_sess_timeout_ticker(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void
-vca_epoll_pass(void *priv, const struct sess *sp)
+vwe_pass(void *priv, const struct sess *sp)
 {
+	struct vwe *vwe;
 
-	(void)priv;
-	assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
+	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
+	assert(sizeof sp == write(vwe->pipes[1], &sp, sizeof sp));
 }
 
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_epoll_init(void)
+vwe_init(void)
 {
 	int i;
+	struct vwe *vwe;
+
+	ALLOC_OBJ(vwe, VWE_MAGIC);
+	AN(vwe);
+	VTAILQ_INIT(&vwe->sesshead);
+	AZ(pipe(vwe->pipes));
+	AZ(pipe(vwe->timer_pipes));
 
-	i = fcntl(vca_pipes[0], F_GETFL);
+	i = fcntl(vwe->pipes[0], F_GETFL);
 	assert(i != -1);
 	i |= O_NONBLOCK;
-	i = fcntl(vca_pipes[0], F_SETFL, i);
+	i = fcntl(vwe->pipes[0], F_SETFL, i);
 	assert(i != -1);
 
-	AZ(pipe(dotimer_pipe));
-	i = fcntl(dotimer_pipe[0], F_GETFL);
+	i = fcntl(vwe->timer_pipes[0], F_GETFL);
 	assert(i != -1);
 	i |= O_NONBLOCK;
-	i = fcntl(dotimer_pipe[0], F_SETFL, i);
+	i = fcntl(vwe->timer_pipes[0], F_SETFL, i);
 	assert(i != -1);
 
-	AZ(pthread_create(&vca_epoll_timeout_thread,
-	    NULL, vca_sess_timeout_ticker, NULL));
-	AZ(pthread_create(&vca_epoll_thread, NULL, vca_main, NULL));
-	return(NULL);
+	AZ(pthread_create(&vwe->timer_thread,
+	    NULL, vwe_sess_timeout_ticker, vwe));
+	AZ(pthread_create(&vwe->epoll_thread, NULL, vwe_thread, vwe));
+	return(vwe);
 }
 
 /*--------------------------------------------------------------------*/
 
 struct waiter waiter_epoll = {
 	.name =		"epoll",
-	.init =		vca_epoll_init,
-	.pass =		vca_epoll_pass,
+	.init =		vwe_init,
+	.pass =		vwe_pass,
 };
 
 #endif /* defined(HAVE_EPOLL_CTL) */



From phk at varnish-cache.org  Sat Sep 17 11:43:52 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 13:43:52 +0200
Subject: [master] 012200a Eliminate the global vca_pipes.
Message-ID: <E1R4tJI-0007P0-L9@project.varnish-software.com>

commit 012200abc78909d6c5b511a39895659897eab09d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 11:43:39 2011 +0000

    Eliminate the global vca_pipes.

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 2afc975..f7147cc 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -95,8 +95,6 @@ static const struct linger linger = {
 
 static unsigned char	need_sndtimeo, need_rcvtimeo, need_linger, need_test;
 
-int vca_pipes[2] = { -1, -1 };
-
 static void
 sock_test(int fd)
 {
@@ -407,7 +405,6 @@ ccf_start(struct cli *cli, const char * const *av, void *priv)
 	AN(vca_act->init);
 	AN(vca_act->pass);
 
-	AZ(pipe(vca_pipes)); 	/* XXX */
 	waiter_priv = vca_act->init();
 	AZ(pthread_create(&VCA_thread, NULL, vca_acct, NULL));
 	VSL(SLT_Debug, 0, "Acceptor is %s", vca_act->name);
diff --git a/bin/varnishd/cache_waiter.h b/bin/varnishd/cache_waiter.h
index 9bf139c..ab54fe7 100644
--- a/bin/varnishd/cache_waiter.h
+++ b/bin/varnishd/cache_waiter.h
@@ -33,8 +33,6 @@ struct sess;
 typedef void* waiter_init_f(void);
 typedef void waiter_pass_f(void *priv, const struct sess *);
 
-extern int vca_pipes[2];
-
 struct waiter {
 	const char		*name;
 	waiter_init_f		*init;



From phk at varnish-cache.org  Sat Sep 17 12:43:43 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 14:43:43 +0200
Subject: [master] 74a0b31 Attempt to convert to object (VTLA = VWS)
Message-ID: <E1R4uFD-0001Ou-In@project.varnish-software.com>

commit 74a0b3112158ac4eea3b74893bde55c1b4808200
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 12:43:22 2011 +0000

    Attempt to convert to object (VTLA = VWS)

diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index 6a587cc..d607b8a 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -47,44 +47,48 @@
 #include "cache_waiter.h"
 
 #define MAX_EVENTS 256
-static pthread_t vca_ports_thread;
-int solaris_dport = -1;
 
-static VTAILQ_HEAD(,sess) sesshead = VTAILQ_HEAD_INITIALIZER(sesshead);
+struct vws {
+	unsigned		magic;
+#define VWS_MAGIC		0x0b771473
+	pthread_t		ports_thread;
+	int			dport;
+	VTAILQ_HEAD(,sess)	sesshead;
+};
 
 static inline void
-vca_add(int fd, void *data)
+vws_add(struct vws *vws, int fd, void *data)
 {
 	/*
 	 * POLLIN should be all we need here
 	 *
 	 */
-	AZ(port_associate(solaris_dport, PORT_SOURCE_FD, fd, POLLIN, data));
+	AZ(port_associate(vws->dport, PORT_SOURCE_FD, fd, POLLIN, data));
 }
 
 static inline void
-vca_del(int fd)
+vws_del(struct vws *vws, int fd)
 {
-	port_dissociate(solaris_dport, PORT_SOURCE_FD, fd);
+	port_dissociate(vws->dport, PORT_SOURCE_FD, fd);
 }
 
 static inline void
-vca_port_ev(port_event_t *ev) {
+vws_port_ev(struct vws *vws, port_event_t *ev) {
 	struct sess *sp;
 	if(ev->portev_source == PORT_SOURCE_USER) {
 		CAST_OBJ_NOTNULL(sp, ev->portev_user, SESS_MAGIC);
 		assert(sp->fd >= 0);
 		AZ(sp->obj);
-		VTAILQ_INSERT_TAIL(&sesshead, sp, list);
-		vca_add(sp->fd, sp);
+		VTAILQ_INSERT_TAIL(&vws->sesshead, sp, list);
+		vws_add(vw, sp->fd, sp);
 	} else {
 		int i;
 		assert(ev->portev_source == PORT_SOURCE_FD);
 		CAST_OBJ_NOTNULL(sp, ev->portev_user, SESS_MAGIC);
 		assert(sp->fd >= 0);
 		if(ev->portev_events & POLLERR) {
-			vca_del(sp->fd);
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			vws_del(vws->sp->fd);
+			VTAILQ_REMOVE(&vws->sesshead, sp, list);
 			vca_close_session(sp, "EOF");
 			SES_Delete(sp);
 			return;
@@ -93,7 +97,7 @@ vca_port_ev(port_event_t *ev) {
 
 		if (i == 0) {
 			/* incomplete header, wait for more data */
-			vca_add(sp->fd, sp);
+			vws_add(sp->fd, sp);
 			return;
 		}
 
@@ -109,8 +113,8 @@ vca_port_ev(port_event_t *ev) {
 		 *
 		 * Ref: http://opensolaris.org/jive/thread.jspa?threadID=129476&tstart=0
 		 */
-		vca_del(sp->fd);
-		VTAILQ_REMOVE(&sesshead, sp, list);
+		vws_del(vws->sp->fd);
+		VTAILQ_REMOVE(&vws->sesshead, sp, list);
 
 		/* vca_handover will also handle errors */
 		vca_handover(sp, i);
@@ -119,10 +123,12 @@ vca_port_ev(port_event_t *ev) {
 }
 
 static void *
-vca_main(void *arg)
+vws_thread(void *priv)
 {
 	struct sess *sp;
+	struct vws *vws;
 
+	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
 	/*
 	 * timeouts:
 	 *
@@ -147,13 +153,12 @@ vca_main(void *arg)
 	static struct timespec max_ts = {1L, 0L};		/* 1 second */
 	static double	       max_t  = 1.0;			/* 1 second */
 
+	/* XXX: These should probably go in vws ? */
 	struct timespec ts;
 	struct timespec *timeout;
 
-	(void)arg;
-
-	solaris_dport = port_create();
-	assert(solaris_dport >= 0);
+	vws->dport = port_create();
+	assert(vws->dport >= 0);
 
 	timeout = &max_ts;
 
@@ -184,14 +189,13 @@ vca_main(void *arg)
 		 *
 		 */
 
-		ret = port_getn(solaris_dport, ev, MAX_EVENTS, &nevents, timeout);
+		ret = port_getn(vws->dport, ev, MAX_EVENTS, &nevents, timeout);
 
 		if (ret < 0)
 			assert((errno == EINTR) || (errno == ETIME));
 
-		for (ei=0; ei<nevents; ei++) {
-			vca_port_ev(ev + ei);
-		}
+		for (ei = 0; ei < nevents; ei++)
+			vws_port_ev(vws, ev + ei);
 
 		/* check for timeouts */
 		now = TIM_real();
@@ -205,15 +209,15 @@ vca_main(void *arg)
 		 */
 
 		for (;;) {
-			sp = VTAILQ_FIRST(&sesshead);
+			sp = VTAILQ_FIRST(&vws->sesshead);
 			if (sp == NULL)
 				break;
 			if (sp->t_open > deadline) {
 				break;
 			}
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			VTAILQ_REMOVE(&vws->sesshead, sp, list);
 			if(sp->fd != -1) {
-				vca_del(sp->fd);
+				vws_del(vws, sp->fd);
 			}
 			vca_close_session(sp, "timeout");
 			SES_Delete(sp);
@@ -246,12 +250,13 @@ vca_main(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void
-vca_ports_pass(void *priv, const struct sess *sp)
+vws_pass(void *priv, const struct sess *sp)
 {
 	int r;
+	struct vws *vws;
 
-	(void)priv;
-	while((r = port_send(solaris_dport, 0, sp)) == -1 &&
+	CAST_OBJ_NOTNULL(vws, priv, VWS_MAGIC);
+	while((r = port_send(vws->dport, 0, TRUST_ME(sp))) == -1 &&
 		errno == EAGAIN);
 	AZ(r);
 }
@@ -259,19 +264,23 @@ vca_ports_pass(void *priv, const struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_ports_init(void)
+vws_init(void)
 {
+	struct vws *vws;
 
-	AZ(pthread_create(&vca_ports_thread, NULL, vca_main, NULL));
-	return (NULL);
+	ALLOC_OBJ(vws, VWS_MAGIC);
+	AN(vws);
+	VTAILQ_INIT(&vws->sesshead);
+	AZ(pthread_create(&vws_ports_thread, NULL, vws_thread, vws));
+	return (vws);
 }
 
 /*--------------------------------------------------------------------*/
 
 struct waiter waiter_ports = {
 	.name =		"ports",
-	.init =		vca_ports_init,
-	.pass =		vca_ports_pass
+	.init =		vws_init,
+	.pass =		vws_pass
 };
 
 #endif /* defined(HAVE_PORT_CREATE) */



From phk at varnish-cache.org  Sat Sep 17 13:20:30 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 15:20:30 +0200
Subject: [master] 0ad2c29 Move vca_handover() to SES_Handle() Move
	vca_close_session() to SES_Close() If SES_Delete() gets a
	reason, it calls SES_Close()
Message-ID: <E1R4uoo-000350-9k@project.varnish-software.com>

commit 0ad2c29e9716f357f1000e40198aa3bbaf2b0a19
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 13:19:19 2011 +0000

    Move vca_handover() to SES_Handle()
    Move vca_close_session() to SES_Close()
    If SES_Delete() gets a reason, it calls SES_Close()
    
    More work on the solaris ports waiter.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index a6b1786..2f43ff8 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -646,7 +646,6 @@ struct vbc {
 
 /* cache_acceptor.c */
 void vca_return_session(struct sess *sp);
-void vca_close_session(struct sess *sp, const char *why);
 void VCA_Prep(struct sess *sp);
 void VCA_Init(void);
 void VCA_Shutdown(void);
@@ -861,7 +860,8 @@ void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 void SES_Init(void);
 struct sess *SES_New(void);
 struct sess *SES_Alloc(void);
-void SES_Delete(struct sess *sp);
+void SES_Close(struct sess *sp, const char *reason);
+void SES_Delete(struct sess *sp, const char *reason);
 void SES_Charge(struct sess *sp);
 
 /* cache_shmlog.c */
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index f7147cc..46f955b 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -330,47 +330,10 @@ vca_acct(void *arg)
 	NEEDLESS_RETURN(NULL);
 }
 
-/*--------------------------------------------------------------------*/
-
-void
-vca_handover(struct sess *sp, int status)
-{
-
-	switch (status) {
-	case -2:
-		vca_close_session(sp, "blast");
-		SES_Delete(sp);
-		break;
-	case -1:
-		vca_close_session(sp, "no request");
-		SES_Delete(sp);
-		break;
-	case 1:
-		sp->step = STP_START;
-		if (Pool_QueueSession(sp))
-			VSC_C_main->client_drop_late++;
-		break;
-	default:
-		INCOMPL();
-	}
-}
 
 /*--------------------------------------------------------------------*/
 
 void
-vca_close_session(struct sess *sp, const char *why)
-{
-	int i;
-
-	VSL(SLT_SessionClose, sp->id, "%s", why);
-	if (sp->fd >= 0) {
-		i = close(sp->fd);
-		assert(i == 0 || errno != EBADF);	/* XXX EINVAL seen */
-	}
-	sp->fd = -1;
-}
-
-void
 vca_return_session(struct sess *sp)
 {
 
@@ -383,7 +346,7 @@ vca_return_session(struct sess *sp)
 	 * acceptor thread, to reduce syscall density of the latter.
 	 */
 	if (VTCP_nonblocking(sp->fd))
-		vca_close_session(sp, "remote closed");
+		SES_Close(sp, "remote closed");
 	vca_act->pass(waiter_priv, sp);
 }
 
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 39a2104..24b0ddc 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -116,14 +116,14 @@ cnt_wait(struct sess *sp)
 		return (0);
 	}
 	if (i == -2) {
-		vca_close_session(sp, "overflow");
+		SES_Close(sp, "overflow");
 		return (0);
 	}
 	if (i == -1 && Tlen(sp->htc->rxbuf) == 0 &&
 	    (errno == 0 || errno == ECONNRESET))
-		vca_close_session(sp, "EOF");
+		SES_Close(sp, "EOF");
 	else
-		vca_close_session(sp, "error");
+		SES_Close(sp, "error");
 	sp->step = STP_DONE;
 	return (0);
 }
@@ -364,13 +364,13 @@ cnt_done(struct sess *sp)
 		 * before we close, to get queued data transmitted.
 		 */
 		// XXX: not yet (void)VTCP_linger(sp->fd, 0);
-		vca_close_session(sp, sp->doclose);
+		SES_Close(sp, sp->doclose);
 	}
 
 	if (sp->fd < 0) {
 		sp->wrk->stats.sess_closed++;
 		sp->wrk = NULL;
-		SES_Delete(sp);
+		SES_Delete(sp, NULL);
 		return (1);
 	}
 
@@ -1483,7 +1483,7 @@ cnt_start(struct sess *sp)
 	/* If we could not even parse the request, just close */
 	if (done == 400) {
 		sp->step = STP_DONE;
-		vca_close_session(sp, "junk");
+		SES_Close(sp, "junk");
 		return (0);
 	}
 
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 61555bf..ea4951f 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -335,8 +335,7 @@ ESI_Deliver(struct sess *sp)
 						st->ptr + off, l2,
 						obuf, sizeof obuf, &obufl);
 					if (WRW_Error(sp->wrk)) {
-						vca_close_session(sp,
-						    "remote closed");
+						SES_Close(sp, "remote closed");
 						p = e;
 						break;
 					}
@@ -388,7 +387,7 @@ ESI_Deliver(struct sess *sp)
 				obufl = 0;
 			}
 			if (WRW_Flush(sp->wrk)) {
-				vca_close_session(sp, "remote closed");
+				SES_Close(sp, "remote closed");
 				p = e;
 				break;
 			}
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
index 0175937..ddb5c15 100644
--- a/bin/varnishd/cache_pipe.c
+++ b/bin/varnishd/cache_pipe.c
@@ -87,7 +87,7 @@ PipeSession(struct sess *sp)
 	i = WRW_FlushRelease(w);
 
 	if (i) {
-		vca_close_session(sp, "pipe");
+		SES_Close(sp, "pipe");
 		VDI_CloseFd(sp);
 		return;
 	}
@@ -127,6 +127,6 @@ PipeSession(struct sess *sp)
 			fds[1].fd = -1;
 		}
 	}
-	vca_close_session(sp, "pipe");
+	SES_Close(sp, "pipe");
 	VDI_CloseFd(sp);
 }
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index a80cd9b..92e81b1 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -232,7 +232,6 @@ Pool_QueueSession(struct sess *sp)
 	 * XXX: sleep whichever thread got us here
 	 */
 	sp->t_end = TIM_real();
-	vca_close_session(sp, "dropped");
 	if (sp->vcl != NULL) {
 		/*
 		 * A session parked on a busy object can come here
@@ -240,7 +239,7 @@ Pool_QueueSession(struct sess *sp)
 		 */
 		VCL_Rel(&sp->vcl);
 	}
-	SES_Delete(sp);
+	SES_Delete(sp, "dropped");
 	return (1);
 }
 
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 73338f7..982d8fe 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -323,7 +323,7 @@ RES_WriteObj(struct sess *sp)
 		WRW_EndChunk(sp->wrk);
 
 	if (WRW_FlushRelease(sp->wrk))
-		vca_close_session(sp, "remote closed");
+		SES_Close(sp, "remote closed");
 }
 
 /*--------------------------------------------------------------------*/
@@ -423,5 +423,5 @@ RES_StreamEnd(struct sess *sp)
 	    !(sp->wrk->res_mode & RES_ESI_CHILD))
 		WRW_EndChunk(sp->wrk);
 	if (WRW_FlushRelease(sp->wrk))
-		vca_close_session(sp, "remote closed");
+		SES_Close(sp, "remote closed");
 }
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 1ebca44..15373a4 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -48,6 +48,7 @@
 
 #include "cache.h"
 #include "cache_backend.h"
+#include "cache_waiter.h"
 
 /*--------------------------------------------------------------------*/
 
@@ -238,12 +239,55 @@ SES_Alloc(void)
 }
 
 /*--------------------------------------------------------------------
- * Recycle a session.  If the workspace has changed, deleted it,
+ * Handle a session (from waiter)
+ *
+ * Status: see HTC_Rx()
+ */
+
+void
+SES_Handle(struct sess *sp, int status)
+{
+
+	switch (status) {
+	case -2:
+		SES_Delete(sp, "blast");
+		break;
+	case -1:
+		SES_Delete(sp, "no request");
+		break;
+	case 1:
+		sp->step = STP_START;
+		if (Pool_QueueSession(sp))
+			VSC_C_main->client_drop_late++;
+		break;
+	default:
+		WRONG("Unexpected return from HTC_Rx()");
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Close a sessions connection.
+ */
+
+void
+SES_Close(struct sess *sp, const char *reason)
+{
+	int i;
+
+	assert(sp->fd >= 0);
+	VSL(SLT_SessionClose, sp->id, "%s", reason);
+	i = close(sp->fd);
+	assert(i == 0 || errno != EBADF); /* XXX EINVAL seen */
+	sp->fd = -1;
+}
+
+/*--------------------------------------------------------------------
+ * (Close &) Recycle a session.  If the workspace has changed, deleted it,
  * otherwise wash it, and put it up for adoption.
  */
 
 void
-SES_Delete(struct sess *sp)
+SES_Delete(struct sess *sp, const char *reason)
 {
 	struct acct *b = &sp->acct_ses;
 	struct sessmem *sm;
@@ -253,6 +297,10 @@ SES_Delete(struct sess *sp)
 	sm = sp->mem;
 	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
 
+	if (reason != NULL)
+		SES_Close(sp, reason);
+	assert(sp->fd < 0);
+
 	AZ(sp->obj);
 	AZ(sp->vcl);
 	VSC_C_main->n_sess--;			/* XXX: locking ? */
diff --git a/bin/varnishd/cache_waiter.h b/bin/varnishd/cache_waiter.h
index ab54fe7..86eb2cb 100644
--- a/bin/varnishd/cache_waiter.h
+++ b/bin/varnishd/cache_waiter.h
@@ -53,5 +53,6 @@ extern struct waiter waiter_poll;
 extern struct waiter waiter_ports;
 #endif
 
-/* vca_acceptor.c */
-void vca_handover(struct sess *sp, int bad);
+/* cache_session.c */
+void SES_Handle(struct sess *sp, int status);
+
diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index 05c8bc2..acb8e13 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -138,19 +138,16 @@ vwe_eev(struct vwe *vwe, const struct epoll_event *ep)
 				return;	/* more needed */
 			}
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			vca_handover(sp, i);
+			SES_Handle(sp, i);
 		} else if (ep->events & EPOLLERR) {
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			vca_close_session(sp, "ERR");
-			SES_Delete(sp);
+			SES_Delete(sp, "ERR");
 		} else if (ep->events & EPOLLHUP) {
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			vca_close_session(sp, "HUP");
-			SES_Delete(sp);
+			SES_Delete(sp, "HUP");
 		} else if (ep->events & EPOLLRDHUP) {
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			vca_close_session(sp, "RHUP");
-			SES_Delete(sp);
+			SES_Delete(sp, "RHUP");
 		}
 	}
 }
@@ -202,8 +199,7 @@ vwe_thread(void *priv)
 				break;
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
 			// XXX: not yet VTCP_linger(sp->fd, 0);
-			vca_close_session(sp, "timeout");
-			SES_Delete(sp);
+			SES_Delete(sp, "timeout");
 		}
 	}
 	return NULL;
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index 8febec7..7e6b10f 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -125,12 +125,11 @@ vwk_kev(struct vwk *vwk, const struct kevent *kp)
 			return;	/* more needed */
 		}
 		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
-		vca_handover(sp, i);
+		SES_Handle(sp, i);
 		return;
 	} else if (kp->flags & EV_EOF) {
 		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
-		vca_close_session(sp, "EOF");
-		SES_Delete(sp);
+		SES_Delete(sp, "EOF");
 		return;
 	} else {
 		VSL(SLT_Debug, sp->id, "KQ: sp %p kev data %lu flags 0x%x%s",
@@ -196,8 +195,7 @@ vwk_thread(void *priv)
 				break;
 			VTAILQ_REMOVE(&vwk->sesshead, sp, list);
 			// XXX: not yet (void)VTCP_linger(sp->fd, 0);
-			vca_close_session(sp, "timeout");
-			SES_Delete(sp);
+			SES_Delete(sp, "timeout");
 		}
 	}
 }
diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index c51776d..f44daa9 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -165,14 +165,13 @@ vwp_main(void *priv)
 					VTAILQ_INSERT_HEAD(&vwp->sesshead, sp, list);
 				} else {
 					vwp_unpoll(vwp, fd);
-					vca_handover(sp, i);
+					SES_Handle(sp, i);
 				}
 			} else if (sp->t_open <= deadline) {
 				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
 				vwp_unpoll(vwp, fd);
 				// XXX: not yet (void)VTCP_linger(sp->fd, 0);
-				vca_close_session(sp, "timeout");
-				SES_Delete(sp);
+				SES_Delete(sp, "timeout");
 			}
 		}
 		if (v && vwp->pollfd[vwp->pipes[0]].revents) {
diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index d607b8a..329ee23 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -80,24 +80,23 @@ vws_port_ev(struct vws *vws, port_event_t *ev) {
 		assert(sp->fd >= 0);
 		AZ(sp->obj);
 		VTAILQ_INSERT_TAIL(&vws->sesshead, sp, list);
-		vws_add(vw, sp->fd, sp);
+		vws_add(vws, sp->fd, sp);
 	} else {
 		int i;
 		assert(ev->portev_source == PORT_SOURCE_FD);
 		CAST_OBJ_NOTNULL(sp, ev->portev_user, SESS_MAGIC);
 		assert(sp->fd >= 0);
 		if(ev->portev_events & POLLERR) {
-			vws_del(vws->sp->fd);
+			vws_del(vws, sp->fd);
 			VTAILQ_REMOVE(&vws->sesshead, sp, list);
-			vca_close_session(sp, "EOF");
-			SES_Delete(sp);
+			SES_Delete(sp, "EOF");
 			return;
 		}
 		i = HTC_Rx(sp->htc);
 
 		if (i == 0) {
 			/* incomplete header, wait for more data */
-			vws_add(sp->fd, sp);
+			vws_add(vws, sp->fd, sp);
 			return;
 		}
 
@@ -113,11 +112,11 @@ vws_port_ev(struct vws *vws, port_event_t *ev) {
 		 *
 		 * Ref: http://opensolaris.org/jive/thread.jspa?threadID=129476&tstart=0
 		 */
-		vws_del(vws->sp->fd);
+		vws_del(vws, sp->fd);
 		VTAILQ_REMOVE(&vws->sesshead, sp, list);
 
-		/* vca_handover will also handle errors */
-		vca_handover(sp, i);
+		/* SES_Handle will also handle errors */
+		SES_Handle(sp, i);
 	}
 	return;
 }
@@ -128,7 +127,7 @@ vws_thread(void *priv)
 	struct sess *sp;
 	struct vws *vws;
 
-	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
+	CAST_OBJ_NOTNULL(vws, priv, VWS_MAGIC);
 	/*
 	 * timeouts:
 	 *
@@ -219,8 +218,7 @@ vws_thread(void *priv)
 			if(sp->fd != -1) {
 				vws_del(vws, sp->fd);
 			}
-			vca_close_session(sp, "timeout");
-			SES_Delete(sp);
+			SES_Delete(sp, "timeout");
 		}
 
 		/*
@@ -271,7 +269,7 @@ vws_init(void)
 	ALLOC_OBJ(vws, VWS_MAGIC);
 	AN(vws);
 	VTAILQ_INIT(&vws->sesshead);
-	AZ(pthread_create(&vws_ports_thread, NULL, vws_thread, vws));
+	AZ(pthread_create(&vws->ports_thread, NULL, vws_thread, vws));
 	return (vws);
 }
 
diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index 45dfac4..30203f8 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -557,7 +557,6 @@ const struct stevedore smf_stevedore = {
 #ifdef INCLUDE_TEST_DRIVER
 
 void vca_flush(struct sess *sp) {}
-void vca_close_session(struct sess *sp, const char *why) {}
 
 #define N	100
 #define M	(128*1024)



From phk at varnish-cache.org  Sat Sep 17 14:09:28 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 16:09:28 +0200
Subject: [master] c105995 Move waiter related code from cache_acceptor.c to
	its own file.
Message-ID: <E1R4vaC-0005Nm-Hz@project.varnish-software.com>

commit c10599526e8f43d9c999a470d530f643752d2564
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 14:08:17 2011 +0000

    Move waiter related code from cache_acceptor.c to its own file.
    
    (Sort Makefile.am::SOURCES which had become unsorted)

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 288f1f1..e4ff980 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -11,10 +11,6 @@ dist_man_MANS = varnishd.1
 
 varnishd_SOURCES = \
 	cache_acceptor.c \
-	cache_waiter_epoll.c \
-	cache_waiter_kqueue.c \
-	cache_waiter_poll.c \
-	cache_waiter_ports.c \
 	cache_backend.c \
 	cache_backend_cfg.c \
 	cache_backend_poll.c \
@@ -22,20 +18,20 @@ varnishd_SOURCES = \
 	cache_center.c \
 	cache_cli.c \
 	cache_dir.c \
-	cache_dir_random.c \
 	cache_dir_dns.c \
+	cache_dir_random.c \
 	cache_dir_round_robin.c \
+	cache_esi_deliver.c \
 	cache_esi_fetch.c \
 	cache_esi_parse.c \
-	cache_esi_deliver.c \
 	cache_expire.c \
 	cache_fetch.c \
 	cache_gzip.c \
 	cache_hash.c \
 	cache_http.c \
 	cache_httpconn.c \
-	cache_main.c \
 	cache_lck.c \
+	cache_main.c \
 	cache_panic.c \
 	cache_pipe.c \
 	cache_pool.c \
@@ -48,6 +44,11 @@ varnishd_SOURCES = \
 	cache_vrt_re.c \
 	cache_vrt_var.c \
 	cache_vrt_vmod.c \
+	cache_waiter.c \
+	cache_waiter_epoll.c \
+	cache_waiter_kqueue.c \
+	cache_waiter_poll.c \
+	cache_waiter_ports.c \
 	cache_wrk.c \
 	cache_wrw.c \
 	cache_ws.c \
@@ -63,6 +64,7 @@ varnishd_SOURCES = \
 	mgt_vcc.c \
 	rfc2616.c \
 	stevedore.c \
+	stevedore_utils.c \
 	storage_file.c \
 	storage_malloc.c \
 	storage_persistent.c \
@@ -71,7 +73,6 @@ varnishd_SOURCES = \
 	storage_persistent_subr.c \
 	storage_synth.c \
 	storage_umem.c \
-	stevedore_utils.c \
 	varnishd.c \
 	vsm.c
 
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 2f43ff8..5d44092 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -649,7 +649,6 @@ void vca_return_session(struct sess *sp);
 void VCA_Prep(struct sess *sp);
 void VCA_Init(void);
 void VCA_Shutdown(void);
-const char *VCA_waiter_name(void);
 extern pthread_t VCA_thread;
 
 /* cache_backend.c */
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 46f955b..3781f47 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -30,14 +30,10 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <errno.h>
 #include <poll.h>
-#include <string.h>
 #include <stdlib.h>
-#include <unistd.h>
 
-#include <sys/uio.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 
@@ -46,22 +42,6 @@
 #include "cache.h"
 #include "cache_waiter.h"
 
-static struct waiter * const vca_waiters[] = {
-#if defined(HAVE_KQUEUE)
-	&waiter_kqueue,
-#endif
-#if defined(HAVE_EPOLL_CTL)
-	&waiter_epoll,
-#endif
-#if defined(HAVE_PORT_CREATE)
-	&waiter_ports,
-#endif
-	&waiter_poll,
-	NULL,
-};
-
-static struct waiter const *vca_act;
-
 static void *waiter_priv;
 
 pthread_t		VCA_thread;
@@ -69,21 +49,6 @@ static struct timeval	tv_sndtimeo;
 static struct timeval	tv_rcvtimeo;
 
 /*--------------------------------------------------------------------
- * Report waiter name to panics
- */
-
-const char *
-VCA_waiter_name(void)
-{
-
-	if (vca_act != NULL)
-		return (vca_act->name);
-	else
-		return ("no_waiter");
-}
-
-
-/*--------------------------------------------------------------------
  * We want to get out of any kind of trouble-hit TCP connections as fast
  * as absolutely possible, so we set them LINGER enabled with zero timeout,
  * so that even if there are outstanding write data on the socket, a close(2)
@@ -347,7 +312,7 @@ vca_return_session(struct sess *sp)
 	 */
 	if (VTCP_nonblocking(sp->fd))
 		SES_Close(sp, "remote closed");
-	vca_act->pass(waiter_priv, sp);
+	waiter->pass(waiter_priv, sp);
 }
 
 /*--------------------------------------------------------------------*/
@@ -360,17 +325,14 @@ ccf_start(struct cli *cli, const char * const *av, void *priv)
 	(void)av;
 	(void)priv;
 
-	if (vca_act == NULL)
-		vca_act = vca_waiters[0];
-
-	AN(vca_act);
-	AN(vca_act->name);
-	AN(vca_act->init);
-	AN(vca_act->pass);
+	AN(waiter);
+	AN(waiter->name);
+	AN(waiter->init);
+	AN(waiter->pass);
 
-	waiter_priv = vca_act->init();
+	waiter_priv = waiter->init();
 	AZ(pthread_create(&VCA_thread, NULL, vca_acct, NULL));
-	VSL(SLT_Debug, 0, "Acceptor is %s", vca_act->name);
+	VSL(SLT_Debug, 0, "Acceptor is %s", waiter->name);
 }
 
 /*--------------------------------------------------------------------*/
@@ -424,37 +386,3 @@ VCA_Shutdown(void)
 		(void)close(i);
 	}
 }
-
-void
-VCA_tweak_waiter(struct cli *cli, const char *arg)
-{
-	int i;
-
-	 ASSERT_MGT();
-
-	if (arg == NULL) {
-		if (vca_act == NULL)
-			VCLI_Out(cli, "default");
-		else
-			VCLI_Out(cli, "%s", vca_act->name);
-
-		VCLI_Out(cli, " (");
-		for (i = 0; vca_waiters[i] != NULL; i++)
-			VCLI_Out(cli, "%s%s", i == 0 ? "" : ", ",
-			    vca_waiters[i]->name);
-		VCLI_Out(cli, ")");
-		return;
-	}
-	if (!strcmp(arg, "default")) {
-		vca_act = NULL;
-		return;
-	}
-	for (i = 0; vca_waiters[i]; i++) {
-		if (!strcmp(arg, vca_waiters[i]->name)) {
-			vca_act = vca_waiters[i];
-			return;
-		}
-	}
-	VCLI_Out(cli, "Unknown waiter");
-	VCLI_SetResult(cli, CLIS_PARAM);
-}
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index 4312393..3489530 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -37,6 +37,7 @@
 #include "cache.h"
 #include "stevedore.h"
 #include "hash_slinger.h"
+#include "cache_waiter.h"
 
 /*--------------------------------------------------------------------
  * Per thread storage for the session currently being processed by
@@ -103,6 +104,7 @@ child_main(void)
 
 	LCK_Init();	/* Second, locking */
 
+	WAIT_Init();
 	PAN_Init();
 	CLI_Init();
 	Fetch_Init();
diff --git a/bin/varnishd/cache_panic.c b/bin/varnishd/cache_panic.c
index 2f0921d..b243832 100644
--- a/bin/varnishd/cache_panic.c
+++ b/bin/varnishd/cache_panic.c
@@ -44,6 +44,7 @@
 #include "cache.h"
 #include "vsm.h"
 #include "cache_backend.h"
+#include "cache_waiter.h"
 #include "vcl.h"
 #include "libvcl.h"
 
@@ -341,7 +342,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 		VSB_printf(vsp, "thread = (%s)\n", q);
 
 	VSB_printf(vsp, "ident = %s,%s\n",
-	    VSB_data(vident) + 1, VCA_waiter_name());
+	    VSB_data(vident) + 1, WAIT_GetName());
 
 	pan_backtrace();
 
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 982d8fe..eb2e62d 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -322,7 +322,7 @@ RES_WriteObj(struct sess *sp)
 	    !(sp->wrk->res_mode & RES_ESI_CHILD))
 		WRW_EndChunk(sp->wrk);
 
-	if (WRW_FlushRelease(sp->wrk))
+	if (WRW_FlushRelease(sp->wrk) && sp->fd >= 0)
 		SES_Close(sp, "remote closed");
 }
 
diff --git a/bin/varnishd/cache_waiter.c b/bin/varnishd/cache_waiter.c
new file mode 100644
index 0000000..cd3e007
--- /dev/null
+++ b/bin/varnishd/cache_waiter.c
@@ -0,0 +1,109 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include "vcli.h"
+#include "cli_priv.h"
+#include "cache.h"
+#include "cache_waiter.h"
+
+static const struct waiter * const vca_waiters[] = {
+    #if defined(HAVE_KQUEUE)
+	&waiter_kqueue,
+    #endif
+    #if defined(HAVE_EPOLL_CTL)
+	&waiter_epoll,
+    #endif
+    #if defined(HAVE_PORT_CREATE)
+	&waiter_ports,
+    #endif
+	&waiter_poll,
+	NULL,
+};
+
+struct waiter const * waiter;
+
+const char *
+WAIT_GetName(void)
+{
+
+	if (waiter != NULL)
+		return (waiter->name);
+	else
+		return ("no_waiter");
+}
+
+void
+WAIT_tweak_waiter(struct cli *cli, const char *arg)
+{
+	int i;
+
+	ASSERT_MGT();
+
+	if (arg == NULL) {
+		if (waiter == NULL)
+			VCLI_Out(cli, "default");
+		else
+			VCLI_Out(cli, "%s", waiter->name);
+
+		VCLI_Out(cli, " (");
+		for (i = 0; vca_waiters[i] != NULL; i++)
+			VCLI_Out(cli, "%s%s", i == 0 ? "" : ", ",
+			    vca_waiters[i]->name);
+		VCLI_Out(cli, ")");
+		return;
+	}
+	if (!strcmp(arg, "default")) {
+		waiter = NULL;
+		return;
+	}
+	for (i = 0; vca_waiters[i]; i++) {
+		if (!strcmp(arg, vca_waiters[i]->name)) {
+			waiter = vca_waiters[i];
+			return;
+		}
+	}
+	VCLI_Out(cli, "Unknown waiter");
+	VCLI_SetResult(cli, CLIS_PARAM);
+}
+
+void
+WAIT_Init(void)
+{
+
+	if (waiter == NULL)
+		waiter = vca_waiters[0];
+
+	AN(waiter);
+	AN(waiter->name);
+	AN(waiter->init);
+	AN(waiter->pass);
+}
diff --git a/bin/varnishd/cache_waiter.h b/bin/varnishd/cache_waiter.h
index 86eb2cb..4377f82 100644
--- a/bin/varnishd/cache_waiter.h
+++ b/bin/varnishd/cache_waiter.h
@@ -39,20 +39,25 @@ struct waiter {
 	waiter_pass_f		*pass;
 };
 
+extern struct waiter const * waiter;
+
 #if defined(HAVE_EPOLL_CTL)
-extern struct waiter waiter_epoll;
+extern const struct waiter waiter_epoll;
 #endif
 
 #if defined(HAVE_KQUEUE)
-extern struct waiter waiter_kqueue;
+extern const struct waiter waiter_kqueue;
 #endif
 
-extern struct waiter waiter_poll;
-
 #if defined(HAVE_PORT_CREATE)
-extern struct waiter waiter_ports;
+extern const struct waiter waiter_ports;
 #endif
 
+extern const struct waiter waiter_poll;
+
 /* cache_session.c */
 void SES_Handle(struct sess *sp, int status);
 
+const char *WAIT_GetName(void);
+void WAIT_tweak_waiter(struct cli *cli, const char *arg);
+void WAIT_Init(void);
diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index acb8e13..28776cd 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -269,7 +269,7 @@ vwe_init(void)
 
 /*--------------------------------------------------------------------*/
 
-struct waiter waiter_epoll = {
+const struct waiter waiter_epoll = {
 	.name =		"epoll",
 	.init =		vwe_init,
 	.pass =		vwe_pass,
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index 7e6b10f..7decfea 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -237,7 +237,7 @@ vwk_init(void)
 
 /*--------------------------------------------------------------------*/
 
-struct waiter waiter_kqueue = {
+const struct waiter waiter_kqueue = {
 	.name =		"kqueue",
 	.init =		vwk_init,
 	.pass =		vwk_pass,
diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index f44daa9..3a211bd 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -227,7 +227,7 @@ vwp_poll_init(void)
 
 /*--------------------------------------------------------------------*/
 
-struct waiter waiter_poll = {
+const struct waiter waiter_poll = {
 	.name =		"poll",
 	.init =		vwp_poll_init,
 	.pass =		vwp_poll_pass,
diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index 329ee23..695e9c0 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -275,7 +275,7 @@ vws_init(void)
 
 /*--------------------------------------------------------------------*/
 
-struct waiter waiter_ports = {
+const struct waiter waiter_ports = {
 	.name =		"ports",
 	.init =		vws_init,
 	.pass =		vws_pass
diff --git a/bin/varnishd/common.h b/bin/varnishd/common.h
index 1a64cf3..3708b45 100644
--- a/bin/varnishd/common.h
+++ b/bin/varnishd/common.h
@@ -33,9 +33,6 @@ struct cli;
 extern pid_t mgt_pid;
 #define ASSERT_MGT() do { assert(getpid() == mgt_pid);} while (0)
 
-/* cache_acceptor.c */
-void VCA_tweak_waiter(struct cli *cli, const char *arg);
-
 /* mgt_shmem.c */
 extern struct VSC_C_main *VSC_C_main;
 
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index 33101bb..b9067e7 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -48,6 +48,7 @@
 
 #include "heritage.h"
 #include "vparam.h"
+#include "cache_waiter.h"
 
 #include "vss.h"
 
@@ -438,7 +439,7 @@ tweak_waiter(struct cli *cli, const struct parspec *par, const char *arg)
 
 	/* XXX should have tweak_generic_string */
 	(void)par;
-	VCA_tweak_waiter(cli, arg);
+	WAIT_tweak_waiter(cli, arg);
 }
 
 /*--------------------------------------------------------------------*/



From phk at varnish-cache.org  Sat Sep 17 14:45:18 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 16:45:18 +0200
Subject: [master] 0841b65 Move vca_return_session() to Pool_Wait()
Message-ID: <E1R4w8s-0007Se-Gy@project.varnish-software.com>

commit 0841b65418d21eed601be56abefb4f56b462d750
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 14:44:53 2011 +0000

    Move vca_return_session() to Pool_Wait()

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 5d44092..4fd7086 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -645,7 +645,6 @@ struct vbc {
 /* Prototypes etc ----------------------------------------------------*/
 
 /* cache_acceptor.c */
-void vca_return_session(struct sess *sp);
 void VCA_Prep(struct sess *sp);
 void VCA_Init(void);
 void VCA_Shutdown(void);
@@ -840,6 +839,7 @@ void PipeSession(struct sess *sp);
 void Pool_Init(void);
 int Pool_QueueSession(struct sess *sp);
 void Pool_Work_Thread(void *priv, struct worker *w);
+void Pool_Wait(struct sess *sp);
 
 #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
 int WRW_Error(const struct worker *w);
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 3781f47..23752f3 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -40,9 +40,6 @@
 #include "vcli.h"
 #include "cli_priv.h"
 #include "cache.h"
-#include "cache_waiter.h"
-
-static void *waiter_priv;
 
 pthread_t		VCA_thread;
 static struct timeval	tv_sndtimeo;
@@ -298,25 +295,6 @@ vca_acct(void *arg)
 
 /*--------------------------------------------------------------------*/
 
-void
-vca_return_session(struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AZ(sp->obj);
-	AZ(sp->vcl);
-	assert(sp->fd >= 0);
-	/*
-	 * Set nonblocking in the worker-thread, before passing to the
-	 * acceptor thread, to reduce syscall density of the latter.
-	 */
-	if (VTCP_nonblocking(sp->fd))
-		SES_Close(sp, "remote closed");
-	waiter->pass(waiter_priv, sp);
-}
-
-/*--------------------------------------------------------------------*/
-
 static void
 ccf_start(struct cli *cli, const char * const *av, void *priv)
 {
@@ -325,14 +303,7 @@ ccf_start(struct cli *cli, const char * const *av, void *priv)
 	(void)av;
 	(void)priv;
 
-	AN(waiter);
-	AN(waiter->name);
-	AN(waiter->init);
-	AN(waiter->pass);
-
-	waiter_priv = waiter->init();
 	AZ(pthread_create(&VCA_thread, NULL, vca_acct, NULL));
-	VSL(SLT_Debug, 0, "Acceptor is %s", waiter->name);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 24b0ddc..b561c84 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -108,7 +108,7 @@ cnt_wait(struct sess *sp)
 		sp->wrk->stats.sess_herd++;
 		SES_Charge(sp);
 		sp->wrk = NULL;
-		vca_return_session(sp);
+		Pool_Wait(sp);
 		return (1);
 	}
 	if (i == 1) {
@@ -398,7 +398,7 @@ cnt_done(struct sess *sp)
 	}
 	sp->wrk->stats.sess_herd++;
 	sp->wrk = NULL;
-	vca_return_session(sp);
+	Pool_Wait(sp);
 	return (1);
 }
 
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 92e81b1..0bbb76d 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -54,9 +54,12 @@
 #include "vcl.h"
 #include "cli_priv.h"
 #include "cache.h"
+#include "cache_waiter.h"
 #include "stevedore.h"
 #include "hash_slinger.h"
 
+static void *waiter_priv;
+
 VTAILQ_HEAD(workerhead, worker);
 
 /* Number of work requests queued in excess of worker threads available */
@@ -454,6 +457,27 @@ wrk_herder_thread(void *priv)
 	NEEDLESS_RETURN(NULL);
 }
 
+/*--------------------------------------------------------------------
+ * Wait for another request
+ */
+
+void
+Pool_Wait(struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	AZ(sp->obj);
+	AZ(sp->vcl);
+	assert(sp->fd >= 0);
+	/*
+	 * Set nonblocking in the worker-thread, before passing to the
+	 * acceptor thread, to reduce syscall density of the latter.
+	 */
+	if (VTCP_nonblocking(sp->fd))
+		SES_Close(sp, "remote closed");
+	waiter->pass(waiter_priv, sp);
+}
+
 /*--------------------------------------------------------------------*/
 
 void
@@ -464,6 +488,8 @@ Pool_Init(void)
 	AZ(pthread_cond_init(&herder_cond, NULL));
 	Lck_New(&herder_mtx, lck_herder);
 
+	waiter_priv = waiter->init();
+
 	wrk_addpools(params->wthread_pools);
 	AZ(pthread_create(&tp, NULL, wrk_herdtimer_thread, NULL));
 	AZ(pthread_detach(tp));
diff --git a/bin/varnishd/cache_waiter.h b/bin/varnishd/cache_waiter.h
index 4377f82..28bc39d 100644
--- a/bin/varnishd/cache_waiter.h
+++ b/bin/varnishd/cache_waiter.h
@@ -53,11 +53,12 @@ extern const struct waiter waiter_kqueue;
 extern const struct waiter waiter_ports;
 #endif
 
-extern const struct waiter waiter_poll;
 
 /* cache_session.c */
 void SES_Handle(struct sess *sp, int status);
 
+/* cache_waiter.c */
+extern const struct waiter waiter_poll;
 const char *WAIT_GetName(void);
 void WAIT_tweak_waiter(struct cli *cli, const char *arg);
 void WAIT_Init(void);



From phk at varnish-cache.org  Sat Sep 17 15:59:43 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 17:59:43 +0200
Subject: [master] 12f5366 Rename struct wq to struct pool,
	and tag worker threads with it.
Message-ID: <E1R4xIt-0000AU-2x@project.varnish-software.com>

commit 12f5366e05dd0b1cb52ceea1e75f80cd8858b642
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 15:59:24 2011 +0000

    Rename struct wq to struct pool, and tag worker threads with it.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 4fd7086..0a566f2 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -116,6 +116,7 @@ struct SHA256Context;
 struct VSC_C_lck;
 struct waitinglist;
 struct vef_priv;
+struct pool;
 
 #define DIGEST_LEN		32
 
@@ -291,6 +292,7 @@ struct stream_ctx {
 struct worker {
 	unsigned		magic;
 #define WORKER_MAGIC		0x6391adcf
+	struct pool		*pool;
 	struct objhead		*nobjhead;
 	struct objcore		*nobjcore;
 	struct waitinglist	*nwaitinglist;
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 0bbb76d..63245d6 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -64,9 +64,9 @@ VTAILQ_HEAD(workerhead, worker);
 
 /* Number of work requests queued in excess of worker threads available */
 
-struct wq {
+struct pool {
 	unsigned		magic;
-#define WQ_MAGIC		0x606658fa
+#define POOL_MAGIC		0x606658fa
 	struct lock		mtx;
 	struct workerhead	idle;
 	VTAILQ_HEAD(, workreq)	queue;
@@ -77,7 +77,7 @@ struct wq {
 	uintmax_t		nqueue;
 };
 
-static struct wq		**wq;
+static struct pool		**wq;
 static unsigned			nwq;
 static unsigned			queue_max;
 static unsigned			nthr_max;
@@ -90,10 +90,11 @@ static struct lock		herder_mtx;
 void
 Pool_Work_Thread(void *priv, struct worker *w)
 {
-	struct wq *qp;
+	struct pool *qp;
 	int stats_clean;
 
-	CAST_OBJ_NOTNULL(qp, priv, WQ_MAGIC);
+	CAST_OBJ_NOTNULL(qp, priv, POOL_MAGIC);
+	w->pool = qp;
 	Lck_Lock(&qp->mtx);
 	qp->nthr++;
 	stats_clean = 1;
@@ -145,6 +146,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 	}
 	qp->nthr--;
 	Lck_Unlock(&qp->mtx);
+	w->pool = NULL;
 }
 
 /*--------------------------------------------------------------------
@@ -157,7 +159,7 @@ static int
 WRK_Queue(struct workreq *wrq)
 {
 	struct worker *w;
-	struct wq *qp;
+	struct pool *qp;
 	static unsigned nq = 0;
 	unsigned onq;
 
@@ -253,7 +255,7 @@ Pool_QueueSession(struct sess *sp)
 static void
 wrk_addpools(const unsigned pools)
 {
-	struct wq **pwq, **owq;
+	struct pool **pwq, **owq;
 	unsigned u;
 
 	pwq = calloc(sizeof *pwq, pools);
@@ -266,7 +268,7 @@ wrk_addpools(const unsigned pools)
 	for (u = nwq; u < pools; u++) {
 		wq[u] = calloc(sizeof *wq[0], 1);
 		XXXAN(wq[u]);
-		wq[u]->magic = WQ_MAGIC;
+		wq[u]->magic = POOL_MAGIC;
 		Lck_New(&wq[u]->mtx, lck_wq);
 		VTAILQ_INIT(&wq[u]->queue);
 		VTAILQ_INIT(&wq[u]->idle);
@@ -280,7 +282,7 @@ wrk_addpools(const unsigned pools)
  */
 
 static void
-wrk_decimate_flock(struct wq *qp, double t_idle, struct VSC_C_main *vs)
+wrk_decimate_flock(struct pool *qp, double t_idle, struct VSC_C_main *vs)
 {
 	struct worker *w = NULL;
 
@@ -380,7 +382,7 @@ wrk_herdtimer_thread(void *priv)
  */
 
 static void
-wrk_breed_flock(struct wq *qp, const pthread_attr_t *tp_attr)
+wrk_breed_flock(struct pool *qp, const pthread_attr_t *tp_attr)
 {
 	pthread_t tp;
 
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 66ba04b..7b731ba 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -175,6 +175,7 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
 	VSL(SLT_WorkThread, 0, "%p start", w);
 
 	Pool_Work_Thread(priv, w);
+	AZ(w->pool);
 
 	VSL(SLT_WorkThread, 0, "%p end", w);
 	if (w->vcl != NULL)



From phk at varnish-cache.org  Sat Sep 17 16:45:19 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 18:45:19 +0200
Subject: [master] 738cafa Drop struct workreq, we don't use it.
Message-ID: <E1R4y11-0001b0-LE@project.varnish-software.com>

commit 738cafa04be5cca08cbb472603974732357ce112
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 16:45:07 2011 +0000

    Drop struct workreq, we don't use it.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 0a566f2..65d1886 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -108,7 +108,6 @@ struct objhead;
 struct objcore;
 struct busyobj;
 struct storage;
-struct workreq;
 struct vrt_backend;
 struct cli_proto;
 struct ban;
@@ -307,7 +306,7 @@ struct worker {
 	pthread_cond_t		cond;
 
 	VTAILQ_ENTRY(worker)	list;
-	struct workreq		*wrq;
+	struct sess		*sp;
 
 	struct VCL_conf		*vcl;
 
@@ -369,21 +368,6 @@ struct worker {
 	struct acct		acct_tmp;
 };
 
-/* Work Request for worker thread ------------------------------------*/
-
-/*
- * This is a worker-function.
- * XXX: typesafety is probably not worth fighting for
- */
-
-typedef void workfunc(struct worker *, void *priv);
-
-struct workreq {
-	VTAILQ_ENTRY(workreq)	list;
-	workfunc		*func;
-	void			*priv;
-};
-
 /* Storage -----------------------------------------------------------*/
 
 struct storage {
@@ -614,7 +598,7 @@ struct sess {
 	/* Various internal stuff */
 	struct sessmem		*mem;
 
-	struct workreq		workreq;
+	VTAILQ_ENTRY(sess)	poollist;
 	struct acct		acct_req;
 	struct acct		acct_ses;
 
diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index a3d1851..dd26303 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -131,7 +131,6 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv)
 	SZOF(struct http_conn);
 	SZOF(struct acct);
 	SZOF(struct worker);
-	SZOF(struct workreq);
 	SZOF(struct storage);
 	SZOF(struct object);
 	SZOF(struct objcore);
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 63245d6..6fc7737 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -69,7 +69,7 @@ struct pool {
 #define POOL_MAGIC		0x606658fa
 	struct lock		mtx;
 	struct workerhead	idle;
-	VTAILQ_HEAD(, workreq)	queue;
+	VTAILQ_HEAD(, sess)	queue;
 	unsigned		nthr;
 	unsigned		lqueue;
 	unsigned		last_lqueue;
@@ -105,9 +105,9 @@ Pool_Work_Thread(void *priv, struct worker *w)
 		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 
 		/* Process queued requests, if any */
-		w->wrq = VTAILQ_FIRST(&qp->queue);
-		if (w->wrq != NULL) {
-			VTAILQ_REMOVE(&qp->queue, w->wrq, list);
+		w->sp = VTAILQ_FIRST(&qp->queue);
+		if (w->sp != NULL) {
+			VTAILQ_REMOVE(&qp->queue, w->sp, poollist);
 			qp->lqueue--;
 		} else {
 			if (isnan(w->lastused))
@@ -117,17 +117,21 @@ Pool_Work_Thread(void *priv, struct worker *w)
 				WRK_SumStat(w);
 			Lck_CondWait(&w->cond, &qp->mtx);
 		}
-		if (w->wrq == NULL)
+		if (w->sp == NULL)
 			break;
 		Lck_Unlock(&qp->mtx);
 		stats_clean = 0;
-		AN(w->wrq);
-		AN(w->wrq->func);
 		w->lastused = NAN;
 		WS_Reset(w->ws, NULL);
 		w->storage_hint = NULL;
 
-		w->wrq->func(w, w->wrq->priv);
+		AZ(w->sp->wrk);
+		THR_SetSession(w->sp);
+		w->sp->wrk = w;
+		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+		CNT_Session(w->sp);
+		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+		THR_SetSession(NULL);
 
 		WS_Assert(w->ws);
 		AZ(w->bereq->ws);
@@ -136,7 +140,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 		AZ(w->wrw.wfd);
 		AZ(w->storage_hint);
 		assert(w->wlp == w->wlb);
-		w->wrq = NULL;
+		w->sp = NULL;
 		if (params->diag_bitmap & 0x00040000) {
 			if (w->vcl != NULL)
 				VCL_Rel(&w->vcl);
@@ -156,7 +160,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
  */
 
 static int
-WRK_Queue(struct workreq *wrq)
+WRK_Queue(struct sess *sp)
 {
 	struct worker *w;
 	struct pool *qp;
@@ -181,7 +185,7 @@ WRK_Queue(struct workreq *wrq)
 	if (w != NULL) {
 		VTAILQ_REMOVE(&qp->idle, w, list);
 		Lck_Unlock(&qp->mtx);
-		w->wrq = wrq;
+		w->sp = sp;
 		AZ(pthread_cond_signal(&w->cond));
 		return (0);
 	}
@@ -193,7 +197,7 @@ WRK_Queue(struct workreq *wrq)
 		return (-1);
 	}
 
-	VTAILQ_INSERT_TAIL(&qp->queue, wrq, list);
+	VTAILQ_INSERT_TAIL(&qp->queue, sp, poollist);
 	qp->nqueue++;
 	qp->lqueue++;
 	Lck_Unlock(&qp->mtx);
@@ -203,31 +207,12 @@ WRK_Queue(struct workreq *wrq)
 
 /*--------------------------------------------------------------------*/
 
-static void
-wrk_do_cnt_sess(struct worker *w, void *priv)
-{
-	struct sess *sess;
-
-	CAST_OBJ_NOTNULL(sess, priv, SESS_MAGIC);
-	AZ(sess->wrk);
-	THR_SetSession(sess);
-	sess->wrk = w;
-	CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
-	CNT_Session(sess);
-	CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
-	THR_SetSession(NULL);
-}
-
-/*--------------------------------------------------------------------*/
-
 int
 Pool_QueueSession(struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk);
-	sp->workreq.func = wrk_do_cnt_sess;
-	sp->workreq.priv = sp;
-	if (WRK_Queue(&sp->workreq) == 0)
+	if (WRK_Queue(sp) == 0)
 		return (0);
 
 	/*
@@ -303,7 +288,7 @@ wrk_decimate_flock(struct pool *qp, double t_idle, struct VSC_C_main *vs)
 
 	/* And give it a kiss on the cheek... */
 	if (w != NULL) {
-		AZ(w->wrq);
+		AZ(w->sp);
 		AZ(pthread_cond_signal(&w->cond));
 		TIM_sleep(params->wthread_purge_delay * 1e-3);
 	}



From phk at varnish-cache.org  Sat Sep 17 16:57:37 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 18:57:37 +0200
Subject: [master] fc59c74 Give each pool a list of sockets to accept() from
Message-ID: <E1R4yCv-0002AV-Dd@project.varnish-software.com>

commit fc59c7455129e230fbead7627d522f6a7e8c07d1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 16:57:20 2011 +0000

    Give each pool a list of sockets to accept() from

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 6fc7737..05ce529 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -62,6 +62,13 @@ static void *waiter_priv;
 
 VTAILQ_HEAD(workerhead, worker);
 
+struct poolsock {
+	unsigned			magic;
+#define POOLSOCK_MAGIC			0x1b0a2d38
+	VTAILQ_ENTRY(poolsock)		list;
+	int				sock;
+};
+
 /* Number of work requests queued in excess of worker threads available */
 
 struct pool {
@@ -70,6 +77,7 @@ struct pool {
 	struct lock		mtx;
 	struct workerhead	idle;
 	VTAILQ_HEAD(, sess)	queue;
+	VTAILQ_HEAD(, poolsock)	socks;
 	unsigned		nthr;
 	unsigned		lqueue;
 	unsigned		last_lqueue;
@@ -237,6 +245,29 @@ Pool_QueueSession(struct sess *sp)
  * Add (more) thread pools
  */
 
+static struct pool *
+pool_mkpool(void)
+{
+	struct pool *pp;
+	struct listen_sock *ls;
+	struct poolsock *ps;
+
+	ALLOC_OBJ(pp, POOL_MAGIC);
+	XXXAN(pp);
+	Lck_New(&pp->mtx, lck_wq);
+	VTAILQ_INIT(&pp->queue);
+	VTAILQ_INIT(&pp->idle);
+	VTAILQ_INIT(&pp->socks);
+
+	VTAILQ_FOREACH(ls, &heritage.socks, list) {
+		ALLOC_OBJ(ps, POOLSOCK_MAGIC);
+		XXXAN(ps);
+		ps->sock = ls->sock;
+		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
+	}
+	return (pp);
+}
+
 static void
 wrk_addpools(const unsigned pools)
 {
@@ -251,12 +282,8 @@ wrk_addpools(const unsigned pools)
 	owq = wq;
 	wq = pwq;
 	for (u = nwq; u < pools; u++) {
-		wq[u] = calloc(sizeof *wq[0], 1);
+		wq[u] = pool_mkpool();
 		XXXAN(wq[u]);
-		wq[u]->magic = POOL_MAGIC;
-		Lck_New(&wq[u]->mtx, lck_wq);
-		VTAILQ_INIT(&wq[u]->queue);
-		VTAILQ_INIT(&wq[u]->idle);
 	}
 	(void)owq;	/* XXX: avoid race, leak it. */
 	nwq = pools;



From phk at varnish-cache.org  Sat Sep 17 17:36:52 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 19:36:52 +0200
Subject: [master] 5d5e873 Prepare the acceptor code to become multithreaded by
	moving the pacing under a dedicated lock.
Message-ID: <E1R4you-00044X-Mb@project.varnish-software.com>

commit 5d5e8732ad589a0b417e8f78585e4c252ea07c15
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 17:36:06 2011 +0000

    Prepare the acceptor code to become multithreaded by moving the
    pacing under a dedicated lock.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 65d1886..6c0c525 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -299,8 +299,13 @@ struct worker {
 	void			*nhashpriv;
 	struct dstat		stats;
 
+	/* Pool stuff */
 	double			lastused;
 
+	/* Accept stuff */
+	struct sockaddr_storage	acceptaddr;
+	int			acceptsock;
+
 	struct wrw		wrw;
 
 	pthread_cond_t		cond;
@@ -634,6 +639,7 @@ struct vbc {
 void VCA_Prep(struct sess *sp);
 void VCA_Init(void);
 void VCA_Shutdown(void);
+int VCA_Accept(int sock, socklen_t *slp, struct sockaddr_storage *sap);
 extern pthread_t VCA_thread;
 
 /* cache_backend.c */
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 23752f3..0022037 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -153,6 +153,85 @@ VCA_Prep(struct sess *sp)
 #endif
 }
 
+/*--------------------------------------------------------------------
+ * If accept(2)'ing fails, we pace ourselves to relive any resource
+ * shortage if possible.
+ */
+
+static double vca_pace = 0.0;
+static struct lock pace_mtx;
+
+static void
+vca_pace_check(void)
+{
+	double p;
+
+	if (vca_pace == 0.0) 
+		return;
+	Lck_Lock(&pace_mtx);
+	p = vca_pace;
+	Lck_Unlock(&pace_mtx);
+	if (p > 0.0)
+		TIM_sleep(p);
+}
+
+static void
+vca_pace_bad(void)
+{
+	Lck_Lock(&pace_mtx);
+	vca_pace += params->acceptor_sleep_incr;
+	if (vca_pace > params->acceptor_sleep_max)
+		vca_pace = params->acceptor_sleep_max;
+	Lck_Unlock(&pace_mtx);
+}
+
+static void
+vca_pace_good(void)
+{
+
+	if (vca_pace == 0.0) 
+		return;
+	Lck_Lock(&pace_mtx);
+	vca_pace *= params->acceptor_sleep_decay;
+	if (vca_pace < params->acceptor_sleep_incr)
+		vca_pace = 0.0;
+	Lck_Unlock(&pace_mtx);
+}
+
+/*--------------------------------------------------------------------
+ * Accept on a listen socket, and handle error returns.
+ */
+
+int
+VCA_Accept(int sock, socklen_t *slp, struct sockaddr_storage *sap)
+{
+	int i;
+
+	vca_pace_check();
+
+	*slp = sizeof *sap;
+	i = accept(sock, (void*)sap, slp);
+
+	if (i < 0) {
+		VSC_C_main->accept_fail++;
+		switch (errno) {
+		case EAGAIN:
+		case ECONNABORTED:
+			break;
+		case EMFILE:
+			VSL(SLT_Debug, sock, "Too many open files");
+			vca_pace_bad();
+			break;
+		default:
+			VSL(SLT_Debug, sock, "Accept failed: %s",
+			    strerror(errno));
+			vca_pace_bad();
+			break;
+		}
+	}
+	return (i);
+}
+
 /*--------------------------------------------------------------------*/
 
 static void *
@@ -172,7 +251,7 @@ vca_acct(void *arg)
 	struct pollfd *pfd;
 	struct listen_sock *ls;
 	unsigned u;
-	double t0, now, pace;
+	double t0, now;
 
 	THR_SetName("cache-acceptor");
 	(void)arg;
@@ -192,7 +271,6 @@ vca_acct(void *arg)
 	}
 
 	need_test = 1;
-	pace = 0;
 	t0 = TIM_real();
 	while (1) {
 #ifdef SO_SNDTIMEO_WORKS
@@ -223,13 +301,6 @@ vca_acct(void *arg)
 			}
 		}
 #endif
-		/* Bound the pacing delay by parameter */
-		if (pace > params->acceptor_sleep_max)
-			pace = params->acceptor_sleep_max;
-		if (pace < params->acceptor_sleep_incr)
-			pace = 0.0;
-		if (pace > 0.0)
-			TIM_sleep(pace);
 		i = poll(pfd, heritage.nsocks, 1000);
 		now = TIM_real();
 		VSC_C_main->uptime = (uint64_t)(now - t0);
@@ -242,33 +313,14 @@ vca_acct(void *arg)
 			VSC_C_main->client_conn++;
 			l = sizeof addr_s;
 			addr = (void*)&addr_s;
-			i = accept(ls->sock, addr, &l);
-			if (i < 0) {
-				VSC_C_main->accept_fail++;
-				switch (errno) {
-				case EAGAIN:
-				case ECONNABORTED:
-					break;
-				case EMFILE:
-					VSL(SLT_Debug, ls->sock,
-					    "Too many open files "
-					    "when accept(2)ing. Sleeping.");
-					pace += params->acceptor_sleep_incr;
-					break;
-				default:
-					VSL(SLT_Debug, ls->sock,
-					    "Accept failed: %s",
-					    strerror(errno));
-					pace += params->acceptor_sleep_incr;
-					break;
-				}
+			i = VCA_Accept(ls->sock, &l, &addr_s);
+			if (i < 0) 
 				continue;
-			}
 			sp = SES_New();
 			if (sp == NULL) {
 				AZ(close(i));
 				VSC_C_main->client_drop++;
-				pace += params->acceptor_sleep_incr;
+				vca_pace_bad();
 				continue;
 			}
 			sp->fd = i;
@@ -283,9 +335,9 @@ vca_acct(void *arg)
 			sp->step = STP_FIRST;
 			if (Pool_QueueSession(sp)) {
 				VSC_C_main->client_drop++;
-				pace += params->acceptor_sleep_incr;
+				vca_pace_bad();
 			} else {
-				pace *= params->acceptor_sleep_decay;
+				vca_pace_good();
 			}
 		}
 	}
@@ -341,6 +393,7 @@ VCA_Init(void)
 {
 
 	CLI_AddFuncs(vca_cmds);
+	Lck_New(&pace_mtx, lck_vcapace);
 }
 
 void
diff --git a/bin/varnishd/locks.h b/bin/varnishd/locks.h
index e0a6bd5..6cbf91f 100644
--- a/bin/varnishd/locks.h
+++ b/bin/varnishd/locks.h
@@ -49,4 +49,5 @@ LOCK(ban)
 LOCK(vbp)
 LOCK(vbe)
 LOCK(backend)
+LOCK(vcapace)
 /*lint -restore */



From phk at varnish-cache.org  Sat Sep 17 18:41:50 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 20:41:50 +0200
Subject: [master] 761d0c0 Add body_status.h to includes
Message-ID: <E1R4zpm-0002HJ-MD@project.varnish-software.com>

commit 761d0c0d2bd78a6b094d4283064851436fbfbd19
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 18:41:41 2011 +0000

    Add body_status.h to includes

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index e4ff980..c13f4e5 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -78,6 +78,7 @@ varnishd_SOURCES = \
 
 noinst_HEADERS = \
 	acct_fields.h \
+	body_status.h \
 	cache.h \
 	cache_backend.h \
 	cache_backend_poll.h \



From phk at varnish-cache.org  Sat Sep 17 20:07:18 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 22:07:18 +0200
Subject: [master] ac1fae5 Screw up the session allocator performance,
	in order to prepare it for per-pool operation.
Message-ID: <E1R51AU-0006Ig-E9@project.varnish-software.com>

commit ac1fae53f4b5c66db6db5d7e3ade181b2ab6dc50
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 20:06:36 2011 +0000

    Screw up the session allocator performance, in order to prepare it
    for per-pool operation.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 6c0c525..71cf6b3 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -108,6 +108,7 @@ struct objhead;
 struct objcore;
 struct busyobj;
 struct storage;
+struct sesspool;
 struct vrt_backend;
 struct cli_proto;
 struct ban;
@@ -849,7 +850,7 @@ void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 
 /* cache_session.c [SES] */
 void SES_Init(void);
-struct sess *SES_New(void);
+struct sess *SES_New(struct sesspool *pp);
 struct sess *SES_Alloc(void);
 void SES_Close(struct sess *sp, const char *reason);
 void SES_Delete(struct sess *sp, const char *reason);
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 0022037..37cc857 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -316,7 +316,7 @@ vca_acct(void *arg)
 			i = VCA_Accept(ls->sock, &l, &addr_s);
 			if (i < 0) 
 				continue;
-			sp = SES_New();
+			sp = SES_New(NULL);
 			if (sp == NULL) {
 				AZ(close(i));
 				VSC_C_main->client_drop++;
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 15373a4..6b34928 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -26,16 +26,8 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * Session and Client management.
+ * Session management
  *
- * XXX: The two-list session management is actually not a good idea
- * XXX: come to think of it, because we want the sessions reused in
- * XXX: Most Recently Used order.
- * XXX: Another and maybe more interesting option would be to cache
- * XXX: free sessions in the worker threads and postpone session
- * XXX: allocation until then.  This does not quite implment MRU order
- * XXX: but it does save some locking, although not that much because
- * XXX: we still have to do the source-addr lookup.
  */
 
 #include "config.h"
@@ -56,6 +48,7 @@ struct sessmem {
 	unsigned		magic;
 #define SESSMEM_MAGIC		0x555859c5
 
+	struct sesspool		*pool;
 	struct sess		sess;
 	unsigned		workspace;
 	void			*wsp;
@@ -64,13 +57,16 @@ struct sessmem {
 	struct sockaddr_storage	sockaddr[2];
 };
 
-static VTAILQ_HEAD(,sessmem)	ses_free_mem[2] = {
-    VTAILQ_HEAD_INITIALIZER(ses_free_mem[0]),
-    VTAILQ_HEAD_INITIALIZER(ses_free_mem[1]),
+struct sesspool {
+	unsigned		magic;
+#define SESSPOOL_MAGIC		0xd916e202
+	VTAILQ_HEAD(,sessmem)	freelist;
+	struct lock		mtx;
+	unsigned		nsess;
+	unsigned		maxsess;
 };
 
-static unsigned ses_qp;
-static struct lock		ses_mem_mtx;
+static struct sesspool *sesspool;
 
 /*--------------------------------------------------------------------*/
 
@@ -106,8 +102,6 @@ ses_sm_alloc(void)
 	uint16_t nhttp;
 	unsigned l, hl;
 
-	if (VSC_C_main->n_sess_mem >= params->max_sess)
-		return (NULL);
 	/*
 	 * It is not necessary to lock these, but we need to
 	 * cache them locally, to make sure we get a consistent
@@ -115,6 +109,7 @@ ses_sm_alloc(void)
 	 */
 	nws = params->sess_workspace;
 	nhttp = (uint16_t)params->http_max_hdr;
+
 	hl = HTTP_estimate(nhttp);
 	l = sizeof *sm + nws + 2 * hl;
 	p = malloc(l);
@@ -122,6 +117,7 @@ ses_sm_alloc(void)
 		return (NULL);
 	q = p + l;
 
+	/* XXX Stats */
 	Lck_Lock(&stat_mtx);
 	VSC_C_main->n_sess_mem++;
 	Lck_Unlock(&stat_mtx);
@@ -153,7 +149,6 @@ ses_setup(struct sessmem *sm)
 {
 	struct sess *sp;
 
-
 	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
 	sp = &sm->sess;
 	memset(sp, 0, sizeof *sp);
@@ -184,39 +179,38 @@ ses_setup(struct sessmem *sm)
  */
 
 struct sess *
-SES_New(void)
+SES_New(struct sesspool *pp)
 {
 	struct sessmem *sm;
 	struct sess *sp;
+	int do_alloc = 0;
 
-	assert(pthread_self() == VCA_thread);
-	assert(ses_qp <= 1);
-	sm = VTAILQ_FIRST(&ses_free_mem[ses_qp]);
-	if (sm == NULL) {
-		/*
-		 * If that queue is empty, flip queues holding the lock
-		 * and try the new unlocked queue.
-		 */
-		Lck_Lock(&ses_mem_mtx);
-		ses_qp = 1 - ses_qp;
-		Lck_Unlock(&ses_mem_mtx);
-		sm = VTAILQ_FIRST(&ses_free_mem[ses_qp]);
-	}
+	if (pp == NULL)
+		pp = sesspool;
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+
+	Lck_Lock(&pp->mtx);
+	sm = VTAILQ_FIRST(&pp->freelist);
 	if (sm != NULL) {
-		VTAILQ_REMOVE(&ses_free_mem[ses_qp], sm, list);
-		sp = &sm->sess;
-		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	} else {
+		VTAILQ_REMOVE(&pp->freelist, sm, list);
+	} else if (pp->nsess < pp->maxsess) {
+		pp->nsess++;
+		do_alloc = 1;
+	}
+	Lck_Unlock(&pp->mtx);
+	if (do_alloc) {
 		sm = ses_sm_alloc();
-		if (sm == NULL)
-			return (NULL);
-		ses_setup(sm);
-		sp = &sm->sess;
-		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+		if (sm != NULL) {
+			sm->pool = pp;
+			ses_setup(sm);
+		}
 	}
-
+	if (sm == NULL)
+		return (NULL);
+	sp = &sm->sess;
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	/* XXX Stats */
 	VSC_C_main->n_sess++;		/* XXX: locking  ? */
-
 	return (sp);
 }
 
@@ -292,10 +286,14 @@ SES_Delete(struct sess *sp, const char *reason)
 	struct acct *b = &sp->acct_ses;
 	struct sessmem *sm;
 	static char noaddr[] = "-";
+	struct sesspool *pp;
+
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	sm = sp->mem;
 	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
+	pp = sm->pool;
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 
 	if (reason != NULL)
 		SES_Close(sp, reason);
@@ -314,28 +312,21 @@ SES_Delete(struct sess *sp, const char *reason)
 	    sp->addr, sp->port, sp->t_end - b->first,
 	    b->sess, b->req, b->pipe, b->pass,
 	    b->fetch, b->hdrbytes, b->bodybytes);
+
 	if (sm->workspace != params->sess_workspace) {
 		Lck_Lock(&stat_mtx);
 		VSC_C_main->n_sess_mem--;
 		Lck_Unlock(&stat_mtx);
 		free(sm);
+		Lck_Lock(&pp->mtx);
+		sesspool->nsess--;
+		Lck_Unlock(&pp->mtx);
 	} else {
 		/* Clean and prepare for reuse */
 		ses_setup(sm);
-		Lck_Lock(&ses_mem_mtx);
-		VTAILQ_INSERT_HEAD(&ses_free_mem[1 - ses_qp], sm, list);
-		Lck_Unlock(&ses_mem_mtx);
-	}
-
-	/* Try to precreate some ses-mem so the acceptor will not have to */
-	if (VSC_C_main->n_sess_mem < VSC_C_main->n_sess + 10) {
-		sm = ses_sm_alloc();
-		if (sm != NULL) {
-			ses_setup(sm);
-			Lck_Lock(&ses_mem_mtx);
-			VTAILQ_INSERT_HEAD(&ses_free_mem[1 - ses_qp], sm, list);
-			Lck_Unlock(&ses_mem_mtx);
-		}
+		Lck_Lock(&pp->mtx);
+		VTAILQ_INSERT_HEAD(&sesspool->freelist, sm, list);
+		Lck_Unlock(&pp->mtx);
 	}
 }
 
@@ -345,6 +336,10 @@ void
 SES_Init()
 {
 
+	ALLOC_OBJ(sesspool, SESSPOOL_MAGIC);
+	VTAILQ_INIT(&sesspool->freelist);
+	Lck_New(&sesspool->mtx, lck_sessmem);
+	sesspool->maxsess = params->max_sess;
+
 	Lck_New(&stat_mtx, lck_stat);
-	Lck_New(&ses_mem_mtx, lck_sessmem);
 }



From phk at varnish-cache.org  Sat Sep 17 20:25:33 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Sep 2011 22:25:33 +0200
Subject: [master] ea74343 A little bit of spit and polish...
Message-ID: <E1R51S9-0007Ac-1J@project.varnish-software.com>

commit ea7434334778be8c38052b6090c25abc63bcc37b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 20:25:20 2011 +0000

    A little bit of spit and polish...

diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 6b34928..a930c07 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -319,27 +319,36 @@ SES_Delete(struct sess *sp, const char *reason)
 		Lck_Unlock(&stat_mtx);
 		free(sm);
 		Lck_Lock(&pp->mtx);
-		sesspool->nsess--;
+		pp->nsess--;
 		Lck_Unlock(&pp->mtx);
 	} else {
 		/* Clean and prepare for reuse */
 		ses_setup(sm);
 		Lck_Lock(&pp->mtx);
-		VTAILQ_INSERT_HEAD(&sesspool->freelist, sm, list);
+		VTAILQ_INSERT_HEAD(&pp->freelist, sm, list);
 		Lck_Unlock(&pp->mtx);
 	}
 }
 
 /*--------------------------------------------------------------------*/
 
+static struct sesspool *
+SES_NewPool(unsigned maxsess)
+{
+	struct sesspool *sp;
+
+	ALLOC_OBJ(sp, SESSPOOL_MAGIC);
+	VTAILQ_INIT(&sp->freelist);
+	Lck_New(&sp->mtx, lck_sessmem);
+	sp->maxsess = maxsess;
+	return (sp);
+}
+
+
 void
 SES_Init()
 {
 
-	ALLOC_OBJ(sesspool, SESSPOOL_MAGIC);
-	VTAILQ_INIT(&sesspool->freelist);
-	Lck_New(&sesspool->mtx, lck_sessmem);
-	sesspool->maxsess = params->max_sess;
-
+	sesspool = SES_NewPool(params->max_sess);
 	Lck_New(&stat_mtx, lck_stat);
 }



From phk at varnish-cache.org  Sun Sep 18 07:24:37 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 18 Sep 2011 09:24:37 +0200
Subject: [master] 12bf085 Get rid of the single acceptor thread.
Message-ID: <E1R5Bjx-0007be-9J@project.varnish-software.com>

commit 12bf085b48f787b4744be61197136cd6ee983b5d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 07:19:44 2011 +0000

    Get rid of the single acceptor thread.
    
    Instead of a single thread which accepts all sockets with a poll/accept
    sequence, each thread-pool will have a thread accepting on each socket.
    
    If no threads are available, the sockets will not be accepted on.
    
    CAVEATS:
    
    This commit undoubtedly leaves a number of minor issues dangling,
    they will be cleaned up as we find them.
    
    Please notice that there are changes to stats counters (some don't
    work right now a nd more changes will be coming)
    
    Changing certain acceptor-related params on the fly may not work.
    
    Testing would be very welcome.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 71cf6b3..49ca3a4 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -305,7 +305,9 @@ struct worker {
 
 	/* Accept stuff */
 	struct sockaddr_storage	acceptaddr;
+	socklen_t		acceptaddrlen;
 	int			acceptsock;
+	struct listen_sock	*acceptlsock;
 
 	struct wrw		wrw;
 
@@ -641,7 +643,7 @@ void VCA_Prep(struct sess *sp);
 void VCA_Init(void);
 void VCA_Shutdown(void);
 int VCA_Accept(int sock, socklen_t *slp, struct sockaddr_storage *sap);
-extern pthread_t VCA_thread;
+void VCA_SetupSess(struct worker *w);
 
 /* cache_backend.c */
 void VBE_UseHealth(const struct director *vdi);
@@ -849,12 +851,13 @@ void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 
 
 /* cache_session.c [SES] */
-void SES_Init(void);
-struct sess *SES_New(struct sesspool *pp);
+struct sess *SES_New(struct worker *wrk, struct sesspool *pp);
 struct sess *SES_Alloc(void);
 void SES_Close(struct sess *sp, const char *reason);
 void SES_Delete(struct sess *sp, const char *reason);
 void SES_Charge(struct sess *sp);
+struct sesspool *SES_NewPool(void);
+
 
 /* cache_shmlog.c */
 void VSL_Init(void);
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 37cc857..b2a8015 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -33,6 +33,7 @@
 #include <errno.h>
 #include <poll.h>
 #include <stdlib.h>
+#include <stdio.h>
 
 #include <sys/types.h>
 #include <sys/socket.h>
@@ -41,7 +42,7 @@
 #include "cli_priv.h"
 #include "cache.h"
 
-pthread_t		VCA_thread;
+static pthread_t	VCA_thread;
 static struct timeval	tv_sndtimeo;
 static struct timeval	tv_rcvtimeo;
 
@@ -178,6 +179,7 @@ vca_pace_check(void)
 static void
 vca_pace_bad(void)
 {
+
 	Lck_Lock(&pace_mtx);
 	vca_pace += params->acceptor_sleep_incr;
 	if (vca_pace > params->acceptor_sleep_max)
@@ -202,13 +204,19 @@ vca_pace_good(void)
  * Accept on a listen socket, and handle error returns.
  */
 
+static int hack_ready;
+
 int
 VCA_Accept(int sock, socklen_t *slp, struct sockaddr_storage *sap)
 {
 	int i;
 
+	assert(sock >= 0);
 	vca_pace_check();
 
+	while(!hack_ready)
+		(void)usleep(100*1000);
+
 	*slp = sizeof *sap;
 	i = accept(sock, (void*)sap, slp);
 
@@ -234,45 +242,67 @@ VCA_Accept(int sock, socklen_t *slp, struct sockaddr_storage *sap)
 
 /*--------------------------------------------------------------------*/
 
+void
+VCA_SetupSess(struct worker *w)
+{
+	struct sess *sp;
+
+	sp = w->sp;
+	if (sp == NULL) {
+		AZ(close(w->acceptsock));
+		w->acceptsock = -1;
+		VSC_C_main->client_drop++;
+		/* XXX: 50x Reply ? */
+		vca_pace_bad();
+		INCOMPL();
+	}
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	sp->fd = w->acceptsock;
+	sp->id = w->acceptsock;
+	w->acceptsock = -1;
+	sp->t_open = TIM_real();
+	sp->t_end = sp->t_end;
+	sp->mylsock = w->acceptlsock;
+	assert(w->acceptaddrlen <= sp->sockaddrlen);
+	memcpy(sp->sockaddr, &w->acceptaddr, w->acceptaddrlen);
+	sp->sockaddrlen = w->acceptaddrlen;
+	sp->step = STP_FIRST;
+	vca_pace_good();
+	w->sp = sp;
+	w->stats.client_conn++;
+}
+
+/*--------------------------------------------------------------------*/
+
 static void *
 vca_acct(void *arg)
 {
-	struct sess *sp;
-	socklen_t l;
-	struct sockaddr_storage addr_s;
-	struct sockaddr *addr;
 #ifdef SO_RCVTIMEO_WORKS
 	double sess_timeout = 0;
 #endif
 #ifdef SO_SNDTIMEO_WORKS
 	double send_timeout = 0;
 #endif
-	int i;
-	struct pollfd *pfd;
 	struct listen_sock *ls;
-	unsigned u;
 	double t0, now;
 
 	THR_SetName("cache-acceptor");
 	(void)arg;
 
-	/* Set up the poll argument */
-	pfd = calloc(sizeof *pfd, heritage.nsocks);
-	AN(pfd);
-	i = 0;
 	VTAILQ_FOREACH(ls, &heritage.socks, list) {
 		if (ls->sock < 0)
 			continue;
 		AZ(listen(ls->sock, params->listen_depth));
 		AZ(setsockopt(ls->sock, SOL_SOCKET, SO_LINGER,
 		    &linger, sizeof linger));
-		pfd[i].events = POLLIN;
-		pfd[i++].fd = ls->sock;
 	}
 
+	hack_ready = 1;
+
 	need_test = 1;
 	t0 = TIM_real();
 	while (1) {
+		(void)sleep(1);
 #ifdef SO_SNDTIMEO_WORKS
 		if (params->send_timeout != send_timeout) {
 			need_test = 1;
@@ -301,45 +331,8 @@ vca_acct(void *arg)
 			}
 		}
 #endif
-		i = poll(pfd, heritage.nsocks, 1000);
 		now = TIM_real();
 		VSC_C_main->uptime = (uint64_t)(now - t0);
-		u = 0;
-		VTAILQ_FOREACH(ls, &heritage.socks, list) {
-			if (ls->sock < 0)
-				continue;
-			if (pfd[u++].revents == 0)
-				continue;
-			VSC_C_main->client_conn++;
-			l = sizeof addr_s;
-			addr = (void*)&addr_s;
-			i = VCA_Accept(ls->sock, &l, &addr_s);
-			if (i < 0) 
-				continue;
-			sp = SES_New(NULL);
-			if (sp == NULL) {
-				AZ(close(i));
-				VSC_C_main->client_drop++;
-				vca_pace_bad();
-				continue;
-			}
-			sp->fd = i;
-			sp->id = i;
-			sp->t_open = now;
-			sp->t_end = now;
-			sp->mylsock = ls;
-			assert(l < sp->sockaddrlen);
-			memcpy(sp->sockaddr, addr, l);
-			sp->sockaddrlen = l;
-
-			sp->step = STP_FIRST;
-			if (Pool_QueueSession(sp)) {
-				VSC_C_main->client_drop++;
-				vca_pace_bad();
-			} else {
-				vca_pace_good();
-			}
-		}
 	}
 	NEEDLESS_RETURN(NULL);
 }
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index b561c84..db51927 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -369,7 +369,6 @@ cnt_done(struct sess *sp)
 
 	if (sp->fd < 0) {
 		sp->wrk->stats.sess_closed++;
-		sp->wrk = NULL;
 		SES_Delete(sp, NULL);
 		return (1);
 	}
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 16e5cf0..8e0fd4e 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -509,7 +509,6 @@ hsh_rush(struct objhead *oh)
 			 * We could not schedule the session, leave the
 			 * rest on the busy list.
 			 */
-			VSC_C_main->client_drop_late++;
 			break;
 		}
 	}
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index 3489530..5660893 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -113,7 +113,6 @@ child_main(void)
 	VCL_Init();
 
 	HTTP_Init();
-	SES_Init();
 
 	VBE_Init();
 	VBP_Init();
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 05ce529..1bfc6b4 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -66,6 +66,7 @@ struct poolsock {
 	unsigned			magic;
 #define POOLSOCK_MAGIC			0x1b0a2d38
 	VTAILQ_ENTRY(poolsock)		list;
+	struct listen_sock		*lsock;
 	int				sock;
 };
 
@@ -83,6 +84,7 @@ struct pool {
 	unsigned		last_lqueue;
 	uintmax_t		ndrop;
 	uintmax_t		nqueue;
+	struct sesspool		*sesspool;
 };
 
 static struct pool		**wq;
@@ -95,11 +97,46 @@ static struct lock		herder_mtx;
 
 /*--------------------------------------------------------------------*/
 
+static void
+pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
+{
+	struct worker *w2;
+
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(ps, POOLSOCK_MAGIC);
+
+	assert(ps->sock >= 0);
+	Lck_AssertHeld(&pp->mtx);
+	Lck_Unlock(&pp->mtx);
+	while (1) {
+		w->acceptsock =
+		    VCA_Accept(ps->sock, &w->acceptaddrlen, &w->acceptaddr);
+		if (w->acceptsock == -1)
+			continue;
+		w->acceptlsock = ps->lsock;
+		Lck_Lock(&pp->mtx);
+		if (VTAILQ_EMPTY(&pp->idle))
+			return;
+		w2 = VTAILQ_FIRST(&pp->idle);
+		VTAILQ_REMOVE(&pp->idle, w2, list);
+		Lck_Unlock(&pp->mtx);
+		w2->acceptaddr = w->acceptaddr;
+		w2->acceptaddrlen = w->acceptaddrlen;
+		w2->acceptsock = w->acceptsock;
+		w2->acceptlsock = w->acceptlsock;
+		AZ(pthread_cond_signal(&w2->cond));
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
 void
 Pool_Work_Thread(void *priv, struct worker *w)
 {
 	struct pool *qp;
 	int stats_clean;
+	struct poolsock *ps;
 
 	CAST_OBJ_NOTNULL(qp, priv, POOL_MAGIC);
 	w->pool = qp;
@@ -107,6 +144,9 @@ Pool_Work_Thread(void *priv, struct worker *w)
 	qp->nthr++;
 	stats_clean = 1;
 	while (1) {
+
+		Lck_AssertHeld(&qp->mtx);
+
 		CHECK_OBJ_NOTNULL(w->bereq, HTTP_MAGIC);
 		CHECK_OBJ_NOTNULL(w->beresp, HTTP_MAGIC);
 		CHECK_OBJ_NOTNULL(w->resp, HTTP_MAGIC);
@@ -117,17 +157,29 @@ Pool_Work_Thread(void *priv, struct worker *w)
 		if (w->sp != NULL) {
 			VTAILQ_REMOVE(&qp->queue, w->sp, poollist);
 			qp->lqueue--;
-		} else {
+		} else if (VTAILQ_EMPTY(&qp->socks)) {
 			if (isnan(w->lastused))
 				w->lastused = TIM_real();
 			VTAILQ_INSERT_HEAD(&qp->idle, w, list);
 			if (!stats_clean)
 				WRK_SumStat(w);
 			Lck_CondWait(&w->cond, &qp->mtx);
+		} else {
+			ps = VTAILQ_FIRST(&qp->socks);
+			VTAILQ_REMOVE(&qp->socks, ps, list);
+			pool_accept(qp, w, ps);
+			Lck_AssertHeld(&qp->mtx);
+			VTAILQ_INSERT_TAIL(&qp->socks, ps, list);
 		}
-		if (w->sp == NULL)
+		if (w->sp == NULL && w->acceptsock == -1)
 			break;
 		Lck_Unlock(&qp->mtx);
+		if (w->sp == NULL) {
+			w->sp = SES_New(w, qp->sesspool);
+			VCA_SetupSess(w);
+		}
+		AN(w->sp);
+		assert(w->acceptsock == -1);
 		stats_clean = 0;
 		w->lastused = NAN;
 		WS_Reset(w->ws, NULL);
@@ -221,7 +273,9 @@ Pool_QueueSession(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk);
 	if (WRK_Queue(sp) == 0)
-		return (0);
+		return(0);
+
+	VSC_C_main->client_drop_late++;
 
 	/*
 	 * Couldn't queue it -- kill it.
@@ -258,11 +312,16 @@ pool_mkpool(void)
 	VTAILQ_INIT(&pp->queue);
 	VTAILQ_INIT(&pp->idle);
 	VTAILQ_INIT(&pp->socks);
+	pp->sesspool = SES_NewPool();
+	AN(pp->sesspool);
 
 	VTAILQ_FOREACH(ls, &heritage.socks, list) {
+		if (ls->sock < 0)
+			continue;
 		ALLOC_OBJ(ps, POOLSOCK_MAGIC);
 		XXXAN(ps);
 		ps->sock = ls->sock;
+		ps->lsock = ls;
 		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
 	}
 	return (pp);
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index a930c07..4c01f34 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -28,6 +28,9 @@
  *
  * Session management
  *
+ * This is a little bit of a mixed back, containing both memory management
+ * and various state-change functions.
+ *
  */
 
 #include "config.h"
@@ -39,7 +42,6 @@
 #include <sys/socket.h>
 
 #include "cache.h"
-#include "cache_backend.h"
 #include "cache_waiter.h"
 
 /*--------------------------------------------------------------------*/
@@ -63,16 +65,12 @@ struct sesspool {
 	VTAILQ_HEAD(,sessmem)	freelist;
 	struct lock		mtx;
 	unsigned		nsess;
-	unsigned		maxsess;
+	unsigned		dly_free_cnt;
 };
 
-static struct sesspool *sesspool;
-
-/*--------------------------------------------------------------------*/
-
-static struct lock		stat_mtx;
-
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Charge statistics from worker to request and session.
+ */
 
 void
 SES_Charge(struct sess *sp)
@@ -112,16 +110,12 @@ ses_sm_alloc(void)
 
 	hl = HTTP_estimate(nhttp);
 	l = sizeof *sm + nws + 2 * hl;
+	VSC_C_main->g_sessmem_size = l;
 	p = malloc(l);
 	if (p == NULL)
 		return (NULL);
 	q = p + l;
 
-	/* XXX Stats */
-	Lck_Lock(&stat_mtx);
-	VSC_C_main->n_sess_mem++;
-	Lck_Unlock(&stat_mtx);
-
 	/* Don't waste time zeroing the workspace */
 	memset(p, 0, l - nws);
 
@@ -179,38 +173,42 @@ ses_setup(struct sessmem *sm)
  */
 
 struct sess *
-SES_New(struct sesspool *pp)
+SES_New(struct worker *wrk, struct sesspool *pp)
 {
 	struct sessmem *sm;
 	struct sess *sp;
-	int do_alloc = 0;
+	int do_alloc;
 
-	if (pp == NULL)
-		pp = sesspool;
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 
+	do_alloc = 0;
 	Lck_Lock(&pp->mtx);
 	sm = VTAILQ_FIRST(&pp->freelist);
 	if (sm != NULL) {
 		VTAILQ_REMOVE(&pp->freelist, sm, list);
-	} else if (pp->nsess < pp->maxsess) {
+	} else if (pp->nsess < params->max_sess) {
 		pp->nsess++;
 		do_alloc = 1;
 	}
+	wrk->stats.c_sessmem_free += pp->dly_free_cnt;
+	pp->dly_free_cnt = 0;
 	Lck_Unlock(&pp->mtx);
 	if (do_alloc) {
 		sm = ses_sm_alloc();
 		if (sm != NULL) {
+			wrk->stats.c_sessmem_alloc++;
 			sm->pool = pp;
 			ses_setup(sm);
+		} else {
+			wrk->stats.c_sessmem_fail++;
 		}
+	} else if (sm == NULL) {
+		wrk->stats.c_sessmem_limit++;
 	}
 	if (sm == NULL)
 		return (NULL);
 	sp = &sm->sess;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	/* XXX Stats */
-	VSC_C_main->n_sess++;		/* XXX: locking  ? */
 	return (sp);
 }
 
@@ -251,8 +249,7 @@ SES_Handle(struct sess *sp, int status)
 		break;
 	case 1:
 		sp->step = STP_START;
-		if (Pool_QueueSession(sp))
-			VSC_C_main->client_drop_late++;
+		(void)Pool_QueueSession(sp);
 		break;
 	default:
 		WRONG("Unexpected return from HTC_Rx()");
@@ -276,16 +273,21 @@ SES_Close(struct sess *sp, const char *reason)
 }
 
 /*--------------------------------------------------------------------
- * (Close &) Recycle a session.  If the workspace has changed, deleted it,
- * otherwise wash it, and put it up for adoption.
+ * (Close &) Free or Recycle a session.
+ *
+ * If the workspace has changed, deleted it, otherwise wash it, and put
+ * it up for adoption.
+ *
+ * XXX: We should also check nhttp
  */
 
 void
 SES_Delete(struct sess *sp, const char *reason)
 {
-	struct acct *b = &sp->acct_ses;
+	struct acct *b;
 	struct sessmem *sm;
 	static char noaddr[] = "-";
+	struct worker *wrk;
 	struct sesspool *pp;
 
 
@@ -294,6 +296,9 @@ SES_Delete(struct sess *sp, const char *reason)
 	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
 	pp = sm->pool;
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC);
+
 
 	if (reason != NULL)
 		SES_Close(sp, reason);
@@ -301,54 +306,55 @@ SES_Delete(struct sess *sp, const char *reason)
 
 	AZ(sp->obj);
 	AZ(sp->vcl);
-	VSC_C_main->n_sess--;			/* XXX: locking ? */
-	assert(!isnan(b->first));
-	assert(!isnan(sp->t_end));
 	if (sp->addr == NULL)
 		sp->addr = noaddr;
 	if (sp->port == NULL)
 		sp->port = noaddr;
+
+	b = &sp->acct_ses;
+	assert(!isnan(b->first));
+	assert(!isnan(sp->t_end));
+
 	VSL(SLT_StatSess, sp->id, "%s %s %.0f %ju %ju %ju %ju %ju %ju %ju",
 	    sp->addr, sp->port, sp->t_end - b->first,
 	    b->sess, b->req, b->pipe, b->pass,
 	    b->fetch, b->hdrbytes, b->bodybytes);
 
-	if (sm->workspace != params->sess_workspace) {
-		Lck_Lock(&stat_mtx);
-		VSC_C_main->n_sess_mem--;
-		Lck_Unlock(&stat_mtx);
+	if (sm->workspace != params->sess_workspace ||
+	    pp->nsess > params->max_sess) {
 		free(sm);
 		Lck_Lock(&pp->mtx);
+		if (wrk != NULL)
+			wrk->stats.c_sessmem_free++;
+		else
+			pp->dly_free_cnt++;
 		pp->nsess--;
 		Lck_Unlock(&pp->mtx);
 	} else {
 		/* Clean and prepare for reuse */
 		ses_setup(sm);
 		Lck_Lock(&pp->mtx);
+		if (wrk != NULL) {
+			wrk->stats.c_sessmem_free += pp->dly_free_cnt;
+			pp->dly_free_cnt = 0;
+		}
 		VTAILQ_INSERT_HEAD(&pp->freelist, sm, list);
 		Lck_Unlock(&pp->mtx);
 	}
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Create a new pool to allocate from
+ */
 
-static struct sesspool *
-SES_NewPool(unsigned maxsess)
+struct sesspool *
+SES_NewPool(void)
 {
 	struct sesspool *sp;
 
 	ALLOC_OBJ(sp, SESSPOOL_MAGIC);
+	AN(sp);
 	VTAILQ_INIT(&sp->freelist);
 	Lck_New(&sp->mtx, lck_sessmem);
-	sp->maxsess = maxsess;
 	return (sp);
 }
-
-
-void
-SES_Init()
-{
-
-	sesspool = SES_NewPool(params->max_sess);
-	Lck_New(&stat_mtx, lck_stat);
-}
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 7b731ba..631342f 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -165,6 +165,7 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
 	w->bereq = HTTP_create(http0, nhttp);
 	w->beresp = HTTP_create(http1, nhttp);
 	w->resp = HTTP_create(http2, nhttp);
+	w->acceptsock = -1;
 	w->wrw.iov = iov;
 	w->wrw.siov = siov;
 	w->wrw.ciov = siov;
diff --git a/bin/varnishd/locks.h b/bin/varnishd/locks.h
index 6cbf91f..f1b634b 100644
--- a/bin/varnishd/locks.h
+++ b/bin/varnishd/locks.h
@@ -36,7 +36,6 @@ LOCK(hsl)
 LOCK(hcb)
 LOCK(hcl)
 LOCK(vcl)
-LOCK(stat)
 LOCK(sessmem)
 LOCK(wstat)
 LOCK(herder)
diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index 9d6966a..4181a43 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -241,7 +241,6 @@ open_sockets(void)
 		 * closes before we call accept(2) and nobody else are in
 		 * the listen queue to release us.
 		 */
-		(void)VTCP_nonblocking(ls->sock);
 		(void)VTCP_filter_http(ls->sock);
 		good++;
 	}
diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index c35acec..e337321 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -41,7 +41,7 @@
 
 #ifdef VSC_DO_MAIN
 
-VSC_F(client_conn,		uint64_t, 0, 'a', "Client connections accepted")
+VSC_F(client_conn,		uint64_t, 1, 'a', "Client connections accepted")
 VSC_F(client_drop,		uint64_t, 0, 'a',
 					"Connection dropped, no sess/wrk")
 VSC_F(client_req,		uint64_t, 1, 'a', "Client requests received")
@@ -72,6 +72,12 @@ VSC_F(fetch_1xx,		uint64_t, 1, 'a', "Fetch no body (1xx)")
 VSC_F(fetch_204,		uint64_t, 1, 'a', "Fetch no body (204)")
 VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)")
 
+/* Sessmem cache_session.c */
+VSC_F(g_sessmem_size,		uint64_t, 1, 'i', "Session mem size")
+VSC_F(c_sessmem_alloc,		uint64_t, 1, 'a', "Session mem allocated")
+VSC_F(c_sessmem_free,		uint64_t, 1, 'a', "Session mem freed")
+VSC_F(c_sessmem_fail,		uint64_t, 1, 'a', "Session mem alloc failed")
+VSC_F(c_sessmem_limit,		uint64_t, 1, 'a', "Session mem alloc limited")
 
 VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem")
 VSC_F(n_sess,			uint64_t, 0, 'i', "N struct sess")



From phk at varnish-cache.org  Sun Sep 18 07:40:34 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 18 Sep 2011 09:40:34 +0200
Subject: [master] eafb66a Add a "long description field" to the VSC_F() macro,
	the intent is that they should be mined and dumped into the
	documentation.
Message-ID: <E1R5BzO-0000JJ-AP@project.varnish-software.com>

commit eafb66a243531c88b82ecae1b5ee0c34d1d95e16
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 07:39:30 2011 +0000

    Add a "long description field" to the VSC_F() macro, the intent is
    that they should be mined and dumped into the documentation.
    
    Feel free to add (or send patches with descriptions, preferably
    only one line or paragraph for each.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 49ca3a4..5db137a 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -220,7 +220,7 @@ struct acct {
 
 #define L0(n)
 #define L1(n)			uint64_t n;
-#define VSC_F(n, t, l, f, e)	L##l(n)
+#define VSC_F(n, t, l, f, e,d)	L##l(n)
 #define VSC_DO_MAIN
 struct dstat {
 #include "vsc_fields.h"
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 631342f..3cfaba0 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -58,7 +58,7 @@ wrk_sumstat(struct worker *w)
 #define L0(n)
 #define L1(n) (VSC_C_main->n += w->stats.n)
 #define VSC_DO_MAIN
-#define VSC_F(n, t, l, f, d) L##l(n);
+#define VSC_F(n, t, l, f, d, e) L##l(n);
 #include "vsc_fields.h"
 #undef VSC_F
 #undef VSC_DO_MAIN
diff --git a/include/vsc.h b/include/vsc.h
index 1256dae..7cb3235 100644
--- a/include/vsc.h
+++ b/include/vsc.h
@@ -38,7 +38,7 @@
 #define VSC_TYPE_VBE	"VBE"
 #define VSC_TYPE_LCK	"LCK"
 
-#define VSC_F(n, t, l, f, e)	t n;
+#define VSC_F(n, t, l, f, e, d)	t n;
 
 #define VSC_DO(u,l,t) struct VSC_C_##l {
 #define VSC_DONE(u,l,t) };
diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index e337321..79ad315 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -20,7 +20,7 @@
  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION, "")
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
@@ -33,140 +33,165 @@
  * XXX: turned into a major mess, causing trouble already for backends.
  * XXX:
  * XXX: Please converge on:
- * XXX:		c_* counter	(total bytes ever allocated from sma)
- * XXX:		g_* gauge	(presently allocated bytes from sma)
+ * XXX:		c_* counter	(total bytes ever allocated from sma, "")
+ * XXX:		g_* gauge	(presently allocated bytes from sma, "")
  */
 
 /**********************************************************************/
 
 #ifdef VSC_DO_MAIN
 
-VSC_F(client_conn,		uint64_t, 1, 'a', "Client connections accepted")
+VSC_F(client_conn,		uint64_t, 1, 'a', "Client connections accepted", "")
 VSC_F(client_drop,		uint64_t, 0, 'a',
-					"Connection dropped, no sess/wrk")
-VSC_F(client_req,		uint64_t, 1, 'a', "Client requests received")
-
-VSC_F(cache_hit,		uint64_t, 1, 'a', "Cache hits")
-VSC_F(cache_hitpass,	uint64_t, 1, 'a', "Cache hits for pass")
-VSC_F(cache_miss,		uint64_t, 1, 'a', "Cache misses")
-
-VSC_F(backend_conn,	uint64_t, 0, 'a', "Backend conn. success")
-VSC_F(backend_unhealthy,	uint64_t, 0, 'a', "Backend conn. not attempted")
-VSC_F(backend_busy,	uint64_t, 0, 'a', "Backend conn. too many")
-VSC_F(backend_fail,	uint64_t, 0, 'a', "Backend conn. failures")
-VSC_F(backend_reuse,	uint64_t, 0, 'a', "Backend conn. reuses")
-VSC_F(backend_toolate,	uint64_t, 0, 'a', "Backend conn. was closed")
-VSC_F(backend_recycle,	uint64_t, 0, 'a', "Backend conn. recycles")
-VSC_F(backend_retry,	uint64_t, 0, 'a', "Backend conn. retry")
-
-VSC_F(fetch_head,		uint64_t, 1, 'a', "Fetch head")
-VSC_F(fetch_length,		uint64_t, 1, 'a', "Fetch with Length")
-VSC_F(fetch_chunked,		uint64_t, 1, 'a', "Fetch chunked")
-VSC_F(fetch_eof,		uint64_t, 1, 'a', "Fetch EOF")
-VSC_F(fetch_bad,		uint64_t, 1, 'a', "Fetch had bad headers")
-VSC_F(fetch_close,		uint64_t, 1, 'a', "Fetch wanted close")
-VSC_F(fetch_oldhttp,		uint64_t, 1, 'a', "Fetch pre HTTP/1.1 closed")
-VSC_F(fetch_zero,		uint64_t, 1, 'a', "Fetch zero len")
-VSC_F(fetch_failed,		uint64_t, 1, 'a', "Fetch failed")
-VSC_F(fetch_1xx,		uint64_t, 1, 'a', "Fetch no body (1xx)")
-VSC_F(fetch_204,		uint64_t, 1, 'a', "Fetch no body (204)")
-VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)")
-
-/* Sessmem cache_session.c */
-VSC_F(g_sessmem_size,		uint64_t, 1, 'i', "Session mem size")
-VSC_F(c_sessmem_alloc,		uint64_t, 1, 'a', "Session mem allocated")
-VSC_F(c_sessmem_free,		uint64_t, 1, 'a', "Session mem freed")
-VSC_F(c_sessmem_fail,		uint64_t, 1, 'a', "Session mem alloc failed")
-VSC_F(c_sessmem_limit,		uint64_t, 1, 'a', "Session mem alloc limited")
-
-VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem")
-VSC_F(n_sess,			uint64_t, 0, 'i', "N struct sess")
-VSC_F(n_object,			uint64_t, 1, 'i', "N struct object")
-VSC_F(n_vampireobject,		uint64_t, 1, 'i', "N unresurrected objects")
-VSC_F(n_objectcore,		uint64_t, 1, 'i', "N struct objectcore")
-VSC_F(n_objecthead,		uint64_t, 1, 'i', "N struct objecthead")
-VSC_F(n_waitinglist,		uint64_t, 1, 'i', "N struct waitinglist")
-
-VSC_F(n_vbc,		uint64_t, 0, 'i', "N struct vbc")
-VSC_F(n_wrk,		uint64_t, 0, 'i', "N worker threads")
-VSC_F(n_wrk_create,	uint64_t, 0, 'a', "N worker threads created")
+					"Connection dropped, no sess/wrk", "")
+VSC_F(client_req,		uint64_t, 1, 'a', "Client requests received", "")
+
+VSC_F(cache_hit,		uint64_t, 1, 'a', "Cache hits", "")
+VSC_F(cache_hitpass,	uint64_t, 1, 'a', "Cache hits for pass", "")
+VSC_F(cache_miss,		uint64_t, 1, 'a', "Cache misses", "")
+
+VSC_F(backend_conn,	uint64_t, 0, 'a', "Backend conn. success", "")
+VSC_F(backend_unhealthy,	uint64_t, 0, 'a', "Backend conn. not attempted", "")
+VSC_F(backend_busy,	uint64_t, 0, 'a', "Backend conn. too many", "")
+VSC_F(backend_fail,	uint64_t, 0, 'a', "Backend conn. failures", "")
+VSC_F(backend_reuse,	uint64_t, 0, 'a', "Backend conn. reuses", "")
+VSC_F(backend_toolate,	uint64_t, 0, 'a', "Backend conn. was closed", "")
+VSC_F(backend_recycle,	uint64_t, 0, 'a', "Backend conn. recycles", "")
+VSC_F(backend_retry,	uint64_t, 0, 'a', "Backend conn. retry", "")
+
+VSC_F(fetch_head,		uint64_t, 1, 'a', "Fetch head", "")
+VSC_F(fetch_length,		uint64_t, 1, 'a', "Fetch with Length", "")
+VSC_F(fetch_chunked,		uint64_t, 1, 'a', "Fetch chunked", "")
+VSC_F(fetch_eof,		uint64_t, 1, 'a', "Fetch EOF", "")
+VSC_F(fetch_bad,		uint64_t, 1, 'a', "Fetch had bad headers", "")
+VSC_F(fetch_close,		uint64_t, 1, 'a', "Fetch wanted close", "")
+VSC_F(fetch_oldhttp,		uint64_t, 1, 'a', "Fetch pre HTTP/1.1 closed", "")
+VSC_F(fetch_zero,		uint64_t, 1, 'a', "Fetch zero len", "")
+VSC_F(fetch_failed,		uint64_t, 1, 'a', "Fetch failed", "")
+VSC_F(fetch_1xx,		uint64_t, 1, 'a', "Fetch no body (1xx)", "")
+VSC_F(fetch_204,		uint64_t, 1, 'a', "Fetch no body (204)", "")
+VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)", "")
+
+/*---------------------------------------------------------------------
+ * Session Memory
+ *    see: cache_session.c
+ */
+
+VSC_F(g_sessmem_size,		uint64_t, 1, 'i',
+    "Session mem size",
+	"Bytes of memory allocated for last allocated session."
+)
+
+VSC_F(c_sessmem_alloc,		uint64_t, 1, 'a',
+    "Session mem allocated",
+	"Count of all allocations of session memory."
+)
+
+VSC_F(c_sessmem_free,		uint64_t, 1, 'a',
+    "Session mem freed",
+	"Count of all frees of session memory."
+)
+
+VSC_F(c_sessmem_fail,		uint64_t, 1, 'a',
+    "Session mem alloc failed",
+	"Count of session memory allocation failures."
+)
+
+VSC_F(c_sessmem_limit,		uint64_t, 1, 'a',
+    "Session mem alloc limited",
+	"Count of session memory allocations blocked by limit (max_sess)."
+)
+
+/*---------------------------------------------------------------------*/
+
+VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem", "")
+VSC_F(n_sess,			uint64_t, 0, 'i', "N struct sess", "")
+VSC_F(n_object,			uint64_t, 1, 'i', "N struct object", "")
+VSC_F(n_vampireobject,		uint64_t, 1, 'i', "N unresurrected objects", "")
+VSC_F(n_objectcore,		uint64_t, 1, 'i', "N struct objectcore", "")
+VSC_F(n_objecthead,		uint64_t, 1, 'i', "N struct objecthead", "")
+VSC_F(n_waitinglist,		uint64_t, 1, 'i', "N struct waitinglist", "")
+
+VSC_F(n_vbc,		uint64_t, 0, 'i', "N struct vbc", "")
+VSC_F(n_wrk,		uint64_t, 0, 'i', "N worker threads", "")
+VSC_F(n_wrk_create,	uint64_t, 0, 'a', "N worker threads created", "")
 VSC_F(n_wrk_failed,	uint64_t, 0, 'a',
-					"N worker threads not created")
-VSC_F(n_wrk_max,		uint64_t, 0, 'a', "N worker threads limited")
-VSC_F(n_wrk_lqueue,		uint64_t, 0, 'a', "work request queue length")
-VSC_F(n_wrk_queued,		uint64_t, 0, 'a', "N queued work requests")
-VSC_F(n_wrk_drop,		uint64_t, 0, 'a', "N dropped work requests")
-VSC_F(n_backend,		uint64_t, 0, 'i', "N backends")
+					"N worker threads not created", "")
+VSC_F(n_wrk_max,		uint64_t, 0, 'a', "N worker threads limited", "")
+VSC_F(n_wrk_lqueue,		uint64_t, 0, 'a', "work request queue length", "")
+VSC_F(n_wrk_queued,		uint64_t, 0, 'a', "N queued work requests", "")
+VSC_F(n_wrk_drop,		uint64_t, 0, 'a', "N dropped work requests", "")
+VSC_F(n_backend,		uint64_t, 0, 'i', "N backends", "")
 
-VSC_F(n_expired,		uint64_t, 0, 'i', "N expired objects")
-VSC_F(n_lru_nuked,		uint64_t, 0, 'i', "N LRU nuked objects")
-VSC_F(n_lru_moved,		uint64_t, 0, 'i', "N LRU moved objects")
+VSC_F(n_expired,		uint64_t, 0, 'i', "N expired objects", "")
+VSC_F(n_lru_nuked,		uint64_t, 0, 'i', "N LRU nuked objects", "")
+VSC_F(n_lru_moved,		uint64_t, 0, 'i', "N LRU moved objects", "")
 
-VSC_F(losthdr,		uint64_t, 0, 'a', "HTTP header overflows")
+VSC_F(losthdr,		uint64_t, 0, 'a', "HTTP header overflows", "")
 
-VSC_F(n_objsendfile,	uint64_t, 0, 'a', "Objects sent with sendfile")
-VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write")
+VSC_F(n_objsendfile,	uint64_t, 0, 'a', "Objects sent with sendfile", "")
+VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write", "")
 VSC_F(n_objoverflow,	uint64_t, 1, 'a',
-					"Objects overflowing workspace")
-
-VSC_F(s_sess,		uint64_t, 1, 'a', "Total Sessions")
-VSC_F(s_req,		uint64_t, 1, 'a', "Total Requests")
-VSC_F(s_pipe,		uint64_t, 1, 'a', "Total pipe")
-VSC_F(s_pass,		uint64_t, 1, 'a', "Total pass")
-VSC_F(s_fetch,		uint64_t, 1, 'a', "Total fetch")
-VSC_F(s_hdrbytes,		uint64_t, 1, 'a', "Total header bytes")
-VSC_F(s_bodybytes,		uint64_t, 1, 'a', "Total body bytes")
-
-VSC_F(sess_closed,		uint64_t, 1, 'a', "Session Closed")
-VSC_F(sess_pipeline,	uint64_t, 1, 'a', "Session Pipeline")
-VSC_F(sess_readahead,	uint64_t, 1, 'a', "Session Read Ahead")
-VSC_F(sess_linger,		uint64_t, 1, 'a', "Session Linger")
-VSC_F(sess_herd,		uint64_t, 1, 'a', "Session herd")
-
-VSC_F(shm_records,		uint64_t, 0, 'a', "SHM records")
-VSC_F(shm_writes,		uint64_t, 0, 'a', "SHM writes")
-VSC_F(shm_flushes,		uint64_t, 0, 'a', "SHM flushes due to overflow")
-VSC_F(shm_cont,		uint64_t, 0, 'a', "SHM MTX contention")
-VSC_F(shm_cycles,		uint64_t, 0, 'a', "SHM cycles through buffer")
-
-VSC_F(sms_nreq,		uint64_t, 0, 'a', "SMS allocator requests")
-VSC_F(sms_nobj,		uint64_t, 0, 'i', "SMS outstanding allocations")
-VSC_F(sms_nbytes,		uint64_t, 0, 'i', "SMS outstanding bytes")
-VSC_F(sms_balloc,		uint64_t, 0, 'i', "SMS bytes allocated")
-VSC_F(sms_bfree,		uint64_t, 0, 'i', "SMS bytes freed")
-
-VSC_F(backend_req,		uint64_t, 0, 'a', "Backend requests made")
-
-VSC_F(n_vcl,		uint64_t, 0, 'a', "N vcl total")
-VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available")
-VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded")
-
-VSC_F(n_ban,		uint64_t, 0, 'i', "N total active bans")
-VSC_F(n_ban_add,		uint64_t, 0, 'a', "N new bans added")
-VSC_F(n_ban_retire,	uint64_t, 0, 'a', "N old bans deleted")
-VSC_F(n_ban_obj_test,	uint64_t, 0, 'a', "N objects tested")
-VSC_F(n_ban_re_test,	uint64_t, 0, 'a', "N regexps tested against")
-VSC_F(n_ban_dups,	uint64_t, 0, 'a', "N duplicate bans removed")
-
-VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock")
-VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock")
-VSC_F(hcb_insert,		uint64_t, 0, 'a', "HCB Inserts")
-
-VSC_F(esi_errors,		uint64_t, 0, 'a', "ESI parse errors (unlock)")
-VSC_F(esi_warnings,		uint64_t, 0, 'a', "ESI parse warnings (unlock)")
-VSC_F(accept_fail,		uint64_t, 0, 'a', "Accept failures")
-VSC_F(client_drop_late,	uint64_t, 0, 'a', "Connection dropped late")
-VSC_F(uptime,		uint64_t, 0, 'a', "Client uptime")
-
-VSC_F(dir_dns_lookups,	uint64_t, 0, 'a', "DNS director lookups")
-VSC_F(dir_dns_failed,	uint64_t, 0, 'a', "DNS director failed lookups")
-VSC_F(dir_dns_hit,		uint64_t, 0, 'a', "DNS director cached lookups hit")
-VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a', "DNS director full dnscache")
-
-VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs")
-
-VSC_F(n_gzip,			uint64_t, 0, 'a', "Gzip operations")
-VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations")
+					"Objects overflowing workspace", "")
+
+VSC_F(s_sess,		uint64_t, 1, 'a', "Total Sessions", "")
+VSC_F(s_req,		uint64_t, 1, 'a', "Total Requests", "")
+VSC_F(s_pipe,		uint64_t, 1, 'a', "Total pipe", "")
+VSC_F(s_pass,		uint64_t, 1, 'a', "Total pass", "")
+VSC_F(s_fetch,		uint64_t, 1, 'a', "Total fetch", "")
+VSC_F(s_hdrbytes,		uint64_t, 1, 'a', "Total header bytes", "")
+VSC_F(s_bodybytes,		uint64_t, 1, 'a', "Total body bytes", "")
+
+VSC_F(sess_closed,		uint64_t, 1, 'a', "Session Closed", "")
+VSC_F(sess_pipeline,	uint64_t, 1, 'a', "Session Pipeline", "")
+VSC_F(sess_readahead,	uint64_t, 1, 'a', "Session Read Ahead", "")
+VSC_F(sess_linger,		uint64_t, 1, 'a', "Session Linger", "")
+VSC_F(sess_herd,		uint64_t, 1, 'a', "Session herd", "")
+
+VSC_F(shm_records,		uint64_t, 0, 'a', "SHM records", "")
+VSC_F(shm_writes,		uint64_t, 0, 'a', "SHM writes", "")
+VSC_F(shm_flushes,		uint64_t, 0, 'a', "SHM flushes due to overflow", "")
+VSC_F(shm_cont,		uint64_t, 0, 'a', "SHM MTX contention", "")
+VSC_F(shm_cycles,		uint64_t, 0, 'a', "SHM cycles through buffer", "")
+
+VSC_F(sms_nreq,		uint64_t, 0, 'a', "SMS allocator requests", "")
+VSC_F(sms_nobj,		uint64_t, 0, 'i', "SMS outstanding allocations", "")
+VSC_F(sms_nbytes,		uint64_t, 0, 'i', "SMS outstanding bytes", "")
+VSC_F(sms_balloc,		uint64_t, 0, 'i', "SMS bytes allocated", "")
+VSC_F(sms_bfree,		uint64_t, 0, 'i', "SMS bytes freed", "")
+
+VSC_F(backend_req,		uint64_t, 0, 'a', "Backend requests made", "")
+
+VSC_F(n_vcl,		uint64_t, 0, 'a', "N vcl total", "")
+VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available", "")
+VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded", "")
+
+VSC_F(n_ban,		uint64_t, 0, 'i', "N total active bans", "")
+VSC_F(n_ban_add,		uint64_t, 0, 'a', "N new bans added", "")
+VSC_F(n_ban_retire,	uint64_t, 0, 'a', "N old bans deleted", "")
+VSC_F(n_ban_obj_test,	uint64_t, 0, 'a', "N objects tested", "")
+VSC_F(n_ban_re_test,	uint64_t, 0, 'a', "N regexps tested against", "")
+VSC_F(n_ban_dups,	uint64_t, 0, 'a', "N duplicate bans removed", "")
+
+VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock", "")
+VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock", "")
+VSC_F(hcb_insert,		uint64_t, 0, 'a', "HCB Inserts", "")
+
+VSC_F(esi_errors,		uint64_t, 0, 'a', "ESI parse errors (unlock)", "")
+VSC_F(esi_warnings,		uint64_t, 0, 'a', "ESI parse warnings (unlock)", "")
+VSC_F(accept_fail,		uint64_t, 0, 'a', "Accept failures", "")
+VSC_F(client_drop_late,	uint64_t, 0, 'a', "Connection dropped late", "")
+VSC_F(uptime,		uint64_t, 0, 'a', "Client uptime", "")
+
+VSC_F(dir_dns_lookups,	uint64_t, 0, 'a', "DNS director lookups", "")
+VSC_F(dir_dns_failed,	uint64_t, 0, 'a', "DNS director failed lookups", "")
+VSC_F(dir_dns_hit,		uint64_t, 0, 'a', "DNS director cached lookups hit", "")
+VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a', "DNS director full dnscache", "")
+
+VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs", "")
+
+VSC_F(n_gzip,			uint64_t, 0, 'a', "Gzip operations", "")
+VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations", "")
 
 #endif
 
@@ -174,10 +199,10 @@ VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations")
 
 #ifdef VSC_DO_LCK
 
-VSC_F(creat,		uint64_t, 0, 'a', "Created locks")
-VSC_F(destroy,		uint64_t, 0, 'a', "Destroyed locks")
-VSC_F(locks,		uint64_t, 0, 'a', "Lock Operations")
-VSC_F(colls,		uint64_t, 0, 'a', "Collisions")
+VSC_F(creat,		uint64_t, 0, 'a', "Created locks", "")
+VSC_F(destroy,		uint64_t, 0, 'a', "Destroyed locks", "")
+VSC_F(locks,		uint64_t, 0, 'a', "Lock Operations", "")
+VSC_F(colls,		uint64_t, 0, 'a', "Collisions", "")
 
 #endif
 
@@ -186,13 +211,13 @@ VSC_F(colls,		uint64_t, 0, 'a', "Collisions")
  */
 
 #if defined(VSC_DO_SMA) || defined (VSC_DO_SMF)
-VSC_F(c_req,		uint64_t, 0, 'a', "Allocator requests")
-VSC_F(c_fail,		uint64_t, 0, 'a', "Allocator failures")
-VSC_F(c_bytes,		uint64_t, 0, 'a', "Bytes allocated")
-VSC_F(c_freed,		uint64_t, 0, 'a', "Bytes freed")
-VSC_F(g_alloc,		uint64_t, 0, 'i', "Allocations outstanding")
-VSC_F(g_bytes,		uint64_t, 0, 'i', "Bytes outstanding")
-VSC_F(g_space,		uint64_t, 0, 'i', "Bytes available")
+VSC_F(c_req,		uint64_t, 0, 'a', "Allocator requests", "")
+VSC_F(c_fail,		uint64_t, 0, 'a', "Allocator failures", "")
+VSC_F(c_bytes,		uint64_t, 0, 'a', "Bytes allocated", "")
+VSC_F(c_freed,		uint64_t, 0, 'a', "Bytes freed", "")
+VSC_F(g_alloc,		uint64_t, 0, 'i', "Allocations outstanding", "")
+VSC_F(g_bytes,		uint64_t, 0, 'i', "Bytes outstanding", "")
+VSC_F(g_space,		uint64_t, 0, 'i', "Bytes available", "")
 #endif
 
 
@@ -205,17 +230,17 @@ VSC_F(g_space,		uint64_t, 0, 'i', "Bytes available")
 /**********************************************************************/
 
 #ifdef VSC_DO_SMF
-VSC_F(g_smf,			uint64_t, 0, 'i', "N struct smf")
-VSC_F(g_smf_frag,		uint64_t, 0, 'i', "N small free smf")
-VSC_F(g_smf_large,		uint64_t, 0, 'i', "N large free smf")
+VSC_F(g_smf,			uint64_t, 0, 'i', "N struct smf", "")
+VSC_F(g_smf_frag,		uint64_t, 0, 'i', "N small free smf", "")
+VSC_F(g_smf_large,		uint64_t, 0, 'i', "N large free smf", "")
 #endif
 
 /**********************************************************************/
 
 #ifdef VSC_DO_VBE
 
-VSC_F(vcls,			uint64_t, 0, 'i', "VCL references")
-VSC_F(happy,		uint64_t, 0, 'b', "Happy health probes")
+VSC_F(vcls,			uint64_t, 0, 'i', "VCL references", "")
+VSC_F(happy,		uint64_t, 0, 'b', "Happy health probes", "")
 
 #endif
 
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index fc18cb3..25a1948 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -295,7 +295,7 @@ iter_call(const struct vsc *vsc, VSC_iter_f *func, void *priv,
 		sp.class = t;						\
 		sp.ident = sha->ident;
 
-#define VSC_F(nn,tt,ll,ff,dd)						\
+#define VSC_F(nn,tt,ll,ff,dd,ee)					\
 		sp.name = #nn;						\
 		sp.fmt = #tt;						\
 		sp.flag = ff;						\
@@ -331,7 +331,7 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 		if (strcmp(sha->class, VSC_CLASS))
 			continue;
 		/*lint -save -e525 -e539 */
-#define VSC_F(a,b,c,d,e)
+#define VSC_F(n,t,l,f,d,e)
 #define VSC_DONE(a,b,c)
 #define VSC_DO(U,l,t)						\
 		if (!strcmp(sha->type, t)) {			\



From phk at varnish-cache.org  Sun Sep 18 07:59:20 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 18 Sep 2011 09:59:20 +0200
Subject: [master] 4f610d3 I've spent some time trying to think more clearly
	about stats counters and it's all a mess.
Message-ID: <E1R5CHY-000200-Is@project.varnish-software.com>

commit 4f610d3a28939cc2bbc74d4c7100a6613a3d9555
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 07:58:27 2011 +0000

    I've spent some time trying to think more clearly about stats counters
    and it's all a mess.
    
    Try to get it right, and document in the head-comment what I mean with
    "right".
    
    Goal:  Cleaned up for 3.1

diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 4c01f34..80e8da5 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -110,7 +110,7 @@ ses_sm_alloc(void)
 
 	hl = HTTP_estimate(nhttp);
 	l = sizeof *sm + nws + 2 * hl;
-	VSC_C_main->g_sessmem_size = l;
+	VSC_C_main->sessmem_size = l;
 	p = malloc(l);
 	if (p == NULL)
 		return (NULL);
@@ -190,20 +190,20 @@ SES_New(struct worker *wrk, struct sesspool *pp)
 		pp->nsess++;
 		do_alloc = 1;
 	}
-	wrk->stats.c_sessmem_free += pp->dly_free_cnt;
+	wrk->stats.sessmem_free += pp->dly_free_cnt;
 	pp->dly_free_cnt = 0;
 	Lck_Unlock(&pp->mtx);
 	if (do_alloc) {
 		sm = ses_sm_alloc();
 		if (sm != NULL) {
-			wrk->stats.c_sessmem_alloc++;
+			wrk->stats.sessmem_alloc++;
 			sm->pool = pp;
 			ses_setup(sm);
 		} else {
-			wrk->stats.c_sessmem_fail++;
+			wrk->stats.sessmem_fail++;
 		}
 	} else if (sm == NULL) {
-		wrk->stats.c_sessmem_limit++;
+		wrk->stats.sessmem_limit++;
 	}
 	if (sm == NULL)
 		return (NULL);
@@ -325,7 +325,7 @@ SES_Delete(struct sess *sp, const char *reason)
 		free(sm);
 		Lck_Lock(&pp->mtx);
 		if (wrk != NULL)
-			wrk->stats.c_sessmem_free++;
+			wrk->stats.sessmem_free++;
 		else
 			pp->dly_free_cnt++;
 		pp->nsess--;
@@ -335,7 +335,7 @@ SES_Delete(struct sess *sp, const char *reason)
 		ses_setup(sm);
 		Lck_Lock(&pp->mtx);
 		if (wrk != NULL) {
-			wrk->stats.c_sessmem_free += pp->dly_free_cnt;
+			wrk->stats.sessmem_free += pp->dly_free_cnt;
 			pp->dly_free_cnt = 0;
 		}
 		VTAILQ_INSERT_HEAD(&pp->freelist, sm, list);
diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index b5b1e6f..a8b1d5f 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -110,7 +110,7 @@ do_once_cb(void *priv, const struct VSC_point * const pt)
 	if (i > op->pad)
 		op->pad = i + 1;
 	printf("%*.*s", op->pad - i, op->pad - i, "");
-	if (pt->flag == 'a')
+	if (pt->flag == 'a' || pt->flag == 'c')
 		printf("%12ju %12.2f %s\n", val, val / op->up, pt->desc);
 	else
 		printf("%12ju %12s %s\n", val, ".  ", pt->desc);
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index d0eb001..d0db9f0 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -59,7 +59,7 @@ struct pt {
 	VTAILQ_ENTRY(pt)	next;
 	const volatile uint64_t	*ptr;
 	uint64_t		ref;
-	int			type;
+	int			flag;
 	char			seen;
 	char			*name;
 };
@@ -81,7 +81,7 @@ do_curses_cb(void *priv, const struct VSC_point * const sp)
 
 	pt->ptr = sp->ptr;
 	pt->ref = *pt->ptr;
-	pt->type = sp->flag;
+	pt->flag = sp->flag;
 
 	*buf = '\0';
 	if (strcmp(sp->class, "")) {
@@ -210,13 +210,13 @@ do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main,
 				line++;
 				if (line >= LINES)
 					break;
-				if (pt->type == 'a') {
+				if (pt->flag == 'a' || pt->flag == 'c') {
 					AC(mvprintw(line, 0,
 					    "%12ju %12.2f %12.2f %s\n",
 					    ju, (ju - (intmax_t)pt->ref)/lt,
 					    ju / up, pt->name));
 					pt->ref = ju;
-				} else if (pt->type == 'b') {
+				} else if (pt->flag == 'b') {
 					AC(mvprintw(line, 0, "  %010.10jx <",
 					    (ju >> 24) & 0xffffffffffLL));
 					for (ch = 0x800000; ch; ch >>= 1)
diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index 79ad315..6f910de 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -26,15 +26,31 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * 3rd argument marks fields for inclusion in the per worker-thread
- * stats structure.
+ * Definition of all shared memory statistics below.
+ *
+ * Fields (n, t, l, f, e, d):
+ *    n - Name:		Field name, in C-source and stats programs
+ *    t - Type:		C-type, uint64_t, unless marked in 'f'
+ *    l - Local:	Local counter in worker thread.
+ *    f - Format: 	Semantics of the value in this field
+ *				'a' - Accumulator (deprecated, use 'c')
+ *				'b' - Bitmap
+ *				'c' - Counter, never decreases.
+ *				'g' - Gauge, goes up and down
+ *				'i' - Integer (deprecated, use 'g')
+ *    e - Explantion:	Short explanation of field (for screen use)
+ *    d - Description:	Long explanation of field (for doc use)
+ *	
+ * -----------------------
+ * NB: Cleanup in progress
+ * -----------------------
+ *
+ * Insufficient attention has caused this to become a swamp of conflicting
+ * conventions, shorthands and general mumbo-jumbo.  I'm trying to clean
+ * it up as I go over the code in other business.
+ *
+ * Please see the sessmem section for how it should look.
  *
- * XXX: We need a much more consistent naming of these fields, this has
- * XXX: turned into a major mess, causing trouble already for backends.
- * XXX:
- * XXX: Please converge on:
- * XXX:		c_* counter	(total bytes ever allocated from sma, "")
- * XXX:		g_* gauge	(presently allocated bytes from sma, "")
  */
 
 /**********************************************************************/
@@ -77,27 +93,27 @@ VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)", "")
  *    see: cache_session.c
  */
 
-VSC_F(g_sessmem_size,		uint64_t, 1, 'i',
+VSC_F(sessmem_size,		uint64_t, 1, 'g',
     "Session mem size",
 	"Bytes of memory allocated for last allocated session."
 )
 
-VSC_F(c_sessmem_alloc,		uint64_t, 1, 'a',
+VSC_F(sessmem_alloc,		uint64_t, 1, 'c',
     "Session mem allocated",
 	"Count of all allocations of session memory."
 )
 
-VSC_F(c_sessmem_free,		uint64_t, 1, 'a',
+VSC_F(sessmem_free,		uint64_t, 1, 'c',
     "Session mem freed",
 	"Count of all frees of session memory."
 )
 
-VSC_F(c_sessmem_fail,		uint64_t, 1, 'a',
+VSC_F(sessmem_fail,		uint64_t, 1, 'c',
     "Session mem alloc failed",
 	"Count of session memory allocation failures."
 )
 
-VSC_F(c_sessmem_limit,		uint64_t, 1, 'a',
+VSC_F(sessmem_limit,		uint64_t, 1, 'c',
     "Session mem alloc limited",
 	"Count of session memory allocations blocked by limit (max_sess)."
 )



From phk at varnish-cache.org  Sun Sep 18 08:14:52 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 18 Sep 2011 10:14:52 +0200
Subject: [master] 4571689 Shave 16 bytes of the session memory requirement by
	putting the sockaddr's directly into struct sess.
Message-ID: <E1R5CWa-0004aN-Be@project.varnish-software.com>

commit 4571689fdc7a7fff5f1b13ace0979bc0085181f3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 08:14:22 2011 +0000

    Shave 16 bytes of the session memory requirement by putting the
    sockaddr's directly into struct sess.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 5db137a..122ab34 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -548,8 +548,8 @@ struct sess {
 
 	socklen_t		sockaddrlen;
 	socklen_t		mysockaddrlen;
-	struct sockaddr_storage	*sockaddr;
-	struct sockaddr_storage	*mysockaddr;
+	struct sockaddr_storage	sockaddr;
+	struct sockaddr_storage	mysockaddr;
 	struct listen_sock	*mylsock;
 
 	/* formatted ascii client address */
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index b2a8015..50cc824 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -122,13 +122,13 @@ VCA_Prep(struct sess *sp)
 	char addr[VTCP_ADDRBUFSIZE];
 	char port[VTCP_PORTBUFSIZE];
 
-	VTCP_name(sp->sockaddr, sp->sockaddrlen,
+	VTCP_name(&sp->sockaddr, sp->sockaddrlen,
 	    addr, sizeof addr, port, sizeof port);
 	sp->addr = WS_Dup(sp->ws, addr);
 	sp->port = WS_Dup(sp->ws, port);
 	if (params->log_local_addr) {
-		AZ(getsockname(sp->fd, (void*)sp->mysockaddr, &sp->mysockaddrlen));
-		VTCP_name(sp->mysockaddr, sp->mysockaddrlen,
+		AZ(getsockname(sp->fd, (void*)&sp->mysockaddr, &sp->mysockaddrlen));
+		VTCP_name(&sp->mysockaddr, sp->mysockaddrlen,
 		    addr, sizeof addr, port, sizeof port);
 		VSL(SLT_SessionOpen, sp->fd, "%s %s %s %s",
 		    sp->addr, sp->port, addr, port);
@@ -264,7 +264,7 @@ VCA_SetupSess(struct worker *w)
 	sp->t_end = sp->t_end;
 	sp->mylsock = w->acceptlsock;
 	assert(w->acceptaddrlen <= sp->sockaddrlen);
-	memcpy(sp->sockaddr, &w->acceptaddr, w->acceptaddrlen);
+	memcpy(&sp->sockaddr, &w->acceptaddr, w->acceptaddrlen);
 	sp->sockaddrlen = w->acceptaddrlen;
 	sp->step = STP_FIRST;
 	vca_pace_good();
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 80e8da5..7012263 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -56,7 +56,6 @@ struct sessmem {
 	void			*wsp;
 	struct http		*http[2];
 	VTAILQ_ENTRY(sessmem)	list;
-	struct sockaddr_storage	sockaddr[2];
 };
 
 struct sesspool {
@@ -152,11 +151,9 @@ ses_setup(struct sessmem *sm)
 
 	sp->magic = SESS_MAGIC;
 	sp->mem = sm;
-	sp->sockaddr = (void*)(&sm->sockaddr[0]);
-	sp->sockaddrlen = sizeof(sm->sockaddr[0]);
-	sp->mysockaddr = (void*)(&sm->sockaddr[1]);
-	sp->mysockaddrlen = sizeof(sm->sockaddr[1]);
-	sp->sockaddr->ss_family = sp->mysockaddr->ss_family = PF_UNSPEC;
+	sp->sockaddrlen = sizeof(sp->sockaddr);
+	sp->mysockaddrlen = sizeof(sp->mysockaddr);
+	sp->sockaddr.ss_family = sp->mysockaddr.ss_family = PF_UNSPEC;
 	sp->t_open = NAN;
 	sp->t_req = NAN;
 	sp->t_resp = NAN;
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index 2054953..e26b433 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -457,10 +457,10 @@ REQ_BOOL(hash_always_miss)
 /*--------------------------------------------------------------------*/
 
 struct sockaddr_storage *
-VRT_r_client_ip(const struct sess *sp)
+VRT_r_client_ip(struct sess *sp)
 {
 
-	return (sp->sockaddr);
+	return (&sp->sockaddr);
 }
 
 struct sockaddr_storage *
@@ -468,12 +468,13 @@ VRT_r_server_ip(struct sess *sp)
 {
 	int i;
 
-	if (sp->mysockaddr->ss_family == AF_UNSPEC) {
-		i = getsockname(sp->fd, (void*)sp->mysockaddr, &sp->mysockaddrlen);
+	if (sp->mysockaddr.ss_family == AF_UNSPEC) {
+		i = getsockname(sp->fd,
+		    (void*)&sp->mysockaddr, &sp->mysockaddrlen);
 		assert(VTCP_Check(i));
 	}
 
-	return (sp->mysockaddr);
+	return (&sp->mysockaddr);
 }
 
 const char*
@@ -506,10 +507,14 @@ VRT_r_server_hostname(struct sess *sp)
 int
 VRT_r_server_port(struct sess *sp)
 {
+	int i;
 
-	if (sp->mysockaddr->ss_family == AF_UNSPEC)
-		AZ(getsockname(sp->fd, (void*)sp->mysockaddr, &sp->mysockaddrlen));
-	return (VTCP_port(sp->mysockaddr));
+	if (sp->mysockaddr.ss_family == AF_UNSPEC) {
+		i = getsockname(sp->fd,
+		    (void*)&sp->mysockaddr, &sp->mysockaddrlen);
+		assert(VTCP_Check(i));
+	}
+	return (VTCP_port(&sp->mysockaddr));
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index fde8bb2..779091e 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -107,7 +107,7 @@ sp_variables = (
 		'IP',
 		( 'proc',),
 		( ),
-		'const struct sess *'
+		'struct sess *'
 	),
 	('client.identity',
 		'STRING',



From phk at varnish-cache.org  Sun Sep 18 08:31:40 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 18 Sep 2011 10:31:40 +0200
Subject: [master] dd0e30e Finish polishing the session memory management:
Message-ID: <E1R5Cmq-0005tz-Oa@project.varnish-software.com>

commit dd0e30ee0b53f1a2a0f2f7de460a2dd16431efc3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 08:29:58 2011 +0000

    Finish polishing the session memory management:
    
    Also check if the http_max_hdr param changes.
    
    Describe http_max_hdr as per pool.
    
    Add SES_DeletePool() function.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 122ab34..2a7578d 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -849,7 +849,6 @@ unsigned WRW_WriteH(struct worker *w, const txt *hh, const char *suf);
 void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 #endif  /* SENDFILE_WORKS */
 
-
 /* cache_session.c [SES] */
 struct sess *SES_New(struct worker *wrk, struct sesspool *pp);
 struct sess *SES_Alloc(void);
@@ -857,7 +856,7 @@ void SES_Close(struct sess *sp, const char *reason);
 void SES_Delete(struct sess *sp, const char *reason);
 void SES_Charge(struct sess *sp);
 struct sesspool *SES_NewPool(void);
-
+void SES_DeletePool(struct sesspool *sp, struct worker *wrk);
 
 /* cache_shmlog.c */
 void VSL_Init(void);
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 7012263..23d15e7 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -51,11 +51,14 @@ struct sessmem {
 #define SESSMEM_MAGIC		0x555859c5
 
 	struct sesspool		*pool;
-	struct sess		sess;
+
 	unsigned		workspace;
+	uint16_t		nhttp;
 	void			*wsp;
 	struct http		*http[2];
 	VTAILQ_ENTRY(sessmem)	list;
+
+	struct sess		sess;
 };
 
 struct sesspool {
@@ -120,14 +123,20 @@ ses_sm_alloc(void)
 
 	sm = (void*)p;
 	p += sizeof *sm;
+
 	sm->magic = SESSMEM_MAGIC;
 	sm->workspace = nws;
+	sm->nhttp = nhttp;
+
 	sm->http[0] = HTTP_create(p, nhttp);
 	p += hl;
+
 	sm->http[1] = HTTP_create(p, nhttp);
 	p += hl;
+
 	sm->wsp = p;
 	p += nws;
+
 	assert(p == q);
 
 	return (sm);
@@ -318,6 +327,7 @@ SES_Delete(struct sess *sp, const char *reason)
 	    b->fetch, b->hdrbytes, b->bodybytes);
 
 	if (sm->workspace != params->sess_workspace ||
+	    sm->nhttp != (uint16_t)params->http_max_hdr ||
 	    pp->nsess > params->max_sess) {
 		free(sm);
 		Lck_Lock(&pp->mtx);
@@ -341,7 +351,7 @@ SES_Delete(struct sess *sp, const char *reason)
 }
 
 /*--------------------------------------------------------------------
- * Create a new pool to allocate from
+ * Create and delete pools
  */
 
 struct sesspool *
@@ -355,3 +365,25 @@ SES_NewPool(void)
 	Lck_New(&sp->mtx, lck_sessmem);
 	return (sp);
 }
+
+void
+SES_DeletePool(struct sesspool *sp, struct worker *wrk)
+{
+	struct sessmem *sm;
+
+	CHECK_OBJ_NOTNULL(sp, SESSPOOL_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	Lck_Lock(&sp->mtx);
+	while (!VTAILQ_EMPTY(&sp->freelist)) {
+		sm = VTAILQ_FIRST(&sp->freelist);
+		CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
+		VTAILQ_REMOVE(&sp->freelist, sm, list);
+		FREE_OBJ(sm);
+		wrk->stats.sessmem_free++;
+		sp->nsess--;
+	}
+	AZ(sp->nsess);
+	Lck_Unlock(&sp->mtx);
+	Lck_Delete(&sp->mtx);
+	FREE_OBJ(sp);
+}
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index b9067e7..ec38b11 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -777,7 +777,7 @@ static const struct parspec input_parspec[] = {
 		"off", "bool" },
 	{ "session_max", tweak_uint,
 		&master.max_sess, 1000, UINT_MAX,
-		"Maximum number of sessions we will allocate "
+		"Maximum number of sessions we will allocate from one pool "
 		"before just dropping connections.\n"
 		"This is mostly an anti-DoS measure, and setting it plenty "
 		"high should not hurt, as long as you have the memory for "



From phk at varnish-cache.org  Sun Sep 18 10:49:47 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 18 Sep 2011 12:49:47 +0200
Subject: [master] 260574a Make listen_sock a miniobj
Message-ID: <E1R5EwV-0006i2-KW@project.varnish-software.com>

commit 260574ac96af6db89a5374144d84f168b8e76645
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 09:04:55 2011 +0000

    Make listen_sock a miniobj

diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index 6571e31..13fce78 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -32,6 +32,8 @@
 #include <pthread.h>
 
 struct listen_sock {
+	unsigned			magic;
+#define LISTEN_SOCK_MAGIC		0x999e4b57
 	VTAILQ_ENTRY(listen_sock)	list;
 	int				sock;
 	char				*name;
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index ec38b11..96a83fc 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -333,10 +333,11 @@ clean_listen_sock_head(struct listen_sock_head *lsh)
 	struct listen_sock *ls, *ls2;
 
 	VTAILQ_FOREACH_SAFE(ls, lsh, list, ls2) {
+		CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
 		VTAILQ_REMOVE(lsh, ls, list);
 		free(ls->name);
 		free(ls->addr);
-		free(ls);
+		FREE_OBJ(ls);
 	}
 }
 
@@ -386,7 +387,7 @@ tweak_listen_address(struct cli *cli, const struct parspec *par,
 			break;
 		}
 		for (j = 0; j < n; ++j) {
-			ls = calloc(sizeof *ls, 1);
+			ALLOC_OBJ(ls, LISTEN_SOCK_MAGIC);
 			AN(ls);
 			ls->sock = -1;
 			ls->addr = ta[j];
@@ -410,6 +411,7 @@ tweak_listen_address(struct cli *cli, const struct parspec *par,
 	while (!VTAILQ_EMPTY(&lsh)) {
 		ls = VTAILQ_FIRST(&lsh);
 		VTAILQ_REMOVE(&lsh, ls, list);
+		CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
 		VTAILQ_INSERT_TAIL(&heritage.socks, ls, list);
 		heritage.nsocks++;
 	}



From phk at varnish-cache.org  Sun Sep 18 10:49:48 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 18 Sep 2011 12:49:48 +0200
Subject: [master] 6810e7c First cleanup sweep over new per-pool acceptor code:
Message-ID: <E1R5EwW-0006i7-4P@project.varnish-software.com>

commit 6810e7c339f46aadd5b1e6676f99663cc4b2ccd5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 10:46:08 2011 +0000

    First cleanup sweep over new per-pool acceptor code:
    
    Store the data from accept(2) on the worker->ws, we only need it for
    a brief moment until the worker configures a session with it.
    
    If we fail to allocate a session after accepting the connection,
    we silently close the connection again:  Presumably it is a DoS
    situation.  (false positive:  Extremely popular busy objects)
    
    Make three clearly defined counters for sessions:  Accepted, Dropped
    and failed (accept(2) failure).
    
    Lots of spit&polish.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 2a7578d..d5e4311 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -289,6 +289,20 @@ struct stream_ctx {
 };
 
 /*--------------------------------------------------------------------*/
+
+struct wrk_accept {
+	unsigned		magic;
+#define WRK_ACCEPT_MAGIC	0x8c4b4d59
+
+	/* Accept stuff */
+	struct sockaddr_storage	acceptaddr;
+	socklen_t		acceptaddrlen;
+	int			acceptsock;
+	struct listen_sock	*acceptlsock;
+};
+
+/*--------------------------------------------------------------------*/
+
 struct worker {
 	unsigned		magic;
 #define WORKER_MAGIC		0x6391adcf
@@ -303,12 +317,6 @@ struct worker {
 	/* Pool stuff */
 	double			lastused;
 
-	/* Accept stuff */
-	struct sockaddr_storage	acceptaddr;
-	socklen_t		acceptaddrlen;
-	int			acceptsock;
-	struct listen_sock	*acceptlsock;
-
 	struct wrw		wrw;
 
 	pthread_cond_t		cond;
@@ -642,8 +650,9 @@ struct vbc {
 void VCA_Prep(struct sess *sp);
 void VCA_Init(void);
 void VCA_Shutdown(void);
-int VCA_Accept(int sock, socklen_t *slp, struct sockaddr_storage *sap);
+int VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa);
 void VCA_SetupSess(struct worker *w);
+void VCA_FailSess(struct worker *w);
 
 /* cache_backend.c */
 void VBE_UseHealth(const struct director *vdi);
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 50cc824..ba4e84a 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -167,7 +167,7 @@ vca_pace_check(void)
 {
 	double p;
 
-	if (vca_pace == 0.0) 
+	if (vca_pace == 0.0)
 		return;
 	Lck_Lock(&pace_mtx);
 	p = vca_pace;
@@ -191,7 +191,7 @@ static void
 vca_pace_good(void)
 {
 
-	if (vca_pace == 0.0) 
+	if (vca_pace == 0.0)
 		return;
 	Lck_Lock(&pace_mtx);
 	vca_pace *= params->acceptor_sleep_decay;
@@ -207,69 +207,89 @@ vca_pace_good(void)
 static int hack_ready;
 
 int
-VCA_Accept(int sock, socklen_t *slp, struct sockaddr_storage *sap)
+VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa)
 {
 	int i;
 
-	assert(sock >= 0);
+	CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
+	assert(ls->sock >= 0);
 	vca_pace_check();
 
 	while(!hack_ready)
 		(void)usleep(100*1000);
 
-	*slp = sizeof *sap;
-	i = accept(sock, (void*)sap, slp);
+	wa->acceptaddrlen = sizeof wa->acceptaddr;
+	i = accept(ls->sock, (void*)&wa->acceptaddr, &wa->acceptaddrlen);
 
 	if (i < 0) {
-		VSC_C_main->accept_fail++;
 		switch (errno) {
-		case EAGAIN:
 		case ECONNABORTED:
 			break;
 		case EMFILE:
-			VSL(SLT_Debug, sock, "Too many open files");
+			VSL(SLT_Debug, ls->sock, "Too many open files");
 			vca_pace_bad();
 			break;
 		default:
-			VSL(SLT_Debug, sock, "Accept failed: %s",
+			VSL(SLT_Debug, ls->sock, "Accept failed: %s",
 			    strerror(errno));
 			vca_pace_bad();
 			break;
 		}
 	}
+	wa->acceptlsock = ls;
+	wa->acceptsock = i;
 	return (i);
 }
 
+/*--------------------------------------------------------------------
+ * Fail a session
+ *
+ * This happens if we accept the socket, but cannot get a session
+ * structure.
+ *
+ * We consider this a DoS situation (false positive:  Extremely popular
+ * busy objects) and silently close the connection with minimum effort
+ * and fuzz, rather than try to send an intelligent message back.
+ */
+
+void
+VCA_FailSess(struct worker *w)
+{
+	struct wrk_accept *wa;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(wa, (void*)w->ws->f, WRK_ACCEPT_MAGIC);
+	AZ(w->sp);
+	AZ(close(wa->acceptsock));
+	w->stats.sess_drop++;
+	vca_pace_bad();
+}
+
 /*--------------------------------------------------------------------*/
 
 void
 VCA_SetupSess(struct worker *w)
 {
 	struct sess *sp;
+	struct wrk_accept *wa;
 
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(wa, (void*)w->ws->f, WRK_ACCEPT_MAGIC);
 	sp = w->sp;
-	if (sp == NULL) {
-		AZ(close(w->acceptsock));
-		w->acceptsock = -1;
-		VSC_C_main->client_drop++;
-		/* XXX: 50x Reply ? */
-		vca_pace_bad();
-		INCOMPL();
-	}
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	sp->fd = w->acceptsock;
-	sp->id = w->acceptsock;
-	w->acceptsock = -1;
+	sp->fd = wa->acceptsock;
+	sp->id = wa->acceptsock;
+	wa->acceptsock = -1;
 	sp->t_open = TIM_real();
 	sp->t_end = sp->t_end;
-	sp->mylsock = w->acceptlsock;
-	assert(w->acceptaddrlen <= sp->sockaddrlen);
-	memcpy(&sp->sockaddr, &w->acceptaddr, w->acceptaddrlen);
-	sp->sockaddrlen = w->acceptaddrlen;
+	sp->mylsock = wa->acceptlsock;
+	CHECK_OBJ_NOTNULL(sp->mylsock, LISTEN_SOCK_MAGIC);
+	assert(wa->acceptaddrlen <= sp->sockaddrlen);
+	memcpy(&sp->sockaddr, &wa->acceptaddr, wa->acceptaddrlen);
+	sp->sockaddrlen = wa->acceptaddrlen;
 	sp->step = STP_FIRST;
 	vca_pace_good();
-	w->sp = sp;
-	w->stats.client_conn++;
+	w->stats.sess_conn++;
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index db51927..122bccf 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -1115,7 +1115,7 @@ cnt_lookup(struct sess *sp)
 			WS_ReleaseP(sp->ws, (void*)sp->vary_l);
 		} else {
 			AZ(oc->busyobj->vary);
-			WS_Release(sp->ws, 0); 
+			WS_Release(sp->ws, 0);
 		}
 		sp->vary_b = NULL;
 		sp->vary_l = NULL;
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 1bfc6b4..64aad05 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -67,7 +67,6 @@ struct poolsock {
 #define POOLSOCK_MAGIC			0x1b0a2d38
 	VTAILQ_ENTRY(poolsock)		list;
 	struct listen_sock		*lsock;
-	int				sock;
 };
 
 /* Number of work requests queued in excess of worker threads available */
@@ -95,121 +94,163 @@ static unsigned			nthr_max;
 static pthread_cond_t		herder_cond;
 static struct lock		herder_mtx;
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Nobody is accepting on this socket, so we do.
+ *
+ * As long as we can stick the accepted connection to another thread
+ * we do so, otherwise we return and handle it ourselves.
+ *
+ * Notice calling convention:  Called locked and returns locked, but
+ * works lock in the meantime.
+ *
+ * We store data about the accept in reserved workspace, it is only used
+ * for a brief moment and it takes up around 144 bytes.
+ */
 
 static void
 pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
 {
 	struct worker *w2;
+	struct wrk_accept *wa, *wa2;
 
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(ps, POOLSOCK_MAGIC);
 
-	assert(ps->sock >= 0);
+	CHECK_OBJ_NOTNULL(ps->lsock, LISTEN_SOCK_MAGIC);
 	Lck_AssertHeld(&pp->mtx);
 	Lck_Unlock(&pp->mtx);
+	assert(sizeof *wa == WS_Reserve(w->ws, sizeof *wa));
+	wa = (void*)w->ws->f;
 	while (1) {
-		w->acceptsock =
-		    VCA_Accept(ps->sock, &w->acceptaddrlen, &w->acceptaddr);
-		if (w->acceptsock == -1)
+		memset(wa, 0, sizeof *wa);
+		wa->magic = WRK_ACCEPT_MAGIC;
+
+		if (VCA_Accept(ps->lsock, wa) < 0) {
+			w->stats.sess_fail++;
+			/* We're going to pace in vca anyway... */
+			(void)WRK_TrySumStat(w);
 			continue;
-		w->acceptlsock = ps->lsock;
+		}
+
 		Lck_Lock(&pp->mtx);
 		if (VTAILQ_EMPTY(&pp->idle))
 			return;
 		w2 = VTAILQ_FIRST(&pp->idle);
 		VTAILQ_REMOVE(&pp->idle, w2, list);
 		Lck_Unlock(&pp->mtx);
-		w2->acceptaddr = w->acceptaddr;
-		w2->acceptaddrlen = w->acceptaddrlen;
-		w2->acceptsock = w->acceptsock;
-		w2->acceptlsock = w->acceptlsock;
+		assert(sizeof *wa2 == WS_Reserve(w2->ws, sizeof *wa2));
+		wa2 = (void*)w2->ws->f;
+		memcpy(wa2, wa, sizeof *wa);
 		AZ(pthread_cond_signal(&w2->cond));
 	}
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * This is the work function for worker threads in the pool.
+ */
 
 void
 Pool_Work_Thread(void *priv, struct worker *w)
 {
-	struct pool *qp;
+	struct pool *pp;
 	int stats_clean;
 	struct poolsock *ps;
 
-	CAST_OBJ_NOTNULL(qp, priv, POOL_MAGIC);
-	w->pool = qp;
-	Lck_Lock(&qp->mtx);
-	qp->nthr++;
+	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
+	w->pool = pp;
+	Lck_Lock(&pp->mtx);
+	pp->nthr++;
 	stats_clean = 1;
 	while (1) {
 
-		Lck_AssertHeld(&qp->mtx);
+		Lck_AssertHeld(&pp->mtx);
 
+		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 		CHECK_OBJ_NOTNULL(w->bereq, HTTP_MAGIC);
 		CHECK_OBJ_NOTNULL(w->beresp, HTTP_MAGIC);
 		CHECK_OBJ_NOTNULL(w->resp, HTTP_MAGIC);
-		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 
-		/* Process queued requests, if any */
-		w->sp = VTAILQ_FIRST(&qp->queue);
+		WS_Reset(w->ws, NULL);
+
+		w->sp = VTAILQ_FIRST(&pp->queue);
 		if (w->sp != NULL) {
-			VTAILQ_REMOVE(&qp->queue, w->sp, poollist);
-			qp->lqueue--;
-		} else if (VTAILQ_EMPTY(&qp->socks)) {
+			/* Process queued requests, if any */
+			assert(pp->lqueue > 0);
+			VTAILQ_REMOVE(&pp->queue, w->sp, poollist);
+			pp->lqueue--;
+		} else if (!VTAILQ_EMPTY(&pp->socks)) {
+			/* Accept on a socket */
+			ps = VTAILQ_FIRST(&pp->socks);
+			VTAILQ_REMOVE(&pp->socks, ps, list);
+			pool_accept(pp, w, ps);
+			Lck_AssertHeld(&pp->mtx);
+			VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
+		} else if (VTAILQ_EMPTY(&pp->socks)) {
+			/* Nothing to do: To sleep, perchance to dream ... */
 			if (isnan(w->lastused))
 				w->lastused = TIM_real();
-			VTAILQ_INSERT_HEAD(&qp->idle, w, list);
+			VTAILQ_INSERT_HEAD(&pp->idle, w, list);
 			if (!stats_clean)
 				WRK_SumStat(w);
-			Lck_CondWait(&w->cond, &qp->mtx);
-		} else {
-			ps = VTAILQ_FIRST(&qp->socks);
-			VTAILQ_REMOVE(&qp->socks, ps, list);
-			pool_accept(qp, w, ps);
-			Lck_AssertHeld(&qp->mtx);
-			VTAILQ_INSERT_TAIL(&qp->socks, ps, list);
+			Lck_CondWait(&w->cond, &pp->mtx);
 		}
-		if (w->sp == NULL && w->acceptsock == -1)
+
+		/*
+		 * If we got neither session or accepted a socket, we were
+		 * woken up to die to cull the herd.
+		 */
+		if (w->sp == NULL && w->ws->r == NULL)
 			break;
-		Lck_Unlock(&qp->mtx);
+
+		Lck_Unlock(&pp->mtx);
+
 		if (w->sp == NULL) {
-			w->sp = SES_New(w, qp->sesspool);
-			VCA_SetupSess(w);
+			/* Turn accepted socket into a session */
+			assert(w->ws->r != NULL);
+			w->sp = SES_New(w, pp->sesspool);
+			if (w->sp == NULL)
+				VCA_FailSess(w);
+			else
+				VCA_SetupSess(w);
+			WS_Release(w->ws, 0);
 		}
-		AN(w->sp);
-		assert(w->acceptsock == -1);
-		stats_clean = 0;
-		w->lastused = NAN;
-		WS_Reset(w->ws, NULL);
-		w->storage_hint = NULL;
-
-		AZ(w->sp->wrk);
-		THR_SetSession(w->sp);
-		w->sp->wrk = w;
-		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
-		CNT_Session(w->sp);
-		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
-		THR_SetSession(NULL);
-
-		WS_Assert(w->ws);
-		AZ(w->bereq->ws);
-		AZ(w->beresp->ws);
-		AZ(w->resp->ws);
-		AZ(w->wrw.wfd);
-		AZ(w->storage_hint);
-		assert(w->wlp == w->wlb);
-		w->sp = NULL;
-		if (params->diag_bitmap & 0x00040000) {
-			if (w->vcl != NULL)
-				VCL_Rel(&w->vcl);
+		assert(w->ws->r == NULL);
+
+		if (w->sp != NULL) {
+			CHECK_OBJ_NOTNULL(w->sp, SESS_MAGIC);
+
+			stats_clean = 0;
+			w->lastused = NAN;
+			w->storage_hint = NULL;
+
+			AZ(w->sp->wrk);
+			THR_SetSession(w->sp);
+			w->sp->wrk = w;
+			CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+			CNT_Session(w->sp);
+			CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+			THR_SetSession(NULL);
+			w->sp = NULL;
+
+			WS_Assert(w->ws);
+			AZ(w->bereq->ws);
+			AZ(w->beresp->ws);
+			AZ(w->resp->ws);
+			AZ(w->wrw.wfd);
+			AZ(w->storage_hint);
+			assert(w->wlp == w->wlb);
+			if (params->diag_bitmap & 0x00040000) {
+				if (w->vcl != NULL)
+					VCL_Rel(&w->vcl);
+			}
 		}
 		stats_clean = WRK_TrySumStat(w);
-		Lck_Lock(&qp->mtx);
+		Lck_Lock(&pp->mtx);
 	}
-	qp->nthr--;
-	Lck_Unlock(&qp->mtx);
+	assert(pp->nthr > 0);
+	pp->nthr--;
+	Lck_Unlock(&pp->mtx);
 	w->pool = NULL;
 }
 
@@ -320,7 +361,6 @@ pool_mkpool(void)
 			continue;
 		ALLOC_OBJ(ps, POOLSOCK_MAGIC);
 		XXXAN(ps);
-		ps->sock = ls->sock;
 		ps->lsock = ls;
 		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
 	}
diff --git a/bin/varnishd/cache_vrt_vmod.c b/bin/varnishd/cache_vrt_vmod.c
index bd2e172..557b01b 100644
--- a/bin/varnishd/cache_vrt_vmod.c
+++ b/bin/varnishd/cache_vrt_vmod.c
@@ -81,7 +81,7 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 		if (v->hdl == NULL) {
 			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
 			VCLI_Out(cli, "dlopen() failed: %s\n", dlerror());
-			VCLI_Out(cli, "Check child process permissions.\n"); 
+			VCLI_Out(cli, "Check child process permissions.\n");
 			FREE_OBJ(v);
 			return (1);
 		}
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index 7decfea..6e2cb05 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -52,9 +52,9 @@
 struct vwk {
 	unsigned		magic;
 #define VWK_MAGIC		0x1cc2acc2
-	pthread_t 		thread;
+	pthread_t		thread;
 	int			pipes[2];
-	int 			kq;
+	int			kq;
 	struct kevent		ki[NKEV];
 	unsigned		nki;
 	VTAILQ_HEAD(,sess)	sesshead;
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 3cfaba0..8e88e1d 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -79,7 +79,7 @@ WRK_SumStat(struct worker *w)
 int
 WRK_TrySumStat(struct worker *w)
 {
-	if (Lck_Trylock(&wstat_mtx)) 
+	if (Lck_Trylock(&wstat_mtx))
 		return (0);
 	wrk_sumstat(w);
 	Lck_Unlock(&wstat_mtx);
@@ -165,7 +165,6 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
 	w->bereq = HTTP_create(http0, nhttp);
 	w->beresp = HTTP_create(http1, nhttp);
 	w->resp = HTTP_create(http2, nhttp);
-	w->acceptsock = -1;
 	w->wrw.iov = iov;
 	w->wrw.siov = siov;
 	w->wrw.ciov = siov;
diff --git a/bin/varnishtest/tests/b00000.vtc b/bin/varnishtest/tests/b00000.vtc
index 1f4f766..2018235 100644
--- a/bin/varnishtest/tests/b00000.vtc
+++ b/bin/varnishtest/tests/b00000.vtc
@@ -10,7 +10,7 @@ varnish v1 -storage "-smalloc,1m" -vcl+backend {} -start
 varnish v1 -cliok "param.set diag_bitmap 0x2"
 
 varnish v1 -expect n_object == 0
-varnish v1 -expect client_conn == 0
+varnish v1 -expect sess_conn == 0
 varnish v1 -expect client_req == 0
 varnish v1 -expect cache_miss == 0
 
@@ -21,7 +21,7 @@ client c1 {
 } -run
 
 varnish v1 -expect n_object == 1
-varnish v1 -expect client_conn == 1
+varnish v1 -expect sess_conn == 1
 varnish v1 -expect client_req == 1
 varnish v1 -expect cache_miss == 1
 varnish v1 -expect s_sess == 1
diff --git a/bin/varnishtest/tests/b00001.vtc b/bin/varnishtest/tests/b00001.vtc
index 41dbe63..edec601 100644
--- a/bin/varnishtest/tests/b00001.vtc
+++ b/bin/varnishtest/tests/b00001.vtc
@@ -18,7 +18,7 @@ client c1 {
 } -run
 
 varnish v1 -expect n_object == 0
-varnish v1 -expect client_conn == 1
+varnish v1 -expect sess_conn == 1
 varnish v1 -expect client_req == 1
 varnish v1 -expect s_sess == 1
 varnish v1 -expect s_req == 1
diff --git a/bin/varnishtest/tests/b00002.vtc b/bin/varnishtest/tests/b00002.vtc
index 54a7ab7..8bb7ad3 100644
--- a/bin/varnishtest/tests/b00002.vtc
+++ b/bin/varnishtest/tests/b00002.vtc
@@ -22,7 +22,7 @@ delay .1
 
 varnish v1 -expect n_object == 0
 varnish v1 -expect SMA.Transient.g_alloc == 0
-varnish v1 -expect client_conn == 1
+varnish v1 -expect sess_conn == 1
 varnish v1 -expect client_req == 1
 varnish v1 -expect s_sess == 1
 varnish v1 -expect s_req == 1
diff --git a/bin/varnishtest/tests/b00003.vtc b/bin/varnishtest/tests/b00003.vtc
index 8b9375d..8c2bf00 100644
--- a/bin/varnishtest/tests/b00003.vtc
+++ b/bin/varnishtest/tests/b00003.vtc
@@ -24,7 +24,7 @@ client c2 {
 # Give varnish a chance to update stats
 delay .1
 
-varnish v1 -expect client_conn == 2
+varnish v1 -expect sess_conn == 2
 varnish v1 -expect cache_hit == 1
 varnish v1 -expect cache_miss == 1
 varnish v1 -expect client_req == 2
diff --git a/bin/varnishtest/tests/c00020.vtc b/bin/varnishtest/tests/c00020.vtc
index 31875b2..7fc707c 100644
--- a/bin/varnishtest/tests/c00020.vtc
+++ b/bin/varnishtest/tests/c00020.vtc
@@ -50,7 +50,7 @@ client c2 {
         expect resp.http.X-Varnish == "1006 1003"
 } -run
 
-varnish v1 -expect client_conn == 3
+varnish v1 -expect sess_conn == 3
 varnish v1 -expect cache_hit == 3
 varnish v1 -expect cache_miss == 3
 varnish v1 -expect client_req == 6
diff --git a/bin/varnishtest/tests/c00023.vtc b/bin/varnishtest/tests/c00023.vtc
index 6bed8e3..8c65eaa 100644
--- a/bin/varnishtest/tests/c00023.vtc
+++ b/bin/varnishtest/tests/c00023.vtc
@@ -148,7 +148,7 @@ client c1 {
 
 varnish v1 -cliok "hcb.dump"
 
-varnish v1 -expect client_conn == 2
+varnish v1 -expect sess_conn == 2
 varnish v1 -expect cache_hit == 9
 varnish v1 -expect cache_miss == 9
 varnish v1 -expect client_req == 18
diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index 6f910de..fc81919 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -32,7 +32,7 @@
  *    n - Name:		Field name, in C-source and stats programs
  *    t - Type:		C-type, uint64_t, unless marked in 'f'
  *    l - Local:	Local counter in worker thread.
- *    f - Format: 	Semantics of the value in this field
+ *    f - Format:	Semantics of the value in this field
  *				'a' - Accumulator (deprecated, use 'c')
  *				'b' - Bitmap
  *				'c' - Counter, never decreases.
@@ -40,7 +40,7 @@
  *				'i' - Integer (deprecated, use 'g')
  *    e - Explantion:	Short explanation of field (for screen use)
  *    d - Description:	Long explanation of field (for doc use)
- *	
+ *
  * -----------------------
  * NB: Cleanup in progress
  * -----------------------
@@ -57,9 +57,30 @@
 
 #ifdef VSC_DO_MAIN
 
-VSC_F(client_conn,		uint64_t, 1, 'a', "Client connections accepted", "")
-VSC_F(client_drop,		uint64_t, 0, 'a',
-					"Connection dropped, no sess/wrk", "")
+/*---------------------------------------------------------------------
+ * Sessions
+ *    see: cache_acceptor.c and cache_pool.c
+ */
+
+VSC_F(sess_conn,		uint64_t, 1, 'c',
+    "Sessions accepted",
+	"Count of sessions succesfully accepted"
+)
+VSC_F(sess_drop,		uint64_t, 1, 'c',
+    "Sessions dropped",
+	"Count of sessions silently dropped due to lack of session memory."
+	"  See parameter 'max_sess'."
+)
+
+VSC_F(sess_fail,		uint64_t, 1, 'c',
+    "Session accept failures",
+	"Count of failures to accept TCP connection."
+	"  Either the client changed its mind, or the kernel ran out of"
+	" some resource like filedescriptors."
+)
+
+/*---------------------------------------------------------------------*/
+
 VSC_F(client_req,		uint64_t, 1, 'a', "Client requests received", "")
 
 VSC_F(cache_hit,		uint64_t, 1, 'a', "Cache hits", "")
@@ -195,7 +216,6 @@ VSC_F(hcb_insert,		uint64_t, 0, 'a', "HCB Inserts", "")
 
 VSC_F(esi_errors,		uint64_t, 0, 'a', "ESI parse errors (unlock)", "")
 VSC_F(esi_warnings,		uint64_t, 0, 'a', "ESI parse warnings (unlock)", "")
-VSC_F(accept_fail,		uint64_t, 0, 'a', "Accept failures", "")
 VSC_F(client_drop_late,	uint64_t, 0, 'a', "Connection dropped late", "")
 VSC_F(uptime,		uint64_t, 0, 'a', "Client uptime", "")
 



From phk at varnish-cache.org  Sun Sep 18 20:12:37 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 18 Sep 2011 22:12:37 +0200
Subject: [master] 56c353f Give session memory pools a back pointer to their
	(thread) pool.
Message-ID: <E1R5NjB-0000y9-5V@project.varnish-software.com>

commit 56c353f7242d483ad710a0599f6d0000d16126dc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 20:09:43 2011 +0000

    Give session memory pools a back pointer to their (thread) pool.
    
    Schedule sessions from busy-lists and from the waiter via
    SES_Schedule(), to Pool_Schedule() in the right pool.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index d5e4311..9de746c 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -841,9 +841,9 @@ void PipeSession(struct sess *sp);
 
 /* cache_pool.c */
 void Pool_Init(void);
-int Pool_QueueSession(struct sess *sp);
 void Pool_Work_Thread(void *priv, struct worker *w);
 void Pool_Wait(struct sess *sp);
+int Pool_Schedule(struct pool *pp, struct sess *sp);
 
 #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
 int WRW_Error(const struct worker *w);
@@ -864,8 +864,10 @@ struct sess *SES_Alloc(void);
 void SES_Close(struct sess *sp, const char *reason);
 void SES_Delete(struct sess *sp, const char *reason);
 void SES_Charge(struct sess *sp);
-struct sesspool *SES_NewPool(void);
+struct sesspool *SES_NewPool(struct pool *pp);
 void SES_DeletePool(struct sesspool *sp, struct worker *wrk);
+int SES_Schedule(struct sess *sp);
+
 
 /* cache_shmlog.c */
 void VSL_Init(void);
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 8e0fd4e..faf867a 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -504,7 +504,7 @@ hsh_rush(struct objhead *oh)
 		AZ(sp->wrk);
 		VTAILQ_REMOVE(&wl->list, sp, list);
 		DSL(0x20, SLT_Debug, sp->id, "off waiting list");
-		if (Pool_QueueSession(sp)) {
+		if (SES_Schedule(sp)) {
 			/*
 			 * We could not schedule the session, leave the
 			 * rest on the busy list.
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 64aad05..f8c6871 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -261,47 +261,33 @@ Pool_Work_Thread(void *priv, struct worker *w)
  */
 
 static int
-WRK_Queue(struct sess *sp)
+WRK_Queue(struct pool *pp, struct sess *sp)
 {
 	struct worker *w;
-	struct pool *qp;
-	static unsigned nq = 0;
-	unsigned onq;
 
-	/*
-	 * Select which pool we issue to
-	 * XXX: better alg ?
-	 * XXX: per CPU ?
-	 */
-	onq = nq + 1;
-	if (onq >= nwq)
-		onq = 0;
-	qp = wq[onq];
-	nq = onq;
-
-	Lck_Lock(&qp->mtx);
+	Lck_Lock(&pp->mtx);
 
 	/* If there are idle threads, we tickle the first one into action */
-	w = VTAILQ_FIRST(&qp->idle);
+	w = VTAILQ_FIRST(&pp->idle);
 	if (w != NULL) {
-		VTAILQ_REMOVE(&qp->idle, w, list);
-		Lck_Unlock(&qp->mtx);
+		VTAILQ_REMOVE(&pp->idle, w, list);
+		Lck_Unlock(&pp->mtx);
 		w->sp = sp;
 		AZ(pthread_cond_signal(&w->cond));
 		return (0);
 	}
 
 	/* If we have too much in the queue already, refuse. */
-	if (qp->lqueue > queue_max) {
-		qp->ndrop++;
-		Lck_Unlock(&qp->mtx);
+	if (pp->lqueue > queue_max) {
+		pp->ndrop++;
+		Lck_Unlock(&pp->mtx);
 		return (-1);
 	}
 
-	VTAILQ_INSERT_TAIL(&qp->queue, sp, poollist);
-	qp->nqueue++;
-	qp->lqueue++;
-	Lck_Unlock(&qp->mtx);
+	VTAILQ_INSERT_TAIL(&pp->queue, sp, poollist);
+	pp->nqueue++;
+	pp->lqueue++;
+	Lck_Unlock(&pp->mtx);
 	AZ(pthread_cond_signal(&herder_cond));
 	return (0);
 }
@@ -309,11 +295,11 @@ WRK_Queue(struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 int
-Pool_QueueSession(struct sess *sp)
+Pool_Schedule(struct pool *pp, struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk);
-	if (WRK_Queue(sp) == 0)
+	if (WRK_Queue(pp, sp) == 0)
 		return(0);
 
 	VSC_C_main->client_drop_late++;
@@ -332,7 +318,6 @@ Pool_QueueSession(struct sess *sp)
 		 */
 		VCL_Rel(&sp->vcl);
 	}
-	SES_Delete(sp, "dropped");
 	return (1);
 }
 
@@ -353,7 +338,7 @@ pool_mkpool(void)
 	VTAILQ_INIT(&pp->queue);
 	VTAILQ_INIT(&pp->idle);
 	VTAILQ_INIT(&pp->socks);
-	pp->sesspool = SES_NewPool();
+	pp->sesspool = SES_NewPool(pp);
 	AN(pp->sesspool);
 
 	VTAILQ_FOREACH(ls, &heritage.socks, list) {
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 23d15e7..e8352f6 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -64,6 +64,7 @@ struct sessmem {
 struct sesspool {
 	unsigned		magic;
 #define SESSPOOL_MAGIC		0xd916e202
+	struct pool		*pool;
 	VTAILQ_HEAD(,sessmem)	freelist;
 	struct lock		mtx;
 	unsigned		nsess;
@@ -237,6 +238,35 @@ SES_Alloc(void)
 }
 
 /*--------------------------------------------------------------------
+ * Schedule a session back on a work-thread from its pool
+ */
+
+int
+SES_Schedule(struct sess *sp)
+{
+	struct sessmem *sm;
+	struct sesspool *pp;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	AZ(sp->wrk);
+
+	sm = sp->mem;
+	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
+
+	pp = sm->pool;
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+
+	AN(pp->pool);
+
+	if (Pool_Schedule(pp->pool, sp)) {
+		SES_Delete(sp, "dropped");
+		return (1);
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
  * Handle a session (from waiter)
  *
  * Status: see HTC_Rx()
@@ -255,7 +285,7 @@ SES_Handle(struct sess *sp, int status)
 		break;
 	case 1:
 		sp->step = STP_START;
-		(void)Pool_QueueSession(sp);
+		(void)SES_Schedule(sp);
 		break;
 	default:
 		WRONG("Unexpected return from HTC_Rx()");
@@ -355,12 +385,13 @@ SES_Delete(struct sess *sp, const char *reason)
  */
 
 struct sesspool *
-SES_NewPool(void)
+SES_NewPool(struct pool *pp)
 {
 	struct sesspool *sp;
 
 	ALLOC_OBJ(sp, SESSPOOL_MAGIC);
 	AN(sp);
+	sp->pool = pp;
 	VTAILQ_INIT(&sp->freelist);
 	Lck_New(&sp->mtx, lck_sessmem);
 	return (sp);



From phk at varnish-cache.org  Sun Sep 18 20:26:15 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 18 Sep 2011 22:26:15 +0200
Subject: [master] d885b81 Organize thread pools on a linked list instead of in
	an array.
Message-ID: <E1R5NwN-0001vq-Of@project.varnish-software.com>

commit d885b8182a4a4d6dcd57d158bfbefd1bbc251496
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 20:25:54 2011 +0000

    Organize thread pools on a linked list instead of in an array.

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index f8c6871..b593617 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -74,6 +74,7 @@ struct poolsock {
 struct pool {
 	unsigned		magic;
 #define POOL_MAGIC		0x606658fa
+	VTAILQ_ENTRY(pool)	list;
 	struct lock		mtx;
 	struct workerhead	idle;
 	VTAILQ_HEAD(, sess)	queue;
@@ -86,11 +87,13 @@ struct pool {
 	struct sesspool		*sesspool;
 };
 
-static struct pool		**wq;
-static unsigned			nwq;
+static VTAILQ_HEAD(,pool)	pools = VTAILQ_HEAD_INITIALIZER(pools);
+
 static unsigned			queue_max;
 static unsigned			nthr_max;
 
+static unsigned			nwq;
+
 static pthread_cond_t		herder_cond;
 static struct lock		herder_mtx;
 
@@ -349,28 +352,21 @@ pool_mkpool(void)
 		ps->lsock = ls;
 		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
 	}
+	VTAILQ_INSERT_TAIL(&pools, pp, list);
 	return (pp);
 }
 
 static void
-wrk_addpools(const unsigned pools)
+wrk_addpools(const unsigned npools)
 {
-	struct pool **pwq, **owq;
+	struct pool *pp;
 	unsigned u;
 
-	pwq = calloc(sizeof *pwq, pools);
-	if (pwq == NULL)
-		return;
-	if (wq != NULL)
-		memcpy(pwq, wq, sizeof *pwq * nwq);
-	owq = wq;
-	wq = pwq;
-	for (u = nwq; u < pools; u++) {
-		wq[u] = pool_mkpool();
-		XXXAN(wq[u]);
+	for (u = nwq; u < npools; u++) {
+		pp = pool_mkpool();
+		XXXAN(pp);
 	}
-	(void)owq;	/* XXX: avoid race, leak it. */
-	nwq = pools;
+	nwq = npools;
 }
 
 /*--------------------------------------------------------------------
@@ -422,6 +418,7 @@ wrk_herdtimer_thread(void *priv)
 	double t_idle;
 	struct VSC_C_main vsm, *vs;
 	int errno_is_multi_threaded;
+	struct pool *pp;
 
 	THR_SetName("wrk_herdtimer");
 
@@ -460,8 +457,8 @@ wrk_herdtimer_thread(void *priv)
 		vs->n_wrk_queued = 0;
 
 		t_idle = TIM_real() - params->wthread_timeout;
-		for (u = 0; u < nwq; u++)
-			wrk_decimate_flock(wq[u], t_idle, vs);
+		VTAILQ_FOREACH(pp, &pools, list)
+			wrk_decimate_flock(pp, t_idle, vs);
 
 		VSC_C_main->n_wrk= vs->n_wrk;
 		VSC_C_main->n_wrk_lqueue = vs->n_wrk_lqueue;
@@ -521,8 +518,8 @@ wrk_breed_flock(struct pool *qp, const pthread_attr_t *tp_attr)
 static void *
 wrk_herder_thread(void *priv)
 {
-	unsigned u, w;
 	pthread_attr_t tp_attr;
+	struct pool *pp, *pp2;
 
 	/* Set the stacksize for worker threads */
 	AZ(pthread_attr_init(&tp_attr));
@@ -530,19 +527,19 @@ wrk_herder_thread(void *priv)
 	THR_SetName("wrk_herder");
 	(void)priv;
 	while (1) {
-		for (u = 0 ; u < nwq; u++) {
+		VTAILQ_FOREACH(pp, &pools, list) {
 			if (params->wthread_stacksize != UINT_MAX)
 				AZ(pthread_attr_setstacksize(&tp_attr,
 				    params->wthread_stacksize));
 
-			wrk_breed_flock(wq[u], &tp_attr);
+			wrk_breed_flock(pp, &tp_attr);
 
 			/*
 			 * Make sure all pools have their minimum complement
 			 */
-			for (w = 0 ; w < nwq; w++)
-				while (wq[w]->nthr < params->wthread_min)
-					wrk_breed_flock(wq[w], &tp_attr);
+			VTAILQ_FOREACH(pp2, &pools, list)
+				while (pp2->nthr < params->wthread_min)
+					wrk_breed_flock(pp2, &tp_attr);
 			/*
 			 * We cannot avoid getting a mutex, so we have a
 			 * bogo mutex just for POSIX_STUPIDITY



From phk at varnish-cache.org  Sun Sep 18 22:52:05 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Sep 2011 00:52:05 +0200
Subject: [master] ac393c4 Revamp the code which rides herd on the thread pools:
Message-ID: <E1R5QDV-0000A9-OZ@project.varnish-software.com>

commit ac393c4b825a23abc428b4226920a0adc8bec67e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 22:48:01 2011 +0000

    Revamp the code which rides herd on the thread pools:
    
    Make one single thread, which creates thread pools, and since that
    is not particularly ardous work, also have it maintain the stats
    gauge of number of sessions queued.
    
    (Removing pools should now be possible, but the code is not there yet.)
    
    Each pool has its own herder thread, which creates and destroys
    threads as required, and updates stats for dropped and queued
    sessions.
    
    Sanitize VSC fields associated with this area.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 9de746c..4f18a08 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -821,7 +821,7 @@ void Lck__Assert(const struct lock *lck, int held);
 /* public interface: */
 void LCK_Init(void);
 void Lck_Delete(struct lock *lck);
-void Lck_CondWait(pthread_cond_t *cond, struct lock *lck);
+int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, struct timespec *ts);
 
 #define Lck_New(a, b) Lck__New(a, b, #b)
 #define Lck_Lock(a) Lck__Lock(a, __func__, __FILE__, __LINE__)
diff --git a/bin/varnishd/cache_lck.c b/bin/varnishd/cache_lck.c
index 3e0c640..b9229b3 100644
--- a/bin/varnishd/cache_lck.c
+++ b/bin/varnishd/cache_lck.c
@@ -139,19 +139,26 @@ Lck__Assert(const struct lock *lck, int held)
 		    !pthread_equal(ilck->owner, pthread_self()));
 }
 
-void __match_proto__()
-Lck_CondWait(pthread_cond_t *cond, struct lock *lck)
+int __match_proto__()
+Lck_CondWait(pthread_cond_t *cond, struct lock *lck, struct timespec *ts)
 {
 	struct ilck *ilck;
+	int retval = 0;
 
 	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
 	AN(ilck->held);
 	assert(pthread_equal(ilck->owner, pthread_self()));
 	ilck->held = 0;
-	AZ(pthread_cond_wait(cond, &ilck->mtx));
+	if (ts == NULL) {
+		AZ(pthread_cond_wait(cond, &ilck->mtx));
+	} else {
+		retval = pthread_cond_timedwait(cond, &ilck->mtx, ts);
+		assert(retval == 0 || retval == ETIMEDOUT);
+	}
 	AZ(ilck->held);
 	ilck->held = 1;
 	ilck->owner = pthread_self();
+	return (retval);
 }
 
 void
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index b593617..2aeb1b9 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -75,6 +75,11 @@ struct pool {
 	unsigned		magic;
 #define POOL_MAGIC		0x606658fa
 	VTAILQ_ENTRY(pool)	list;
+
+	pthread_cond_t		herder_cond;
+	struct lock		herder_mtx;
+	pthread_t		herder_thr;
+
 	struct lock		mtx;
 	struct workerhead	idle;
 	VTAILQ_HEAD(, sess)	queue;
@@ -82,20 +87,13 @@ struct pool {
 	unsigned		nthr;
 	unsigned		lqueue;
 	unsigned		last_lqueue;
-	uintmax_t		ndrop;
-	uintmax_t		nqueue;
+	uintmax_t		ndropped;
+	uintmax_t		nqueued;
 	struct sesspool		*sesspool;
 };
 
-static VTAILQ_HEAD(,pool)	pools = VTAILQ_HEAD_INITIALIZER(pools);
-
-static unsigned			queue_max;
-static unsigned			nthr_max;
-
-static unsigned			nwq;
-
-static pthread_cond_t		herder_cond;
-static struct lock		herder_mtx;
+static struct lock		pool_mtx;
+static pthread_t		thr_pool_herder;
 
 /*--------------------------------------------------------------------
  * Nobody is accepting on this socket, so we do.
@@ -163,7 +161,6 @@ Pool_Work_Thread(void *priv, struct worker *w)
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 	w->pool = pp;
 	Lck_Lock(&pp->mtx);
-	pp->nthr++;
 	stats_clean = 1;
 	while (1) {
 
@@ -196,7 +193,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 			VTAILQ_INSERT_HEAD(&pp->idle, w, list);
 			if (!stats_clean)
 				WRK_SumStat(w);
-			Lck_CondWait(&w->cond, &pp->mtx);
+			(void)Lck_CondWait(&w->cond, &pp->mtx, NULL);
 		}
 
 		/*
@@ -251,8 +248,6 @@ Pool_Work_Thread(void *priv, struct worker *w)
 		stats_clean = WRK_TrySumStat(w);
 		Lck_Lock(&pp->mtx);
 	}
-	assert(pp->nthr > 0);
-	pp->nthr--;
 	Lck_Unlock(&pp->mtx);
 	w->pool = NULL;
 }
@@ -281,17 +276,17 @@ WRK_Queue(struct pool *pp, struct sess *sp)
 	}
 
 	/* If we have too much in the queue already, refuse. */
-	if (pp->lqueue > queue_max) {
-		pp->ndrop++;
+	if (pp->lqueue > (params->queue_max * pp->nthr) / 100) {
+		pp->ndropped++;
 		Lck_Unlock(&pp->mtx);
 		return (-1);
 	}
 
 	VTAILQ_INSERT_TAIL(&pp->queue, sp, poollist);
-	pp->nqueue++;
+	pp->nqueued++;
 	pp->lqueue++;
 	Lck_Unlock(&pp->mtx);
-	AZ(pthread_cond_signal(&herder_cond));
+	AZ(pthread_cond_signal(&pp->herder_cond));
 	return (0);
 }
 
@@ -325,149 +320,24 @@ Pool_Schedule(struct pool *pp, struct sess *sp)
 }
 
 /*--------------------------------------------------------------------
- * Add (more) thread pools
- */
-
-static struct pool *
-pool_mkpool(void)
-{
-	struct pool *pp;
-	struct listen_sock *ls;
-	struct poolsock *ps;
-
-	ALLOC_OBJ(pp, POOL_MAGIC);
-	XXXAN(pp);
-	Lck_New(&pp->mtx, lck_wq);
-	VTAILQ_INIT(&pp->queue);
-	VTAILQ_INIT(&pp->idle);
-	VTAILQ_INIT(&pp->socks);
-	pp->sesspool = SES_NewPool(pp);
-	AN(pp->sesspool);
-
-	VTAILQ_FOREACH(ls, &heritage.socks, list) {
-		if (ls->sock < 0)
-			continue;
-		ALLOC_OBJ(ps, POOLSOCK_MAGIC);
-		XXXAN(ps);
-		ps->lsock = ls;
-		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
-	}
-	VTAILQ_INSERT_TAIL(&pools, pp, list);
-	return (pp);
-}
-
-static void
-wrk_addpools(const unsigned npools)
-{
-	struct pool *pp;
-	unsigned u;
-
-	for (u = nwq; u < npools; u++) {
-		pp = pool_mkpool();
-		XXXAN(pp);
-	}
-	nwq = npools;
-}
-
-/*--------------------------------------------------------------------
- * If a thread is idle or excess, pick it out of the pool.
- */
-
-static void
-wrk_decimate_flock(struct pool *qp, double t_idle, struct VSC_C_main *vs)
-{
-	struct worker *w = NULL;
-
-	Lck_Lock(&qp->mtx);
-	vs->n_wrk += qp->nthr;
-	vs->n_wrk_lqueue += qp->lqueue;
-	vs->n_wrk_drop += qp->ndrop;
-	vs->n_wrk_queued += qp->nqueue;
-
-	if (qp->nthr > params->wthread_min) {
-		w = VTAILQ_LAST(&qp->idle, workerhead);
-		if (w != NULL && (w->lastused < t_idle || qp->nthr > nthr_max))
-			VTAILQ_REMOVE(&qp->idle, w, list);
-		else
-			w = NULL;
-	}
-	Lck_Unlock(&qp->mtx);
-
-	/* And give it a kiss on the cheek... */
-	if (w != NULL) {
-		AZ(w->sp);
-		AZ(pthread_cond_signal(&w->cond));
-		TIM_sleep(params->wthread_purge_delay * 1e-3);
-	}
-}
-
-/*--------------------------------------------------------------------
- * Periodic pool herding thread
- *
- * Do things which we can do at our leisure:
- *  Add pools
- *  Scale constants
- *  Get rid of excess threads
- *  Aggregate stats across pools
+ * Wait for another request
  */
 
-static void *
-wrk_herdtimer_thread(void *priv)
+void
+Pool_Wait(struct sess *sp)
 {
-	volatile unsigned u;
-	double t_idle;
-	struct VSC_C_main vsm, *vs;
-	int errno_is_multi_threaded;
-	struct pool *pp;
-
-	THR_SetName("wrk_herdtimer");
 
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	AZ(sp->obj);
+	AZ(sp->vcl);
+	assert(sp->fd >= 0);
 	/*
-	 * This is one of the first threads created, test to see that
-	 * errno is really per thread.  If this fails, your C-compiler
-	 * needs some magic argument (-mt, -pthread, -pthreads etc etc).
+	 * Set nonblocking in the worker-thread, before passing to the
+	 * acceptor thread, to reduce syscall density of the latter.
 	 */
-	errno = 0;
-	AN(unlink("/"));		/* This had better fail */
-	errno_is_multi_threaded = errno;
-	assert(errno_is_multi_threaded != 0);
-
-	memset(&vsm, 0, sizeof vsm);
-	vs = &vsm;
-
-	(void)priv;
-	while (1) {
-		/* Add Pools */
-		u = params->wthread_pools;
-		if (u > nwq)
-			wrk_addpools(u);
-
-		/* Scale parameters */
-
-		u = params->wthread_max;
-		if (u < params->wthread_min)
-			u = params->wthread_min;
-		nthr_max = u;
-
-		queue_max = (nthr_max * params->queue_max) / 100;
-
-		vs->n_wrk = 0;
-		vs->n_wrk_lqueue = 0;
-		vs->n_wrk_drop = 0;
-		vs->n_wrk_queued = 0;
-
-		t_idle = TIM_real() - params->wthread_timeout;
-		VTAILQ_FOREACH(pp, &pools, list)
-			wrk_decimate_flock(pp, t_idle, vs);
-
-		VSC_C_main->n_wrk= vs->n_wrk;
-		VSC_C_main->n_wrk_lqueue = vs->n_wrk_lqueue;
-		VSC_C_main->n_wrk_drop = vs->n_wrk_drop;
-		VSC_C_main->n_wrk_queued = vs->n_wrk_queued;
-
-		TIM_sleep(params->wthread_purge_delay * 1e-3);
-	}
-	NEEDLESS_RETURN(NULL);
+	if (VTCP_nonblocking(sp->fd))
+		SES_Close(sp, "remote closed");
+	waiter->pass(waiter_priv, sp);
 }
 
 /*--------------------------------------------------------------------
@@ -486,23 +356,33 @@ wrk_breed_flock(struct pool *qp, const pthread_attr_t *tp_attr)
 	if (qp->nthr < params->wthread_min ||	/* Not enough threads yet */
 	    (qp->lqueue > params->wthread_add_threshold && /* more needed */
 	    qp->lqueue > qp->last_lqueue)) {	/* not getting better since last */
-		if (qp->nthr >= nthr_max) {
-			VSC_C_main->n_wrk_max++;
+		if (qp->nthr > params->wthread_max) {
+			Lck_Lock(&pool_mtx);
+			VSC_C_main->threads_limited++;
+			Lck_Unlock(&pool_mtx);
 		} else if (pthread_create(&tp, tp_attr, WRK_thread, qp)) {
 			VSL(SLT_Debug, 0, "Create worker thread failed %d %s",
 			    errno, strerror(errno));
-			VSC_C_main->n_wrk_failed++;
+			Lck_Lock(&pool_mtx);
+			VSC_C_main->threads_limited++;
+			Lck_Unlock(&pool_mtx);
 			TIM_sleep(params->wthread_fail_delay * 1e-3);
 		} else {
 			AZ(pthread_detach(tp));
-			VSC_C_main->n_wrk_create++;
 			TIM_sleep(params->wthread_add_delay * 1e-3);
+			qp->nthr++;
+			Lck_Lock(&pool_mtx);
+			VSC_C_main->threads++;
+			VSC_C_main->threads_created++;
+			Lck_Unlock(&pool_mtx);
 		}
 	}
 	qp->last_lqueue = qp->lqueue;
 }
 
 /*--------------------------------------------------------------------
+ * Herd a single pool
+ *
  * This thread wakes up whenever a pool queues.
  *
  * The trick here is to not be too aggressive about creating threads.
@@ -515,62 +395,154 @@ wrk_breed_flock(struct pool *qp, const pthread_attr_t *tp_attr)
  *
  */
 
-static void *
-wrk_herder_thread(void *priv)
+static void*
+pool_herder(void *priv)
 {
+	struct pool *pp;
 	pthread_attr_t tp_attr;
-	struct pool *pp, *pp2;
+	struct timespec ts;
+	double t_idle;
+	struct worker *w;
+	int i;
 
-	/* Set the stacksize for worker threads */
+	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 	AZ(pthread_attr_init(&tp_attr));
 
-	THR_SetName("wrk_herder");
-	(void)priv;
 	while (1) {
-		VTAILQ_FOREACH(pp, &pools, list) {
-			if (params->wthread_stacksize != UINT_MAX)
-				AZ(pthread_attr_setstacksize(&tp_attr,
-				    params->wthread_stacksize));
-
-			wrk_breed_flock(pp, &tp_attr);
-
-			/*
-			 * Make sure all pools have their minimum complement
-			 */
-			VTAILQ_FOREACH(pp2, &pools, list)
-				while (pp2->nthr < params->wthread_min)
-					wrk_breed_flock(pp2, &tp_attr);
-			/*
-			 * We cannot avoid getting a mutex, so we have a
-			 * bogo mutex just for POSIX_STUPIDITY
-			 */
-			Lck_Lock(&herder_mtx);
-			Lck_CondWait(&herder_cond, &herder_mtx);
-			Lck_Unlock(&herder_mtx);
+		/* Set the stacksize for worker threads we create */
+		if (params->wthread_stacksize != UINT_MAX)
+			AZ(pthread_attr_setstacksize(&tp_attr,
+			    params->wthread_stacksize));
+		else {
+			AZ(pthread_attr_destroy(&tp_attr));
+			AZ(pthread_attr_init(&tp_attr));
+		}
+
+		wrk_breed_flock(pp, &tp_attr);
+		
+		if (pp->nthr < params->wthread_min)
+			continue;
+
+		AZ(clock_gettime(CLOCK_MONOTONIC, &ts));
+		ts.tv_sec += params->wthread_purge_delay / 1000;
+		ts.tv_nsec += 
+		    (params->wthread_purge_delay % 1000) * 1000000;
+		if (ts.tv_nsec >= 1000000000) {
+			ts.tv_sec++;
+			ts.tv_nsec -= 1000000000;
+		}
+
+		Lck_Lock(&pp->herder_mtx);
+		i = Lck_CondWait(&pp->herder_cond, &pp->herder_mtx, &ts);
+		Lck_Unlock(&pp->herder_mtx);
+		if (!i)
+			continue;
+
+		if (pp->nthr <= params->wthread_min) 
+			continue;
+
+		t_idle = TIM_real() - params->wthread_timeout;
+
+		Lck_Lock(&pp->mtx);
+		VSC_C_main->sess_queued += pp->nqueued;
+		VSC_C_main->sess_dropped += pp->ndropped;
+		pp->nqueued = pp->ndropped = 0;
+		w = VTAILQ_LAST(&pp->idle, workerhead);
+		if (w != NULL &&  
+		    (w->lastused < t_idle || pp->nthr > params->wthread_max)) {
+			VTAILQ_REMOVE(&pp->idle, w, list);
+		} else
+			w = NULL;
+		Lck_Unlock(&pp->mtx);
+
+		/* And give it a kiss on the cheek... */
+		if (w != NULL) {
+			pp->nthr--;
+			Lck_Lock(&pool_mtx);
+			VSC_C_main->threads--;
+			VSC_C_main->threads_destroyed++;
+			Lck_Unlock(&pool_mtx);
+			AZ(w->sp);
+			AZ(pthread_cond_signal(&w->cond));
 		}
 	}
-	NEEDLESS_RETURN(NULL);
 }
 
 /*--------------------------------------------------------------------
- * Wait for another request
+ * Add a thread pool
  */
 
-void
-Pool_Wait(struct sess *sp)
+static struct pool *
+pool_mkpool(void)
 {
+	struct pool *pp;
+	struct listen_sock *ls;
+	struct poolsock *ps;
+	pthread_condattr_t cv_attr;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AZ(sp->obj);
-	AZ(sp->vcl);
-	assert(sp->fd >= 0);
-	/*
-	 * Set nonblocking in the worker-thread, before passing to the
-	 * acceptor thread, to reduce syscall density of the latter.
-	 */
-	if (VTCP_nonblocking(sp->fd))
-		SES_Close(sp, "remote closed");
-	waiter->pass(waiter_priv, sp);
+	ALLOC_OBJ(pp, POOL_MAGIC);
+	XXXAN(pp);
+	Lck_New(&pp->mtx, lck_wq);
+
+	VTAILQ_INIT(&pp->queue);
+	VTAILQ_INIT(&pp->idle);
+	VTAILQ_INIT(&pp->socks);
+	pp->sesspool = SES_NewPool(pp);
+	AN(pp->sesspool);
+
+	VTAILQ_FOREACH(ls, &heritage.socks, list) {
+		if (ls->sock < 0)
+			continue;
+		ALLOC_OBJ(ps, POOLSOCK_MAGIC);
+		XXXAN(ps);
+		ps->lsock = ls;
+		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
+	}
+
+	AZ(pthread_condattr_init(&cv_attr));
+	AZ(pthread_condattr_setclock(&cv_attr, CLOCK_MONOTONIC));
+	AZ(pthread_cond_init(&pp->herder_cond, &cv_attr));
+	AZ(pthread_condattr_destroy(&cv_attr));
+	Lck_New(&pp->herder_mtx, lck_herder);
+	AZ(pthread_create(&pp->herder_thr, NULL, pool_herder, pp));
+
+	return (pp);
+}
+
+/*--------------------------------------------------------------------
+ * This thread adjusts the number of pools to match the parameter.
+ *
+ */
+
+static void *
+pool_poolherder(void *priv)
+{
+	unsigned nwq;
+	VTAILQ_HEAD(,pool)	pools = VTAILQ_HEAD_INITIALIZER(pools);
+	struct pool *pp;
+	uint64_t u;
+
+	THR_SetName("pool_herder");
+	(void)priv;
+
+	nwq = 0;
+	while (1) {
+		if (nwq < params->wthread_pools) {
+			pp = pool_mkpool();
+			if (pp != NULL) {
+				VTAILQ_INSERT_TAIL(&pools, pp, list);
+				VSC_C_main->pools++;
+				nwq++;
+				continue;
+			} 
+		}
+		/* XXX: remove pools */
+		(void)sleep(1);
+		u = 0;
+		VTAILQ_FOREACH(pp, &pools, list) 
+			u += pp->lqueue;
+		VSC_C_main->thread_queue_len = u;
+	}
 }
 
 /*--------------------------------------------------------------------*/
@@ -578,18 +550,10 @@ Pool_Wait(struct sess *sp)
 void
 Pool_Init(void)
 {
-	pthread_t tp;
-
-	AZ(pthread_cond_init(&herder_cond, NULL));
-	Lck_New(&herder_mtx, lck_herder);
 
 	waiter_priv = waiter->init();
-
-	wrk_addpools(params->wthread_pools);
-	AZ(pthread_create(&tp, NULL, wrk_herdtimer_thread, NULL));
-	AZ(pthread_detach(tp));
-	AZ(pthread_create(&tp, NULL, wrk_herder_thread, NULL));
-	AZ(pthread_detach(tp));
+	Lck_New(&pool_mtx, lck_wq);
+	AZ(pthread_create(&thr_pool_herder, NULL, pool_poolherder, NULL));
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishtest/tests/c00002.vtc b/bin/varnishtest/tests/c00002.vtc
index 88862b3..6914736 100644
--- a/bin/varnishtest/tests/c00002.vtc
+++ b/bin/varnishtest/tests/c00002.vtc
@@ -16,4 +16,4 @@ client c1 {
 	expect resp.status == 200
 } -run
 
-varnish v1 -expect n_wrk == 8
+varnish v1 -expect threads == 8
diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index fc81919..ea668cc 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -139,6 +139,68 @@ VSC_F(sessmem_limit,		uint64_t, 1, 'c',
 	"Count of session memory allocations blocked by limit (max_sess)."
 )
 
+/*---------------------------------------------------------------------
+ * Pools, threads, and sessions
+ *    see: cache_pool.c
+ *
+ */
+
+VSC_F(pools,			uint64_t, 1, 'g',
+    "Number of thread pools",
+	"Number of thread pools.  See also param wthread_pools."
+	"  NB: Presently pools cannot be removed once created."
+)
+
+VSC_F(threads,			uint64_t, 1, 'g',
+    "Total number of threads",
+	"Number of threads in all pools."
+	"  See also params thread_pools, thread_pool_min & thread_pool_max."
+)
+
+VSC_F(threads_limited,		uint64_t, 1, 'c',
+    "Threads hit max",
+	"Number of times more threads were needed, but limit was reached"
+	" in a thread pool."
+	"  See also param thread_pool_max."
+)
+
+VSC_F(threads_created,		uint64_t, 1, 'c',
+    "Threads created",
+	"Total number of threads created in all pools."
+)
+
+VSC_F(threads_destroyed,	uint64_t, 1, 'c',
+    "Threads destoryed",
+	"Total number of threads destroyed in all pools."
+)
+
+VSC_F(threads_failed,		uint64_t, 1, 'c',
+    "Thread creation failed",
+	"Number of times creating a thread failed."
+	"  See VSL::Debug for diagnostics."
+	"  See also param thread_fail_delay."
+)
+
+VSC_F(thread_queue_len,		uint64_t, 1, 'g',
+    "Length of session queue",
+	"Length of session queue waiting for threads."
+	"  NB: Only updates once per second."
+	"  See also param queue_max."
+)
+
+VSC_F(sess_queued,		uint64_t, 1, 'c',
+    "Sessions queued for thread",
+	"Number of times session was queued waiting for a thread."
+	"  See also param queue_max."
+)
+
+VSC_F(sess_dropped,		uint64_t, 1, 'c',
+    "Sessions dropped for thread",
+	"Number of times session was dropped because the queue were too"
+	" long already."
+	"  See also param queue_max."
+)
+
 /*---------------------------------------------------------------------*/
 
 VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem", "")
@@ -150,14 +212,7 @@ VSC_F(n_objecthead,		uint64_t, 1, 'i', "N struct objecthead", "")
 VSC_F(n_waitinglist,		uint64_t, 1, 'i', "N struct waitinglist", "")
 
 VSC_F(n_vbc,		uint64_t, 0, 'i', "N struct vbc", "")
-VSC_F(n_wrk,		uint64_t, 0, 'i', "N worker threads", "")
-VSC_F(n_wrk_create,	uint64_t, 0, 'a', "N worker threads created", "")
-VSC_F(n_wrk_failed,	uint64_t, 0, 'a',
-					"N worker threads not created", "")
-VSC_F(n_wrk_max,		uint64_t, 0, 'a', "N worker threads limited", "")
-VSC_F(n_wrk_lqueue,		uint64_t, 0, 'a', "work request queue length", "")
-VSC_F(n_wrk_queued,		uint64_t, 0, 'a', "N queued work requests", "")
-VSC_F(n_wrk_drop,		uint64_t, 0, 'a', "N dropped work requests", "")
+
 VSC_F(n_backend,		uint64_t, 0, 'i', "N backends", "")
 
 VSC_F(n_expired,		uint64_t, 0, 'i', "N expired objects", "")



From phk at varnish-cache.org  Mon Sep 19 08:36:24 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Sep 2011 10:36:24 +0200
Subject: [master] 8faf587 Various minor polishing
Message-ID: <E1R5ZKy-0007Jh-Fk@project.varnish-software.com>

commit 8faf587064362d52586affdc36bfcb27a6c1d947
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 19 08:36:14 2011 +0000

    Various minor polishing

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 2aeb1b9..2d64299 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -259,7 +259,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
  */
 
 static int
-WRK_Queue(struct pool *pp, struct sess *sp)
+pool_queue(struct pool *pp, struct sess *sp)
 {
 	struct worker *w;
 
@@ -297,7 +297,7 @@ Pool_Schedule(struct pool *pp, struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk);
-	if (WRK_Queue(pp, sp) == 0)
+	if (pool_queue(pp, sp) == 0)
 		return(0);
 
 	VSC_C_main->client_drop_late++;
@@ -345,7 +345,7 @@ Pool_Wait(struct sess *sp)
  */
 
 static void
-wrk_breed_flock(struct pool *qp, const pthread_attr_t *tp_attr)
+pool_breed(struct pool *qp, const pthread_attr_t *tp_attr)
 {
 	pthread_t tp;
 
@@ -418,7 +418,7 @@ pool_herder(void *priv)
 			AZ(pthread_attr_init(&tp_attr));
 		}
 
-		wrk_breed_flock(pp, &tp_attr);
+		pool_breed(pp, &tp_attr);
 		
 		if (pp->nthr < params->wthread_min)
 			continue;
@@ -466,6 +466,7 @@ pool_herder(void *priv)
 			AZ(pthread_cond_signal(&w->cond));
 		}
 	}
+	NEEDLESS_RETURN(NULL);
 }
 
 /*--------------------------------------------------------------------
@@ -543,6 +544,7 @@ pool_poolherder(void *priv)
 			u += pp->lqueue;
 		VSC_C_main->thread_queue_len = u;
 	}
+	NEEDLESS_RETURN(NULL);
 }
 
 /*--------------------------------------------------------------------*/
@@ -555,5 +557,3 @@ Pool_Init(void)
 	Lck_New(&pool_mtx, lck_wq);
 	AZ(pthread_create(&thr_pool_herder, NULL, pool_poolherder, NULL));
 }
-
-/*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 8e88e1d..0c3b18e 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -26,7 +26,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * Worker thread stuff unrealted to the worker thread pools.
+ * Worker thread stuff unrelated to the worker thread pools.
  */
 
 #include "config.h"



From phk at varnish-cache.org  Mon Sep 19 09:45:42 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Sep 2011 11:45:42 +0200
Subject: [master] bc1d749 Shave acct_req down to a bodybytes counter,
	it's all we use it for.
Message-ID: <E1R5aQ2-0004Lp-D3@project.varnish-software.com>

commit bc1d749a6618aecc2c075ce30e1366fa71c525da
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 19 09:45:00 2011 +0000

    Shave acct_req down to a bodybytes counter, it's all we use it for.
    
    Make pipe mode use the acct_tmp like everybody else.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 4f18a08..6c80714 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -218,9 +218,9 @@ struct acct {
 
 /*--------------------------------------------------------------------*/
 
-#define L0(n)
-#define L1(n)			uint64_t n;
-#define VSC_F(n, t, l, f, e,d)	L##l(n)
+#define L0(t, n)
+#define L1(t, n)		t n;
+#define VSC_F(n, t, l, f, e,d)	L##l(t, n)
 #define VSC_DO_MAIN
 struct dstat {
 #include "vsc_fields.h"
@@ -615,7 +615,7 @@ struct sess {
 	struct sessmem		*mem;
 
 	VTAILQ_ENTRY(sess)	poollist;
-	struct acct		acct_req;
+	uint64_t		req_bodybytes;
 	struct acct		acct_ses;
 
 #if defined(HAVE_EPOLL_CTL)
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 122bccf..32a4215 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -338,7 +338,8 @@ cnt_done(struct sess *sp)
 		/* XXX: Add StatReq == StatSess */
 		/* XXX: Workaround for pipe */
 		if (sp->fd >= 0) {
-			WSP(sp, SLT_Length, "%ju", (uintmax_t)sp->acct_req.bodybytes);
+			WSP(sp, SLT_Length, "%ju",
+			    (uintmax_t)sp->req_bodybytes);
 		}
 		WSL(sp->wrk, SLT_ReqEnd, sp->id, "%u %.9f %.9f %.9f %.9f %.9f",
 		    sp->xid, sp->t_req, sp->t_end, dh, dp, da);
@@ -352,7 +353,7 @@ cnt_done(struct sess *sp)
 	if (sp->esi_level > 0)
 		return (1);
 
-	memset(&sp->acct_req, 0, sizeof sp->acct_req);
+	sp->req_bodybytes = 0;
 
 	sp->t_req = NAN;
 	sp->hash_always_miss = 0;
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
index ddb5c15..3968767 100644
--- a/bin/varnishd/cache_pipe.c
+++ b/bin/varnishd/cache_pipe.c
@@ -78,10 +78,10 @@ PipeSession(struct sess *sp)
 	(void)VTCP_blocking(vc->fd);
 
 	WRW_Reserve(w, &vc->fd);
-	sp->acct_req.hdrbytes += http_Write(w, sp->wrk->bereq, 0);
+	sp->wrk->acct_tmp.hdrbytes += http_Write(w, sp->wrk->bereq, 0);
 
 	if (sp->htc->pipeline.b != NULL)
-		sp->acct_req.bodybytes +=
+		sp->wrk->acct_tmp.bodybytes +=
 		    WRW_Write(w, sp->htc->pipeline.b, Tlen(sp->htc->pipeline));
 
 	i = WRW_FlushRelease(w);
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index e8352f6..bbc5706 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -80,9 +80,10 @@ SES_Charge(struct sess *sp)
 {
 	struct acct *a = &sp->wrk->acct_tmp;
 
+	sp->req_bodybytes += a->bodybytes;
+
 #define ACCT(foo)				\
 	sp->wrk->stats.s_##foo += a->foo;	\
-	sp->acct_req.foo += a->foo;		\
 	sp->acct_ses.foo += a->foo;		\
 	a->foo = 0;
 #include "acct_fields.h"



From phk at varnish-cache.org  Mon Sep 19 10:04:36 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Sep 2011 12:04:36 +0200
Subject: [master] f0746df OS/X does not have CLOCK_MONOTONIC,
	use CLOCK_REALTIME and hope nobody steps the clock...
Message-ID: <E1R5aiK-0002RI-J2@project.varnish-software.com>

commit f0746df953ca90e11a48e10d9b6383feb63ac1f6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 19 10:04:00 2011 +0000

    OS/X does not have CLOCK_MONOTONIC, use CLOCK_REALTIME and hope nobody
    steps the clock...
    
    Fixes	#1022

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 2d64299..675fea6 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -58,6 +58,10 @@
 #include "stevedore.h"
 #include "hash_slinger.h"
 
+#ifndef CLOCK_MONOTONIC
+#define CLOCK_MONOTONIC CLOCK_REALTIME		/* OS/X is silly */
+#endif
+
 static void *waiter_priv;
 
 VTAILQ_HEAD(workerhead, worker);



From phk at varnish-cache.org  Mon Sep 19 13:38:42 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Sep 2011 15:38:42 +0200
Subject: [master] 652b2f6 Take another stab at getting OS/X to compile again.
Message-ID: <E1R5e3W-0007iM-G8@project.varnish-software.com>

commit 652b2f61a59a5698bb5f15ccf0a8d1b31d519f69
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 19 13:36:53 2011 +0000

    Take another stab at getting OS/X to compile again.

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 675fea6..811b233 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -49,6 +49,7 @@
 #include <math.h>
 #include <stdlib.h>
 #include <string.h>
+#include <time.h>
 #include <unistd.h>
 
 #include "vcl.h"



From phk at varnish-cache.org  Mon Sep 19 18:26:49 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Sep 2011 20:26:49 +0200
Subject: [master] e5975a5 Fix OS/X compilation for good.
Message-ID: <E1R5iYL-0005a8-Vs@project.varnish-software.com>

commit e5975a5f721fa91b954735b54121d54554696cba
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 19 18:26:20 2011 +0000

    Fix OS/X compilation for good.
    
    Fixes	#1022

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 811b233..7b79873 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -59,9 +59,32 @@
 #include "stevedore.h"
 #include "hash_slinger.h"
 
+/*--------------------------------------------------------------------
+ * MAC OS/X is incredibly moronic when it comes to time and such...
+ */
+
 #ifndef CLOCK_MONOTONIC
-#define CLOCK_MONOTONIC CLOCK_REALTIME		/* OS/X is silly */
-#endif
+#define CLOCK_MONOTONIC	0
+
+static int
+clock_gettime(int foo, struct timespec *ts)
+{
+	struct timeval tv;
+
+	gettimeofday(&tv, NULL);
+	ts->tv_sec = tv.tv_sec;
+	ts->tv_nsec = tv.tv_usec * 1000;
+	return (0);
+}
+
+static int
+pthread_condattr_setclock(pthread_condattr_t *attr, int foo)
+{
+	(void)attr;
+	(void)foo;
+	return (0);
+}
+#endif /* !CLOCK_MONOTONIC */
 
 static void *waiter_priv;
 



From phk at varnish-cache.org  Mon Sep 19 20:38:59 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Sep 2011 22:38:59 +0200
Subject: [master] 319d601 More OSX compat hackery, this time for LLVM
Message-ID: <E1R5kcF-0000cw-D4@project.varnish-software.com>

commit 319d60136f8bfbe52bda54b2a1c0e0c97a048a35
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 19 20:38:39 2011 +0000

    More OSX compat hackery, this time for LLVM

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 7b79873..4dbf8bd 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -71,6 +71,7 @@ clock_gettime(int foo, struct timespec *ts)
 {
 	struct timeval tv;
 
+	(void)foo;
 	gettimeofday(&tv, NULL);
 	ts->tv_sec = tv.tv_sec;
 	ts->tv_nsec = tv.tv_usec * 1000;



From phk at varnish-cache.org  Tue Sep 20 07:43:43 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 09:43:43 +0200
Subject: [master] 9e39eb2 Increase the servers listen depth to 10.
Message-ID: <E1R5uzX-0003tv-FI@project.varnish-software.com>

commit 9e39eb25c239961513f53138d21cc1f275f8f9b3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 07:43:29 2011 +0000

    Increase the servers listen depth to 10.

diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 061d9cc..542a266 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -132,7 +132,7 @@ server_new(const char *name)
 	bprintf(s->listen, "127.0.0.1:%d", 0);
 	AZ(VSS_parse(s->listen, &s->addr, &s->port));
 	s->repeat = 1;
-	s->depth = 1;
+	s->depth = 10;
 	s->sock = -1;
 	VTAILQ_INSERT_TAIL(&servers, s, list);
 	return (s);



From phk at varnish-cache.org  Tue Sep 20 08:03:13 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 10:03:13 +0200
Subject: [master] c4a27e7 Work around a bug in OSX
Message-ID: <E1R5vIP-00053n-50@project.varnish-software.com>

commit c4a27e7243249f33aeaf84ded84b007c145eaee8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 08:03:03 2011 +0000

    Work around a bug in OSX

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index c53482e..2ebef93 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -353,7 +353,15 @@ varnish_launch(struct varnish *v)
 	fd[0].events = POLLIN;
 	fd[1].fd = v->fds[0];
 	fd[1].events = POLLHUP;
+#ifdef __APPLE__
+	/*
+	 * OSX cannot poll a pipe for POLLHUP only, poll just returns
+	 * zero with no revents.
+	 */
+	i = poll(fd, 1, 10000);
+#else
 	i = poll(fd, 2, 10000);
+#endif
 	vtc_log(v->vl, 4, "CLIPOLL %d 0x%x 0x%x",
 	    i, fd[0].revents, fd[1].revents);
 	if (i == 0) {



From phk at varnish-cache.org  Tue Sep 20 08:11:40 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 10:11:40 +0200
Subject: [master] d5eedbe A missing type-cast
Message-ID: <E1R5vQa-0005pA-KN@project.varnish-software.com>

commit d5eedbe40b0388cf4b8d5001b4a2ee957cfd312b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 08:11:29 2011 +0000

    A missing type-cast

diff --git a/bin/varnishd/storage_persistent_mgt.c b/bin/varnishd/storage_persistent_mgt.c
index c4f2191..3bd5809 100644
--- a/bin/varnishd/storage_persistent_mgt.c
+++ b/bin/varnishd/storage_persistent_mgt.c
@@ -109,7 +109,7 @@ smp_metrics(struct smp_sc *sc)
 	 */
 	sc->free_reserve = sc->aim_segl * 10;
 
-	fprintf(stderr, "free_reserve = %ju\n", sc->free_reserve);
+	fprintf(stderr, "free_reserve = %ju\n", (uintmax_t)sc->free_reserve);
 }
 
 /*--------------------------------------------------------------------



From phk at varnish-cache.org  Tue Sep 20 08:57:56 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 10:57:56 +0200
Subject: [master] b99540d Fix a typo.
Message-ID: <E1R5w9M-0000NG-38@project.varnish-software.com>

commit b99540df697ff4780cad40384a863f7c428e4b76
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 08:57:46 2011 +0000

    Fix a typo.

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index ba4e84a..0808010 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -281,7 +281,7 @@ VCA_SetupSess(struct worker *w)
 	sp->id = wa->acceptsock;
 	wa->acceptsock = -1;
 	sp->t_open = TIM_real();
-	sp->t_end = sp->t_end;
+	sp->t_end = sp->t_open;
 	sp->mylsock = wa->acceptlsock;
 	CHECK_OBJ_NOTNULL(sp->mylsock, LISTEN_SOCK_MAGIC);
 	assert(wa->acceptaddrlen <= sp->sockaddrlen);



From phk at varnish-cache.org  Tue Sep 20 09:06:41 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 11:06:41 +0200
Subject: [master] 6734c86 Clear the poll structure beforehand in order to make
	sure there are no lingering bits.
Message-ID: <E1R5wHp-000117-EP@project.varnish-software.com>

commit 6734c8682f4a32c5240ccba10dbc61876a2de89c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:05:58 2011 +0000

    Clear the poll structure beforehand in order to make sure there
    are no lingering bits.
    
    Convert an assert to a failure.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 9c0f175..d01240b 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -899,8 +899,9 @@ cmd_http_send(CMD_ARGS)
 	AZ(av[2]);
 	vtc_dump(hp->vl, 4, "send", av[1], -1);
 	i = write(hp->fd, av[1], strlen(av[1]));
-	assert(i == strlen(av[1]));
-
+	if (i != strlen(av[1]))
+		vtc_log(hp->vl, 0, "Write error in http_send(): %s",
+		    strerror(errno));
 }
 
 /**********************************************************************
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 2ebef93..5ef4beb 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -349,6 +349,7 @@ varnish_launch(struct varnish *v)
 	AZ(pthread_create(&v->tp_vsl, NULL, varnishlog_thread, v));
 
 	/* Wait for the varnish to call home */
+	memset(fd, 0, sizeof fd);
 	fd[0].fd = v->cli_fd;
 	fd[0].events = POLLIN;
 	fd[1].fd = v->fds[0];



From tfheen at varnish-cache.org  Tue Sep 20 09:10:07 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 20 Sep 2011 11:10:07 +0200
Subject: [master] 8ff8091 Update zope-plone.vcl to v3 syntax
Message-ID: <E1R5wL9-0001LP-Qb@project.varnish-software.com>

commit 8ff8091eb4239f2e257288d02a84b67241cb3ad4
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 20 11:03:08 2011 +0200

    Update zope-plone.vcl to v3 syntax
    
    Thanks to cleberjsantos for initial patch which this is based on.
    
    Fixes: #976

diff --git a/etc/zope-plone.vcl b/etc/zope-plone.vcl
index 8391499..b69f583 100644
--- a/etc/zope-plone.vcl
+++ b/etc/zope-plone.vcl
@@ -14,8 +14,8 @@
 
 # Default backend is the Zope CMS
 backend default {
-	set backend.host = "127.0.0.1";
-	set backend.port = "9673";
+	.host = "127.0.0.1";
+	.port = "9673";
 }
 
 acl purge {
@@ -29,10 +29,10 @@ sub vcl_recv {
 	# requests for unknown hosts
         if (req.http.host ~ "(www.)?example.com") {
                 set req.http.host = "example.com";
-                set req.url = "/VirtualHostBase/http/example.com:80/example.com/VirtualHostRoot" req.url;
+                set req.url = "/VirtualHostBase/http/example.com:80/example.com/VirtualHostRoot" + req.url;
         } elsif (req.http.host ~ "(www.)?example.org") {
                 set req.http.host = "example.org";
-                set req.url = "/VirtualHostBase/http/example.org:80/example.org/VirtualHostRoot" req.url;
+                set req.url = "/VirtualHostBase/http/example.org:80/example.org/VirtualHostRoot" + req.url;
         } else {
                 error 404 "Unknown virtual host.";
         }
@@ -42,7 +42,7 @@ sub vcl_recv {
 
                 # POST - Logins and edits
                 if (req.request == "POST") {
-                        pass;
+                        return(pass);
                 }
                 
                 # PURGE - The CacheFu product can invalidate updated URLs
@@ -50,7 +50,7 @@ sub vcl_recv {
                         if (!client.ip ~ purge) {
                                 error 405 "Not allowed.";
                         }
-                        lookup;
+                        return(lookup);
                 }
         }
 
@@ -60,9 +60,9 @@ sub vcl_recv {
 		# Force lookup of specific urls unlikely to need protection
 		if (req.url ~ "\.(js|css)") {
                         remove req.http.cookie;
-                        lookup;
+                        return(lookup);
                 }
-                pass;
+                return(pass);
         }
 
         # The default vcl_recv is used from here.
@@ -71,13 +71,14 @@ sub vcl_recv {
 # Do the PURGE thing
 sub vcl_hit {
         if (req.request == "PURGE") {
-                set obj.ttl = 0s;
+                purge;
                 error 200 "Purged";
         }
 }
 sub vcl_miss {
         if (req.request == "PURGE") {
-                error 404 "Not in cache";
+                purge;
+                error 200 "Purged";
         }
 }
 
@@ -85,7 +86,7 @@ sub vcl_miss {
 # from Zope by using the CacheFu product
 
 sub vcl_fetch {
-        if (obj.ttl < 3600s) {
-                set obj.ttl = 3600s;
+        if (beresp.ttl < 3600s) {
+                set beresp.ttl = 3600s;
         }
 }



From tfheen at varnish-cache.org  Tue Sep 20 09:10:08 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 20 Sep 2011 11:10:08 +0200
Subject: [master] e1fea86 Increase http_req_size, http_resp_size to 8192 bytes
Message-ID: <E1R5wLA-0001LZ-Mb@project.varnish-software.com>

commit e1fea86a22813af16533edcab374a1043fdb3593
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 20 11:07:21 2011 +0200

    Increase http_req_size, http_resp_size to 8192 bytes
    
    Apache (and nginx) uses 8k, so use the same default size to avoid
    people being surprised by more 413 responses than necessary.
    
    Fixes: #1016

diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index 96a83fc..decd130 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -525,7 +525,7 @@ static const struct parspec input_parspec[] = {
 		"Maximum length of any HTTP client request header we will "
 		"allow.  The limit is inclusive its continuation lines.\n",
 		0,
-		"4096", "bytes" },
+		"8192", "bytes" },
 	{ "http_req_size", tweak_uint, &master.http_req_size,
 		256, UINT_MAX,
 		"Maximum number of bytes of HTTP client request we will deal "
@@ -541,7 +541,7 @@ static const struct parspec input_parspec[] = {
 		"Maximum length of any HTTP backend response header we will "
 		"allow.  The limit is inclusive its continuation lines.\n",
 		0,
-		"4096", "bytes" },
+		"8192", "bytes" },
 	{ "http_resp_size", tweak_uint, &master.http_resp_size,
 		256, UINT_MAX,
 		"Maximum number of bytes of HTTP backend resonse we will deal "



From phk at varnish-cache.org  Tue Sep 20 09:28:25 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 11:28:25 +0200
Subject: [master] c94a809 Kqueue is go for Darwin11. Somebody should attempt
	Darwin10 also.
Message-ID: <E1R5wcr-0002SA-ST@project.varnish-software.com>

commit c94a8092e4b3b2935a80e8234d6de73602cbb60c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:27:39 2011 +0000

    Kqueue is go for Darwin11.  Somebody should attempt Darwin10 also.

diff --git a/configure.ac b/configure.ac
index 1cdaa0c..c8b61e5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -332,7 +332,7 @@ AC_ARG_ENABLE(kqueue,
 
 if test "$enable_kqueue" = yes; then
 	case $target in
-	*-*-freebsd* | *-*-darwin9* | *-*-netbsd* )
+	*-*-freebsd* | *-*-darwin9* | *-*-darwin11* | *-*-netbsd* )
 		AC_CHECK_FUNCS([kqueue])
 		;;
 	*-*-bsd*)



From phk at varnish-cache.org  Tue Sep 20 09:28:26 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 11:28:26 +0200
Subject: [master] 0bc47b7 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1R5wcs-0002SF-20@project.varnish-software.com>

commit 0bc47b73e62173e985390fd53d40af23a1aeabaf
Merge: c94a809 e1fea86
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:28:22 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From phk at varnish-cache.org  Tue Sep 20 09:34:46 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 11:34:46 +0200
Subject: [master] d5adf88 Fix poll waiter,
	so that we don't terminate the search for poll'ed fd's early in the
	case of a timeout.
Message-ID: <E1R5wj0-0002pW-P3@project.varnish-software.com>

commit d5adf8877389d51455e2754d5fb21e24c9772e0e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:33:58 2011 +0000

    Fix poll waiter, so that we don't terminate the search for poll'ed
    fd's early in the case of a timeout.
    
    Fixes	#1023

diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index 3a211bd..4678255 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -122,7 +122,7 @@ vwp_unpoll(struct vwp *vwp, int fd)
 static void *
 vwp_main(void *priv)
 {
-	int v;
+	int v, v2;
 	struct vwp *vwp;
 	struct sess *ss[NEEV], *sp, *sp2;
 	double deadline;
@@ -143,8 +143,9 @@ vwp_main(void *priv)
 		v = poll(vwp->pollfd, vwp->hpoll + 1, 100);
 		assert(v >= 0);
 		deadline = TIM_real() - params->sess_timeout;
+		v2 = v;
 		VTAILQ_FOREACH_SAFE(sp, &vwp->sesshead, list, sp2) {
-			if (v == 0)
+			if (v != 0 && v2 == 0)
 				break;
 			CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 			fd = sp->fd;
@@ -153,7 +154,7 @@ vwp_main(void *priv)
 			assert(fd < vwp->npoll);
 			assert(vwp->pollfd[fd].fd == fd);
 			if (vwp->pollfd[fd].revents) {
-				v--;
+				v2--;
 				i = HTC_Rx(sp->htc);
 				if (vwp->pollfd[fd].revents != POLLIN)
 					VSL(SLT_Debug, fd, "Poll: %x / %d",
@@ -174,14 +175,14 @@ vwp_main(void *priv)
 				SES_Delete(sp, "timeout");
 			}
 		}
-		if (v && vwp->pollfd[vwp->pipes[0]].revents) {
+		if (v2 && vwp->pollfd[vwp->pipes[0]].revents) {
 
 			if (vwp->pollfd[vwp->pipes[0]].revents != POLLIN)
 				VSL(SLT_Debug, 0, "pipe.revents= 0x%x",
 				    vwp->pollfd[vwp->pipes[0]].revents);
 			assert(vwp->pollfd[vwp->pipes[0]].revents == POLLIN);
 			vwp->pollfd[vwp->pipes[0]].revents = 0;
-			v--;
+			v2--;
 			i = read(vwp->pipes[0], ss, sizeof ss);
 			assert(i >= 0);
 			assert(((unsigned)i % sizeof ss[0]) == 0);
@@ -192,7 +193,7 @@ vwp_main(void *priv)
 				vwp_poll(vwp, ss[j]->fd);
 			}
 		}
-		assert(v == 0);
+		assert(v2 == 0);
 	}
 	NEEDLESS_RETURN(NULL);
 }



From phk at varnish-cache.org  Tue Sep 20 09:40:55 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 11:40:55 +0200
Subject: [master] 2a8a885 More details about failing test subprocesses.
Message-ID: <E1R5wox-0003AM-0Z@project.varnish-software.com>

commit 2a8a885a1998cb4a31e45c61b00d8bc5017dc6c6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:40:44 2011 +0000

    More details about failing test subprocesses.

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index b280022..b189d1e 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -224,8 +224,11 @@ tst_cb(const struct vev *ve, int what)
 		free(jp->tmpdir);
 
 		if (stx) {
-			printf("#     top  TEST %s FAILED (%.3f)\n",
+			printf("#     top  TEST %s FAILED (%.3f)",
 			    jp->tst->filename, t);
+			if (WIFSIGNALED(stx))
+				printf(" signal=%d", WTERMSIG(stx));
+			printf(" exit=%d", WEXITSTATUS(stx));
 			if (!vtc_continue) {
 				/* XXX kill -9 other jobs ? */
 				exit(2);



From phk at varnish-cache.org  Tue Sep 20 09:41:34 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 11:41:34 +0200
Subject: [master] c43c726 Missing newline
Message-ID: <E1R5wpa-0003EU-9h@project.varnish-software.com>

commit c43c7265d4b931ddca971743dc6c75a6d56824b7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:41:28 2011 +0000

    Missing newline

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index b189d1e..01a5fe2 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -228,7 +228,7 @@ tst_cb(const struct vev *ve, int what)
 			    jp->tst->filename, t);
 			if (WIFSIGNALED(stx))
 				printf(" signal=%d", WTERMSIG(stx));
-			printf(" exit=%d", WEXITSTATUS(stx));
+			printf(" exit=%d\n", WEXITSTATUS(stx));
 			if (!vtc_continue) {
 				/* XXX kill -9 other jobs ? */
 				exit(2);



From phk at varnish-cache.org  Tue Sep 20 09:49:01 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 11:49:01 +0200
Subject: [master] ff3d342 Wait for the health-probe to finish before we even
	start client c2,
	this should solve issues where the c2-v1 connection times out before it
	ever got used.
Message-ID: <E1R5wwn-0003eo-1D@project.varnish-software.com>

commit ff3d342e34a8d779f4f8b83540248bde193d1906
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:48:16 2011 +0000

    Wait for the health-probe to finish before we even start client c2,
    this should solve issues where the c2-v1 connection times out before
    it ever got used.

diff --git a/bin/varnishtest/tests/s00002.vtc b/bin/varnishtest/tests/s00002.vtc
index f7b8820..939d0ee 100644
--- a/bin/varnishtest/tests/s00002.vtc
+++ b/bin/varnishtest/tests/s00002.vtc
@@ -62,8 +62,9 @@ client c1 {
 	expect resp.status == 200
 } -run
 
+sema r1 sync 2
+
 client c2 {
-	sema r1 sync 2
 	txreq -url "/"
 	rxresp
 	expect resp.http.foo == "bar"



From apj at varnish-cache.org  Tue Sep 20 10:25:57 2011
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Tue, 20 Sep 2011 12:25:57 +0200
Subject: [master] 8b925e3 Make -M help text better
Message-ID: <E1R5xWX-0007o1-Ax@project.varnish-software.com>

commit 8b925e389329874f4cca79718f5cbac7491fc0ae
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Sep 20 12:24:06 2011 +0200

    Make -M help text better

diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 5fb054a..49e0f81 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -145,7 +145,7 @@ usage(void)
 	fprintf(stderr, FMT, "", "  shl: space for SHL records [80m]");
 	fprintf(stderr, FMT, "", "  free: space for other allocations [1m]");
 	fprintf(stderr, FMT, "", "  fill: prefill new file [+]");
-	fprintf(stderr, FMT, "-M address:port", "CLI-master to connect to.");
+	fprintf(stderr, FMT, "-M address:port", "Reverse CLI destination.");
 	fprintf(stderr, FMT, "-n dir", "varnishd working directory");
 	fprintf(stderr, FMT, "-P file", "PID file");
 	fprintf(stderr, FMT, "-p param=value", "set parameter");



From phk at varnish-cache.org  Tue Sep 20 12:16:09 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 14:16:09 +0200
Subject: [master] e749a3e Make the VSL skeptical of zero-filled VSM.
Message-ID: <E1R5zFB-0000RJ-2e@project.varnish-software.com>

commit e749a3ee4960ad05c3fc70901a472f2e75379b92
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 11:58:37 2011 +0000

    Make the VSL skeptical of zero-filled VSM.

diff --git a/include/vsl.h b/include/vsl.h
index 288c5c7..45406c0 100644
--- a/include/vsl.h
+++ b/include/vsl.h
@@ -66,6 +66,7 @@
  * on this file to extract the table rather than handcode it
  */
 enum VSL_tag_e {
+	SLT_Bogus = 0,
 #define SLTM(foo)	SLT_##foo,
 #include "vsl_tags.h"
 #undef SLTM
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 6791888..848fe65 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -173,6 +173,11 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 	for (w = 0; w < TIMEOUT_USEC;) {
 		t = *vsl->log_ptr;
 
+		if (t == 0) {
+			/* Zero-initialized VSL */
+			VRMB();
+			continue;
+		}
 		if (t == VSL_WRAPMARKER) {
 			/* Wrap around not possible at front */
 			assert(vsl->log_ptr != vsl->log_start + 1);



From phk at varnish-cache.org  Tue Sep 20 12:16:14 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 14:16:14 +0200
Subject: [master] d4455ed Correctly handle listen socket shutdown: Delete the
	poolsock instance, pretend nothing else happened.
Message-ID: <E1R5zFG-0000RR-E4@project.varnish-software.com>

commit d4455ed4774c7433a9c70453bd54202daf299992
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 12:10:42 2011 +0000

    Correctly handle listen socket shutdown:  Delete the poolsock instance,
    pretend nothing else happened.

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 0808010..1d73a1a 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -212,7 +212,6 @@ VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa)
 	int i;
 
 	CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
-	assert(ls->sock >= 0);
 	vca_pace_check();
 
 	while(!hack_ready)
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 4dbf8bd..5ff635b 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -137,7 +137,7 @@ static pthread_t		thr_pool_herder;
  * for a brief moment and it takes up around 144 bytes.
  */
 
-static void
+static int
 pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
 {
 	struct worker *w2;
@@ -156,6 +156,11 @@ pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
 		memset(wa, 0, sizeof *wa);
 		wa->magic = WRK_ACCEPT_MAGIC;
 
+		if (ps->lsock->sock < 0) {
+			/* Socket Shutdown */
+			Lck_Lock(&pp->mtx);
+			return (-1);
+		}
 		if (VCA_Accept(ps->lsock, wa) < 0) {
 			w->stats.sess_fail++;
 			/* We're going to pace in vca anyway... */
@@ -165,7 +170,7 @@ pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
 
 		Lck_Lock(&pp->mtx);
 		if (VTAILQ_EMPTY(&pp->idle))
-			return;
+			return (0);
 		w2 = VTAILQ_FIRST(&pp->idle);
 		VTAILQ_REMOVE(&pp->idle, w2, list);
 		Lck_Unlock(&pp->mtx);
@@ -184,7 +189,7 @@ void
 Pool_Work_Thread(void *priv, struct worker *w)
 {
 	struct pool *pp;
-	int stats_clean;
+	int stats_clean, i;
 	struct poolsock *ps;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
@@ -212,8 +217,13 @@ Pool_Work_Thread(void *priv, struct worker *w)
 			/* Accept on a socket */
 			ps = VTAILQ_FIRST(&pp->socks);
 			VTAILQ_REMOVE(&pp->socks, ps, list);
-			pool_accept(pp, w, ps);
+			i = pool_accept(pp, w, ps);
 			Lck_AssertHeld(&pp->mtx);
+			if (i < 0) {
+				/* Socket Shutdown */
+				FREE_OBJ(ps);
+				continue;
+			}
 			VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
 		} else if (VTAILQ_EMPTY(&pp->socks)) {
 			/* Nothing to do: To sleep, perchance to dream ... */



From phk at varnish-cache.org  Tue Sep 20 12:16:25 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 14:16:25 +0200
Subject: [master] bc2759d Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1R5zFR-0000SE-Ei@project.varnish-software.com>

commit bc2759d426afe6423cb94f66c6c5771378a6c177
Merge: d4455ed 8b925e3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 12:11:24 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From phk at varnish-cache.org  Tue Sep 20 12:16:31 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 14:16:31 +0200
Subject: [master] 96a4a90 Try to make it easier to see why this test fails.
Message-ID: <E1R5zFX-0000T3-QL@project.varnish-software.com>

commit 96a4a9034bb0766228720bccba04cba3726f24b1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 12:15:54 2011 +0000

    Try to make it easier to see why this test fails.

diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc
index 66f447b..faf4b4b 100644
--- a/bin/varnishtest/tests/r00962.vtc
+++ b/bin/varnishtest/tests/r00962.vtc
@@ -32,6 +32,12 @@ varnish v1 -cliok "debug.persistent s0 dump"
 varnish v1 -cliok "debug.persistent s0 sync"
 varnish v1 -stop
 
+server s1 {
+	rxreq 
+	txresp -status 400 -msg "Persistent Object Not Found"
+} -start
+
+
 varnish v2 \
 	-arg "-pdiag_bitmap=0x20000" \
 	-storage "-spersistent,${tmpdir}/_.per2,10m -spersistent,${tmpdir}/_.per1,10m" \
@@ -40,6 +46,7 @@ varnish v2 \
 client c1 -connect ${v2_sock} {
 	txreq -url "/"
 	rxresp
+	expect resp.msg != "Persistent Object Not Found"
 	expect resp.status == 200
 	expect resp.http.X-Varnish == "1001"
 } -run



From phk at varnish-cache.org  Tue Sep 20 12:29:15 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 14:29:15 +0200
Subject: [master] f72c709 Disable this test on OSX,
	the VM mapping is too random to get us the same address again,
	even with hinting.
Message-ID: <E1R5zRr-0001Cw-HJ@project.varnish-software.com>

commit f72c7094e338c77650aaf4e00fef9c9344c2c8ae
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 12:28:43 2011 +0000

    Disable this test on OSX, the VM mapping is too random to get us
    the same address again, even with hinting.

diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc
index faf4b4b..aa0fb10 100644
--- a/bin/varnishtest/tests/r00962.vtc
+++ b/bin/varnishtest/tests/r00962.vtc
@@ -1,5 +1,8 @@
 varnishtest "Test address remapping"
 
+# VM-remapping is to random on OSX
+feature	not-OSX
+
 server s1 {
 	rxreq 
 	txresp 
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index a034fef..3bee42a 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -492,7 +492,13 @@ cmd_feature(CMD_ARGS)
 		if (sizeof(void*) == 8 && !strcmp(av[i], "64bit"))
 			continue;
 
-		vtc_log(vl, 1, "SKIPPING test, missing feature %s", av[i]);
+		if (!strcmp(av[i], "!OSX")) {
+#if !defined(__APPLE__) || !defined(__MACH__)
+			continue;
+#endif
+		}
+
+		vtc_log(vl, 1, "SKIPPING test, missing feature: %s", av[i]);
 		vtc_stop = 1;
 		return;
 	}



From phk at varnish-cache.org  Tue Sep 20 13:45:17 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 15:45:17 +0200
Subject: [master] fb5477b Don't ignore the return value of the read(2) call on
	a random dev.
Message-ID: <E1R60dR-0008K4-4u@project.varnish-software.com>

commit fb5477bec8dad639f3db0bd2ac6900b3010a61a6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 13:44:59 2011 +0000

    Don't ignore the return value of the read(2) call on a random dev.

diff --git a/lib/libvarnishcompat/srandomdev.c b/lib/libvarnishcompat/srandomdev.c
index 79f774d..d0332c9 100644
--- a/lib/libvarnishcompat/srandomdev.c
+++ b/lib/libvarnishcompat/srandomdev.c
@@ -40,6 +40,22 @@
 
 #include "compat/srandomdev.h"
 
+static int
+trydev(const char *fn, unsigned long *seed)
+{
+	int fd;
+	ssize_t sz;
+
+	fd = open(fn, O_RDONLY);
+	if (fd < 0)
+		return (-1);
+	sz = read(fd, seed, sizeof *seed);
+	(void)close(fd);
+	if (sz != sizeof *seed)
+		return (-1);
+	return (0);
+}
+
 void
 srandomdev(void)
 {
@@ -47,13 +63,11 @@ srandomdev(void)
 	unsigned long seed;
 	int fd;
 
-	if ((fd = open("/dev/urandom", O_RDONLY)) >= 0 ||
-	    (fd = open("/dev/random", O_RDONLY)) >= 0) {
-		read(fd, &seed, sizeof seed);
-		close(fd);
-	} else {
-		gettimeofday(&tv, NULL);
-		seed = (getpid() << 16) ^ tv.tv_sec ^ tv.tv_usec;
+	if (trydev("/dev/urandom", &seed)) {
+		if (trydev("/dev/random", &seed)) {
+			gettimeofday(&tv, NULL);
+			seed = (getpid() << 16) ^ tv.tv_sec ^ tv.tv_usec;
+		}
 	}
 	srandom(seed);
 }



From phk at varnish-cache.org  Tue Sep 20 13:56:19 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 15:56:19 +0200
Subject: [master] db36b58 This cast is a tad too creative for my taste,
	make it safer.
Message-ID: <E1R60o7-0002Vr-K4@project.varnish-software.com>

commit db36b58c3237c0a63912781595516e3cf9ad36f8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 13:55:51 2011 +0000

    This cast is a tad too creative for my taste, make it safer.

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index 9c6cb99..8e1218a 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -59,8 +59,11 @@ VRT_re_init(void **rep, const char *re)
 void
 VRT_re_fini(void *rep)
 {
+	vre_t *vv;
+
+	vv = rep;
 	if (rep != NULL)
-		VRE_free((vre_t**)&rep);
+		VRE_free(&vv);
 }
 
 int



From phk at varnish-cache.org  Tue Sep 20 14:21:08 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 16:21:08 +0200
Subject: [master] 8b4cc65 Remember to release workspace on listen socket close.
Message-ID: <E1R61C8-0004eS-I2@project.varnish-software.com>

commit 8b4cc6576a5e4869cd133fef7d2b002dffc49af9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 14:20:51 2011 +0000

    Remember to release workspace on listen socket close.

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 5ff635b..6be7613 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -222,6 +222,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 			if (i < 0) {
 				/* Socket Shutdown */
 				FREE_OBJ(ps);
+				WS_Release(w->ws, 0);
 				continue;
 			}
 			VTAILQ_INSERT_TAIL(&pp->socks, ps, list);



From phk at varnish-cache.org  Tue Sep 20 14:25:48 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 16:25:48 +0200
Subject: [master] f98e2b8 remove unused variable
Message-ID: <E1R61Ge-00052j-K2@project.varnish-software.com>

commit f98e2b83dd9683d2081a0ad6d8f219e98e8023ee
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 14:25:40 2011 +0000

    remove unused variable

diff --git a/lib/libvarnishcompat/srandomdev.c b/lib/libvarnishcompat/srandomdev.c
index d0332c9..83b22c4 100644
--- a/lib/libvarnishcompat/srandomdev.c
+++ b/lib/libvarnishcompat/srandomdev.c
@@ -61,7 +61,6 @@ srandomdev(void)
 {
 	struct timeval tv;
 	unsigned long seed;
-	int fd;
 
 	if (trydev("/dev/urandom", &seed)) {
 		if (trydev("/dev/random", &seed)) {



From phk at varnish-cache.org  Tue Sep 20 14:32:24 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 16:32:24 +0200
Subject: [master] 2da6772 Remember to sleep and terminate,
	rather than loop for ever on a zero VSL record. Still not quite sure
	how they happen, but we shouldn't hang no matter what.
Message-ID: <E1R61N2-0005U4-An@project.varnish-software.com>

commit 2da67725c55a9aa1c7719048b5bb9642129f0dcf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 14:31:44 2011 +0000

    Remember to sleep and terminate, rather than loop for ever on a zero
    VSL record.  Still not quite sure how they happen, but we shouldn't hang
    no matter what.

diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 848fe65..1bc46dd 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -175,6 +175,8 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 
 		if (t == 0) {
 			/* Zero-initialized VSL */
+			w += SLEEP_USEC;
+			assert(usleep(SLEEP_USEC) == 0 || errno == EINTR);
 			VRMB();
 			continue;
 		}
@@ -197,6 +199,7 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 				return (-1);
 			w += SLEEP_USEC;
 			assert(usleep(SLEEP_USEC) == 0 || errno == EINTR);
+			VRMB();
 			continue;
 		}
 		if (vsl->log_ptr == vsl->log_start + 1)



From phk at varnish-cache.org  Tue Sep 20 15:26:06 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 17:26:06 +0200
Subject: [master] 223ff71 Do the zero check last,
	so a varnishd restart has a chance to recover it.
Message-ID: <E1R62D0-00077J-Ve@project.varnish-software.com>

commit 223ff71a0ce19a7d33d3e06f800fa6e7b8a713e4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 15:25:44 2011 +0000

    Do the zero check last, so a varnishd restart has a chance to recover it.

diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 1bc46dd..58a454d 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -173,13 +173,6 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 	for (w = 0; w < TIMEOUT_USEC;) {
 		t = *vsl->log_ptr;
 
-		if (t == 0) {
-			/* Zero-initialized VSL */
-			w += SLEEP_USEC;
-			assert(usleep(SLEEP_USEC) == 0 || errno == EINTR);
-			VRMB();
-			continue;
-		}
 		if (t == VSL_WRAPMARKER) {
 			/* Wrap around not possible at front */
 			assert(vsl->log_ptr != vsl->log_start + 1);
@@ -202,6 +195,13 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 			VRMB();
 			continue;
 		}
+		if (t == 0) {
+			/* Zero-initialized VSL */
+			w += SLEEP_USEC;
+			assert(usleep(SLEEP_USEC) == 0 || errno == EINTR);
+			VRMB();
+			continue;
+		}
 		if (vsl->log_ptr == vsl->log_start + 1)
 			vsl->last_seq = vsl->log_start[0];
 



From phk at varnish-cache.org  Tue Sep 20 19:16:01 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 21:16:01 +0200
Subject: [master] 6d76ccd Don't rely on the test executing fast enough and
	repeatably enough that two separate timestamps just happen to
	be the same, just make them the same.
Message-ID: <E1R65nV-0001By-Bs@project.varnish-software.com>

commit 6d76ccd9fbd3b5022e5d55ae1779b7ea33f0365d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 19:15:11 2011 +0000

    Don't rely on the test executing fast enough and repeatably enough that
    two separate timestamps just happen to be the same, just make them
    the same.

diff --git a/bin/varnishtest/tests/r00907.vtc b/bin/varnishtest/tests/r00907.vtc
index 8bbcd2e..b64a160 100644
--- a/bin/varnishtest/tests/r00907.vtc
+++ b/bin/varnishtest/tests/r00907.vtc
@@ -3,7 +3,8 @@ varnishtest "Ticket #907 200/304 handling with Etags + Last-Modified"
 server s1 {
 	rxreq 
 	txresp \
-	       -hdr "ETag: saengei1Ohshicich4iteesu"
+		-hdr "ETag: saengei1Ohshicich4iteesu" \
+		-hdr "Last-Modified: Tue, 20 Sep 2011 18:55:00 GMT"
 } -start
 
 varnish v1 -vcl+backend {
@@ -16,8 +17,15 @@ client c1 {
 	txreq -hdr "If-None-Match: saengei1Ohshicich4iteesu"
 	rxresp
 	expect resp.status == 304
+
 	txreq -hdr "If-None-Match: saengei1Ohshicich4iteesu" \
-	      -hdr "If-Modified-Since: ${date}"
+	      -hdr "If-Modified-Since: Tue, 20 Sep 2011 18:54:59 GMT"
+	rxresp
+	expect resp.status == 200
+
+	txreq -hdr "If-None-Match: saengei1Ohshicich4iteesu" \
+	      -hdr "If-Modified-Since: Tue, 20 Sep 2011 18:55:00 GMT"
 	rxresp
 	expect resp.status == 304
+
 } -run



From phk at varnish-cache.org  Tue Sep 20 19:50:05 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 21:50:05 +0200
Subject: [master] 4a976ec Remove never effected test
Message-ID: <E1R66KT-0003j1-DY@project.varnish-software.com>

commit 4a976ecab7fdcbd58c79c6a1d5b589e3b9605e69
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 19:48:49 2011 +0000

    Remove never effected test

diff --git a/bin/varnishtest/tests/g00002.vtc b/bin/varnishtest/tests/g00002.vtc
index 574f1f2..1a0ab35 100644
--- a/bin/varnishtest/tests/g00002.vtc
+++ b/bin/varnishtest/tests/g00002.vtc
@@ -47,11 +47,4 @@ client c1 {
 	rxresp
 	expect resp.http.content-encoding == "resp.http.content-encoding"
 	expect resp.bodylen == 4109
-
-	# See varnish can deliver gzip'ed ESI (NOTYET)
-	#txreq -url /bar -hdr "Accept-Encoding: gzip"
-	#rxresp
-	# expect resp.http.content-encoding == "gzip"
-	#gunzip
-	#expect resp.bodylen == 4109
 } -run



From phk at varnish-cache.org  Tue Sep 20 19:50:05 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 21:50:05 +0200
Subject: [master] ff6f655 Increase varnishtest's HTTP rx timeout to 15 seconds,
	a lot of test failures seems to be timeouts on stressed machines.
Message-ID: <E1R66KT-0003j6-Pv@project.varnish-software.com>

commit ff6f655bfd040ec05000b8be39849c5a3c437f59
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 19:49:04 2011 +0000

    Increase varnishtest's HTTP rx timeout to 15 seconds, a lot of
    test failures seems to be timeouts on stressed machines.
    
    Teport timeouts more clearly.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index d01240b..7dad4ed 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -343,7 +343,10 @@ http_rxchar_eof(struct http *hp, int n)
 		pfd[0].events = POLLIN;
 		pfd[0].revents = 0;
 		i = poll(pfd, 1, hp->timeout);
-		if (i <= 0)
+		if (i < 0)
+			vtc_log(hp->vl, 0, "HTTP rx timeout (%u ms)",
+			    hp->timeout);
+		if (i < 0)
 			vtc_log(hp->vl, 0, "HTTP rx failed (poll: %s)",
 			    strerror(errno));
 		assert(i > 0);
@@ -1143,7 +1146,7 @@ http_process(struct vtclog *vl, const char *spec, int sock, int sfd)
 	ALLOC_OBJ(hp, HTTP_MAGIC);
 	AN(hp);
 	hp->fd = sock;
-	hp->timeout = 5000;
+	hp->timeout = 15000;
 	hp->nrxbuf = 2048*1024;
 	hp->vsb = VSB_new_auto();
 	hp->rxbuf = malloc(hp->nrxbuf);		/* XXX */



From phk at varnish-cache.org  Tue Sep 20 20:29:45 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 22:29:45 +0200
Subject: [master] 4351233 Fix a long-standing bug in varnishtest:
Message-ID: <E1R66wr-0006hR-MO@project.varnish-software.com>

commit 43512331ff01fad7f5078cda5339ba169bd1d81f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 20:22:09 2011 +0000

    Fix a long-standing bug in varnishtest:
    
    It used to be that we'd call http processing on a filedescriptor and
    then that was that.  Then we added keywords to do new accepts in the
    server and suddenly the calling code closes a wrong filedesc which
    belongs to somebody else and things get really inconvenient fast.
    
    This made all test-cases which use the "accept" server directive flakey.

diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index eb9a02a..5e5cce7 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -64,7 +64,7 @@ extern pthread_t	vtc_thread;
 
 void init_sema(void);
 
-void http_process(struct vtclog *vl, const char *spec, int sock, int sfd);
+int http_process(struct vtclog *vl, const char *spec, int sock, int *sfd);
 
 void cmd_server_genvcl(struct vsb *vsb);
 
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index 20b152b..3bd430b 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -103,7 +103,7 @@ client_thread(void *priv)
 		VTCP_myname(fd, mabuf, sizeof mabuf, mpbuf, sizeof mpbuf);
 		vtc_log(vl, 3, "connected fd %d from %s %s to %s",
 		    fd, mabuf, mpbuf, VSB_data(vsb));
-		http_process(vl, c->spec, fd, -1);
+		fd = http_process(vl, c->spec, fd, NULL);
 		vtc_log(vl, 3, "closing fd %d", fd);
 		VTCP_close(&fd);
 	}
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 7dad4ed..32c7e0c 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -54,7 +54,7 @@ struct http {
 	unsigned		magic;
 #define HTTP_MAGIC		0x2f02169c
 	int			fd;
-	int			sfd;
+	int			*sfd;
 	int			timeout;
 	struct vtclog		*vl;
 
@@ -77,14 +77,14 @@ struct http {
 
 #define ONLY_CLIENT(hp, av)						\
 	do {								\
-		if (hp->sfd >= 0)					\
+		if (hp->sfd != NULL)					\
 			vtc_log(hp->vl, 0,				\
 			    "\"%s\" only possible in client", av[0]);	\
 	} while (0)
 
 #define ONLY_SERVER(hp, av)						\
 	do {								\
-		if (hp->sfd < 0)					\
+		if (hp->sfd == NULL)					\
 			vtc_log(hp->vl, 0,				\
 			    "\"%s\" only possible in server", av[0]);	\
 	} while (0)
@@ -344,19 +344,22 @@ http_rxchar_eof(struct http *hp, int n)
 		pfd[0].revents = 0;
 		i = poll(pfd, 1, hp->timeout);
 		if (i < 0)
-			vtc_log(hp->vl, 0, "HTTP rx timeout (%u ms)",
-			    hp->timeout);
+			vtc_log(hp->vl, 0, "HTTP rx timeout (fd:%d %u ms)",
+			    hp->fd, hp->timeout);
 		if (i < 0)
-			vtc_log(hp->vl, 0, "HTTP rx failed (poll: %s)",
-			    strerror(errno));
+			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d poll: %s)",
+			    hp->fd, strerror(errno));
 		assert(i > 0);
+		if (pfd[0].revents & ~POLLIN)
+			vtc_log(hp->vl, 4, "HTTP rx poll (fd:%d revents: %x)",
+			    hp->fd, pfd[0].revents);
 		assert(hp->prxbuf + n < hp->nrxbuf);
 		i = read(hp->fd, hp->rxbuf + hp->prxbuf, n);
 		if (i == 0)
 			return (i);
 		if (i <= 0)
-			vtc_log(hp->vl, 0, "HTTP rx failed (read: %s)",
-			    strerror(errno));
+			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d read: %s)",
+			    hp->fd, strerror(errno));
 		hp->prxbuf += i;
 		hp->rxbuf[hp->prxbuf] = '\0';
 		n -= i;
@@ -1036,7 +1039,7 @@ cmd_http_expect_close(CMD_ARGS)
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	AZ(av[1]);
-	assert(hp->sfd >= 0);
+	assert(hp->sfd != NULL);
 
 	vtc_log(vl, 4, "Expecting close (fd = %d)", hp->fd);
 	while (1) {
@@ -1074,10 +1077,11 @@ cmd_http_accept(CMD_ARGS)
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	AZ(av[1]);
+	assert(hp->sfd != NULL);
 	assert(hp->sfd >= 0);
 	VTCP_close(&hp->fd);
 	vtc_log(vl, 4, "Accepting");
-	hp->fd = accept(hp->sfd, NULL, NULL);
+	hp->fd = accept(*hp->sfd, NULL, NULL);
 	if (hp->fd < 0)
 		vtc_log(vl, 0, "Accepted failed: %s", strerror(errno));
 	vtc_log(vl, 3, "Accepted socket fd is %d", hp->fd);
@@ -1136,11 +1140,12 @@ static const struct cmds http_cmds[] = {
 	{ NULL,			NULL }
 };
 
-void
-http_process(struct vtclog *vl, const char *spec, int sock, int sfd)
+int
+http_process(struct vtclog *vl, const char *spec, int sock, int *sfd)
 {
 	struct http *hp;
 	char *s, *q;
+	int retval;
 
 	(void)sfd;
 	ALLOC_OBJ(hp, HTTP_MAGIC);
@@ -1162,9 +1167,11 @@ http_process(struct vtclog *vl, const char *spec, int sock, int sfd)
 	assert(q > s);
 	AN(s);
 	parse_string(s, http_cmds, hp, vl);
+	retval = hp->fd;
 	VSB_delete(hp->vsb);
 	free(hp->rxbuf);
 	free(hp);
+	return (retval);
 }
 
 /**********************************************************************
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 542a266..d1787b1 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -100,7 +100,7 @@ server_thread(void *priv)
 		if (fd < 0)
 			vtc_log(vl, 0, "Accepted failed: %s", strerror(errno));
 		vtc_log(vl, 3, "accepted fd %d", fd);
-		http_process(vl, s->spec, fd, s->sock);
+		fd = http_process(vl, s->spec, fd, &s->sock);
 		vtc_log(vl, 3, "shutting fd %d", fd);
 		j = shutdown(fd, SHUT_WR);
 		if (!VTCP_Check(j))



From phk at varnish-cache.org  Tue Sep 20 21:29:26 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Sep 2011 23:29:26 +0200
Subject: [master] 1bb41f9 Add a missing derefence.
Message-ID: <E1R67sc-00026b-Cx@project.varnish-software.com>

commit 1bb41f9b58e51adc4d5cc9f4854d12915bb2ba46
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 21:29:03 2011 +0000

    Add a missing derefence.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 32c7e0c..8f96577 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1078,7 +1078,7 @@ cmd_http_accept(CMD_ARGS)
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	AZ(av[1]);
 	assert(hp->sfd != NULL);
-	assert(hp->sfd >= 0);
+	assert(*hp->sfd >= 0);
 	VTCP_close(&hp->fd);
 	vtc_log(vl, 4, "Accepting");
 	hp->fd = accept(*hp->sfd, NULL, NULL);



From phk at varnish-cache.org  Tue Sep 20 22:00:34 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 21 Sep 2011 00:00:34 +0200
Subject: [master] abc5dd2 Go over this code once more, and fix various nits.
Message-ID: <E1R68Mk-0003cO-LW@project.varnish-software.com>

commit abc5dd2a72249fe49d3f1071d8268d3708d7fd3e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 22:00:12 2011 +0000

    Go over this code once more, and fix various nits.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 8f96577..e6aaf33 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -107,7 +107,7 @@ synth_body(const char *len, int rnd)
 	AN(len);
 	i = strtoul(len, NULL, 0);
 	assert(i > 0);
-	b = malloc(i + 1);
+	b = malloc(i + 1L);
 	AN(b);
 	l = k = '!';
 	for (j = 0; j < i; j++) {
@@ -333,7 +333,7 @@ http_splitheader(struct http *hp, int req)
  */
 
 static int
-http_rxchar_eof(struct http *hp, int n)
+http_rxchar(struct http *hp, int n, int eof)
 {
 	int i;
 	struct pollfd pfd[1];
@@ -343,21 +343,25 @@ http_rxchar_eof(struct http *hp, int n)
 		pfd[0].events = POLLIN;
 		pfd[0].revents = 0;
 		i = poll(pfd, 1, hp->timeout);
-		if (i < 0)
+		if (i == 0)
 			vtc_log(hp->vl, 0, "HTTP rx timeout (fd:%d %u ms)",
 			    hp->fd, hp->timeout);
 		if (i < 0)
 			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d poll: %s)",
 			    hp->fd, strerror(errno));
 		assert(i > 0);
-		if (pfd[0].revents & ~POLLIN)
-			vtc_log(hp->vl, 4, "HTTP rx poll (fd:%d revents: %x)",
-			    hp->fd, pfd[0].revents);
 		assert(hp->prxbuf + n < hp->nrxbuf);
 		i = read(hp->fd, hp->rxbuf + hp->prxbuf, n);
-		if (i == 0)
+		if (!(pfd[0].revents & POLLIN))
+			vtc_log(hp->vl, 4,
+			    "HTTP rx poll (fd:%d revents: %x n=%d, i=%d)",
+			    hp->fd, pfd[0].revents, n, i);
+		if (i == 0 && eof)
 			return (i);
-		if (i <= 0)
+		if (i == 0)
+			vtc_log(hp->vl, 0, "HTTP rx EOF (fd:%d read: %s)",
+			    hp->fd, strerror(errno));
+		if (i < 0)
 			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d read: %s)",
 			    hp->fd, strerror(errno));
 		hp->prxbuf += i;
@@ -367,17 +371,6 @@ http_rxchar_eof(struct http *hp, int n)
 	return (1);
 }
 
-static void
-http_rxchar(struct http *hp, int n)
-{
-	int i;
-
-	i = http_rxchar_eof(hp, n);
-	if (i <= 0)
-		vtc_log(hp->vl, 0, "HTTP rx failed (%s)", strerror(errno));
-	assert(i > 0);
-}
-
 static int
 http_rxchunk(struct http *hp)
 {
@@ -386,7 +379,7 @@ http_rxchunk(struct http *hp)
 
 	l = hp->prxbuf;
 	do
-		http_rxchar(hp, 1);
+		(void)http_rxchar(hp, 1, 0);
 	while (hp->rxbuf[hp->prxbuf - 1] != '\n');
 	vtc_dump(hp->vl, 4, "len", hp->rxbuf + l, -1);
 	i = strtoul(hp->rxbuf + l, &q, 16);
@@ -400,12 +393,12 @@ http_rxchunk(struct http *hp)
 	assert(*q == '\0' || vct_islws(*q));
 	hp->prxbuf = l;
 	if (i > 0) {
-		http_rxchar(hp, i);
+		(void)http_rxchar(hp, i, 0);
 		vtc_dump(hp->vl, 4, "chunk",
 		    hp->rxbuf + l, i);
 	}
 	l = hp->prxbuf;
-	http_rxchar(hp, 2);
+	(void)http_rxchar(hp, 2, 0);
 	if(!vct_iscrlf(hp->rxbuf[l]))
 		vtc_log(hp->vl, 0,
 		    "Wrong chunk tail[0] = %02x",
@@ -433,7 +426,7 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
 	p = http_find_header(hh, "content-length");
 	if (p != NULL) {
 		l = strtoul(p, NULL, 0);
-		http_rxchar(hp, l);
+		(void)http_rxchar(hp, l, 0);
 		vtc_dump(hp->vl, 4, "body", hp->body, l);
 		hp->bodyl = l;
 		sprintf(hp->bodylen, "%d", l);
@@ -452,7 +445,7 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
 	if (body) {
 		hp->body = hp->rxbuf + hp->prxbuf;
 		do  {
-			i = http_rxchar_eof(hp, 1);
+			i = http_rxchar(hp, 1, 1);
 			ll += i;
 		} while (i > 0);
 		vtc_dump(hp->vl, 4, "rxeof", hp->body, ll);
@@ -475,7 +468,7 @@ http_rxhdr(struct http *hp)
 	hp->prxbuf = 0;
 	hp->body = NULL;
 	while (1) {
-		http_rxchar(hp, 1);
+		(void)http_rxchar(hp, 1, 0);
 		p = hp->rxbuf + hp->prxbuf - 1;
 		for (i = 0; p > hp->rxbuf; p--) {
 			if (*p != '\n')
@@ -533,7 +526,7 @@ cmd_http_rxresp(CMD_ARGS)
 
 #define TRUST_ME(ptr)   ((void*)(uintptr_t)(ptr))
 
-#define OVERHEAD 64
+#define OVERHEAD 64L
 
 
 static void
@@ -942,7 +935,7 @@ cmd_http_sendhex(CMD_ARGS)
 		if (!vct_ishex(buf[0]) || !vct_ishex(buf[1]))
 			vtc_log(hp->vl, 0, "Illegal Hex char \"%c%c\"",
 			    buf[0], buf[1]);
-		p[i] = strtoul(buf, NULL, 16);
+		p[i] = (uint8_t)strtoul(buf, NULL, 16);
 	}
 	vtc_hexdump(hp->vl, 4, "sendhex", (void*)p, i);
 	j = write(hp->fd, p, i);



From apj at varnish-cache.org  Wed Sep 21 08:03:17 2011
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Wed, 21 Sep 2011 10:03:17 +0200
Subject: [master] ef4c867 Cleanup sphinx warnings
Message-ID: <E1R6Hm1-00053J-R8@project.varnish-software.com>

commit ef4c867ae4635362c6a01596c20113638bc5ea27
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Wed Sep 21 10:02:49 2011 +0200

    Cleanup sphinx warnings

diff --git a/doc/sphinx/conf.py.in b/doc/sphinx/conf.py.in
index d6b98ab..0b07a2b 100644
--- a/doc/sphinx/conf.py.in
+++ b/doc/sphinx/conf.py.in
@@ -64,7 +64,7 @@ release = '@VERSION@'
 
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
-exclude_trees = ['=build']
+exclude_patterns = ['=build','reference/params.rst']
 
 # The reST default role (used for this markup: `text`) to use for all documents.
 #default_role = None
@@ -147,7 +147,7 @@ html_title = "Varnish version @VERSION@ documentation"
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['=static']
+#html_static_path = ['=static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
diff --git a/doc/sphinx/faq/general.rst b/doc/sphinx/faq/general.rst
index d40ef9f..b30f093 100644
--- a/doc/sphinx/faq/general.rst
+++ b/doc/sphinx/faq/general.rst
@@ -337,7 +337,7 @@ Varnish has a feature called **hit for pass**, which is used when Varnish gets a
           * Client 2..N are now given the **hit for pass** object instructing them to go to the backend
 
 The **hit for pass** object will stay cached for the duration of its ttl. This means that subsequent clients requesting /foo will be sent straight to the backend as long as the **hit for pass** object exists.
-The :command:`varnishstat` can tell you how many **hit for pass** objects varnish has served. The default vcl will set ttl for a hit_for_pass object to 120s. But you can override this, using the following logic:
+The :command:`varnishstat` can tell you how many **hit for pass** objects varnish has served. The default vcl will set ttl for a hit_for_pass object to 120s. But you can override this, using the following logic::
 
         sub vcl_fetch {
           if (!obj.cacheable) {
diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst
index d00e9b0..eda1515 100644
--- a/doc/sphinx/installation/upgrade.rst
+++ b/doc/sphinx/installation/upgrade.rst
@@ -42,7 +42,7 @@ becomes
 
 	ban("req.url = " + req.url);
 
-``purge`` does not take any arguments anymore, but can be used in vcl_hit or vcl_miss to purge the item from the cache, where you would reduce ttl to 0 in Varnish 2.1.
+``purge`` does not take any arguments anymore, but can be used in vcl_hit or vcl_miss to purge the item from the cache, where you would reduce ttl to 0 in Varnish 2.1::
 
 	sub vcl_hit {
 	  if (req.request == "PURGE") {
@@ -51,7 +51,7 @@ becomes
 	  }
 	}
 
-becomes
+becomes::
 
 	sub vcl_hit {
 	  if (req.request == "PURGE") {
@@ -68,13 +68,13 @@ becomes
 returns are now done with the ``return()`` function
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``pass``, ``pipe``, ``lookup``, ``deliver``, ``fetch``, ``hash``, ``pipe`` and ``restart`` are no longer keywords, but arguments to ``return()``, so
+``pass``, ``pipe``, ``lookup``, ``deliver``, ``fetch``, ``hash``, ``pipe`` and ``restart`` are no longer keywords, but arguments to ``return()``, so::
 
 	sub vcl_pass {
 	  pass;
 	}
 
-becomes
+becomes::
 
 	sub vcl_pass {
 	  return(pass);
diff --git a/doc/sphinx/reference/params.rst b/doc/sphinx/reference/params.rst
index ce7da4b..f161596 100644
--- a/doc/sphinx/reference/params.rst
+++ b/doc/sphinx/reference/params.rst
@@ -128,6 +128,7 @@ diag_bitmap
 	  0x00020000 - synchronous start of persistence.
 	  0x00040000 - release VCL early.
 	  0x80000000 - do edge-detection on digest.
+
 	Use 0x notation and do the bitor in your head :-)
 
 esi_syntax
@@ -140,6 +141,7 @@ esi_syntax
 	  0x00000002 - Ignore non-esi elements
 	  0x00000004 - Emit parsing debug records
 	  0x00000008 - Force-split parser input (debugging)
+
 	Use 0x notation and do the bitor in your head :-)
 
 expiry_sleep
@@ -203,6 +205,7 @@ gzip_tmp_space
 	  0 - malloc
 	  1 - session workspace
 	  2 - thread workspace
+
 	If you have much gzip/gunzip activity, it may be an advantage to use workspace for these allocations to reduce malloc activity.  Be aware that gzip needs 256+KB and gunzip needs 32+KB of workspace (64+KB if ESI processing).
 
 gzip_window
@@ -216,7 +219,7 @@ http_gzip_support
 	- Default: on
 	- Flags: experimental
 
-	Enable gzip support. When enabled Varnish will compress uncompressed objects before they are stored in the cache. If a client does not support gzip encoding Varnish will uncompress compressed objects on demand. Varnish will also rewrite the Accept-Encoding header of clients indicating support for gzip to::
+	Enable gzip support. When enabled Varnish will compress uncompressed objects before they are stored in the cache. If a client does not support gzip encoding Varnish will uncompress compressed objects on demand. Varnish will also rewrite the Accept-Encoding header of clients indicating support for gzip to:
 
 	Accept-Encoding: gzip
 



From phk at varnish-cache.org  Wed Sep 21 09:55:37 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 21 Sep 2011 11:55:37 +0200
Subject: [master] 63af998 Unrace this testcase a little bit more.
Message-ID: <E1R6JWj-0004vQ-5b@project.varnish-software.com>

commit 63af9987adee7bca0bd008ce2334835295ec3c90
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 21 09:55:09 2011 +0000

    Unrace this testcase a little bit more.

diff --git a/bin/varnishtest/tests/s00002.vtc b/bin/varnishtest/tests/s00002.vtc
index 939d0ee..3cb56cf 100644
--- a/bin/varnishtest/tests/s00002.vtc
+++ b/bin/varnishtest/tests/s00002.vtc
@@ -53,9 +53,9 @@ varnish v1 -vcl {
 	}
 } -start
 
+sema r1 sync 2
 
 client c1 {
-	sema r1 sync 2
 	txreq -url "/"
 	rxresp
 	expect resp.http.foo == "bar"



From phk at varnish-cache.org  Wed Sep 21 09:55:38 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 21 Sep 2011 11:55:38 +0200
Subject: [master] 0d50550 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1R6JWk-0004vW-8z@project.varnish-software.com>

commit 0d5055017657c821d1a5709176eed4dd922d6aff
Merge: 63af998 ef4c867
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 21 09:55:26 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From phk at varnish-cache.org  Wed Sep 21 10:40:14 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 21 Sep 2011 12:40:14 +0200
Subject: [master] 08b8b27 More detail on write errors.
Message-ID: <E1R6KDu-0007bx-Ga@project.varnish-software.com>

commit 08b8b27eedd5d52ccd9ca608cda762332d944ebe
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 21 10:40:05 2011 +0000

    More detail on write errors.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index e6aaf33..024e376 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -143,7 +143,8 @@ http_write(const struct http *hp, int lvl, const char *pfx)
 	vtc_dump(hp->vl, lvl, pfx, VSB_data(hp->vsb), VSB_len(hp->vsb));
 	l = write(hp->fd, VSB_data(hp->vsb), VSB_len(hp->vsb));
 	if (l != VSB_len(hp->vsb))
-		vtc_log(hp->vl, 0, "Write failed: %s", strerror(errno));
+		vtc_log(hp->vl, 0, "Write failed: (%d vs %d) %s",
+		    l, VSB_len(hp->vsb), strerror(errno));
 }
 
 /**********************************************************************



From tfheen at varnish-cache.org  Wed Sep 21 11:15:32 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Wed, 21 Sep 2011 13:15:32 +0200
Subject: [master] 5ba20ed Add varnish-counters(7) man page
Message-ID: <E1R6Km4-0001aF-Ft@project.varnish-software.com>

commit 5ba20ed333c5597c46f51c1371ae7c471f1feee3
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Sep 21 13:03:54 2011 +0200

    Add varnish-counters(7) man page

diff --git a/.gitignore b/.gitignore
index 6cf0950..f35e990 100644
--- a/.gitignore
+++ b/.gitignore
@@ -54,8 +54,10 @@ TAGS
 /lib/libvmod_std/vcc_if.h
 
 # Man-files and binaries
+/man/vsc2rst
 /man/vcl.7
 /man/varnish-cli.7
+/man/varnish-counters.7
 /bin/varnishadm/varnishadm
 /bin/varnishadm/varnishadm.1
 /bin/varnishd/varnishd
diff --git a/man/Makefile.am b/man/Makefile.am
index bc662b6..da95e8c 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -1,6 +1,12 @@
 #
 
-dist_man_MANS = vcl.7 varnish-cli.7
+noinst_PROGRAMS = vsc2rst
+vsc2rst_SOURCES = vsc2rst.c \
+	$(top_srcdir)/include/vsc_fields.h
+
+INCLUDES = -I$(top_srcdir)/include
+
+dist_man_MANS = vcl.7 varnish-cli.7 varnish-counters.7
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 
 vcl.7: $(top_srcdir)/doc/sphinx/reference/vcl.rst \
@@ -23,3 +29,13 @@ else
 	@echo "========================================"
 	@false
 endif
+
+varnish-counters.7: vsc2rst
+if HAVE_RST2MAN
+	./vsc2rst | ${RST2MAN} - $@
+else
+	@echo "========================================"
+	@echo "You need rst2man installed to make dist"
+	@echo "========================================"
+	@false
+endif
diff --git a/man/vsc2rst.c b/man/vsc2rst.c
new file mode 100644
index 0000000..9b89cce
--- /dev/null
+++ b/man/vsc2rst.c
@@ -0,0 +1,65 @@
+
+#include <stdio.h>
+
+#define P(x, ...) printf(x "\n", ##__VA_ARGS__)
+#define VSC_F(n, t, l, f, e, d) printf("%s ? %s\n\t%s\n\n", #n, e, d);
+
+int main(int argc, char **argv)
+{
+	P("================");
+	P("varnish-counters");
+	P("================");
+	P("");
+
+	P("---------------------------------");
+	P("Varnish counter field definitions");
+	P("---------------------------------");
+
+	P(":Author: Tollef Fog Heen");
+	P(":Date:   2011-09-20");
+	P(":Version: 1.0");
+	P(":Manual section: 7");
+	P("");
+
+	P("MAIN COUNTERS");
+	P("=============");
+	P("");
+#define VSC_DO_MAIN
+#include "vsc_fields.h"
+#undef VSC_DO_MAIN
+
+	P("");
+	P("LOCK COUNTERS");
+	P("=============");
+	P("");
+#define VSC_DO_LCK
+#include "vsc_fields.h"
+#undef VSC_DO_LCK
+
+	P("");
+	P("PER MALLOC STORAGE COUNTERS");
+	P("===========================");
+	P("");
+#define VSC_DO_SMA
+#include "vsc_fields.h"
+#undef  VSC_DO_SMA
+
+	P("");
+	P("PER FILE STORAGE COUNTERS");
+	P("=========================");
+	P("");
+#define VSC_DO_SMF
+#include "vsc_fields.h"
+#undef VSC_DO_SMF
+
+	P("");
+	P("PER BACKEND COUNTERS");
+	P("====================");
+	P("");
+#define VSC_DO_VBE
+#include "vsc_fields.h"
+#undef VSC_DO_VBE
+
+	return 0;
+}
+



From tfheen at varnish-cache.org  Wed Sep 21 11:43:02 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Wed, 21 Sep 2011 13:43:02 +0200
Subject: [master] c2e5ccb Look for ncurses/curses.h before curses.h
Message-ID: <E1R6LCg-00008L-O9@project.varnish-software.com>

commit c2e5ccb1325922975eeb2ee1dadaeb6c069e790e
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Sep 21 13:41:58 2011 +0200

    Look for ncurses/curses.h before curses.h
    
    Solaris puts the ncurses header in /usr/include/ncurses, and we need
    that to compile with -Werror.
    
    Fixes: #889

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index d0db9f0..22d21fc 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -34,7 +34,11 @@
 
 #include <sys/time.h>
 
+#ifdef HAVE_NCURSES_CURSES_H
+#include <ncurses/curses.h>
+#elif HAVE_CURSES_H
 #include <curses.h>
+#endif
 #include <errno.h>
 #include <signal.h>
 #include <stdio.h>
diff --git a/configure.ac b/configure.ac
index c8b61e5..ef7d76c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -87,6 +87,7 @@ AC_SUBST(CURSES_LIBS)
 if test "$have_curses" = no; then
 	AC_MSG_WARN([curses not found; some tools will not be built])
 fi
+AC_CHECK_HEADERS([ncurses/curses.h curses.h])
 AM_CONDITIONAL([HAVE_CURSES], [test x$have_curses = xyes])
 
 save_LIBS="${LIBS}"



From tfheen at varnish-cache.org  Wed Sep 21 11:52:42 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Wed, 21 Sep 2011 13:52:42 +0200
Subject: [master] e0ee2a2 Add file_read to the privilege set we need on Solaris
Message-ID: <E1R6LM2-0003QW-LV@project.varnish-software.com>

commit e0ee2a2e69654a9df74aaf3dcadc9639659cf42b
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Sep 21 13:52:03 2011 +0200

    Add file_read to the privilege set we need on Solaris
    
    Fixes: #912

diff --git a/bin/varnishd/mgt_sandbox.c b/bin/varnishd/mgt_sandbox.c
index 0299cfd..a5eee2f 100644
--- a/bin/varnishd/mgt_sandbox.c
+++ b/bin/varnishd/mgt_sandbox.c
@@ -98,6 +98,7 @@ mgt_sandbox(void)
 		 * silently ignore any errors if it doesn't exist
 		 */
 		priv_addset(minimal, "net_access");
+		priv_addset(minimal, "file_read");
 
 #define SETPPRIV(which, set)						\
 		if (setppriv(PRIV_SET, which, set))			\



From tfheen at varnish-cache.org  Wed Sep 21 12:28:14 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Wed, 21 Sep 2011 14:28:14 +0200
Subject: [3.0] 0010135 Allow relational comparisons on REAL type.
Message-ID: <E1R6LuQ-0003u8-1J@project.varnish-software.com>

commit 00101351cd05f444da422ddb770cc1cb11892287
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 31 15:17:43 2011 +0000

    Allow relational comparisons on REAL type.
    
    Fixes	#1002

diff --git a/bin/varnishtest/tests/r01002.vtc b/bin/varnishtest/tests/r01002.vtc
new file mode 100644
index 0000000..12eb636
--- /dev/null
+++ b/bin/varnishtest/tests/r01002.vtc
@@ -0,0 +1,12 @@
+varnishtest "Real relational comparisons"
+
+varnish v1 -vcl {
+	import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ;
+
+	backend foo { .host = "${bad_ip}"; }
+	sub vcl_recv {
+		if (std.random(0,5) < 1.0) {
+			return (pipe);
+		}
+	}
+}
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 0426299..3883dd2 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -886,6 +886,7 @@ static const struct cmps {
 	NUM_REL(INT),
 	NUM_REL(DURATION),
 	NUM_REL(BYTES),
+	NUM_REL(REAL),
 
 	{STRING,	T_EQ,	"!VRT_strcmp(\v1, \v2)" },
 	{STRING,	T_NEQ,	"VRT_strcmp(\v1, \v2)" },



From tfheen at varnish-cache.org  Wed Sep 21 12:28:14 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Wed, 21 Sep 2011 14:28:14 +0200
Subject: [3.0] a75c5e5 When headers are surplus to limits,
	only log the first 20 char.
Message-ID: <E1R6LuQ-0003uJ-7U@project.varnish-software.com>

commit a75c5e516c82f3746bfe219ff26d7a50fe036467
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 31 15:27:41 2011 +0000

    When headers are surplus to limits, only log the first 20 char.

diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index 7fe0a8a..f77ab74 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -522,7 +522,8 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p,
 
 		if (q - p > htc->maxhdr) {
 			VSC_C_main->losthdr++;
-			WSL(w, SLT_LostHeader, fd, "%.*s", q - p, p);
+			WSL(w, SLT_LostHeader, fd, "%.*s",
+			    q - p > 20 ? 20 : q - p, p);
 			return (413);
 		}
 
@@ -547,7 +548,8 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p,
 			hp->nhd++;
 		} else {
 			VSC_C_main->losthdr++;
-			WSL(w, SLT_LostHeader, fd, "%.*s", q - p, p);
+			WSL(w, SLT_LostHeader, fd, "%.*s",
+			    q - p > 20 ? 20 : q - p, p);
 			return (413);
 		}
 	}



From tfheen at varnish-cache.org  Wed Sep 21 12:28:14 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Wed, 21 Sep 2011 14:28:14 +0200
Subject: [3.0] ec3097a Reset the "built vary spec" (also) if we came back from
	the waiting list, otherwise it might turn into garbage.
Message-ID: <E1R6LuQ-0003uZ-DC@project.varnish-software.com>

commit ec3097a5d70b54fffe594cea7f3267ae7c56063b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 07:55:46 2011 +0000

    Reset the "built vary spec" (also) if we came back from the waiting
    list, otherwise it might turn into garbage.
    
    Fixes	#994
    Fixes	#1001

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index df9941c..0f69861 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -1080,10 +1080,12 @@ cnt_lookup(struct sess *sp)
 		AZ(sp->vary_l);
 		AZ(sp->vary_e);
 		(void)WS_Reserve(sp->ws, 0);
-		sp->vary_b = (void*)sp->ws->f;
-		sp->vary_e = (void*)sp->ws->r;
-		sp->vary_b[2] = '\0';
+	} else {
+		AN(sp->ws->r);
 	}
+	sp->vary_b = (void*)sp->ws->f;
+	sp->vary_e = (void*)sp->ws->r;
+	sp->vary_b[2] = '\0';
 
 	oc = HSH_Lookup(sp, &oh);
 



From apj at varnish-cache.org  Thu Sep 22 09:10:38 2011
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Thu, 22 Sep 2011 11:10:38 +0200
Subject: [master] 701e5c3 Format all code examples
Message-ID: <E1R6fIk-00053d-N3@project.varnish-software.com>

commit 701e5c35e7962be65cca4261b7811b33f0244515
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Thu Sep 22 11:10:12 2011 +0200

    Format all code examples

diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst
index eda1515..0ed4f9f 100644
--- a/doc/sphinx/installation/upgrade.rst
+++ b/doc/sphinx/installation/upgrade.rst
@@ -20,11 +20,11 @@ To simplify strings, the %-encoding has been removed. If you need non-printable
 ``log`` moved to the std vmod
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``log`` has moved to the std vmod:
+``log`` has moved to the std vmod::
 
 	log "log something";
 
-becomes
+becomes::
 
 	import std;
 	std.log("log something");
@@ -34,11 +34,11 @@ You only need to import std once.
 purges are now called bans
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``purge()`` and ``purge_url()`` are now respectively ``ban()`` and ``ban_url()``, so you should replace all occurences:
+``purge()`` and ``purge_url()`` are now respectively ``ban()`` and ``ban_url()``, so you should replace all occurences::
 
 	purge("req.url = " req.url);
 
-becomes
+becomes::
 
 	ban("req.url = " + req.url);
 
@@ -84,22 +84,22 @@ becomes::
 ``req.hash`` is replaced with ``hash_data()``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You no longer append to the hash with ``+=``, so
+You no longer append to the hash with ``+=``, so::
 
 	set req.hash += req.url;
 
-becomes
+becomes::
 
 	hash_data(req.url);
 
 ``esi`` is replaced with ``beresp.do_esi``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You no longer enable ESI with ``esi``, so
+You no longer enable ESI with ``esi``, so::
 
 	esi;
 
-in ``vcl_fetch`` becomes
+in ``vcl_fetch`` becomes::
 
 	set beresp.do_esi = true;
 



From tfheen at varnish-cache.org  Thu Sep 22 12:01:30 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:01:30 +0200
Subject: [3.0] c97264d Pass warnings and error flags to distcheck
Message-ID: <E1R6hy6-00046S-Hj@project.varnish-software.com>

commit c97264d4c5fc7cd88bb50f1f193f38e45d3fa417
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Sep 21 14:36:55 2011 +0200

    Pass warnings and error flags to distcheck

diff --git a/Makefile.am b/Makefile.am
index bc0bfc2..dad3621 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -11,6 +11,15 @@ pkgconfig_DATA = varnishapi.pc
 
 EXTRA_DIST = LICENSE autogen.sh varnishapi.pc.in
 
+DISTCHECK_CONFIGURE_FLAGS = \
+    --enable-developer-warnings \
+    --enable-debugging-symbols \
+    --enable-dependency-tracking \
+    --enable-diagnostics \
+    --enable-extra-developer-warnings \
+    --enable-tests \
+    --enable-werror
+
 install-data-local:
 	$(install_sh) -d -m 0755 $(DESTDIR)$(localstatedir)/varnish
 



From tfheen at varnish-cache.org  Thu Sep 22 12:01:31 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:01:31 +0200
Subject: [3.0] 57cfe63 Also remove DIAGNISTICS around the prototype for
	MCF_DumpRst()
Message-ID: <E1R6hy7-00046Y-JJ@project.varnish-software.com>

commit 57cfe6301b56db6a13d25f28c922ee2fb6e5b798
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 08:37:08 2011 +0000

    Also remove DIAGNISTICS around the prototype for MCF_DumpRst()

diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
index 30b90ba..485de26 100644
--- a/bin/varnishd/mgt.h
+++ b/bin/varnishd/mgt.h
@@ -66,9 +66,7 @@ void mgt_cli_close_all(void);
 void MCF_ParamSync(void);
 void MCF_ParamInit(struct cli *);
 void MCF_ParamSet(struct cli *, const char *param, const char *val);
-#ifdef DIAGNOSTICS
 void MCF_DumpRst(void);
-#endif
 
 /* mgt_sandbox.c */
 void mgt_sandbox(void);



From tfheen at varnish-cache.org  Thu Sep 22 12:01:34 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:01:34 +0200
Subject: [3.0] 784a26e Take another stab at #1001/#994
Message-ID: <E1R6hyA-00046h-FW@project.varnish-software.com>

commit 784a26e1deb38179582bcad03805fbb6108f2f5e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 09:59:22 2011 +0000

    Take another stab at #1001/#994

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 0509d99..85e7c76 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -898,6 +898,7 @@ void RES_StreamPoll(const struct sess *sp);
 /* cache_vary.c */
 struct vsb *VRY_Create(const struct sess *sp, const struct http *hp);
 int VRY_Match(struct sess *sp, const uint8_t *vary);
+void VRY_Validate(const uint8_t *vary);
 
 /* cache_vcl.c */
 void VCL_Init(void);
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 0f69861..fc883e3 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -802,6 +802,7 @@ cnt_fetchbody(struct sess *sp)
 		    (void *)WS_Alloc(sp->obj->http->ws, varyl);
 		AN(sp->obj->vary);
 		memcpy(sp->obj->vary, VSB_data(vary), varyl);
+		VRY_Validate(sp->obj->vary);
 		VSB_delete(vary);
 	}
 
@@ -1107,10 +1108,14 @@ cnt_lookup(struct sess *sp)
 	if (oc->flags & OC_F_BUSY) {
 		sp->wrk->stats.cache_miss++;
 
-		if (sp->vary_l != NULL)
+		if (sp->vary_l != NULL) {
+			assert(oc->busyobj->vary == sp->vary_b);
+			VRY_Validate(oc->busyobj->vary);
 			WS_ReleaseP(sp->ws, (void*)sp->vary_l);
-		else
-			WS_Release(sp->ws, 0);
+		} else {
+			AZ(oc->busyobj->vary);
+			WS_Release(sp->ws, 0); 
+		}
 		sp->vary_b = NULL;
 		sp->vary_l = NULL;
 		sp->vary_e = NULL;
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index b070b38..328964a 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -461,7 +461,12 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	AN(oc->flags & OC_F_BUSY);
 	oc->refcnt = 1;
 
-	w->nbusyobj->vary = sp->vary_b;
+	/* XXX: clear w->nbusyobj before use */
+	VRY_Validate(sp->vary_b);
+	if (sp->vary_l != NULL)
+		w->nbusyobj->vary = sp->vary_b;
+	else
+		w->nbusyobj->vary = NULL;
 	oc->busyobj = w->nbusyobj;
 	w->nbusyobj = NULL;
 
diff --git a/bin/varnishd/cache_vary.c b/bin/varnishd/cache_vary.c
index 73d6354..b7a6b01 100644
--- a/bin/varnishd/cache_vary.c
+++ b/bin/varnishd/cache_vary.c
@@ -247,3 +247,13 @@ VRY_Match(struct sess *sp, const uint8_t *vary)
 		sp->vary_l = vsp + 3;
 	return (retval);
 }
+
+void
+VRY_Validate(const uint8_t *vary)
+{
+
+	while (vary[2] != 0) {
+		assert(strlen((const char*)vary+3) == vary[2]);
+		vary += vry_len(vary);
+	}
+}



From tfheen at varnish-cache.org  Thu Sep 22 12:01:38 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:01:38 +0200
Subject: [3.0] a504a3f Document hit_for_pass in the tutorial
Message-ID: <E1R6hyE-00047O-4J@project.varnish-software.com>

commit a504a3f8a29daaf3cf9566d4bd977db5fb566bb4
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Thu Sep 1 15:20:36 2011 +0200

    Document hit_for_pass in the tutorial

diff --git a/doc/sphinx/tutorial/vcl.rst b/doc/sphinx/tutorial/vcl.rst
index f52c1ed..3ed78c5 100644
--- a/doc/sphinx/tutorial/vcl.rst
+++ b/doc/sphinx/tutorial/vcl.rst
@@ -58,7 +58,15 @@ The most common actions to return are these:
 *pass*
  When you return pass the request and subsequent response will be passed to
  and from the backend server. It won't be cached. pass can be returned from
- both vcl_recv and vcl_fetch.
+ vcl_recv
+
+*hit_for_pass*
+  Similar to pass, but accessible from vcl_fetch. Unlike pass, hit_for_pass
+  will create a hitforpass object in the cache. This has the side-effect of
+  caching the decision not to cache. This is to allow would-be uncachable
+  requests to be passed to the backend at the same time. The same logic is
+  not necessary in vcl_recv because this happens before any potential
+  queueing for an object takes place.
 
 *lookup*
   When you return lookup from vcl_recv you tell Varnish to deliver content 



From tfheen at varnish-cache.org  Thu Sep 22 12:01:42 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:01:42 +0200
Subject: [3.0] d2d5427 Move acct_tmp to worker instead of session and report #
	gunziped bytes rather than storage size when we gunzip for delivery.
Message-ID: <E1R6hyI-00048K-Hk@project.varnish-software.com>

commit d2d5427fe12950b8ede6b6f39fc458a15a04b7c4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 08:38:33 2011 +0000

    Move acct_tmp to worker instead of session and report # gunziped
    bytes rather than storage size when we gunzip for delivery.
    
    Fixes	#992

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 85e7c76..f9989fb 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -353,6 +353,8 @@ struct worker {
 #define RES_ESI_CHILD		(1<<5)
 #define RES_GUNZIP		(1<<6)
 
+	/* Temporary accounting */
+	struct acct		acct_tmp;
 };
 
 /* Work Request for worker thread ------------------------------------*/
@@ -601,7 +603,6 @@ struct sess {
 	struct sessmem		*mem;
 
 	struct workreq		workreq;
-	struct acct		acct_tmp;
 	struct acct		acct_req;
 	struct acct		acct_ses;
 
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index fc883e3..6bb8324 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -870,7 +870,7 @@ cnt_fetchbody(struct sess *sp)
 		AN(sp->obj->objcore->ban);
 		HSH_Unbusy(sp);
 	}
-	sp->acct_tmp.fetch++;
+	sp->wrk->acct_tmp.fetch++;
 	sp->step = STP_PREPRESP;
 	return (0);
 }
@@ -927,7 +927,7 @@ cnt_streambody(struct sess *sp)
 	} else {
 		sp->doclose = "Stream error";
 	}
-	sp->acct_tmp.fetch++;
+	sp->wrk->acct_tmp.fetch++;
 	sp->director = NULL;
 	sp->restarts = 0;
 
@@ -969,7 +969,7 @@ cnt_first(struct sess *sp)
 	HTC_Init(sp->htc, sp->ws, sp->fd, params->http_req_size,
 	    params->http_req_hdr_len);
 	sp->wrk->lastused = sp->t_open;
-	sp->acct_tmp.sess++;
+	sp->wrk->acct_tmp.sess++;
 
 	sp->step = STP_WAIT;
 	return (0);
@@ -1277,7 +1277,7 @@ cnt_pass(struct sess *sp)
 		return (0);
 	}
 	assert(sp->handling == VCL_RET_PASS);
-	sp->acct_tmp.pass++;
+	sp->wrk->acct_tmp.pass++;
 	sp->sendbody = 1;
 	sp->step = STP_FETCH;
 	return (0);
@@ -1315,7 +1315,7 @@ cnt_pipe(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
-	sp->acct_tmp.pipe++;
+	sp->wrk->acct_tmp.pipe++;
 	WS_Reset(sp->wrk->ws, NULL);
 	http_Setup(sp->wrk->bereq, sp->wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_PIPE);
@@ -1464,7 +1464,7 @@ cnt_start(struct sess *sp)
 	sp->wrk->stats.client_req++;
 	sp->t_req = TIM_real();
 	sp->wrk->lastused = sp->t_req;
-	sp->acct_tmp.req++;
+	sp->wrk->acct_tmp.req++;
 
 	/* Assign XID and log */
 	sp->xid = ++xids;				/* XXX not locked */
@@ -1619,6 +1619,9 @@ CNT_Session(struct sess *sp)
 	AZ(w->is_gunzip);
 	AZ(w->do_gunzip);
 	AZ(w->do_esi);
+#define ACCT(foo)	AZ(w->acct_tmp.foo);
+#include "acct_fields.h"
+#undef ACCT
 	assert(WRW_IsReleased(w));
 }
 
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 40a8bdf..bed3104 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -378,6 +378,7 @@ VGZ_WrwGunzip(const struct sess *sp, struct vgz *vg, const void *ibuf,
 			return (-1);
 		}
 		if (obufl == *obufp || i == VGZ_STUCK) {
+			sp->wrk->acct_tmp.bodybytes += *obufp;
 			(void)WRW_Write(sp->wrk, obuf, *obufp);
 			(void)WRW_Flush(sp->wrk);
 			*obufp = 0;
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 17d1525..c592909 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -175,7 +175,6 @@ res_WriteGunzipObj(struct sess *sp)
 		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
 		u += st->len;
 
-		sp->acct_tmp.bodybytes += st->len;	/* XXX ? */
 		VSC_C_main->n_objwrite++;
 
 		i = VGZ_WrwGunzip(sp, vg,
@@ -195,7 +194,7 @@ res_WriteGunzipObj(struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 static void
-res_WriteDirObj(struct sess *sp, ssize_t low, ssize_t high)
+res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high)
 {
 	ssize_t u = 0;
 	size_t ptr, off, len;
@@ -227,7 +226,7 @@ res_WriteDirObj(struct sess *sp, ssize_t low, ssize_t high)
 
 		ptr += len;
 
-		sp->acct_tmp.bodybytes += len;
+		sp->wrk->acct_tmp.bodybytes += len;
 #ifdef SENDFILE_WORKS
 		/*
 		 * XXX: the overhead of setting up sendfile is not
@@ -292,7 +291,7 @@ RES_WriteObj(struct sess *sp)
 	 * Send HTTP protocol header, unless interior ESI object
 	 */
 	if (!(sp->wrk->res_mode & RES_ESI_CHILD))
-		sp->acct_tmp.hdrbytes +=
+		sp->wrk->acct_tmp.hdrbytes +=
 		    http_Write(sp->wrk, sp->wrk->resp, 1);
 
 	if (!sp->wantbody)
@@ -348,7 +347,7 @@ RES_StreamStart(struct sess *sp)
 		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
 		    "Content-Length: %s", sp->wrk->h_content_length);
 
-	sp->acct_tmp.hdrbytes +=
+	sp->wrk->acct_tmp.hdrbytes +=
 	    http_Write(sp->wrk, sp->wrk->resp, 1);
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index af422a6..1ebca44 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -80,7 +80,7 @@ static struct lock		stat_mtx;
 void
 SES_Charge(struct sess *sp)
 {
-	struct acct *a = &sp->acct_tmp;
+	struct acct *a = &sp->wrk->acct_tmp;
 
 #define ACCT(foo)				\
 	sp->wrk->stats.s_##foo += a->foo;	\



From tfheen at varnish-cache.org  Thu Sep 22 12:01:46 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:01:46 +0200
Subject: [3.0] 2f57b83 A bit of spit and polish inspired by DocWilcos
	varnishtest patch.
Message-ID: <E1R6hyM-00049H-0I@project.varnish-software.com>

commit 2f57b83a1bc4de75768d9dd17cbf8eec2c49f3b4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 15:19:33 2011 +0000

    A bit of spit and polish inspired by DocWilcos varnishtest patch.

diff --git a/bin/varnishlog/flint.lnt b/bin/varnishlog/flint.lnt
index 149eec1..3a0395c 100644
--- a/bin/varnishlog/flint.lnt
+++ b/bin/varnishlog/flint.lnt
@@ -1,4 +1,5 @@
-
+-efile(451, "../../include/vsc_all.h")
+-e835   // A zero has been given as ___ argument to operator '___'  (<<)
 -e712	// 14      Info    712     Loss of precision (___) (___ to ___)
 -e747	// 16      Info    747     Significant prototype coercion (___) ___ to ___
 
@@ -17,3 +18,4 @@
 -e788	// enum constant '___' not used within defaulted switch
 -e641	// Converting enum '___' to '___'
 
+-esym(785,VSL_tags);	// Sparse array
diff --git a/bin/varnishlog/flint.sh b/bin/varnishlog/flint.sh
index a94710d..dc9e9ba 100755
--- a/bin/varnishlog/flint.sh
+++ b/bin/varnishlog/flint.sh
@@ -1,11 +1,28 @@
 #!/bin/sh
 
+if [ "x$1" = "x-ok" -a -f _.fl ] ; then
+	echo "Saved as reference"
+	mv _.fl _.fl.old
+	exit 0
+fi
+
 flexelint \
-	-I/usr/include \
+	../flint.lnt \
+	flint.lnt \
 	-I. \
 	-I../../include \
 	-I../.. \
-	../flint.lnt \
-	flint.lnt \
+	-I/usr/local/include \
+	-DVARNISH_STATE_DIR=\"foo\" \
 	*.c \
-	../../lib/libvarnishapi/*.c
+	../../lib/libvarnishapi/*.c \
+	2>&1 | tee _.fl
+
+if [ -f _.fl.old ] ; then
+	diff -u _.fl.old _.fl
+fi
+
+if [ "x$1" = "x-ok" ] ; then
+	echo "Saved as reference"
+	mv _.fl _.fl.old
+fi
diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index ed29865..c7749cb 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -46,7 +46,6 @@
 
 #include "libvarnish.h"
 #include "vsl.h"
-#include "vre.h"
 #include "varnishapi.h"
 
 static int	b_flag, c_flag;
@@ -60,7 +59,7 @@ static uint64_t       bitmap[65536];
 #define F_INVCL		(1 << 0)
 
 static void
-h_order_finish(int fd, struct VSM_data *vd)
+h_order_finish(int fd, const struct VSM_data *vd)
 {
 
 	AZ(VSB_finish(ob[fd]));
@@ -72,7 +71,7 @@ h_order_finish(int fd, struct VSM_data *vd)
 }
 
 static void
-clean_order(struct VSM_data *vd)
+clean_order(const struct VSM_data *vd)
 {
 	unsigned u;
 
@@ -321,7 +320,8 @@ main(int argc, char * const *argv)
 			w_arg = optarg;
 			break;
 		case 'm':
-			m_flag = 1; /* fall through */
+			m_flag = 1;
+			/* FALLTHROUGH */
 		default:
 			if (VSL_Arg(vd, c, optarg) > 0)
 				break;
diff --git a/include/varnishapi.h b/include/varnishapi.h
index 0e97755..dcdd74f 100644
--- a/include/varnishapi.h
+++ b/include/varnishapi.h
@@ -109,7 +109,7 @@ int VSM_ReOpen(struct VSM_data *vd, int diag);
 	 *	-1 failure to reopen.
 	 */
 
-unsigned VSM_Seq(struct VSM_data *vd);
+unsigned VSM_Seq(const struct VSM_data *vd);
 	/*
 	 * Return the allocation sequence number
 	 */
@@ -260,6 +260,7 @@ void VSL_NonBlocking(const struct VSM_data *vd, int nb);
 int VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv);
 int VSL_NextLog(const struct VSM_data *lh, uint32_t **pp, uint64_t *bitmap);
 int VSL_Matched(const struct VSM_data *vd, uint64_t bitmap);
+int VSL_Name2Tag(const char *name, int l);
 extern const char *VSL_tags[256];
 
 #endif
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index d9624b9..fc18cb3 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -330,7 +330,7 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 		CHECK_OBJ_NOTNULL(sha, VSM_CHUNK_MAGIC);
 		if (strcmp(sha->class, VSC_CLASS))
 			continue;
-
+		/*lint -save -e525 -e539 */
 #define VSC_F(a,b,c,d,e)
 #define VSC_DONE(a,b,c)
 #define VSC_DO(U,l,t)						\
@@ -343,6 +343,7 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 #undef VSC_F
 #undef VSC_DO
 #undef VSC_DONE
+		/*lint -restore */
 		break;
 	}
 	return (i);
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index f4e633b..6791888 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -206,7 +206,7 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 }
 
 int
-VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *mb)
+VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 {
 	struct vsl *vsl;
 	uint32_t *p;
@@ -268,7 +268,7 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *mb)
 			if (i != VRE_ERROR_NOMATCH)
 				continue;
 		}
-		if (mb != NULL) {
+		if (bits != NULL) {
 			struct vsl_re_match *vrm;
 			int j = 0;
 			VTAILQ_FOREACH(vrm, &vsl->matchers, next) {
@@ -276,7 +276,7 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *mb)
 					i = VRE_exec(vrm->re, VSL_DATA(p),
 						     VSL_LEN(p), 0, 0, NULL, 0);
 					if (i >= 0)
-						*mb |= 1 << j;
+						*bits |= (uintmax_t)1 << j;
 				}
 				j++;
 			}
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 49930c8..21cd8e9 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -41,7 +41,6 @@
 #include <unistd.h>
 
 #include "vas.h"
-#include "vin.h"
 #include "vre.h"
 #include "vbm.h"
 #include "miniobj.h"
@@ -50,6 +49,34 @@
 #include "vsm_api.h"
 #include "vsl_api.h"
 
+/*--------------------------------------------------------------------
+ * Look up a tag
+ *   0..255	tag number
+ *   -1		no tag matches
+ *   -2		multiple tags match
+ */
+
+int
+VSL_Name2Tag(const char *name, int l)
+{
+	int i, n;
+
+	if (l == -1)
+		l = strlen(name);
+	n = -1;
+	for (i = 0; i < 256; i++) {
+		if (VSL_tags[i] != NULL &&
+		    !strncasecmp(name, VSL_tags[i], l)) {
+			if (n == -1)
+				n = i;
+			else
+				n = -2;
+		}
+	}
+	return (n);
+}
+
+
 /*--------------------------------------------------------------------*/
 
 static int
@@ -99,8 +126,8 @@ vsl_IX_arg(const struct VSM_data *vd, const char *opt, int arg)
 static int
 vsl_ix_arg(const struct VSM_data *vd, const char *opt, int arg)
 {
-	int i, j, l;
-	const char *b, *e, *p, *q;
+	int i, l;
+	const char *b, *e;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	/* If first option is 'i', set all bits for supression */
@@ -120,24 +147,17 @@ vsl_ix_arg(const struct VSM_data *vd, const char *opt, int arg)
 			e++;
 		while (isspace(b[l - 1]))
 			l--;
-		for (i = 0; i < 256; i++) {
-			if (VSL_tags[i] == NULL)
-				continue;
-			p = VSL_tags[i];
-			q = b;
-			for (j = 0; j < l; j++)
-				if (tolower(*q++) != tolower(*p++))
-					break;
-			if (j != l || *p != '\0')
-				continue;
-
+		i = VSL_Name2Tag(b, l);
+		if (i >= 0) {
 			if (arg == 'x')
 				vbit_set(vd->vsl->vbm_supress, i);
 			else
 				vbit_clr(vd->vsl->vbm_supress, i);
-			break;
-		}
-		if (i == 256) {
+		} else if (i == -2) {
+			fprintf(stderr,
+			    "\"%*.*s\" matches multiple tags\n", l, l, b);
+			return (-1);
+		} else {
 			fprintf(stderr,
 			    "Could not match \"%*.*s\" to any tag\n", l, l, b);
 			return (-1);
@@ -148,19 +168,6 @@ vsl_ix_arg(const struct VSM_data *vd, const char *opt, int arg)
 
 /*--------------------------------------------------------------------*/
 
-static int
-name2tag(const char *n)
-{
-	int i;
-
-	for (i = 0; i < 256; i++) {
-		if (VSL_tags[i] == NULL)
-			continue;
-		if (!strcasecmp(n, VSL_tags[i]))
-			return (i);
-	}
-	return (-1);
-}
 
 static int
 vsl_m_arg(const struct VSM_data *vd, const char *opt)
@@ -171,8 +178,6 @@ vsl_m_arg(const struct VSM_data *vd, const char *opt)
 	int erroroffset;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	ALLOC_OBJ(m, VSL_RE_MATCH_MAGIC);
-	AN(m);
 
 	if (!strchr(opt, ':')) {
 		fprintf(stderr, "No : found in -o option %s\n", opt);
@@ -185,10 +190,13 @@ vsl_m_arg(const struct VSM_data *vd, const char *opt)
 	*regex = '\0';
 	regex++;
 
-	m->tag = name2tag(o);
-	if (m->tag == -1) {
+	ALLOC_OBJ(m, VSL_RE_MATCH_MAGIC);
+	AN(m);
+	m->tag = VSL_Name2Tag(o, -1);
+	if (m->tag < 0) {
 		fprintf(stderr, "Illegal tag %s specified\n", o);
 		free(o);
+		FREE_OBJ(m);
 		return (-1);
 	}
 	/* Get tag, regex */
@@ -196,6 +204,7 @@ vsl_m_arg(const struct VSM_data *vd, const char *opt)
 	if (m->re == NULL) {
 		fprintf(stderr, "Illegal regex: %s\n", error);
 		free(o);
+		FREE_OBJ(m);
 		return (-1);
 	}
 	vd->vsl->num_matchers++;
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index efb7592..888a0d7 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -224,7 +224,7 @@ VSM_Open(struct VSM_data *vd, int diag)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	AZ(vd->VSM_head);
 	if (!vd->n_opt)
-		VSM_n_Arg(vd, "");
+		(void)VSM_n_Arg(vd, "");
 	return (vsm_open(vd, diag));
 }
 
@@ -331,7 +331,7 @@ VSM_iter0(struct VSM_data *vd)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	vd->alloc_seq = vd->VSM_head->alloc_seq;
 	while (vd->alloc_seq == 0) {
-		usleep(50000);
+		(void)usleep(50000);
 		vd->alloc_seq = vd->VSM_head->alloc_seq;
 	}
 	CHECK_OBJ_NOTNULL(&vd->VSM_head->head, VSM_CHUNK_MAGIC);
@@ -358,7 +358,7 @@ VSM_itern(const struct VSM_data *vd, struct VSM_chunk **pp)
 
 /*--------------------------------------------------------------------*/
 unsigned
-VSM_Seq(struct VSM_data *vd)
+VSM_Seq(const struct VSM_data *vd)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);



From tfheen at varnish-cache.org  Thu Sep 22 12:01:53 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:01:53 +0200
Subject: [3.0] bc40c1a Ignore invalid HTTP headers
Message-ID: <E1R6hyT-0004AE-53@project.varnish-software.com>

commit bc40c1afc3ffba2d64a3bdfd60380aa9bd589ace
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Sep 5 12:05:24 2011 +0200

    Ignore invalid HTTP headers

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 9ebb096..25c63ee 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -250,7 +250,7 @@ static int
 collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
     const char *ptr, unsigned len)
 {
-	const char *end, *next;
+	const char *end, *next, *split;
 
 	assert(spec & VSL_S_BACKEND);
 	end = ptr + len;
@@ -330,18 +330,19 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		break;
 
 	case SLT_TxHeader:
+		split = strchr(ptr, ':');
 		if (!lp->active)
 			break;
+		if (split == NULL)
+			break;
 		if (isprefix(ptr, "authorization:", end, &next) &&
 		    isprefix(next, "basic", end, &next)) {
 			lp->df_u = trimline(next, end);
 		} else {
 			struct hdr *h;
-			const char *split;
 			size_t l;
 			h = malloc(sizeof(struct hdr));
 			AN(h);
-			split = strchr(ptr, ':');
 			AN(split);
 			l = strlen(split);
 			h->key = trimline(ptr, split-1);
@@ -369,7 +370,7 @@ static int
 collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
     const char *ptr, unsigned len)
 {
-	const char *end, *next;
+	const char *end, *next, *split;
 	long l;
 	time_t t;
 
@@ -437,8 +438,11 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 
 	case SLT_TxHeader:
 	case SLT_RxHeader:
+		split = strchr(ptr, ':');
 		if (!lp->active)
 			break;
+		if (split == NULL)
+			break;
 		if (tag == SLT_RxHeader &&
 		    isprefix(ptr, "authorization:", end, &next) &&
 		    isprefix(next, "basic", end, &next)) {
@@ -446,10 +450,8 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 			lp->df_u = trimline(next, end);
 		} else {
 			struct hdr *h;
-			const char *split;
 			h = malloc(sizeof(struct hdr));
 			AN(h);
-			split = strchr(ptr, ':');
 			AN(split);
 			h->key = trimline(ptr, split);
 			h->value = trimline(split+1, end);



From tfheen at varnish-cache.org  Thu Sep 22 12:01:53 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:01:53 +0200
Subject: [3.0] 8d3ad6e Small optimization: don't process header when inactive
Message-ID: <E1R6hyT-0004Af-Es@project.varnish-software.com>

commit 8d3ad6e2ceb05d017995c57951f0df1eb3101aa3
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Sep 5 13:04:37 2011 +0200

    Small optimization: don't process header when inactive

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 25c63ee..28f6f21 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -330,9 +330,9 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		break;
 
 	case SLT_TxHeader:
-		split = strchr(ptr, ':');
 		if (!lp->active)
 			break;
+		split = strchr(ptr, ':');
 		if (split == NULL)
 			break;
 		if (isprefix(ptr, "authorization:", end, &next) &&
@@ -438,9 +438,9 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 
 	case SLT_TxHeader:
 	case SLT_RxHeader:
-		split = strchr(ptr, ':');
 		if (!lp->active)
 			break;
+		split = strchr(ptr, ':');
 		if (split == NULL)
 			break;
 		if (tag == SLT_RxHeader &&



From tfheen at varnish-cache.org  Thu Sep 22 12:01:53 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:01:53 +0200
Subject: [3.0] 050d0ff Add documentation of string syntax, rollback,
	panic and synthetic keywords.
Message-ID: <E1R6hyT-0004Aw-Lf@project.varnish-software.com>

commit 050d0ff33dfb7667c77dbc08d7ff00fd91c45633
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Sep 5 20:09:59 2011 +0200

    Add documentation of string syntax, rollback, panic and synthetic keywords.
    
    Fixes: #991

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 4f005d0..82e20f4 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -39,6 +39,12 @@ In addition to the C-like assignment (=), comparison (==, !=) and
 boolean (!, && and \|\|) operators, VCL supports both regular
 expression and ACL matching using the ~ and the !~ operators.
 
+Basic strings are enclosed in " ... " and uses URL-style %-escapes.
+
+Long strings are enclosed in {" ... "} and do not have an escape
+character. They may contain any character including ", newline and
+other control characters except for the NUL (0x00) character.
+
 Unlike C and Perl, the backslash (\) character has no special meaning
 in strings in VCL, so it can be freely used in regular expressions
 without doubling.
@@ -55,6 +61,15 @@ You can use the *set* keyword to arbitrary HTTP headers. You can
 remove headers with the *remove* or *unset* keywords, which are
 synonym.
 
+You can use the *rollback* keyword to revert any changes to req at
+any time.
+
+The *synthetic* keyword is used to produce a synthetic response
+body in vcl_error. It takes a single string as argument.
+
+You can force a crash of the client process with the *panic* keyword.
+*panic* takes a string as argument.
+
 The ``return(action)`` keyword terminates the subroutine. *action* can be,
 depending on context one of
 



From tfheen at varnish-cache.org  Thu Sep 22 12:01:54 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:01:54 +0200
Subject: [3.0] 81c7a6e Unlink -s file with autogenerated names
Message-ID: <E1R6hyU-0004BV-C1@project.varnish-software.com>

commit 81c7a6e729e8f2d833699c3f79384f5d201a30e4
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 6 13:49:59 2011 +0200

    Unlink -s file with autogenerated names
    
    If just -s file or -s file,/path/to/directory is given, the file
    stevedore will generate a name.  Make sure to unlink this to not leak
    disk space over time.
    
    Fixes: #1008

diff --git a/bin/varnishd/stevedore_utils.c b/bin/varnishd/stevedore_utils.c
index a38588d..bd4e368 100644
--- a/bin/varnishd/stevedore_utils.c
+++ b/bin/varnishd/stevedore_utils.c
@@ -108,6 +108,7 @@ STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx)
 		if (fd < 0)
 			ARGV_ERR("(%s) \"%s\" mkstemp(%s) failed (%s)\n",
 			    ctx, fn, buf, strerror(errno));
+		AZ(unlink(buf));
 		*fnp = strdup(buf);
 		AN(*fnp);
 		retval = 2;



From tfheen at varnish-cache.org  Thu Sep 22 12:01:54 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:01:54 +0200
Subject: [3.0] efaea24 Make the default -s file size 100MB rather than 50% of
	free disk space
Message-ID: <E1R6hyU-0004Bh-GL@project.varnish-software.com>

commit efaea244aecc402a9fe92e9820ae50572fceb4fc
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 6 13:51:04 2011 +0200

    Make the default -s file size 100MB rather than 50% of free disk space

diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index ea78526..7687179 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -116,7 +116,7 @@ smf_initfile(struct stevedore *st, struct smf_sc *sc, const char *size)
 	/* XXX: force block allocation here or in open ? */
 }
 
-static const char default_size[] = "50%";
+static const char default_size[] = "100M";
 static const char default_filename[] = ".";
 
 static void



From tfheen at varnish-cache.org  Thu Sep 22 12:01:58 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:01:58 +0200
Subject: [3.0] 693b27c Fix up default format string in man page and code
	comments
Message-ID: <E1R6hyY-0004CU-18@project.varnish-software.com>

commit 693b27c3332e1dda737976f405b3a7e12712ef7b
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 6 15:09:14 2011 +0200

    Fix up default format string in man page and code comments
    
    We're actually using %s, not %>s, since we don't support >.
    
    Thanks to Kai for patch
    Fixes: #999

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 70e915f..7c35c3e 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -31,7 +31,7 @@
  * Obtain log data from the shared memory log, order it by session ID, and
  * display it in Apache / NCSA combined log format:
  *
- *	%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"
+ *	%h %l %u %t "%r" %s %b "%{Referer}i" "%{User-agent}i"
  *
  * where the fields are defined as follows:
  *
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 65f08dd..a4bac0a 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -52,7 +52,7 @@ The following options are available:
 -F format   Specify the log format used. If no format is specified the  
    	    default log format is used. Currently it is:
 
-            %h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"
+            %h %l %u %t "%r" %s %b "%{Referer}i" "%{User-agent}i"
 
 	    Supported formatters are:
 



From tfheen at varnish-cache.org  Thu Sep 22 12:02:18 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:18 +0200
Subject: [3.0] 4077127 Make it possible for VCL initializtion to fail
	gracefully,
	particularly so VMOD loading can emit sensible diagnostics.
Message-ID: <E1R6hys-0004GZ-Lu@project.varnish-software.com>

commit 4077127a21db2145bb3224153b72c610343a7a9e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 19:30:48 2011 +0000

    Make it possible for VCL initializtion to fail gracefully, particularly
    so VMOD loading can emit sensible diagnostics.

diff --git a/bin/varnishd/cache_vcl.c b/bin/varnishd/cache_vcl.c
index 49f3fbb..ac712fb 100644
--- a/bin/varnishd/cache_vcl.c
+++ b/bin/varnishd/cache_vcl.c
@@ -170,10 +170,15 @@ VCL_Load(const char *fn, const char *name, struct cli *cli)
 		FREE_OBJ(vcl);
 		return (1);
 	}
+	if (vcl->conf->init_vcl(cli)) {
+		VCLI_Out(cli, "VCL \"%s\" Failed to initialize", name);
+		(void)dlclose(vcl->dlh);
+		FREE_OBJ(vcl);
+		return (1);
+	}
 	REPLACE(vcl->name, name);
-	VTAILQ_INSERT_TAIL(&vcl_head, vcl, list);
 	VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name);
-	vcl->conf->init_vcl(cli);
+	VTAILQ_INSERT_TAIL(&vcl_head, vcl, list);
 	(void)vcl->conf->init_func(NULL);
 	Lck_Lock(&vcl_mtx);
 	if (vcl_active == NULL)
diff --git a/bin/varnishd/cache_vrt_vmod.c b/bin/varnishd/cache_vrt_vmod.c
index 4a4d230..84a2bb7 100644
--- a/bin/varnishd/cache_vrt_vmod.c
+++ b/bin/varnishd/cache_vrt_vmod.c
@@ -60,12 +60,12 @@ struct vmod {
 
 static VTAILQ_HEAD(,vmod)	vmods = VTAILQ_HEAD_INITIALIZER(vmods);
 
-void
-VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, const char *path)
+int
+VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
+    const char *path, struct cli *cli)
 {
 	struct vmod *v;
-	void *x;
-	const int *i;
+	void *x, *y, *z;
 
 	ASSERT_CLI();
 
@@ -76,31 +76,46 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, const char *path)
 		ALLOC_OBJ(v, VMOD_MAGIC);
 		AN(v);
 
-		VTAILQ_INSERT_TAIL(&vmods, v, list);
-		VSC_C_main->vmods++;
-
-		REPLACE(v->nm, nm);
-		REPLACE(v->path, path);
-
-		v->hdl = dlopen(v->path, RTLD_NOW | RTLD_LOCAL);
-		if (! v->hdl) {
-			char buf[1024];
-			sprintf(buf, "dlopen failed (child process lacks permission?): %.512s", dlerror());
-			VAS_Fail(__func__, __FILE__, __LINE__, buf, 0, 0);
+		v->hdl = dlopen(path, RTLD_NOW | RTLD_LOCAL);
+		if (v->hdl == NULL) {
+			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
+			VCLI_Out(cli, "dlopen() failed: %s\n", dlerror());
+			VCLI_Out(cli, "Check child process permissions.\n"); 
+			FREE_OBJ(v);
+			return (1);
 		}
 
 		x = dlsym(v->hdl, "Vmod_Name");
+		y = dlsym(v->hdl, "Vmod_Len");
+		z = dlsym(v->hdl, "Vmod_Func");
+		if (x == NULL || y == NULL || z == NULL) {
+			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
+			VCLI_Out(cli, "VMOD symbols not found\n");
+			VCLI_Out(cli, "Check relative pathnames.\n");
+			(void)dlclose(v->hdl);
+			FREE_OBJ(v);
+			return (1);
+		}
 		AN(x);
-		/* XXX: check that name is correct */
+		AN(y);
+		AN(z);
+		if (strcmp(x, nm)) {
+			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
+			VCLI_Out(cli, "File contain wrong VMOD (\"%s\")\n", x);
+			VCLI_Out(cli, "Check relative pathnames ?.\n");
+			(void)dlclose(v->hdl);
+			FREE_OBJ(v);
+			return (1);
+		}
 
-		x = dlsym(v->hdl, "Vmod_Len");
-		AN(x);
-		i = x;
-		v->funclen = *i;
+		v->funclen = *(const int *)y;
+		v->funcs = z;
 
-		x = dlsym(v->hdl, "Vmod_Func");
-		AN(x);
-		v->funcs = x;
+		REPLACE(v->nm, nm);
+		REPLACE(v->path, path);
+
+		VSC_C_main->vmods++;
+		VTAILQ_INSERT_TAIL(&vmods, v, list);
 	}
 
 	assert(len == v->funclen);
@@ -108,6 +123,7 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, const char *path)
 	v->ref++;
 
 	*hdl = v;
+	return (0);
 }
 
 void
diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index 2664931..ebdf57b 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -385,7 +385,7 @@ start_child(struct cli *cli)
 	mgt_cli_start_child(child_cli_in, child_VCLI_Out);
 	child_pid = pid;
 	if (mgt_push_vcls_and_start(&u, &p)) {
-		REPORT(LOG_ERR, "Pushing vcls failed: %s", p);
+		REPORT(LOG_ERR, "Pushing vcls failed:\n%s", p);
 		free(p);
 		child_state = CH_RUNNING;
 		mgt_stop_child();
diff --git a/include/vrt.h b/include/vrt.h
index cfafe75..9f0228f 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -183,8 +183,8 @@ void VRT_init_dir(struct cli *, struct director **, const char *name,
 void VRT_fini_dir(struct cli *, struct director *);
 
 /* VMOD/Modules related */
-void VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
-    const char *path);
+int VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
+    const char *path, struct cli *cli);
 void VRT_Vmod_Fini(void **hdl);
 
 struct vmod_priv;
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 5407a51..fde8bb2 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -712,7 +712,7 @@ fo.write("""
 struct sess;
 struct cli;
 
-typedef void vcl_init_f(struct cli *);
+typedef int vcl_init_f(struct cli *);
 typedef void vcl_fini_f(struct cli *);
 typedef int vcl_func_f(struct sess *sp);
 """)
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index b0019ca..c2a52d4 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -310,9 +310,10 @@ static void
 EmitInitFunc(const struct vcc *tl)
 {
 
-	Fc(tl, 0, "\nstatic void\nVGC_Init(struct cli *cli)\n{\n\n");
+	Fc(tl, 0, "\nstatic int\nVGC_Init(struct cli *cli)\n{\n\n");
 	AZ(VSB_finish(tl->fi));
 	VSB_cat(tl->fc, VSB_data(tl->fi));
+	Fc(tl, 0, "\treturn(0);\n");
 	Fc(tl, 0, "}\n");
 }
 
@@ -321,7 +322,7 @@ EmitFiniFunc(const struct vcc *tl)
 {
 	unsigned u;
 
-	Fc(tl, 0, "\nstatic void\nVGC_Fini(struct cli *cli)\n{\n\n");
+	Fc(tl, 0, "\nstatic int\nVGC_Fini(struct cli *cli)\n{\n\n");
 
 	/*
 	 * We do this here, so we are sure they happen before any
@@ -332,6 +333,7 @@ EmitFiniFunc(const struct vcc *tl)
 
 	AZ(VSB_finish(tl->ff));
 	VSB_cat(tl->fc, VSB_data(tl->ff));
+	Fc(tl, 0, "\treturn(0);\n");
 	Fc(tl, 0, "}\n");
 }
 
diff --git a/lib/libvcl/vcc_vmod.c b/lib/libvcl/vcc_vmod.c
index 35cc702..f74f874 100644
--- a/lib/libvcl/vcc_vmod.c
+++ b/lib/libvcl/vcc_vmod.c
@@ -102,13 +102,15 @@ vcc_ParseImport(struct vcc *tl)
 
 	Fh(tl, 0, "static void *VGC_vmod_%.*s;\n", PF(mod));
 
-	Fi(tl, 0, "\tVRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod));
+	Fi(tl, 0, "\tif (VRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod));
 	Fi(tl, 0, "\t    &Vmod_Func_%.*s,\n", PF(mod));
 	Fi(tl, 0, "\t    sizeof(Vmod_Func_%.*s),\n", PF(mod));
 	Fi(tl, 0, "\t    \"%.*s\",\n", PF(mod));
 	Fi(tl, 0, "\t    ");
 	EncString(tl->fi, fn, NULL, 0);
-	Fi(tl, 0, ");\n");
+	Fi(tl, 0, ",\n\t    ");
+	Fi(tl, 0, "cli))\n");
+	Fi(tl, 0, "\t\treturn(1);\n");
 
 	SkipToken(tl, ';');
 



From tfheen at varnish-cache.org  Thu Sep 22 12:02:32 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:32 +0200
Subject: [3.0] 784396f Don't leave pid file in weird error case.
Message-ID: <E1R6hz6-0004Id-HG@project.varnish-software.com>

commit 784396fd1b3c2481ed0d99c294afb3102f64747d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 20:16:30 2011 +0000

    Don't leave pid file in weird error case.

diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index ebdf57b..9d6966a 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -595,8 +595,10 @@ MGT_Run(void)
 		REPORT0(LOG_ERR, "No VCL loaded yet");
 	else if (!d_flag) {
 		start_child(NULL);
-		if (child_state == CH_STOPPED)
-			exit(2);
+		if (child_state == CH_STOPPED) {
+			exit_status = 2;
+			return;
+		}
 	}
 
 	i = vev_schedule(mgt_evb);



From tfheen at varnish-cache.org  Thu Sep 22 12:02:39 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:39 +0200
Subject: [3.0] 4442b28 Add Vmod_Id handle which VMOD can identify themselves
	to VRT with.
Message-ID: <E1R6hzD-0004Jj-2W@project.varnish-software.com>

commit 4442b286af0fca9ee6a3073cb45ef3aa39e4ac17
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 8 11:53:05 2011 +0000

    Add Vmod_Id handle which VMOD can identify themselves to VRT with.

diff --git a/bin/varnishd/cache_vrt_vmod.c b/bin/varnishd/cache_vrt_vmod.c
index 84a2bb7..bd2e172 100644
--- a/bin/varnishd/cache_vrt_vmod.c
+++ b/bin/varnishd/cache_vrt_vmod.c
@@ -56,6 +56,7 @@ struct vmod {
 	void			*hdl;
 	const void		*funcs;
 	int			funclen;
+	const void		*idptr;
 };
 
 static VTAILQ_HEAD(,vmod)	vmods = VTAILQ_HEAD_INITIALIZER(vmods);
@@ -65,12 +66,12 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
     const char *path, struct cli *cli)
 {
 	struct vmod *v;
-	void *x, *y, *z;
+	void *x, *y, *z, *w;
 
 	ASSERT_CLI();
 
 	VTAILQ_FOREACH(v, &vmods, list)
-		if (!strcmp(v->nm, nm))
+		if (!strcmp(v->nm, nm))	// Also path, len ?
 			break;
 	if (v == NULL) {
 		ALLOC_OBJ(v, VMOD_MAGIC);
@@ -88,7 +89,8 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 		x = dlsym(v->hdl, "Vmod_Name");
 		y = dlsym(v->hdl, "Vmod_Len");
 		z = dlsym(v->hdl, "Vmod_Func");
-		if (x == NULL || y == NULL || z == NULL) {
+		w = dlsym(v->hdl, "Vmod_Id");
+		if (x == NULL || y == NULL || z == NULL || w == NULL) {
 			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
 			VCLI_Out(cli, "VMOD symbols not found\n");
 			VCLI_Out(cli, "Check relative pathnames.\n");
@@ -99,6 +101,7 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 		AN(x);
 		AN(y);
 		AN(z);
+		AN(w);
 		if (strcmp(x, nm)) {
 			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
 			VCLI_Out(cli, "File contain wrong VMOD (\"%s\")\n", x);
@@ -116,6 +119,7 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 
 		VSC_C_main->vmods++;
 		VTAILQ_INSERT_TAIL(&vmods, v, list);
+		v->idptr = w;
 	}
 
 	assert(len == v->funclen);
diff --git a/lib/libvmod_std/vmod.py b/lib/libvmod_std/vmod.py
index 83f8eca..1b0f1c0 100755
--- a/lib/libvmod_std/vmod.py
+++ b/lib/libvmod_std/vmod.py
@@ -308,5 +308,9 @@ fc.write("\n");
 fc.write('const char * const Vmod_Spec[] = {\n' + slist + '\t0\n};\n')
 
 fc.write('const char Vmod_Varnish_ABI[] = VMOD_ABI_Version;\n')
+
+fh.write('extern const void * const Vmod_Id;\n')
+fc.write('const void * const Vmod_Id = &Vmod_Id;\n')
+
 fc.write("\n")
 



From tfheen at varnish-cache.org  Thu Sep 22 12:02:45 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:45 +0200
Subject: [3.0] bb4163f Tabulate probe properties and add all missing ones
Message-ID: <E1R6hzJ-0004Kx-DL@project.varnish-software.com>

commit bb4163fb218445eeec60ac62b8ab70e1437aff40
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Fri Sep 9 17:43:31 2011 +0200

    Tabulate probe properties and add all missing ones
    
    Really fixes #997

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index aae16bc..517f4fb 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -280,12 +280,36 @@ Backend probes
 
 Backends can be probed to see whether they should be considered
 healthy or not.  The return status can also be checked by using
-req.backend.healthy .window is how many of the latest polls we
-examine, while .threshold is how many of those must have succeeded for
-us to consider the backend healthy.  .initial is how many of the
-probes are considered good when Varnish starts - defaults to the same
-amount as the threshold. .expected_response is the expected backend
-HTTP response code.
+req.backend.healthy.
+
+Probes take the following parameters:
+
+.url
+  Specify a URL to request from the backend.
+  Defaults to "/".
+.request
+  Specify a full HTTP request using multiple strings.
+  .request will have \r\n automatically inserted after every string.
+  If specified, .request will take precedence over .url.
+.window
+  How many of the latest polls we examine to determine backend health.
+  Defaults to 8.
+.threshold 
+  How many of the polls in .window must have succeeded for us to consider
+  the backend healthy.
+  Defaults to 3.
+.initial
+  How many of the probes are considered good when Varnish starts.
+  Defaults to the same amount as the threshold.
+.expected_response
+  The expected backend HTTP response code.
+  Defaults to 200.
+.interval
+  Defines how often the probe should check the backend.
+  Default is every 5 seconds.
+.timeout
+  How fast each probe times out.
+  Default is 2 seconds.
 
 A backend with a probe can be defined like this, together with the
 backend or director:::



From tfheen at varnish-cache.org  Thu Sep 22 12:02:45 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:45 +0200
Subject: [3.0] 3b5e7c4 Add undocumented variables
Message-ID: <E1R6hzJ-0004LG-Mz@project.varnish-software.com>

commit 3b5e7c4b2e1232ed8e1be5da24e0fe9edd8320ec
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Sun Sep 11 12:52:01 2011 +0200

    Add undocumented variables

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index e3169ea..9fd526c 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -687,6 +687,9 @@ The following variables are available while processing a request:
 client.ip
   The client's IP address.
 
+client.identity
+  Identification of the client, used to load balance in the client director.
+
 server.hostname
   The host name of the server.
 
@@ -733,6 +736,21 @@ req.hash_ignore_busy
 req.can_gzip
   Does the client accept the gzip transfer encoding.
 
+req.restarts
+  A count of how many times this request has been restarted.
+
+req.esi
+  True if the request is an ESI request.
+
+req.esi_level
+  A count of how many levels of ESI requests we're currently at.
+
+req.grace
+  Set to a period to enable grace.
+
+req.xid
+  Unique ID of this request.
+
 The following variables are available while preparing a backend
 request (either for a cache miss or for pass or pipe mode):
 
@@ -792,6 +810,25 @@ beresp.response
 beresp.ttl
   The object's remaining time to live, in seconds. beresp.ttl is writable.
 
+beresp.grace
+  Set to a period to enable grace.
+
+beresp.saintmode
+  Set to a period to enable saint mode.
+
+beresp.backend.name
+  Name of the backend this response was fetched from.
+
+beresp.backend.ip
+  IP of the backend this response was fetched from.
+
+beresp.backend.port
+  Port of the backend this response was fetched from.
+
+beresp.storage
+  Set to force Varnish to save this object to a particular storage
+  backend.
+
 After the object is entered into the cache, the following (mostly
 read-only) variables are available when the object has been located in
 cache, typically in vcl_hit and vcl_deliver.
@@ -816,6 +853,12 @@ obj.hits
   The approximate number of times the object has been delivered. A value 
   of 0 indicates a cache miss.
 
+obj.grace
+  The object's grace period in seconds. obj.grace is writable.
+
+obj.http.header
+  The corresponding HTTP header.
+
 The following variables are available while determining the hash key
 of an object:
 
@@ -1000,8 +1043,8 @@ and Per Buer.
 COPYRIGHT
 =========
 
-This document is licensed under the same licence as Varnish
-itself. See LICENCE for details.
+This document is licensed under the same license as Varnish
+itself. See LICENSE for details.
 
 * Copyright (c) 2006 Verdens Gang AS
 * Copyright (c) 2006-2011 Varnish Software AS



From tfheen at varnish-cache.org  Thu Sep 22 12:02:46 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:46 +0200
Subject: [3.0] 192e6d0 Commit the actual fix for #1014 and not just the
	test-case.
Message-ID: <E1R6hzK-0004LZ-AE@project.varnish-software.com>

commit 192e6d06882d753212f81622faa43b1c6f975c26
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 12 10:14:27 2011 +0000

    Commit the actual fix for #1014 and not just the test-case.
    
    Fixes	#1014

diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 36df887..73edaa1 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -202,20 +202,15 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 	ssize_t cl;
 
 	assert(sp->wrk->body_status == BS_LENGTH);
+
 	cl = fetch_number(b, 10);
+	sp->wrk->vfp->begin(sp, cl > 0 ? cl : 0);
 	if (cl < 0) {
 		WSP(sp, SLT_FetchError, "straight length field bogus");
 		return (-1);
-	}
-	/*
-	 * XXX: we shouldn't need this if we have cl==0
-	 * XXX: but we must also conditionalize the vfp->end()
-	 */
-	sp->wrk->vfp->begin(sp, cl);
-	if (cl == 0)
+	} else if (cl == 0)
 		return (0);
 
-
 	i = sp->wrk->vfp->bytes(sp, htc, cl);
 	if (i <= 0) {
 		WSP(sp, SLT_FetchError, "straight read_error: %d %d (%s)",
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 675718a..b32b5cf 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -600,6 +600,7 @@ vfp_testgzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
 	sp->wrk->vgz_rx = VGZ_NewUngzip(sp, "u F -");
+	CHECK_OBJ_NOTNULL(sp->wrk->vgz_rx, VGZ_MAGIC);
 }
 
 static int __match_proto__()



From tfheen at varnish-cache.org  Thu Sep 22 12:02:46 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:46 +0200
Subject: [3.0] c593859 Always call fetch-processor begin,
	also if C-L header is bogus.
Message-ID: <E1R6hzK-0004Lp-Ps@project.varnish-software.com>

commit c59385938d80c6655fc31f774122d895c65d0c45
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 12 07:36:12 2011 +0000

    Always call fetch-processor begin, also if C-L header is bogus.
    
    Fixes	#1014

diff --git a/bin/varnishtest/tests/r01014.vtc b/bin/varnishtest/tests/r01014.vtc
new file mode 100644
index 0000000..53a4473
--- /dev/null
+++ b/bin/varnishtest/tests/r01014.vtc
@@ -0,0 +1,13 @@
+varnishtest "bug 1014, Invalid C-L header with gzip"
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Content-Encoding: gzip" -hdr "Content-Length:"
+} -start
+
+varnish v1 -vcl+backend { } -start
+
+client c1 {
+	txreq
+	rxresp
+} -run



From tfheen at varnish-cache.org  Thu Sep 22 12:02:47 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:47 +0200
Subject: [3.0] fc15a1c Updating configure line for ppc64,
	as --disabled-jemalloc has changed to --without-jemalloc
Message-ID: <E1R6hzL-0004Ly-10@project.varnish-software.com>

commit fc15a1cde13810cc87a69beb22f0ebd4e6b65944
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Sep 12 12:31:13 2011 +0200

    Updating configure line for ppc64, as --disabled-jemalloc has changed
    to --without-jemalloc
    
    Patch from P?l-Eivind Johnsen
    
    Fixes: #1011

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 3cef8f5..115ff54 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -87,7 +87,7 @@ cp bin/varnishd/default.vcl etc/zope-plone.vcl examples
 # Remove "--disable static" if you want to build static libraries 
 # jemalloc is not compatible with Red Hat's ppc64 RHEL kernel :-(
 %ifarch ppc64 ppc
-	%configure --disable-static --localstatedir=/var/lib --disable-jemalloc
+	%configure --disable-static --localstatedir=/var/lib --without-jemalloc
 %else
 	%configure --disable-static --localstatedir=/var/lib
 %endif



From tfheen at varnish-cache.org  Thu Sep 22 12:02:48 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:48 +0200
Subject: [3.0] 6fdc5c8 Minor flexelint'ing
Message-ID: <E1R6hzM-0004Ma-L4@project.varnish-software.com>

commit 6fdc5c81fc4c10beb54bd5c043453d8f7920bb21
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 14 07:50:40 2011 +0000

    Minor flexelint'ing

diff --git a/bin/varnishtest/flint.lnt b/bin/varnishtest/flint.lnt
index f50040c..8f3d880 100644
--- a/bin/varnishtest/flint.lnt
+++ b/bin/varnishtest/flint.lnt
@@ -22,3 +22,4 @@
 -e737	// [45]  Loss of sign in promotion from int to unsigned
 -e713	// Loss of precision (assignment) (unsigned long long to long long)
 -e574	// Signed-unsigned mix with relational
+-e835	//  A zero has been given as ___ argument to operator '___'  (<<)
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 5b14edc..a034fef 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -60,7 +60,6 @@ volatile sig_atomic_t	vtc_error;	/* Error encountered */
 int			vtc_stop;	/* Stops current test without error */
 pthread_t		vtc_thread;
 static struct vtclog	*vltop;
-int			in_tree = 0;	/* Are we running in-tree */
 
 /**********************************************************************
  * Macro facility
@@ -238,18 +237,6 @@ extmacro_def(const char *name, const char *fmt, ...)
 	}
 }
 
-const char *
-extmacro_get(const char *name)
-{
-	struct extmacro *m;
-
-	VTAILQ_FOREACH(m, &extmacro_list, list)
-		if (!strcmp(name, m->name))
-			return (m->val);
-
-	return (NULL);
-}
-
 /**********************************************************************
  * Execute a file
  */
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 16d577f..eb9a02a 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -85,4 +85,3 @@ void macro_def(struct vtclog *vl, const char *instance, const char *name,
 struct vsb *macro_expand(struct vtclog *vl, const char *text);
 
 void extmacro_def(const char *name, const char *fmt, ...);
-const char *extmacro_get(const char *name);
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 758113b..9c0f175 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -586,7 +586,7 @@ cmd_http_gunzip_body(CMD_ARGS)
  */
 
 static void
-gzip_body(struct http *hp, const char *txt, char **body, int *bodylen)
+gzip_body(const struct http *hp, const char *txt, char **body, int *bodylen)
 {
 	int l, i;
 	z_stream vz;



From tfheen at varnish-cache.org  Thu Sep 22 12:03:22 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:22 +0200
Subject: [3.0] 5b48f8a Missing newline
Message-ID: <E1R6hzu-0004Qw-BK@project.varnish-software.com>

commit 5b48f8a64bf663f5a9ac25580ff0a0fb500a6b29
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:41:28 2011 +0000

    Missing newline

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index b189d1e..01a5fe2 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -228,7 +228,7 @@ tst_cb(const struct vev *ve, int what)
 			    jp->tst->filename, t);
 			if (WIFSIGNALED(stx))
 				printf(" signal=%d", WTERMSIG(stx));
-			printf(" exit=%d", WEXITSTATUS(stx));
+			printf(" exit=%d\n", WEXITSTATUS(stx));
 			if (!vtc_continue) {
 				/* XXX kill -9 other jobs ? */
 				exit(2);



From tfheen at varnish-cache.org  Thu Sep 22 12:03:23 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:23 +0200
Subject: [3.0] 787ce07 Make -M help text better
Message-ID: <E1R6hzv-0004RH-Jl@project.varnish-software.com>

commit 787ce07404477a1ffd45a5b86bcfde4fd3445e22
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Sep 20 12:24:06 2011 +0200

    Make -M help text better

diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 5fb054a..49e0f81 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -145,7 +145,7 @@ usage(void)
 	fprintf(stderr, FMT, "", "  shl: space for SHL records [80m]");
 	fprintf(stderr, FMT, "", "  free: space for other allocations [1m]");
 	fprintf(stderr, FMT, "", "  fill: prefill new file [+]");
-	fprintf(stderr, FMT, "-M address:port", "CLI-master to connect to.");
+	fprintf(stderr, FMT, "-M address:port", "Reverse CLI destination.");
 	fprintf(stderr, FMT, "-n dir", "varnishd working directory");
 	fprintf(stderr, FMT, "-P file", "PID file");
 	fprintf(stderr, FMT, "-p param=value", "set parameter");



From tfheen at varnish-cache.org  Thu Sep 22 12:03:30 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:30 +0200
Subject: [3.0] f4c29c7 Disable this test on OSX,
	the VM mapping is too random to get us the same address again,
	even with hinting.
Message-ID: <E1R6i02-0004Rw-Ew@project.varnish-software.com>

commit f4c29c725cf963c8bf3d3445170bf83dff84b220
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 12:28:43 2011 +0000

    Disable this test on OSX, the VM mapping is too random to get us
    the same address again, even with hinting.

diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc
index faf4b4b..aa0fb10 100644
--- a/bin/varnishtest/tests/r00962.vtc
+++ b/bin/varnishtest/tests/r00962.vtc
@@ -1,5 +1,8 @@
 varnishtest "Test address remapping"
 
+# VM-remapping is to random on OSX
+feature	not-OSX
+
 server s1 {
 	rxreq 
 	txresp 
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index a034fef..3bee42a 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -492,7 +492,13 @@ cmd_feature(CMD_ARGS)
 		if (sizeof(void*) == 8 && !strcmp(av[i], "64bit"))
 			continue;
 
-		vtc_log(vl, 1, "SKIPPING test, missing feature %s", av[i]);
+		if (!strcmp(av[i], "!OSX")) {
+#if !defined(__APPLE__) || !defined(__MACH__)
+			continue;
+#endif
+		}
+
+		vtc_log(vl, 1, "SKIPPING test, missing feature: %s", av[i]);
 		vtc_stop = 1;
 		return;
 	}



From tfheen at varnish-cache.org  Thu Sep 22 12:03:42 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:42 +0200
Subject: [3.0] 5cb89d9 Don't ignore the return value of the read(2) call on a
	random dev.
Message-ID: <E1R6i0E-0004Si-Tg@project.varnish-software.com>

commit 5cb89d9a486ffb66b9505e4f7a4cc0064e45915f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 13:44:59 2011 +0000

    Don't ignore the return value of the read(2) call on a random dev.

diff --git a/lib/libvarnishcompat/srandomdev.c b/lib/libvarnishcompat/srandomdev.c
index 79f774d..d0332c9 100644
--- a/lib/libvarnishcompat/srandomdev.c
+++ b/lib/libvarnishcompat/srandomdev.c
@@ -40,6 +40,22 @@
 
 #include "compat/srandomdev.h"
 
+static int
+trydev(const char *fn, unsigned long *seed)
+{
+	int fd;
+	ssize_t sz;
+
+	fd = open(fn, O_RDONLY);
+	if (fd < 0)
+		return (-1);
+	sz = read(fd, seed, sizeof *seed);
+	(void)close(fd);
+	if (sz != sizeof *seed)
+		return (-1);
+	return (0);
+}
+
 void
 srandomdev(void)
 {
@@ -47,13 +63,11 @@ srandomdev(void)
 	unsigned long seed;
 	int fd;
 
-	if ((fd = open("/dev/urandom", O_RDONLY)) >= 0 ||
-	    (fd = open("/dev/random", O_RDONLY)) >= 0) {
-		read(fd, &seed, sizeof seed);
-		close(fd);
-	} else {
-		gettimeofday(&tv, NULL);
-		seed = (getpid() << 16) ^ tv.tv_sec ^ tv.tv_usec;
+	if (trydev("/dev/urandom", &seed)) {
+		if (trydev("/dev/random", &seed)) {
+			gettimeofday(&tv, NULL);
+			seed = (getpid() << 16) ^ tv.tv_sec ^ tv.tv_usec;
+		}
 	}
 	srandom(seed);
 }



From tfheen at varnish-cache.org  Thu Sep 22 12:03:45 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:45 +0200
Subject: [3.0] 697832e remove unused variable
Message-ID: <E1R6i0H-0004Ul-1B@project.varnish-software.com>

commit 697832e965ebcabbd8aeb03fa79addd033de5d6c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 14:25:40 2011 +0000

    remove unused variable

diff --git a/lib/libvarnishcompat/srandomdev.c b/lib/libvarnishcompat/srandomdev.c
index d0332c9..83b22c4 100644
--- a/lib/libvarnishcompat/srandomdev.c
+++ b/lib/libvarnishcompat/srandomdev.c
@@ -61,7 +61,6 @@ srandomdev(void)
 {
 	struct timeval tv;
 	unsigned long seed;
-	int fd;
 
 	if (trydev("/dev/urandom", &seed)) {
 		if (trydev("/dev/random", &seed)) {



From tfheen at varnish-cache.org  Thu Sep 22 12:03:53 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:53 +0200
Subject: [3.0] 2e50ed1 Don't rely on the test executing fast enough and
	repeatably enough that two separate timestamps just happen to
	be the same, just make them the same.
Message-ID: <E1R6i0P-0004VU-Nk@project.varnish-software.com>

commit 2e50ed19062cfb5773f63a329f09f56445f6930d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 19:15:11 2011 +0000

    Don't rely on the test executing fast enough and repeatably enough that
    two separate timestamps just happen to be the same, just make them
    the same.

diff --git a/bin/varnishtest/tests/r00907.vtc b/bin/varnishtest/tests/r00907.vtc
index 8bbcd2e..b64a160 100644
--- a/bin/varnishtest/tests/r00907.vtc
+++ b/bin/varnishtest/tests/r00907.vtc
@@ -3,7 +3,8 @@ varnishtest "Ticket #907 200/304 handling with Etags + Last-Modified"
 server s1 {
 	rxreq 
 	txresp \
-	       -hdr "ETag: saengei1Ohshicich4iteesu"
+		-hdr "ETag: saengei1Ohshicich4iteesu" \
+		-hdr "Last-Modified: Tue, 20 Sep 2011 18:55:00 GMT"
 } -start
 
 varnish v1 -vcl+backend {
@@ -16,8 +17,15 @@ client c1 {
 	txreq -hdr "If-None-Match: saengei1Ohshicich4iteesu"
 	rxresp
 	expect resp.status == 304
+
 	txreq -hdr "If-None-Match: saengei1Ohshicich4iteesu" \
-	      -hdr "If-Modified-Since: ${date}"
+	      -hdr "If-Modified-Since: Tue, 20 Sep 2011 18:54:59 GMT"
+	rxresp
+	expect resp.status == 200
+
+	txreq -hdr "If-None-Match: saengei1Ohshicich4iteesu" \
+	      -hdr "If-Modified-Since: Tue, 20 Sep 2011 18:55:00 GMT"
 	rxresp
 	expect resp.status == 304
+
 } -run



From tfheen at varnish-cache.org  Thu Sep 22 12:03:56 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:56 +0200
Subject: [3.0] dde4ee0 Remove never effected test
Message-ID: <E1R6i0S-0004WL-67@project.varnish-software.com>

commit dde4ee036a6539a6c65d66a6edd8ad6fb104df5b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 19:48:49 2011 +0000

    Remove never effected test

diff --git a/bin/varnishtest/tests/g00002.vtc b/bin/varnishtest/tests/g00002.vtc
index 574f1f2..1a0ab35 100644
--- a/bin/varnishtest/tests/g00002.vtc
+++ b/bin/varnishtest/tests/g00002.vtc
@@ -47,11 +47,4 @@ client c1 {
 	rxresp
 	expect resp.http.content-encoding == "resp.http.content-encoding"
 	expect resp.bodylen == 4109
-
-	# See varnish can deliver gzip'ed ESI (NOTYET)
-	#txreq -url /bar -hdr "Accept-Encoding: gzip"
-	#rxresp
-	# expect resp.http.content-encoding == "gzip"
-	#gunzip
-	#expect resp.bodylen == 4109
 } -run



From tfheen at varnish-cache.org  Thu Sep 22 12:04:00 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:04:00 +0200
Subject: [3.0] 902bdf4 Fix a long-standing bug in varnishtest:
Message-ID: <E1R6i0W-0004XD-Ra@project.varnish-software.com>

commit 902bdf49b5465e5336dc6ce51593112c289d1499
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 20:22:09 2011 +0000

    Fix a long-standing bug in varnishtest:
    
    It used to be that we'd call http processing on a filedescriptor and
    then that was that.  Then we added keywords to do new accepts in the
    server and suddenly the calling code closes a wrong filedesc which
    belongs to somebody else and things get really inconvenient fast.
    
    This made all test-cases which use the "accept" server directive flakey.

diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index eb9a02a..5e5cce7 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -64,7 +64,7 @@ extern pthread_t	vtc_thread;
 
 void init_sema(void);
 
-void http_process(struct vtclog *vl, const char *spec, int sock, int sfd);
+int http_process(struct vtclog *vl, const char *spec, int sock, int *sfd);
 
 void cmd_server_genvcl(struct vsb *vsb);
 
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index 20b152b..3bd430b 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -103,7 +103,7 @@ client_thread(void *priv)
 		VTCP_myname(fd, mabuf, sizeof mabuf, mpbuf, sizeof mpbuf);
 		vtc_log(vl, 3, "connected fd %d from %s %s to %s",
 		    fd, mabuf, mpbuf, VSB_data(vsb));
-		http_process(vl, c->spec, fd, -1);
+		fd = http_process(vl, c->spec, fd, NULL);
 		vtc_log(vl, 3, "closing fd %d", fd);
 		VTCP_close(&fd);
 	}
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 7dad4ed..32c7e0c 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -54,7 +54,7 @@ struct http {
 	unsigned		magic;
 #define HTTP_MAGIC		0x2f02169c
 	int			fd;
-	int			sfd;
+	int			*sfd;
 	int			timeout;
 	struct vtclog		*vl;
 
@@ -77,14 +77,14 @@ struct http {
 
 #define ONLY_CLIENT(hp, av)						\
 	do {								\
-		if (hp->sfd >= 0)					\
+		if (hp->sfd != NULL)					\
 			vtc_log(hp->vl, 0,				\
 			    "\"%s\" only possible in client", av[0]);	\
 	} while (0)
 
 #define ONLY_SERVER(hp, av)						\
 	do {								\
-		if (hp->sfd < 0)					\
+		if (hp->sfd == NULL)					\
 			vtc_log(hp->vl, 0,				\
 			    "\"%s\" only possible in server", av[0]);	\
 	} while (0)
@@ -344,19 +344,22 @@ http_rxchar_eof(struct http *hp, int n)
 		pfd[0].revents = 0;
 		i = poll(pfd, 1, hp->timeout);
 		if (i < 0)
-			vtc_log(hp->vl, 0, "HTTP rx timeout (%u ms)",
-			    hp->timeout);
+			vtc_log(hp->vl, 0, "HTTP rx timeout (fd:%d %u ms)",
+			    hp->fd, hp->timeout);
 		if (i < 0)
-			vtc_log(hp->vl, 0, "HTTP rx failed (poll: %s)",
-			    strerror(errno));
+			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d poll: %s)",
+			    hp->fd, strerror(errno));
 		assert(i > 0);
+		if (pfd[0].revents & ~POLLIN)
+			vtc_log(hp->vl, 4, "HTTP rx poll (fd:%d revents: %x)",
+			    hp->fd, pfd[0].revents);
 		assert(hp->prxbuf + n < hp->nrxbuf);
 		i = read(hp->fd, hp->rxbuf + hp->prxbuf, n);
 		if (i == 0)
 			return (i);
 		if (i <= 0)
-			vtc_log(hp->vl, 0, "HTTP rx failed (read: %s)",
-			    strerror(errno));
+			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d read: %s)",
+			    hp->fd, strerror(errno));
 		hp->prxbuf += i;
 		hp->rxbuf[hp->prxbuf] = '\0';
 		n -= i;
@@ -1036,7 +1039,7 @@ cmd_http_expect_close(CMD_ARGS)
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	AZ(av[1]);
-	assert(hp->sfd >= 0);
+	assert(hp->sfd != NULL);
 
 	vtc_log(vl, 4, "Expecting close (fd = %d)", hp->fd);
 	while (1) {
@@ -1074,10 +1077,11 @@ cmd_http_accept(CMD_ARGS)
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	AZ(av[1]);
+	assert(hp->sfd != NULL);
 	assert(hp->sfd >= 0);
 	VTCP_close(&hp->fd);
 	vtc_log(vl, 4, "Accepting");
-	hp->fd = accept(hp->sfd, NULL, NULL);
+	hp->fd = accept(*hp->sfd, NULL, NULL);
 	if (hp->fd < 0)
 		vtc_log(vl, 0, "Accepted failed: %s", strerror(errno));
 	vtc_log(vl, 3, "Accepted socket fd is %d", hp->fd);
@@ -1136,11 +1140,12 @@ static const struct cmds http_cmds[] = {
 	{ NULL,			NULL }
 };
 
-void
-http_process(struct vtclog *vl, const char *spec, int sock, int sfd)
+int
+http_process(struct vtclog *vl, const char *spec, int sock, int *sfd)
 {
 	struct http *hp;
 	char *s, *q;
+	int retval;
 
 	(void)sfd;
 	ALLOC_OBJ(hp, HTTP_MAGIC);
@@ -1162,9 +1167,11 @@ http_process(struct vtclog *vl, const char *spec, int sock, int sfd)
 	assert(q > s);
 	AN(s);
 	parse_string(s, http_cmds, hp, vl);
+	retval = hp->fd;
 	VSB_delete(hp->vsb);
 	free(hp->rxbuf);
 	free(hp);
+	return (retval);
 }
 
 /**********************************************************************
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 542a266..d1787b1 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -100,7 +100,7 @@ server_thread(void *priv)
 		if (fd < 0)
 			vtc_log(vl, 0, "Accepted failed: %s", strerror(errno));
 		vtc_log(vl, 3, "accepted fd %d", fd);
-		http_process(vl, s->spec, fd, s->sock);
+		fd = http_process(vl, s->spec, fd, &s->sock);
 		vtc_log(vl, 3, "shutting fd %d", fd);
 		j = shutdown(fd, SHUT_WR);
 		if (!VTCP_Check(j))



From tfheen at varnish-cache.org  Thu Sep 22 12:04:15 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:04:15 +0200
Subject: [3.0] 656e038 More detail on write errors.
Message-ID: <E1R6i0l-0004ZY-1z@project.varnish-software.com>

commit 656e038e1657c0384d48789833747a228be9bcd6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 21 10:40:05 2011 +0000

    More detail on write errors.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index e6aaf33..024e376 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -143,7 +143,8 @@ http_write(const struct http *hp, int lvl, const char *pfx)
 	vtc_dump(hp->vl, lvl, pfx, VSB_data(hp->vsb), VSB_len(hp->vsb));
 	l = write(hp->fd, VSB_data(hp->vsb), VSB_len(hp->vsb));
 	if (l != VSB_len(hp->vsb))
-		vtc_log(hp->vl, 0, "Write failed: %s", strerror(errno));
+		vtc_log(hp->vl, 0, "Write failed: (%d vs %d) %s",
+		    l, VSB_len(hp->vsb), strerror(errno));
 }
 
 /**********************************************************************



From tfheen at varnish-cache.org  Thu Sep 22 12:02:36 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:36 +0200
Subject: [3.0] da4c405 Add return(restart) in vcl_error to the dot-graph
Message-ID: <E1R6hzA-0004J8-A7@project.varnish-software.com>

commit da4c405f98dfaa8f14ae5edf498b504726b58cca
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Thu Sep 8 09:48:50 2011 +0200

    Add return(restart) in vcl_error to the dot-graph
    
    Thanks to scoof for pointing this out.

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 6bb8324..39a2104 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -413,6 +413,8 @@ DOT	]
 DOT	ERROR -> vcl_error
 DOT	vcl_error-> prepresp [label=deliver]
 DOT }
+DOT vcl_error-> rsterr [label="restart",color=purple]
+DOT rsterr [label="RESTART",shape=plaintext]
  */
 
 static int



From tfheen at varnish-cache.org  Thu Sep 22 12:04:13 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:04:13 +0200
Subject: [3.0] 2bec68e Unrace this testcase a little bit more.
Message-ID: <E1R6i0j-0004ZH-G2@project.varnish-software.com>

commit 2bec68e672ba0994f671f9df9712f030ee931325
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 21 09:55:09 2011 +0000

    Unrace this testcase a little bit more.

diff --git a/bin/varnishtest/tests/s00002.vtc b/bin/varnishtest/tests/s00002.vtc
index 939d0ee..3cb56cf 100644
--- a/bin/varnishtest/tests/s00002.vtc
+++ b/bin/varnishtest/tests/s00002.vtc
@@ -53,9 +53,9 @@ varnish v1 -vcl {
 	}
 } -start
 
+sema r1 sync 2
 
 client c1 {
-	sema r1 sync 2
 	txreq -url "/"
 	rxresp
 	expect resp.http.foo == "bar"



From tfheen at varnish-cache.org  Thu Sep 22 12:02:58 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:58 +0200
Subject: [3.0] 7337e71 Clear the poll structure beforehand in order to make
	sure there are no lingering bits.
Message-ID: <E1R6hzW-0004NU-BU@project.varnish-software.com>

commit 7337e71623be053d89b022d0a6beab35730f264b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:05:58 2011 +0000

    Clear the poll structure beforehand in order to make sure there
    are no lingering bits.
    
    Convert an assert to a failure.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 9c0f175..d01240b 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -899,8 +899,9 @@ cmd_http_send(CMD_ARGS)
 	AZ(av[2]);
 	vtc_dump(hp->vl, 4, "send", av[1], -1);
 	i = write(hp->fd, av[1], strlen(av[1]));
-	assert(i == strlen(av[1]));
-
+	if (i != strlen(av[1]))
+		vtc_log(hp->vl, 0, "Write error in http_send(): %s",
+		    strerror(errno));
 }
 
 /**********************************************************************
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index c53482e..2ff6ea9 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -349,6 +349,7 @@ varnish_launch(struct varnish *v)
 	AZ(pthread_create(&v->tp_vsl, NULL, varnishlog_thread, v));
 
 	/* Wait for the varnish to call home */
+	memset(fd, 0, sizeof fd);
 	fd[0].fd = v->cli_fd;
 	fd[0].events = POLLIN;
 	fd[1].fd = v->fds[0];



From tfheen at varnish-cache.org  Thu Sep 22 12:03:48 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:48 +0200
Subject: [3.0] d57f12d This cast is a tad too creative for my taste,
	make it safer.
Message-ID: <E1R6i0K-0004V4-0Z@project.varnish-software.com>

commit d57f12d9c73f240fda943e5ad3bfedb381ab6ead
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 13:55:51 2011 +0000

    This cast is a tad too creative for my taste, make it safer.

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index 9c6cb99..8e1218a 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -59,8 +59,11 @@ VRT_re_init(void **rep, const char *re)
 void
 VRT_re_fini(void *rep)
 {
+	vre_t *vv;
+
+	vv = rep;
 	if (rep != NULL)
-		VRE_free((vre_t**)&rep);
+		VRE_free(&vv);
 }
 
 int



From tfheen at varnish-cache.org  Thu Sep 22 12:04:10 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:04:10 +0200
Subject: [3.0] 300097c Cleanup sphinx warnings
Message-ID: <E1R6i0g-0004YX-Ly@project.varnish-software.com>

commit 300097cb220d13a7d9b73774d04ab0bb9c123acc
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Wed Sep 21 10:02:49 2011 +0200

    Cleanup sphinx warnings

diff --git a/doc/sphinx/conf.py.in b/doc/sphinx/conf.py.in
index d6b98ab..0b07a2b 100644
--- a/doc/sphinx/conf.py.in
+++ b/doc/sphinx/conf.py.in
@@ -64,7 +64,7 @@ release = '@VERSION@'
 
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
-exclude_trees = ['=build']
+exclude_patterns = ['=build','reference/params.rst']
 
 # The reST default role (used for this markup: `text`) to use for all documents.
 #default_role = None
@@ -147,7 +147,7 @@ html_title = "Varnish version @VERSION@ documentation"
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['=static']
+#html_static_path = ['=static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
diff --git a/doc/sphinx/faq/general.rst b/doc/sphinx/faq/general.rst
index d40ef9f..b30f093 100644
--- a/doc/sphinx/faq/general.rst
+++ b/doc/sphinx/faq/general.rst
@@ -337,7 +337,7 @@ Varnish has a feature called **hit for pass**, which is used when Varnish gets a
           * Client 2..N are now given the **hit for pass** object instructing them to go to the backend
 
 The **hit for pass** object will stay cached for the duration of its ttl. This means that subsequent clients requesting /foo will be sent straight to the backend as long as the **hit for pass** object exists.
-The :command:`varnishstat` can tell you how many **hit for pass** objects varnish has served. The default vcl will set ttl for a hit_for_pass object to 120s. But you can override this, using the following logic:
+The :command:`varnishstat` can tell you how many **hit for pass** objects varnish has served. The default vcl will set ttl for a hit_for_pass object to 120s. But you can override this, using the following logic::
 
         sub vcl_fetch {
           if (!obj.cacheable) {
diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst
index d00e9b0..eda1515 100644
--- a/doc/sphinx/installation/upgrade.rst
+++ b/doc/sphinx/installation/upgrade.rst
@@ -42,7 +42,7 @@ becomes
 
 	ban("req.url = " + req.url);
 
-``purge`` does not take any arguments anymore, but can be used in vcl_hit or vcl_miss to purge the item from the cache, where you would reduce ttl to 0 in Varnish 2.1.
+``purge`` does not take any arguments anymore, but can be used in vcl_hit or vcl_miss to purge the item from the cache, where you would reduce ttl to 0 in Varnish 2.1::
 
 	sub vcl_hit {
 	  if (req.request == "PURGE") {
@@ -51,7 +51,7 @@ becomes
 	  }
 	}
 
-becomes
+becomes::
 
 	sub vcl_hit {
 	  if (req.request == "PURGE") {
@@ -68,13 +68,13 @@ becomes
 returns are now done with the ``return()`` function
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``pass``, ``pipe``, ``lookup``, ``deliver``, ``fetch``, ``hash``, ``pipe`` and ``restart`` are no longer keywords, but arguments to ``return()``, so
+``pass``, ``pipe``, ``lookup``, ``deliver``, ``fetch``, ``hash``, ``pipe`` and ``restart`` are no longer keywords, but arguments to ``return()``, so::
 
 	sub vcl_pass {
 	  pass;
 	}
 
-becomes
+becomes::
 
 	sub vcl_pass {
 	  return(pass);
diff --git a/doc/sphinx/reference/params.rst b/doc/sphinx/reference/params.rst
index ce7da4b..f161596 100644
--- a/doc/sphinx/reference/params.rst
+++ b/doc/sphinx/reference/params.rst
@@ -128,6 +128,7 @@ diag_bitmap
 	  0x00020000 - synchronous start of persistence.
 	  0x00040000 - release VCL early.
 	  0x80000000 - do edge-detection on digest.
+
 	Use 0x notation and do the bitor in your head :-)
 
 esi_syntax
@@ -140,6 +141,7 @@ esi_syntax
 	  0x00000002 - Ignore non-esi elements
 	  0x00000004 - Emit parsing debug records
 	  0x00000008 - Force-split parser input (debugging)
+
 	Use 0x notation and do the bitor in your head :-)
 
 expiry_sleep
@@ -203,6 +205,7 @@ gzip_tmp_space
 	  0 - malloc
 	  1 - session workspace
 	  2 - thread workspace
+
 	If you have much gzip/gunzip activity, it may be an advantage to use workspace for these allocations to reduce malloc activity.  Be aware that gzip needs 256+KB and gunzip needs 32+KB of workspace (64+KB if ESI processing).
 
 gzip_window
@@ -216,7 +219,7 @@ http_gzip_support
 	- Default: on
 	- Flags: experimental
 
-	Enable gzip support. When enabled Varnish will compress uncompressed objects before they are stored in the cache. If a client does not support gzip encoding Varnish will uncompress compressed objects on demand. Varnish will also rewrite the Accept-Encoding header of clients indicating support for gzip to::
+	Enable gzip support. When enabled Varnish will compress uncompressed objects before they are stored in the cache. If a client does not support gzip encoding Varnish will uncompress compressed objects on demand. Varnish will also rewrite the Accept-Encoding header of clients indicating support for gzip to:
 
 	Accept-Encoding: gzip
 



From tfheen at varnish-cache.org  Thu Sep 22 12:02:15 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:15 +0200
Subject: [3.0] 513da1e Improve the FetchError VSL messages.
Message-ID: <E1R6hyp-0004GB-Re@project.varnish-software.com>

commit 513da1ed1ad1cfab1f2c28293e4372841eb35dc9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 12:08:02 2011 +0000

    Improve the FetchError VSL messages.
    
    Fixes	#1005

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 09c94f3..1c1952d 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -203,6 +203,7 @@ struct http_conn {
 	struct ws		*ws;
 	txt			rxbuf;
 	txt			pipeline;
+	const char		*error;
 };
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 24463f5..36df887 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -88,8 +88,10 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 
 	while (bytes > 0) {
 		st = FetchStorage(sp, 0);
-		if (st == NULL)
+		if (st == NULL) {
+			htc->error = "Could not get storage";
 			return (-1);
+		}
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
@@ -202,7 +204,7 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 	assert(sp->wrk->body_status == BS_LENGTH);
 	cl = fetch_number(b, 10);
 	if (cl < 0) {
-		WSP(sp, SLT_FetchError, "straight length syntax");
+		WSP(sp, SLT_FetchError, "straight length field bogus");
 		return (-1);
 	}
 	/*
@@ -217,7 +219,7 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 	i = sp->wrk->vfp->bytes(sp, htc, cl);
 	if (i <= 0) {
 		WSP(sp, SLT_FetchError, "straight read_error: %d %d (%s)",
-		    i, errno, strerror(errno));
+		    i, errno, htc->error);
 		return (-1);
 	}
 	return (0);
@@ -230,7 +232,7 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 		if (i != 1) {					\
 			WSP(sp, SLT_FetchError,			\
 			    "chunked read_error: %d (%s)",	\
-			    errno, strerror(errno));		\
+			    errno, htc->error);			\
 			return (-1);				\
 		}						\
 	} while (0)
@@ -315,7 +317,7 @@ fetch_eof(struct sess *sp, struct http_conn *htc)
 	i = sp->wrk->vfp->bytes(sp, htc, SSIZE_MAX);
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "eof read_error: %d (%s)",
-		    errno, strerror(errno));
+		    errno, htc->error);
 		return (-1);
 	}
 	return (0);
@@ -396,7 +398,7 @@ FetchHdr(struct sess *sp)
 		AN(sp->objcore->flags & OC_F_BUSY);
 	}
 
-	hp = sp->wrk->bereq;
+	hp = w->bereq;
 
 	sp->vbc = VDI_GetFd(NULL, sp);
 	if (sp->vbc == NULL) {
@@ -437,16 +439,16 @@ FetchHdr(struct sess *sp)
 
 	/* Receive response */
 
-	HTC_Init(sp->wrk->htc, sp->wrk->ws, vc->fd, params->http_resp_size,
+	HTC_Init(w->htc, w->ws, vc->fd, params->http_resp_size,
 	    params->http_resp_hdr_len);
 
 	VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout);
 
-	i = HTC_Rx(sp->wrk->htc);
+	i = HTC_Rx(w->htc);
 
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "http first read error: %d %d (%s)",
-		    i, errno, strerror(errno));
+		    i, errno, w->htc->error);
 		VDI_CloseFd(sp);
 		/* XXX: other cleanup ? */
 		/* Retryable if we never received anything */
@@ -456,20 +458,20 @@ FetchHdr(struct sess *sp)
 	VTCP_set_read_timeout(vc->fd, vc->between_bytes_timeout);
 
 	while (i == 0) {
-		i = HTC_Rx(sp->wrk->htc);
+		i = HTC_Rx(w->htc);
 		if (i < 0) {
 			WSP(sp, SLT_FetchError,
 			    "http first read error: %d %d (%s)",
-			    i, errno, strerror(errno));
+			    i, errno, w->htc->error);
 			VDI_CloseFd(sp);
 			/* XXX: other cleanup ? */
 			return (-1);
 		}
 	}
 
-	hp = sp->wrk->beresp;
+	hp = w->beresp;
 
-	if (http_DissectResponse(sp->wrk, sp->wrk->htc, hp)) {
+	if (http_DissectResponse(w, w->htc, hp)) {
 		WSP(sp, SLT_FetchError, "http format error");
 		VDI_CloseFd(sp);
 		/* XXX: other cleanup ? */
@@ -485,22 +487,24 @@ FetchBody(struct sess *sp)
 {
 	int cls;
 	struct storage *st;
+	struct worker *w;
 	int mklen;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+	w = sp->wrk;
 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->obj->http, HTTP_MAGIC);
 
-	if (sp->wrk->vfp == NULL)
-		sp->wrk->vfp = &vfp_nop;
+	if (w->vfp == NULL)
+		w->vfp = &vfp_nop;
 
 	AN(sp->director);
 	AssertObjCorePassOrBusy(sp->obj->objcore);
 
-	AZ(sp->wrk->vgz_rx);
+	AZ(w->vgz_rx);
 	AZ(VTAILQ_FIRST(&sp->obj->store));
-	switch (sp->wrk->body_status) {
+	switch (w->body_status) {
 	case BS_NONE:
 		cls = 0;
 		mklen = 0;
@@ -510,20 +514,20 @@ FetchBody(struct sess *sp)
 		mklen = 1;
 		break;
 	case BS_LENGTH:
-		cls = fetch_straight(sp, sp->wrk->htc,
-		    sp->wrk->h_content_length);
+		cls = fetch_straight(sp, w->htc,
+		    w->h_content_length);
 		mklen = 1;
-		XXXAZ(sp->wrk->vfp->end(sp));
+		XXXAZ(w->vfp->end(sp));
 		break;
 	case BS_CHUNKED:
-		cls = fetch_chunked(sp, sp->wrk->htc);
+		cls = fetch_chunked(sp, w->htc);
 		mklen = 1;
-		XXXAZ(sp->wrk->vfp->end(sp));
+		XXXAZ(w->vfp->end(sp));
 		break;
 	case BS_EOF:
-		cls = fetch_eof(sp, sp->wrk->htc);
+		cls = fetch_eof(sp, w->htc);
 		mklen = 1;
-		XXXAZ(sp->wrk->vfp->end(sp));
+		XXXAZ(w->vfp->end(sp));
 		break;
 	case BS_ERROR:
 		cls = 1;
@@ -534,26 +538,26 @@ FetchBody(struct sess *sp)
 		mklen = 0;
 		INCOMPL();
 	}
-	AZ(sp->wrk->vgz_rx);
+	AZ(w->vgz_rx);
 
 	/*
 	 * It is OK for ->end to just leave the last storage segment
-	 * sitting on sp->wrk->storage, we will always call vfp_nop_end()
+	 * sitting on w->storage, we will always call vfp_nop_end()
 	 * to get it trimmed or thrown out if empty.
 	 */
 	AZ(vfp_nop_end(sp));
 
-	WSL(sp->wrk, SLT_Fetch_Body, sp->vbc->fd, "%u(%s) %d %u",
-	    sp->wrk->body_status, body_status(sp->wrk->body_status),
+	WSL(w, SLT_Fetch_Body, sp->vbc->fd, "%u(%s) cls %d mklen %u",
+	    w->body_status, body_status(w->body_status),
 	    cls, mklen);
 
-	if (sp->wrk->body_status == BS_ERROR) {
+	if (w->body_status == BS_ERROR) {
 		VDI_CloseFd(sp);
 		return (__LINE__);
 	}
 
 	if (cls < 0) {
-		sp->wrk->stats.fetch_failed++;
+		w->stats.fetch_failed++;
 		/* XXX: Wouldn't this store automatically be released ? */
 		while (!VTAILQ_EMPTY(&sp->obj->store)) {
 			st = VTAILQ_FIRST(&sp->obj->store);
@@ -565,10 +569,10 @@ FetchBody(struct sess *sp)
 		return (__LINE__);
 	}
 
-	if (cls == 0 && sp->wrk->do_close)
+	if (cls == 0 && w->do_close)
 		cls = 1;
 
-	WSL(sp->wrk, SLT_Length, sp->vbc->fd, "%u", sp->obj->len);
+	WSL(w, SLT_Length, sp->vbc->fd, "%u", sp->obj->len);
 
 	{
 	/* Sanity check fetch methods accounting */
@@ -586,7 +590,7 @@ FetchBody(struct sess *sp)
 
 	if (mklen > 0) {
 		http_Unset(sp->obj->http, H_Content_Length);
-		http_PrintfHeader(sp->wrk, sp->fd, sp->obj->http,
+		http_PrintfHeader(w, sp->fd, sp->obj->http,
 		    "Content-Length: %jd", (intmax_t)sp->obj->len);
 	}
 
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index bed3104..675718a 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -469,8 +469,10 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			bytes -= w;
 		}
 
-		if (VGZ_ObufStorage(sp, vg))
+		if (VGZ_ObufStorage(sp, vg)) {
+			htc->error = "Could not get storage";
 			return (-1);
+		}
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		assert(i == VGZ_OK || i == VGZ_END);
 		sp->obj->len += dl;
@@ -479,6 +481,8 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	}
 	if (i == Z_OK || i == Z_STREAM_END)
 		return (1);
+	htc->error = "See other message";
+	WSP(sp, SLT_FetchError, "Gunzip trouble (%d)", i);
 	return (-1);
 }
 
@@ -540,8 +544,10 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
-		if (VGZ_ObufStorage(sp, vg))
+		if (VGZ_ObufStorage(sp, vg)) {
+			htc->error = "Could not get storage";
 			return (-1);
+		}
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
 		sp->obj->len += dl;
@@ -612,8 +618,10 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	AZ(vg->vz.avail_in);
 	while (bytes > 0) {
 		st = FetchStorage(sp, 0);
-		if (st == NULL)
+		if (st == NULL) {
+			htc->error = "Could not get storage";
 			return (-1);
+		}
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
@@ -631,10 +639,11 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			VGZ_Obuf(vg, obuf, sizeof obuf);
 			i = VGZ_Gunzip(vg, &dp, &dl);
 			if (i == VGZ_END && !VGZ_IbufEmpty(vg)) {
-				WSP(sp, SLT_FetchError, "Junk after gzip data");
+				htc->error = "Junk after gzip data";
 				return (-1);
 			}
 			if (i != VGZ_OK && i != VGZ_END) {
+				htc->error = "See other message";
 				WSP(sp, SLT_FetchError,
 				    "Invalid Gzip data: %s", vg->vz.msg);
 				return (-1);
@@ -643,6 +652,7 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	}
 	if (i == VGZ_OK || i == VGZ_END)
 		return (1);
+	htc->error = "See other message";
 	WSP(sp, SLT_FetchError, "Gunzip trouble (%d)", i);
 	return (-1);
 }
@@ -665,5 +675,3 @@ struct vfp vfp_testgzip = {
         .bytes  =       vfp_testgzip_bytes,
         .end    =       vfp_testgzip_end,
 };
-
-
diff --git a/bin/varnishd/cache_httpconn.c b/bin/varnishd/cache_httpconn.c
index 3840ed6..9d6d926 100644
--- a/bin/varnishd/cache_httpconn.c
+++ b/bin/varnishd/cache_httpconn.c
@@ -88,6 +88,7 @@ HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned maxbytes,
 	htc->fd = fd;
 	htc->maxbytes = maxbytes;
 	htc->maxhdr = maxhdr;
+	htc->error = "No error recorded";
 
 	(void)WS_Reserve(htc->ws, htc->maxbytes);
 	htc->rxbuf.b = ws->f;
@@ -109,6 +110,7 @@ HTC_Reinit(struct http_conn *htc)
 	unsigned l;
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	htc->error = "No error recorded";
 	(void)WS_Reserve(htc->ws, htc->maxbytes);
 	htc->rxbuf.b = htc->ws->f;
 	htc->rxbuf.e = htc->ws->f;
@@ -201,7 +203,10 @@ HTC_Read(struct http_conn *htc, void *d, size_t len)
 	if (len == 0)
 		return (l);
 	i = read(htc->fd, p, len);
-	if (i < 0)
+	if (i < 0) {
+		htc->error = strerror(errno);
 		return (i);
+	} else if (i == 0)
+		htc->error = "Remote closed connection";
 	return (i + l);
 }



From tfheen at varnish-cache.org  Thu Sep 22 12:02:45 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:45 +0200
Subject: [3.0] 9b1fea2 Add type to connect_timeout
Message-ID: <E1R6hzJ-0004Kp-7s@project.varnish-software.com>

commit 9b1fea27154c30e314c859c58e2a1e0946de5c53
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Fri Sep 9 15:30:55 2011 +0200

    Add type to connect_timeout
    
    References #971

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 0e39187..aae16bc 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -232,7 +232,7 @@ random or round-robin director or using .list::
           .list = {
                   .host_header = "www.example.com";
                   .port = "80";
-                  .connect_timeout = 0.4;
+                  .connect_timeout = 0.4s;
                   "192.168.15.0"/24;
                   "192.168.16.128"/25;
           }



From tfheen at varnish-cache.org  Thu Sep 22 12:03:01 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:01 +0200
Subject: [3.0] aab7c10 Update zope-plone.vcl to v3 syntax
Message-ID: <E1R6hzZ-0004OC-SN@project.varnish-software.com>

commit aab7c102985f497091f279562b28538d313cf034
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 20 11:03:08 2011 +0200

    Update zope-plone.vcl to v3 syntax
    
    Thanks to cleberjsantos for initial patch which this is based on.
    
    Fixes: #976

diff --git a/etc/zope-plone.vcl b/etc/zope-plone.vcl
index 8391499..b69f583 100644
--- a/etc/zope-plone.vcl
+++ b/etc/zope-plone.vcl
@@ -14,8 +14,8 @@
 
 # Default backend is the Zope CMS
 backend default {
-	set backend.host = "127.0.0.1";
-	set backend.port = "9673";
+	.host = "127.0.0.1";
+	.port = "9673";
 }
 
 acl purge {
@@ -29,10 +29,10 @@ sub vcl_recv {
 	# requests for unknown hosts
         if (req.http.host ~ "(www.)?example.com") {
                 set req.http.host = "example.com";
-                set req.url = "/VirtualHostBase/http/example.com:80/example.com/VirtualHostRoot" req.url;
+                set req.url = "/VirtualHostBase/http/example.com:80/example.com/VirtualHostRoot" + req.url;
         } elsif (req.http.host ~ "(www.)?example.org") {
                 set req.http.host = "example.org";
-                set req.url = "/VirtualHostBase/http/example.org:80/example.org/VirtualHostRoot" req.url;
+                set req.url = "/VirtualHostBase/http/example.org:80/example.org/VirtualHostRoot" + req.url;
         } else {
                 error 404 "Unknown virtual host.";
         }
@@ -42,7 +42,7 @@ sub vcl_recv {
 
                 # POST - Logins and edits
                 if (req.request == "POST") {
-                        pass;
+                        return(pass);
                 }
                 
                 # PURGE - The CacheFu product can invalidate updated URLs
@@ -50,7 +50,7 @@ sub vcl_recv {
                         if (!client.ip ~ purge) {
                                 error 405 "Not allowed.";
                         }
-                        lookup;
+                        return(lookup);
                 }
         }
 
@@ -60,9 +60,9 @@ sub vcl_recv {
 		# Force lookup of specific urls unlikely to need protection
 		if (req.url ~ "\.(js|css)") {
                         remove req.http.cookie;
-                        lookup;
+                        return(lookup);
                 }
-                pass;
+                return(pass);
         }
 
         # The default vcl_recv is used from here.
@@ -71,13 +71,14 @@ sub vcl_recv {
 # Do the PURGE thing
 sub vcl_hit {
         if (req.request == "PURGE") {
-                set obj.ttl = 0s;
+                purge;
                 error 200 "Purged";
         }
 }
 sub vcl_miss {
         if (req.request == "PURGE") {
-                error 404 "Not in cache";
+                purge;
+                error 200 "Purged";
         }
 }
 
@@ -85,7 +86,7 @@ sub vcl_miss {
 # from Zope by using the CacheFu product
 
 sub vcl_fetch {
-        if (obj.ttl < 3600s) {
-                set obj.ttl = 3600s;
+        if (beresp.ttl < 3600s) {
+                set beresp.ttl = 3600s;
         }
 }



From tfheen at varnish-cache.org  Thu Sep 22 12:01:54 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:01:54 +0200
Subject: [3.0] 21b7258 I have not been able to reproduce this exact
	circumstance, but the fix is readily obvious,
	so:  Tighten requirement for range delivery.
Message-ID: <E1R6hyU-0004B8-0C@project.varnish-software.com>

commit 21b7258b2b618350a46d96ea07ff5fee3a653b28
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 6 08:15:06 2011 +0000

    I have not been able to reproduce this exact circumstance, but the
    fix is readily obvious, so:  Tighten requirement for range delivery.
    
    Fixes	#1007

diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index c592909..73338f7 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -276,9 +276,13 @@ RES_WriteObj(struct sess *sp)
 	 */
 	low = 0;
 	high = sp->obj->len - 1;
-	if (!(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
-	    params->http_range_support && sp->obj->response == 200 &&
-	    sp->wantbody && http_GetHdr(sp->http, H_Range, &r))
+	if (
+	    sp->wantbody &&
+	    (sp->wrk->res_mode & RES_LEN) &&
+	    !(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
+	    params->http_range_support &&
+	    sp->obj->response == 200 &&
+	    http_GetHdr(sp->http, H_Range, &r))
 		res_dorange(sp, r, &low, &high);
 
 	/*



From tfheen at varnish-cache.org  Thu Sep 22 12:02:45 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:45 +0200
Subject: [3.0] 407ed75 Fix escapes, and workaround man-page formatting issues
Message-ID: <E1R6hzJ-0004L4-Hf@project.varnish-software.com>

commit 407ed754bfea9cc5551fd81130d97d61c3c76869
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Sat Sep 10 15:16:37 2011 +0200

    Fix escapes, and workaround man-page formatting issues

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 517f4fb..e3169ea 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -288,8 +288,8 @@ Probes take the following parameters:
   Specify a URL to request from the backend.
   Defaults to "/".
 .request
-  Specify a full HTTP request using multiple strings.
-  .request will have \r\n automatically inserted after every string.
+  Specify a full HTTP request using multiple strings. .request will
+  have \\r\\n automatically inserted after every string.
   If specified, .request will take precedence over .url.
 .window
   How many of the latest polls we examine to determine backend health.



From tfheen at varnish-cache.org  Thu Sep 22 12:03:58 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:58 +0200
Subject: [3.0] bc568b8 Increase varnishtest's HTTP rx timeout to 15 seconds,
	a lot of test failures seems to be timeouts on stressed machines.
Message-ID: <E1R6i0U-0004Wh-7V@project.varnish-software.com>

commit bc568b808c422ac3404f026ddaa7af2636239916
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 19:49:04 2011 +0000

    Increase varnishtest's HTTP rx timeout to 15 seconds, a lot of
    test failures seems to be timeouts on stressed machines.
    
    Teport timeouts more clearly.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index d01240b..7dad4ed 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -343,7 +343,10 @@ http_rxchar_eof(struct http *hp, int n)
 		pfd[0].events = POLLIN;
 		pfd[0].revents = 0;
 		i = poll(pfd, 1, hp->timeout);
-		if (i <= 0)
+		if (i < 0)
+			vtc_log(hp->vl, 0, "HTTP rx timeout (%u ms)",
+			    hp->timeout);
+		if (i < 0)
 			vtc_log(hp->vl, 0, "HTTP rx failed (poll: %s)",
 			    strerror(errno));
 		assert(i > 0);
@@ -1143,7 +1146,7 @@ http_process(struct vtclog *vl, const char *spec, int sock, int sfd)
 	ALLOC_OBJ(hp, HTTP_MAGIC);
 	AN(hp);
 	hp->fd = sock;
-	hp->timeout = 5000;
+	hp->timeout = 15000;
 	hp->nrxbuf = 2048*1024;
 	hp->vsb = VSB_new_auto();
 	hp->rxbuf = malloc(hp->nrxbuf);		/* XXX */



From tfheen at varnish-cache.org  Thu Sep 22 12:02:14 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:14 +0200
Subject: [3.0] 989645a Add body_status.h to includes
Message-ID: <E1R6hyo-0004FT-G4@project.varnish-software.com>

commit 989645adb055f5a373045d4b3fbec77738aebfef
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 18:41:41 2011 +0000

    Add body_status.h to includes

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 0e0bc83..7d21fcd 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -76,6 +76,7 @@ varnishd_SOURCES = \
 
 noinst_HEADERS = \
 	acct_fields.h \
+	body_status.h \
 	cache.h \
 	cache_backend.h \
 	cache_backend_poll.h \



From tfheen at varnish-cache.org  Thu Sep 22 12:03:19 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:19 +0200
Subject: [3.0] 8a37450 Increase http_req_size, http_resp_size to 8192 bytes
Message-ID: <E1R6hzr-0004Pa-KN@project.varnish-software.com>

commit 8a3745074a83a3dbfca9ff65f5bf22ef56fe220d
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 20 11:07:21 2011 +0200

    Increase http_req_size, http_resp_size to 8192 bytes
    
    Apache (and nginx) uses 8k, so use the same default size to avoid
    people being surprised by more 413 responses than necessary.
    
    Fixes: #1016

diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index 33101bb..4ca1b7b 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -522,7 +522,7 @@ static const struct parspec input_parspec[] = {
 		"Maximum length of any HTTP client request header we will "
 		"allow.  The limit is inclusive its continuation lines.\n",
 		0,
-		"4096", "bytes" },
+		"8192", "bytes" },
 	{ "http_req_size", tweak_uint, &master.http_req_size,
 		256, UINT_MAX,
 		"Maximum number of bytes of HTTP client request we will deal "
@@ -538,7 +538,7 @@ static const struct parspec input_parspec[] = {
 		"Maximum length of any HTTP backend response header we will "
 		"allow.  The limit is inclusive its continuation lines.\n",
 		0,
-		"4096", "bytes" },
+		"8192", "bytes" },
 	{ "http_resp_size", tweak_uint, &master.http_resp_size,
 		256, UINT_MAX,
 		"Maximum number of bytes of HTTP backend resonse we will deal "



From tfheen at varnish-cache.org  Thu Sep 22 12:02:07 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:07 +0200
Subject: [3.0] c8a05bf Add text representation to Fetch_Body VSL message.
Message-ID: <E1R6hyh-0004EH-Og@project.varnish-software.com>

commit c8a05bf6a74750d02955c2e857c687c2d76d0b7c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 11:03:35 2011 +0000

    Add text representation to Fetch_Body VSL message.

diff --git a/bin/varnishd/body_status.h b/bin/varnishd/body_status.h
new file mode 100644
index 0000000..33803f9
--- /dev/null
+++ b/bin/varnishd/body_status.h
@@ -0,0 +1,38 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Various ways to handle the body coming from the backend.
+ */
+
+/*lint -save -e525 -e539 */
+BODYSTATUS(NONE, none)
+BODYSTATUS(ZERO, zero)
+BODYSTATUS(ERROR, error)
+BODYSTATUS(CHUNKED, chunked)
+BODYSTATUS(LENGTH, length)
+BODYSTATUS(EOF, eof)
+/*lint -restore */
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index f9989fb..09c94f3 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -66,14 +66,23 @@
 #include "vsl.h"
 
 enum body_status {
-	BS_NONE,
-	BS_ZERO,
-	BS_ERROR,
-	BS_CHUNKED,
-	BS_LENGTH,
-	BS_EOF
+#define BODYSTATUS(U,l)	BS_##U,
+#include "body_status.h"
+#undef BODYSTATUS
 };
 
+static inline const char *
+body_status(enum body_status e)
+{
+	switch(e) {
+#define BODYSTATUS(U,l)	case BS_##U: return (#l);
+#include "body_status.h"
+#undef BODYSTATUS
+	default:
+		return ("?");
+	}
+}
+
 /*
  * NB: HDR_STATUS is only used in cache_http.c, everybody else uses the
  * http->status integer field.
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 657392f..24463f5 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -543,8 +543,9 @@ FetchBody(struct sess *sp)
 	 */
 	AZ(vfp_nop_end(sp));
 
-	WSL(sp->wrk, SLT_Fetch_Body, sp->vbc->fd, "%u %d %u",
-	    sp->wrk->body_status, cls, mklen);
+	WSL(sp->wrk, SLT_Fetch_Body, sp->vbc->fd, "%u(%s) %d %u",
+	    sp->wrk->body_status, body_status(sp->wrk->body_status),
+	    cls, mklen);
 
 	if (sp->wrk->body_status == BS_ERROR) {
 		VDI_CloseFd(sp);
diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index 1e88f1b..f2d2a6b 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -86,6 +86,7 @@
 -efile(451, "sys/\*.h")	// No include guard
 -efile(451, "machine/\*.h")	// No include guard
 -efile(451, "vcl_returns.h")	// No include guard
+-efile(451, "body_status.h")	// No include guard
 -efile(451, "locks.h")	// No include guard
 -efile(451, "cache_backend_poll.h")	// No include guard
 -efile(451, "steps.h")	// No include guard



From tfheen at varnish-cache.org  Thu Sep 22 12:04:06 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:04:06 +0200
Subject: [3.0] 2481d4b Add a missing derefence.
Message-ID: <E1R6i0c-0004Y1-9o@project.varnish-software.com>

commit 2481d4b647613bd0ab58c7ef54ac63598c027f12
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 21:29:03 2011 +0000

    Add a missing derefence.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 32c7e0c..8f96577 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1078,7 +1078,7 @@ cmd_http_accept(CMD_ARGS)
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	AZ(av[1]);
 	assert(hp->sfd != NULL);
-	assert(hp->sfd >= 0);
+	assert(*hp->sfd >= 0);
 	VTCP_close(&hp->fd);
 	vtc_log(vl, 4, "Accepting");
 	hp->fd = accept(*hp->sfd, NULL, NULL);



From tfheen at varnish-cache.org  Thu Sep 22 12:03:21 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:21 +0200
Subject: [3.0] 1a21466 More details about failing test subprocesses.
Message-ID: <E1R6hzt-0004Qk-O2@project.varnish-software.com>

commit 1a2146647ca4e98ddfd2057958651c4c2678cab0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:40:44 2011 +0000

    More details about failing test subprocesses.

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index b280022..b189d1e 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -224,8 +224,11 @@ tst_cb(const struct vev *ve, int what)
 		free(jp->tmpdir);
 
 		if (stx) {
-			printf("#     top  TEST %s FAILED (%.3f)\n",
+			printf("#     top  TEST %s FAILED (%.3f)",
 			    jp->tst->filename, t);
+			if (WIFSIGNALED(stx))
+				printf(" signal=%d", WTERMSIG(stx));
+			printf(" exit=%d", WEXITSTATUS(stx));
 			if (!vtc_continue) {
 				/* XXX kill -9 other jobs ? */
 				exit(2);



From tfheen at varnish-cache.org  Thu Sep 22 12:03:23 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:23 +0200
Subject: [3.0] 277ad79 Wait for the health-probe to finish before we even
	start client c2,
	this should solve issues where the c2-v1 connection times out before it
	ever got used.
Message-ID: <E1R6hzv-0004R6-2i@project.varnish-software.com>

commit 277ad79137c4a21ee045435e7eb306e91cbb8d97
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:48:16 2011 +0000

    Wait for the health-probe to finish before we even start client c2,
    this should solve issues where the c2-v1 connection times out before
    it ever got used.

diff --git a/bin/varnishtest/tests/s00002.vtc b/bin/varnishtest/tests/s00002.vtc
index f7b8820..939d0ee 100644
--- a/bin/varnishtest/tests/s00002.vtc
+++ b/bin/varnishtest/tests/s00002.vtc
@@ -62,8 +62,9 @@ client c1 {
 	expect resp.status == 200
 } -run
 
+sema r1 sync 2
+
 client c2 {
-	sema r1 sync 2
 	txreq -url "/"
 	rxresp
 	expect resp.http.foo == "bar"



From tfheen at varnish-cache.org  Thu Sep 22 12:02:02 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:02 +0200
Subject: [3.0] 0a7a30d Fix string documentation for 3.0
Message-ID: <E1R6hyc-0004Dd-1y@project.varnish-software.com>

commit 0a7a30da941e461cfb7761ca46a9076500e6a0d2
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Sep 6 19:30:40 2011 +0200

    Fix string documentation for 3.0

diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst
index ee57df4..d00e9b0 100644
--- a/doc/sphinx/installation/upgrade.rst
+++ b/doc/sphinx/installation/upgrade.rst
@@ -13,6 +13,10 @@ string concatenation operator
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 String concatenation did not have an operator previously, but this has now been changed to ``+``.
 
+no more %-escapes in strings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+To simplify strings, the %-encoding has been removed. If you need non-printable characters, you need to use inline C.
+
 ``log`` moved to the std vmod
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 82e20f4..7359e6f 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -39,11 +39,11 @@ In addition to the C-like assignment (=), comparison (==, !=) and
 boolean (!, && and \|\|) operators, VCL supports both regular
 expression and ACL matching using the ~ and the !~ operators.
 
-Basic strings are enclosed in " ... " and uses URL-style %-escapes.
+Basic strings are enclosed in " ... ", and may not contain newlines.
 
-Long strings are enclosed in {" ... "} and do not have an escape
-character. They may contain any character including ", newline and
-other control characters except for the NUL (0x00) character.
+Long strings are enclosed in {" ... "}. They may contain any
+character including ", newline and other control characters except
+for the NUL (0x00) character.
 
 Unlike C and Perl, the backslash (\) character has no special meaning
 in strings in VCL, so it can be freely used in regular expressions
diff --git a/lib/libvcl/vcc_token.c b/lib/libvcl/vcc_token.c
index 5a1b00c..c5766c0 100644
--- a/lib/libvcl/vcc_token.c
+++ b/lib/libvcl/vcc_token.c
@@ -492,7 +492,7 @@ vcc_Lexer(struct vcc *tl, struct source *sp)
 			continue;
 		}
 
-		/* Match strings, with \\ and \" escapes */
+		/* Match strings */
 		if (*p == '"') {
 			for (q = p + 1; q < sp->e; q++) {
 				if (*q == '"') {



From tfheen at varnish-cache.org  Thu Sep 22 12:02:47 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:47 +0200
Subject: [3.0] df412e7 Dump the VSL into the varnishtest output.
Message-ID: <E1R6hzL-0004MG-IN@project.varnish-software.com>

commit df412e73ee45c56e9d638c3c6a963eede23829fb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 14 07:44:17 2011 +0000

    Dump the VSL into the varnishtest output.
    
    Based on code from:	DocWilco

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 2b6e3d4..c53482e 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -47,7 +47,6 @@
 #include "libvarnish.h"
 #include "varnishapi.h"
 #include "vcli.h"
-#include "cli_common.h"
 #include "vss.h"
 #include "vsb.h"
 
@@ -58,7 +57,6 @@ struct varnish {
 #define VARNISH_MAGIC		0x208cd8e3
 	char			*name;
 	struct vtclog		*vl;
-	struct vtclog		*vl1;
 	VTAILQ_ENTRY(varnish)	list;
 
 	struct vsb		*storage;
@@ -68,16 +66,22 @@ struct varnish {
 	pid_t			pid;
 
 	pthread_t		tp;
+	pthread_t		tp_vsl;
 
 	int			cli_fd;
 	int			vcl_nbr;
 	char			*workdir;
 
-	struct VSM_data		*vd;
+	struct VSM_data		*vd;		/* vsc use */
+
+	unsigned		vsl_tag_count[256];
+	unsigned		vsl_sleep;
 };
 
 #define NONSENSE	"%XJEIFLH|)Xspa8P"
 
+#define VSL_SLEEP_USEC	(50*1000)
+
 static VTAILQ_HEAD(, varnish)	varnishes =
     VTAILQ_HEAD_INITIALIZER(varnishes);
 
@@ -116,6 +120,57 @@ varnish_ask_cli(const struct varnish *v, const char *cmd, char **repl)
 }
 
 /**********************************************************************
+ * Varnishlog gatherer + thread
+ */
+
+static int
+h_addlog(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
+    unsigned spec, const char *ptr, uint64_t bitmap)
+{
+	struct varnish *v;
+	int type;
+
+	(void) bitmap;
+
+	type = (spec & VSL_S_CLIENT) ? 'c' :
+	    (spec & VSL_S_BACKEND) ? 'b' : '-';
+	CAST_OBJ_NOTNULL(v, priv, VARNISH_MAGIC);
+
+	v->vsl_tag_count[tag]++;
+
+	vtc_log(v->vl, 4, "vsl| %5u %-12s %c %.*s", fd,
+	    VSL_tags[tag], type, len, ptr);
+	v->vsl_sleep = 100;
+	return (0);
+}
+
+static void *
+varnishlog_thread(void *priv)
+{
+	struct varnish *v;
+	struct VSM_data	*vsl;
+
+	CAST_OBJ_NOTNULL(v, priv, VARNISH_MAGIC);
+	vsl = VSM_New();
+	VSL_Setup(vsl);
+	(void)VSL_Arg(vsl, 'n', v->workdir);
+	VSL_NonBlocking(vsl, 1);
+	while (v->pid  && VSL_Open(vsl, 0) != 0) {
+		assert(usleep(VSL_SLEEP_USEC) == 0 || errno == EINTR);
+	}
+	while (v->pid) {
+		if (VSL_Dispatch(vsl, h_addlog, v) < 0) {
+			assert(usleep(v->vsl_sleep) == 0 || errno == EINTR);
+			v->vsl_sleep += v->vsl_sleep;
+			if (v->vsl_sleep > VSL_SLEEP_USEC)
+				v->vsl_sleep = VSL_SLEEP_USEC;
+		}
+	}
+	VSM_Delete(vsl);
+	return (NULL);
+}
+
+/**********************************************************************
  * Allocate and initialize a varnish
  */
 
@@ -145,9 +200,6 @@ varnish_new(const char *name)
 	    v->workdir, v->workdir, random(), v->workdir);
 	AZ(system(buf));
 
-	v->vl1 = vtc_logopen(name);
-	AN(v->vl1);
-
 	if (*v->name != 'v')
 		vtc_log(v->vl, 0, "Varnish name must start with 'v'");
 
@@ -218,7 +270,7 @@ varnish_thread(void *priv)
 		if (i <= 0)
 			break;
 		buf[i] = '\0';
-		vtc_dump(v->vl1, 3, "debug", buf, -1);
+		vtc_dump(v->vl, 3, "debug", buf, -1);
 	}
 	return (NULL);
 }
@@ -294,6 +346,7 @@ varnish_launch(struct varnish *v)
 	v->fds[2] = v->fds[3] = -1;
 	VSB_delete(vsb);
 	AZ(pthread_create(&v->tp, NULL, varnish_thread, v));
+	AZ(pthread_create(&v->tp_vsl, NULL, varnishlog_thread, v));
 
 	/* Wait for the varnish to call home */
 	fd[0].fd = v->cli_fd;
@@ -458,7 +511,9 @@ varnish_wait(struct varnish *v)
 	AZ(pthread_join(v->tp, &p));
 	AZ(close(v->fds[0]));
 	r = wait4(v->pid, &status, 0, NULL);
+	v->pid = 0;
 	vtc_log(v->vl, 2, "R %d Status: %04x", r, status);
+	AZ(pthread_join(v->tp_vsl, &p));
 	if (WIFEXITED(status) && WEXITSTATUS(status) == 0)
 		return;
 #ifdef WCOREDUMP



From tfheen at varnish-cache.org  Thu Sep 22 12:03:25 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:25 +0200
Subject: [3.0] 38116cf Try to make it easier to see why this test fails.
Message-ID: <E1R6hzx-0004Ra-Gm@project.varnish-software.com>

commit 38116cf37cd4b6b69979b1e6aa7742a7dd8509f3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 12:15:54 2011 +0000

    Try to make it easier to see why this test fails.

diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc
index 66f447b..faf4b4b 100644
--- a/bin/varnishtest/tests/r00962.vtc
+++ b/bin/varnishtest/tests/r00962.vtc
@@ -32,6 +32,12 @@ varnish v1 -cliok "debug.persistent s0 dump"
 varnish v1 -cliok "debug.persistent s0 sync"
 varnish v1 -stop
 
+server s1 {
+	rxreq 
+	txresp -status 400 -msg "Persistent Object Not Found"
+} -start
+
+
 varnish v2 \
 	-arg "-pdiag_bitmap=0x20000" \
 	-storage "-spersistent,${tmpdir}/_.per2,10m -spersistent,${tmpdir}/_.per1,10m" \
@@ -40,6 +46,7 @@ varnish v2 \
 client c1 -connect ${v2_sock} {
 	txreq -url "/"
 	rxresp
+	expect resp.msg != "Persistent Object Not Found"
 	expect resp.status == 200
 	expect resp.http.X-Varnish == "1001"
 } -run



From tfheen at varnish-cache.org  Thu Sep 22 12:04:16 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:04:16 +0200
Subject: [3.0] 63ec044 Look for ncurses/curses.h before curses.h
Message-ID: <E1R6i0m-0004Zn-5u@project.varnish-software.com>

commit 63ec044368d1a8a8a5b700c99e1c5ce4ea84bbbd
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Sep 21 13:41:58 2011 +0200

    Look for ncurses/curses.h before curses.h
    
    Solaris puts the ncurses header in /usr/include/ncurses, and we need
    that to compile with -Werror.
    
    Fixes: #889

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index d0eb001..1127e6a 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -34,7 +34,11 @@
 
 #include <sys/time.h>
 
+#ifdef HAVE_NCURSES_CURSES_H
+#include <ncurses/curses.h>
+#elif HAVE_CURSES_H
 #include <curses.h>
+#endif
 #include <errno.h>
 #include <signal.h>
 #include <stdio.h>
diff --git a/configure.ac b/configure.ac
index ab745fa..9ef4c67 100644
--- a/configure.ac
+++ b/configure.ac
@@ -87,6 +87,7 @@ AC_SUBST(CURSES_LIBS)
 if test "$have_curses" = no; then
 	AC_MSG_WARN([curses not found; some tools will not be built])
 fi
+AC_CHECK_HEADERS([ncurses/curses.h curses.h])
 AM_CONDITIONAL([HAVE_CURSES], [test x$have_curses = xyes])
 
 save_LIBS="${LIBS}"



From tfheen at varnish-cache.org  Thu Sep 22 12:02:42 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:42 +0200
Subject: [3.0] 184ad94 Restore old nuke_limit, as 10 seems to be too low
Message-ID: <E1R6hzG-0004K1-Ob@project.varnish-software.com>

commit 184ad943d98c64892734cca0144bed7198b5e688
Author: Andreas Plesner Jacobsen <apj at marvin.nerd.dk>
Date:   Fri Sep 9 13:33:38 2011 +0200

    Restore old nuke_limit, as 10 seems to be too low
    
    Fixes #1012

diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index de93679..33101bb 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -622,7 +622,7 @@ static const struct parspec input_parspec[] = {
 		"Maximum number of objects we attempt to nuke in order"
 		"to make space for a object body.",
 		EXPERIMENTAL,
-		"10", "allocations" },
+		"50", "allocations" },
 	{ "fetch_chunksize",
 		tweak_uint, &master.fetch_chunksize, 4, UINT_MAX / 1024.,
 		"The default chunksize used by fetcher. "



From tfheen at varnish-cache.org  Thu Sep 22 12:04:07 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:04:07 +0200
Subject: [3.0] cc7a81e Go over this code once more, and fix various nits.
Message-ID: <E1R6i0d-0004YJ-6y@project.varnish-software.com>

commit cc7a81e96dd5ac604b1dd8046effd2efa87aaa2e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 22:00:12 2011 +0000

    Go over this code once more, and fix various nits.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 8f96577..e6aaf33 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -107,7 +107,7 @@ synth_body(const char *len, int rnd)
 	AN(len);
 	i = strtoul(len, NULL, 0);
 	assert(i > 0);
-	b = malloc(i + 1);
+	b = malloc(i + 1L);
 	AN(b);
 	l = k = '!';
 	for (j = 0; j < i; j++) {
@@ -333,7 +333,7 @@ http_splitheader(struct http *hp, int req)
  */
 
 static int
-http_rxchar_eof(struct http *hp, int n)
+http_rxchar(struct http *hp, int n, int eof)
 {
 	int i;
 	struct pollfd pfd[1];
@@ -343,21 +343,25 @@ http_rxchar_eof(struct http *hp, int n)
 		pfd[0].events = POLLIN;
 		pfd[0].revents = 0;
 		i = poll(pfd, 1, hp->timeout);
-		if (i < 0)
+		if (i == 0)
 			vtc_log(hp->vl, 0, "HTTP rx timeout (fd:%d %u ms)",
 			    hp->fd, hp->timeout);
 		if (i < 0)
 			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d poll: %s)",
 			    hp->fd, strerror(errno));
 		assert(i > 0);
-		if (pfd[0].revents & ~POLLIN)
-			vtc_log(hp->vl, 4, "HTTP rx poll (fd:%d revents: %x)",
-			    hp->fd, pfd[0].revents);
 		assert(hp->prxbuf + n < hp->nrxbuf);
 		i = read(hp->fd, hp->rxbuf + hp->prxbuf, n);
-		if (i == 0)
+		if (!(pfd[0].revents & POLLIN))
+			vtc_log(hp->vl, 4,
+			    "HTTP rx poll (fd:%d revents: %x n=%d, i=%d)",
+			    hp->fd, pfd[0].revents, n, i);
+		if (i == 0 && eof)
 			return (i);
-		if (i <= 0)
+		if (i == 0)
+			vtc_log(hp->vl, 0, "HTTP rx EOF (fd:%d read: %s)",
+			    hp->fd, strerror(errno));
+		if (i < 0)
 			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d read: %s)",
 			    hp->fd, strerror(errno));
 		hp->prxbuf += i;
@@ -367,17 +371,6 @@ http_rxchar_eof(struct http *hp, int n)
 	return (1);
 }
 
-static void
-http_rxchar(struct http *hp, int n)
-{
-	int i;
-
-	i = http_rxchar_eof(hp, n);
-	if (i <= 0)
-		vtc_log(hp->vl, 0, "HTTP rx failed (%s)", strerror(errno));
-	assert(i > 0);
-}
-
 static int
 http_rxchunk(struct http *hp)
 {
@@ -386,7 +379,7 @@ http_rxchunk(struct http *hp)
 
 	l = hp->prxbuf;
 	do
-		http_rxchar(hp, 1);
+		(void)http_rxchar(hp, 1, 0);
 	while (hp->rxbuf[hp->prxbuf - 1] != '\n');
 	vtc_dump(hp->vl, 4, "len", hp->rxbuf + l, -1);
 	i = strtoul(hp->rxbuf + l, &q, 16);
@@ -400,12 +393,12 @@ http_rxchunk(struct http *hp)
 	assert(*q == '\0' || vct_islws(*q));
 	hp->prxbuf = l;
 	if (i > 0) {
-		http_rxchar(hp, i);
+		(void)http_rxchar(hp, i, 0);
 		vtc_dump(hp->vl, 4, "chunk",
 		    hp->rxbuf + l, i);
 	}
 	l = hp->prxbuf;
-	http_rxchar(hp, 2);
+	(void)http_rxchar(hp, 2, 0);
 	if(!vct_iscrlf(hp->rxbuf[l]))
 		vtc_log(hp->vl, 0,
 		    "Wrong chunk tail[0] = %02x",
@@ -433,7 +426,7 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
 	p = http_find_header(hh, "content-length");
 	if (p != NULL) {
 		l = strtoul(p, NULL, 0);
-		http_rxchar(hp, l);
+		(void)http_rxchar(hp, l, 0);
 		vtc_dump(hp->vl, 4, "body", hp->body, l);
 		hp->bodyl = l;
 		sprintf(hp->bodylen, "%d", l);
@@ -452,7 +445,7 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
 	if (body) {
 		hp->body = hp->rxbuf + hp->prxbuf;
 		do  {
-			i = http_rxchar_eof(hp, 1);
+			i = http_rxchar(hp, 1, 1);
 			ll += i;
 		} while (i > 0);
 		vtc_dump(hp->vl, 4, "rxeof", hp->body, ll);
@@ -475,7 +468,7 @@ http_rxhdr(struct http *hp)
 	hp->prxbuf = 0;
 	hp->body = NULL;
 	while (1) {
-		http_rxchar(hp, 1);
+		(void)http_rxchar(hp, 1, 0);
 		p = hp->rxbuf + hp->prxbuf - 1;
 		for (i = 0; p > hp->rxbuf; p--) {
 			if (*p != '\n')
@@ -533,7 +526,7 @@ cmd_http_rxresp(CMD_ARGS)
 
 #define TRUST_ME(ptr)   ((void*)(uintptr_t)(ptr))
 
-#define OVERHEAD 64
+#define OVERHEAD 64L
 
 
 static void
@@ -942,7 +935,7 @@ cmd_http_sendhex(CMD_ARGS)
 		if (!vct_ishex(buf[0]) || !vct_ishex(buf[1]))
 			vtc_log(hp->vl, 0, "Illegal Hex char \"%c%c\"",
 			    buf[0], buf[1]);
-		p[i] = strtoul(buf, NULL, 16);
+		p[i] = (uint8_t)strtoul(buf, NULL, 16);
 	}
 	vtc_hexdump(hp->vl, 4, "sendhex", (void*)p, i);
 	j = write(hp->fd, p, i);



From tfheen at varnish-cache.org  Thu Sep 22 12:02:50 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:50 +0200
Subject: [3.0] f8e7612 Increase the servers listen depth to 10.
Message-ID: <E1R6hzO-0004Mr-Fv@project.varnish-software.com>

commit f8e7612b6b449c2af0d4db14e780ad6a19d9af12
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 07:43:29 2011 +0000

    Increase the servers listen depth to 10.

diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 061d9cc..542a266 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -132,7 +132,7 @@ server_new(const char *name)
 	bprintf(s->listen, "127.0.0.1:%d", 0);
 	AZ(VSS_parse(s->listen, &s->addr, &s->port));
 	s->repeat = 1;
-	s->depth = 1;
+	s->depth = 10;
 	s->sock = -1;
 	VTAILQ_INSERT_TAIL(&servers, s, list);
 	return (s);



From tfheen at varnish-cache.org  Thu Sep 22 12:02:46 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:46 +0200
Subject: [3.0] b6fcb3d Avoid 32 bit variables in stats aggregation.
Message-ID: <E1R6hzK-0004LR-08@project.varnish-software.com>

commit b6fcb3de0f8fcdaedcf3151067d2aa3e8f59e575
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 12 10:13:35 2011 +0000

    Avoid 32 bit variables in stats aggregation.
    
    Fixes	#993

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 1c1952d..e2faf30 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -218,7 +218,7 @@ struct acct {
 /*--------------------------------------------------------------------*/
 
 #define L0(n)
-#define L1(n)			int n;
+#define L1(n)			uint64_t n;
 #define VSC_F(n, t, l, f, e)	L##l(n)
 #define VSC_DO_MAIN
 struct dstat {



From tfheen at varnish-cache.org  Thu Sep 22 12:02:28 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:28 +0200
Subject: [3.0] 12095cf Quench a pointless debug printf
Message-ID: <E1R6hz2-0004Hs-Kk@project.varnish-software.com>

commit 12095cf8e58868528b084e7598666c534813f209
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 20:13:07 2011 +0000

    Quench a pointless debug printf

diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index 7687179..45dfac4 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -104,13 +104,10 @@ struct smf_sc {
 /*--------------------------------------------------------------------*/
 
 static void
-smf_initfile(struct stevedore *st, struct smf_sc *sc, const char *size)
+smf_initfile(struct smf_sc *sc, const char *size)
 {
 	sc->filesize = STV_FileSize(sc->fd, size, &sc->pagesize, "-sfile");
 
-	printf("SMF.%s: filename: %s size %ju MB.\n",
-	      st->ident, sc->filename, sc->filesize / (1024 * 1024));
-
 	AZ(ftruncate(sc->fd, (off_t)sc->filesize));
 
 	/* XXX: force block allocation here or in open ? */
@@ -162,7 +159,7 @@ smf_init(struct stevedore *parent, int ac, char * const *av)
 	(void)STV_GetFile(fn, &sc->fd, &sc->filename, "-sfile");
 
 	mgt_child_inherit(sc->fd, "storage_file");
-	smf_initfile(parent, sc, size);
+	smf_initfile(sc, size);
 }
 
 /*--------------------------------------------------------------------



From tfheen at varnish-cache.org  Thu Sep 22 12:04:17 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:04:17 +0200
Subject: [3.0] 0056676 Add file_read to the privilege set we need on Solaris
Message-ID: <E1R6i0n-0004Zy-69@project.varnish-software.com>

commit 00566769df8606de4bfeac6fac18986b93aaf168
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Sep 21 13:52:03 2011 +0200

    Add file_read to the privilege set we need on Solaris
    
    Fixes: #912

diff --git a/bin/varnishd/mgt_sandbox.c b/bin/varnishd/mgt_sandbox.c
index 0299cfd..a5eee2f 100644
--- a/bin/varnishd/mgt_sandbox.c
+++ b/bin/varnishd/mgt_sandbox.c
@@ -98,6 +98,7 @@ mgt_sandbox(void)
 		 * silently ignore any errors if it doesn't exist
 		 */
 		priv_addset(minimal, "net_access");
+		priv_addset(minimal, "file_read");
 
 #define SETPPRIV(which, set)						\
 		if (setppriv(PRIV_SET, which, set))			\



From tfheen at varnish-cache.org  Thu Sep 22 12:02:44 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:44 +0200
Subject: [3.0] 1a80763 Add .expected_response to probe documentation. Fix
	error message for .expected_response.
Message-ID: <E1R6hzI-0004Kb-ON@project.varnish-software.com>

commit 1a807636089475f0f0eaf50476fb3ded42bf355e
Author: Andreas Plesner Jacobsen <apj at marvin.nerd.dk>
Date:   Fri Sep 9 14:54:24 2011 +0200

    Add .expected_response to probe documentation.
    Fix error message for .expected_response.
    
    Fixes #997

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 7359e6f..0e39187 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -284,7 +284,8 @@ req.backend.healthy .window is how many of the latest polls we
 examine, while .threshold is how many of those must have succeeded for
 us to consider the backend healthy.  .initial is how many of the
 probes are considered good when Varnish starts - defaults to the same
-amount as the threshold.
+amount as the threshold. .expected_response is the expected backend
+HTTP response code.
 
 A backend with a probe can be defined like this, together with the
 backend or director:::
@@ -310,6 +311,7 @@ Or it can be defined separately and then referenced:::
      .window = 8;
      .threshold = 3;
      .initial = 3;
+     .expected_response = 200;
   }	
 
   backend www {
diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c
index 1b8f8a0..f79447f 100644
--- a/lib/libvcl/vcc_backend.c
+++ b/lib/libvcl/vcc_backend.c
@@ -305,8 +305,9 @@ vcc_ParseProbeSpec(struct vcc *tl)
 			status = vcc_UintVal(tl);
 			if (status < 100 || status > 999) {
 				VSB_printf(tl->sb,
-				    "Must specify .status with exactly three "
-				    " digits (100 <= x <= 999)\n");
+				    "Must specify .expected_response with "
+				    "exactly three digits "
+				    "(100 <= x <= 999)\n");
 				vcc_ErrWhere(tl, tl->t);
 				return;
 			}



From tfheen at varnish-cache.org  Thu Sep 22 12:02:21 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:02:21 +0200
Subject: [3.0] 1914ab2 Prototype adjustment for last commit
Message-ID: <E1R6hyv-0004HC-Nu@project.varnish-software.com>

commit 1914ab231616a517d7fa8a088f70698d2ccd6e2f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 19:54:35 2011 +0000

    Prototype adjustment for last commit

diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index c2a52d4..f4a85a0 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -322,7 +322,7 @@ EmitFiniFunc(const struct vcc *tl)
 {
 	unsigned u;
 
-	Fc(tl, 0, "\nstatic int\nVGC_Fini(struct cli *cli)\n{\n\n");
+	Fc(tl, 0, "\nstatic void\nVGC_Fini(struct cli *cli)\n{\n\n");
 
 	/*
 	 * We do this here, so we are sure they happen before any
@@ -333,7 +333,6 @@ EmitFiniFunc(const struct vcc *tl)
 
 	AZ(VSB_finish(tl->ff));
 	VSB_cat(tl->fc, VSB_data(tl->ff));
-	Fc(tl, 0, "\treturn(0);\n");
 	Fc(tl, 0, "}\n");
 }
 



From tfheen at varnish-cache.org  Thu Sep 22 12:01:54 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:01:54 +0200
Subject: [3.0] ea3eff1 Fix handling vs Varnish:handling in varnishncsa
Message-ID: <E1R6hyU-0004By-TV@project.varnish-software.com>

commit ea3eff1e19cd88dbee9bdeea28269ff4e127ca97
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 6 15:07:03 2011 +0200

    Fix handling vs Varnish:handling in varnishncsa
    
    Thanks to Kai for patch
    Fixes: #999

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 28f6f21..70e915f 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -716,7 +716,7 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 					VSB_cat(os, (lp->df_hitmiss ? lp->df_hitmiss : "-"));
 					p = tmp;
 					break;
-				} else if (strcmp(fname, "handling") == 0) {
+				} else if (strcmp(fname, "Varnish:handling") == 0) {
 					VSB_cat(os, (lp->df_handling ? lp->df_handling : "-"));
 					p = tmp;
 					break;



From tfheen at varnish-cache.org  Thu Sep 22 12:03:08 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 22 Sep 2011 14:03:08 +0200
Subject: [3.0] d4ece17 A missing type-cast
Message-ID: <E1R6hzg-0004Oh-93@project.varnish-software.com>

commit d4ece17a31688111dffb5699783689dcc8a8185d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 08:11:29 2011 +0000

    A missing type-cast

diff --git a/bin/varnishd/storage_persistent_mgt.c b/bin/varnishd/storage_persistent_mgt.c
index c4f2191..3bd5809 100644
--- a/bin/varnishd/storage_persistent_mgt.c
+++ b/bin/varnishd/storage_persistent_mgt.c
@@ -109,7 +109,7 @@ smp_metrics(struct smp_sc *sc)
 	 */
 	sc->free_reserve = sc->aim_segl * 10;
 
-	fprintf(stderr, "free_reserve = %ju\n", sc->free_reserve);
+	fprintf(stderr, "free_reserve = %ju\n", (uintmax_t)sc->free_reserve);
 }
 
 /*--------------------------------------------------------------------



From phk at varnish-cache.org  Thu Sep 22 14:57:33 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 22 Sep 2011 16:57:33 +0200
Subject: [master] 6c696ad Add more expressive VSL's for VCL related errors.
Message-ID: <E1R6kiT-000281-V4@project.varnish-software.com>

commit 6c696adb19040d271bf783590a52d24e1fc24fc8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 11:26:34 2011 +0000

    Add more expressive VSL's for VCL related errors.

diff --git a/include/vsl_tags.h b/include/vsl_tags.h
index bae0a4a..21ca1df 100644
--- a/include/vsl_tags.h
+++ b/include/vsl_tags.h
@@ -95,6 +95,9 @@ SLTM(ESI_xmlerror)
 SLTM(Hash)
 
 SLTM(Backend_health)
+
+SLTM(VCL_Debug)
 SLTM(VCL_Log)
+SLTM(VCL_Error)
 
 SLTM(Gzip)



From phk at varnish-cache.org  Thu Sep 22 14:57:34 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 22 Sep 2011 16:57:34 +0200
Subject: [master] 74f3c13 A quick style polish
Message-ID: <E1R6kiU-000286-2W@project.varnish-software.com>

commit 74f3c13ec5b0a451480431cc1a7db58d56bb5b90
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 11:41:39 2011 +0000

    A quick style polish

diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index e97758a..667d177 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -38,41 +38,47 @@ struct vre {
 	pcre *re;
 };
 
-vre_t *VRE_compile(const char *pattern, int options,
-		    const char **errptr, int *erroffset) {
+vre_t *
+VRE_compile(const char *pattern, int options,
+		    const char **errptr, int *erroffset)
+{
 	vre_t *v;
 	*errptr = NULL; *erroffset = 0;
 
 	ALLOC_OBJ(v, VRE_MAGIC);
-	AN(v);
+	if (v == NULL)
+		return (NULL);
 	v->re = pcre_compile(pattern, options, errptr, erroffset, NULL);
 	if (v->re == NULL) {
 		VRE_free(&v);
-		return NULL;
+		return (NULL);
 	}
-	return v;
+	return (v);
 }
 
-int VRE_exec(const vre_t *code, const char *subject, int length,
-	     int startoffset, int options, int *ovector, int ovecsize) {
+int
+VRE_exec(const vre_t *code, const char *subject, int length,
+    int startoffset, int options, int *ovector, int ovecsize)
+{
 	CHECK_OBJ_NOTNULL(code, VRE_MAGIC);
 	int ov[30];
+
 	if (ovector == NULL) {
 		ovector = ov;
 		ovecsize = sizeof(ov)/sizeof(ov[0]);
 	}
 
-	return pcre_exec(code->re, NULL, subject, length,
-			 startoffset, options, ovector, ovecsize);
+	return (pcre_exec(code->re, NULL, subject, length,
+	    startoffset, options, ovector, ovecsize));
 }
 
-void VRE_free(vre_t **vv) {
-
+void
+VRE_free(vre_t **vv)
+{
 	vre_t *v = *vv;
 
 	*vv = NULL;
 	CHECK_OBJ(v, VRE_MAGIC);
 	pcre_free(v->re);
-	v->magic = 0;
 	FREE_OBJ(v);
 }



From phk at varnish-cache.org  Thu Sep 22 14:57:34 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 22 Sep 2011 16:57:34 +0200
Subject: [master] 9816a23 Give VRT_re_match a sess* arg and report VRE errors
	using SLT_VCL_Error instead of asserting.
Message-ID: <E1R6kiU-00028F-66@project.varnish-software.com>

commit 9816a237e8c62ed2992fa6b86931dbb0793e2b01
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 11:48:16 2011 +0000

    Give VRT_re_match a sess* arg and report VRE errors using SLT_VCL_Error
    instead of asserting.
    
    Inspired by:	Patch from DocWilco

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index 8e1218a..f41b809 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -67,7 +67,7 @@ VRT_re_fini(void *rep)
 }
 
 int
-VRT_re_match(const char *s, void *re)
+VRT_re_match(const struct sess *sp, const char *s, void *re)
 {
 	vre_t *t;
 	int i;
@@ -79,7 +79,8 @@ VRT_re_match(const char *s, void *re)
 	i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0);
 	if (i >= 0)
 		return (1);
-	assert(i == VRE_ERROR_NOMATCH);
+	if (i < VRE_ERROR_NOMATCH )
+		WSP(sp, SLT_VCL_Error, "Regexp matching returned %d", i);
 	return (0);
 }
 
@@ -105,6 +106,10 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 	/* If it didn't match, we can return the original string */
 	if (i == VRE_ERROR_NOMATCH)
 		return(str);
+	if (i < VRE_ERROR_NOMATCH ) {
+		WSP(sp, SLT_VCL_Error, "Regexp matching returned %d", i);
+		return(str);
+	}
 
 	u = WS_Reserve(sp->http->ws, 0);
 	res.e = res.b = b0 = sp->http->ws->f;
@@ -135,6 +140,11 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 			break;
 		memset(&ovector, 0, sizeof(ovector));
 		i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30);
+		if (i < VRE_ERROR_NOMATCH ) {
+			WSP(sp, SLT_VCL_Error,
+			    "Regexp matching returned %d", i);
+			return(str);
+		}
 	} while (i != VRE_ERROR_NOMATCH);
 
 	/* Copy suffix to match */
diff --git a/include/vrt.h b/include/vrt.h
index 9f0228f..72c4cfe 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -145,7 +145,7 @@ void VRT_acl_log(const struct sess *, const char *msg);
 /* Regexp related */
 void VRT_re_init(void **, const char *);
 void VRT_re_fini(void *);
-int VRT_re_match(const char *, void *re);
+int VRT_re_match(const struct sess *sp, const char *, void *re);
 const char *VRT_regsub(const struct sess *sp, int all, const char *,
     void *, const char *);
 
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 3883dd2..1f1c274 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -941,7 +941,7 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, enum var_type fmt)
 		re = vcc_regexp(tl);
 		ERRCHK(tl);
 		vcc_NextToken(tl);
-		bprintf(buf, "%sVRT_re_match(\v1, %s)", not, re);
+		bprintf(buf, "%sVRT_re_match(sp, \v1, %s)", not, re);
 		*e = vcc_expr_edit(BOOL, buf, *e, NULL);
 		return;
 	}



From phk at varnish-cache.org  Thu Sep 22 14:57:34 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 22 Sep 2011 16:57:34 +0200
Subject: [master] cb4bf6d Add an XXX comment
Message-ID: <E1R6kiU-00028R-CH@project.varnish-software.com>

commit cb4bf6db68a974355745a7f73c1b72d2facd6f6c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 11:53:56 2011 +0000

    Add an XXX comment

diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index 4c95592..fed958c 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -286,6 +286,8 @@ ban_parse_http(const struct ban *b, const char *a1)
 
 /*--------------------------------------------------------------------
  * Parse and add a ban test specification
+ *
+ * XXX: This should vector through VRE.
  */
 
 static int



From phk at varnish-cache.org  Thu Sep 22 14:57:35 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 22 Sep 2011 16:57:35 +0200
Subject: [master] 8a385bf Make it possible to set limits for VRE matching.
Message-ID: <E1R6kiV-00028e-1P@project.varnish-software.com>

commit 8a385bfc803409428e37e23c4d7f844d03bb74c4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 12:15:52 2011 +0000

    Make it possible to set limits for VRE matching.

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index f41b809..b2afaa5 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -76,7 +76,7 @@ VRT_re_match(const struct sess *sp, const char *s, void *re)
 		s = "";
 	AN(re);
 	t = re;
-	i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0);
+	i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0, &params->vre_limits);
 	if (i >= 0)
 		return (1);
 	if (i < VRE_ERROR_NOMATCH )
@@ -101,7 +101,8 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 		str = "";
 	t = re;
 	memset(ovector, 0, sizeof(ovector));
-	i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30);
+	i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30,
+	    &params->vre_limits);
 
 	/* If it didn't match, we can return the original string */
 	if (i == VRE_ERROR_NOMATCH)
@@ -139,7 +140,8 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 		if (!all)
 			break;
 		memset(&ovector, 0, sizeof(ovector));
-		i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30);
+		i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30,
+		    &params->vre_limits);
 		if (i < VRE_ERROR_NOMATCH ) {
 			WSP(sp, SLT_VCL_Error,
 			    "Regexp matching returned %d", i);
diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index 13fce78..7aebf11 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -30,6 +30,7 @@
  */
 
 #include <pthread.h>
+#include "vre.h"
 
 struct listen_sock {
 	unsigned			magic;
@@ -213,6 +214,8 @@ struct params {
 	double			critbit_cooloff;
 
 	double			shortlived;
+
+	struct vre_limits	vre_limits;
 };
 
 /*
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index decd130..f247b71 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -951,6 +951,24 @@ static const struct parspec input_parspec[] = {
 		"Unreferenced VCL objects result in error.\n",
 		0,
 		"on", "bool" },
+
+
+	{ "pcre_match_limit", tweak_uint,
+		&master.vre_limits.match,
+		1, UINT_MAX,
+		"The limit for the  number of internal matching function"
+		" calls in a pcre_exec() execution.",
+		0,
+		"10000", ""},
+
+	{ "pcre_match_limit_recursion", tweak_uint,
+		&master.vre_limits.match_recursion,
+		1, UINT_MAX,
+		"The limit for the  number of internal matching function"
+		" recursions in a pcre_exec() execution.",
+		0,
+		"10000", ""},
+
 	{ NULL, NULL, NULL }
 };
 
diff --git a/include/vre.h b/include/vre.h
index 47dea3f..29c1b08 100644
--- a/include/vre.h
+++ b/include/vre.h
@@ -27,9 +27,21 @@
  *
  * Regular expression support
  *
+ * We wrap PCRE in VRE to make to make it feasible to use something else
+ * without hunting down stuff through out the Varnish source code.
+ *
  */
 
+#ifndef VRE_H_INCLUDED
+#define VRE_H_INCLUDED
+
 struct vre;
+
+struct vre_limits {
+	unsigned	match;
+	unsigned	match_recursion;
+};
+
 typedef struct vre vre_t;
 
 /* This maps to PCRE error codes */
@@ -39,5 +51,9 @@ typedef struct vre vre_t;
 #define VRE_CASELESS           0x00000001
 
 vre_t *VRE_compile(const char *, int, const char **, int *);
-int VRE_exec(const vre_t *, const char *, int, int, int, int *, int);
+int VRE_exec(const vre_t *code, const char *subject, int length,
+    int startoffset, int options, int *ovector, int ovecsize,
+    const volatile struct vre_limits *lim);
 void VRE_free(vre_t **);
+
+#endif /* VRE_H_INCLUDED */
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 667d177..68beaf8 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -27,6 +27,7 @@
  */
 
 #include <pcre.h>
+#include <string.h>
 
 #include "libvarnish.h"
 #include "miniobj.h"
@@ -58,17 +59,27 @@ VRE_compile(const char *pattern, int options,
 
 int
 VRE_exec(const vre_t *code, const char *subject, int length,
-    int startoffset, int options, int *ovector, int ovecsize)
+    int startoffset, int options, int *ovector, int ovecsize,
+    const volatile struct vre_limits *lim)
 {
 	CHECK_OBJ_NOTNULL(code, VRE_MAGIC);
 	int ov[30];
+	pcre_extra extra;
 
 	if (ovector == NULL) {
 		ovector = ov;
 		ovecsize = sizeof(ov)/sizeof(ov[0]);
 	}
 
-	return (pcre_exec(code->re, NULL, subject, length,
+	memset(&extra, 0, sizeof extra);
+	if (lim != NULL) {
+		extra.match_limit = lim->match;
+		extra.flags |= PCRE_EXTRA_MATCH_LIMIT;
+		extra.match_limit_recursion = lim->match_recursion;
+		extra.flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION;
+	}
+
+	return (pcre_exec(code->re, &extra, subject, length,
 	    startoffset, options, ovector, ovecsize));
 }
 
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 58a454d..8473d31 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -266,13 +266,13 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 			continue;
 		if (vsl->regincl != NULL) {
 			i = VRE_exec(vsl->regincl, VSL_DATA(p), VSL_LEN(p),
-			    0, 0, NULL, 0);
+			    0, 0, NULL, 0, NULL);
 			if (i == VRE_ERROR_NOMATCH)
 				continue;
 		}
 		if (vsl->regexcl != NULL) {
 			i = VRE_exec(vsl->regexcl, VSL_DATA(p), VSL_LEN(p),
-			    0, 0, NULL, 0);
+			    0, 0, NULL, 0, NULL);
 			if (i != VRE_ERROR_NOMATCH)
 				continue;
 		}
@@ -282,7 +282,7 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 			VTAILQ_FOREACH(vrm, &vsl->matchers, next) {
 				if (vrm->tag == t) {
 					i = VRE_exec(vrm->re, VSL_DATA(p),
-						     VSL_LEN(p), 0, 0, NULL, 0);
+					    VSL_LEN(p), 0, 0, NULL, 0, NULL);
 					if (i >= 0)
 						*bits |= (uintmax_t)1 << j;
 				}



From phk at varnish-cache.org  Thu Sep 22 14:57:35 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 22 Sep 2011 16:57:35 +0200
Subject: [master] 194d664 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1R6kiV-00028t-8e@project.varnish-software.com>

commit 194d664e2308d3caca6f465d99aef0638fa0ce36
Merge: 8a385bf 701e5c3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 14:57:11 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From tfheen at varnish-cache.org  Fri Sep 23 12:21:41 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Fri, 23 Sep 2011 14:21:41 +0200
Subject: [master] 5fc5c45 Pass warnings and error flags to distcheck
Message-ID: <E1R74lB-0004Bq-Uq@project.varnish-software.com>

commit 5fc5c45ca001714f6f64ebca5a4c720f4dc81bac
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Sep 21 14:36:55 2011 +0200

    Pass warnings and error flags to distcheck

diff --git a/Makefile.am b/Makefile.am
index bc0bfc2..dad3621 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -11,6 +11,15 @@ pkgconfig_DATA = varnishapi.pc
 
 EXTRA_DIST = LICENSE autogen.sh varnishapi.pc.in
 
+DISTCHECK_CONFIGURE_FLAGS = \
+    --enable-developer-warnings \
+    --enable-debugging-symbols \
+    --enable-dependency-tracking \
+    --enable-diagnostics \
+    --enable-extra-developer-warnings \
+    --enable-tests \
+    --enable-werror
+
 install-data-local:
 	$(install_sh) -d -m 0755 $(DESTDIR)$(localstatedir)/varnish
 



From phk at varnish-cache.org  Fri Sep 30 13:32:10 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 30 Sep 2011 15:32:10 +0200
Subject: [master] f837fbc Split solaris sandboxing out to a separate source
	file, and apply patch received from Nils Goroll <nils.goroll@uplex.de>
Message-ID: <E1R9dCE-0004H3-Rd@project.varnish-software.com>

commit f837fbca893cc09458482c5283456bf8990aeee6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Sep 30 13:30:43 2011 +0000

    Split solaris sandboxing out to a separate source file, and apply
    patch received from Nils Goroll <nils.goroll at uplex.de>
    
    - [e0ee2a2e69654a9df74aaf3dcadc9639659cf42b] adds the file_read
      privilege needed for onnv_140 and newer (see #912), but we also need
      the file_write privilege for stevedore access.
    
    - If available, keep sys_resource in the permitted/limited set to
      allow cache_waiter_ports to raise the process.max-port-events
      resource control (feature to be added later).
    
    - When starting varnish with euid 0 on Solaris, privilege seperation
      prohibited preserving additional privileges (in excess of the basic
      set) in the child, because, for a non privilege aware process,
      setuid() resets the effective, inheritable and permitted sets to the
      basic set.
    
      To achieve interoperability between solaris privileges and
      setuid()/setgid(), we now make the varnish child privilege aware
      before calling setuid() by trying to add all privileges we will need
      plus proc_setid.
    
    - On solaris, check for proc_setid rather than checking the euid as a
      prerequisite for changing the uid/gid and only change the uid/gid if
      we need to (for a privilege aware process, [ers]uid 0 loose their
      magic powers).
    
      Note that setuid() will always set SNOCD on Solaris, which will
      prevent core dumps from being written, unless setuid core dumps are
      explicitly enabled using coreadm(1M).
    
      To avoid setuid() (and the SNOCD flag, consequently), start varnish
      as the user you intend to run the child as, but with additional
      privileges, e.g. using
    
      ppriv -e -s A=basic,net_privaddr,sys_resource varnishd ...
    
    - setppriv(PRIV_SET, ...) failed when the privileges to be applied
      were not available in the permitted set.
    
      We change the logic to only clear the privileges which are not
      needed by inverting the sets and removing all unneeded privileges
      using setppriv(PRIV_OFF, ...).
    
      So the child might end up with less privileges than given initially,

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index c13f4e5..0dbc745 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -60,6 +60,7 @@ varnishd_SOURCES = \
 	mgt_param.c \
 	mgt_pool.c \
 	mgt_sandbox.c \
+	mgt_sandbox_solaris.c \
 	mgt_shmem.c \
 	mgt_vcc.c \
 	rfc2616.c \
diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
index 485de26..b6dac01 100644
--- a/bin/varnishd/mgt.h
+++ b/bin/varnishd/mgt.h
@@ -71,6 +71,12 @@ void MCF_DumpRst(void);
 /* mgt_sandbox.c */
 void mgt_sandbox(void);
 
+/* mgt_sandbox_solaris.c */
+#ifdef HAVE_SETPPRIV
+void mgt_sandbox_solaris_init(void)
+void mgt_sandbox_solaris_fini(void)
+#endif
+
 /* mgt_shmem.c */
 void mgt_SHM_Init(const char *arg);
 void mgt_SHM_Pid(void);
diff --git a/bin/varnishd/mgt_sandbox.c b/bin/varnishd/mgt_sandbox.c
index a5eee2f..94f9f4d 100644
--- a/bin/varnishd/mgt_sandbox.c
+++ b/bin/varnishd/mgt_sandbox.c
@@ -48,10 +48,6 @@
 #include <syslog.h>
 #include <unistd.h>
 
-#ifdef HAVE_PRIV_H
-#include <priv.h>
-#endif
-
 #ifdef __linux__
 #include <sys/prctl.h>
 #endif
@@ -67,6 +63,10 @@ void
 mgt_sandbox(void)
 {
 
+#ifdef HAVE_SETPPRIV
+	mgt_sandbox_solaris_init();
+#endif
+
 	if (geteuid() == 0) {
 		XXXAZ(setgid(params->gid));
 		XXXAZ(setuid(params->uid));
@@ -84,35 +84,7 @@ mgt_sandbox(void)
 #endif
 
 #ifdef HAVE_SETPPRIV
-	priv_set_t *empty, *minimal;
-
-	if (!(empty = priv_allocset()) ||
-	    !(minimal = priv_allocset())) {
-		REPORT0(LOG_ERR, "priv_allocset_failed");
-	} else {
-		priv_emptyset(empty);
-		priv_emptyset(minimal);
-
-		/*
-		 * new privilege,
-		 * silently ignore any errors if it doesn't exist
-		 */
-		priv_addset(minimal, "net_access");
-		priv_addset(minimal, "file_read");
-
-#define SETPPRIV(which, set)						\
-		if (setppriv(PRIV_SET, which, set))			\
-			REPORT0(LOG_ERR,				\
-			    "Waiving privileges failed on " #which)
-
-		/* need to set I after P to avoid SNOCD being set */
-		SETPPRIV(PRIV_LIMIT, minimal);
-		SETPPRIV(PRIV_PERMITTED, minimal); /* implies PRIV_EFFECTIVE */
-		SETPPRIV(PRIV_INHERITABLE, empty);
-
-		priv_freeset(empty);
-		priv_freeset(minimal);
-	}
+	mgt_sandbox_solaris_fini();
 #endif
 
 }
diff --git a/bin/varnishd/mgt_sandbox_solaris.c b/bin/varnishd/mgt_sandbox_solaris.c
new file mode 100644
index 0000000..15b7c95
--- /dev/null
+++ b/bin/varnishd/mgt_sandbox_solaris.c
@@ -0,0 +1,219 @@
+/*-
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *	   Nils Goroll <nils.goroll at uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Sandboxing child processes on Solaris
+ *
+ */
+
+#include "config.h"
+
+#ifdef HAVE_SETPPRIV
+
+#include <stdio.h>
+#include <syslog.h>
+#include <unistd.h>
+#include <string.h>
+
+#ifdef HAVE_PRIV_H
+#include <priv.h>
+#endif
+
+#include "mgt.h"
+#include "heritage.h"
+
+/*--------------------------------------------------------------------
+ * SOLARIS PRIVILEGES: Note on use of symbolic PRIV_* constants
+ * 
+ * For privileges which existed in Solaris 10 FCS, we may use the constants from
+ * sys/priv_names.h
+ *
+ * For privileges which have been added later, we need to use strings in order
+ * not to break builds of varnish on these platforms. To remain binary
+ * compatible, we need to silently ignore errors from priv_addset when using
+ * these strings.
+ *
+ * For optimal build and binary forward comatibility, we could use subtractive
+ * set specs like
+ *
+ *       basic,!file_link_any,!proc_exec,!proc_fork,!proc_info,!proc_session
+ *
+ * but I (Nils) have a preference for making an informed decision about which
+ * privileges the varnish child should have and which it shouldn't.
+ *
+ * Newly introduced privileges should be annotated with their PSARC / commit ID
+ * (as long as Oracle reveils these :/ )
+ *
+ * SOLARIS PRIVILEGES: Note on accidentally setting the SNOCD flag
+ *
+ * When setting privileges, we need to take care not to accidentally set the
+ * SNOCD flag which will disable core dumps unnecessarily. (see
+ * https://www.varnish-cache.org/trac/ticket/671 )
+ *
+ * When changing the logic herein, always check with mdb -k. Replace _PID_ with
+ * the pid of your varnish child, the result should be 0, otherwise a regression
+ * has been introduced.
+ *
+ * > 0t_PID_::pid2proc | ::print proc_t p_flag | >a
+ * > (<a & 0x10000000)=X
+ *                 0
+ *
+ * (a value of 0x10000000 indicates that SNOCD is set)
+ *
+ * NOTE that on Solaris changing the uid will _always_ set SNOCD, so make sure
+ * you run this test with appropriate privileges, but without proc_setid, so
+ * varnish won't setuid(), e.g.
+ *
+ * pfexec ppriv -e -s A=basic,net_privaddr,sys_resource varnish ...
+ *
+ * SOLARIS COREDUMPS with setuid(): See coreadm(1M) - global-setid / proc-setid
+ *
+ */
+
+/* effective during runtime of the child */
+static inline void
+mgt_sandbox_solaris_add_effective(priv_set_t *pset)
+{
+	/* PSARC/2009/685 - 8eca52188202 - onnv_132 */
+	priv_addset(pset, "net_access");
+
+	/* PSARC/2009/378 - 63678502e95e - onnv_140 */
+	priv_addset(pset, "file_read");
+	priv_addset(pset, "file_write");
+}
+
+/* permitted during runtime of the child - for privilege bracketing */
+static inline void
+mgt_sandbox_solaris_add_permitted(priv_set_t *pset)
+{
+	/* for raising limits in cache_waiter_ports.c */
+	priv_addset(pset, PRIV_SYS_RESOURCE);
+}
+
+/* effective during mgt_sandbox */
+static inline void
+mgt_sandbox_solaris_add_initial(priv_set_t *pset)
+{
+	/* for setgid/setuid */
+	priv_addset(pset, PRIV_PROC_SETID);
+}
+
+/*
+ * if we are not yet privilege-aware already (ie we have been started
+ * not-privilege aware wird euid 0), we need to grab any additional privileges
+ * needed during mgt_standbox, until we reduce to least privileges in
+ * mgt_sandbox_waive, otherwise we would loose them with setuid()
+ */
+
+void
+mgt_sandbox_solaris_init(void)
+{
+	priv_set_t *priv_all;
+
+	if (! (priv_all = priv_allocset())) {
+		REPORT(LOG_ERR,
+		    "Child start warning: mgt_sandbox_init - priv_allocset failed: errno=%d (%s)",
+		    errno, strerror(errno));
+		return;
+	}
+	
+	priv_emptyset(priv_all);
+
+	mgt_sandbox_solaris_add_effective(priv_all);
+	mgt_sandbox_solaris_add_permitted(priv_all);
+	mgt_sandbox_solaris_add_initial(priv_all);
+
+	setppriv(PRIV_ON, PRIV_PERMITTED, priv_all);
+	setppriv(PRIV_ON, PRIV_EFFECTIVE, priv_all);
+	setppriv(PRIV_ON, PRIV_INHERITABLE, priv_all);
+
+	priv_freeset(priv_all);
+}
+
+/* 
+ * Waive most privileges in the child
+ *
+ * as of onnv_151a, we should end up with:
+ *
+ * > ppriv -v #pid of varnish child
+ * PID:  .../varnishd ...
+ * flags = PRIV_AWARE
+ *      E: file_read,file_write,net_access
+ *      I: none
+ *      P: file_read,file_write,net_access,sys_resource
+ *      L: file_read,file_write,net_access,sys_resource
+ *
+ * We should keep sys_resource in P in order to adjust our limits if we need to
+ */
+
+void
+mgt_sandbox_solaris_fini(void)
+{
+	priv_set_t *effective, *inheritable, *permitted;
+
+	if (!(effective = priv_allocset()) ||
+	    !(inheritable = priv_allocset()) ||
+	    !(permitted = priv_allocset())) {
+		REPORT(LOG_ERR,
+		    "Child start warning: mgt_sandbox_waive - priv_allocset failed: errno=%d (%s)",
+		    errno, strerror(errno));
+		return;
+	}
+
+	priv_emptyset(inheritable);
+
+	priv_emptyset(effective);
+	mgt_sandbox_solaris_add_effective(effective);
+
+	priv_copyset(effective, permitted);
+	mgt_sandbox_solaris_add_permitted(permitted);
+
+	/* 
+	 * invert the sets and clear privileges such that setppriv will always
+	 * succeed
+	 */
+	priv_inverse(inheritable);
+	priv_inverse(effective);
+	priv_inverse(permitted);
+
+#define SETPPRIV(which, set)						\
+	if (setppriv(PRIV_OFF, which, set))				\
+		REPORT(LOG_ERR,						\
+		    "Child start warning: Waiving privileges failed on %s: errno=%d (%s)", \
+		    #which, errno, strerror(errno));
+
+	SETPPRIV(PRIV_INHERITABLE, inheritable);
+	SETPPRIV(PRIV_EFFECTIVE, effective);
+	SETPPRIV(PRIV_PERMITTED, permitted);
+	SETPPRIV(PRIV_LIMIT, permitted);
+#undef SETPPRIV
+
+	priv_freeset(inheritable);
+	priv_freeset(effective);
+}
+
+#endif /* HAVE_SETPPRIV */



From phk at varnish-cache.org  Fri Sep 30 15:11:46 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 30 Sep 2011 17:11:46 +0200
Subject: [master] 009eb1c Give VSL records a backend and client tag (2^30 fd's
	is enough for everybody!) and tag them at generation,
	rather than having varnishapi try to deduce each VSL's relationship.
Message-ID: <E1R9ekc-0002vM-Sx@project.varnish-software.com>

commit 009eb1c50e5dc3f6f1419636621327348ce22eb2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Sep 30 15:09:35 2011 +0000

    Give VSL records a backend and client tag (2^30 fd's is enough for
    everybody!) and tag them at generation, rather than having varnishapi
    try to deduce each VSL's relationship.
    
    Please yell if you have records in your varnishlog output which need
    tagging or which are tagged wrong.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 6c80714..5c793e9 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -199,6 +199,7 @@ struct http_conn {
 #define HTTP_CONN_MAGIC		0x3e19edd1
 
 	int			fd;
+	unsigned		vsl_id;
 	unsigned		maxbytes;
 	unsigned		maxhdr;
 	struct ws		*ws;
@@ -542,7 +543,7 @@ struct sess {
 	unsigned		magic;
 #define SESS_MAGIC		0x2c2f9c5a
 	int			fd;
-	int			id;
+	unsigned		vsl_id;
 	unsigned		xid;
 
 	int			restarts;
@@ -632,6 +633,7 @@ struct vbc {
 	VTAILQ_ENTRY(vbc)	list;
 	struct backend		*backend;
 	struct vdi_simple	*vdis;
+	unsigned		vsl_id;
 	int			fd;
 
 	struct sockaddr_storage	*addr;
@@ -756,7 +758,8 @@ const char *http_StatusMessage(unsigned);
 unsigned http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd);
 void HTTP_Init(void);
 void http_ClrHeader(struct http *to);
-unsigned http_Write(struct worker *w, const struct http *hp, int resp);
+unsigned http_Write(struct worker *w, unsigned vsl_id, const struct http *hp,
+    int resp);
 void http_CopyResp(struct http *to, const struct http *fm);
 void http_SetResp(struct http *to, const char *proto, uint16_t status,
     const char *response);
@@ -792,8 +795,8 @@ void http_Unset(struct http *hp, const char *hdr);
 void http_CollectHdr(struct http *hp, const char *hdr);
 
 /* cache_httpconn.c */
-void HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned maxbytes,
-    unsigned maxhdr);
+void HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned vsl_id,
+    unsigned maxbytes, unsigned maxhdr);
 int HTC_Reinit(struct http_conn *htc);
 int HTC_Rx(struct http_conn *htc);
 ssize_t HTC_Read(struct http_conn *htc, void *d, size_t len);
@@ -887,10 +890,10 @@ void WSL_Flush(struct worker *w, int overflow);
 	} while (0)
 
 #define WSP(sess, tag, ...)					\
-	WSL((sess)->wrk, tag, (sess)->fd, __VA_ARGS__)
+	WSL((sess)->wrk, tag, (sess)->vsl_id, __VA_ARGS__)
 
 #define WSPR(sess, tag, txt)					\
-	WSLR((sess)->wrk, tag, (sess)->fd, txt)
+	WSLR((sess)->wrk, tag, (sess)->vsl_id, txt)
 
 #define INCOMPL() do {							\
 	VSL(SLT_Debug, 0, "INCOMPLETE AT: %s(%d)", __func__, __LINE__); \
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 1d73a1a..ba28a39 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -277,7 +277,7 @@ VCA_SetupSess(struct worker *w)
 	sp = w->sp;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	sp->fd = wa->acceptsock;
-	sp->id = wa->acceptsock;
+	sp->vsl_id = wa->acceptsock | VSL_CLIENTMARKER ;
 	wa->acceptsock = -1;
 	sp->t_open = TIM_real();
 	sp->t_end = sp->t_open;
diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index 3d6a405..d2bc860 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -118,8 +118,6 @@ vbe_TryConnect(const struct sess *sp, int pf, const struct sockaddr_storage *sa,
 {
 	int s, i, tmo;
 	double tmod;
-	char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE];
-	char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE];
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
@@ -139,11 +137,6 @@ vbe_TryConnect(const struct sess *sp, int pf, const struct sockaddr_storage *sa,
 		return (-1);
 	}
 
-	VTCP_myname(s, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
-	VTCP_name(sa, salen, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2);
-	WSL(sp->wrk, SLT_BackendOpen, s, "%s %s %s %s %s",
-	    vs->backend->vcl_name, abuf1, pbuf1, abuf2, pbuf2);
-
 	return (s);
 }
 
@@ -154,6 +147,8 @@ bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs)
 {
 	int s;
 	struct backend *bp = vs->backend;
+	char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE];
+	char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE];
 
 	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
 
@@ -191,7 +186,15 @@ bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs)
 		Lck_Unlock(&bp->mtx);
 		vc->addr = NULL;
 		vc->addrlen = 0;
+	} else {
+		vc->vsl_id = s | VSL_BACKENDMARKER;
+		VTCP_myname(s, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
+		VTCP_name(vc->addr, vc->addrlen,
+		    abuf2, sizeof abuf2, pbuf2, sizeof pbuf2);
+		WSL(sp->wrk, SLT_BackendOpen, vc->vsl_id, "%s %s %s %s %s",
+		    vs->backend->vcl_name, abuf1, pbuf1, abuf2, pbuf2);
 	}
+
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 32a4215..38ec4f2 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -341,7 +341,7 @@ cnt_done(struct sess *sp)
 			WSP(sp, SLT_Length, "%ju",
 			    (uintmax_t)sp->req_bodybytes);
 		}
-		WSL(sp->wrk, SLT_ReqEnd, sp->id, "%u %.9f %.9f %.9f %.9f %.9f",
+		WSP(sp, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f",
 		    sp->xid, sp->t_req, sp->t_end, dh, dp, da);
 	}
 	sp->xid = 0;
@@ -818,9 +818,9 @@ cnt_fetchbody(struct sess *sp)
 
 	hp2->logtag = HTTP_Obj;
 	http_CopyResp(hp2, hp);
-	http_FilterFields(sp->wrk, sp->fd, hp2, hp,
+	http_FilterFields(sp->wrk, sp->vsl_id, hp2, hp,
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS);
-	http_CopyHome(sp->wrk, sp->fd, hp2);
+	http_CopyHome(sp->wrk, sp->vsl_id, hp2);
 
 	if (http_GetHdr(hp, H_Last_Modified, &b))
 		sp->obj->last_modified = TIM_parse(b);
@@ -968,7 +968,7 @@ cnt_first(struct sess *sp)
 	sp->ws_ses = WS_Snapshot(sp->ws);
 
 	/* Receive a HTTP protocol request */
-	HTC_Init(sp->htc, sp->ws, sp->fd, params->http_req_size,
+	HTC_Init(sp->htc, sp->ws, sp->fd, sp->vsl_id, params->http_req_size,
 	    params->http_req_hdr_len);
 	sp->wrk->lastused = sp->t_open;
 	sp->wrk->acct_tmp.sess++;
@@ -1537,12 +1537,11 @@ static void
 cnt_diag(struct sess *sp, const char *state)
 {
 	if (sp->wrk != NULL) {
-		WSL(sp->wrk, SLT_Debug, sp->id,
-		    "thr %p STP_%s sp %p obj %p vcl %p",
+		WSP(sp, SLT_Debug, "thr %p STP_%s sp %p obj %p vcl %p",
 		    pthread_self(), state, sp, sp->obj, sp->vcl);
 		WSL_Flush(sp->wrk, 0);
 	} else {
-		VSL(SLT_Debug, sp->id,
+		VSL(SLT_Debug, sp->vsl_id,
 		    "thr %p STP_%s sp %p obj %p vcl %p",
 		    pthread_self(), state, sp, sp->obj, sp->vcl);
 	}
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index ea4951f..8b2edba 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -105,7 +105,7 @@ ved_include(struct sess *sp, const char *src, const char *host)
 			break;
 		AZ(sp->wrk);
 		WSL_Flush(w, 0);
-		DSL(0x20, SLT_Debug, sp->id, "loop waiting for ESI");
+		DSL(0x20, SLT_Debug, sp->vsl_id, "loop waiting for ESI");
 		(void)usleep(10000);
 	}
 	sp->xid = sxid;
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 73edaa1..6451f10 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -414,7 +414,7 @@ FetchHdr(struct sess *sp)
 
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
 	WRW_Reserve(w, &vc->fd);
-	(void)http_Write(w, hp, 0);	/* XXX: stats ? */
+	(void)http_Write(w, vc->vsl_id, hp, 0);	/* XXX: stats ? */
 
 	/* Deal with any message-body the request might have */
 	i = FetchReqBody(sp);
@@ -434,7 +434,7 @@ FetchHdr(struct sess *sp)
 
 	/* Receive response */
 
-	HTC_Init(w->htc, w->ws, vc->fd, params->http_resp_size,
+	HTC_Init(w->htc, w->ws, vc->fd, vc->vsl_id, params->http_resp_size,
 	    params->http_resp_hdr_len);
 
 	VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout);
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index faf867a..58c3b6a 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -503,7 +503,7 @@ hsh_rush(struct objhead *oh)
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 		AZ(sp->wrk);
 		VTAILQ_REMOVE(&wl->list, sp, list);
-		DSL(0x20, SLT_Debug, sp->id, "off waiting list");
+		DSL(0x20, SLT_Debug, sp->vsl_id, "off waiting list");
 		if (SES_Schedule(sp)) {
 			/*
 			 * We could not schedule the session, leave the
diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index f77ab74..587b6bb 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -77,10 +77,11 @@ http2shmlog(const struct http *hp, int t)
 }
 
 static void
-WSLH(struct worker *w, int fd, const struct http *hp, unsigned hdr)
+WSLH(struct worker *w, int vsl_id, const struct http *hp, unsigned hdr)
 {
 
-	WSLR(w, http2shmlog(hp, hdr), fd, hp->hd[hdr]);
+	AN(vsl_id & (VSL_CLIENTMARKER|VSL_BACKENDMARKER));
+	WSLR(w, http2shmlog(hp, hdr), vsl_id, hp->hd[hdr]);
 }
 
 /*--------------------------------------------------------------------*/
@@ -485,7 +486,7 @@ http_GetReq(const struct http *hp)
  */
 
 static uint16_t
-http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p,
+http_dissect_hdrs(struct worker *w, struct http *hp, int vsl_id, char *p,
     const struct http_conn *htc)
 {
 	char *q, *r;
@@ -522,7 +523,7 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p,
 
 		if (q - p > htc->maxhdr) {
 			VSC_C_main->losthdr++;
-			WSL(w, SLT_LostHeader, fd, "%.*s",
+			WSL(w, SLT_LostHeader, vsl_id, "%.*s",
 			    q - p > 20 ? 20 : q - p, p);
 			return (413);
 		}
@@ -544,11 +545,11 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p,
 			hp->hdf[hp->nhd] = 0;
 			hp->hd[hp->nhd].b = p;
 			hp->hd[hp->nhd].e = q;
-			WSLH(w, fd, hp, hp->nhd);
+			WSLH(w, vsl_id, hp, hp->nhd);
 			hp->nhd++;
 		} else {
 			VSC_C_main->losthdr++;
-			WSL(w, SLT_LostHeader, fd, "%.*s",
+			WSL(w, SLT_LostHeader, vsl_id, "%.*s",
 			    q - p > 20 ? 20 : q - p, p);
 			return (413);
 		}
@@ -561,7 +562,7 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p,
  */
 
 static uint16_t
-http_splitline(struct worker *w, int fd, struct http *hp,
+http_splitline(struct worker *w, int vsl_id, struct http *hp,
     const struct http_conn *htc, int h1, int h2, int h3)
 {
 	char *p, *q;
@@ -623,17 +624,17 @@ http_splitline(struct worker *w, int fd, struct http *hp,
 	p += vct_skipcrlf(p);
 
 	*hp->hd[h1].e = '\0';
-	WSLH(w, fd, hp, h1);
+	WSLH(w, vsl_id, hp, h1);
 
 	*hp->hd[h2].e = '\0';
-	WSLH(w, fd, hp, h2);
+	WSLH(w, vsl_id, hp, h2);
 
 	if (hp->hd[h3].e != NULL) {
 		*hp->hd[h3].e = '\0';
-		WSLH(w, fd, hp, h3);
+		WSLH(w, vsl_id, hp, h3);
 	}
 
-	return (http_dissect_hdrs(w, hp, fd, p, htc));
+	return (http_dissect_hdrs(w, hp, vsl_id, p, htc));
 }
 
 /*--------------------------------------------------------------------*/
@@ -668,7 +669,7 @@ http_DissectRequest(struct sess *sp)
 
 	hp->logtag = HTTP_Rx;
 
-	retval = http_splitline(sp->wrk, sp->fd, hp, htc,
+	retval = http_splitline(sp->wrk, sp->vsl_id, hp, htc,
 	    HTTP_HDR_REQ, HTTP_HDR_URL, HTTP_HDR_PROTO);
 	if (retval != 0) {
 		WSPR(sp, SLT_HttpGarbage, htc->rxbuf);
@@ -693,7 +694,7 @@ http_DissectResponse(struct worker *w, const struct http_conn *htc,
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	hp->logtag = HTTP_Rx;
 
-	if (http_splitline(w, htc->fd, hp, htc,
+	if (http_splitline(w, htc->vsl_id, hp, htc,
 	    HTTP_HDR_PROTO, HTTP_HDR_STATUS, HTTP_HDR_RESPONSE))
 		retval = 503;
 
@@ -719,7 +720,7 @@ http_DissectResponse(struct worker *w, const struct http_conn *htc,
 	}
 
 	if (retval != 0) {
-		WSLR(w, SLT_HttpGarbage, htc->fd, htc->rxbuf);
+		WSLR(w, SLT_HttpGarbage, htc->vsl_id, htc->rxbuf);
 		assert(retval >= 100 && retval <= 999);
 		hp->status = retval;
 	} else {
@@ -791,7 +792,7 @@ http_SetResp(struct http *to, const char *proto, uint16_t status,
 }
 
 static void
-http_copyheader(struct worker *w, int fd, struct http *to,
+http_copyheader(struct worker *w, int vsl_id, struct http *to,
     const struct http *fm, unsigned n)
 {
 
@@ -805,7 +806,7 @@ http_copyheader(struct worker *w, int fd, struct http *to,
 		to->nhd++;
 	} else  {
 		VSC_C_main->losthdr++;
-		WSLR(w, SLT_LostHeader, fd, fm->hd[n]);
+		WSLR(w, SLT_LostHeader, vsl_id, fm->hd[n]);
 	}
 }
 
@@ -842,7 +843,7 @@ http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd)
 /*--------------------------------------------------------------------*/
 
 void
-http_FilterFields(struct worker *w, int fd, struct http *to,
+http_FilterFields(struct worker *w, int vsl_id, struct http *to,
     const struct http *fm, unsigned how)
 {
 	unsigned u;
@@ -861,7 +862,7 @@ http_FilterFields(struct worker *w, int fd, struct http *to,
 			continue;
 #include "http_headers.h"
 #undef HTTPH
-		http_copyheader(w, fd, to, fm, u);
+		http_copyheader(w, vsl_id, to, fm, u);
 	}
 }
 
@@ -882,8 +883,8 @@ http_FilterHeader(const struct sess *sp, unsigned how)
 		http_SetH(hp, HTTP_HDR_PROTO, "HTTP/1.1");
 	else
 		http_copyh(hp, sp->http, HTTP_HDR_PROTO);
-	http_FilterFields(sp->wrk, sp->fd, hp, sp->http, how);
-	http_PrintfHeader(sp->wrk, sp->fd, hp, "X-Varnish: %u", sp->xid);
+	http_FilterFields(sp->wrk, sp->vsl_id, hp, sp->http, how);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, hp, "X-Varnish: %u", sp->xid);
 }
 
 /*--------------------------------------------------------------------
@@ -892,7 +893,7 @@ http_FilterHeader(const struct sess *sp, unsigned how)
  */
 
 void
-http_CopyHome(struct worker *w, int fd, const struct http *hp)
+http_CopyHome(struct worker *w, int vsl_id, const struct http *hp)
 {
 	unsigned u, l;
 	char *p;
@@ -901,20 +902,20 @@ http_CopyHome(struct worker *w, int fd, const struct http *hp)
 		if (hp->hd[u].b == NULL)
 			continue;
 		if (hp->hd[u].b >= hp->ws->s && hp->hd[u].e <= hp->ws->e) {
-			WSLH(w, fd, hp, u);
+			WSLH(w, vsl_id, hp, u);
 			continue;
 		}
 		l = Tlen(hp->hd[u]);
 		p = WS_Alloc(hp->ws, l + 1);
 		if (p != NULL) {
-			WSLH(w, fd, hp, u);
+			WSLH(w, vsl_id, hp, u);
 			memcpy(p, hp->hd[u].b, l + 1L);
 			hp->hd[u].b = p;
 			hp->hd[u].e = p + l;
 		} else {
 			/* XXX This leaves a slot empty */
 			VSC_C_main->losthdr++;
-			WSLR(w, SLT_LostHeader, fd, hp->hd[u]);
+			WSLR(w, SLT_LostHeader, vsl_id, hp->hd[u]);
 			hp->hd[u].b = NULL;
 			hp->hd[u].e = NULL;
 		}
@@ -938,13 +939,13 @@ http_ClrHeader(struct http *to)
 /*--------------------------------------------------------------------*/
 
 void
-http_SetHeader(struct worker *w, int fd, struct http *to, const char *hdr)
+http_SetHeader(struct worker *w, int vsl_id, struct http *to, const char *hdr)
 {
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
 	if (to->nhd >= to->shd) {
 		VSC_C_main->losthdr++;
-		WSL(w, SLT_LostHeader, fd, "%s", hdr);
+		WSL(w, SLT_LostHeader, vsl_id, "%s", hdr);
 		return;
 	}
 	http_SetH(to, to->nhd++, hdr);
@@ -953,7 +954,7 @@ http_SetHeader(struct worker *w, int fd, struct http *to, const char *hdr)
 /*--------------------------------------------------------------------*/
 
 static void
-http_PutField(struct worker *w, int fd, const struct http *to, int field,
+http_PutField(struct worker *w, int vsl_id, const struct http *to, int field,
     const char *string)
 {
 	char *p;
@@ -963,7 +964,7 @@ http_PutField(struct worker *w, int fd, const struct http *to, int field,
 	l = strlen(string);
 	p = WS_Alloc(to->ws, l + 1);
 	if (p == NULL) {
-		WSL(w, SLT_LostHeader, fd, "%s", string);
+		WSL(w, SLT_LostHeader, vsl_id, "%s", string);
 		to->hd[field].b = NULL;
 		to->hd[field].e = NULL;
 		to->hdf[field] = 0;
@@ -976,11 +977,11 @@ http_PutField(struct worker *w, int fd, const struct http *to, int field,
 }
 
 void
-http_PutProtocol(struct worker *w, int fd, const struct http *to,
+http_PutProtocol(struct worker *w, int vsl_id, const struct http *to,
     const char *protocol)
 {
 
-	http_PutField(w, fd, to, HTTP_HDR_PROTO, protocol);
+	http_PutField(w, vsl_id, to, HTTP_HDR_PROTO, protocol);
 }
 
 void
@@ -992,15 +993,15 @@ http_PutStatus(struct http *to, uint16_t status)
 }
 
 void
-http_PutResponse(struct worker *w, int fd, const struct http *to,
+http_PutResponse(struct worker *w, int vsl_id, const struct http *to,
     const char *response)
 {
 
-	http_PutField(w, fd, to, HTTP_HDR_RESPONSE, response);
+	http_PutField(w, vsl_id, to, HTTP_HDR_RESPONSE, response);
 }
 
 void
-http_PrintfHeader(struct worker *w, int fd, struct http *to,
+http_PrintfHeader(struct worker *w, int vsl_id, struct http *to,
     const char *fmt, ...)
 {
 	va_list ap;
@@ -1013,7 +1014,7 @@ http_PrintfHeader(struct worker *w, int fd, struct http *to,
 	va_end(ap);
 	if (n + 1 >= l || to->nhd >= to->shd) {
 		VSC_C_main->losthdr++;
-		WSL(w, SLT_LostHeader, fd, "%s", to->ws->f);
+		WSL(w, SLT_LostHeader, vsl_id, "%s", to->ws->f);
 		WS_Release(to->ws, 0);
 	} else {
 		to->hd[to->nhd].b = to->ws->f;
@@ -1063,14 +1064,13 @@ HTTP_Copy(struct http *to, const struct http * const fm)
 /*--------------------------------------------------------------------*/
 
 unsigned
-http_Write(struct worker *w, const struct http *hp, int resp)
+http_Write(struct worker *w, unsigned vsl_id, const struct http *hp, int resp)
 {
 	unsigned u, l;
-	int fd = *(w->wrw.wfd);
 
 	if (resp) {
 		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], " ");
-		WSLH(w, fd, hp, HTTP_HDR_PROTO);
+		WSLH(w, vsl_id, hp, HTTP_HDR_PROTO);
 
 		hp->hd[HTTP_HDR_STATUS].b = WS_Alloc(w->ws, 4);
 		AN(hp->hd[HTTP_HDR_STATUS].b);
@@ -1079,18 +1079,18 @@ http_Write(struct worker *w, const struct http *hp, int resp)
 		hp->hd[HTTP_HDR_STATUS].e = hp->hd[HTTP_HDR_STATUS].b + 3;
 
 		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " ");
-		WSLH(w, fd, hp, HTTP_HDR_STATUS);
+		WSLH(w, vsl_id, hp, HTTP_HDR_STATUS);
 
 		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_RESPONSE], "\r\n");
-		WSLH(w, fd, hp, HTTP_HDR_RESPONSE);
+		WSLH(w, vsl_id, hp, HTTP_HDR_RESPONSE);
 	} else {
 		AN(hp->hd[HTTP_HDR_URL].b);
 		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_REQ], " ");
-		WSLH(w, fd, hp, HTTP_HDR_REQ);
+		WSLH(w, vsl_id, hp, HTTP_HDR_REQ);
 		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_URL], " ");
-		WSLH(w, fd, hp, HTTP_HDR_URL);
+		WSLH(w, vsl_id, hp, HTTP_HDR_URL);
 		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], "\r\n");
-		WSLH(w, fd, hp, HTTP_HDR_PROTO);
+		WSLH(w, vsl_id, hp, HTTP_HDR_PROTO);
 	}
 	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
 		if (hp->hd[u].b == NULL)
@@ -1098,7 +1098,7 @@ http_Write(struct worker *w, const struct http *hp, int resp)
 		AN(hp->hd[u].b);
 		AN(hp->hd[u].e);
 		l += WRW_WriteH(w, &hp->hd[u], "\r\n");
-		WSLH(w, fd, hp, u);
+		WSLH(w, vsl_id, hp, u);
 	}
 	l += WRW_Write(w, "\r\n", -1);
 	return (l);
diff --git a/bin/varnishd/cache_httpconn.c b/bin/varnishd/cache_httpconn.c
index 9d6d926..3ba5d7f 100644
--- a/bin/varnishd/cache_httpconn.c
+++ b/bin/varnishd/cache_httpconn.c
@@ -79,13 +79,14 @@ htc_header_complete(txt *t)
 /*--------------------------------------------------------------------*/
 
 void
-HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned maxbytes,
-    unsigned maxhdr)
+HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned vsl_id,
+    unsigned maxbytes, unsigned maxhdr)
 {
 
 	htc->magic = HTTP_CONN_MAGIC;
 	htc->ws = ws;
 	htc->fd = fd;
+	htc->vsl_id = vsl_id;
 	htc->maxbytes = maxbytes;
 	htc->maxhdr = maxhdr;
 	htc->error = "No error recorded";
diff --git a/bin/varnishd/cache_panic.c b/bin/varnishd/cache_panic.c
index b243832..8ca1b4f 100644
--- a/bin/varnishd/cache_panic.c
+++ b/bin/varnishd/cache_panic.c
@@ -216,7 +216,8 @@ pan_sess(const struct sess *sp)
 
 	VSB_printf(vsp, "sp = %p {\n", sp);
 	VSB_printf(vsp,
-	    "  fd = %d, id = %d, xid = %u,\n", sp->fd, sp->id, sp->xid);
+	    "  fd = %d, id = %d, xid = %u,\n",
+	    sp->fd, sp->vsl_id & VSL_IDENTMASK, sp->xid);
 	VSB_printf(vsp, "  client = %s %s,\n",
 	    sp->addr ? sp->addr : "?.?.?.?",
 	    sp->port ? sp->port : "?");
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
index 3968767..ea5189d 100644
--- a/bin/varnishd/cache_pipe.c
+++ b/bin/varnishd/cache_pipe.c
@@ -78,7 +78,8 @@ PipeSession(struct sess *sp)
 	(void)VTCP_blocking(vc->fd);
 
 	WRW_Reserve(w, &vc->fd);
-	sp->wrk->acct_tmp.hdrbytes += http_Write(w, sp->wrk->bereq, 0);
+	sp->wrk->acct_tmp.hdrbytes +=
+	    http_Write(w, sp->vsl_id, sp->wrk->bereq, 0);
 
 	if (sp->htc->pipeline.b != NULL)
 		sp->wrk->acct_tmp.bodybytes +=
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index eb2e62d..e490bbf 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -296,7 +296,7 @@ RES_WriteObj(struct sess *sp)
 	 */
 	if (!(sp->wrk->res_mode & RES_ESI_CHILD))
 		sp->wrk->acct_tmp.hdrbytes +=
-		    http_Write(sp->wrk, sp->wrk->resp, 1);
+		    http_Write(sp->wrk, sp->vsl_id, sp->wrk->resp, 1);
 
 	if (!sp->wantbody)
 		sp->wrk->res_mode &= ~RES_CHUNKED;
@@ -352,7 +352,7 @@ RES_StreamStart(struct sess *sp)
 		    "Content-Length: %s", sp->wrk->h_content_length);
 
 	sp->wrk->acct_tmp.hdrbytes +=
-	    http_Write(sp->wrk, sp->wrk->resp, 1);
+	    http_Write(sp->wrk, sp->vsl_id, sp->wrk->resp, 1);
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
 		WRW_Chunked(sp->wrk);
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index bbc5706..4956154 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -303,7 +303,7 @@ SES_Close(struct sess *sp, const char *reason)
 	int i;
 
 	assert(sp->fd >= 0);
-	VSL(SLT_SessionClose, sp->id, "%s", reason);
+	VSL(SLT_SessionClose, sp->vsl_id, "%s", reason);
 	i = close(sp->fd);
 	assert(i == 0 || errno != EBADF); /* XXX EINVAL seen */
 	sp->fd = -1;
@@ -352,7 +352,7 @@ SES_Delete(struct sess *sp, const char *reason)
 	assert(!isnan(b->first));
 	assert(!isnan(sp->t_end));
 
-	VSL(SLT_StatSess, sp->id, "%s %s %.0f %ju %ju %ju %ju %ju %ju %ju",
+	VSL(SLT_StatSess, sp->vsl_id, "%s %s %.0f %ju %ju %ju %ju %ju %ju %ju",
 	    sp->addr, sp->port, sp->t_end - b->first,
 	    b->sess, b->req, b->pipe, b->pass,
 	    b->fetch, b->hdrbytes, b->bodybytes);
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index 6e2cb05..9575dc8 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -80,7 +80,7 @@ vwk_kq_sess(struct vwk *vwk, struct sess *sp, short arm)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	assert(sp->fd >= 0);
-	DSL(0x04, SLT_Debug, sp->fd, "KQ: EV_SET sp %p arm %x", sp, arm);
+	DSL(0x04, SLT_Debug, sp->vsl_id, "KQ: EV_SET sp %p arm %x", sp, arm);
 	EV_SET(&vwk->ki[vwk->nki], sp->fd, EVFILT_READ, arm, 0, 0, sp);
 	if (++vwk->nki == NKEV)
 		vwk_kq_flush(vwk);
@@ -112,12 +112,12 @@ vwk_kev(struct vwk *vwk, const struct kevent *kp)
 		return;
 	}
 	CAST_OBJ_NOTNULL(sp, kp->udata, SESS_MAGIC);
-	DSL(0x04, SLT_Debug, sp->id, "KQ: sp %p kev data %lu flags 0x%x%s",
+	DSL(0x04, SLT_Debug, sp->vsl_id, "KQ: sp %p kev data %lu flags 0x%x%s",
 	    sp, (unsigned long)kp->data, kp->flags,
 	    (kp->flags & EV_EOF) ? " EOF" : "");
 
-	assert(sp->id == kp->ident);
-	assert(sp->fd == sp->id);
+	assert((sp->vsl_id & VSL_IDENTMASK) == kp->ident);
+	assert((sp->vsl_id & VSL_IDENTMASK) == sp->fd);
 	if (kp->data > 0) {
 		i = HTC_Rx(sp->htc);
 		if (i == 0) {
@@ -132,7 +132,7 @@ vwk_kev(struct vwk *vwk, const struct kevent *kp)
 		SES_Delete(sp, "EOF");
 		return;
 	} else {
-		VSL(SLT_Debug, sp->id, "KQ: sp %p kev data %lu flags 0x%x%s",
+		VSL(SLT_Debug, sp->vsl_id, "KQ: sp %p kev data %lu flags 0x%x%s",
 		    sp, (unsigned long)kp->data, kp->flags,
 		    (kp->flags & EV_EOF) ? " EOF" : "");
 	}
diff --git a/include/vsl.h b/include/vsl.h
index 45406c0..7f771ce 100644
--- a/include/vsl.h
+++ b/include/vsl.h
@@ -50,12 +50,18 @@
  *	[n + 2] ... [m]	= content
  */
 
+#define VSL_CLIENTMARKER	(1U<<30)
+#define VSL_BACKENDMARKER	(1U<<31)
+#define VSL_IDENTMASK		(~(3U<<30))
+
 #define VSL_WORDS(len)		(((len) + 3) / 4)
 #define VSL_END(ptr, len)	((ptr) + 2 + VSL_WORDS(len))
 #define VSL_NEXT(ptr)		VSL_END(ptr, VSL_LEN(ptr))
 #define VSL_LEN(ptr)		((ptr)[0] & 0xffff)
 #define VSL_TAG(ptr)		((ptr)[0] >> 24)
-#define VSL_ID(ptr)		((ptr)[1])
+#define VSL_ID(ptr)		(((ptr)[1]) & VSL_IDENTMASK)
+#define VSL_CLIENT(ptr)		(((ptr)[1]) & VSL_CLIENTMARKER)
+#define VSL_BACKEND(ptr)	(((ptr)[1]) & VSL_BACKENDMARKER)
 #define VSL_DATA(ptr)		((char*)((ptr)+2))
 
 #define VSL_ENDMARKER	(((uint32_t)SLT_Reserved << 24) | 0x454545) /* "EEE" */
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 8473d31..a34ea1e 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -75,8 +75,6 @@ VSL_Setup(struct VSM_data *vd)
 	vsl->regflags = 0;
 
 	/* XXX: Allocate only if log access */
-	vsl->vbm_client = vbit_init(4096);
-	vsl->vbm_backend = vbit_init(4096);
 	vsl->vbm_supress = vbit_init(256);
 	vsl->vbm_select = vbit_init(256);
 
@@ -102,8 +100,6 @@ VSL_Delete(struct VSM_data *vd)
 	vd->vsl = NULL;
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 
-	vbit_destroy(vsl->vbm_client);
-	vbit_destroy(vsl->vbm_backend);
 	vbit_destroy(vsl->vbm_supress);
 	vbit_destroy(vsl->vbm_select);
 	free(vsl->rbuf);
@@ -232,20 +228,6 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 			return (i);
 		u = VSL_ID(p);
 		t = VSL_TAG(p);
-		switch(t) {
-		case SLT_SessionOpen:
-		case SLT_ReqStart:
-			vbit_set(vsl->vbm_client, u);
-			vbit_clr(vsl->vbm_backend, u);
-			break;
-		case SLT_BackendOpen:
-		case SLT_BackendXID:
-			vbit_clr(vsl->vbm_client, u);
-			vbit_set(vsl->vbm_backend, u);
-			break;
-		default:
-			break;
-		}
 		if (vsl->skip) {
 			--vsl->skip;
 			continue;
@@ -260,9 +242,9 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 		}
 		if (vbit_test(vsl->vbm_supress, t))
 			continue;
-		if (vsl->b_opt && !vbit_test(vsl->vbm_backend, u))
+		if (vsl->b_opt && !VSL_BACKEND(p))
 			continue;
-		if (vsl->c_opt && !vbit_test(vsl->vbm_client, u))
+		if (vsl->c_opt && !VSL_CLIENT(p))
 			continue;
 		if (vsl->regincl != NULL) {
 			i = VRE_exec(vsl->regincl, VSL_DATA(p), VSL_LEN(p),
@@ -319,10 +301,10 @@ VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv)
 		u = VSL_ID(p);
 		l = VSL_LEN(p);
 		s = 0;
-		if (vbit_test(vsl->vbm_backend, u))
-			s |= VSL_S_BACKEND;
-		if (vbit_test(vsl->vbm_client, u))
+		if (VSL_CLIENT(p))
 			s |= VSL_S_CLIENT;
+		if (VSL_BACKEND(p))
+			s |= VSL_S_BACKEND;
 		if (func(priv, VSL_TAG(p), u, l, s, VSL_DATA(p), bitmap))
 			return (1);
 	}
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index e1684fd..aa3d0f4 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -64,13 +64,6 @@ struct vsl {
 #define F_NON_BLOCKING		(1 << 1)
 
 	/*
-	 * These two bitmaps mark fd's as belonging to client or backend
-	 * transactions respectively.
-	 */
-	struct vbitmap		*vbm_client;
-	struct vbitmap		*vbm_backend;
-
-	/*
 	 * Bit map of programatically selected tags, that cannot be suppressed.
 	 * This way programs can make sure they will see certain tags, even
 	 * if the user tries to supress them with -x/-X



From phk at varnish-cache.org  Fri Sep 30 15:35:31 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 30 Sep 2011 17:35:31 +0200
Subject: [master] 0d58f56 Second pass to tag VSL's with backend or client bits.
Message-ID: <E1R9f7b-0004uJ-0g@project.varnish-software.com>

commit 0d58f56e1bcc7ca1c17b23a370afb3da4e21b989
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Sep 30 15:35:02 2011 +0000

    Second pass to tag VSL's with backend or client bits.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 5c793e9..5a91dd8 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -763,17 +763,18 @@ unsigned http_Write(struct worker *w, unsigned vsl_id, const struct http *hp,
 void http_CopyResp(struct http *to, const struct http *fm);
 void http_SetResp(struct http *to, const char *proto, uint16_t status,
     const char *response);
-void http_FilterFields(struct worker *w, int fd, struct http *to,
+void http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
     const struct http *fm, unsigned how);
 void http_FilterHeader(const struct sess *sp, unsigned how);
-void http_PutProtocol(struct worker *w, int fd, const struct http *to,
+void http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
     const char *protocol);
 void http_PutStatus(struct http *to, uint16_t status);
-void http_PutResponse(struct worker *w, int fd, const struct http *to,
+void http_PutResponse(struct worker *w, unsigned vsl_id, const struct http *to,
     const char *response);
-void http_PrintfHeader(struct worker *w, int fd, struct http *to,
+void http_PrintfHeader(struct worker *w, unsigned vsl_id, struct http *to,
     const char *fmt, ...);
-void http_SetHeader(struct worker *w, int fd, struct http *to, const char *hdr);
+void http_SetHeader(struct worker *w, unsigned vsl_id, struct http *to,
+    const char *hdr);
 void http_SetH(const struct http *to, unsigned n, const char *fm);
 void http_ForceGet(const struct http *to);
 void http_Setup(struct http *ht, struct ws *ws);
@@ -790,7 +791,7 @@ uint16_t http_DissectRequest(struct sess *sp);
 uint16_t http_DissectResponse(struct worker *w, const struct http_conn *htc,
     struct http *sp);
 const char *http_DoConnection(const struct http *hp);
-void http_CopyHome(struct worker *w, int fd, const struct http *hp);
+void http_CopyHome(struct worker *w, unsigned vsl_id, const struct http *hp);
 void http_Unset(struct http *hp, const char *hdr);
 void http_CollectHdr(struct http *hp, const char *hdr);
 
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index ba28a39..e2be560 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -130,10 +130,10 @@ VCA_Prep(struct sess *sp)
 		AZ(getsockname(sp->fd, (void*)&sp->mysockaddr, &sp->mysockaddrlen));
 		VTCP_name(&sp->mysockaddr, sp->mysockaddrlen,
 		    addr, sizeof addr, port, sizeof port);
-		VSL(SLT_SessionOpen, sp->fd, "%s %s %s %s",
+		WSP(sp, SLT_SessionOpen, "%s %s %s %s",
 		    sp->addr, sp->port, addr, port);
 	} else {
-		VSL(SLT_SessionOpen, sp->fd, "%s %s %s",
+		WSP(sp, SLT_SessionOpen, "%s %s %s",
 		    sp->addr, sp->port, sp->mylsock->name);
 	}
 	sp->acct_ses.first = sp->t_open;
diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index d2bc860..1061612 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -70,7 +70,7 @@ VDI_AddHostHeader(const struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp->wrk->bereq, HTTP_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vbc, VBC_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vbc->vdis, VDI_SIMPLE_MAGIC);
-	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->bereq,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
 	    "Host: %s", sp->vbc->vdis->vrt->hosthdr);
 }
 
@@ -357,7 +357,7 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs)
 			return (vc);
 		}
 		VSC_C_main->backend_toolate++;
-		WSL(sp->wrk, SLT_BackendClose, vc->fd, "%s", bp->vcl_name);
+		WSL(sp->wrk, SLT_BackendClose, vc->vsl_id, "%s", bp->vcl_name);
 
 		/* Checkpoint log to flush all info related to this connection
 		   before the OS reuses the FD */
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 38ec4f2..b4a056d 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -104,7 +104,7 @@ cnt_wait(struct sess *sp)
 			i = HTC_Rx(sp->htc);
 	}
 	if (i == 0) {
-		WSL(sp->wrk, SLT_Debug, sp->fd, "herding");
+		WSP(sp, SLT_Debug, "herding");
 		sp->wrk->stats.sess_herd++;
 		SES_Charge(sp);
 		sp->wrk = NULL;
@@ -464,16 +464,16 @@ cnt_error(struct sess *sp)
 	if (sp->err_code < 100 || sp->err_code > 999)
 		sp->err_code = 501;
 
-	http_PutProtocol(w, sp->fd, h, "HTTP/1.1");
+	http_PutProtocol(w, sp->vsl_id, h, "HTTP/1.1");
 	http_PutStatus(h, sp->err_code);
 	TIM_format(TIM_real(), date);
-	http_PrintfHeader(w, sp->fd, h, "Date: %s", date);
-	http_PrintfHeader(w, sp->fd, h, "Server: Varnish");
+	http_PrintfHeader(w, sp->vsl_id, h, "Date: %s", date);
+	http_PrintfHeader(w, sp->vsl_id, h, "Server: Varnish");
 
 	if (sp->err_reason != NULL)
-		http_PutResponse(w, sp->fd, h, sp->err_reason);
+		http_PutResponse(w, sp->vsl_id, h, sp->err_reason);
 	else
-		http_PutResponse(w, sp->fd, h,
+		http_PutResponse(w, sp->vsl_id, h,
 		    http_StatusMessage(sp->err_code));
 	VCL_error_method(sp);
 
@@ -728,7 +728,7 @@ cnt_fetchbody(struct sess *sp)
 
 	/* If we do gzip, add the C-E header */
 	if (sp->wrk->do_gzip)
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->beresp,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->beresp,
 		    "Content-Encoding: %s", "gzip");
 
 	/* But we can't do both at the same time */
@@ -1194,7 +1194,7 @@ cnt_miss(struct sess *sp)
 		 * the minority of clients which don't.
 		 */
 		http_Unset(sp->wrk->bereq, H_Accept_Encoding);
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->bereq,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
 		    "Accept-Encoding: gzip");
 	}
 	sp->wrk->connect_timeout = 0;
@@ -1398,7 +1398,7 @@ cnt_recv(struct sess *sp)
 	     (recv_handling != VCL_RET_PASS)) {
 		if (RFC2616_Req_Gzip(sp)) {
 			http_Unset(sp->http, H_Accept_Encoding);
-			http_PrintfHeader(sp->wrk, sp->fd, sp->http,
+			http_PrintfHeader(sp->wrk, sp->vsl_id, sp->http,
 			    "Accept-Encoding: gzip");
 		} else {
 			http_Unset(sp->http, H_Accept_Encoding);
diff --git a/bin/varnishd/cache_dir.c b/bin/varnishd/cache_dir.c
index 2463aa8..03fdf26 100644
--- a/bin/varnishd/cache_dir.c
+++ b/bin/varnishd/cache_dir.c
@@ -48,7 +48,7 @@ VDI_CloseFd(struct sess *sp)
 
 	bp = sp->vbc->backend;
 
-	WSL(sp->wrk, SLT_BackendClose, sp->vbc->fd, "%s", bp->vcl_name);
+	WSL(sp->wrk, SLT_BackendClose, sp->vbc->vsl_id, "%s", bp->vcl_name);
 
 	/* Checkpoint log to flush all info related to this connection
 	   before the OS reuses the FD */
@@ -76,7 +76,7 @@ VDI_RecycleFd(struct sess *sp)
 
 	bp = sp->vbc->backend;
 
-	WSL(sp->wrk, SLT_BackendReuse, sp->vbc->fd, "%s", bp->vcl_name);
+	WSL(sp->wrk, SLT_BackendReuse, sp->vbc->vsl_id, "%s", bp->vcl_name);
 	/*
 	 * Flush the shmlog, so that another session reusing this backend
 	 * will log chronologically later than our use of it.
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 8b2edba..5bfa654 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -72,7 +72,7 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	if (host != NULL && *host != '\0')  {
 		http_Unset(sp->http, H_Host);
 		http_Unset(sp->http, H_If_Modified_Since);
-		http_SetHeader(w, sp->fd, sp->http, host);
+		http_SetHeader(w, sp->vsl_id, sp->http, host);
 	}
 	/*
 	 * XXX: We should decide if we should cache the director
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 6451f10..7c6e2e5 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -63,8 +63,7 @@ vfp_nop_begin(struct sess *sp, size_t estimate)
 
 	if (fetchfrag > 0) {
 		estimate = fetchfrag;
-		WSL(sp->wrk, SLT_Debug, sp->fd,
-		    "Fetch %d byte segments:", fetchfrag);
+		WSP(sp, SLT_Debug, "Fetch %d byte segments:", fetchfrag);
 	}
 	if (estimate > 0)
 		(void)FetchStorage(sp, estimate);
@@ -355,7 +354,7 @@ FetchReqBody(struct sess *sp)
 	}
 	if (http_GetHdr(sp->http, H_Transfer_Encoding, NULL)) {
 		/* XXX: Handle chunked encoding. */
-		WSL(sp->wrk, SLT_Debug, sp->fd, "Transfer-Encoding in request");
+		WSP(sp, SLT_Debug, "Transfer-Encoding in request");
 		return (1);
 	}
 	return (0);
@@ -542,7 +541,7 @@ FetchBody(struct sess *sp)
 	 */
 	AZ(vfp_nop_end(sp));
 
-	WSL(w, SLT_Fetch_Body, sp->vbc->fd, "%u(%s) cls %d mklen %u",
+	WSL(w, SLT_Fetch_Body, sp->vbc->vsl_id, "%u(%s) cls %d mklen %u",
 	    w->body_status, body_status(w->body_status),
 	    cls, mklen);
 
@@ -567,7 +566,7 @@ FetchBody(struct sess *sp)
 	if (cls == 0 && w->do_close)
 		cls = 1;
 
-	WSL(w, SLT_Length, sp->vbc->fd, "%u", sp->obj->len);
+	WSL(w, SLT_Length, sp->vbc->vsl_id, "%u", sp->obj->len);
 
 	{
 	/* Sanity check fetch methods accounting */
@@ -585,7 +584,7 @@ FetchBody(struct sess *sp)
 
 	if (mklen > 0) {
 		http_Unset(sp->obj->http, H_Content_Length);
-		http_PrintfHeader(w, sp->fd, sp->obj->http,
+		http_PrintfHeader(w, sp->vsl_id, sp->obj->http,
 		    "Content-Length: %jd", (intmax_t)sp->obj->len);
 	}
 
diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index 587b6bb..c8a5415 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -77,7 +77,7 @@ http2shmlog(const struct http *hp, int t)
 }
 
 static void
-WSLH(struct worker *w, int vsl_id, const struct http *hp, unsigned hdr)
+WSLH(struct worker *w, unsigned vsl_id, const struct http *hp, unsigned hdr)
 {
 
 	AN(vsl_id & (VSL_CLIENTMARKER|VSL_BACKENDMARKER));
@@ -486,7 +486,7 @@ http_GetReq(const struct http *hp)
  */
 
 static uint16_t
-http_dissect_hdrs(struct worker *w, struct http *hp, int vsl_id, char *p,
+http_dissect_hdrs(struct worker *w, struct http *hp, unsigned vsl_id, char *p,
     const struct http_conn *htc)
 {
 	char *q, *r;
@@ -562,7 +562,7 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int vsl_id, char *p,
  */
 
 static uint16_t
-http_splitline(struct worker *w, int vsl_id, struct http *hp,
+http_splitline(struct worker *w, unsigned vsl_id, struct http *hp,
     const struct http_conn *htc, int h1, int h2, int h3)
 {
 	char *p, *q;
@@ -792,7 +792,7 @@ http_SetResp(struct http *to, const char *proto, uint16_t status,
 }
 
 static void
-http_copyheader(struct worker *w, int vsl_id, struct http *to,
+http_copyheader(struct worker *w, unsigned vsl_id, struct http *to,
     const struct http *fm, unsigned n)
 {
 
@@ -843,7 +843,7 @@ http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd)
 /*--------------------------------------------------------------------*/
 
 void
-http_FilterFields(struct worker *w, int vsl_id, struct http *to,
+http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
     const struct http *fm, unsigned how)
 {
 	unsigned u;
@@ -893,7 +893,7 @@ http_FilterHeader(const struct sess *sp, unsigned how)
  */
 
 void
-http_CopyHome(struct worker *w, int vsl_id, const struct http *hp)
+http_CopyHome(struct worker *w, unsigned vsl_id, const struct http *hp)
 {
 	unsigned u, l;
 	char *p;
@@ -939,7 +939,8 @@ http_ClrHeader(struct http *to)
 /*--------------------------------------------------------------------*/
 
 void
-http_SetHeader(struct worker *w, int vsl_id, struct http *to, const char *hdr)
+http_SetHeader(struct worker *w, unsigned vsl_id, struct http *to,
+    const char *hdr)
 {
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
@@ -954,8 +955,8 @@ http_SetHeader(struct worker *w, int vsl_id, struct http *to, const char *hdr)
 /*--------------------------------------------------------------------*/
 
 static void
-http_PutField(struct worker *w, int vsl_id, const struct http *to, int field,
-    const char *string)
+http_PutField(struct worker *w, unsigned vsl_id, const struct http *to,
+    int field, const char *string)
 {
 	char *p;
 	unsigned l;
@@ -977,7 +978,7 @@ http_PutField(struct worker *w, int vsl_id, const struct http *to, int field,
 }
 
 void
-http_PutProtocol(struct worker *w, int vsl_id, const struct http *to,
+http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
     const char *protocol)
 {
 
@@ -993,7 +994,7 @@ http_PutStatus(struct http *to, uint16_t status)
 }
 
 void
-http_PutResponse(struct worker *w, int vsl_id, const struct http *to,
+http_PutResponse(struct worker *w, unsigned vsl_id, const struct http *to,
     const char *response)
 {
 
@@ -1001,7 +1002,7 @@ http_PutResponse(struct worker *w, int vsl_id, const struct http *to,
 }
 
 void
-http_PrintfHeader(struct worker *w, int vsl_id, struct http *to,
+http_PrintfHeader(struct worker *w, unsigned vsl_id, struct http *to,
     const char *fmt, ...)
 {
 	va_list ap;
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index e490bbf..4d7c88b 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -92,12 +92,12 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 	if (low > high)
 		return;
 
-	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 	    "Content-Range: bytes %jd-%jd/%jd",
 	    (intmax_t)low, (intmax_t)high, (intmax_t)sp->obj->len);
 	http_Unset(sp->wrk->resp, H_Content_Length);
 	assert(sp->wrk->res_mode & RES_LEN);
-	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 	    "Content-Length: %jd", (intmax_t)(1 + high - low));
 	http_SetResp(sp->wrk->resp, "HTTP/1.1", 206, "Partial Content");
 
@@ -118,34 +118,34 @@ RES_BuildHttp(const struct sess *sp)
 	http_ClrHeader(sp->wrk->resp);
 	sp->wrk->resp->logtag = HTTP_Tx;
 	http_CopyResp(sp->wrk->resp, sp->obj->http);
-	http_FilterFields(sp->wrk, sp->fd, sp->wrk->resp, sp->obj->http,
+	http_FilterFields(sp->wrk, sp->vsl_id, sp->wrk->resp, sp->obj->http,
 	    HTTPH_A_DELIVER);
 
 	if (!(sp->wrk->res_mode & RES_LEN)) {
 		http_Unset(sp->wrk->resp, H_Content_Length);
 	} else if (params->http_range_support) {
 		/* We only accept ranges if we know the length */
-		http_SetHeader(sp->wrk, sp->fd, sp->wrk->resp,
+		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "Accept-Ranges: bytes");
 	}
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "Transfer-Encoding: chunked");
 
 	TIM_format(TIM_real(), time_str);
-	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Date: %s", time_str);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Date: %s", time_str);
 
 	if (sp->xid != sp->obj->xid)
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "X-Varnish: %u %u", sp->xid, sp->obj->xid);
 	else
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "X-Varnish: %u", sp->xid);
-	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Age: %.0f",
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Age: %.0f",
 	    sp->obj->exp.age + sp->t_resp - sp->obj->exp.entered);
-	http_SetHeader(sp->wrk, sp->fd, sp->wrk->resp, "Via: 1.1 varnish");
-	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Connection: %s",
+	http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Via: 1.1 varnish");
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Connection: %s",
 	    sp->doclose ? "close" : "keep-alive");
 }
 
@@ -348,7 +348,7 @@ RES_StreamStart(struct sess *sp)
 
 	if (!(sp->wrk->res_mode & RES_CHUNKED) &&
 	    sp->wrk->h_content_length != NULL)
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "Content-Length: %s", sp->wrk->h_content_length);
 
 	sp->wrk->acct_tmp.hdrbytes +=
diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c
index c17c361..86cb10f 100644
--- a/bin/varnishd/cache_vrt.c
+++ b/bin/varnishd/cache_vrt.c
@@ -86,7 +86,7 @@ VRT_count(const struct sess *sp, unsigned u)
 void
 VRT_acl_log(const struct sess *sp, const char *msg)
 {
-	WSL(sp->wrk, SLT_VCL_acl, sp->fd, msg);
+	WSP(sp, SLT_VCL_acl, msg);
 }
 
 /*--------------------------------------------------------------------*/
@@ -232,7 +232,7 @@ VRT_SetHdr(const struct sess *sp , enum gethdr_e where, const char *hdr,
 			WSP(sp, SLT_LostHeader, "%s", hdr + 1);
 		} else {
 			http_Unset(hp, hdr);
-			http_SetHeader(sp->wrk, sp->fd, hp, b);
+			http_SetHeader(sp->wrk, sp->vsl_id, hp, b);
 		}
 	}
 	va_end(ap);
@@ -418,7 +418,7 @@ VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...)
 	va_end(ap);
 	SMS_Finish(sp->obj);
 	http_Unset(sp->obj->http, H_Content_Length);
-	http_PrintfHeader(sp->wrk, sp->fd, sp->obj->http,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->obj->http,
 	    "Content-Length: %d", sp->obj->len);
 }
 



From phk at varnish-cache.org  Fri Sep 30 15:40:09 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 30 Sep 2011 17:40:09 +0200
Subject: [master] 558ee90 Add mising ;'s
Message-ID: <E1R9fC5-0005X2-1X@project.varnish-software.com>

commit 558ee90f00a157c6452b35ff9eda974f53d993c6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Sep 30 15:39:55 2011 +0000

    Add mising ;'s

diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
index b6dac01..24293c2 100644
--- a/bin/varnishd/mgt.h
+++ b/bin/varnishd/mgt.h
@@ -73,8 +73,8 @@ void mgt_sandbox(void);
 
 /* mgt_sandbox_solaris.c */
 #ifdef HAVE_SETPPRIV
-void mgt_sandbox_solaris_init(void)
-void mgt_sandbox_solaris_fini(void)
+void mgt_sandbox_solaris_init(void);
+void mgt_sandbox_solaris_fini(void);
 #endif
 
 /* mgt_shmem.c */



From phk at varnish-cache.org  Fri Sep 30 18:25:43 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 30 Sep 2011 20:25:43 +0200
Subject: [master] d4dbc04 This test is much to brittle due to dependence on
	random timing. Tighten it up with use of semaphores and
	collapse s1/s2/s3 using "accept" keyword.
Message-ID: <E1R9hmJ-0002Hv-0n@project.varnish-software.com>

commit d4dbc04e9664a72baf6d41742617df259c6b1502
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Sep 30 18:24:34 2011 +0000

    This test is much to brittle due to dependence on random timing.
    Tighten it up with use of semaphores and collapse s1/s2/s3 using
    "accept" keyword.

diff --git a/bin/varnishtest/tests/v00036.vtc b/bin/varnishtest/tests/v00036.vtc
index ff2ecd3..46bcca8 100644
--- a/bin/varnishtest/tests/v00036.vtc
+++ b/bin/varnishtest/tests/v00036.vtc
@@ -4,12 +4,46 @@ server s1 {
 	rxreq
 	expect req.url == "/"
 	txresp -body "slash"
+	accept
+
+	sema r1 sync 3
+
+	rxreq
+	expect req.url == "/"
+	txresp -body "slash"
+	accept
+
+	rxreq
+	expect req.url == "/"
+	txresp -body "slash"
+
+	sema r3 sync 2
+	accept
+
+	rxreq
+	expect req.url == "/foo"
+	txresp -hdr "Foo: 1" -body "foobar"
+
 } -start
 
 server s2 {
 	rxreq
 	expect req.url == "/"
 	txresp -body "slash"
+	accept
+
+	sema r1 sync 3
+
+	rxreq
+	expect req.url == "/"
+	txresp -body "slash"
+
+	sema r2 sync 2
+	accept
+
+	rxreq
+	expect req.url == "/foo"
+	txresp -hdr "Foo: 2" -body "foobar"
 } -start
 
 server s3 {
@@ -65,66 +99,23 @@ varnish v1 -vcl {
 	}
 } -start
 
-# s1 & s2 have both had 1 probe, so both are unhealthy
 
 client c1 {
+	# s1 & s2 are both sick, expect response from s3
 	txreq -url "/foo"
 	rxresp
 	expect resp.http.foo == "3"
-} -run
-
-# setup for probe #2
-
-server s1 {
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-} -start
-
-server s2 {
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-} -start
 
-# if we muck with a running server, the test will wait until it's done,
-# which will be after probe #2 completes. b2 will then be healthy.
+	sema r1 sync 3
 
-server s2 {
-	rxreq
-	expect req.url == "/foo"
-	txresp -hdr "Foo: 2" -body "foobar"
-} -start
-
-client c1 {
+	# wait for s2 to become healthy
+	sema r2 sync 2
 	txreq -url "/foo"
 	rxresp
 	expect resp.http.foo == "2"
-} -run
 
-# setup for probe #3
-
-server s1 {
-        rxreq
-        expect req.url == "/"
-        txresp -body "slash"
-} -start
-
-server s2 {
-        rxreq
-        expect req.url == "/"
-        txresp -body "slash"
-} -start
-
-# after probe #3 b1 should be healthy.
-
-server s1 {
-	rxreq
-	expect req.url == "/foo"
-	txresp -hdr "Foo: 1" -body "foobar"
-} -start
-
-client c1 {
+	# wait for s1 to become healthy
+	sema r3 sync 2
 	txreq -url "/foo"
 	rxresp
 	expect resp.http.foo == "1"



From phk at varnish-cache.org  Fri Sep 30 19:09:32 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 30 Sep 2011 21:09:32 +0200
Subject: [master] 7f56b9d Try harder to make v00036 deterministic
Message-ID: <E1R9iSi-0007X0-LJ@project.varnish-software.com>

commit 7f56b9d20f0516ae6b78e7c46fa7c010ba1be87b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Sep 30 19:09:16 2011 +0000

    Try harder to make v00036 deterministic

diff --git a/bin/varnishtest/tests/v00036.vtc b/bin/varnishtest/tests/v00036.vtc
index 46bcca8..e83acef 100644
--- a/bin/varnishtest/tests/v00036.vtc
+++ b/bin/varnishtest/tests/v00036.vtc
@@ -4,22 +4,25 @@ server s1 {
 	rxreq
 	expect req.url == "/"
 	txresp -body "slash"
-	accept
+	close
 
 	sema r1 sync 3
 
+	accept
 	rxreq
 	expect req.url == "/"
 	txresp -body "slash"
-	accept
+	close
 
+	accept
 	rxreq
 	expect req.url == "/"
 	txresp -body "slash"
+	close
 
 	sema r3 sync 2
-	accept
 
+	accept
 	rxreq
 	expect req.url == "/foo"
 	txresp -hdr "Foo: 1" -body "foobar"
@@ -30,17 +33,19 @@ server s2 {
 	rxreq
 	expect req.url == "/"
 	txresp -body "slash"
-	accept
+	close
 
 	sema r1 sync 3
 
+	accept
 	rxreq
 	expect req.url == "/"
 	txresp -body "slash"
+	close
 
 	sema r2 sync 2
-	accept
 
+	accept
 	rxreq
 	expect req.url == "/foo"
 	txresp -hdr "Foo: 2" -body "foobar"
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 024e376..fddcd26 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1059,6 +1059,25 @@ cmd_http_expect_close(CMD_ARGS)
 }
 
 /**********************************************************************
+ * close a new connection  (server only)
+ */
+
+static void
+cmd_http_close(CMD_ARGS)
+{
+	struct http *hp;
+
+	(void)cmd;
+	(void)vl;
+	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
+	AZ(av[1]);
+	assert(hp->sfd != NULL);
+	assert(*hp->sfd >= 0);
+	VTCP_close(&hp->fd);
+	vtc_log(vl, 4, "Closed");
+}
+
+/**********************************************************************
  * close and accept a new connection  (server only)
  */
 
@@ -1073,7 +1092,8 @@ cmd_http_accept(CMD_ARGS)
 	AZ(av[1]);
 	assert(hp->sfd != NULL);
 	assert(*hp->sfd >= 0);
-	VTCP_close(&hp->fd);
+	if (hp->fd >= 0)
+		VTCP_close(&hp->fd);
 	vtc_log(vl, 4, "Accepting");
 	hp->fd = accept(*hp->sfd, NULL, NULL);
 	if (hp->fd < 0)
@@ -1129,6 +1149,7 @@ static const struct cmds http_cmds[] = {
 	{ "delay",		cmd_delay },
 	{ "sema",		cmd_sema },
 	{ "expect_close",	cmd_http_expect_close },
+	{ "close",		cmd_http_close },
 	{ "accept",		cmd_http_accept },
 	{ "loop",		cmd_http_loop },
 	{ NULL,			NULL }