From perbu at varnish-cache.org  Wed Apr  4 08:17:03 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Wed, 04 Apr 2012 10:17:03 +0200
Subject: [master] 9cd5b47 escape \0
Message-ID: <E1SFLOp-0008D3-JH@project.varnish-software.com>

commit 9cd5b47122ebc1219ee3f86eb88a2e027e7a5f68
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Apr 4 10:17:01 2012 +0200

    escape \0

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 9e71fbd..5a5ca6e 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -408,7 +408,7 @@ hash_data(str)
 
 regsub(str, regex, sub)
   Returns a copy of str with the first occurrence of the regular 
-  expression regex replaced with sub. Within sub, \0 (which can 
+  expression regex replaced with sub. Within sub, \\0 (which can 
   also be spelled &) is replaced with the entire matched string, 
   and \n is replaced with the contents of subgroup n in the 
   matched string.



From tfheen at varnish-cache.org  Tue Apr 10 08:57:00 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 10 Apr 2012 10:57:00 +0200
Subject: [master] 8b8adc6 Fix typo in HTTP header name
Message-ID: <E1SHWsm-0003la-To@project.varnish-software.com>

commit 8b8adc6c79cfd64578ca5bdf4a8f1f4768f7a68b
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Apr 10 10:27:33 2012 +0200

    Fix typo in HTTP header name

diff --git a/include/tbl/http_headers.h b/include/tbl/http_headers.h
index 3f2c50a..758c7e6 100644
--- a/include/tbl/http_headers.h
+++ b/include/tbl/http_headers.h
@@ -56,7 +56,7 @@ HTTPH("Authorization",		H_Authorization,	0					  )	/* RFC2616 14.8 */
 HTTPH("Cache-Control",		H_Cache_Control,		       HTTPH_R_FETCH		  )	/* RFC2616 14.9 */
 HTTPH("Connection",		H_Connection,		HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.10 */
 HTTPH("Content-Encoding",	H_Content_Encoding,	0					  )	/* RFC2616 14.11 */
-HTTPH("Content-Langugae",	H_Content_Language,	0					  )	/* RFC2616 14.12 */
+HTTPH("Content-Language",	H_Content_Language,	0					  )	/* RFC2616 14.12 */
 HTTPH("Content-Length",		H_Content_Length,		       HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.13 */
 HTTPH("Content-Location",	H_Content_Location,	0					  )	/* RFC2616 14.14 */
 HTTPH("Content-MD5",		H_Content_MD5,		0					  )	/* RFC2616 14.15 */



From phk at varnish-cache.org  Tue Apr 10 11:42:50 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 10 Apr 2012 13:42:50 +0200
Subject: [master] 3cf4384 Adapt to FreeBSD 10 #include changes
Message-ID: <E1SHZTG-0003Zg-2f@project.varnish-software.com>

commit 3cf4384ea84b8da3a5ff3850d45c51fa06754608
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 10 11:42:37 2012 +0000

    Adapt to FreeBSD 10 #include changes

diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index 0c9f5d0..4ba4dd2 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -90,7 +90,8 @@
 -efunc(1791, pdiff)	// return last on line
 //////////////
 -efile(451, "sys/\*.h")	// No include guard
--efile(451, "machine/\*.h")	// No include guard
+-efile(451, "machine/*.h")	// No include guard
+-efile(451, "stdarg.h")	// No include guard
 -efile(451, "tbl/*.h")	// No include guard
 -efile(451, "vcc_types.h")	// No include guard
 -efile(451, "symbol_kind.h")	// No include guard



From phk at varnish-cache.org  Tue Apr 10 12:04:02 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 10 Apr 2012 14:04:02 +0200
Subject: [master] 9f9429c move FreeBSD hacks to the (out of varnish-tree)
	freebsd configs
Message-ID: <E1SHZnm-0006Mk-Ic@project.varnish-software.com>

commit 9f9429c35841712811d84e3e7a97a37b871445ff
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 10 12:01:53 2012 +0000

    move FreeBSD hacks to the (out of varnish-tree) freebsd configs

diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index 4ba4dd2..1569271 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -89,9 +89,6 @@
 //////////////
 -efunc(1791, pdiff)	// return last on line
 //////////////
--efile(451, "sys/\*.h")	// No include guard
--efile(451, "machine/*.h")	// No include guard
--efile(451, "stdarg.h")	// No include guard
 -efile(451, "tbl/*.h")	// No include guard
 -efile(451, "vcc_types.h")	// No include guard
 -efile(451, "symbol_kind.h")	// No include guard



From phk at varnish-cache.org  Tue Apr 10 12:04:02 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 10 Apr 2012 14:04:02 +0200
Subject: [master] e6d7f46 Fix for #1109
Message-ID: <E1SHZnm-0006Mp-Jm@project.varnish-software.com>

commit e6d7f462483e518400c8a0ae151ed70438b3a56c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 10 12:03:16 2012 +0000

    Fix for #1109
    
    I've opted for a less intrusive fix that Martin proposed, but my fix
    is largely a rework of his suggested patch.
    
    Testcase by:	Martin

diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index dd3d98a..0e8b11a 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -421,12 +421,10 @@ ved_deliver_byterange(const struct sess *sp, ssize_t low, ssize_t high)
 	ssize_t l, lx;
 	u_char *p;
 
-//printf("BR %jd %jd\n", low, high);
 	lx = 0;
 	VTAILQ_FOREACH(st, &sp->req->obj->store, list) {
 		p = st->ptr;
 		l = st->len;
-//printf("[0-] %jd %jd\n", lx, lx + l);
 		if (lx + l < low) {
 			lx += l;
 			continue;
@@ -439,16 +437,14 @@ ved_deliver_byterange(const struct sess *sp, ssize_t low, ssize_t high)
 			l -= (low - lx);
 			lx = low;
 		}
-//printf("[1-] %jd %jd\n", lx, lx + l);
 		if (lx + l >= high)
 			l = high - lx;
-//printf("[2-] %jd %jd\n", lx, lx + l);
 		assert(lx >= low && lx + l <= high);
 		if (l != 0)
 			(void)WRW_Write(sp->wrk, p, l);
-		if (lx + st->len > high)
+		if (p + l < st->ptr + st->len)
 			return(p[l]);
-		lx += st->len;
+		lx += l;
 	}
 	INCOMPL();
 }
@@ -459,10 +455,12 @@ ESI_DeliverChild(const struct sess *sp)
 	struct storage *st;
 	struct object *obj;
 	ssize_t start, last, stop, lpad;
-	u_char *p, cc;
+	u_char cc;
 	uint32_t icrc;
 	uint32_t ilen;
 	uint8_t *dbits;
+	int i, j;
+	uint8_t tailbuf[8];
 
 	if (!sp->req->obj->gziped) {
 		VTAILQ_FOREACH(st, &sp->req->obj->store, list)
@@ -542,12 +540,20 @@ ESI_DeliverChild(const struct sess *sp)
 	}
 	if (lpad > 0)
 		(void)WRW_Write(sp->wrk, dbits + 1, lpad);
+
+	/* We need the entire tail, but it may not be in one storage segment */
 	st = VTAILQ_LAST(&sp->req->obj->store, storagehead);
-	assert(st->len > 8);
+	for (i = sizeof tailbuf; i > 0; i -= j) {
+		j = st->len;
+		if (j > i)
+			j = i;
+		memcpy(tailbuf + i - j, st->ptr + st->len - j, j);
+		st = VTAILQ_PREV(st, storagehead, list);
+		assert(i == j || st != NULL);
+	}
 
-	p = st->ptr + st->len - 8;
-	icrc = vle32dec(p);
-	ilen = vle32dec(p + 4);
+	icrc = vle32dec(tailbuf);
+	ilen = vle32dec(tailbuf + 4);
 	sp->req->crc = crc32_combine(sp->req->crc, icrc, ilen);
 	sp->req->l_crc += ilen;
 }
diff --git a/bin/varnishtest/tests/r01109.vtc b/bin/varnishtest/tests/r01109.vtc
new file mode 100644
index 0000000..50f0a0e
--- /dev/null
+++ b/bin/varnishtest/tests/r01109.vtc
@@ -0,0 +1,43 @@
+varnishtest "Test case for #1109 - Gzip+ESI broken for large included objects"
+
+server s1 {
+	rxreq
+	expect req.url == "/test1"
+	txresp -body {<html>start<esi:include src="/include1"/>end}
+	rxreq
+	expect req.url == "/include1"
+	# This tests ESI+gzip delivery when the ESI-included object
+	# has more than one storage chunk
+	txresp -bodylen 4082
+
+	rxreq
+	txresp -body {<html>start<esi:include src="/include2"/>end}
+	expect req.url == "/test2"
+
+	rxreq
+	expect req.url == "/include2"
+	# This tests gzip trailer extraction for ESI+gzip CRC calculation
+	# when the trailer spans two storage chunks
+	txresp -bodylen 4074
+} -start
+
+varnish v1 -arg "-pfetch_chunksize=4k" -arg "-pgzip_level=0" -vcl+backend {
+	sub vcl_fetch {
+		if (req.url ~ "/test") {
+			set beresp.do_esi = true;
+		}
+		set beresp.do_gzip = true;
+	}
+} -start
+
+client c1 {
+	txreq -url "/test1" -hdr "Accept-Encoding: gzip"
+	rxresp
+	gunzip
+	expect resp.bodylen == 4096
+
+	txreq -url "/test2" -hdr "Accept-Encoding: gzip"
+	rxresp
+	gunzip
+	expect resp.bodylen == 4088
+} -run



From tfheen at varnish-cache.org  Tue Apr 10 13:12:30 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 10 Apr 2012 15:12:30 +0200
Subject: [master] 9dc28b2 Fix libedit detection on *BSD OS's
Message-ID: <E1SHas2-0003RV-B8@project.varnish-software.com>

commit 9dc28b2ce920274432da72f78419562d9365bc2b
Author: Brad <brad at comstyle.com>
Date:   Mon Jan 2 14:03:46 2012 -0500

    Fix libedit detection on *BSD OS's
    
    The following patch allows the autoconf script to detect the presence of
    libedit when there isn't a pkg-config file present and thus allowing
    Varnish to detect libedit on OpenBSD/FreeBSD/NetBSD/DragonFly.
    
    Fixes: #1003

diff --git a/configure.ac b/configure.ac
index 7e5e57e..33b0a60 100644
--- a/configure.ac
+++ b/configure.ac
@@ -144,7 +144,14 @@ AC_SUBST(PCRE_LIBS)
 
 PKG_CHECK_MODULES([LIBEDIT], [libedit], 
 	[AC_DEFINE([HAVE_LIBEDIT], [1], [Define we have libedit])],
-	[AC_MSG_WARN([libedit not found, disabling libedit support])])
+	[AC_CHECK_HEADERS([readline/readline.h])
+	 AC_CHECK_LIB(edit, el_init,
+	        [ AC_DEFINE([HAVE_LIBEDIT], [1], [Define we have libedit])
+		  LIBEDIT_CFLAGS=""
+		  LIBEDIT_LIBS="-ledit ${CURSES_LIBS}"
+		],
+               [AC_MSG_WARN([libedit not found, disabling libedit support])],
+               [${CURSES_LIBS}])])
 
 # Checks for header files.
 AC_HEADER_STDC



From phk at varnish-cache.org  Thu Apr 12 07:24:27 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 12 Apr 2012 09:24:27 +0200
Subject: [master] 7e51d6f Stopgap fix to get FreeBSD 10-current compiling
	again.
Message-ID: <E1SIEOJ-0007dh-7D@project.varnish-software.com>

commit 7e51d6ff15713f5e4d424b179428c6fd8afe29ca
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Apr 12 07:24:02 2012 +0000

    Stopgap fix to get FreeBSD 10-current compiling again.

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index bb5cc8e..f3b8b69 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -32,8 +32,12 @@
 #include <sys/socket.h>
 
 #ifdef HAVE_LIBEDIT
-#include <stdio.h>
-#include <editline/readline.h>
+#  include <stdio.h>
+#  ifdef HAVE_EDIT_READLINE_READLINE_H
+#    include <edit/readline/readline.h>
+#  else
+#    include <editline/readline.h>
+#  endif
 #endif
 
 #include <errno.h>
diff --git a/configure.ac b/configure.ac
index 33b0a60..e6ecfb9 100644
--- a/configure.ac
+++ b/configure.ac
@@ -145,6 +145,7 @@ AC_SUBST(PCRE_LIBS)
 PKG_CHECK_MODULES([LIBEDIT], [libedit], 
 	[AC_DEFINE([HAVE_LIBEDIT], [1], [Define we have libedit])],
 	[AC_CHECK_HEADERS([readline/readline.h])
+	 AC_CHECK_HEADERS([edit/readline/readline.h])
 	 AC_CHECK_LIB(edit, el_init,
 	        [ AC_DEFINE([HAVE_LIBEDIT], [1], [Define we have libedit])
 		  LIBEDIT_CFLAGS=""



From phk at varnish-cache.org  Thu Apr 12 08:09:18 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 12 Apr 2012 10:09:18 +0200
Subject: [master] 480c0a5 Go over the return(restart) code.
Message-ID: <E1SIF5i-0004Rq-SC@project.varnish-software.com>

commit 480c0a5d0b7b11e8e6b66f90dd66a2d8b22d235f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Apr 12 08:06:09 2012 +0000

    Go over the return(restart) code.
    
    This is vastly inspired by #1113 and scoof+martins testcase+patch.
    
    Add a new cnt* state "cnt_restart" to do the mandatory restart work,
    and to move the restart limit check out of vcl_recv{}, getting rid
    of a nasty XXX comment.
    
    Set the explicit 503 when over limit, reset in other cases.
    
    Expand scoof+martins test-case to cover more vcl methods, which
    were also affected, but not fixed in their patch.
    
    Fixes	#1113

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 7104e0f..5530f1b 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -293,10 +293,8 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req)
 			(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
 		}
 		AZ(req->obj);
-		req->restarts++;
-		req->director = NULL;
 		http_Teardown(req->resp);
-		sp->step = STP_RECV;
+		sp->step = STP_RESTART;
 		return (0);
 	default:
 		WRONG("Illegal action in vcl_deliver{}");
@@ -524,9 +522,7 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req)
 	    req->restarts <  cache_param->max_restarts) {
 		HSH_Drop(wrk, &sp->req->obj);
 		VBO_DerefBusyObj(wrk, &req->busyobj);
-		req->director = NULL;
-		req->restarts++;
-		sp->step = STP_RECV;
+		sp->step = STP_RESTART;
 		return (0);
 	} else if (req->handling == VCL_RET_RESTART)
 		req->handling = VCL_RET_DELIVER;
@@ -660,8 +656,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 
 	switch (req->handling) {
 	case VCL_RET_RESTART:
-		req->restarts++;
-		sp->step = STP_RECV;
+		sp->step = STP_RESTART;
 		return (0);
 	case VCL_RET_ERROR:
 		sp->step = STP_ERROR;
@@ -1045,9 +1040,7 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req)
 		sp->step = STP_ERROR;
 		return (0);
 	case VCL_RET_RESTART:
-		req->director = NULL;
-		req->restarts++;
-		sp->step = STP_RECV;
+		sp->step = STP_RESTART;
 		return (0);
 	default:
 		WRONG("Illegal action in vcl_hit{}");
@@ -1231,9 +1224,7 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
 		sp->step = STP_PASS;
 		break;
 	case VCL_RET_RESTART:
-		req->restarts++;
-		req->director = NULL;
-		sp->step = STP_RECV;
+		sp->step = STP_RESTART;
 		break;
 	default:
 		WRONG("Illegal action in vcl_miss{}");
@@ -1350,6 +1341,37 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req)
 }
 
 /*--------------------------------------------------------------------
+ *
+DOT subgraph xcluster_restart {
+DOT	restart [
+DOT		shape=record
+DOT		label="{cnt_restart}"
+DOT	]
+DOT }
+DOT RESTART -> restart [color=purple]
+DOT restart -> recv [color=purple]
+ */
+
+static int
+cnt_restart(struct sess *sp, const struct worker *wrk, struct req *req)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	
+	req->director = NULL;
+	if (++req->restarts >= cache_param->max_restarts) {
+		req->err_code = 503;
+		sp->step = STP_ERROR;
+	} else {
+		req->err_code = 0;
+		sp->step = STP_RECV;
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
  * RECV
  * We have a complete request, set everything up and start it.
  * We can come here both with a request from the client and with
@@ -1368,7 +1390,6 @@ DOT		label="{cnt_recv:|{vcl_hash\{\}|req.*}}"
 DOT	]
 DOT }
 DOT ESI_REQ [ shape=hexagon ]
-DOT RESTART -> recv [color=purple]
 DOT ESI_REQ -> recv
 DOT recv:pipe -> pipe [style=bold,color=orange]
 DOT recv:pass -> pass [style=bold,color=red]
@@ -1406,18 +1427,6 @@ cnt_recv(struct sess *sp, const struct worker *wrk, struct req *req)
 	VCL_recv_method(sp);
 	recv_handling = req->handling;
 
-	if (req->restarts >= cache_param->max_restarts) {
-		/*
-		 * XXX: Why not before vcl_recv{} ?  We go to vcl_error{}
-		 * XXX: without vcl_recv{} on 413 and 417 already.
-		 * XXX tell vcl_error why we come
-		 */
-		if (req->err_code == 0)
-			req->err_code = 503;
-		sp->step = STP_ERROR;
-		return (0);
-	}
-
 	if (cache_param->http_gzip_support &&
 	     (recv_handling != VCL_RET_PIPE) &&
 	     (recv_handling != VCL_RET_PASS)) {
diff --git a/bin/varnishtest/tests/r01113.vtc b/bin/varnishtest/tests/r01113.vtc
new file mode 100644
index 0000000..abcee83
--- /dev/null
+++ b/bin/varnishtest/tests/r01113.vtc
@@ -0,0 +1,62 @@
+varnishtest "HTTP status code when hitting max_restarts"
+
+server s1 {
+	rxreq
+	txresp
+	accept
+	rxreq
+	txresp
+	accept
+	rxreq
+	txresp
+	accept
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_hit {
+		if (req.url == "/hit") {
+			return(restart);
+		}
+	}
+	sub vcl_fetch {
+		if (req.url == "/fetch") {
+			return(restart);
+		}
+	}
+	sub vcl_deliver {
+		if (req.url == "/deliver") {
+			return(restart);
+		}
+	}
+	sub vcl_miss {
+		if (req.url == "/miss") {
+			return(restart);
+		}
+	}
+} -start -cliok "param.set max_restarts 1"
+
+client c1 {
+	txreq -url /hit
+	rxresp
+	expect resp.status == 200
+	txreq -url /hit
+	rxresp
+	expect resp.status == 503
+} -run
+client c1 {
+	txreq -url /miss
+	rxresp
+	expect resp.status == 503
+} -run
+client c1 {
+	txreq -url /fetch
+	rxresp
+	expect resp.status == 503
+} -run
+client c1 {
+	txreq -url /deliver
+	rxresp
+	expect resp.status == 503
+} -run
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index e6738cb..ee70874 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -31,6 +31,7 @@
 /*lint -save -e525 -e539 */
 STEP(wait,		WAIT,		(sp, sp->wrk, sp->req))
 STEP(first,		FIRST,		(sp, sp->wrk))
+STEP(restart,		RESTART,	(sp, sp->wrk, sp->req))
 STEP(recv,		RECV,		(sp, sp->wrk, sp->req))
 STEP(start,		START,		(sp, sp->wrk, sp->req))
 STEP(pipe,		PIPE,		(sp, sp->wrk, sp->req))



From phk at varnish-cache.org  Mon Apr 16 07:05:37 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 16 Apr 2012 09:05:37 +0200
Subject: [master] 10f1de6 Fix #1126 by properly setting the mask token to the
	IP number token.
Message-ID: <E1SJg0H-0003NV-Kx@project.varnish-software.com>

commit 10f1de6232d60c8cd2d9081a7680dad857c6d5aa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 16 07:04:40 2012 +0000

    Fix #1126 by properly setting the mask token to the IP number token.
    
    Varnishtest doesn't see the difference between a VCC core dump and
    a VCC error, so a test-case is non-trivial.
    
    Fixes	#1126

diff --git a/lib/libvcl/vcc_acl.c b/lib/libvcl/vcc_acl.c
index e6a1065..b42f9de 100644
--- a/lib/libvcl/vcc_acl.c
+++ b/lib/libvcl/vcc_acl.c
@@ -263,8 +263,10 @@ vcc_acl_try_netnotation(struct vcc *tl, struct acl_e *ae)
 			return (0);
 		p += k + 1;
 	}
-	if (ae->t_mask == NULL)
+	if (ae->t_mask == NULL) {
 		ae->mask = 8 + 8 * i;
+		ae->t_mask = ae->t_addr;
+	}
 	vcc_acl_add_entry(tl, ae, 4, b, AF_INET);
 	return (1);
 }



From phk at varnish-cache.org  Mon Apr 16 07:12:31 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 16 Apr 2012 09:12:31 +0200
Subject: [master] 35475f8 Missing errorchecks incompilation of regsub()
Message-ID: <E1SJg6x-0004Az-2q@project.varnish-software.com>

commit 35475f8c5db39fa47a0d5862f00737ccd9985575
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 16 07:12:10 2012 +0000

    Missing errorchecks incompilation of regsub()
    
    Fixes #1125

diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 203d76f..30e63d9 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -454,6 +454,8 @@ vcc_Eval_Regsub(struct vcc *tl, struct expr **e, const struct symbol *sym)
 	SkipToken(tl, '(');
 
 	vcc_expr0(tl, &e2, STRING);
+	if (e2 == NULL)
+		return;
 	if (e2->fmt != STRING)
 		vcc_expr_tostring(&e2, STRING);
 
@@ -467,6 +469,8 @@ vcc_Eval_Regsub(struct vcc *tl, struct expr **e, const struct symbol *sym)
 
 	SkipToken(tl, ',');
 	vcc_expr0(tl, &e2, STRING);
+	if (e2 == NULL)
+		return;
 	if (e2->fmt != STRING)
 		vcc_expr_tostring(&e2, STRING);
 	*e = vcc_expr_edit(STRING, "\v1, \v2)", *e, e2);



From phk at varnish-cache.org  Mon Apr 16 07:23:07 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 16 Apr 2012 09:23:07 +0200
Subject: [master] 8bc9a68 Better fix for #1126
Message-ID: <E1SJgHD-0005cj-0b@project.varnish-software.com>

commit 8bc9a68068584feaab4442569f5d815ba609b925
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 16 07:22:57 2012 +0000

    Better fix for #1126

diff --git a/lib/libvcl/vcc_acl.c b/lib/libvcl/vcc_acl.c
index b42f9de..9535f59 100644
--- a/lib/libvcl/vcc_acl.c
+++ b/lib/libvcl/vcc_acl.c
@@ -101,7 +101,10 @@ vcc_acl_add_entry(struct vcc *tl, const struct acl_e *ae, int l,
 	if (fam == PF_INET && ae->mask > 32) {
 		VSB_printf(tl->sb,
 		    "Too wide mask (%u) for IPv4 address", ae->mask);
-		vcc_ErrWhere(tl, ae->t_mask);
+		if (ae->t_mask != NULL)
+			vcc_ErrWhere(tl, ae->t_mask);
+		else
+			vcc_ErrWhere(tl, ae->t_addr);
 		return;
 	}
 	if (fam == PF_INET6 && ae->mask > 128) {
@@ -263,10 +266,8 @@ vcc_acl_try_netnotation(struct vcc *tl, struct acl_e *ae)
 			return (0);
 		p += k + 1;
 	}
-	if (ae->t_mask == NULL) {
+	if (ae->t_mask == NULL)
 		ae->mask = 8 + 8 * i;
-		ae->t_mask = ae->t_addr;
-	}
 	vcc_acl_add_entry(tl, ae, 4, b, AF_INET);
 	return (1);
 }



From tfheen at varnish-cache.org  Mon Apr 16 08:20:33 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:33 +0200
Subject: [3.0] d04d9c9 Fix poll waiter,
	so that we don't terminate the search for poll'ed fd's early in the
	case of a timeout.
Message-ID: <E1SJhAn-0004u9-IS@project.varnish-software.com>

commit d04d9c99c57d45054b923860abcc9dca5509b882
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:33:58 2011 +0000

    Fix poll waiter, so that we don't terminate the search for poll'ed
    fd's early in the case of a timeout.
    
    Fixes	#1023
    
    Conflicts:
    
    	bin/varnishd/cache_waiter_poll.c

diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index 42b54dc..b9ad213 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -115,7 +115,7 @@ vca_unpoll(int fd)
 static void *
 vca_main(void *arg)
 {
-	int v;
+	int v, v2;
 	struct sess *ss[NEEV], *sp, *sp2;
 	double deadline;
 	int i, j, fd;
@@ -135,8 +135,10 @@ vca_main(void *arg)
 		v = poll(pollfd, hpoll + 1, 100);
 		assert(v >= 0);
 		deadline = TIM_real() - params->sess_timeout;
+		v2 = v;
 		VTAILQ_FOREACH_SAFE(sp, &sesshead, list, sp2) {
 			if (v == 0)
+			if (v != 0 && v2 == 0)
 				break;
 			CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 			fd = sp->fd;
@@ -145,7 +147,7 @@ vca_main(void *arg)
 			assert(fd < npoll);
 			assert(pollfd[fd].fd == fd);
 			if (pollfd[fd].revents) {
-				v--;
+				v2--;
 				i = HTC_Rx(sp->htc);
 				if (pollfd[fd].revents != POLLIN)
 					VSL(SLT_Debug, fd, "Poll: %x / %d",
@@ -167,14 +169,14 @@ vca_main(void *arg)
 				SES_Delete(sp);
 			}
 		}
-		if (v && pollfd[vca_pipes[0]].revents) {
+		if (v2 && pollfd[vca_pipes[0]].revents) {
 
 			if (pollfd[vca_pipes[0]].revents != POLLIN)
 				VSL(SLT_Debug, 0, "pipe.revents= 0x%x",
 				    pollfd[vca_pipes[0]].revents);
 			assert(pollfd[vca_pipes[0]].revents == POLLIN);
 			pollfd[vca_pipes[0]].revents = 0;
-			v--;
+			v2--;
 			i = read(vca_pipes[0], ss, sizeof ss);
 			assert(i >= 0);
 			assert(((unsigned)i % sizeof ss[0]) == 0);
@@ -185,7 +187,7 @@ vca_main(void *arg)
 				vca_poll(ss[j]->fd);
 			}
 		}
-		assert(v == 0);
+		assert(v2 == 0);
 	}
 	NEEDLESS_RETURN(NULL);
 }



From tfheen at varnish-cache.org  Mon Apr 16 08:20:33 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:33 +0200
Subject: [3.0] a8e7773 A quick style polish
Message-ID: <E1SJhAn-0004uE-JE@project.varnish-software.com>

commit a8e7773228bf6ff9d797bf1b39c5cfaed7c22359
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 11:41:39 2011 +0000

    A quick style polish

diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index e97758a..667d177 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -38,41 +38,47 @@ struct vre {
 	pcre *re;
 };
 
-vre_t *VRE_compile(const char *pattern, int options,
-		    const char **errptr, int *erroffset) {
+vre_t *
+VRE_compile(const char *pattern, int options,
+		    const char **errptr, int *erroffset)
+{
 	vre_t *v;
 	*errptr = NULL; *erroffset = 0;
 
 	ALLOC_OBJ(v, VRE_MAGIC);
-	AN(v);
+	if (v == NULL)
+		return (NULL);
 	v->re = pcre_compile(pattern, options, errptr, erroffset, NULL);
 	if (v->re == NULL) {
 		VRE_free(&v);
-		return NULL;
+		return (NULL);
 	}
-	return v;
+	return (v);
 }
 
-int VRE_exec(const vre_t *code, const char *subject, int length,
-	     int startoffset, int options, int *ovector, int ovecsize) {
+int
+VRE_exec(const vre_t *code, const char *subject, int length,
+    int startoffset, int options, int *ovector, int ovecsize)
+{
 	CHECK_OBJ_NOTNULL(code, VRE_MAGIC);
 	int ov[30];
+
 	if (ovector == NULL) {
 		ovector = ov;
 		ovecsize = sizeof(ov)/sizeof(ov[0]);
 	}
 
-	return pcre_exec(code->re, NULL, subject, length,
-			 startoffset, options, ovector, ovecsize);
+	return (pcre_exec(code->re, NULL, subject, length,
+	    startoffset, options, ovector, ovecsize));
 }
 
-void VRE_free(vre_t **vv) {
-
+void
+VRE_free(vre_t **vv)
+{
 	vre_t *v = *vv;
 
 	*vv = NULL;
 	CHECK_OBJ(v, VRE_MAGIC);
 	pcre_free(v->re);
-	v->magic = 0;
 	FREE_OBJ(v);
 }



From tfheen at varnish-cache.org  Mon Apr 16 08:20:33 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:33 +0200
Subject: [3.0] 62ab683 Make it possible to set limits for VRE matching.
Message-ID: <E1SJhAn-0004uQ-L8@project.varnish-software.com>

commit 62ab6836a23294b4600027ca7396aaacc587230e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 12:15:52 2011 +0000

    Make it possible to set limits for VRE matching.

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index a02b6ff..30ae5ca 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -76,7 +76,7 @@ VRT_re_match(const struct sess *sp, const char *s, void *re)
 		s = "";
 	AN(re);
 	t = re;
-	i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0);
+	i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0, &params->vre_limits);
 	if (i >= 0)
 		return (1);
 	if (i < VRE_ERROR_NOMATCH )
@@ -101,7 +101,8 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 		str = "";
 	t = re;
 	memset(ovector, 0, sizeof(ovector));
-	i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30);
+	i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30,
+	    &params->vre_limits);
 
 	/* If it didn't match, we can return the original string */
 	if (i == VRE_ERROR_NOMATCH)
@@ -139,7 +140,8 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 		if (!all)
 			break;
 		memset(&ovector, 0, sizeof(ovector));
-		i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30);
+		i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30,
+		    &params->vre_limits);
 		if (i < VRE_ERROR_NOMATCH ) {
 			WSP(sp, SLT_VCL_error,
 			    "Regexp matching returned %d", i);
diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index 6571e31..99d5747 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -30,6 +30,7 @@
  */
 
 #include <pthread.h>
+#include "vre.h"
 
 struct listen_sock {
 	VTAILQ_ENTRY(listen_sock)	list;
@@ -211,6 +212,8 @@ struct params {
 	double			critbit_cooloff;
 
 	double			shortlived;
+
+	struct vre_limits	vre_limits;
 };
 
 /*
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index 4ca1b7b..6113978 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -948,6 +948,24 @@ static const struct parspec input_parspec[] = {
 		"Unreferenced VCL objects result in error.\n",
 		0,
 		"on", "bool" },
+
+
+	{ "pcre_match_limit", tweak_uint,
+		&master.vre_limits.match,
+		1, UINT_MAX,
+		"The limit for the  number of internal matching function"
+		" calls in a pcre_exec() execution.",
+		0,
+		"10000", ""},
+
+	{ "pcre_match_limit_recursion", tweak_uint,
+		&master.vre_limits.match_recursion,
+		1, UINT_MAX,
+		"The limit for the  number of internal matching function"
+		" recursions in a pcre_exec() execution.",
+		0,
+		"10000", ""},
+
 	{ NULL, NULL, NULL }
 };
 
diff --git a/include/vre.h b/include/vre.h
index 47dea3f..29c1b08 100644
--- a/include/vre.h
+++ b/include/vre.h
@@ -27,9 +27,21 @@
  *
  * Regular expression support
  *
+ * We wrap PCRE in VRE to make to make it feasible to use something else
+ * without hunting down stuff through out the Varnish source code.
+ *
  */
 
+#ifndef VRE_H_INCLUDED
+#define VRE_H_INCLUDED
+
 struct vre;
+
+struct vre_limits {
+	unsigned	match;
+	unsigned	match_recursion;
+};
+
 typedef struct vre vre_t;
 
 /* This maps to PCRE error codes */
@@ -39,5 +51,9 @@ typedef struct vre vre_t;
 #define VRE_CASELESS           0x00000001
 
 vre_t *VRE_compile(const char *, int, const char **, int *);
-int VRE_exec(const vre_t *, const char *, int, int, int, int *, int);
+int VRE_exec(const vre_t *code, const char *subject, int length,
+    int startoffset, int options, int *ovector, int ovecsize,
+    const volatile struct vre_limits *lim);
 void VRE_free(vre_t **);
+
+#endif /* VRE_H_INCLUDED */
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 667d177..68beaf8 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -27,6 +27,7 @@
  */
 
 #include <pcre.h>
+#include <string.h>
 
 #include "libvarnish.h"
 #include "miniobj.h"
@@ -58,17 +59,27 @@ VRE_compile(const char *pattern, int options,
 
 int
 VRE_exec(const vre_t *code, const char *subject, int length,
-    int startoffset, int options, int *ovector, int ovecsize)
+    int startoffset, int options, int *ovector, int ovecsize,
+    const volatile struct vre_limits *lim)
 {
 	CHECK_OBJ_NOTNULL(code, VRE_MAGIC);
 	int ov[30];
+	pcre_extra extra;
 
 	if (ovector == NULL) {
 		ovector = ov;
 		ovecsize = sizeof(ov)/sizeof(ov[0]);
 	}
 
-	return (pcre_exec(code->re, NULL, subject, length,
+	memset(&extra, 0, sizeof extra);
+	if (lim != NULL) {
+		extra.match_limit = lim->match;
+		extra.flags |= PCRE_EXTRA_MATCH_LIMIT;
+		extra.match_limit_recursion = lim->match_recursion;
+		extra.flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION;
+	}
+
+	return (pcre_exec(code->re, &extra, subject, length,
 	    startoffset, options, ovector, ovecsize));
 }
 
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 6791888..8ffff30 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -258,13 +258,13 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 			continue;
 		if (vsl->regincl != NULL) {
 			i = VRE_exec(vsl->regincl, VSL_DATA(p), VSL_LEN(p),
-			    0, 0, NULL, 0);
+			    0, 0, NULL, 0, NULL);
 			if (i == VRE_ERROR_NOMATCH)
 				continue;
 		}
 		if (vsl->regexcl != NULL) {
 			i = VRE_exec(vsl->regexcl, VSL_DATA(p), VSL_LEN(p),
-			    0, 0, NULL, 0);
+			    0, 0, NULL, 0, NULL);
 			if (i != VRE_ERROR_NOMATCH)
 				continue;
 		}
@@ -274,7 +274,7 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 			VTAILQ_FOREACH(vrm, &vsl->matchers, next) {
 				if (vrm->tag == t) {
 					i = VRE_exec(vrm->re, VSL_DATA(p),
-						     VSL_LEN(p), 0, 0, NULL, 0);
+					    VSL_LEN(p), 0, 0, NULL, 0, NULL);
 					if (i >= 0)
 						*bits |= (uintmax_t)1 << j;
 				}



From tfheen at varnish-cache.org  Mon Apr 16 08:20:33 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:33 +0200
Subject: [3.0] 0d77e83 Give VRT_re_match a sess* arg and report VRE errors
	using SLT_VCL_Error instead of asserting.
Message-ID: <E1SJhAn-0004uK-K9@project.varnish-software.com>

commit 0d77e8383c08895d98b04a84e6619f7a0cc638dc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 11:48:16 2011 +0000

    Give VRT_re_match a sess* arg and report VRE errors using SLT_VCL_Error
    instead of asserting.
    
    Inspired by:	Patch from DocWilco

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index 8e1218a..a02b6ff 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -67,7 +67,7 @@ VRT_re_fini(void *rep)
 }
 
 int
-VRT_re_match(const char *s, void *re)
+VRT_re_match(const struct sess *sp, const char *s, void *re)
 {
 	vre_t *t;
 	int i;
@@ -79,7 +79,8 @@ VRT_re_match(const char *s, void *re)
 	i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0);
 	if (i >= 0)
 		return (1);
-	assert(i == VRE_ERROR_NOMATCH);
+	if (i < VRE_ERROR_NOMATCH )
+		WSP(sp, SLT_VCL_error, "Regexp matching returned %d", i);
 	return (0);
 }
 
@@ -105,6 +106,10 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 	/* If it didn't match, we can return the original string */
 	if (i == VRE_ERROR_NOMATCH)
 		return(str);
+	if (i < VRE_ERROR_NOMATCH ) {
+		WSP(sp, SLT_VCL_error, "Regexp matching returned %d", i);
+		return(str);
+	}
 
 	u = WS_Reserve(sp->http->ws, 0);
 	res.e = res.b = b0 = sp->http->ws->f;
@@ -135,6 +140,11 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 			break;
 		memset(&ovector, 0, sizeof(ovector));
 		i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30);
+		if (i < VRE_ERROR_NOMATCH ) {
+			WSP(sp, SLT_VCL_error,
+			    "Regexp matching returned %d", i);
+			return(str);
+		}
 	} while (i != VRE_ERROR_NOMATCH);
 
 	/* Copy suffix to match */
diff --git a/include/vrt.h b/include/vrt.h
index 9f0228f..72c4cfe 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -145,7 +145,7 @@ void VRT_acl_log(const struct sess *, const char *msg);
 /* Regexp related */
 void VRT_re_init(void **, const char *);
 void VRT_re_fini(void *);
-int VRT_re_match(const char *, void *re);
+int VRT_re_match(const struct sess *sp, const char *, void *re);
 const char *VRT_regsub(const struct sess *sp, int all, const char *,
     void *, const char *);
 
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 3883dd2..1f1c274 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -941,7 +941,7 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, enum var_type fmt)
 		re = vcc_regexp(tl);
 		ERRCHK(tl);
 		vcc_NextToken(tl);
-		bprintf(buf, "%sVRT_re_match(\v1, %s)", not, re);
+		bprintf(buf, "%sVRT_re_match(sp, \v1, %s)", not, re);
 		*e = vcc_expr_edit(BOOL, buf, *e, NULL);
 		return;
 	}



From tfheen at varnish-cache.org  Mon Apr 16 08:20:33 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:33 +0200
Subject: [3.0] 5e46b7c Bump minimum number of threads for RPMs
Message-ID: <E1SJhAn-0004uY-MR@project.varnish-software.com>

commit 5e46b7c0f5b27a72ad3aec8ba1c6329fa4f8c362
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 3 11:15:19 2011 +0200

    Bump minimum number of threads for RPMs
    
    1 thread is a bit on the low side, make the default on RPM based
    distros 50.

diff --git a/redhat/varnish.sysconfig b/redhat/varnish.sysconfig
index 8ab3c22..d88712f 100644
--- a/redhat/varnish.sysconfig
+++ b/redhat/varnish.sysconfig
@@ -70,7 +70,7 @@ VARNISH_ADMIN_LISTEN_PORT=6082
 VARNISH_SECRET_FILE=/etc/varnish/secret
 #
 # # The minimum number of worker threads to start
-VARNISH_MIN_THREADS=1
+VARNISH_MIN_THREADS=50
 #
 # # The Maximum number of worker threads to start
 VARNISH_MAX_THREADS=1000



From tfheen at varnish-cache.org  Mon Apr 16 08:20:33 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:33 +0200
Subject: [3.0] 98f1c8c Add a "wait-running" primitive to varnish instances,
	so we can avoid fixed sleeps waiting for the child process to start.
Message-ID: <E1SJhAn-0004uj-No@project.varnish-software.com>

commit 98f1c8c743e3b848c1191b0adfdaec39b64a35d2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 6 09:02:18 2011 +0000

    Add a "wait-running" primitive to varnish instances, so we can
    avoid fixed sleeps waiting for the child process to start.

diff --git a/bin/varnishtest/tests/v00010.vtc b/bin/varnishtest/tests/v00010.vtc
index ec7b947..0421956 100644
--- a/bin/varnishtest/tests/v00010.vtc
+++ b/bin/varnishtest/tests/v00010.vtc
@@ -5,6 +5,8 @@ server s1 {
 	txresp -hdr "Foo: bar" -body "abcdef\n"
 	rxreq
 	txresp -hdr "Panic: please" -body "012345\n"
+	close
+
 	sema r1 sync 2
 	accept
 	rxreq
@@ -33,10 +35,10 @@ client c1 {
 	rxresp
 	txreq -url "/foo"
 	# Don't expect answer, the server crashed.
-	sema r1 sync 2
 } -run
 
-delay 2.5
+varnish v1 -wait-running
+sema r1 sync 2
 
 client c1 {
 	txreq -url "/"
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 2ff6ea9..d4b4569 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -120,6 +120,30 @@ varnish_ask_cli(const struct varnish *v, const char *cmd, char **repl)
 }
 
 /**********************************************************************
+ *
+ */
+
+static void
+wait_running(const struct varnish *v)
+{
+	char *r;
+	enum VCLI_status_e st;
+
+	while (1) {
+		st = varnish_ask_cli(v, "status", &r);
+		if (st != CLIS_OK)
+			vtc_log(v->vl, 0,
+			    "CLI status command failed: %u %s", st, r);
+		if (!strcmp(r, "Child in state running")) {
+			free(r);
+			break;
+		}
+		free(r);
+		(void)usleep(200000);
+	}
+}
+
+/**********************************************************************
  * Varnishlog gatherer + thread
  */
 
@@ -431,6 +455,7 @@ varnish_start(struct varnish *v)
 		return;
 	if (u != CLIS_OK)
 		vtc_log(v->vl, 0, "CLI start command failed: %u %s", u, resp);
+	wait_running(v);
 	free(resp);
 	u = varnish_ask_cli(v, "debug.xid 1000", &resp);
 	if (vtc_error)
@@ -815,6 +840,10 @@ cmd_varnish(CMD_ARGS)
 			varnish_stop(v);
 			continue;
 		}
+		if (!strcmp(*av, "-wait-running")) {
+			wait_running(v);
+			continue;
+		}
 		if (!strcmp(*av, "-wait")) {
 			varnish_wait(v);
 			continue;



From tfheen at varnish-cache.org  Mon Apr 16 08:20:33 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:33 +0200
Subject: [3.0] f697e67 Try to firm up v00010 even more,
	by explicitly waiting for the crashing child to do so,
	and then explicitly start it again.
Message-ID: <E1SJhAn-0004uw-Oc@project.varnish-software.com>

commit f697e673e108373930884e3b294b1d36541d4c8d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 6 10:48:57 2011 +0000

    Try to firm up v00010 even more, by explicitly waiting for the
    crashing child to do so, and then explicitly start it again.

diff --git a/bin/varnishtest/tests/v00010.vtc b/bin/varnishtest/tests/v00010.vtc
index 0421956..45dbc79 100644
--- a/bin/varnishtest/tests/v00010.vtc
+++ b/bin/varnishtest/tests/v00010.vtc
@@ -27,9 +27,6 @@ varnish v1 -cliok "param.set diag_bitmap 0x00001000"
 # Force the (random) port selected to be used again after restart.
 varnish v1 -cliok "param.set listen_address ${v1_addr}:${v1_port}"
 
-# varnishtest defaults to auto_restart off, to avoid masking bugs.
-varnish v1 -cliok "param.set auto_restart on" 
-
 client c1 {
 	txreq -url "/"
 	rxresp
@@ -37,6 +34,11 @@ client c1 {
 	# Don't expect answer, the server crashed.
 } -run
 
+varnish v1 -wait-stopped
+varnish v1 -cliok "panic.show"
+varnish v1 -cliok "panic.clear"
+varnish v1 -clierr 300 "panic.clear"
+varnish v1 -cliok "start"
 varnish v1 -wait-running
 sema r1 sync 2
 
@@ -45,7 +47,3 @@ client c1 {
 	rxresp
 	expect resp.http.foo == "foo"
 } -run
-
-varnish v1 -cliok "panic.show"
-varnish v1 -cliok "panic.clear"
-varnish v1 -clierr 300 "panic.clear"
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index d4b4569..6871b5f 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -124,12 +124,37 @@ varnish_ask_cli(const struct varnish *v, const char *cmd, char **repl)
  */
 
 static void
+wait_stopped(const struct varnish *v)
+{
+	char *r;
+	enum VCLI_status_e st;
+
+	while (1) {
+		vtc_log(v->vl, 3, "wait-stopped");
+		st = varnish_ask_cli(v, "status", &r);
+		if (st != CLIS_OK)
+			vtc_log(v->vl, 0,
+			    "CLI status command failed: %u %s", st, r);
+		if (!strcmp(r, "Child in state stopped")) {
+			free(r);
+			break;
+		}
+		free(r);
+		(void)usleep(200000);
+	}
+}
+/**********************************************************************
+ *
+ */
+
+static void
 wait_running(const struct varnish *v)
 {
 	char *r;
 	enum VCLI_status_e st;
 
 	while (1) {
+		vtc_log(v->vl, 3, "wait-running");
 		st = varnish_ask_cli(v, "status", &r);
 		if (st != CLIS_OK)
 			vtc_log(v->vl, 0,
@@ -840,6 +865,10 @@ cmd_varnish(CMD_ARGS)
 			varnish_stop(v);
 			continue;
 		}
+		if (!strcmp(*av, "-wait-stopped")) {
+			wait_stopped(v);
+			continue;
+		}
 		if (!strcmp(*av, "-wait-running")) {
 			wait_running(v);
 			continue;



From tfheen at varnish-cache.org  Mon Apr 16 08:20:33 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:33 +0200
Subject: [3.0] b4ef285 Elminiate pthread poisoning of manager process.
Message-ID: <E1SJhAn-0004v2-Pl@project.varnish-software.com>

commit b4ef285fc16e7e46c02315ba972d637440ec60b4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 7 10:03:50 2011 +0000

    Elminiate pthread poisoning of manager process.

diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index 99d5747..daf4a1c 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -29,7 +29,6 @@
  * This file contains the heritage passed when mgt forks cache
  */
 
-#include <pthread.h>
 #include "vre.h"
 
 struct listen_sock {
diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
index 578fcad..41fb525 100644
--- a/bin/varnishd/mgt.h
+++ b/bin/varnishd/mgt.h
@@ -108,3 +108,7 @@ extern unsigned mgt_vcc_err_unref;
 #define VSM_Free(a)		VSM__Free(a)
 #define VSM_Clean()		VSM__Clean()
 
+
+#if defined(PTHREAD_CANCELED) || defined(PTHREAD_MUTEX_DEFAULT)
+#error "Keep pthreads out of in manager process"
+#endif
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 49e0f81..fd91d08 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -653,3 +653,7 @@ main(int argc, char * const *argv)
 		(void)VPF_Remove(pfh);
 	exit(exit_status);
 }
+
+#if defined(PTHREAD_CANCELED) || defined(PTHREAD_MUTEX_DEFAULT)
+#error "Keep pthreads out of in manager process"
+#endif



From tfheen at varnish-cache.org  Mon Apr 16 08:20:33 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:33 +0200
Subject: [3.0] 12d4a11 WRK_Flush was renamed to WRW_Flush a long time ago;
	update some ifdefed code and comments to match
Message-ID: <E1SJhAn-0004vC-R2@project.varnish-software.com>

commit 12d4a11f66a3f2ca2e3908d0fbaa776c8ef02037
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 10 08:14:31 2011 +0200

    WRK_Flush was renamed to WRW_Flush a long time ago; update some ifdefed code and comments to match

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 3d6982d..a00e5e1 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -260,7 +260,7 @@ struct exp {
 
 struct wrw {
 	int			*wfd;
-	unsigned		werr;	/* valid after WRK_Flush() */
+	unsigned		werr;	/* valid after WRW_Flush() */
 	struct iovec		*iov;
 	unsigned		siov;
 	unsigned		niov;
diff --git a/bin/varnishd/cache_wrw.c b/bin/varnishd/cache_wrw.c
index 61aa2e5..f25bafb 100644
--- a/bin/varnishd/cache_wrw.c
+++ b/bin/varnishd/cache_wrw.c
@@ -264,7 +264,7 @@ WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len)
 	} while (0);
 #elif defined(__linux__)
 	do {
-		if (WRK_Flush(w) == 0 &&
+		if (WRW_Flush(w) == 0 &&
 		    sendfile(*wrw->wfd, fd, &off, len) != len)
 			wrw->werr++;
 	} while (0);
@@ -293,7 +293,7 @@ WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len)
 	} while (0);
 #elif defined(__sun) && defined(HAVE_SENDFILE)
 	do {
-		if (WRK_Flush(w) == 0 &&
+		if (WRW_Flush(w) == 0 &&
 		    sendfile(*wrw->wfd, fd, &off, len) != len)
 			wrw->werr++;
 	} while (0);



From tfheen at varnish-cache.org  Mon Apr 16 08:20:33 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:33 +0200
Subject: [3.0] 09b66bd Use = rather than := in Makefile to make solaris make
	stop whining on distcheck
Message-ID: <E1SJhAn-0004vQ-SK@project.varnish-software.com>

commit 09b66bd7060ab2cb90bef7608ccfa23068543180
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Oct 13 08:33:36 2011 +0200

    Use = rather than := in Makefile to make solaris make stop whining on distcheck

diff --git a/lib/libjemalloc/Makefile.am b/lib/libjemalloc/Makefile.am
index 32b3462..63f5c7a 100644
--- a/lib/libjemalloc/Makefile.am
+++ b/lib/libjemalloc/Makefile.am
@@ -1,6 +1,6 @@
 # See source code comments to avoid memory leaks when enabling MALLOC_MAG.
-#CPPFLAGS := -DMALLOC_PRODUCTION -DMALLOC_MAG
-AM_CPPFLAGS := -DMALLOC_PRODUCTION
+#CPPFLAGS = -DMALLOC_PRODUCTION -DMALLOC_MAG
+AM_CPPFLAGS = -DMALLOC_PRODUCTION
 
 #all: libjemalloc.so.0 libjemalloc_mt.so.0
 



From tfheen at varnish-cache.org  Mon Apr 16 08:20:33 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:33 +0200
Subject: [3.0] ef21769 Quit early if setting blocking mode fails.
Message-ID: <E1SJhAn-0004vZ-Tw@project.varnish-software.com>

commit ef2176972b9249b84ffb08bfb6b0ae162146e66b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 13 10:11:42 2011 +0000

    Quit early if setting blocking mode fails.
    
    Submitted by:	Nils Goroll

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 4d94d88..bc64a73 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -1575,14 +1575,21 @@ CNT_Session(struct sess *sp)
 	AZ(w->do_esi);
 
 	/*
-	 * Whenever we come in from the acceptor we need to set blocking
-	 * mode, but there is no point in setting it when we come from
+	 * Whenever we come in from the acceptor or waiter, we need to set
+	 * blocking mode, but there is no point in setting it when we come from
 	 * ESI or when a parked sessions returns.
-	 * It would be simpler to do this in the acceptor, but we'd rather
-	 * do the syscall in the worker thread.
+	 * It would be simpler to do this in the acceptor or waiter, but we'd
+	 * rather do the syscall in the worker thread.
+	 * On systems which return errors for ioctl, we close early
 	 */
-	if (sp->step == STP_FIRST || sp->step == STP_START)
-		(void)VTCP_blocking(sp->fd);
+	if ((sp->step == STP_FIRST || sp->step == STP_START) &&
+	    VTCP_blocking(sp->fd)) {
+		if (errno == ECONNRESET)
+			vca_close_session(sp, "remote closed");
+		else
+			vca_close_session(sp, "error");
+		sp->step = STP_DONE;
+	}
 
 	/*
 	 * NB: Once done is set, we can no longer touch sp!



From tfheen at varnish-cache.org  Mon Apr 16 08:20:33 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:33 +0200
Subject: [3.0] 50b79ba Drop explicit dependeny on Makefile for default_vcl.h
	to avoid spurious rebuilds
Message-ID: <E1SJhAn-0004vn-VS@project.varnish-software.com>

commit 50b79ba587926cc6d9be90dd44d4ed4f982d1cc9
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Fri Oct 14 12:39:57 2011 +0200

    Drop explicit dependeny on Makefile for default_vcl.h to avoid spurious rebuilds

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 00dc716..bfeb6b9 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -118,7 +118,7 @@ DISTCLEANFILES = default_vcl.h
 #
 # Turn the default.vcl file into a C-string we can include in the program.
 #
-default_vcl.h:	default.vcl Makefile
+default_vcl.h:	default.vcl
 	echo '/*' > $@
 	echo ' * NB:  This file is machine generated, DO NOT EDIT!' >> $@
 	echo ' *' >> $@



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] f85a52c Attempt to document the different options for
	invalidating cached content
Message-ID: <E1SJhAo-0004vv-0I@project.varnish-software.com>

commit f85a52c00a9387db0682b921f7ef004472f3a264
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Wed Oct 19 22:43:25 2011 +0200

    Attempt to document the different options for invalidating cached content

diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index 84449da..9e41430 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -21,6 +21,7 @@ The Varnish Reference Manual
 	shmem.rst
 	vmod.rst
 	vmod_std.rst
+	purging_banning.rst
 	vsl.rst
 
 .. todo::
diff --git a/doc/sphinx/reference/purging_banning.rst b/doc/sphinx/reference/purging_banning.rst
new file mode 100644
index 0000000..4d63d8a
--- /dev/null
+++ b/doc/sphinx/reference/purging_banning.rst
@@ -0,0 +1,140 @@
+%%%%%%%%%%%%%%%%%%%%%%%%%%%
+Purging and banning content
+%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+Varnish has three ways to invalidate content in varnish. You can either purge an object and all the variants of it from the cache, or you can prevent past versions of an object from being served by banning it. Finally, you can force a cache miss to force a backend fetch and override an object in the cache.
+
+Purging
+=======
+
+To purge an object, you need to access that object explicitly, which is why the purge method is only available in VCL in the methods `vcl_hit` and `vcl_miss`. Purging is available in `vcl_miss` to allow for purging of all other variants of this object, even when this is particular request didn't hit a variant. Purging explicitly evicts that object and all variants from the cache immediately. An example implementation of the HTTP PURGE method in VCL::
+
+  acl purgers {
+    "localhost";
+    "192.0.2.1"/24;
+  }
+
+  sub vcl_recv {
+    if (req.request == "PURGE") {
+      if (!client.ip ~ purgers) {
+        error 405 "Not allowed.";
+      }
+      return(lookup);
+    }
+  }
+
+  sub vcl_hit {
+    if (req.request == "PURGE") {
+      purge;
+      error 200 "Purged.";
+    }
+  }
+
+  sub vcl_miss {
+    if (req.request == "PURGE") {
+      purge;
+      error 200 "Purged.";
+    }
+  }
+
+Banning
+=======
+
+Banning prevents varnish from serving all matching objects in the cache at the time of the ban. Banning is not a permanent operation, it is only used to invalidate and evict content already in the cache. It does not immediately evict objects from the cache, but will compare all future hits to this ban, and evict the objects if they match. When a ban has been matched against all objects in the cache, or when all objects in the cache is newer than the ban, it is deleted.
+
+Bans are added to the ban list using the `ban` CLI command or the `ban()` VCL method. They both take a ban expression that matches the objects that should be banned.
+
+Varnish also does duplicate ban detection if `ban_dups` is enabled.
+
+Ban Expressions
+---------------
+
+A ban expression is a list of conditions that needs to be fulfilled to invalidate content. You can match the content of the following variables using equality comparison or regular expressions:
+
+ * req.url
+ * req.http.*
+ * obj.http.*
+
+Conditions are combined using logical and: &&
+
+To ban any content served from an Apache backend, you could use this expression::
+
+ obj.http.Server ~ Apache
+
+To ban a particular URL and hostname::
+
+ req.url == /this/url && req.http.Host == example.com
+
+Banning From CLI
+----------------
+
+To ban from CLI, use the ban or the ban.url commands::
+
+ ban obj.http.Server ~ Apache
+ ban.url /images/.*
+
+ban.url is equivelant to "ban req.url ~"
+
+Banning From VCL
+----------------
+
+To ban from VCL, use the ban() or ban_url() functions. You can use the full arsenal of varnish string manipulation functions to build your ban expression. For example to let users execute requests that purge based on regular expressions::
+
+ sub vcl_recv {
+  if (req.url ~ "^/purgere/") {
+   if (!client.ip ~ purge) {
+    error 405 "Not allowed.";
+   }
+   set req.url = regsub(req.url, "^/purgere/", "/");
+   ban("obj.http.x-url ~ " req.url);
+   error 200 "Banned.";
+  }
+ }
+
+The Ban List
+------------
+
+The ban list can be inspected via the CLI command ban.list.
+
+Example output::
+
+  0xb75096d0 1318329475.377475    10      obj.http.x-url ~ test
+  0xb7509610 1318329470.785875    20G     obj.http.x-url ~ test
+
+The ban list contains the ID of the ban, the timestamp when the ban entered the ban list. A count of the objects that has reached this point in the ban list, optionally postfixed with a 'G' for "Gone", if the ban is no longer valid. Finally, the ban expression is listed. The ban can be marked as Gone if it is a duplicate ban, but is still kept in the list for optimization purposes.
+
+The Ban Lurker
+--------------
+
+Since a ban needs to be be matched against all objects in the cache, one way to speed up the eviction process is to enable the ban lurker. The ban lurker will walk the cache and match all objects to the bans in the ban list, and evict matching objects. The ban lurker is enabled by setting `ban_lurker_sleep` to a value above 0.
+
+Since Varnish 3.0, the ban lurker is enabled by default.
+
+Writing Ban Lurker Friendly Bans
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To fully utilize the ban lurker, bans need to be written without the use of any req.* parameters, since there is no request to match against when the ban lurker walks the cache.
+
+A simple mode to avoid req.* in bans is to add headers to the cached object containing the parts of the request on which you want to ban, e.g.::
+
+  sub vcl_fetch {
+      set obj.http.x-url = req.url;
+  }
+
+  sub vcl_deliver {
+      unset resp.http.x-url; # Optional
+  }
+
+  sub vcl_recv {
+      if (req.request == "PURGE") {
+            if (client.ip !~ purgers) {
+                error 401 "Not allowed";
+             }
+            purge("obj.http.x-url ~ " req.url); # Assumes req.url is a regex. This might be a bit too simple
+      }
+  }
+
+req.hash_always_miss
+====================
+
+The final way to invalidate an object is a method that allows you to refresh an object by forcing a hash miss for a single request. If you set `req.hash_always_miss` to true, varnish will miss the current object in the cache, thus forcing a fetch from the backend. This can in turn add the freshly fetched object to the cache, thus overriding the current one. The old object will stay in the cache until ttl expires or it is evicted by some other means.



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] 000f199 Make EXP_NukeOne() make do with a struct worker arg
	instead of sess.
Message-ID: <E1SJhAo-0004w4-1e@project.varnish-software.com>

commit 000f1991af68acb1ea68a45d8c492ba7bba72c88
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 21 12:19:16 2011 +0000

    Make EXP_NukeOne() make do with a struct worker arg instead of sess.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index a00e5e1..3cb0af9 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -714,7 +714,7 @@ void EXP_Inject(struct objcore *oc, struct lru *lru, double when);
 void EXP_Init(void);
 void EXP_Rearm(const struct object *o);
 int EXP_Touch(struct objcore *oc);
-int EXP_NukeOne(const struct sess *sp, struct lru *lru);
+int EXP_NukeOne(struct worker *w, struct lru *lru);
 
 /* cache_fetch.c */
 struct storage *FetchStorage(const struct sess *sp, ssize_t sz);
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index bc64a73..7f9198f 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -441,8 +441,8 @@ cnt_error(struct sess *sp)
 		     &w->exp, (uint16_t)params->http_max_hdr);
 		if (sp->obj == NULL)
 			sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE,
-			     params->http_resp_size , &w->exp,
-			     (uint16_t)params->http_max_hdr);
+			    params->http_resp_size, &w->exp,
+			    (uint16_t)params->http_max_hdr);
 		if (sp->obj == NULL) {
 			sp->doclose = "Out of objects";
 			sp->director = NULL;
diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c
index a07ab6d..7fa48da 100644
--- a/bin/varnishd/cache_expire.c
+++ b/bin/varnishd/cache_expire.c
@@ -417,7 +417,7 @@ exp_timer(struct sess *sp, void *priv)
  */
 
 int
-EXP_NukeOne(const struct sess *sp, struct lru *lru)
+EXP_NukeOne(struct worker *w, struct lru *lru)
 {
 	struct objcore *oc;
 	struct object *o;
@@ -449,9 +449,9 @@ EXP_NukeOne(const struct sess *sp, struct lru *lru)
 		return (-1);
 
 	/* XXX: bad idea for -spersistent */
-	o = oc_getobj(sp->wrk, oc);
-	WSL(sp->wrk, SLT_ExpKill, 0, "%u LRU", o->xid);
-	(void)HSH_Deref(sp->wrk, NULL, &o);
+	o = oc_getobj(w, oc);
+	WSL(w, SLT_ExpKill, 0, "%u LRU", o->xid);
+	(void)HSH_Deref(w, NULL, &o);
 	return (1);
 }
 
diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index 844e71b..5ce1b6a 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -1003,7 +1003,6 @@ http_PutResponse(struct worker *w, int fd, const struct http *to,
 	if (to->hd[HTTP_HDR_RESPONSE].b == NULL)
 		http_SetH(to, HTTP_HDR_RESPONSE, "Lost Response");
 	Tcheck(to->hd[HTTP_HDR_RESPONSE]);
-
 }
 
 void
diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c
index 624d2b1..00eb645 100644
--- a/bin/varnishd/stevedore.c
+++ b/bin/varnishd/stevedore.c
@@ -183,7 +183,7 @@ stv_alloc(const struct sess *sp, size_t size)
 		}
 
 		/* no luck; try to free some space and keep trying */
-		if (EXP_NukeOne(sp, stv->lru) == -1)
+		if (EXP_NukeOne(sp->wrk, stv->lru) == -1)
 			break;
 
 		/* Enough is enough: try another if we have one */
@@ -335,7 +335,7 @@ STV_NewObject(struct sess *sp, const char *hint, unsigned wsl, struct exp *ep,
 	if (o == NULL) {
 		/* no luck; try to free some space and keep trying */
 		for (i = 0; o == NULL && i < params->nuke_limit; i++) {
-			if (EXP_NukeOne(sp, stv->lru) == -1)
+			if (EXP_NukeOne(sp->wrk, stv->lru) == -1)
 				break;
 			o = stv->allocobj(stv, sp, ltot, &soc);
 		}



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] 910518a Merge new material from reference/purging_banning,
	and do a little house keeping
Message-ID: <E1SJhAo-0004wH-2v@project.varnish-software.com>

commit 910518a0cb708486c2a85d61121948cfc1b38061
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Sat Oct 22 10:58:22 2011 +0200

    Merge new material from reference/purging_banning, and do a little house keeping
    
    Remove reference/purging_banning, since all info is now in tutorial/purging

diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index 9e41430..84449da 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -21,7 +21,6 @@ The Varnish Reference Manual
 	shmem.rst
 	vmod.rst
 	vmod_std.rst
-	purging_banning.rst
 	vsl.rst
 
 .. todo::
diff --git a/doc/sphinx/reference/purging_banning.rst b/doc/sphinx/reference/purging_banning.rst
deleted file mode 100644
index 4d63d8a..0000000
--- a/doc/sphinx/reference/purging_banning.rst
+++ /dev/null
@@ -1,140 +0,0 @@
-%%%%%%%%%%%%%%%%%%%%%%%%%%%
-Purging and banning content
-%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-Varnish has three ways to invalidate content in varnish. You can either purge an object and all the variants of it from the cache, or you can prevent past versions of an object from being served by banning it. Finally, you can force a cache miss to force a backend fetch and override an object in the cache.
-
-Purging
-=======
-
-To purge an object, you need to access that object explicitly, which is why the purge method is only available in VCL in the methods `vcl_hit` and `vcl_miss`. Purging is available in `vcl_miss` to allow for purging of all other variants of this object, even when this is particular request didn't hit a variant. Purging explicitly evicts that object and all variants from the cache immediately. An example implementation of the HTTP PURGE method in VCL::
-
-  acl purgers {
-    "localhost";
-    "192.0.2.1"/24;
-  }
-
-  sub vcl_recv {
-    if (req.request == "PURGE") {
-      if (!client.ip ~ purgers) {
-        error 405 "Not allowed.";
-      }
-      return(lookup);
-    }
-  }
-
-  sub vcl_hit {
-    if (req.request == "PURGE") {
-      purge;
-      error 200 "Purged.";
-    }
-  }
-
-  sub vcl_miss {
-    if (req.request == "PURGE") {
-      purge;
-      error 200 "Purged.";
-    }
-  }
-
-Banning
-=======
-
-Banning prevents varnish from serving all matching objects in the cache at the time of the ban. Banning is not a permanent operation, it is only used to invalidate and evict content already in the cache. It does not immediately evict objects from the cache, but will compare all future hits to this ban, and evict the objects if they match. When a ban has been matched against all objects in the cache, or when all objects in the cache is newer than the ban, it is deleted.
-
-Bans are added to the ban list using the `ban` CLI command or the `ban()` VCL method. They both take a ban expression that matches the objects that should be banned.
-
-Varnish also does duplicate ban detection if `ban_dups` is enabled.
-
-Ban Expressions
----------------
-
-A ban expression is a list of conditions that needs to be fulfilled to invalidate content. You can match the content of the following variables using equality comparison or regular expressions:
-
- * req.url
- * req.http.*
- * obj.http.*
-
-Conditions are combined using logical and: &&
-
-To ban any content served from an Apache backend, you could use this expression::
-
- obj.http.Server ~ Apache
-
-To ban a particular URL and hostname::
-
- req.url == /this/url && req.http.Host == example.com
-
-Banning From CLI
-----------------
-
-To ban from CLI, use the ban or the ban.url commands::
-
- ban obj.http.Server ~ Apache
- ban.url /images/.*
-
-ban.url is equivelant to "ban req.url ~"
-
-Banning From VCL
-----------------
-
-To ban from VCL, use the ban() or ban_url() functions. You can use the full arsenal of varnish string manipulation functions to build your ban expression. For example to let users execute requests that purge based on regular expressions::
-
- sub vcl_recv {
-  if (req.url ~ "^/purgere/") {
-   if (!client.ip ~ purge) {
-    error 405 "Not allowed.";
-   }
-   set req.url = regsub(req.url, "^/purgere/", "/");
-   ban("obj.http.x-url ~ " req.url);
-   error 200 "Banned.";
-  }
- }
-
-The Ban List
-------------
-
-The ban list can be inspected via the CLI command ban.list.
-
-Example output::
-
-  0xb75096d0 1318329475.377475    10      obj.http.x-url ~ test
-  0xb7509610 1318329470.785875    20G     obj.http.x-url ~ test
-
-The ban list contains the ID of the ban, the timestamp when the ban entered the ban list. A count of the objects that has reached this point in the ban list, optionally postfixed with a 'G' for "Gone", if the ban is no longer valid. Finally, the ban expression is listed. The ban can be marked as Gone if it is a duplicate ban, but is still kept in the list for optimization purposes.
-
-The Ban Lurker
---------------
-
-Since a ban needs to be be matched against all objects in the cache, one way to speed up the eviction process is to enable the ban lurker. The ban lurker will walk the cache and match all objects to the bans in the ban list, and evict matching objects. The ban lurker is enabled by setting `ban_lurker_sleep` to a value above 0.
-
-Since Varnish 3.0, the ban lurker is enabled by default.
-
-Writing Ban Lurker Friendly Bans
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To fully utilize the ban lurker, bans need to be written without the use of any req.* parameters, since there is no request to match against when the ban lurker walks the cache.
-
-A simple mode to avoid req.* in bans is to add headers to the cached object containing the parts of the request on which you want to ban, e.g.::
-
-  sub vcl_fetch {
-      set obj.http.x-url = req.url;
-  }
-
-  sub vcl_deliver {
-      unset resp.http.x-url; # Optional
-  }
-
-  sub vcl_recv {
-      if (req.request == "PURGE") {
-            if (client.ip !~ purgers) {
-                error 401 "Not allowed";
-             }
-            purge("obj.http.x-url ~ " req.url); # Assumes req.url is a regex. This might be a bit too simple
-      }
-  }
-
-req.hash_always_miss
-====================
-
-The final way to invalidate an object is a method that allows you to refresh an object by forcing a hash miss for a single request. If you set `req.hash_always_miss` to true, varnish will miss the current object in the cache, thus forcing a fetch from the backend. This can in turn add the freshly fetched object to the cache, thus overriding the current one. The old object will stay in the cache until ttl expires or it is evicted by some other means.
diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst
index ce1d945..e5da595 100644
--- a/doc/sphinx/tutorial/purging.rst
+++ b/doc/sphinx/tutorial/purging.rst
@@ -10,8 +10,8 @@ of, in this twitterific day of age serving content that is outdated is
 bad for business.
 
 The solution is to notify Varnish when there is fresh content
-available. This can be done through two mechanisms. HTTP purging and
-bans. First, let me explain the HTTP purges. 
+available. This can be done through three mechanisms. HTTP purging,
+banning and forced cache misses. First, let me explain the HTTP purges. 
 
 
 HTTP Purges
@@ -61,7 +61,9 @@ As you can see we have used to new VCL subroutines, vcl_hit and
 vcl_miss. When we call lookup Varnish will try to lookup the object in
 its cache. It will either hit an object or miss it and so the
 corresponding subroutine is called. In vcl_hit the object that is
-stored in cache is available and we can set the TTL.
+stored in cache is available and we can set the TTL. The purge in
+vcl_miss is necessary to purge all variants in the cases where you hit an
+object, but miss a particular variant.
 
 So for example.com to invalidate their front page they would call out
 to Varnish like this::
@@ -75,14 +77,16 @@ variants as defined by Vary.
 Bans
 ====
 
-There is another way to invalidate content. Bans. You can think of
-bans as a sort of a filter. You *ban* certain content from being
-served from your cache. You can ban content based on any metadata we
-have.
+There is another way to invalidate content: Bans. You can think of
+bans as a sort of a filter on objects already in the cache. You *ban*
+certain content from being served from your cache. You can ban
+content based on any metadata we have.
+A ban will only work on objects already in the cache, it does not
+prevent new content from entering the cache or being served.
 
 Support for bans is built into Varnish and available in the CLI
-interface. For VG to ban every png object belonging on example.com
-they could issue::
+interface. To ban every png object belonging on example.com, issue
+the following command::
 
   ban req.http.host == "example.com" && req.http.url ~ "\.png$"
 
@@ -91,13 +95,14 @@ Quite powerful, really.
 Bans are checked when we hit an object in the cache, but before we
 deliver it. *An object is only checked against newer bans*. 
 
-Bans that only match against beresp.* are also processed by a
-background worker threads called the *ban lurker*. The ban lurker will
-walk the heap and try to match objects and will evict the matching
-objects. How aggressive the ban lurker is can be controlled by the
-parameter ban_lurker_sleep. 
+Bans that only match against obj.* are also processed by a background
+worker threads called the *ban lurker*. The ban lurker will walk the
+heap and try to match objects and will evict the matching objects. How
+aggressive the ban lurker is can be controlled by the parameter
+ban_lurker_sleep. The ban lurker can be disabled by setting
+ban_lurker_sleep to 0.
 
-Bans that are older then the oldest objects in the cache are discarded
+Bans that are older than the oldest objects in the cache are discarded
 without evaluation.  If you have a lot of objects with long TTL, that
 are seldom accessed you might accumulate a lot of bans. This might
 impact CPU usage and thereby performance.
@@ -122,3 +127,49 @@ You can also add bans to Varnish via HTTP. Doing so requires a bit of VCL::
 This VCL sniplet enables Varnish to handle an HTTP BAN method, adding a
 ban on the URL, including the host part.
 
+The ban lurker can help you keep the ban list at a manageable size, so
+we recommend that you avoid using req.* in your bans, as the request
+object is not available in the ban lurker thread.
+
+You can use the following template to write ban lurker friendly bans::
+
+  sub vcl_fetch {
+    set obj.http.x-url = req.url;
+  }
+
+  sub vcl_deliver {
+    unset resp.http.x-url; # Optional
+  }
+
+  sub vcl_recv {
+    if (req.request == "PURGE") {
+      if (client.ip !~ purge) {
+        error 401 "Not allowed";
+      }
+      ban("obj.http.x-url ~ " req.url); # Assumes req.url is a regex. This might be a bit too simple
+    }
+  }
+
+To inspect the current ban list, issue the ban.list command in CLI. This
+will produce a status of all current bans::
+
+  0xb75096d0 1318329475.377475    10      obj.http.x-url ~ test
+  0xb7509610 1318329470.785875    20G     obj.http.x-url ~ test
+
+The ban list contains the ID of the ban, the timestamp when the ban
+entered the ban list. A count of the objects that has reached this point
+in the ban list, optionally postfixed with a 'G' for "Gone", if the ban
+is no longer valid.  Finally, the ban expression is listed. The ban can
+be marked as Gone if it is a duplicate ban, but is still kept in the list
+for optimization purposes.
+
+Forcing a cache miss
+====================
+
+The final way to invalidate an object is a method that allows you to
+refresh an object by forcing a hash miss for a single request. If you set
+req.hash_always_miss to true, varnish will miss the current object in the
+cache, thus forcing a fetch from the backend. This can in turn add the
+freshly fetched object to the cache, thus overriding the current one. The
+old object will stay in the cache until ttl expires or it is evicted by
+some other means.



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] 834dff6 Update to 3.0 syntax
Message-ID: <E1SJhAo-0004wU-4M@project.varnish-software.com>

commit 834dff692cbed1f8ef66d7976a2f6570eb8ee90e
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Oct 24 22:38:48 2011 +0200

    Update to 3.0 syntax

diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst
index e5da595..cea1ccd 100644
--- a/doc/sphinx/tutorial/purging.rst
+++ b/doc/sphinx/tutorial/purging.rst
@@ -134,7 +134,7 @@ object is not available in the ban lurker thread.
 You can use the following template to write ban lurker friendly bans::
 
   sub vcl_fetch {
-    set obj.http.x-url = req.url;
+    set beresp.http.x-url = req.url;
   }
 
   sub vcl_deliver {



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] 8721bc4 Also snapshot the worker thread workspace around
	esi:include processing.
Message-ID: <E1SJhAo-0004wf-5y@project.varnish-software.com>

commit 8721bc447d38d4f4e4faaea0dd8cf95e1cdccc92
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 25 07:52:09 2011 +0000

    Also snapshot the worker thread workspace around esi:include
    processing.
    
    Convert a few http_PrintfHeader() to http_SetHeader() for good
    measure: There is no reason to waste workspace on compiled in strings.
    
    Fixes	#1038
    
    Conflicts:
    
    	bin/varnishd/cache_center.c
    	bin/varnishd/cache_response.c

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 7f9198f..6cbd532 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -468,7 +468,7 @@ cnt_error(struct sess *sp)
 	http_PutStatus(h, sp->err_code);
 	TIM_format(TIM_real(), date);
 	http_PrintfHeader(w, sp->fd, h, "Date: %s", date);
-	http_PrintfHeader(w, sp->fd, h, "Server: Varnish");
+	http_SetHeader(w, sp->fd, h, "Server: Varnish");
 
 	if (sp->err_reason != NULL)
 		http_PutResponse(w, sp->fd, h, sp->err_reason);
@@ -728,8 +728,8 @@ cnt_fetchbody(struct sess *sp)
 
 	/* If we do gzip, add the C-E header */
 	if (sp->wrk->do_gzip)
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->beresp,
-		    "Content-Encoding: %s", "gzip");
+		http_SetHeader(sp->wrk, sp->fd, sp->wrk->beresp,
+		    "Content-Encoding: gzip");
 
 	/* But we can't do both at the same time */
 	assert(sp->wrk->do_gzip == 0 || sp->wrk->do_gunzip == 0);
@@ -1194,7 +1194,7 @@ cnt_miss(struct sess *sp)
 		 * the minority of clients which don't.
 		 */
 		http_Unset(sp->wrk->bereq, H_Accept_Encoding);
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->bereq,
+		http_SetHeader(sp->wrk, sp->fd, sp->wrk->bereq,
 		    "Accept-Encoding: gzip");
 	}
 	sp->wrk->connect_timeout = 0;
@@ -1398,7 +1398,7 @@ cnt_recv(struct sess *sp)
 	     (recv_handling != VCL_RET_PASS)) {
 		if (RFC2616_Req_Gzip(sp)) {
 			http_Unset(sp->http, H_Accept_Encoding);
-			http_PrintfHeader(sp->wrk, sp->fd, sp->http,
+			http_SetHeader(sp->wrk, sp->fd, sp->http,
 			    "Accept-Encoding: gzip");
 		} else {
 			http_Unset(sp->http, H_Accept_Encoding);
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 61555bf..bbf51fd 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -47,7 +47,8 @@ ved_include(struct sess *sp, const char *src, const char *host)
 {
 	struct object *obj;
 	struct worker *w;
-	char *ws_wm;
+	char *sp_ws_wm;
+	char *wrk_ws_wm;
 	unsigned sxid, res_mode;
 
 	w = sp->wrk;
@@ -66,7 +67,8 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	HTTP_Copy(sp->http, sp->http0);
 
 	/* Take a workspace snapshot */
-	ws_wm = WS_Snapshot(sp->ws);
+	sp_ws_wm = WS_Snapshot(sp->ws);
+	wrk_ws_wm = WS_Snapshot(w->ws);
 
 	http_SetH(sp->http, HTTP_HDR_URL, src);
 	if (host != NULL && *host != '\0')  {
@@ -116,7 +118,8 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	sp->wrk->res_mode = res_mode;
 
 	/* Reset the workspace */
-	WS_Reset(sp->ws, ws_wm);
+	WS_Reset(sp->ws, sp_ws_wm);
+	WS_Reset(w->ws, wrk_ws_wm);
 
 	WRW_Reserve(sp->wrk, &sp->fd);
 	if (sp->wrk->res_mode & RES_CHUNKED)
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 73338f7..b27c8af 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -130,7 +130,7 @@ RES_BuildHttp(const struct sess *sp)
 	}
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
+		http_SetHeader(sp->wrk, sp->fd, sp->wrk->resp,
 		    "Transfer-Encoding: chunked");
 
 	TIM_format(TIM_real(), time_str);
diff --git a/bin/varnishtest/tests/r01038.vtc b/bin/varnishtest/tests/r01038.vtc
new file mode 100644
index 0000000..a4173b4
--- /dev/null
+++ b/bin/varnishtest/tests/r01038.vtc
@@ -0,0 +1,62 @@
+varnishtest "ticket 1038 regression test"
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Transfer-encoding: chunked"
+	chunked {<HTML>}
+
+
+	chunked {<esi:include src="xxx0.htm"/>}
+	chunked {<esi:include src="xxx1.htm"/>}
+	chunked {<esi:include src="xxx2.htm"/>}
+	chunked {<esi:include src="xxx3.htm"/>}
+	chunked {<esi:include src="xxx4.htm"/>}
+	chunked {<esi:include src="xxx5.htm"/>}
+	chunked {<esi:include src="xxx6.htm"/>}
+	chunked {<esi:include src="xxx7.htm"/>}
+	chunked {<esi:include src="xxx8.htm"/>}
+	chunkedlen 0
+	rxreq
+	expect req.url == "/xxx0.htm"
+	txresp -body "foo0"
+	rxreq
+	expect req.url == "/xxx1.htm"
+	txresp -body "foo1"
+	rxreq
+	expect req.url == "/xxx2.htm"
+	txresp -body "foo2"
+	rxreq
+	expect req.url == "/xxx3.htm"
+	txresp -body "foo3"
+	rxreq
+	expect req.url == "/xxx4.htm"
+	txresp -body "foo4"
+	rxreq
+	expect req.url == "/xxx5.htm"
+	txresp -body "foo5"
+	rxreq
+	expect req.url == "/xxx6.htm"
+	txresp -body "foo6"
+	rxreq
+	expect req.url == "/xxx7.htm"
+	txresp -body "foo7"
+	rxreq
+	expect req.url == "/xxx8.htm"
+	txresp -body "foo8"
+} -start
+
+varnish v1 -arg "-p thread_pool_workspace=1024" -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_esi = true;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	txreq
+	rxresp
+} -run
+
+varnish v1 -expect losthdr == 0
+



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] d1bf3bb Ignore SIGPIPE,
	it causes the test-executing subprocess to bail out before all
	diagnostics have been gathered.
Message-ID: <E1SJhAo-0004wu-7q@project.varnish-software.com>

commit d1bf3bbace16c95064f8197e1cbcbc3173533c88
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 25 08:15:35 2011 +0000

    Ignore SIGPIPE, it causes the test-executing subprocess to bail out
    before all diagnostics have been gathered.
    
    Conflicts:
    
    	bin/varnishtest/vtc.c

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 3bee42a..53cbdcd 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -530,6 +530,8 @@ exec_file(const char *fn, const char *script, const char *tmpdir,
 	FILE *f;
 	struct extmacro *m;
 
+	signal(SIGPIPE, SIG_IGN);
+
 	vtc_loginit(logbuf, loglen);
 	vltop = vtc_logopen("top");
 	AN(vltop);



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] f124895 Register buffer allocation failuers on vgz's and make
	failure to clean those up non-fatal.
Message-ID: <E1SJhAo-0004xL-BQ@project.varnish-software.com>

commit f124895fc3e704eb39a16bfa1811c88a0ba631a7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 25 09:44:39 2011 +0000

    Register buffer allocation failuers on vgz's and make failure
    to clean those up non-fatal.
    
    Fixes	#1036
    
    Conflicts:
    
    	bin/varnishd/cache_gzip.c

diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index b32b5cf..1eaa7fa 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -82,6 +82,7 @@ struct vgz {
 	const char		*id;
 	struct ws		*tmp;
 	char			*tmp_snapshot;
+	const char		*error;
 
 	struct storage		*obuf;
 
@@ -263,8 +264,10 @@ VGZ_ObufStorage(const struct sess *sp, struct vgz *vg)
 	struct storage *st;
 
 	st = FetchStorage(sp, 0);
-	if (st == NULL)
+	if (st == NULL) {
+		vg->error = "Could not get ObufStorage";
 		return (-1);
+	}
 
 	vg->obuf = st;
 	VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
@@ -409,6 +412,7 @@ void
 VGZ_Destroy(struct vgz **vgp)
 {
 	struct vgz *vg;
+	const char *err;
 
 	vg = *vgp;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
@@ -421,12 +425,13 @@ VGZ_Destroy(struct vgz **vgp)
 	    (intmax_t)vg->vz.start_bit,
 	    (intmax_t)vg->vz.last_bit,
 	    (intmax_t)vg->vz.stop_bit);
+	err = vg->error;
 	if (vg->tmp != NULL)
 		WS_Reset(vg->tmp, vg->tmp_snapshot);
 	if (vg->dir == VGZ_GZ)
-		AZ(deflateEnd(&vg->vz));
+		assert(deflateEnd(&vg->vz) == 0 || err != NULL);
 	else
-		AZ(inflateEnd(&vg->vz));
+		assert(inflateEnd(&vg->vz) == 0 || err != NULL);
 	FREE_OBJ(vg);
 }
 
@@ -566,18 +571,21 @@ vfp_gzip_end(struct sess *sp)
 	int i;
 
 	vg = sp->wrk->vgz_rx;
-	sp->wrk->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	do {
-		VGZ_Ibuf(vg, "", 0);
-		if (VGZ_ObufStorage(sp, vg))
-			return (-1);
-		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
-		sp->obj->len += dl;
-	} while (i != Z_STREAM_END);
-	if (sp->wrk->do_stream)
-		RES_StreamPoll(sp);
-	VGZ_UpdateObj(vg, sp->obj);
+	sp->wrk->vgz_rx = NULL;
+
+	if (vg->error == NULL) {
+		do {
+			VGZ_Ibuf(vg, "", 0);
+			if (VGZ_ObufStorage(sp, vg))
+				return (-1);
+			i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
+			sp->obj->len += dl;
+		} while (i != Z_STREAM_END);
+		if (sp->wrk->do_stream)
+			RES_StreamPoll(sp);
+		VGZ_UpdateObj(vg, sp->obj);
+	}
 	VGZ_Destroy(&vg);
 	return (0);
 }
@@ -621,6 +629,7 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		st = FetchStorage(sp, 0);
 		if (st == NULL) {
 			htc->error = "Could not get storage";
+			vg->error = htc->error;
 			return (-1);
 		}
 		l = st->space - st->len;
diff --git a/bin/varnishtest/tests/r01036.vtc b/bin/varnishtest/tests/r01036.vtc
new file mode 100644
index 0000000..8fb7600
--- /dev/null
+++ b/bin/varnishtest/tests/r01036.vtc
@@ -0,0 +1,22 @@
+varnishtest "Test case for #1036"
+
+server s1 {
+	rxreq
+	# Send a bodylen of 1,5M, which will exceed cache space of 1M
+	non-fatal
+	txresp -bodylen 1572864
+} -start
+
+varnish v1 -arg "-smalloc,1M" -arg "-pgzip_level=0" -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_gzip = true;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == "503"
+} -run
+
+



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] e5e7c23 Make it possible to mark http stuff "non-fatal".
Message-ID: <E1SJhAo-0004x7-9s@project.varnish-software.com>

commit e5e7c23026e50d36e9041bc7b0ca01a0b47491b5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 25 09:33:57 2011 +0000

    Make it possible to mark http stuff "non-fatal".
    
    The thread (s%d/c%d) still dies, but it does not mark this as
    fatal with respect to the test as such.
    
    This is useful where varnishd shuts the connection on a backend
    prematurely, as part of the correct execution of a test.
    
    Fix a undue recursion problem with asserts in the vtc_log code.

diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 5e5cce7..16861d3 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -71,10 +71,10 @@ void cmd_server_genvcl(struct vsb *vsb);
 void vtc_loginit(char *buf, unsigned buflen);
 struct vtclog *vtc_logopen(const char *id);
 void vtc_logclose(struct vtclog *vl);
-void vtc_log(struct vtclog *vl, unsigned lvl, const char *fmt, ...);
-void vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx,
+void vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...);
+void vtc_dump(struct vtclog *vl, int lvl, const char *pfx,
     const char *str, int len);
-void vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx,
+void vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx,
     const unsigned char *str, int len);
 
 int exec_file(const char *fn, const char *script, const char *tmpdir,
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index fddcd26..c71d9d2 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -73,6 +73,8 @@ struct http {
 
 	int			gziplevel;
 	int			gzipresidual;
+
+	int			fatal;
 };
 
 #define ONLY_CLIENT(hp, av)						\
@@ -143,7 +145,7 @@ http_write(const struct http *hp, int lvl, const char *pfx)
 	vtc_dump(hp->vl, lvl, pfx, VSB_data(hp->vsb), VSB_len(hp->vsb));
 	l = write(hp->fd, VSB_data(hp->vsb), VSB_len(hp->vsb));
 	if (l != VSB_len(hp->vsb))
-		vtc_log(hp->vl, 0, "Write failed: (%d vs %d) %s",
+		vtc_log(hp->vl, hp->fatal, "Write failed: (%d vs %d) %s",
 		    l, VSB_len(hp->vsb), strerror(errno));
 }
 
@@ -345,10 +347,12 @@ http_rxchar(struct http *hp, int n, int eof)
 		pfd[0].revents = 0;
 		i = poll(pfd, 1, hp->timeout);
 		if (i == 0)
-			vtc_log(hp->vl, 0, "HTTP rx timeout (fd:%d %u ms)",
+			vtc_log(hp->vl, hp->fatal,
+			    "HTTP rx timeout (fd:%d %u ms)",
 			    hp->fd, hp->timeout);
 		if (i < 0)
-			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d poll: %s)",
+			vtc_log(hp->vl, hp->fatal,
+			    "HTTP rx failed (fd:%d poll: %s)",
 			    hp->fd, strerror(errno));
 		assert(i > 0);
 		assert(hp->prxbuf + n < hp->nrxbuf);
@@ -360,10 +364,12 @@ http_rxchar(struct http *hp, int n, int eof)
 		if (i == 0 && eof)
 			return (i);
 		if (i == 0)
-			vtc_log(hp->vl, 0, "HTTP rx EOF (fd:%d read: %s)",
+			vtc_log(hp->vl, hp->fatal,
+			    "HTTP rx EOF (fd:%d read: %s)",
 			    hp->fd, strerror(errno));
 		if (i < 0)
-			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d read: %s)",
+			vtc_log(hp->vl, hp->fatal,
+			    "HTTP rx failed (fd:%d read: %s)",
 			    hp->fd, strerror(errno));
 		hp->prxbuf += i;
 		hp->rxbuf[hp->prxbuf] = '\0';
@@ -387,7 +393,7 @@ http_rxchunk(struct http *hp)
 	bprintf(hp->chunklen, "%d", i);
 	if ((q == hp->rxbuf + l) ||
 		(*q != '\0' && !vct_islws(*q))) {
-		vtc_log(hp->vl, 0, "chunked fail %02x @ %d",
+		vtc_log(hp->vl, hp->fatal, "chunked fail %02x @ %d",
 		    *q, q - (hp->rxbuf + l));
 	}
 	assert(q != hp->rxbuf + l);
@@ -401,11 +407,11 @@ http_rxchunk(struct http *hp)
 	l = hp->prxbuf;
 	(void)http_rxchar(hp, 2, 0);
 	if(!vct_iscrlf(hp->rxbuf[l]))
-		vtc_log(hp->vl, 0,
+		vtc_log(hp->vl, hp->fatal,
 		    "Wrong chunk tail[0] = %02x",
 		    hp->rxbuf[l] & 0xff);
 	if(!vct_iscrlf(hp->rxbuf[l + 1]))
-		vtc_log(hp->vl, 0,
+		vtc_log(hp->vl, hp->fatal,
 		    "Wrong chunk tail[1] = %02x",
 		    hp->rxbuf[l + 1] & 0xff);
 	hp->prxbuf = l;
@@ -547,7 +553,8 @@ cmd_http_gunzip_body(CMD_ARGS)
 	memset(&vz, 0, sizeof vz);
 
 	if (hp->body[0] != (char)0x1f || hp->body[1] != (char)0x8b)
-		vtc_log(hp->vl, 0, "Gunzip error: Body lacks gzip magics");
+		vtc_log(hp->vl, hp->fatal,
+		    "Gunzip error: Body lacks gzip magics");
 	vz.next_in = TRUST_ME(hp->body);
 	vz.avail_in = hp->bodyl;
 
@@ -576,7 +583,8 @@ cmd_http_gunzip_body(CMD_ARGS)
 	    (uintmax_t)vz.stop_bit,
 	    (uintmax_t)vz.stop_bit >> 3, (uintmax_t)vz.stop_bit & 7);
 	if (i != Z_STREAM_END)
-		vtc_log(hp->vl, 0, "Gunzip error = %d (%s) in:%jd out:%jd",
+		vtc_log(hp->vl, hp->fatal,
+		    "Gunzip error = %d (%s) in:%jd out:%jd",
 		    i, vz.msg, (intmax_t)vz.total_in, (intmax_t)vz.total_out);
 	assert(Z_OK == inflateEnd(&vz));
 }
@@ -608,7 +616,8 @@ gzip_body(const struct http *hp, const char *txt, char **body, int *bodylen)
 	assert(Z_STREAM_END == deflate(&vz, Z_FINISH));
 	i = vz.stop_bit & 7;
 	if (hp->gzipresidual >= 0 && hp->gzipresidual != i)
-		vtc_log(hp->vl, 0, "Wrong gzip residual got %d wanted %d",
+		vtc_log(hp->vl, hp->fatal,
+		    "Wrong gzip residual got %d wanted %d",
 		    i, hp->gzipresidual);
 	*bodylen = vz.total_out;
 	vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
@@ -900,7 +909,7 @@ cmd_http_send(CMD_ARGS)
 	vtc_dump(hp->vl, 4, "send", av[1], -1);
 	i = write(hp->fd, av[1], strlen(av[1]));
 	if (i != strlen(av[1]))
-		vtc_log(hp->vl, 0, "Write error in http_send(): %s",
+		vtc_log(hp->vl, hp->fatal, "Write error in http_send(): %s",
 		    strerror(errno));
 }
 
@@ -1042,9 +1051,9 @@ cmd_http_expect_close(CMD_ARGS)
 		fds[0].revents = 0;
 		i = poll(fds, 1, 1000);
 		if (i == 0)
-			vtc_log(vl, 0, "Expected close: timeout");
+			vtc_log(vl, hp->fatal, "Expected close: timeout");
 		if (i != 1 || !(fds[0].revents & POLLIN))
-			vtc_log(vl, 0,
+			vtc_log(vl, hp->fatal,
 			    "Expected close: poll = %d, revents = 0x%x",
 			    i, fds[0].revents);
 		i = read(hp->fd, &c, 1);
@@ -1052,7 +1061,7 @@ cmd_http_expect_close(CMD_ARGS)
 			break;
 		if (i == 1 && vct_islws(c))
 			continue;
-		vtc_log(vl, 0,
+		vtc_log(vl, hp->fatal,
 		    "Expecting close: read = %d, c = 0x%02x", i, c);
 	}
 	vtc_log(vl, 4, "fd=%d EOF, as expected", hp->fd);
@@ -1097,7 +1106,7 @@ cmd_http_accept(CMD_ARGS)
 	vtc_log(vl, 4, "Accepting");
 	hp->fd = accept(*hp->sfd, NULL, NULL);
 	if (hp->fd < 0)
-		vtc_log(vl, 0, "Accepted failed: %s", strerror(errno));
+		vtc_log(vl, hp->fatal, "Accepted failed: %s", strerror(errno));
 	vtc_log(vl, 3, "Accepted socket fd is %d", hp->fd);
 }
 
@@ -1126,6 +1135,26 @@ cmd_http_loop(CMD_ARGS)
 }
 
 /**********************************************************************
+ * Control fatality
+ */
+
+static void
+cmd_http_fatal(CMD_ARGS)
+{
+	struct http *hp;
+	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
+
+	AZ(av[1]);
+	if (!strcmp(av[0], "fatal"))
+		hp->fatal = 0;
+	else if (!strcmp(av[0], "non-fatal"))
+		hp->fatal = -1;
+	else {
+		vtc_log(vl, 0, "XXX: fatal %s", cmd->name);
+	}
+}
+
+/**********************************************************************
  * Execute HTTP specifications
  */
 
@@ -1152,6 +1181,8 @@ static const struct cmds http_cmds[] = {
 	{ "close",		cmd_http_close },
 	{ "accept",		cmd_http_accept },
 	{ "loop",		cmd_http_loop },
+	{ "fatal",		cmd_http_fatal },
+	{ "non-fatal",		cmd_http_fatal },
 	{ NULL,			NULL }
 };
 
diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 18f295a..f63a9d8 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -52,6 +52,7 @@ struct vtclog {
 	const char	*id;
 	struct vsb	*vsb;
 	pthread_mutex_t	mtx;
+	int		act;
 };
 
 static pthread_key_t log_key;
@@ -108,10 +109,12 @@ static const char * const lead[] = {
 #define NLEAD (sizeof(lead)/sizeof(lead[0]))
 
 static void
-vtc_log_emit(const struct vtclog *vl, unsigned lvl)
+vtc_log_emit(const struct vtclog *vl, int lvl)
 {
 	int l;
 
+	if (lvl < 0)
+		lvl = 0;
 	if (vtc_stop && lvl == 0)
 		return;
 	l = VSB_len(vl->vsb);
@@ -126,16 +129,18 @@ vtc_log_emit(const struct vtclog *vl, unsigned lvl)
 
 //lint -e{818}
 void
-vtc_log(struct vtclog *vl, unsigned lvl, const char *fmt, ...)
+vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...)
 {
 	double tx;
 
 	CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC);
 	tx = TIM_mono() - t0;
 	AZ(pthread_mutex_lock(&vl->mtx));
+	vl->act = 1;
 	assert(lvl < NLEAD);
 	VSB_clear(vl->vsb);
-	VSB_printf(vl->vsb, "%s %-4s %4.1f ", lead[lvl], vl->id, tx);
+	VSB_printf(vl->vsb, "%s %-4s %4.1f ",
+	    lead[lvl < 0 ? 1: lvl], vl->id, tx);
 	va_list ap;
 	va_start(ap, fmt);
 	(void)VSB_vprintf(vl->vsb, fmt, ap);
@@ -146,12 +151,14 @@ vtc_log(struct vtclog *vl, unsigned lvl, const char *fmt, ...)
 	vtc_log_emit(vl, lvl);
 
 	VSB_clear(vl->vsb);
+	vl->act = 0;
 	AZ(pthread_mutex_unlock(&vl->mtx));
-	if (lvl == 0) {
+	if (lvl > 0)
+		return;
+	if (lvl == 0) 
 		vtc_error = 1;
-		if (pthread_self() != vtc_thread)
-			pthread_exit(NULL);
-	}
+	if (pthread_self() != vtc_thread)
+		pthread_exit(NULL);
 }
 
 /**********************************************************************
@@ -160,7 +167,7 @@ vtc_log(struct vtclog *vl, unsigned lvl, const char *fmt, ...)
 
 //lint -e{818}
 void
-vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx, const char *str, int len)
+vtc_dump(struct vtclog *vl, int lvl, const char *pfx, const char *str, int len)
 {
 	int nl = 1;
 	unsigned l;
@@ -169,7 +176,9 @@ vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx, const char *str, int
 	CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC);
 	tx = TIM_mono() - t0;
 	assert(lvl < NLEAD);
+	assert(lvl >= 0);
 	AZ(pthread_mutex_lock(&vl->mtx));
+	vl->act = 1;
 	VSB_clear(vl->vsb);
 	if (pfx == NULL)
 		pfx = "";
@@ -209,6 +218,7 @@ vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx, const char *str, int
 	vtc_log_emit(vl, lvl);
 
 	VSB_clear(vl->vsb);
+	vl->act = 0;
 	AZ(pthread_mutex_unlock(&vl->mtx));
 	if (lvl == 0) {
 		vtc_error = 1;
@@ -223,7 +233,7 @@ vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx, const char *str, int
 
 //lint -e{818}
 void
-vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned char *str, int len)
+vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx, const unsigned char *str, int len)
 {
 	int nl = 1;
 	unsigned l;
@@ -233,7 +243,9 @@ vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned cha
 	tx = TIM_mono() - t0;
 	assert(len >= 0);
 	assert(lvl < NLEAD);
+	assert(lvl >= 0);
 	AZ(pthread_mutex_lock(&vl->mtx));
+	vl->act = 1;
 	VSB_clear(vl->vsb);
 	if (pfx == NULL)
 		pfx = "";
@@ -265,6 +277,7 @@ vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned cha
 	vtc_log_emit(vl, lvl);
 
 	VSB_clear(vl->vsb);
+	vl->act = 0;
 	AZ(pthread_mutex_unlock(&vl->mtx));
 	if (lvl == 0) {
 		vtc_error = 1;
@@ -284,7 +297,7 @@ vtc_log_VAS_Fail(const char *func, const char *file, int line,
 	(void)err;
 	(void)xxx;
 	vl = pthread_getspecific(log_key);
-	if (vl == NULL) {
+	if (vl == NULL || vl->act) {
 		fprintf(stderr,
 		    "Assert error in %s(), %s line %d:\n"
 		    "  Condition(%s) not true.\n",



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] cbad9e1 Use a better way of checking we have pkg-config
Message-ID: <E1SJhAo-0004xS-Cb@project.varnish-software.com>

commit cbad9e18b433c9bc009d62ac3e66005076a8dbea
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Oct 18 20:29:53 2011 +0200

    Use a better way of checking we have pkg-config

diff --git a/configure.ac b/configure.ac
index 1f58e00..1938c1f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -116,7 +116,7 @@ AC_SUBST(NET_LIBS)
 AC_CHECK_LIBM
 AC_SUBST(LIBM)
 
-m4_pattern_forbid([^_?PKG_[A-Z_]+$],[pkg.m4 missing, please install pkg-config])
+m4_ifndef([PKG_PROG_PKG_CONFIG], [m4_fatal([pkg.m4 missing, please install pkg-config])])
 PKG_PROG_PKG_CONFIG
 if test -n $PKG_CONFIG; then
    PKG_CHECK_MODULES([PCRE], [libpcre])



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] 21335dc Don't abbreviate panic message output.
Message-ID: <E1SJhAo-0004xd-EQ@project.varnish-software.com>

commit 21335dcd3390af13634da92f78dcd4f9ccc48083
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 31 08:09:01 2011 +0000

    Don't abbreviate panic message output.

diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index f63a9d8..a126b48 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -169,7 +169,7 @@ vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...)
 void
 vtc_dump(struct vtclog *vl, int lvl, const char *pfx, const char *str, int len)
 {
-	int nl = 1;
+	int nl = 1, olen;
 	unsigned l;
 	double tx;
 
@@ -186,10 +186,11 @@ vtc_dump(struct vtclog *vl, int lvl, const char *pfx, const char *str, int len)
 		VSB_printf(vl->vsb, "%s %-4s %4.1f %s(null)\n",
 		    lead[lvl], vl->id, tx, pfx);
 	else {
-		if (len == -1)
+		olen = len;
+		if (len < 0)
 			len = strlen(str);
 		for (l = 0; l < len; l++, str++) {
-			if (l > 512) {
+			if (l > 512 && olen != -2) {
 				VSB_printf(vl->vsb, "...");
 				break;
 			}
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 6871b5f..dec9754 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -319,7 +319,7 @@ varnish_thread(void *priv)
 		if (i <= 0)
 			break;
 		buf[i] = '\0';
-		vtc_dump(v->vl, 3, "debug", buf, -1);
+		vtc_dump(v->vl, 3, "debug", buf, -2);
 	}
 	return (NULL);
 }



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] d04a8fb Typographical corrections, spelling
Message-ID: <E1SJhAo-0004xm-G2@project.varnish-software.com>

commit d04a8fb1cf80d4bb488b6abb13145a785d67a620
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 31 09:59:45 2011 +0100

    Typographical corrections, spelling

diff --git a/doc/sphinx/phk/backends.rst b/doc/sphinx/phk/backends.rst
index 2b32353..9ab6123 100644
--- a/doc/sphinx/phk/backends.rst
+++ b/doc/sphinx/phk/backends.rst
@@ -9,10 +9,10 @@ question answered conclusively long time ago, but once you try to
 be efficient, things get hairy fast.
 
 One of the features of Varnish we are very fundamental about, is the
-ability to have multiple VCL's loaded at the same time, and to switch
+ability to have multiple VCLs loaded at the same time, and to switch
 between them instantly and seamlessly.
 
-So Imagine you have 1000 backends in your VCL, not an unreasonable
+So imagine you have 1000 backends in your VCL, not an unreasonable
 number, each configured with health-polling.
 
 Now you fiddle your vcl_recv{} a bit and load the VCL again, but
@@ -25,7 +25,7 @@ be up to date the instant we switch to the other VCL.
 This basically means that either all VCLs poll all their backends,
 or they must share, somehow.
 
-We can dismiss the all VCL's poll all their backends scenario,
+We can dismiss the all VCLs poll all their backends scenario,
 because it scales truly horribly, and would pummel backends with
 probes if people forget to vcl.discard their old dusty VCLs.
 
@@ -39,7 +39,7 @@ It would be truly stupid to close 100 ready and usable connections to
 a backend, and open 100 other, just because we switch to a different
 VCL that has an identical backend definition.
 
-But what is an identical backend definition in this context ?
+But what is an identical backend definition in this context?
 
 It is important to remember that we are not talking physical
 backends:  For instance, there is nothing preventing a VCL for
@@ -48,7 +48,7 @@ backends.
 
 The most obvious thing to do, is to use the VCL name of the backend
 as identifier, but that is not enough.  We can have two different
-VCL's where backend "b1" points at two different physical machines,
+VCLs where backend "b1" points at two different physical machines,
 for instance when we migrate or upgrade the backend.
 
 The identity of the state than can be shared is therefore the triplet:
@@ -78,11 +78,11 @@ is a wildcard-ish scheme, where you can write things like::
 
 	b1()				# All backends named b1
 
-	b1(127.0.0.1)			# All b1's on Ipv4 lookback
+	b1(127.0.0.1)			# All b1s on IPv4 lookback
 
-	b1(:8080)			# All b1's on port 8080, (IPv4 or IPv6)
+	b1(:8080)			# All b1s on port 8080, (IPv4 or IPv6)
 
-	b1(192.168.60.1,192.168.60.2)	# All b1's on one of those addresses.
+	b1(192.168.60.1,192.168.60.2)	# All b1s on one of those addresses.
 
 (Input very much welcome)
 
@@ -112,7 +112,7 @@ the backend gets retired anyway.
 We should either park a thread on each backend, or have a poller thread
 which throws jobs into the work-pool as the backends needs polled.
 
-The patternmatching is confined to CLI and possibly libvarnishapi
+The pattern matching is confined to CLI and possibly libvarnishapi
 
 I think this will work,
 



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] d5117f5 Add new format %{VCL_Log:foo}x which output key:value
	from std.log() in VCL
Message-ID: <E1SJhAo-0004xy-Hl@project.varnish-software.com>

commit d5117f53a650c49c2259ff3ed57479c80a22b59f
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Mon Oct 31 14:28:10 2011 +0100

    Add new format %{VCL_Log:foo}x which output key:value from std.log() in VCL

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index ad9249d..e0d6d32 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -105,6 +105,7 @@ static struct logline {
 	uint64_t bitmap;		/* Bitmap for regex matches */
 	VTAILQ_HEAD(, hdr) req_headers; /* Request headers */
 	VTAILQ_HEAD(, hdr) resp_headers; /* Response headers */
+	VTAILQ_HEAD(, hdr) vcl_log;     /* VLC_Log entries */
 } **ll;
 
 struct VSM_data *vd;
@@ -216,6 +217,19 @@ resp_header(struct logline *l, const char *name)
 	return NULL;
 }
 
+static char *
+vcl_log(struct logline *l, const char *name)
+{
+	struct hdr *h;
+	VTAILQ_FOREACH(h, &l->vcl_log, list) {
+		if (strcasecmp(h->key, name) == 0) {
+			return h->value;
+			break;
+		}
+	}
+	return NULL;
+}
+
 static void
 clean_logline(struct logline *lp)
 {
@@ -242,6 +256,12 @@ clean_logline(struct logline *lp)
 		freez(h->value);
 		freez(h);
 	}
+	VTAILQ_FOREACH_SAFE(h, &lp->vcl_log, list, h2) {
+		VTAILQ_REMOVE(&lp->vcl_log, h, list);
+		freez(h->key);
+		freez(h->value);
+		freez(h);
+	}
 #undef freez
 	memset(lp, 0, sizeof *lp);
 }
@@ -462,6 +482,25 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		}
 		break;
 
+	case SLT_VCL_Log:
+		if(!lp->active)
+			break;
+
+		split = strchr(ptr, ':');
+		if (split == NULL)
+			break;
+
+		struct hdr *h;
+		h = malloc(sizeof(struct hdr));
+		AN(h);
+		AN(split);
+
+		h->key = trimline(ptr, split);
+		h->value = trimline(split+1, end);
+
+		VTAILQ_INSERT_HEAD(&lp->vcl_log, h, list);
+		break;
+
 	case SLT_VCL_call:
 		if(!lp->active)
 			break;
@@ -715,6 +754,14 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 					VSB_cat(os, (lp->df_handling ? lp->df_handling : "-"));
 					p = tmp;
 					break;
+				} else if (strncmp(fname, "VCL_Log:", 8) == 0) {
+					// support pulling entries logged with std.log() into output.
+					// Format: %{VCL_Log:keyname}x
+					// Logging: std.log("keyname:value")
+					h = vcl_log(lp, fname+8);
+					VSB_cat(os, h ? h : "-");
+					p = tmp;
+					break;
 				}
 			default:
 				fprintf(stderr, "Unknown format starting at: %s\n", --p);
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index e654e37..ddb6538 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -116,6 +116,10 @@ The following options are available:
 		   Varnish:handling
 		     How the request was handled, whether it was a
 		     cache hit, miss, pass, pipe or error.
+	
+		   VCL_Log:key
+		     Output value set by std.log("key=value") in VCL.
+		     
 
 -m tag:regex only list records where tag matches regex. Multiple
             -m options are AND-ed together.



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] cad88f4 Support for \t\n in varnishncsa format strings
Message-ID: <E1SJhAo-0004yD-J5@project.varnish-software.com>

commit cad88f4cef11223f8ff39a9907f359487adbfcd6
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 31 14:35:28 2011 +0100

    Support for \t\n in varnishncsa format strings

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index e0d6d32..ae3e4cb 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -626,6 +626,14 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 
 	for (p = format; *p != '\0'; p++) {
 
+		/* allow the most essential escape sequences in format. */
+		if (*p == '\\') {
+			p++;
+			if (*p == 't') VSB_putc(os, '\t');
+			if (*p == 'n') VSB_putc(os, '\n');
+			continue;
+		}
+
 		if (*p != '%') {
 			VSB_putc(os, *p);
 			continue;
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index ddb6538..8499d29 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -54,6 +54,8 @@ The following options are available:
 
             %h %l %u %t "%r" %s %b "%{Referer}i" "%{User-agent}i"
 
+	    Escape sequences \\n and \\t are supported.
+
 	    Supported formatters are:
 
 	      %b 



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] ab7a91e Use scalbn(3) rather than exp2(3),
	it should be faster and more portable.
Message-ID: <E1SJhAp-00050I-51@project.varnish-software.com>

commit ab7a91edd8d8cd8eb040f2bde5ce4b336bada906
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 1 09:41:00 2011 +0000

    Use scalbn(3) rather than exp2(3), it should be faster and more portable.
    
    Fixes	#1046

diff --git a/bin/varnishd/cache_dir_random.c b/bin/varnishd/cache_dir_random.c
index c2b74dd..27f120f 100644
--- a/bin/varnishd/cache_dir_random.c
+++ b/bin/varnishd/cache_dir_random.c
@@ -97,7 +97,7 @@ vdi_random_sha(const char *input, ssize_t len)
 	SHA256_Init(&ctx);
 	SHA256_Update(&ctx, input, len);
 	SHA256_Final(sign, &ctx);
-	return (vle32dec(sign) / exp2(32));
+	return (scalbn(vle32dec(sign), -32));
 }
 
 /*
@@ -119,11 +119,11 @@ vdi_random_init_seed(const struct vdi_random *vs, const struct sess *sp)
 		break;
 	case c_hash:
 		AN(sp->digest);
-		retval = vle32dec(sp->digest) / exp2(32);
+		retval = scalbn(vle32dec(sp->digest), -32);
 		break;
 	case c_random:
 	default:
-		retval = random() / exp2(31);
+		retval = scalbn(random(), -31);
 		break;
 	}
 	return (retval);



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] ce09d2e Add a missing case: ESI parent document gunzip'ed but
	included document gzip'ed.
Message-ID: <E1SJhAp-00050S-6I@project.varnish-software.com>

commit ce09d2ed973b3771d82eac0384295ddf0f0921fc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 1 10:49:09 2011 +0000

    Add a missing case:  ESI parent document gunzip'ed but included
    document gzip'ed.
    
    Fixes	#1029

diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index afef1fb..3f3ab2b 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -801,7 +801,7 @@ VEP_parse(const struct sess *sp, const char *p, size_t l)
 				vep->state = VEP_ATTR;
 			} else if (p < e) {
 				vep_error(vep,
-				    "XML 1.0 Illegal attribute tart char");
+				    "XML 1.0 Illegal attribute start char");
 				vep->state = VEP_TAGERROR;
 			}
 		} else if (vep->state == VEP_TAGERROR) {
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 0f7e435..37459d7 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -312,6 +312,9 @@ RES_WriteObj(struct sess *sp)
 		ESI_Deliver(sp);
 	} else if (sp->wrk->res_mode & RES_ESI_CHILD && sp->wrk->gzip_resp) {
 		ESI_DeliverChild(sp);
+	} else if (sp->wrk->res_mode & RES_ESI_CHILD &&
+	    !sp->wrk->gzip_resp && sp->obj->gziped) {
+		res_WriteGunzipObj(sp);
 	} else if (sp->wrk->res_mode & RES_GUNZIP) {
 		res_WriteGunzipObj(sp);
 	} else {
diff --git a/bin/varnishtest/tests/r01029.vtc b/bin/varnishtest/tests/r01029.vtc
new file mode 100644
index 0000000..2817bba
--- /dev/null
+++ b/bin/varnishtest/tests/r01029.vtc
@@ -0,0 +1,34 @@
+varnishtest "#1029"
+
+server s1 {
+	rxreq
+	expect req.url == "/bar"
+	txresp -gzipbody {[bar]}
+
+	rxreq
+	expect req.url == "/foo"
+	txresp -body {<h1>FOO<esi:include src="/bar"/>BARF</h1>}
+
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_esi = true;
+		if (req.url == "/foo") {
+			set beresp.ttl = 0s;
+		} else {
+			set beresp.ttl = 10m;
+		}
+	}
+} -start
+
+client c1 {
+	txreq -url "/bar" -hdr "Accept-Encoding: gzip"
+	rxresp
+	gunzip
+	expect resp.bodylen == 5
+
+	txreq -url "/foo" -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.bodylen == 21
+} -run



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] 1808856 document the compression behaviour in Varnish 3.0
Message-ID: <E1SJhAp-00050c-8Q@project.varnish-software.com>

commit 1808856f8e2bfce452603caf89a38091e3f5effe
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Nov 3 10:22:13 2011 +0100

    document the compression behaviour in Varnish 3.0

diff --git a/doc/sphinx/tutorial/compression.rst b/doc/sphinx/tutorial/compression.rst
new file mode 100644
index 0000000..0b8d1e8
--- /dev/null
+++ b/doc/sphinx/tutorial/compression.rst
@@ -0,0 +1,75 @@
+.. _tutorial-compression:
+
+Compression
+~~~~~~~~~~~
+
+New in Varnish 3.0 was native support for compression, using gzip
+encoding. *Before* 3.0, Varnish would never compress objects. 
+
+In Varnish 3.0 compression defaults to "on", meaning that it tries to
+be smart and do the sensible thing.
+
+If you don't want Varnish tampering with the encoding you can disable
+compression all together by setting the parameter http_gzip_support to
+*false*. Please see man :ref:`ref-varnishd` for details.
+
+
+Default behaviour
+~~~~~~~~~~~~~~~~~
+
+The default for Varnish is to check if the client supports our
+compression scheme (gzip) and if it does it will override the
+Accept-Encoding header and set it to "gzip".
+
+When Varnish then issues a backend request the Accept-Encoding will
+then only consist of "gzip". If the server responds with gzip'ed
+content it will be stored in memory in its compressed form. If the
+backend sends content in clear text it will be stored like that.
+
+You can make Varnish compress content before storing it in cache in
+vcl_fetch by setting do_gzip to true, like this::
+
+   sub vcl_fetch {
+        if (beresp.http.content-type ~ "text") {
+                set beresp.do_gzip = true;
+        }
+  }
+
+Please make sure that you don't try to compress content that is
+incompressable, like jpgs, gifs and mp3. You'll only waste CPU
+cycles. You can also uncompress objects before storing it in memory by
+setting do_gunzip to *true* but I have no idea why anybody would want
+to do that.
+
+Generally, Varnish doesn't use much CPU so it might make more sense to
+have Varnish spend CPU cycles compressing content than doing it in
+your web- or application servers, which are more likely to be
+CPU-bound.
+
+GZIP and ESI
+~~~~~~~~~~~~
+
+If you are using Edge Side Includes you'll be happy to note that ESI
+and GZIP work together really well. Varnish will magically decompress
+the content to do the ESI-processing, then recompress it for efficient
+storage and delivery. 
+
+
+Clients that don't support gzip
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If the client does not support gzip the Accept-Encoding header is left
+alone and we'll end up serving whatever we get from the backend
+server. Remember that the Backend might tell Varnish to *Vary* on the
+Accept-Encoding.
+
+If the client does not support gzip but we've already got a compressed
+version of the page in memory Varnish will automatically decompress
+the page while delivering it.
+
+
+A random outburst
+~~~~~~~~~~~~~~~~~
+
+Poul has written :ref:`phk_gzip` which talks abit more about how the
+implementation works. 
diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
index 59fd94e..32c2264 100644
--- a/doc/sphinx/tutorial/index.rst
+++ b/doc/sphinx/tutorial/index.rst
@@ -25,6 +25,7 @@ separate topic. Good luck.
 	cookies
 	vary
 	purging
+	compression
 	esi
 	virtualised
 	advanced_backend_servers



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] 494c04a Add health control of backends from CLI
Message-ID: <E1SJhAp-00050q-9p@project.varnish-software.com>

commit 494c04ab8cb20164317c9d44af0b032ec2bee85b
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Nov 3 13:03:51 2011 +0100

    Add health control of backends from CLI
    
    Make it possible to mark backends as sick or healthy by way of
    
      backend.set_health $backendname (sick|healthy|auto)

diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index 3d6a405..b296761 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -259,7 +259,10 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp)
 	backend = vs->backend;
 	CHECK_OBJ_NOTNULL(backend, BACKEND_MAGIC);
 
-	if (!backend->healthy)
+	if (backend->admin_health == from_probe && !backend->healthy)
+		return (0);
+
+	if (backend->admin_health == sick)
 		return (0);
 
 	/* VRT/VCC sets threshold to UINT_MAX to mark that it's not
@@ -270,6 +273,9 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp)
 	else
 		threshold = vs->vrt->saintmode_threshold;
 
+	if (backend->admin_health == healthy)
+		threshold = UINT_MAX;
+
 	/* Saintmode is disabled */
 	if (threshold == 0)
 		return (1);
diff --git a/bin/varnishd/cache_backend.h b/bin/varnishd/cache_backend.h
index 21a7061..25148ac 100644
--- a/bin/varnishd/cache_backend.h
+++ b/bin/varnishd/cache_backend.h
@@ -106,6 +106,12 @@ struct trouble {
  * An instance of a backend from a VCL program.
  */
 
+enum health_status {
+	healthy,
+	sick,
+	from_probe
+};
+
 struct backend {
 	unsigned		magic;
 #define BACKEND_MAGIC		0x64c4c7c6
@@ -129,6 +135,7 @@ struct backend {
 
 	struct vbp_target	*probe;
 	unsigned		healthy;
+	enum health_status	admin_health;
 	VTAILQ_HEAD(, trouble)	troublelist;
 
 	struct VSC_C_vbe	*vsc;
diff --git a/bin/varnishd/cache_backend_cfg.c b/bin/varnishd/cache_backend_cfg.c
index 0582f72..215a4dc 100644
--- a/bin/varnishd/cache_backend_cfg.c
+++ b/bin/varnishd/cache_backend_cfg.c
@@ -41,6 +41,7 @@
 #include <sys/socket.h>
 
 #include "cache.h"
+#include "vcli.h"
 #include "vrt.h"
 #include "cache_backend.h"
 #include "cli_priv.h"
@@ -232,6 +233,7 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 	assert(b->ipv4 != NULL || b->ipv6 != NULL);
 
 	b->healthy = 1;
+	b->admin_health = from_probe;
 
 	VTAILQ_INSERT_TAIL(&backends, b, list);
 	VSC_C_main->n_backend++;
@@ -277,25 +279,166 @@ VRT_fini_dir(struct cli *cli, struct director *b)
 
 /*--------------------------------------------------------------------*/
 
+static int
+backend_find(const char *matcher, struct backend **r, int n)
+{
+	struct backend *b;
+	char *vcl_name;
+	char *s;
+	char *match_ip = NULL;
+	char *match_port = NULL;
+	int found = 0;
+
+	s = strchr(matcher, '(');
+
+	if (s == NULL) {
+		/* Simple match, max one hit */
+		VTAILQ_FOREACH(b, &backends, list) {
+			CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+			if (strcmp(b->vcl_name, matcher) == 0) {
+				if (r && found < n)
+					r[found] = b;
+				found++;
+			}
+		}
+		return found;
+	}
+
+	vcl_name = strndup(matcher, s - matcher);
+	AN(vcl_name);
+	s++;
+	while (*s != ')') {
+		if (*s == ':') {
+			/* Port */
+			s++;
+			match_port = s;
+			if (!(s = strchr(match_port, ','))) {
+				s = strchr(match_port, ')');
+			}
+			XXXAN(s);
+			match_port = strndup(match_port, s - match_port);
+			AN(match_port);
+			if (*s == ',')
+				s++;
+		} else {
+			/* IP */
+			match_ip = s;
+			if (!(s = strchr(match_ip, ','))) {
+				s = strchr(match_ip, ')');
+			}
+			XXXAN(s);
+			match_ip = strndup(match_ip, s - match_ip);
+			AN(match_ip);
+			if (*s == ',')
+				s++;
+		}
+	}
+	VTAILQ_FOREACH(b, &backends, list) {
+		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+		if (match_port && strcmp(b->port, match_port) != 0)
+			continue;
+		if (match_ip &&
+		    (strcmp(b->ipv4_addr, match_ip) != 0) &&
+		    (strcmp(b->ipv6_addr, match_ip) != 0))
+			continue;
+		if (strcmp(b->vcl_name, vcl_name) == 0) {
+			if (r && found < n)
+				r[found] = b;
+			found++;
+		}
+	}
+	return found;
+}
+
 static void
-cli_debug_backend(struct cli *cli, const char * const *av, void *priv)
+cli_backend_list(struct cli *cli, const char * const *av, void *priv)
 {
 	struct backend *b;
+	const char *ah;
 
 	(void)av;
 	(void)priv;
 	ASSERT_CLI();
+	VCLI_Out(cli, "%-30s %10s %15s %15s", "Backend name",
+		 "Conns", "Probed healthy", "Admin health");
 	VTAILQ_FOREACH(b, &backends, list) {
+		char buf[128];
 		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+
+		bprintf(buf, "%s(%s,%s,%s)",
+			b->vcl_name,
+			b->ipv4_addr == NULL ? "" : b->ipv4_addr,
+			b->ipv6_addr == NULL ? "" : b->ipv6_addr, b->port);
+		if (b->admin_health == from_probe) {
+			ah = "Auto";
+		} else if (b->admin_health == sick) {
+			ah = "Sick";
+		} else {
+			ah = "Healthy";
+		}
 		VCLI_Out(cli, "%p %s(%s,%s,:%s) %d %d\n",
 		    b, b->vcl_name, b->ipv4_addr, b->ipv6_addr, b->port,
 		    b->refcount, b->n_conn);
+		VCLI_Out(cli, "\n%-30s %10d %15s %15s",
+			 buf,
+			 b->refcount,
+			 (b ->healthy ? "Yes" : "No"),
+			 ah);
+	}
+}
+
+static void
+cli_backend_set_health(struct cli *cli, const char * const *av, void *priv)
+{
+	struct backend **b;
+	enum health_status state;
+	int n;
+	const char *wstate;
+
+	(void)av;
+	(void)priv;
+	ASSERT_CLI();
+	wstate = av[3];
+	if (strcmp(wstate, "healthy") == 0) {
+		state = healthy;
+	} else if (strcmp(wstate, "sick") == 0) {
+		state = sick;
+	} else if (strcmp(wstate, "auto") == 0) {
+		state = from_probe;
+	} else {
+		VCLI_Out(cli, "Invalid state %s", wstate);
+		VCLI_SetResult(cli, CLIS_CANT);
+		return;
+	}
+	n = backend_find(av[2], NULL, 0);
+	if (n == 0) {
+		VCLI_Out(cli, "No matching backends");
+		VCLI_SetResult(cli, CLIS_CANT);
+		return;
+	}
+
+	b = calloc(n, sizeof(struct backend *));
+	AN(b);
+	n = backend_find(av[2], b, n);
+
+	VCLI_Out(cli, "Set state to %s for the following backends:", wstate);
+	for (int i = 0; i < n; i++) {
+		char buf[128];
+		bprintf(buf, "%s(%s,%s,%s)",
+			b[i]->vcl_name,
+			b[i]->ipv4_addr == NULL ? "" : b[i]->ipv4_addr,
+			b[i]->ipv6_addr == NULL ? "" : b[i]->ipv6_addr, b[i]->port);
+
+		b[i]->admin_health = state;
+		VCLI_Out(cli, "\n\t%s", buf);
 	}
 }
 
-static struct cli_proto debug_cmds[] = {
-	{ "debug.backend", "debug.backend",
-	    "\tExamine Backend internals\n", 0, 0, "d", cli_debug_backend },
+static struct cli_proto backend_cmds[] = {
+	{ "backend.list", "backend.list",
+	    "\tList all backends\n", 0, 0, "d", cli_backend_list },
+	{ "backend.set_health", "backend.set_health matcher state",
+	    "\tShow a backend\n", 2, 2, "d", cli_backend_set_health },
 	{ NULL }
 };
 
@@ -306,5 +449,5 @@ VBE_Init(void)
 {
 
 	Lck_New(&VBE_mtx, lck_vbe);
-	CLI_AddFuncs(debug_cmds);
+	CLI_AddFuncs(backend_cmds);
 }
diff --git a/bin/varnishtest/tests/c00048.vtc b/bin/varnishtest/tests/c00048.vtc
new file mode 100644
index 0000000..ed8671f
--- /dev/null
+++ b/bin/varnishtest/tests/c00048.vtc
@@ -0,0 +1,55 @@
+varnishtest "Forcing health of backends"
+
+server s1 -repeat 3 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl {
+	backend s1 {
+		.host = "${s1_addr}";
+		.port = "${s1_port}";
+		.probe = {
+			.window = 8;
+			.initial = 7;
+			.threshold = 8;
+			.interval = 10s;
+		}
+	}
+
+	sub vcl_recv {
+		return(pass);
+	}
+
+} -start
+
+delay 1
+
+varnish v1 -cliok "backend.set_health s1 auto"
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
+
+varnish v1 -cliok "backend.set_health s1 sick"
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+} -run
+
+varnish v1 -cliok "backend.set_health s1 healthy"
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
+
+varnish v1 -clierr 300 "backend.set_health s1 foo"
+varnish v1 -clierr 300 "backend.set_health s2 foo"
+varnish v1 -clierr 300 "backend.set_health s2 auto"
+



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] 90d322d Expose VSL_Name2Tag in libvarnishapi
Message-ID: <E1SJhAp-000514-BT@project.varnish-software.com>

commit 90d322d922ba4a23318539a35c0f075f9c59542d
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Fri Nov 4 12:02:31 2011 +0100

    Expose VSL_Name2Tag in libvarnishapi

diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 4df8577..83f5169 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -68,3 +68,10 @@ LIBVARNISHAPI_1.0 {
   local:
 	*;
 };
+
+LIBVARNISHAPI_1.1 {
+  global:
+	# Functions:
+	VSL_Name2Tag;
+	# Variables:
+} LIBVARNISHAPI_1.0;



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] c08a29d Rewrite the ban-lurker.
Message-ID: <E1SJhAp-000535-TL@project.varnish-software.com>

commit c08a29d14d63db48dbee64dfc396176c96595738
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 8 13:07:52 2011 +0000

    Rewrite the ban-lurker.
    
    Use a mark&mark strategy to test all obj.* bans even if there are
    req.* bans in the way.

diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index 3a6ac36..6b59c5c 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -65,12 +65,14 @@ struct ban {
 	unsigned		flags;
 #define BAN_F_GONE		(1 << 0)
 #define BAN_F_REQ		(1 << 2)
-#define BAN_F_LURK		(3 << 3)	/* ban-lurker-color */
+#define BAN_F_LURK		(3 << 6)	/* ban-lurker-color */
 	VTAILQ_HEAD(,objcore)	objcore;
 	struct vsb		*vsb;
 	uint8_t			*spec;
 };
 
+#define LURK_SHIFT 6
+
 struct ban_test {
 	uint8_t			arg1;
 	const char		*arg1_spec;
@@ -401,6 +403,7 @@ BAN_Insert(struct ban *b)
 	/* Hunt down duplicates, and mark them as gone */
 	bi = b;
 	pcount = 0;
+	Lck_Lock(&ban_mtx);
 	while(bi != be) {
 		bi = VTAILQ_NEXT(bi, list);
 		if (bi->flags & BAN_F_GONE)
@@ -409,9 +412,9 @@ BAN_Insert(struct ban *b)
 		if (memcmp(b->spec + 8, bi->spec + 8, ln - 8))
 			continue;
 		bi->flags |= BAN_F_GONE;
+		VSC_C_main->n_ban_gone++;
 		pcount++;
 	}
-	Lck_Lock(&ban_mtx);
 	be->refcount--;
 	VSC_C_main->n_ban_dups += pcount;
 	Lck_Unlock(&ban_mtx);
@@ -627,7 +630,13 @@ ban_evaluate(const uint8_t *bs, const struct http *objhttp,
 }
 
 /*--------------------------------------------------------------------
- * Check an object any fresh bans
+ * Check an object against all applicable bans
+ *
+ * Return:
+ *	-1 not all bans checked, but none of the checked matched
+ *		Only if !has_req
+ *	0 No bans matched, object moved to ban_start.
+ *	1 Ban matched, object removed from ban list.
  */
 
 static int
@@ -661,6 +670,12 @@ ban_check_object(struct object *o, const struct sess *sp, int has_req)
 		CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
 		if (b->flags & BAN_F_GONE)
 			continue;
+		if ((b->flags & BAN_F_LURK) &&
+		    (b->flags & BAN_F_LURK) == (oc->flags & OC_F_LURK)) {
+			AZ(b->flags & BAN_F_REQ);
+			/* Lurker already tested this */
+			continue;
+		}
 		if (!has_req && (b->flags & BAN_F_REQ)) {
 			/*
 			 * We cannot test this one, but there might
@@ -671,23 +686,27 @@ ban_check_object(struct object *o, const struct sess *sp, int has_req)
 			break;
 	}
 
+	Lck_Lock(&ban_mtx);
+	VSC_C_main->n_ban_obj_test++;
+	VSC_C_main->n_ban_re_test += tests;
+
 	if (b == oc->ban && skipped > 0) {
+		AZ(has_req);
+		Lck_Unlock(&ban_mtx);
 		/*
 		 * Not banned, but some tests were skipped, so we cannot know
 		 * for certain that it cannot be, so we just have to give up.
 		 */
-		return (0);
+		return (-1);
 	}
 
-	Lck_Lock(&ban_mtx);
 	oc->ban->refcount--;
 	VTAILQ_REMOVE(&oc->ban->objcore, oc, ban_list);
 	if (b == oc->ban) {	/* not banned */
+		b->flags &= ~BAN_F_LURK;
 		VTAILQ_INSERT_TAIL(&b0->objcore, oc, ban_list);
 		b0->refcount++;
 	}
-	VSC_C_main->n_ban_obj_test++;
-	VSC_C_main->n_ban_re_test += tests;
 	Lck_Unlock(&ban_mtx);
 
 	if (b == oc->ban) {	/* not banned */
@@ -709,7 +728,7 @@ int
 BAN_CheckObject(struct object *o, const struct sess *sp)
 {
 
-	return (ban_check_object(o, sp, 1));
+	return (ban_check_object(o, sp, 1) > 0);
 }
 
 static struct ban *
@@ -720,6 +739,8 @@ ban_CheckLast(void)
 	Lck_AssertHeld(&ban_mtx);
 	b = VTAILQ_LAST(&ban_head, banhead_s);
 	if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
+		if (b->flags & BAN_F_GONE)
+			VSC_C_main->n_ban_gone--;
 		VSC_C_main->n_ban--;
 		VSC_C_main->n_ban_retire++;
 		VTAILQ_REMOVE(&ban_head, b, list);
@@ -730,84 +751,144 @@ ban_CheckLast(void)
 }
 
 /*--------------------------------------------------------------------
- * Ban tail lurker thread
+ * Ban lurker thread
  */
 
 static int
-ban_lurker_work(const struct sess *sp)
+ban_lurker_work(const struct sess *sp, unsigned pass)
 {
-	struct ban *b, *bf;
+	struct ban *b, *b0, *b2;
 	struct objhead *oh;
 	struct objcore *oc, *oc2;
 	struct object *o;
 	int i;
 
-	Lck_Lock(&ban_mtx);
+	AN(pass & BAN_F_LURK);
+	AZ(pass & ~BAN_F_LURK);
 
-	/* First try to route the last ban */
-	bf = ban_CheckLast();
-	if (bf != NULL) {
+	/* First route the last ban(s) */
+	do {
+		Lck_Lock(&ban_mtx);
+		b2 = ban_CheckLast();
 		Lck_Unlock(&ban_mtx);
-		BAN_Free(bf);
-		return (0);
-	}
+		if (b2 != NULL)
+			BAN_Free(b2);
+	} while (b2 != NULL);
 
-	/* Find the last ban give up, if we have only one */
-	b = VTAILQ_LAST(&ban_head, banhead_s);
-	if (b == ban_start) {
-		Lck_Unlock(&ban_mtx);
-		return (0);
+	/*
+	 * Find out if we have any bans we can do something about
+	 * If we find any, tag them with our pass number.
+	 */
+	i = 0;
+	b0 = NULL;
+	VTAILQ_FOREACH(b, &ban_head, list) {
+		if (b->flags & BAN_F_GONE)
+			continue;
+		if (b->flags & BAN_F_REQ)
+			continue;
+		if (b == VTAILQ_LAST(&ban_head, banhead_s))
+			continue;
+		if (b0 == NULL)
+			b0 = b;
+		i++;
+		b->flags &= ~BAN_F_LURK;
+		b->flags |= pass;
 	}
-
-	/* Find the first object on it, if any */
-	oc = VTAILQ_FIRST(&b->objcore);
-	if (oc == NULL) {
-		Lck_Unlock(&ban_mtx);
+	if (params->diag_bitmap & 0x80000)
+		VSL(SLT_Debug, 0, "lurker: %d actionable bans", i);
+	if (i == 0)
 		return (0);
-	}
 
-	/* Try to lock the objhead */
-	oh = oc->objhead;
-	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	if (Lck_Trylock(&oh->mtx)) {
+	VTAILQ_FOREACH_REVERSE(b, &ban_head, banhead_s, list) {
+		if (params->diag_bitmap & 0x80000)
+			VSL(SLT_Debug, 0, "lurker doing %f %d",
+			    ban_time(b->spec), b->refcount);
+		while (1) {
+			Lck_Lock(&ban_mtx);
+			oc = VTAILQ_FIRST(&b->objcore);
+			if (oc == NULL)
+				break;
+			CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+			if (params->diag_bitmap & 0x80000)
+				VSL(SLT_Debug, 0, "test: %p %d %d",
+				    oc, oc->flags & OC_F_LURK, pass);
+			if ((oc->flags & OC_F_LURK) == pass)
+				break;
+			oh = oc->objhead;
+			CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+			if (Lck_Trylock(&oh->mtx)) {
+				Lck_Unlock(&ban_mtx);
+				TIM_sleep(params->ban_lurker_sleep);
+				continue;
+			}
+			/*
+			 * See if the objcore is still on the objhead since
+			 * we race against HSH_Deref() which comes in the
+			 * opposite locking order.
+			 */
+			VTAILQ_FOREACH(oc2, &oh->objcs, list)
+				if (oc == oc2)
+					break;
+			if (oc2 == NULL) {
+				Lck_Unlock(&oh->mtx);
+				Lck_Unlock(&ban_mtx);
+				TIM_sleep(params->ban_lurker_sleep);
+				continue;
+			}
+			/*
+			 * Grab a reference to the OC and we can let go of
+			 * the BAN mutex
+			 */
+			AN(oc->refcnt);
+			oc->refcnt++;
+			oc->flags &= ~OC_F_LURK;
+			Lck_Unlock(&ban_mtx);
+			/*
+			 * Get the object and check it against all relevant bans
+			 */
+			o = oc_getobj(sp->wrk, oc);
+			i = ban_check_object(o, sp, 0);
+			if (params->diag_bitmap & 0x80000)
+				VSL(SLT_Debug, 0, "lurker got: %p %d",
+				    oc, i);
+			if (i == -1) {
+				/* Not banned, not moved */
+				oc->flags |= pass;
+				Lck_Lock(&ban_mtx);
+				VTAILQ_REMOVE(&b->objcore, oc, ban_list);
+				VTAILQ_INSERT_TAIL(&b->objcore, oc, ban_list);
+				Lck_Unlock(&ban_mtx);
+			}
+			Lck_Unlock(&oh->mtx);
+			if (params->diag_bitmap & 0x80000)
+				VSL(SLT_Debug, 0, "lurker done: %p %d %d",
+				    oc, oc->flags & OC_F_LURK, pass);
+			(void)HSH_Deref(sp->wrk, NULL, &o);
+			TIM_sleep(params->ban_lurker_sleep);
+		}
+		Lck_AssertHeld(&ban_mtx);
+		if (!(b->flags & BAN_F_REQ)) {
+			if (!(b->flags & BAN_F_GONE)) {
+				b->flags |= BAN_F_GONE;
+				VSC_C_main->n_ban_gone++;
+			}
+			if (params->diag_bitmap & 0x80000)
+				VSL(SLT_Debug, 0, "lurker BAN %f now gone",
+				    ban_time(b->spec));
+		}
 		Lck_Unlock(&ban_mtx);
-		return (0);
-	}
-
-	/*
-	 * See if the objcore is still on the objhead since we race against
-	 * HSH_Deref() which comes in the opposite locking order.
-	 */
-	VTAILQ_FOREACH(oc2, &oh->objcs, list)
-		if (oc == oc2)
+		TIM_sleep(params->ban_lurker_sleep);
+		if (b == b0)
 			break;
-	if (oc2 == NULL) {
-		Lck_Unlock(&oh->mtx);
-		Lck_Unlock(&ban_mtx);
-		return (0);
 	}
-	/*
-	 * Grab a reference to the OC and we can let go of the BAN mutex
-	 */
-	AN(oc->refcnt);
-	oc->refcnt++;
-	Lck_Unlock(&ban_mtx);
-
-	/*
-	 * Get the object and check it against all relevant bans
-	 */
-	o = oc_getobj(sp->wrk, oc);
-	i = ban_check_object(o, sp, 0);
-	Lck_Unlock(&oh->mtx);
-	WSP(sp, SLT_Debug, "lurker: %p %g %d", oc, o->exp.ttl, i);
-	(void)HSH_Deref(sp->wrk, NULL, &o);
-	return (i);
+	return (1);
 }
 
 static void * __match_proto__(bgthread_t)
 ban_lurker(struct sess *sp, void *priv)
 {
 	struct ban *bf;
+	unsigned pass = (1 << LURK_SHIFT);
 
 	int i = 0;
 	(void)priv;
@@ -827,14 +908,18 @@ ban_lurker(struct sess *sp, void *priv)
 				TIM_sleep(1.0);
 		}
 
-		i = ban_lurker_work(sp);
+		i = ban_lurker_work(sp, pass);
 		WSL_Flush(sp->wrk, 0);
 		WRK_SumStat(sp->wrk);
-
-		if (i != 0)
+		if (i) {
+			pass += (1 << LURK_SHIFT);
+			pass &= BAN_F_LURK;
+			if (pass == 0)
+				pass += (1 << LURK_SHIFT);
 			TIM_sleep(params->ban_lurker_sleep);
-		else
+		} else {
 			TIM_sleep(1.0);
+		}
 	}
 	NEEDLESS_RETURN(NULL);
 }
@@ -948,13 +1033,20 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 
 	VCLI_Out(cli, "Present bans:\n");
 	VTAILQ_FOREACH(b, &ban_head, list) {
-		if (b == bl)
+		if (b == bl && !(params->diag_bitmap & 0x80000))
 			break;
 		VCLI_Out(cli, "%10.6f %5u%s\t", ban_time(b->spec),
 		    bl == b ? b->refcount - 1 : b->refcount,
 		    b->flags & BAN_F_GONE ? "G" : " ");
 		ban_render(cli, b->spec);
 		VCLI_Out(cli, "\n");
+		if (params->diag_bitmap & 0x80000) {
+			Lck_Lock(&ban_mtx);
+			struct objcore *oc;
+			VTAILQ_FOREACH(oc, &b->objcore, ban_list)
+				VCLI_Out(cli, "     %p\n", oc);
+			Lck_Unlock(&ban_mtx);
+		}
 	}
 
 	BAN_TailDeref(&bl);
@@ -973,10 +1065,14 @@ BAN_Init(void)
 
 	Lck_New(&ban_mtx, lck_ban);
 	CLI_AddFuncs(ban_cmds);
+	assert(BAN_F_LURK == OC_F_LURK);
+	AN((1 << LURK_SHIFT) & BAN_F_LURK);
+	AN((2 << LURK_SHIFT) & BAN_F_LURK);
 
 	ban_magic = BAN_New();
 	AN(ban_magic);
 	ban_magic->flags |= BAN_F_GONE;
+	VSC_C_main->n_ban_gone++;
 	BAN_Insert(ban_magic);
 	WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL);
 }
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index 6113978..b3ec8b9 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -833,6 +833,7 @@ static const struct parspec input_parspec[] = {
 		"  0x00010000 - synchronize shmlog.\n"
 		"  0x00020000 - synchronous start of persistence.\n"
 		"  0x00040000 - release VCL early.\n"
+		"  0x00080000 - ban-lurker debugging.\n"
 		"  0x80000000 - do edge-detection on digest.\n"
 		"Use 0x notation and do the bitor in your head :-)\n",
 		0,
diff --git a/bin/varnishtest/tests/c00049.vtc b/bin/varnishtest/tests/c00049.vtc
new file mode 100644
index 0000000..facd979
--- /dev/null
+++ b/bin/varnishtest/tests/c00049.vtc
@@ -0,0 +1,131 @@
+varnishtest "ban lurker test"
+
+
+server s1 {
+	rxreq
+	expect req.url == "/alpha"
+	txresp -hdr "Foo: /alpha"
+
+	rxreq
+	expect req.url == "/beta"
+	txresp -hdr "Foo: /beta"
+
+	rxreq
+	expect req.url == "/gamma"
+	txresp -hdr "Foo: /gamma"
+
+	rxreq
+	expect req.url == "/delta"
+	txresp -hdr "Foo: /delta"
+
+	rxreq
+	expect req.url == "/alpha"
+	txresp -hdr "Foo: /alpha2"
+
+	rxreq
+	expect req.url == "/beta"
+	txresp -hdr "Foo: /beta2"
+
+	rxreq
+	expect req.url == "/delta"
+	txresp -hdr "Foo: /delta2"
+
+} -start
+
+varnish v1 -vcl+backend {
+} -start
+
+varnish v1 -cliok "param.set ban_lurker_sleep 0"
+varnish v1 -cliok "param.set diag_bitmap 0x80000"
+
+varnish v1 -cliok "ban.list"
+
+client c1 {
+	txreq -url "/alpha"
+	rxresp
+	expect resp.http.foo == /alpha
+} -run
+
+delay 0.1
+varnish v1 -cliok "ban req.url == /alpha"
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban == 2
+varnish v1 -expect n_ban_gone == 1
+
+client c1 {
+	txreq -url "/beta"
+	rxresp
+	expect resp.http.foo == /beta
+} -run
+
+delay 0.1
+varnish v1 -cliok "ban obj.http.foo == /beta"
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban == 3
+
+client c1 {
+	txreq -url "/gamma"
+	rxresp
+	expect resp.http.foo == /gamma
+} -run
+
+delay 0.1
+varnish v1 -cliok "ban obj.http.foo == /gamma"
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban == 4
+
+client c1 {
+	txreq -url "/delta"
+	rxresp
+	expect resp.http.foo == /delta
+} -run
+
+delay 0.1
+varnish v1 -cliok "ban req.url == /delta"
+
+varnish v1 -expect n_ban_gone == 1
+varnish v1 -cliok "ban obj.http.foo == /gamma"
+# Dup-check should have added one
+varnish v1 -expect n_ban_gone == 2
+
+varnish v1 -cliok "ban req.url == /epsilon"
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban == 7
+varnish v1 -expect n_ban_gone == 2
+
+varnish v1 -cliok "param.set ban_lurker_sleep .01"
+delay 1
+varnish v1 -cliok "param.set ban_lurker_sleep .00"
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban == 7
+varnish v1 -expect n_ban_gone == 4
+
+client c1 {
+	txreq -url "/alpha"
+	rxresp
+	expect resp.http.foo == /alpha2
+} -run
+
+delay 1
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban == 4
+
+client c1 {
+	txreq -url "/beta"
+	rxresp
+	expect resp.http.foo == /beta2
+} -run
+
+varnish v1 -cliok "ban.list"
+
+
+client c1 {
+	txreq -url "/delta"
+	rxresp
+	expect resp.http.foo == /delta2
+} -run
+
+delay 1
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban == 1
+varnish v1 -expect n_ban_gone == 0



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] b1f19fa Add a new stats counter for "gone" marked bans
Message-ID: <E1SJhAp-00053M-VW@project.varnish-software.com>

commit b1f19faa1976250b050313df55a2165d131d9cc1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 8 13:10:24 2011 +0000

    Add a new stats counter for "gone" marked bans

diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index 4b142d2..dddb235 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -139,12 +139,13 @@ VSC_F(n_vcl,		uint64_t, 0, 'a', "N vcl total", "")
 VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available", "")
 VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded", "")
 
-VSC_F(n_ban,		uint64_t, 0, 'i', "N total active bans", "")
+VSC_F(n_ban,			uint64_t, 0, 'i', "N total active bans", "")
+VSC_F(n_ban_gone,		uint64_t, 0, 'i', "N total gone bans", "")
 VSC_F(n_ban_add,		uint64_t, 0, 'a', "N new bans added", "")
-VSC_F(n_ban_retire,	uint64_t, 0, 'a', "N old bans deleted", "")
-VSC_F(n_ban_obj_test,	uint64_t, 0, 'a', "N objects tested", "")
-VSC_F(n_ban_re_test,	uint64_t, 0, 'a', "N regexps tested against", "")
-VSC_F(n_ban_dups,	uint64_t, 0, 'a', "N duplicate bans removed", "")
+VSC_F(n_ban_retire,		uint64_t, 0, 'a', "N old bans deleted", "")
+VSC_F(n_ban_obj_test,		uint64_t, 0, 'a', "N objects tested", "")
+VSC_F(n_ban_re_test,		uint64_t, 0, 'a', "N regexps tested against", "")
+VSC_F(n_ban_dups,		uint64_t, 0, 'a', "N duplicate bans removed", "")
 
 VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock", "")
 VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock", "")



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] a588d04 Don't start the ban-lurker until -spersistent is done
	loading.
Message-ID: <E1SJhAq-00053U-09@project.varnish-software.com>

commit a588d046f480a56103d915c15b9e6b4b3cba9ab8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 8 13:44:29 2011 +0000

    Don't start the ban-lurker until -spersistent is done loading.

diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index 6b59c5c..bab278d 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -86,6 +86,7 @@ static struct lock ban_mtx;
 static struct ban *ban_magic;
 static pthread_t ban_thread;
 static struct ban * volatile ban_start;
+static bgthread_t ban_lurker;
 
 /*--------------------------------------------------------------------
  * BAN string magic markers
@@ -567,6 +568,7 @@ BAN_Compile(void)
 
 	SMP_NewBan(ban_magic->spec, ban_len(ban_magic->spec));
 	ban_start = VTAILQ_FIRST(&ban_head);
+	WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL);
 }
 
 /*--------------------------------------------------------------------
@@ -1074,5 +1076,4 @@ BAN_Init(void)
 	ban_magic->flags |= BAN_F_GONE;
 	VSC_C_main->n_ban_gone++;
 	BAN_Insert(ban_magic);
-	WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL);
 }



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] 37b7329 Add a missing WS_Release()
Message-ID: <E1SJhAq-00053c-1w@project.varnish-software.com>

commit 37b7329b312d45525d12cded3565e1c2e113fd29
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 8 14:50:11 2011 +0000

    Add a missing WS_Release()

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index fe6b814..f2db178 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -148,6 +148,7 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 		i = VRE_exec(t, str, len, 0, options, ovector, 30,
 		    &params->vre_limits);
 		if (i < VRE_ERROR_NOMATCH ) {
+			WS_Release(sp->http->ws, 0);
 			WSP(sp, SLT_VCL_error,
 			    "Regexp matching returned %d", i);
 			return(str);



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] e700b07 Silence a Flexelint warning
Message-ID: <E1SJhAq-00053s-4c@project.varnish-software.com>

commit e700b07f5c1073d274fdbe6c7c696a074888e4f3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 13 10:05:39 2011 +0000

    Silence a Flexelint warning

diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index 45dfac4..21de929 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -232,7 +232,8 @@ alloc_smf(struct smf_sc *sc, size_t bytes)
 	b = bytes / sc->pagesize;
 	if (b >= NBUCKET)
 		b = NBUCKET - 1;
-	for (sp = NULL; b < NBUCKET - 1; b++) {
+	sp = NULL;
+	for (; b < NBUCKET - 1; b++) {
 		sp = VTAILQ_FIRST(&sc->free[b]);
 		if (sp != NULL)
 			break;



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] b2bad5a Explicitly document that concatenation is only
	supported for the builtins.
Message-ID: <E1SJhAq-00053k-3F@project.varnish-software.com>

commit b2bad5a679f91a07d0240f20a284cc03d9511eca
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Thu Nov 10 22:49:57 2011 +0100

    Explicitly document that concatenation is only supported for the builtins.
    
    Fixes #1042

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index aaca549..3aeb001 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -634,8 +634,11 @@ default code.
 
 Multiple subroutines
 ~~~~~~~~~~~~~~~~~~~~
-If multiple subroutines with the same name are defined, they are
-concatenated in the order in which the appear in the source.
+If multiple subroutines with the the name of one of the builtin
+ones are defined, they are concatenated in the order in which they
+appear in the source.
+The default versions distributed with Varnish will be implicitly
+concatenated as a last resort at the end.
 
 Example:::
 
@@ -661,8 +664,6 @@ Example:::
 	  }
 	}
 
-The builtin default subroutines are implicitly appended in this way.
-
 Variables
 ~~~~~~~~~
 



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] 3156777 Implement VRE options with hard linkage to PCRE options
	instead of maintaining magic hex-bit values that must match.
Message-ID: <E1SJhAq-00053x-5i@project.varnish-software.com>

commit 315677722d1c144b1bcd8a607242d21fee608581
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 8 15:04:04 2011 +0000

    Implement VRE options with hard linkage to PCRE options instead
    of maintaining magic hex-bit values that must match.

diff --git a/include/vre.h b/include/vre.h
index a1206e5..59ffeb0 100644
--- a/include/vre.h
+++ b/include/vre.h
@@ -48,8 +48,8 @@ typedef struct vre vre_t;
 #define VRE_ERROR_NOMATCH         (-1)
 
 /* And those to PCRE options */
-#define VRE_CASELESS           0x00000001
-#define VRE_NOTEMPTY_ATSTART   0x10000000
+extern const unsigned VRE_CASELESS;
+extern const unsigned VRE_NOTEMPTY_ATSTART;
 
 vre_t *VRE_compile(const char *, int, const char **, int *);
 int VRE_exec(const vre_t *code, const char *subject, int length,
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 68beaf8..62da7a0 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -39,6 +39,19 @@ struct vre {
 	pcre *re;
 };
 
+/*
+ * We don't want to spread or even expose the majority of PCRE options
+ * so we establish our own options and implement hard linkage to PCRE
+ * here.
+ */
+const unsigned VRE_CASELESS = PCRE_CASELESS;
+const unsigned VRE_NOTEMPTY_ATSTART =
+#ifdef PCRE_NOTEMPTY_ATSTART
+	PCRE_NOTEMPTY_ATSTART;
+#else
+	0;
+#endif
+
 vre_t *
 VRE_compile(const char *pattern, int options,
 		    const char **errptr, int *erroffset)



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] b9830bf Use PCRE_NOTEMPTY rather than NOTEMPTY_ATSTART,
	it suffices for us
Message-ID: <E1SJhAq-000544-7T@project.varnish-software.com>

commit b9830bfa28b60a70caae92787100c675e1c719c6
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Nov 10 12:24:22 2011 +0100

    Use PCRE_NOTEMPTY rather than NOTEMPTY_ATSTART, it suffices for us

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index f2db178..ec9801a 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -144,7 +144,7 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 		if (!all)
 			break;
 		memset(&ovector, 0, sizeof(ovector));
-		options |= VRE_NOTEMPTY_ATSTART;
+		options |= VRE_NOTEMPTY;
 		i = VRE_exec(t, str, len, 0, options, ovector, 30,
 		    &params->vre_limits);
 		if (i < VRE_ERROR_NOMATCH ) {
diff --git a/include/vre.h b/include/vre.h
index 59ffeb0..a59e8d7 100644
--- a/include/vre.h
+++ b/include/vre.h
@@ -49,7 +49,7 @@ typedef struct vre vre_t;
 
 /* And those to PCRE options */
 extern const unsigned VRE_CASELESS;
-extern const unsigned VRE_NOTEMPTY_ATSTART;
+extern const unsigned VRE_NOTEMPTY;
 
 vre_t *VRE_compile(const char *, int, const char **, int *);
 int VRE_exec(const vre_t *code, const char *subject, int length,
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 62da7a0..e11fa29 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -45,12 +45,7 @@ struct vre {
  * here.
  */
 const unsigned VRE_CASELESS = PCRE_CASELESS;
-const unsigned VRE_NOTEMPTY_ATSTART =
-#ifdef PCRE_NOTEMPTY_ATSTART
-	PCRE_NOTEMPTY_ATSTART;
-#else
-	0;
-#endif
+const unsigned VRE_NOTEMPTY = PCRE_NOTEMPTY;
 
 vre_t *
 VRE_compile(const char *pattern, int options,



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] fe1ae1b Update varnishtest(1) documentation somewhat
Message-ID: <E1SJhAq-00054H-A8@project.varnish-software.com>

commit fe1ae1b527b278eeacfb438dc7c9a73d31b74605
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Fri Nov 18 15:09:51 2011 +0100

    Update varnishtest(1) documentation somewhat

diff --git a/doc/sphinx/reference/varnishtest.rst b/doc/sphinx/reference/varnishtest.rst
index c2aaae9..ff68337 100644
--- a/doc/sphinx/reference/varnishtest.rst
+++ b/doc/sphinx/reference/varnishtest.rst
@@ -7,14 +7,16 @@ Test program for Varnish
 ------------------------
 
 :Author: Stig Sandbeck Mathisen
-:Date:   2010-05-31
-:Version: 1.0
+:Author: Kristian Lyngst?l
+:Date:   2011-11-15
+:Version: 1.1
 :Manual section: 1
 
 
 SYNOPSIS
 ========
-     varnishtest [-n iter] [-q] [-v] file [file ...]
+
+     varnishtest [-iklLqv] [-n iter] [-D name=val] [-j jobs] [-t duration] file [file ...]
 
 DESCRIPTION
 ===========
@@ -24,160 +26,95 @@ Varnish Cache.
 
 The varnishtest program, when started and given one or more script
 files, can create a number of threads representing backends, some
-threads representing clients, and a varnishd process.
+threads representing clients, and a varnishd process. This is then used to
+simulate a transaction to provoke a specific behavior.
 
 The following options are available:
 
--n iter     Run iter number of iterations.
+-D name=val      Define macro for use in scripts
+
+-i               Find varnishd in build tree
+
+-j jobs          Run this many tests in parallel
+
+-k               Continue on test failure
+
+-l               Leave /tmp/vtc.* if test fails
+
+-L               Always leave /tmp/vtc.*
+
+-n iterations    Run tests this many times
+
+-q               Quiet mode: report only failures
+
+-t duration      Time tests out after this long
+
+-v               Verbose mode: always report test log
+
+-h               Show help
 
--q          Be quiet.
+file             File to use as a script
 
--v          Be verbose.
 
--t          Dunno.
+Macro definitions that can be overridden.
 
-file        File to use as a script
+varnishd         Path to varnishd to use [varnishd]
 
 
 SCRIPTS
 =======
 
-Example script
-~~~~~~~~~~~~~~
-::
-
-    # Start a varnish instance called "v1"
-    varnish v1 -arg "-b localhost:9080" -start
-    
-    # Create a server thread called "s1"
-    server s1 {
-        # Receive a request
-        rxreq
-        # Send a standard response
-        txresp -hdr "Connection: close" -body "012345\n"
-    }
-    
-    # Start the server thread
-    server s1 -start
-    
-    # Create a client thread called "c1"
-    client c1 {
-        # Send a request
-        txreq -url "/"
-        # Wait for a response
-        rxresp
-    # Insist that it be a success
-    expect resp.status == 200
-    }
-    
-    # Run the client
-    client c1 -run
-    
-    # Wait for the server to die
-    server s1 -wait
-
-    # (Forcefully) Stop the varnish instance.
-    varnish v1 -stop
-
-Example script output
-~~~~~~~~~~~~~~~~~~~~~
-
-The output, running this script looks as follows. The "bargraph" at
-the beginning of the line is an indication of the level of detail in
-the line. The second field where the message comes from. The rest of
-the line is anyones guess :-)
-::
-
-    #  TEST tests/b00000.vtc starting
-    ### v1  CMD: cd ../varnishd && ./varnishd -d -d -n v1 -a :9081 -T :9001 -b localhost:9080
-    ### v1  opening CLI connection
-    #### v1  debug| NB: Storage size limited to 2GB on 32 bit architecture,\n
-    #### v1  debug| NB: otherwise we could run out of address space.\n
-    #### v1  debug| storage_file: filename: ./varnish.Shkoq5 (unlinked) size 2047 MB.\n
-    ### v1  CLI connection fd = 3
-    #### v1  CLI TX| start
-    #### v1  debug| Using old SHMFILE\n
-    #### v1  debug| Notice: locking SHMFILE in core failed: Operation not permitted\n
-    #### v1  debug| bind(): Address already in use\n
-    #### v1  debug| rolling(1)...
-    #### v1  debug| \n
-    #### v1  debug| rolling(2)...\n
-    #### v1  debug| Debugging mode, enter "start" to start child\n
-    ### v1  CLI 200 <start>
-    ##  s1  Starting server
-    ### s1  listen on :9080 (fd 6)
-    ##  c1  Starting client
-    ##  c1  Waiting for client
-    ##  s1  started on :9080
-    ##  c1  started
-    ### c1  connect to :9081
-    ### c1  connected to :9081 fd is 8
-    #### c1  | GET / HTTP/1.1\r\n
-    #### c1  | \r\n
-    ### c1  rxresp
-    #### s1  Accepted socket 7
-    ### s1  rxreq
-    #### s1  | GET / HTTP/1.1\r\n
-    #### s1  | X-Varnish: 422080121\r\n
-    #### s1  | X-Forwarded-For: 127.0.0.1\r\n
-    #### s1  | Host: localhost\r\n
-    #### s1  | \r\n
-    #### s1  http[ 0] | GET
-    #### s1  http[ 1] | /
-    #### s1  http[ 2] | HTTP/1.1
-    #### s1  http[ 3] | X-Varnish: 422080121
-    #### s1  http[ 4] | X-Forwarded-For: 127.0.0.1
-    #### s1  http[ 5] | Host: localhost
-    #### s1  | HTTP/1.1 200 Ok\r\n
-    #### s1  | Connection: close\r\n
-    #### s1  | \r\n
-    #### s1  | 012345\n
-    #### s1  | \r\n
-    ##  s1  ending
-    #### c1  | HTTP/1.1 200 Ok\r\n
-    #### c1  | Content-Length: 9\r\n
-    #### c1  | Date: Mon, 16 Jun 2008 22:16:55 GMT\r\n
-    #### c1  | X-Varnish: 422080121\r\n
-    #### c1  | Age: 0\r\n
-    #### c1  | Via: 1.1 varnish\r\n
-    #### c1  | Connection: keep-alive\r\n
-    #### c1  | \r\n
-    #### c1  http[ 0] | HTTP/1.1
-    #### c1  http[ 1] | 200
-    #### c1  http[ 2] | Ok
-    #### c1  http[ 3] | Content-Length: 9
-    #### c1  http[ 4] | Date: Mon, 16 Jun 2008 22:16:55 GMT
-    #### c1  http[ 5] | X-Varnish: 422080121
-    #### c1  http[ 6] | Age: 0
-    #### c1  http[ 7] | Via: 1.1 varnish
-    #### c1  http[ 8] | Connection: keep-alive
-    #### c1  EXPECT resp.status (200) == 200 (200) match
-    ##  c1  ending
-    ##  s1  Waiting for server
-    #### v1  CLI TX| stop
-    ### v1  CLI 200 <stop>
-    #  TEST tests/b00000.vtc completed
-
-If instead of 200 we had expected 201 with the line:::
-
-  expect resp.status == 201
-
-The output would have ended with:::
-
-  #### c1  http[ 0] | HTTP/1.1
-  #### c1  http[ 1] | 200
-  #### c1  http[ 2] | Ok
-  #### c1  http[ 3] | Content-Length: 9
-  #### c1  http[ 4] | Date: Mon, 16 Jun 2008 22:26:35 GMT
-  #### c1  http[ 5] | X-Varnish: 648043653 648043652
-  #### c1  http[ 6] | Age: 6
-  #### c1  http[ 7] | Via: 1.1 varnish
-  #### c1  http[ 8] | Connection: keep-alive
-  ---- c1  EXPECT resp.status (200) == 201 (201) failed
+The script language used for Varnishtest is not a strictly defined
+language. The best reference for writing scripts is the varnishtest program
+itself. In the Varnish source code repository, under
+`bin/varnishtest/tests/`, all the regression tests for Varnish are kept.
+
+An example::
+
+        varnishtest "#1029"
+
+        server s1 {
+                rxreq
+                expect req.url == "/bar"
+                txresp -gzipbody {[bar]}
+
+                rxreq
+                expect req.url == "/foo"
+                txresp -body {<h1>FOO<esi:include src="/bar"/>BARF</h1>}
+
+        } -start
+
+        varnish v1 -vcl+backend {
+                sub vcl_fetch {
+                        set beresp.do_esi = true;
+                        if (req.url == "/foo") {
+                                set beresp.ttl = 0s;
+                        } else {
+                                set beresp.ttl = 10m;
+                        }
+                }
+        } -start
+
+        client c1 {
+                txreq -url "/bar" -hdr "Accept-Encoding: gzip"
+                rxresp
+                gunzip
+                expect resp.bodylen == 5
+
+                txreq -url "/foo" -hdr "Accept-Encoding: gzip"
+                rxresp
+                expect resp.bodylen == 21
+        } -run
+
+When run, the above script will simulate a server (s1) that expects two
+different requests. It will start a varnish server (v1) and add the backend
+definition to the VCL specified (-vcl+backend). Finally it starts the
+c1-client, which is a single client sending two requests.
 
 SEE ALSO
 ========
 
+* varnishtest source code repository with tests
 * varnishhist(1)
 * varnishlog(1)
 * varnishncsa(1)
@@ -190,9 +127,9 @@ HISTORY
 
 The varnishtest program was developed by Poul-Henning Kamp
 ?phk at phk.freebsd.dk? in cooperation with Varnish Software AS.
-This manual page
-was written by Stig Sandbeck Mathisen ?ssm at linpro.no? using examples
-by Poul-Henning Kamp ?phk at phk.freebsd.dk?.
+This manual page was originally written by Stig Sandbeck Mathisen
+?ssm at linpro.no? and updated by Kristian Lyngst?l
+(kristian at varnish-cache.org).
 
 COPYRIGHT
 =========
@@ -200,4 +137,4 @@ COPYRIGHT
 This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
-* Copyright (c) 2007-2008 Varnish Software AS
+* Copyright (c) 2007-2011 Varnish Software AS



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] 62128e2 fix a block of text rendering as part of source code
Message-ID: <E1SJhAq-00054B-9B@project.varnish-software.com>

commit 62128e2d485479b22429e3fee8738873d20092df
Author: Kolia Morev <kolyuchiy at gmail.com>
Date:   Sun Nov 13 13:40:47 2011 +0400

    fix a block of text rendering as part of source code

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 3aeb001..d7b71c3 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -970,10 +970,10 @@ based on the request URL:::
     }
   }
 
-  The following snippet demonstrates how to force a minimum TTL for
-  all documents.  Note that this is not the same as setting the
-  default_ttl run-time parameter, as that only affects document for
-  which the backend did not specify a TTL:::
+The following snippet demonstrates how to force a minimum TTL for
+all documents.  Note that this is not the same as setting the
+default_ttl run-time parameter, as that only affects document for
+which the backend did not specify a TTL:::
   
   import std; # needed for std.log
 



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] 33d0e41 Make it possible to test for the non-definition of a
	http header.
Message-ID: <E1SJhAq-00054P-BN@project.varnish-software.com>

commit 33d0e410f4391de29ed08904485cd114164a37cd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 28 09:11:28 2011 +0000

    Make it possible to test for the non-definition of a http header.
    
    Fixes #1062

diff --git a/bin/varnishtest/tests/a00012.vtc b/bin/varnishtest/tests/a00012.vtc
new file mode 100644
index 0000000..07e51c1
--- /dev/null
+++ b/bin/varnishtest/tests/a00012.vtc
@@ -0,0 +1,13 @@
+varnishtest "Ensure that we can test non-existence of headers (#1062)"
+
+server s1 {
+        rxreq
+        txresp
+} -start
+
+client c1 -connect ${s1_sock} {
+        txreq
+        rxresp
+        expect resp.http.X-Test == <undef>
+} -run
+
diff --git a/bin/varnishtest/tests/c00016.vtc b/bin/varnishtest/tests/c00016.vtc
index d028a64..99e9c53 100644
--- a/bin/varnishtest/tests/c00016.vtc
+++ b/bin/varnishtest/tests/c00016.vtc
@@ -8,7 +8,7 @@ server s1 {
 
 	rxreq
 	expect req.url == "/bar"
-	expect req.http.Foo == "req.http.Foo"
+	expect req.http.Foo == <undef>
 	txresp -hdr "Bar: fnry,glyf, FOO ,brok" -hdr "Connection: bar" -body "foobar" 
 } -start
 
@@ -21,5 +21,5 @@ client c1 {
 
 	txreq -url "/bar" -hdr "Foo: bar2" -hdr "Connection: foo, close"
 	rxresp
-	expect req.http.Bar == "req.http.Bar"
+	expect req.http.Bar == <undef>
 } -run
diff --git a/bin/varnishtest/tests/e00024.vtc b/bin/varnishtest/tests/e00024.vtc
index 2aa8e3a..f728b3c 100644
--- a/bin/varnishtest/tests/e00024.vtc
+++ b/bin/varnishtest/tests/e00024.vtc
@@ -81,7 +81,7 @@ client c1 {
 
 	txreq 
 	rxresp
-	expect resp.http.content-encoding == resp.http.content-encoding
+	expect resp.http.content-encoding == <undef>
 	expect resp.status == 200
 	expect resp.bodylen == 252
 } -run
diff --git a/bin/varnishtest/tests/e00025.vtc b/bin/varnishtest/tests/e00025.vtc
index 5549f5c..5e8a12b 100644
--- a/bin/varnishtest/tests/e00025.vtc
+++ b/bin/varnishtest/tests/e00025.vtc
@@ -18,7 +18,7 @@ client c1 {
 
 	txreq 
 	rxresp
-	expect resp.http.content-encoding == resp.http.content-encoding
+	expect resp.http.content-encoding == <undef>
 	expect resp.status == 200
 	expect resp.bodylen == 3
 } -run
diff --git a/bin/varnishtest/tests/g00001.vtc b/bin/varnishtest/tests/g00001.vtc
index 34ad00f..e10b743 100644
--- a/bin/varnishtest/tests/g00001.vtc
+++ b/bin/varnishtest/tests/g00001.vtc
@@ -13,7 +13,7 @@ client c1 {
 	txreq 
 	rxresp
 	expect resp.bodylen == "3"
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 
 	txreq -hdr "Accept-encoding: gzip;q=0.1"
 	rxresp
@@ -26,13 +26,13 @@ client c1 {
 	txreq -proto HTTP/1.0
 	rxresp
 	expect resp.bodylen == "3"
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 } -run
 
 client c1 {
 	txreq -req HEAD
 	rxresp -no_obj
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 
 	txreq -req HEAD -hdr "Accept-encoding: gzip;q=0.1"
 	rxresp -no_obj
diff --git a/bin/varnishtest/tests/g00002.vtc b/bin/varnishtest/tests/g00002.vtc
index 1a0ab35..dad45bc 100644
--- a/bin/varnishtest/tests/g00002.vtc
+++ b/bin/varnishtest/tests/g00002.vtc
@@ -39,12 +39,12 @@ client c1 {
 	# See varnish can gunzip it.
 	txreq -url /foo -hdr "Accept-Encoding: null"
 	rxresp
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 	expect resp.bodylen == 4100
 
 	# See varnish can gunzip it, inside ESI
 	txreq -url /bar -hdr "Accept-Encoding: null"
 	rxresp
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 	expect resp.bodylen == 4109
 } -run
diff --git a/bin/varnishtest/tests/g00003.vtc b/bin/varnishtest/tests/g00003.vtc
index dcd8936..05edb5d 100644
--- a/bin/varnishtest/tests/g00003.vtc
+++ b/bin/varnishtest/tests/g00003.vtc
@@ -30,12 +30,12 @@ varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend {
 client c1 {
 	txreq -url /foo -hdr "Accept-Encoding: gzip"
 	rxresp
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 	expect resp.bodylen == 41
 
 	txreq -url /bar -hdr "Accept-Encoding: gzip"
 	rxresp
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 	expect resp.bodylen == 42
 
 	txreq -url /foobar -hdr "Accept-Encoding: gzip"
@@ -46,6 +46,6 @@ client c1 {
 
 	txreq -url /foobar 
 	rxresp
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 	expect resp.bodylen == 43
 } -run
diff --git a/bin/varnishtest/tests/r00292.vtc b/bin/varnishtest/tests/r00292.vtc
index aaa914c..7739e85 100644
--- a/bin/varnishtest/tests/r00292.vtc
+++ b/bin/varnishtest/tests/r00292.vtc
@@ -6,11 +6,11 @@ varnishtest "Header deletion test"
 server s1 {
 	rxreq
 	expect req.url == "/foo"
-	expect req.http.hdr1 == "req.http.hdr1"
+	expect req.http.hdr1 == <undef>
 	expect req.http.hdr2 == "2"
-	expect req.http.hdr3 == "req.http.hdr3"
+	expect req.http.hdr3 == <undef>
 	expect req.http.hdr4 == "4"
-	expect req.http.hdr5 == "req.http.hdr5"
+	expect req.http.hdr5 == <undef>
 	expect req.http.hdr6 == "6"
 	txresp -body "foobar"
 } -start
diff --git a/bin/varnishtest/tests/r00466.vtc b/bin/varnishtest/tests/r00466.vtc
index acff68a..8d753ba 100644
--- a/bin/varnishtest/tests/r00466.vtc
+++ b/bin/varnishtest/tests/r00466.vtc
@@ -3,7 +3,7 @@ varnishtest "Check Range forwarding to backend"
 server s1 {
 	rxreq 
 	expect req.url == "/foo"
-	expect req.http.range == "req.http.range"
+	expect req.http.range == <undef>
 	txresp \
 		-hdr "Foobar: _barf_" \
 		-body "012345\n"
diff --git a/bin/varnishtest/tests/r00494.vtc b/bin/varnishtest/tests/r00494.vtc
index d1dedd1..1a7fec3 100644
--- a/bin/varnishtest/tests/r00494.vtc
+++ b/bin/varnishtest/tests/r00494.vtc
@@ -19,6 +19,6 @@ client c1 {
 	txreq
 	rxresp
 	expect resp.http.bar == "bar,  barf: fail"
-	expect resp.http.barf == resp.http.barf
-	expect resp.http.foo == resp.http.foo
+	expect resp.http.barf == <undef>
+	expect resp.http.foo == <undef>
 } -run
diff --git a/bin/varnishtest/tests/r00693.vtc b/bin/varnishtest/tests/r00693.vtc
index 6cf7b30..37b0a98 100644
--- a/bin/varnishtest/tests/r00693.vtc
+++ b/bin/varnishtest/tests/r00693.vtc
@@ -4,11 +4,11 @@ feature 64bit
 
 server s1 {
 	rxreq
-	expect req.http.baz == "req.http.baz"
+	expect req.http.baz == <undef>
 	txresp -status 201
 
 	rxreq
-	expect req.http.baz == "req.http.baz"
+	expect req.http.baz == <undef>
 	txresp -status 202
 
 	rxreq
@@ -52,7 +52,7 @@ varnish v1 -arg "-p sess_workspace=1024" -vcl+backend {
 			"0123456789abcdef" + "0123456789abcdef" +
 			"0123456789abcdef" + "0123456789abcdef" +
 			"0123456789abcdef" +
-			"01234567";
+			"0123456";
 		set req.http.baz = "BAZ";
 		return (pass);
 	}
diff --git a/bin/varnishtest/tests/r00861.vtc b/bin/varnishtest/tests/r00861.vtc
index 165795b..8a3e8a0 100644
--- a/bin/varnishtest/tests/r00861.vtc
+++ b/bin/varnishtest/tests/r00861.vtc
@@ -34,12 +34,12 @@ varnish v1 \
 client c1 {
 	txreq -url "/1"
 	rxresp
-	expect resp.http.Content-Encoding == resp.http.Content-Encoding
+	expect resp.http.Content-Encoding == <undef>
 	expect resp.bodylen == 22
 
 	txreq -url "/barf" -hdr "Accept-Encoding: gzip"
 	rxresp
-	expect resp.http.Content-Encoding == resp.http.Content-Encoding
+	expect resp.http.Content-Encoding == <undef>
 	expect resp.bodylen == 909
 
 	txreq -url "/2" -hdr "Accept-Encoding: gzip"
diff --git a/bin/varnishtest/tests/r00980.vtc b/bin/varnishtest/tests/r00980.vtc
index 9591786..b7d307e 100644
--- a/bin/varnishtest/tests/r00980.vtc
+++ b/bin/varnishtest/tests/r00980.vtc
@@ -24,6 +24,6 @@ client c1 {
 
 	txreq -url /foobar 
 	rxresp
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 	expect resp.bodylen == 43
 } -run
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index c71d9d2..66e1d23 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -175,7 +175,7 @@ http_find_header(char * const *hh, const char *hdr)
  * Expect
  */
 
-static char *
+static const char *
 cmd_var_resolve(struct http *hp, char *spec)
 {
 	char **hh, *hdr;
@@ -207,16 +207,16 @@ cmd_var_resolve(struct http *hp, char *spec)
 	hdr = http_find_header(hh, hdr);
 	if (hdr != NULL)
 		return (hdr);
-	return (spec);
+	return ("<undef>");
 }
 
 static void
 cmd_http_expect(CMD_ARGS)
 {
 	struct http *hp;
-	char *lhs;
+	const char *lhs;
 	char *cmp;
-	char *rhs;
+	const char *rhs;
 
 	(void)cmd;
 	(void)vl;



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] d8f305f Be more precise about default behaviour in the presence
	of cookie headers
Message-ID: <E1SJhAq-00054W-D8@project.varnish-software.com>

commit d8f305fa4648c9a34056a2cb239b94100c826c91
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Nov 29 20:16:03 2011 +0100

    Be more precise about default behaviour in the presence of cookie headers

diff --git a/doc/sphinx/tutorial/cookies.rst b/doc/sphinx/tutorial/cookies.rst
index 1d6f291..2ee14cd 100644
--- a/doc/sphinx/tutorial/cookies.rst
+++ b/doc/sphinx/tutorial/cookies.rst
@@ -3,9 +3,10 @@
 Cookies
 -------
 
-Varnish will not cache a object coming from the backend with a
-Set-Cookie header present. Also, if the client sends a Cookie header,
-Varnish will bypass the cache and go directly to the backend.
+Varnish will, in the default configuration, not cache a object coming
+from the backend with a Set-Cookie header present. Also, if the client
+sends a Cookie header, Varnish will bypass the cache and go directly to
+the backend.
 
 This can be overly conservative. A lot of sites use Google Analytics
 (GA) to analyze their traffic. GA sets a cookie to track you. This



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] 05da5bc Clean up examples a bit
Message-ID: <E1SJhAq-00054k-Gx@project.varnish-software.com>

commit 05da5bc40bd17b30aab6e5e513f09677f98779de
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Wed Nov 30 15:01:40 2011 +0100

    Clean up examples a bit

diff --git a/doc/sphinx/tutorial/vary.rst b/doc/sphinx/tutorial/vary.rst
index 4da6744..a31e449 100644
--- a/doc/sphinx/tutorial/vary.rst
+++ b/doc/sphinx/tutorial/vary.rst
@@ -14,11 +14,11 @@ the page encoded with the deflate encoding.
 The problem is that the Accept-Encoding field contains a lot of
 different encodings. If one browser sends::
 
-  Accept-Encodign: gzip,deflate
+  Accept-Encoding: gzip,deflate
 
 And another one sends::
 
-  Accept-Encoding:: deflate,gzip
+  Accept-Encoding: deflate,gzip
 
 Varnish will keep two variants of the page requested due to the
 different Accept-Encoding headers. Normalizing the accept-encoding



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] 40236d9 remove minor typos
Message-ID: <E1SJhAq-00054y-KL@project.varnish-software.com>

commit 40236d910fe668f54183db53e9f64c46dc1eeb4a
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Wed Nov 30 15:07:59 2011 +0100

    remove minor typos

diff --git a/doc/sphinx/tutorial/cookies.rst b/doc/sphinx/tutorial/cookies.rst
index 2ee14cd..ab59e21 100644
--- a/doc/sphinx/tutorial/cookies.rst
+++ b/doc/sphinx/tutorial/cookies.rst
@@ -10,7 +10,7 @@ the backend.
 
 This can be overly conservative. A lot of sites use Google Analytics
 (GA) to analyze their traffic. GA sets a cookie to track you. This
-cookie is used by the client side java script and is therefore of no
+cookie is used by the client side javascript and is therefore of no
 interest to the server. 
 
 For a lot of web application it makes sense to completely disregard the
@@ -62,4 +62,4 @@ cookies named COOKIE1 and COOKIE2 and you can marvel at it::
   }
 
 The example is taken from the Varnish Wiki, where you can find other
-scary examples of what can be done i VCL.
+scary examples of what can be done in VCL.



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] 23fa9e5 Fix typo
Message-ID: <E1SJhAr-00056v-AF@project.varnish-software.com>

commit 23fa9e5ab6bbc752dd69a39a8a1ce87381aaf28b
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Jan 2 12:59:42 2012 +0100

    Fix typo

diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index 69b6546..a097248 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -196,7 +196,7 @@ consists of a field, an operator, and an argument.  Conditions can be
 ANDed together with "&&".
 
 A field can be any of the variables from VCL, for instance req.url,
-req.http.host or obj.set-cookie.
+req.http.host or obj.http.set-cookie.
 
 Operators are "==" for direct comparision, "~" for a regular
 expression match, and ">" or "<" for size comparisons.  Prepending



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] ce08e8a Add explanation for Varnish "hashing" in light of
	advisories.
Message-ID: <E1SJhAr-00057C-E6@project.varnish-software.com>

commit ce08e8ae1a010a35ea4b83e812268f3898be2db8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 3 18:10:29 2012 +0000

    Add explanation for Varnish "hashing" in light of advisories.

diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst
index 822434c..58935c7 100644
--- a/doc/sphinx/phk/index.rst
+++ b/doc/sphinx/phk/index.rst
@@ -8,6 +8,7 @@ You may or may not want to know what Poul-Henning think.
 
 .. toctree::
 
+	varnish_does_not_hash.rst
 	thetoolsweworkwith.rst
 	three-zero.rst
 	ssl.rst
diff --git a/doc/sphinx/phk/varnish_does_not_hash.rst b/doc/sphinx/phk/varnish_does_not_hash.rst
new file mode 100644
index 0000000..8393c87
--- /dev/null
+++ b/doc/sphinx/phk/varnish_does_not_hash.rst
@@ -0,0 +1,145 @@
+.. _phk_varnish_does_not_hash:
+
+=====================
+Varnish Does Not Hash
+=====================
+
+A spate of security advisories related to hash-collisions have made
+a lot of people stare at Varnish and wonder if it is affected.
+
+The answer is no, but the explanation is probably not what most of
+you expected:
+
+Varnish does not hash, at least not by default, and
+even if it does, it's still as immune to the attacks as can be.
+
+To understand what is going on, I have to introduce a concept from
+Shannons information theory: "entropy."
+
+Entropy is hard to explain, and according to legend, that is exactly
+why Shannon recycled that term from thermodynamics.
+
+In this context, we can get away with thinking about entropy as how
+much our "keys" differ::
+
+	Low entropy (1 bit):
+		/foo/bar/barf/some/cms/content/article?article=2
+		/foo/bar/barf/some/cms/content/article?article=3
+
+	High entropy (65 bits):
+		/i?ee30d0770eb460634e9d5dcfb562a2c5.html
+		/i?bca3633d52607f38a107cb5297fd66e5.html
+
+Hashing consists of calculating a hash-index from the key and
+storing the objects in an array indexed by that key.
+
+Typically, but not always, the key is a string and the index is a
+(smallish) integer, and the job of the hash-function is to squeeze
+the key into the integer, without loosing any of the entropy.
+
+Needless to say, the more entropy you have to begin with, the more
+of it you can afford to loose, and loose some you almost invariably
+will.
+
+There are two families of hash-functions, the fast ones, and the good
+ones, and the security advisories are about the fast ones.
+
+The good ones are slower, but probably not so much slower that you
+care, and therefore, if you want to fix your web-app:
+
+Change::
+	foo=somedict[$somekey]
+To::
+	foo=somedict[md5($somekey)]
+
+and forget about the advisories.
+
+Yes, that's right: Cryptographic hash algorithms are the good ones,
+they are built to not throw any entropy away, and they are built to
+have very hard to predict collisions, which is exactly the problem
+with the fast hash-functions in the advisories.
+
+-----------------
+What Varnish Does
+-----------------
+
+The way to avoid having hash-collisions is to not use a hash:  Use a
+tree instead, there every object has its own place and there are no
+collisions.
+
+Varnish does that, but with a twist.
+
+The "keys" in varnish can be very long, by default they consist of::
+
+	sub vcl_hash {
+	    hash_data(req.url);
+	    if (req.http.host) {
+		hash_data(req.http.host);
+	    } else {
+		hash_data(server.ip);
+	    }
+	    return (hash);
+	}
+
+But some users will add cookies, user identification and many other
+bits and pieces of string in there, and in the end the keys can be
+kilobytes in length, and quite often, as in the first example above,
+the first difference may not come until pretty far into the keys.
+
+Trees generally need to have a copy of the key around to be able
+to tell if they have a match, and more importantly to compare
+tree-leaves in order to "re-balance" the tree and other such arcanae
+of data structures.
+
+This would add another per-object memory load to Varnish, and it
+would feel particularly silly to store 48 identical characters for
+each object in the far too common case seen above.
+
+But furthermore, we want the tree to be very fast to do lookups in,
+preferably it should be lockless for lookups, and that means that
+we cannot (realistically) use any of the "smart" trees which
+automatically balance themselves etc.
+
+You (generally) don't need a "smart" tree if your keys look
+like random data in the order they arrive, but we can pretty
+much expect the opposite as article number 4, 5, 6 etc are added
+to the CMS in the first example.
+
+But we can make the keys look random, and make them small and fixed
+size at the same time, and the perfect functions designed for just
+that task are the "good" hash-functions, the cryptographic ones.
+
+So what Varnish does is "key-compression":  All the strings hash_data()
+are fed, are pushed through a cryptographic hash algorithm called
+SHA256, which, as the name says, always spits out 256 bits (= 32
+bytes), no matter how many bits you feed it.
+
+This does not eliminate the key-storage requirement, but now all
+the keys are 32 bytes and can be put directly into the data structure::
+
+	struct objhead {
+		[...]
+		unsigned char           digest[DIGEST_LEN];
+	};
+
+In the example above, the output of SHA256 for the 1 bit difference
+in entropy becomes::
+
+	/foo/bar/barf/some/cms/content/article?article=2
+	-> 14f0553caa5c796650ec82256e3f111ae2f20020a4b9029f135a01610932054e
+	/foo/bar/barf/some/cms/content/article?article=3
+	-> 4d45b9544077921575c3c5a2a14c779bff6c4830d1fbafe4bd7e03e5dd93ca05
+
+That should be random enough.
+
+But the key-compression does introduce a risk of collisions, since
+not even SHA256 can guarantee different outputs for all possible
+inputs:  Try pushing all the possible 33 bytes long files through
+SHA256 and sooner or later you will get collisions.
+
+The risk of collision is very small however, and I can all but
+promise you, that you will be fully offset in fame and money for
+any inconvenience a collision might cause, because you will
+be the first person to find a SHA256 collision.
+
+Poul-Henning, 2012-01-03



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] 095c9b9 Added an introduction. Brushed up the chapter on
	varnish in a virtualized environment
Message-ID: <E1SJhAr-00057K-FH@project.varnish-software.com>

commit 095c9b94cfc966d3b5425c609dd4f5af813193e6
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 11 10:26:56 2012 +0100

    Added an introduction. Brushed up the chapter on varnish in a virtualized environment

diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
index 32c2264..0bf3137 100644
--- a/doc/sphinx/tutorial/index.rst
+++ b/doc/sphinx/tutorial/index.rst
@@ -14,6 +14,7 @@ separate topic. Good luck.
 
 .. toctree:: :maxdepth: 1
 
+        indroduction
 	backend_servers
         starting_varnish
 	logging
diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
new file mode 100644
index 0000000..17d7b6a
--- /dev/null
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -0,0 +1,37 @@
+.. _tutorial-intro:
+
+What is Varnish?
+----------------
+
+Varnish Cache is a Varnish Cache is a web application accelerator also
+known as a caching HTTP reverse proxy. You install it in front of any
+server that speaks HTTP and configure it to cache the
+contents. Varnish Cache is really, really fast. It typically speeds up
+delivery with a factor of 300 - 1000x, depending on your architecture.
+
+
+Performance
+~~~~~~~~~~~
+
+Varnish performs really, really well. It is usually bound by the speed
+of the network, effectivly turning performance into a non-issue. We've
+seen Varnish delivering 20 Gbps on regular off-the-shelf hardware.
+
+Flexibility
+~~~~~~~~~~~
+
+One of the key features of Varnish Cache, in addition to it's
+performance, is the flexibility of it's configuration language,
+VCL. VCL enables you to write policies on how incoming requests should
+be handled. In such a policy you can decide what content you want to
+serve, from where you want to get the content and how the request or
+response should be altered. You can read more about this in our
+tutorial.
+
+
+Supported plattforms
+~~~~~~~~~~~~~~~~~~~~
+
+Varnish is written to run on modern versions of Linux and FreeBSD and
+the best experience is had on those plattforms. Thanks to our
+contributors it also runs on NetBSD, OpenBSD and OS X.
diff --git a/doc/sphinx/tutorial/virtualised.rst b/doc/sphinx/tutorial/virtualised.rst
index ce4b9ad..ea8ade5 100644
--- a/doc/sphinx/tutorial/virtualised.rst
+++ b/doc/sphinx/tutorial/virtualised.rst
@@ -1,16 +1,19 @@
 
-Running inside a virtual machine (VM)
--------------------------------------
+Running Varnish in a virtualized environment
+--------------------------------------------
 
-It is possible, but not recommended for high performance, to run Varnish on virtualised 
-hardware.
+It is possible, but not recommended for high performance, to run
+Varnish on virtualised hardware. Reduced disk- and network performance
+will reduce the performance a bit so make sure your system has good IO
+performance.
 
 OpenVZ
-''''''
+~~~~~~
 
-If you are running on 64bit OpenVZ (or Parallels VPS), you must reduce the 
-maximum stack size before starting Varnish. The default allocates to much memory per thread, 
-which will make varnish fail as soon as the number of threads (==traffic) increases.
+If you are running on 64bit OpenVZ (or Parallels VPS), you must reduce
+the maximum stack size before starting Varnish. The default allocates
+to much memory per thread, which will make varnish fail as soon as the
+number of threads (==traffic) increases.
 
 Reduce the maximum stack size by running::
 



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] 6b4dfb8 typo. Thanks scoof
Message-ID: <E1SJhAr-00057R-Gh@project.varnish-software.com>

commit 6b4dfb86333ad16bec47cdcd12c374836833d392
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 11 12:48:23 2012 +0100

    typo. Thanks scoof

diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
index 17d7b6a..0d43623 100644
--- a/doc/sphinx/tutorial/introduction.rst
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -3,11 +3,11 @@
 What is Varnish?
 ----------------
 
-Varnish Cache is a Varnish Cache is a web application accelerator also
-known as a caching HTTP reverse proxy. You install it in front of any
-server that speaks HTTP and configure it to cache the
-contents. Varnish Cache is really, really fast. It typically speeds up
-delivery with a factor of 300 - 1000x, depending on your architecture.
+Varnish Cache is a web application accelerator also known as a caching
+HTTP reverse proxy. You install it in front of any server that speaks
+HTTP and configure it to cache the contents. Varnish Cache is really,
+really fast. It typically speeds up delivery with a factor of 300 -
+1000x, depending on your architecture.
 
 
 Performance



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] 5e2a759 Handle the case of sub being NULL
Message-ID: <E1SJhAr-00057c-I2@project.varnish-software.com>

commit 5e2a759b85c0465542700a90a68a4943db382f49
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Jan 11 14:07:02 2012 +0100

    Handle the case of sub being NULL
    
    if sub is null (because it's the contents of a non-existing header),
    pretend it's "" instead.

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index ec9801a..6eac0da 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -101,6 +101,8 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 	AN(re);
 	if (str == NULL)
 		str = "";
+	if (sub == NULL)
+		sub = "";
 	t = re;
 	memset(ovector, 0, sizeof(ovector));
 	len = strlen(str);
diff --git a/bin/varnishtest/tests/c00001.vtc b/bin/varnishtest/tests/c00001.vtc
index d9b6c41..f1a6824 100644
--- a/bin/varnishtest/tests/c00001.vtc
+++ b/bin/varnishtest/tests/c00001.vtc
@@ -21,6 +21,8 @@ varnish v1 -vcl+backend {
 		    regsub(beresp.http.Foobar, "(b)(a)(r)(f)", "\0\4\3\2\\p");
 		set beresp.http.Snafu6 =
 		    regsub(beresp.http.Foobar, "(b)(a)(r)(f)", "\4\&\3\2p\");
+		set beresp.http.Snafu7 =
+		    regsub(beresp.http.Foobar, "ar", req.http.nosuchheader);
 	}
 } -start 
 
@@ -37,4 +39,5 @@ client c1 {
 	# NB: have to escape the \\ in the next two lines
 	expect resp.http.snafu5 == "_barffra\\p_"
 	expect resp.http.snafu6 == "_f&rap\\_"
+	expect resp.http.snafu7 == "_bf_"
 } -run



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] 0ff7bb2 Update .gitignore
Message-ID: <E1SJhAr-00057j-JT@project.varnish-software.com>

commit 0ff7bb23aee5aa249dc6e266f07f81e61e201f27
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Sat Jan 14 13:36:10 2012 +0100

    Update .gitignore
    
    I work with NetBeans, and it insists on this being present.

diff --git a/.gitignore b/.gitignore
index f35e990..21fe76b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -84,3 +84,6 @@ TAGS
 /doc/*.html
 /doc/sphinx/=build/
 /doc/sphinx/conf.py
+
+# NetBeans insists on this
+/nbproject/private/



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] 079c360 Actually use a default (80) if no port is specified
Message-ID: <E1SJhAr-00057r-LO@project.varnish-software.com>

commit 079c3608fae1d0c2c43097066b96fa3b3617cc68
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Sat Jan 14 13:37:57 2012 +0100

    Actually use a default (80) if no port is specified

diff --git a/lib/libvcl/vcc_dir_dns.c b/lib/libvcl/vcc_dir_dns.c
index 8168a26..45c80c1 100644
--- a/lib/libvcl/vcc_dir_dns.c
+++ b/lib/libvcl/vcc_dir_dns.c
@@ -96,7 +96,7 @@ print_backend(struct vcc *tl,
 	if (serial >= 0)
 		Fb(tl, 0, "[%d]", serial);
 	Fb(tl, 0, "\",\n");
-	Emit_Sockaddr(tl, &tmptok, b_defaults.port);
+	Emit_Sockaddr(tl, &tmptok, b_defaults.port ? b_defaults.port : "80");
 
 	Fb(tl, 0, "\t.hosthdr = \"");
 	if (b_defaults.hostheader != NULL)



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] 9d33363 Update syntax for 3.0
Message-ID: <E1SJhAr-000583-No@project.varnish-software.com>

commit 9d33363d0213a28b2542718f5437f8768f8b30cc
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Sun Jan 15 17:44:17 2012 +0100

    Update syntax for 3.0

diff --git a/doc/sphinx/tutorial/handling_misbehaving_servers.rst b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
index c258f46..6cd7111 100644
--- a/doc/sphinx/tutorial/handling_misbehaving_servers.rst
+++ b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
@@ -69,7 +69,7 @@ can be enabled in VCL:::
   sub vcl_fetch {
     if (beresp.status == 500) { 
       set beresp.saintmode = 10s;
-      restart;
+      return(restart);
     }
     set beresp.grace = 5m;
   } 



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] cf19220 By Slink, via Geoff:
Message-ID: <E1SJhAr-00058A-Qm@project.varnish-software.com>

commit cf192202f9801fff9c689b15eb3e4c2768feb8bf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Jan 15 19:30:53 2012 +0000

    By Slink, via Geoff:
    
    The attached patch is a one-liner for waiter/cache_waiter_ports.c,
    the code was referencing sp->t_open where it should have been
    sp->t_idle, which was causing assertion failures on idle session
    timeouts.

diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index 9ee3aea..e26a80c 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2006 Varnish Software AS
  * Copyright (c) 2007 OmniTI Computer Consulting, Inc.
  * Copyright (c) 2007 Theo Schlossnagle
- * Copyright (c) 2010 UPLEX, Nils Goroll
+ * Copyright (c) 2010-2012 UPLEX, Nils Goroll
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -224,7 +224,8 @@ vca_main(void *arg)
 		 */
 
 		if (sp) {
-			double tmo = (sp->t_open + params->sess_timeout) - now;
+			double tmo =
+			    (sp->t_end + cache_param->timeout_idle) - now;
 
 			/* we should have removed all sps whose timeout has passed */
 			assert(tmo > 0.0);



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] 2cfb4ee speling
Message-ID: <E1SJhAr-00058a-TB@project.varnish-software.com>

commit 2cfb4eeea720d1d3cbe7fdd61b52451fe17d86f7
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Jan 19 10:55:49 2012 +0100

    speling

diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
index 0bf3137..c3a05f5 100644
--- a/doc/sphinx/tutorial/index.rst
+++ b/doc/sphinx/tutorial/index.rst
@@ -14,7 +14,7 @@ separate topic. Good luck.
 
 .. toctree:: :maxdepth: 1
 
-        indroduction
+        introduction
 	backend_servers
         starting_varnish
 	logging



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] 2ee9351 Add support for the %{format}t construct to varnishncsa
Message-ID: <E1SJhAr-00058U-Rv@project.varnish-software.com>

commit 2ee935102094cbc1fab646ccea37694a5516a98c
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Jan 16 10:12:14 2012 +0100

    Add support for the %{format}t construct to varnishncsa

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index ae3e4cb..ff07ed5 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -749,6 +749,11 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 				VSB_cat(os, h ? h : "-");
 				p = tmp;
 				break;
+			case 't':
+				strftime(tbuf, sizeof tbuf, fname, &lp->df_t);
+				VSB_cat(os, tbuf);
+				p = tmp;
+				break;
 			case 'x':
 				if (strcmp(fname, "Varnish:time_firstbyte") == 0) {
 					VSB_cat(os, lp->df_ttfb);
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 0b3f97f..9373f5c 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -90,6 +90,11 @@ The following options are available:
 	         Time when the request was received, in HTTP date/time
 	         format.
 
+	      %{X}t
+	         Time when the request was received, in the format
+		 specified by X.  The time specification format is the
+		 same as for strftime(3).
+
 	      %U
 	         The request URL without any query string. Defaults to
                  '-' if not known.



From tfheen at varnish-cache.org  Mon Apr 16 08:20:38 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:38 +0200
Subject: [3.0] d53674d Make this test case more robust (and faster) by using
	the CLI to control backend health, rather than rely on probes.
Message-ID: <E1SJhAs-00058t-0U@project.varnish-software.com>

commit d53674df134c77557d83c7423cd2c875ca78a645
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 24 19:35:28 2012 +0000

    Make this test case more robust (and faster) by using the
    CLI to control backend health, rather than rely on probes.
    
    Submitted by:	Federico G. Schwindt

diff --git a/bin/varnishtest/tests/v00036.vtc b/bin/varnishtest/tests/v00036.vtc
index e83acef..323f80c 100644
--- a/bin/varnishtest/tests/v00036.vtc
+++ b/bin/varnishtest/tests/v00036.vtc
@@ -1,127 +1,66 @@
 varnishtest "Test fallback director"
 
 server s1 {
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	sema r1 sync 3
-
-	accept
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	accept
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	sema r3 sync 2
-
-	accept
-	rxreq
-	expect req.url == "/foo"
-	txresp -hdr "Foo: 1" -body "foobar"
-
+        rxreq
+        txresp -hdr "Foo: 1"
 } -start
 
 server s2 {
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	sema r1 sync 3
-
-	accept
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	sema r2 sync 2
-
-	accept
-	rxreq
-	expect req.url == "/foo"
-	txresp -hdr "Foo: 2" -body "foobar"
+        rxreq
+        txresp -hdr "Foo: 2"
 } -start
 
 server s3 {
-	rxreq
-	expect req.url == "/foo"
-	txresp -hdr "Foo: 3" -body "foobar"
+        rxreq
+        txresp -hdr "Foo: 3"
 } -start
 
 varnish v1 -vcl {
-
-	probe p1 {
-		.url = "/";
-		.timeout = 1s;
-		.interval = 1s;
-		.window = 4;
-		.threshold = 3;
-		.initial = 0;
-	}
-	probe p2 {
-		.url = "/";
-		.timeout = 1s;
-		.interval = 1s;
-		.window = 3;
-		.threshold = 2;
-		.initial = 0;
-	}
-
-	backend b1 {
-		.host = "${s1_addr}";
-		.port = "${s1_port}";
-		.max_connections = 1;
-		.probe = p1;
-	}
-	backend b2 {
-		.host = "${s2_addr}";
-		.port = "${s2_port}";
-		.max_connections = 1;
-		.probe = p2;
-	}
-	backend b3 {
-		.host = "${s3_addr}";
-		.port = "${s3_port}";
-	}
-	director f1 fallback {
-		{ .backend = b1; }
-		{ .backend = b2; }
-		{ .backend = b3; }
-	}
-
-	sub vcl_recv {
-		set req.backend = f1;
-		return(pass);
-	}
+        backend b1 {
+                .host = "${s1_addr}";
+                .port = "${s1_port}";
+        }
+        backend b2 {
+                .host = "${s2_addr}";
+                .port = "${s2_port}";
+        }
+        backend b3 {
+                .host = "${s3_addr}";
+                .port = "${s3_port}";
+        }
+        director f1 fallback {
+                { .backend = b1; }
+                { .backend = b2; }
+                { .backend = b3; }
+        }
+        sub vcl_recv {
+                set req.backend = f1;
+                return (pass);
+        }
 } -start
 
+varnish v1 -cliok "backend.set_health b1 sick"
+varnish v1 -cliok "backend.set_health b2 sick"
 
 client c1 {
-	# s1 & s2 are both sick, expect response from s3
-	txreq -url "/foo"
-	rxresp
-	expect resp.http.foo == "3"
+        # s1 & s2 are both sick, expect response from s3
+        txreq
+        rxresp
+        expect resp.http.foo == "3"
+} -run
 
-	sema r1 sync 3
+varnish v1 -cliok "backend.set_health b2 healthy"
 
-	# wait for s2 to become healthy
-	sema r2 sync 2
-	txreq -url "/foo"
-	rxresp
-	expect resp.http.foo == "2"
+client c1 {
+        txreq
+        rxresp
+        expect resp.http.foo == "2"
+} -run
 
-	# wait for s1 to become healthy
-	sema r3 sync 2
-	txreq -url "/foo"
-	rxresp
-	expect resp.http.foo == "1"
+varnish v1 -cliok "backend.set_health b1 healthy"
+
+client c1 {
+        txreq
+        rxresp
+        expect resp.http.foo == "1"
 } -run



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] f93325e Fix a test description
Message-ID: <E1SJhAr-00058h-Um@project.varnish-software.com>

commit f93325eab74f2e5238233f3ea539b3c1d9f3b77a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 22:09:04 2012 +0000

    Fix a test description

diff --git a/bin/varnishtest/tests/r00102.vtc b/bin/varnishtest/tests/r00102.vtc
index 6cf81cf..6d2d8aa 100644
--- a/bin/varnishtest/tests/r00102.vtc
+++ b/bin/varnishtest/tests/r00102.vtc
@@ -1,4 +1,4 @@
-varnishtest "Test VCL regsub()"
+varnishtest "Test POST->GET conversion"
 
 server s1 {
 	rxreq 



From tfheen at varnish-cache.org  Mon Apr 16 08:20:38 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:38 +0200
Subject: [3.0] 0f07324 add link
Message-ID: <E1SJhAs-00059G-6L@project.varnish-software.com>

commit 0f0732405022fb9048c97d82a676c03c889121d3
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 25 09:56:57 2012 +0100

    add link

diff --git a/doc/sphinx/tutorial/advanced_backend_servers.rst b/doc/sphinx/tutorial/advanced_backend_servers.rst
index 307f715..b4206d9 100644
--- a/doc/sphinx/tutorial/advanced_backend_servers.rst
+++ b/doc/sphinx/tutorial/advanced_backend_servers.rst
@@ -40,6 +40,8 @@ really arbitrary data. You want to send mobile devices to a different
 backend? No problem. if (req.User-agent ~ /mobile/) .... should do the
 trick. 
 
+.. _tutorial-advanced_backend_servers-directors:
+
 Directors
 ---------
 



From tfheen at varnish-cache.org  Mon Apr 16 08:20:38 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:38 +0200
Subject: [3.0] ebd73a7 cleaned out some cruft. Added links to the actual docs
Message-ID: <E1SJhAs-000592-3y@project.varnish-software.com>

commit ebd73a71cfd3a0ee9ae9aaceb71e006f309c8c50
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 25 09:56:41 2012 +0100

    cleaned out some cruft. Added links to the actual docs

diff --git a/doc/sphinx/faq/general.rst b/doc/sphinx/faq/general.rst
index b30f093..2444394 100644
--- a/doc/sphinx/faq/general.rst
+++ b/doc/sphinx/faq/general.rst
@@ -5,25 +5,7 @@ General questions
 What is Varnish?
 ================
 
-Varnish is a state-of-the-art, high-performance `web accelerator <http://en.wikipedia.org/wiki/Web_accelerator>`_. It uses the advanced features in Linux 2.6, FreeBSD 6/7 and Solaris 10 to achieve its high performance.
-
-Some of the features include
-
-* A modern design
-* VCL - a very flexible configuration language
-* Load balancing with health checking of backends
-* Partial support for ESI
-* URL rewriting
-* Graceful handling of "dead" backends
-
-Features to come (Experimental):
-
-* Support for Ranged headers
-* Support for persistent cache
-	
-
-Varnish is free software and is licenced under a modified BSD licence. Please read the introduction to get started with Varnish.
-
+Please see ref:`tutorial-intro`.
 
 How...
 ======
@@ -59,7 +41,9 @@ This can be achieved by removing the query parameters using a regexp::
 
 **How can I force a refresh on a object cached by varnish?**
 
-Refreshing is often called `purging <http://dictionary.reference.com/browse/PURGE>`_ a document.  You can purge at least 2 different ways in Varnish:
+Refreshing is often called `purging
+<http://dictionary.reference.com/browse/PURGE>`_ a document.  You can
+purge at least 2 different ways in Varnish:
 
 1. Command line
 
@@ -118,23 +102,6 @@ You can use the ``bereq`` object for altering requests going to the backend, but
                 return(fetch);
         }
 
-**How do I force the backend to send Vary headers?**
-
-We have anecdotal evidence of non-RFC2616 compliant backends, which support content negotiation, but which do not emit a Vary header, unless the request contains Accept headers.
-
-It may be appropriate to send no-op Accept headers to trick the backend into sending us the Vary header.
-
-The following should be sufficient for most cases::
-
-        Accept: */*
-        Accept-Language: *
-        Accept-Charset: *
-        Accept-Encoding: identity
-
-Note that Accept-Encoding can not be set to ``*``, as the backend might then send back a compressed response which the client would be unable to process.
-
-This can of course be implemented in VCL.
-
 **How can I customize the error messages that Varnish returns?**
 
 A custom error page can be generated by adding a ``vcl_error`` to your configuration file. The default error page looks like this::
@@ -171,32 +138,9 @@ This can be achieved by removing the query parameters using a regexp::
         }
 
 
-Where...
-========
-
-**Can I find varnish for my operating system?**
-
-We know that Varnish has been packaged for Debian, Ubuntu, RHEL,
-CentOS, (Open)SUSE, Gentoo and FreeBSD, possibly more.  Check whatever
-package manager you use. Or read :ref:`Installing Varnish on your computer <install-doc>`.
-
 Can I...
 ========
 
-**Can I use Varnish as a client-side proxy?**
-
-No.  Varnish needs all backends configured in the VCL.  Look at squid
-instead.
-
-**Can I run Varnish on a 32bit system?**
-
-Yes, recently somebody even claimed to run Varnish on his N900 mobile
-phone recently, but if you have the choice, go 64 bit from the start.
-
-Varnish is written to use Virtual Memory and on a 32bit system that
-really cramps your style, and you will have trouble configuring more
-than 2 GB of storage.
-
 **Can I run Varnish on the same system as Apache?**
 
 Yes, and many people do that with good success.
@@ -243,83 +187,28 @@ Yes, you need VCL code like this::
 		set req.backend = foobar;
 	}
 
-    (XXX: reference to docs, once written)
+    Please see :ref:`tutorial-advanced_backend_servers-directors_`.
+
 
 Why ...
 =======
 
 **Why does it look like Varnish sends all requests to the backend? I thought it was a cache?**
 
-There are 2 common reasons for this:
-	1. The object's ``ttl expired``. A common situation is that the backend does not set an expiry time on the requested image/file/webpage, so Varnish uses the default TTL (normally 120s).
-	2. Your site uses ``cookies``:
-            * By default, varnish will not cache ``responses`` from the backend that come with a ``Set-Cookie``: header.
-            * By default, varnish will not serve ``requests`` with a ``Cookie:`` header, but pass them to the backend instead. Check out [wiki:VCLExamples these VCL examples] on how to make varnish cache cookied/logged in users sanely.
-
-
-**Why are regular expressions case-sensitive?**
-
-Some HTTP headers, such as ``Host:`` and ``Location:`` contain fully
-qualified domain names, which by definition is not case-sensitive.
-Other HTTP headers are case-sensitive, most notably the URLs.
-Therefore a "one size fits all" solution is not possible.
-
-In previous releases, we used the POSIX regular expressions
-supplied with the operating system, and decided, because the
-most common use of regexps were on ```Host:``` headers, that
-they should not be case-sensitive.
-
-From version 2.1.0 and forward, we use PCRE regular expressions,
-where it *is* possible to control case-sensitivity in the
-individual regular expressions, so we decided that it would
-probably confuse people if we made the default case-insensitive.
-(We promise not to change our minds about this again.)
-
-To make a PCRE regex case insensitive, put ``(?i)`` at the start::
-
-	if (req.http.host ~ "(?i)example.com$") {
-		...
-	}
-
-See the `PCRE man pages <http://www.pcre.org/pcre.txt>`_ for more information.
-
-**Are regular expressions case sensitive or not? Can I change it?**
-
-In 2.1 and newer, regular expressions are case sensitive by default.  In earlier versions, they were case insensitive.
-
-To change this for a single regex in 2.1, use ``(?i)`` at the start.
-
-See the `PCRE man pages <http://www.pcre.org/pcre.txt>`_ for more information.
-
-
-**Why does the ``Via:`` header say 1.1 in Varnish 2.1.x?**
-
-The number in the ``Via:`` header is the HTTP protocol version
-supported/applied, not the software's version number.
-
-**Why did you call it *Varnish*?**
-
-Long story, but basically the instigator of Varnish spent a long
-time staring at an art-poster with the word "Vernisage" and ended
-up checking it in a dictionary, which gives the following three
-meanings of the word:
-
-r.v. var?nished, var?nish?ing, var?nish?es
-
-	  1. To cover with varnish.
-	  2. To give a smooth and glossy finish to.
-	  3. To give a deceptively attractive appearance to; gloss over.
 
-The three point describes happens to your backend system when you
-put Varnish in front of it.
+Please see ref:`tutorial-increasing_your_hitrate`.
 
 **Why does Varnish require the system to have a C compiler?**
 
-The :ref:`VCL <faq-vcl>` compiler generates C source as output (your config file), and uses the systems C-compiler to compile that into a shared library.  If there is no C compiler, Varnish will not work.
+The :ref:`VCL <faq-vcl>` compiler generates C source as output (your
+config file), and uses the systems C-compiler to compile that into a
+shared library.  If there is no C compiler, Varnish will not work.
 
 **Isn't that security problem?**
 
-The days when you could prevent people from running non-approved programs by removing the C compiler from your system ended roughly with the VAX 11/780 computer.
+The days when you could prevent people from running non-approved
+programs by removing the C compiler from your system ended roughly
+with the VAX 11/780 computer.
 
 Troubleshooting
 ===============



From tfheen at varnish-cache.org  Mon Apr 16 08:20:38 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:38 +0200
Subject: [3.0] e052321 Literal block colons in fully minimized form
Message-ID: <E1SJhAs-00059Z-9c@project.varnish-software.com>

commit e05232186208c138932600537f88446e374c62e4
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Wed Jan 25 10:03:51 2012 +0100

    Literal block colons in fully minimized form

diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index a097248..668ceab 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -54,7 +54,7 @@ Syntax
 
 Commands are usually terminated with a newline. Long command can be
 entered using sh style *here documents*. The format of here-documents
-is:::
+is::
 
    << word
 	here document
@@ -233,7 +233,7 @@ file on the fly.
 
 Use the unix file permissions to control access to the file.
 
-An authenticated session looks like this:::
+An authenticated session looks like this::
 
    critter phk> telnet localhost 1234
    Trying ::1...
@@ -273,7 +273,7 @@ following byte sequence:
 
 and dumping the resulting digest in lower-case hex.
 
-In the above example, the secret file contained foo\n and thus:::
+In the above example, the secret file contained foo\n and thus::
 
    critter phk> cat > _
    ixslvvxrgkjptxmcgnnsdxsvdmvfympg
@@ -300,18 +300,18 @@ EXAMPLES
 ========
 
 Simple example: All requests where req.url exactly matches the string
-/news are banned from the cache:::
+/news are banned from the cache::
 
     req.url == "/news"
 
 Example: Ban all documents where the name does not end with ".ogg",
-and where the size of the object is greater than 10 megabytes:::
+and where the size of the object is greater than 10 megabytes::
 
     req.url !~ "\.ogg$" && obj.size > 10MB
 
 Example: Ban all documents where the serving host is "example.com"
 or "www.example.com", and where the Set-Cookie header received from
-the backend contains "USERID=1663":::
+the backend contains "USERID=1663"::
 
     req.http.host ~ "^(?i)(www\.)example.com$" && obj.set-cookie ~ "USERID=1663"
 
diff --git a/doc/sphinx/reference/varnishadm.rst b/doc/sphinx/reference/varnishadm.rst
index bfad196..3521474 100644
--- a/doc/sphinx/reference/varnishadm.rst
+++ b/doc/sphinx/reference/varnishadm.rst
@@ -66,7 +66,7 @@ zero if the command succeeded, and non-zero otherwise.
 EXAMPLES
 ========
 
-Some ways you can use varnishadm:::
+Some ways you can use varnishadm::
 
            varnishadm -T localhost:999 -S /var/db/secret vcl.use foo
            echo vcl.use foo | varnishadm -T localhost:999 -S /var/db/secret
diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index b786da9..b37abc3 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -141,11 +141,11 @@ The following log entry tags are currently defined:
 EXAMPLES
 ========
 
-The following command line simply copies all log entries to a log file:::
+The following command line simply copies all log entries to a log file::
 
     $ varnishlog -w /var/log/varnish.log
 
-The following command line reads that same log file and displays requests for the front page:::
+The following command line reads that same log file and displays requests for the front page::
 
     $ varnishlog -r /var/log/varnish.log -c -m 'RxURL:^/$'
 
diff --git a/doc/sphinx/reference/varnishtop.rst b/doc/sphinx/reference/varnishtop.rst
index 9b3ca9f..86e6cf6 100644
--- a/doc/sphinx/reference/varnishtop.rst
+++ b/doc/sphinx/reference/varnishtop.rst
@@ -79,12 +79,12 @@ EXAMPLES
 ========
 
 The following example displays a continuously updated list of the most
-frequently requested URLs:::
+frequently requested URLs::
 
   varnishtop -i RxURL
 
 The following example displays a continuously updated list of the most
-commonly used user agents:::
+commonly used user agents::
 
   varnishtop -i RxHeader -C -I ^User-Agent
 
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index d39128b..5d3030f 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -96,14 +96,14 @@ file as a quoted string.
 Backend declarations
 --------------------
 
-A backend declaration creates and initializes a named backend object:::
+A backend declaration creates and initializes a named backend object::
 
   backend www {
     .host = "www.example.com";
     .port = "http";
   }
 
-The backend object can later be used to select a backend at request time:::
+The backend object can later be used to select a backend at request time::
 
   if (req.http.host ~ "(?i)^(www.)?example.com$") {
     set req.backend = www;
@@ -118,7 +118,7 @@ backend connection, .first_byte_timeout for the time to wait for the
 first byte from the backend and .between_bytes_timeout for time to
 wait between each received byte.
 
-These can be set in the declaration like this:::
+These can be set in the declaration like this::
 
   backend www {
     .host = "www.example.com";
@@ -145,7 +145,7 @@ be used.
 There are several types of directors. The different director types
 use different algorithms to choose which backend to use.
 
-Configuring a director may look like this:::
+Configuring a director may look like this::
 
   director b2 random {
     .retries = 5;
@@ -312,7 +312,7 @@ Probes take the following parameters:
   Default is 2 seconds.
 
 A backend with a probe can be defined like this, together with the
-backend or director:::
+backend or director::
 
   backend www {
     .host = "www.example.com";
@@ -326,7 +326,7 @@ backend or director:::
     }
   }
 
-Or it can be defined separately and then referenced:::
+Or it can be defined separately and then referenced::
 
   probe healthcheck {
      .url = "/status.cgi";
@@ -361,7 +361,7 @@ ACLs
 ----
 
 An ACL declaration creates and initializes a named access control list
-which can later be used to match client addresses:::
+which can later be used to match client addresses::
 
   acl local {
     "localhost";         // myself
@@ -375,7 +375,7 @@ if it is preceded by a negation mark, it will reject any address it is
 compared to, which may not be what you intended.  If the entry is
 enclosed in parentheses, however, it will simply be ignored.
 
-To match an IP address against an ACL, simply use the match operator:::
+To match an IP address against an ACL, simply use the match operator::
 
   if (client.ip ~ local) {
     return (pipe);
@@ -390,7 +390,7 @@ PCRE(3) man page.
 
 To send flags to the PCRE engine, such as to turn on *case
 insensitivity* add the flag within parens following a question mark,
-like this:::
+like this::
 
   if (req.http.host ~ "(?i)example.com$") {
           ...
@@ -424,7 +424,7 @@ ban_url(regex)
 Subroutines
 ~~~~~~~~~~~
 
-A subroutine is used to group code for legibility or reusability:::
+A subroutine is used to group code for legibility or reusability::
   
   sub pipe_if_local {
     if (client.ip ~ local) {
@@ -640,7 +640,7 @@ appear in the source.
 The default versions distributed with Varnish will be implicitly
 concatenated as a last resort at the end.
 
-Example:::
+Example::
 
 	# in file "main.vcl"
 	include "backends.vcl";
@@ -885,7 +885,7 @@ resp.response
 resp.http.header
   The corresponding HTTP header.
 
-Values may be assigned to variables using the set keyword:::
+Values may be assigned to variables using the set keyword::
 
   sub vcl_recv {
     # Normalize the Host: header
@@ -894,7 +894,7 @@ Values may be assigned to variables using the set keyword:::
     }
   }
 
-HTTP headers can be removed entirely using the remove keyword:::
+HTTP headers can be removed entirely using the remove keyword::
 
   sub vcl_fetch {
     # Don't cache cookies
@@ -936,7 +936,7 @@ EXAMPLES
 
 The following code is the equivalent of the default configuration with
 the backend address set to "backend.example.com" and no backend port
-specified:::
+specified::
 
   backend default {
    .host = "backend.example.com";
@@ -948,7 +948,7 @@ specified:::
 
 The following example shows how to support multiple sites running on
 separate backends in the same Varnish instance, by selecting backends
-based on the request URL:::
+based on the request URL::
 
   backend www {
     .host = "www.example.com";
@@ -974,7 +974,7 @@ based on the request URL:::
 The following snippet demonstrates how to force a minimum TTL for
 all documents.  Note that this is not the same as setting the
 default_ttl run-time parameter, as that only affects document for
-which the backend did not specify a TTL:::
+which the backend did not specify a TTL::
   
   import std; # needed for std.log
 
@@ -986,7 +986,7 @@ which the backend did not specify a TTL:::
   }
 
 The following snippet demonstrates how to force Varnish to cache
-documents even when cookies are present:::
+documents even when cookies are present::
 
   sub vcl_recv {
     if (req.request == "GET" && req.http.cookie) {
@@ -1001,7 +1001,7 @@ documents even when cookies are present:::
   }
 
 The following code implements the HTTP PURGE method as used by Squid
-for object invalidation:::
+for object invalidation::
 
   acl purge {
     "localhost";



From tfheen at varnish-cache.org  Mon Apr 16 08:20:38 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:38 +0200
Subject: [3.0] 97ffae1 Rest of the literal block colons in fully minimized form
Message-ID: <E1SJhAs-00059m-C7@project.varnish-software.com>

commit 97ffae102537f98e8f0e535ed3e596d7b30183a4
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Wed Jan 25 12:12:42 2012 +0100

    Rest of the literal block colons in fully minimized form

diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index d725f81..1c6414f 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -116,7 +116,7 @@ Configuring and compiling
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Next, configuration: The configuration will need the dependencies
-above satisfied. Once that is taken care of:::
+above satisfied. Once that is taken care of::
 
 	cd varnish-cache
 	sh autogen.sh
diff --git a/doc/sphinx/tutorial/esi.rst b/doc/sphinx/tutorial/esi.rst
index 1d01cb8..1badd5d 100644
--- a/doc/sphinx/tutorial/esi.rst
+++ b/doc/sphinx/tutorial/esi.rst
@@ -26,7 +26,7 @@ Example: esi include
 ~~~~~~~~~~~~~~~~~~~~
 
 Lets see an example how this could be used. This simple cgi script
-outputs the date:::
+outputs the date::
 
      #!/bin/sh
      
@@ -34,7 +34,7 @@ outputs the date:::
      echo ''
      date "+%Y-%m-%d %H:%M"
 
-Now, lets have an HTML file that has an ESI include statement:::
+Now, lets have an HTML file that has an ESI include statement::
 
      <HTML>
      <BODY>
@@ -43,7 +43,7 @@ Now, lets have an HTML file that has an ESI include statement:::
      </BODY>
      </HTML>
 
-For ESI to work you need to activate ESI processing in VCL, like this:::
+For ESI to work you need to activate ESI processing in VCL, like this::
 
     sub vcl_fetch {
     	if (req.url == "/test.html") {
@@ -59,7 +59,7 @@ Example: esi remove
 ~~~~~~~~~~~~~~~~~~~
 
 The *remove* keyword allows you to remove output. You can use this to make
-a fall back of sorts, when ESI is not available, like this:::
+a fall back of sorts, when ESI is not available, like this::
 
   <esi:include src="http://www.example.com/ad.html"/> 
   <esi:remove> 
diff --git a/doc/sphinx/tutorial/handling_misbehaving_servers.rst b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
index 6cd7111..406b4b3 100644
--- a/doc/sphinx/tutorial/handling_misbehaving_servers.rst
+++ b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
@@ -26,7 +26,7 @@ requests somewhat stale content.
 
 So, in order to serve stale content we must first have some content to
 serve. So to make Varnish keep all objects for 30 minutes beyond their
-TTL use the following VCL:::
+TTL use the following VCL::
 
   sub vcl_fetch {
     set beresp.grace = 30m;
@@ -64,7 +64,7 @@ errors. You can instruct Varnish to try to handle this in a
 more-than-graceful way - enter *Saint mode*. Saint mode enables you to
 discard a certain page from one backend server and either try another
 server or serve stale content from cache. Lets have a look at how this
-can be enabled in VCL:::
+can be enabled in VCL::
 
   sub vcl_fetch {
     if (beresp.status == 500) { 
diff --git a/doc/sphinx/tutorial/troubleshooting.rst b/doc/sphinx/tutorial/troubleshooting.rst
index 7ab9435..5bbcf6c 100644
--- a/doc/sphinx/tutorial/troubleshooting.rst
+++ b/doc/sphinx/tutorial/troubleshooting.rst
@@ -55,7 +55,7 @@ Varnish is crashing
 
 When varnish goes bust the child processes crashes. Usually the mother
 process will manage this by restarting the child process again. Any
-errors will be logged in syslog. It might look like this:::
+errors will be logged in syslog. It might look like this::
 
        Mar  8 13:23:38 smoke varnishd[15670]: Child (15671) not responding to CLI, killing it.
        Mar  8 13:23:43 smoke varnishd[15670]: last message repeated 2 times
@@ -78,13 +78,13 @@ Varnish gives me Guru meditation
 First find the relevant log entries in varnishlog. That will probably
 give you a clue. Since varnishlog logs so much data it might be hard
 to track the entries down. You can set varnishlog to log all your 503
-errors by issuing the following command:::
+errors by issuing the following command::
 
    $ varnishlog -c -m TxStatus:503
 
 If the error happened just a short time ago the transaction might still
 be in the shared memory log segment. To get varnishlog to process the
-whole shared memory log just add the -d option:::
+whole shared memory log just add the -d option::
 
    $ varnishlog -d -c -m TxStatus:503
 
diff --git a/doc/sphinx/tutorial/vcl.rst b/doc/sphinx/tutorial/vcl.rst
index 3ed78c5..7171ab6 100644
--- a/doc/sphinx/tutorial/vcl.rst
+++ b/doc/sphinx/tutorial/vcl.rst
@@ -136,7 +136,7 @@ Example 1 - manipulating headers
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Lets say we want to remove the cookie for all objects in the /static
-directory of our web server:::
+directory of our web server::
 
   sub vcl_recv {
     if (req.url ~ "^/images") {
@@ -154,7 +154,7 @@ Example 2 - manipulating beresp
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Here we override the TTL of a object comming from the backend if it
-matches certain criteria:::
+matches certain criteria::
 
   sub vcl_fetch {
      if (req.url ~ "\.(png|gif|jpg)$") {



From tfheen at varnish-cache.org  Mon Apr 16 08:20:38 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:38 +0200
Subject: [3.0] a94b3a1 Wait a bit before trying to start/use varnishd again
Message-ID: <E1SJhAs-0005AI-G1@project.varnish-software.com>

commit a94b3a11fe922743bbb8ec230e52f15e20226d23
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Wed Jan 18 10:51:16 2012 +0100

    Wait a bit before trying to start/use varnishd again

diff --git a/bin/varnishtest/tests/p00007.vtc b/bin/varnishtest/tests/p00007.vtc
index 78aa8fb..7dc2e68 100644
--- a/bin/varnishtest/tests/p00007.vtc
+++ b/bin/varnishtest/tests/p00007.vtc
@@ -53,6 +53,8 @@ varnish v1 -cliok "debug.persistent s0 dump"
 # Panic worker so second segment does not get closed
 varnish v1 -clierr 400 "debug.panic.worker"
 
+delay 0.5
+
 # start again
 varnish v1 -start
 
diff --git a/bin/varnishtest/tests/v00010.vtc b/bin/varnishtest/tests/v00010.vtc
index 45dbc79..b44fc13 100644
--- a/bin/varnishtest/tests/v00010.vtc
+++ b/bin/varnishtest/tests/v00010.vtc
@@ -42,6 +42,8 @@ varnish v1 -cliok "start"
 varnish v1 -wait-running
 sema r1 sync 2
 
+delay 0.5
+
 client c1 {
 	txreq -url "/"
 	rxresp



From tfheen at varnish-cache.org  Mon Apr 16 08:20:38 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:38 +0200
Subject: [3.0] f7f8e8c Make the invalid domains FQDN to save time if you have
	a long search-list.
Message-ID: <E1SJhAs-00059x-En@project.varnish-software.com>

commit f7f8e8cb515ea7bcb0e6853adec9d7844de45dd5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 25 16:10:51 2012 +0000

    Make the invalid domains FQDN to save time if you have a long search-list.

diff --git a/bin/varnishtest/tests/v00002.vtc b/bin/varnishtest/tests/v00002.vtc
index 586cfac..d9d33d6 100644
--- a/bin/varnishtest/tests/v00002.vtc
+++ b/bin/varnishtest/tests/v00002.vtc
@@ -116,11 +116,11 @@ varnish v1 -badvcl {
 }
 
 varnish v1 -badvcl {
-	backend b1 { .host = "////"; }
+	backend b1 { .host = "////."; }
 }
 
 varnish v1 -badvcl {
-	backend b1 { .host = "127.0.0.1"; .port = "////"; }
+	backend b1 { .host = "127.0.0.1"; .port = "////."; }
 }
 
 varnish v1 -badvcl {



From tfheen at varnish-cache.org  Mon Apr 16 08:20:38 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:38 +0200
Subject: [3.0] 6387403 Make sure we have debug.sizeof in at least one
	test-case.
Message-ID: <E1SJhAs-0005Ac-Il@project.varnish-software.com>

commit 63874031576852805faefa464b4ac7c6578d80b4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 15 09:18:20 2012 +0000

    Make sure we have debug.sizeof in at least one test-case.

diff --git a/bin/varnishtest/tests/b00032.vtc b/bin/varnishtest/tests/b00032.vtc
index 949bcfa..ced05bb 100644
--- a/bin/varnishtest/tests/b00032.vtc
+++ b/bin/varnishtest/tests/b00032.vtc
@@ -1,3 +1,12 @@
 varnishtest "CLI coverage test"
 
 varnish v1 -cliok storage.list
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+varnish v1 -cliok debug.sizeof



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] 62403b5 Misleading use of key= instead of key: which is correct.
Message-ID: <E1SJhAr-00056a-6s@project.varnish-software.com>

commit 62403b5466a61994f7e67cc9bd6bd245821c6cc1
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Wed Dec 28 15:45:36 2011 +0100

    Misleading use of key= instead of key: which is correct.

diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 8499d29..91622d8 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -120,7 +120,7 @@ The following options are available:
 		     cache hit, miss, pass, pipe or error.
 	
 		   VCL_Log:key
-		     Output value set by std.log("key=value") in VCL.
+		     Output value set by std.log("key:value") in VCL.
 		     
 
 -m tag:regex only list records where tag matches regex. Multiple



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] 36b08cc Test more of the backend matching code.
Message-ID: <E1SJhAp-00051l-Gy@project.varnish-software.com>

commit 36b08cc66f6c5363a51c4c50b218fcf719342c1e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 7 12:27:24 2011 +0000

    Test more of the backend matching code.

diff --git a/bin/varnishtest/tests/c00048.vtc b/bin/varnishtest/tests/c00048.vtc
index bf1c3c9..4ddb888 100644
--- a/bin/varnishtest/tests/c00048.vtc
+++ b/bin/varnishtest/tests/c00048.vtc
@@ -59,4 +59,6 @@ varnish v1 -clierr 106 "backend.set_health s1 foo"
 varnish v1 -clierr 106 "backend.set_health s2 foo"
 varnish v1 -clierr 106 "backend.set_health s2 auto"
 varnish v1 -cliok "backend.list (foo)"
+varnish v1 -clierr 300 "backend.list ("
+varnish v1 -clierr 300 {backend.list " ( : ) -"}
 



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] 9dc49ae Clean up examples a bit (#2)
Message-ID: <E1SJhAq-00054r-IU@project.varnish-software.com>

commit 9dc49ae031dcc6a63fa6e42928bc3db2d2c69b52
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Wed Nov 30 15:05:31 2011 +0100

    Clean up examples a bit (#2)

diff --git a/doc/sphinx/tutorial/vary.rst b/doc/sphinx/tutorial/vary.rst
index a31e449..ad7b48d 100644
--- a/doc/sphinx/tutorial/vary.rst
+++ b/doc/sphinx/tutorial/vary.rst
@@ -23,7 +23,7 @@ And another one sends::
 Varnish will keep two variants of the page requested due to the
 different Accept-Encoding headers. Normalizing the accept-encoding
 header will sure that you have as few variants as possible. The
-following VCL code will normalize the Accept-Encoding headers.::
+following VCL code will normalize the Accept-Encoding headers::
 
     if (req.http.Accept-Encoding) {
         if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
@@ -34,7 +34,7 @@ following VCL code will normalize the Accept-Encoding headers.::
         } elsif (req.http.Accept-Encoding ~ "deflate") {
             set req.http.Accept-Encoding = "deflate";
         } else {
-            # unkown algorithm
+            # unknown algorithm
             remove req.http.Accept-Encoding;
         }
     }



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] b1401a8 Simplify json callback
Message-ID: <E1SJhAp-00052X-Nt@project.varnish-software.com>

commit b1401a81db2514e8d04a5f42c20da269c5e28c3b
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Nov 7 13:26:51 2011 +0100

    Simplify json callback

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index d487b54..f8b8ef0 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -92,8 +92,6 @@ do_json_cb(void *priv, const struct VSC_point * const pt)
 {
 	uint64_t val;
 	int *jp;
-	char jsonkey[255];
-	char jsontmp[255];
 
 	jp = priv;
 
@@ -101,18 +99,14 @@ do_json_cb(void *priv, const struct VSC_point * const pt)
 	val = *(const volatile uint64_t*)pt->ptr;
 
 	if (*jp) *jp = 0; else printf(",\n");
-	jsonkey[0] = '\0';
 
+	printf("\t\"");
 	/* build the JSON key name.  */
-	if (strcmp(pt->ident, "")  && strcmp(pt->class, ""))  sprintf(jsonkey, "%s.%s", pt->class, pt->ident);
-	if (strcmp(pt->ident, "")  && !strcmp(pt->class, "")) sprintf(jsonkey, "%s", pt->ident);
-	if (!strcmp(pt->ident, "") && strcmp(pt->class, ""))  sprintf(jsonkey, "%s", pt->class);
-
-	strcpy(jsontmp, jsonkey);
-	if (strcmp(jsonkey, "")) sprintf(jsonkey, "%s.%s", jsontmp, pt->name);
-	else strcpy(jsonkey, pt->name);
-
-	printf("\t\"%s\": {", jsonkey);
+	if (pt->class[0])
+		printf("%s.", pt->class);
+	if (pt->ident[0])
+		printf("%s.", pt->ident);
+	printf("%s\": {", pt->name);
 
 	if (strcmp(pt->class, "")) printf("\"type\": \"%s\", ",  pt->class);
 	if (strcmp(pt->ident, "")) printf("\"ident\": \"%s\", ", pt->ident);



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] d8f894b Remove a bit of stale debugging.
Message-ID: <E1SJhAp-0004zg-0K@project.varnish-software.com>

commit d8f894b5ce902edec57e4d5ce8e50b0a1d0aaf92
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 31 22:59:25 2011 +0000

    Remove a bit of stale debugging.
    
    Conflicts:
    
    	bin/varnishd/cache_fetch.c

diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 53acf25..cdc2fd8 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -95,11 +95,6 @@ static void __match_proto__()
 vfp_nop_begin(struct sess *sp, size_t estimate)
 {
 
-	if (fetchfrag > 0) {
-		estimate = fetchfrag;
-		WSL(sp->wrk, SLT_Debug, sp->fd,
-		    "Fetch %d byte segments:", fetchfrag);
-	}
 	if (estimate > 0)
 		(void)FetchStorage(sp, estimate);
 }



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] f281afc clean up refs
Message-ID: <E1SJhAr-000561-0X@project.varnish-software.com>

commit f281afcc9ea2f71405cf25163650f2121a7288c0
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Dec 19 12:29:33 2011 +0100

    clean up refs

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 5d1fc77..4e2b3f2 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -1,3 +1,5 @@
+.. _ref-varnishd:
+
 =========
  varnishd
 =========
diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index edd8fbc..b786da9 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -1,4 +1,4 @@
-_reference-varnishlog:
+.. _ref-varnishlog:
 
 ==========
 varnishlog



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] 052053b More cleanup and simplification of FetchError reporting.
Message-ID: <E1SJhAo-0004z4-RI@project.varnish-software.com>

commit 052053b1ead8ff3cc682972fbbaae88f28d39871
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 31 21:37:47 2011 +0000

    More cleanup and simplification of FetchError reporting.
    
    Conflicts:
    
    	bin/varnishd/cache_esi_fetch.c
    	bin/varnishd/cache_fetch.c
    	bin/varnishd/cache_gzip.c

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 4c228ba..7177f9f 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -203,7 +203,6 @@ struct http_conn {
 	struct ws		*ws;
 	txt			rxbuf;
 	txt			pipeline;
-	const char		*error;
 };
 
 /*--------------------------------------------------------------------*/
@@ -719,8 +718,8 @@ int EXP_NukeOne(struct worker *w, struct lru *lru);
 
 /* cache_fetch.c */
 struct storage *FetchStorage(const struct sess *sp, ssize_t sz);
-int FetchError(struct sess *sp, const char *error);
-int FetchError2(struct sess *sp, const char *error, const char *more);
+int FetchError(const struct sess *sp, const char *error);
+int FetchError2(const struct sess *sp, const char *error, const char *more);
 int FetchHdr(struct sess *sp);
 int FetchBody(struct sess *sp);
 int FetchReqBody(struct sess *sp);
@@ -798,7 +797,7 @@ void HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned maxbytes,
     unsigned maxhdr);
 int HTC_Reinit(struct http_conn *htc);
 int HTC_Rx(struct http_conn *htc);
-ssize_t HTC_Read(struct http_conn *htc, void *d, size_t len);
+ssize_t HTC_Read(struct worker *w, struct http_conn *htc, void *d, size_t len);
 int HTC_Complete(struct http_conn *htc);
 
 #define HTTPH(a, b, c, d, e, f, g) extern char b[];
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index cb02737..7e636c8 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -46,7 +46,8 @@
  */
 
 static ssize_t
-vef_read(struct http_conn *htc, void *buf, ssize_t buflen, ssize_t bytes)
+vef_read(struct worker *w, struct http_conn *htc, void *buf, ssize_t buflen,
+    ssize_t bytes)
 {
 	ssize_t d;
 
@@ -57,7 +58,7 @@ vef_read(struct http_conn *htc, void *buf, ssize_t buflen, ssize_t bytes)
 		if (d < bytes)
 			bytes = d;
 	}
-	return (HTC_Read(htc, buf, bytes));
+	return (HTC_Read(w, htc, buf, bytes));
 }
 
 /*---------------------------------------------------------------------
@@ -76,7 +77,7 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		st = FetchStorage(sp, 0);
 		if (st == NULL)
 			return (-1);
-		w = vef_read(htc,
+		w = vef_read(sp->wrk, htc,
 		    st->ptr + st->len, st->space - st->len, bytes);
 		if (w <= 0)
 			return (w);
@@ -107,14 +108,14 @@ vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 
 	while (bytes > 0) {
 		if (VGZ_IbufEmpty(vg) && bytes > 0) {
-			w = vef_read(htc, ibuf, sizeof ibuf, bytes);
+			w = vef_read(sp->wrk, htc, ibuf, sizeof ibuf, bytes);
 			if (w <= 0)
 				return (w);
 			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
 		if (VGZ_ObufStorage(sp, vg))
-			return(FetchError(sp, "Could not get storage"));
+			return(-1);
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		xxxassert(i == VGZ_OK || i == VGZ_END);
 		VEP_parse(sp, dp, dl);
@@ -182,7 +183,7 @@ vfp_vep_callback(const struct sess *sp, ssize_t l, enum vgz_flag flg)
 		}
 		do {
 			if (VGZ_ObufStorage(sp, vef->vgz)) {
-				vef->error = errno;
+				vef->error = ENOMEM;
 				vef->tot += l;
 				return (vef->tot);
 			}
@@ -216,7 +217,7 @@ vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 
 	while (bytes > 0) {
-		w = vef_read(htc, ibuf, sizeof ibuf, bytes);
+		w = vef_read(sp->wrk, htc, ibuf, sizeof ibuf, bytes);
 		if (w <= 0)
 			return (w);
 		bytes -= w;
@@ -259,7 +260,7 @@ vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
 	ibuf2[0] = 0; /* For Flexelint */
 
 	while (bytes > 0) {
-		w = vef_read(htc, ibuf, sizeof ibuf, bytes);
+		w = vef_read(sp->wrk, htc, ibuf, sizeof ibuf, bytes);
 		if (w <= 0)
 			return (w);
 		bytes -= w;
@@ -336,6 +337,7 @@ vfp_esi_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk->fetch_failed);
 	AN(sp->wrk->vep);
+	assert(sp->wrk->htc == htc);
 	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
 		i = vfp_esi_bytes_gu(sp, htc, bytes);
 	else if (sp->wrk->is_gunzip && sp->wrk->do_gzip)
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 14ec4f4..29d8b00 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -52,7 +52,7 @@ static unsigned fetchfrag;
  */
 
 int
-FetchError2(struct sess *sp, const char *error, const char *more)
+FetchError2(const struct sess *sp, const char *error, const char *more)
 {
 	struct worker *w;
 
@@ -71,7 +71,7 @@ FetchError2(struct sess *sp, const char *error, const char *more)
 }
 
 int
-FetchError(struct sess *sp, const char *error)
+FetchError(const struct sess *sp, const char *error)
 {
 	return(FetchError2(sp, error, NULL));
 }
@@ -125,15 +125,13 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	while (bytes > 0) {
 		st = FetchStorage(sp, 0);
 		if (st == NULL)
-			return(FetchError(sp, "Could not get storage"));
+			return(-1);
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
-		w = HTC_Read(htc, st->ptr + st->len, l);
-		if (w < 0)
-			return(FetchError(sp, htc->error));
-		if (w == 0)
-			return (w);
+		w = HTC_Read(sp->wrk, htc, st->ptr + st->len, l);
+		if (w <= 0)
+			return(w);
 		st->len += w;
 		sp->obj->len += w;
 		bytes -= w;
@@ -178,7 +176,8 @@ static struct vfp vfp_nop = {
 };
 
 /*--------------------------------------------------------------------
- * Fetch Storage
+ * Fetch Storage to put object into.
+ *
  */
 
 struct storage *
@@ -198,7 +197,7 @@ FetchStorage(const struct sess *sp, ssize_t sz)
 		l = params->fetch_chunksize * 1024LL;
 	st = STV_alloc(sp, l);
 	if (st == NULL) {
-		errno = ENOMEM;
+		(void)FetchError(sp, "Could not get storage");
 		return (NULL);
 	}
 	AZ(st->len);
@@ -250,17 +249,11 @@ fetch_straight(struct sess *sp, struct http_conn *htc, ssize_t cl)
 	return (0);
 }
 
-/*--------------------------------------------------------------------*/
-/* XXX: Cleanup.  It must be possible somehow :-( */
-
-#define CERR() do {						\
-		if (i != 1) {					\
-			WSP(sp, SLT_FetchError,			\
-			    "chunked read_error: %d (%s)",	\
-			    errno, htc->error);			\
-			return (-1);				\
-		}						\
-	} while (0)
+/*--------------------------------------------------------------------
+ * Read a chunked HTTP object.
+ * XXX: Reading one byte at a time is pretty pessimal.
+ */
+ 
 
 static int
 fetch_chunked(struct sess *sp, struct http_conn *htc)
@@ -274,15 +267,17 @@ fetch_chunked(struct sess *sp, struct http_conn *htc)
 	do {
 		/* Skip leading whitespace */
 		do {
-			i = HTC_Read(htc, buf, 1);
-			CERR();
+			i = HTC_Read(sp->wrk, htc, buf, 1);
+			if (i <= 0)
+				return (i);
 		} while (vct_islws(buf[0]));
 
 		/* Collect hex digits, skipping leading zeros */
 		for (u = 1; u < sizeof buf; u++) {
 			do {
-				i = HTC_Read(htc, buf + u, 1);
-				CERR();
+				i = HTC_Read(sp->wrk, htc, buf + u, 1);
+				if (i <= 0)
+					return (i);
 			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
 			if (!vct_ishex(buf[u]))
 				break;
@@ -294,8 +289,9 @@ fetch_chunked(struct sess *sp, struct http_conn *htc)
 
 		/* Skip trailing white space */
 		while(vct_islws(buf[u]) && buf[u] != '\n') {
-			i = HTC_Read(htc, buf + u, 1);
-			CERR();
+			i = HTC_Read(sp->wrk, htc, buf + u, 1);
+			if (i <= 0)
+				return (i);
 		}
 
 		if (buf[u] != '\n') {
@@ -308,13 +304,16 @@ fetch_chunked(struct sess *sp, struct http_conn *htc)
 			return (FetchError(sp,"chunked header number syntax"));
 		} else if (cl > 0) {
 			i = sp->wrk->vfp->bytes(sp, htc, cl);
-			CERR();
+			if (i <= 0)
+				return (-1);
 		}
-		i = HTC_Read(htc, buf, 1);
-		CERR();
+		i = HTC_Read(sp->wrk, htc, buf, 1);
+		if (i <= 0)
+			return (-1);
 		if (buf[0] == '\r') {
-			i = HTC_Read(htc, buf, 1);
-			CERR();
+			i = HTC_Read(sp->wrk, htc, buf, 1);
+			if (i <= 0)
+				return (-1);
 		}
 		if (buf[0] != '\n')
 			return (FetchError(sp,"chunked tail no NL"));
@@ -362,7 +361,7 @@ FetchReqBody(struct sess *sp)
 				rdcnt = sizeof buf;
 			else
 				rdcnt = content_length;
-			rdcnt = HTC_Read(sp->htc, buf, rdcnt);
+			rdcnt = HTC_Read(sp->wrk, sp->htc, buf, rdcnt);
 			if (rdcnt <= 0)
 				return (1);
 			content_length -= rdcnt;
@@ -463,7 +462,7 @@ FetchHdr(struct sess *sp)
 
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "http first read error: %d %d (%s)",
-		    i, errno, w->htc->error);
+		    i, errno, strerror(errno));
 		VDI_CloseFd(sp);
 		/* XXX: other cleanup ? */
 		/* Retryable if we never received anything */
@@ -477,7 +476,7 @@ FetchHdr(struct sess *sp)
 		if (i < 0) {
 			WSP(sp, SLT_FetchError,
 			    "http first read error: %d %d (%s)",
-			    i, errno, w->htc->error);
+			    i, errno, strerror(errno));
 			VDI_CloseFd(sp);
 			/* XXX: other cleanup ? */
 			return (-1);
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 0bd158c..3087a6b 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -474,17 +474,15 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			l = sizeof ibuf;
 			if (l > bytes)
 				l = bytes;
-			w = HTC_Read(htc, ibuf, l);
-			if (w < 0)
-				return(FetchError(sp, htc->error));
-			if (w == 0)
+			w = HTC_Read(sp->wrk, htc, ibuf, l);
+			if (w <= 0)
 				return (w);
 			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
 
 		if (VGZ_ObufStorage(sp, vg))
-			return(FetchError(sp, "Could not get storage"));
+			return(-1);
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		if (i != VGZ_OK && i != VGZ_END)
 			return(FetchError(sp, "Gunzip data error"));
@@ -554,16 +552,14 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			l = sizeof ibuf;
 			if (l > bytes)
 				l = bytes;
-			w = HTC_Read(htc, ibuf, l);
-			if (w < 0)
-				return(FetchError(sp, htc->error));
-			if (w == 0)
+			w = HTC_Read(sp->wrk, htc, ibuf, l);
+			if (w <= 0)
 				return (w);
 			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
 		if (VGZ_ObufStorage(sp, vg))
-			return(FetchError(sp, "Could not get storage"));
+			return(-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
 		sp->obj->len += dl;
@@ -592,7 +588,7 @@ vfp_gzip_end(struct sess *sp)
 	do {
 		VGZ_Ibuf(vg, "", 0);
 		if (VGZ_ObufStorage(sp, vg))
-			return(FetchError(sp, "Could not get storage"));
+			return(-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
 		sp->obj->len += dl;
 	} while (i != Z_STREAM_END);
@@ -643,14 +639,12 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	while (bytes > 0) {
 		st = FetchStorage(sp, 0);
 		if (st == NULL)
-			return(FetchError(sp, "Could not get storage"));
+			return(-1);
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
-		w = HTC_Read(htc, st->ptr + st->len, l);
-		if (w < 0)
-			return(FetchError(sp, htc->error));
-		if (w == 0)
+		w = HTC_Read(sp->wrk, htc, st->ptr + st->len, l);
+		if (w <= 0)
 			return (w);
 		bytes -= w;
 		VGZ_Ibuf(vg, st->ptr + st->len, w);
diff --git a/bin/varnishd/cache_httpconn.c b/bin/varnishd/cache_httpconn.c
index 9d6d926..f185520 100644
--- a/bin/varnishd/cache_httpconn.c
+++ b/bin/varnishd/cache_httpconn.c
@@ -27,6 +27,17 @@
  * SUCH DAMAGE.
  *
  * HTTP protocol requests
+ *
+ * The trouble with the "until magic sequence" design of HTTP protocol messages
+ * is that either you have to read a single character at a time, which is
+ * inefficient, or you risk reading too much, and pre-read some of the object,
+ * or even the next pipelined request, which follows the one you want.
+ *
+ * HTC reads a HTTP protocol header into a workspace, subject to limits,
+ * and stops when we see the magic marker (double [CR]NL), and if we overshoot,
+ * it keeps track of the "pipelined" data.
+ *
+ * We use this both for client and backend connections.
  */
 
 #include "config.h"
@@ -57,7 +68,8 @@ htc_header_complete(txt *t)
 	/* Skip any leading white space */
 	for (p = t->b ; isspace(*p); p++)
 		continue;
-	if (*p == '\0') {
+	if (p == t->e) {
+		/* All white space */
 		t->e = t->b;
 		*t->e = '\0';
 		return (0);
@@ -88,7 +100,6 @@ HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned maxbytes,
 	htc->fd = fd;
 	htc->maxbytes = maxbytes;
 	htc->maxhdr = maxhdr;
-	htc->error = "No error recorded";
 
 	(void)WS_Reserve(htc->ws, htc->maxbytes);
 	htc->rxbuf.b = ws->f;
@@ -110,7 +121,6 @@ HTC_Reinit(struct http_conn *htc)
 	unsigned l;
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-	htc->error = "No error recorded";
 	(void)WS_Reserve(htc->ws, htc->maxbytes);
 	htc->rxbuf.b = htc->ws->f;
 	htc->rxbuf.e = htc->ws->f;
@@ -126,7 +136,7 @@ HTC_Reinit(struct http_conn *htc)
 }
 
 /*--------------------------------------------------------------------
- *
+ * Return 1 if we have a complete HTTP procol header
  */
 
 int
@@ -140,6 +150,8 @@ HTC_Complete(struct http_conn *htc)
 	if (i == 0)
 		return (0);
 	WS_ReleaseP(htc->ws, htc->rxbuf.e);
+	AZ(htc->pipeline.b);
+	AZ(htc->pipeline.e);
 	if (htc->rxbuf.b + i < htc->rxbuf.e) {
 		htc->pipeline.b = htc->rxbuf.b + i;
 		htc->pipeline.e = htc->rxbuf.e;
@@ -171,6 +183,10 @@ HTC_Rx(struct http_conn *htc)
 	}
 	i = read(htc->fd, htc->rxbuf.e, i);
 	if (i <= 0) {
+		/*
+		 * We wouldn't come here if we had a complete HTTP header
+		 * so consequently an EOF can not be OK
+		 */
 		WS_ReleaseP(htc->ws, htc->rxbuf.b);
 		return (-1);
 	}
@@ -179,16 +195,21 @@ HTC_Rx(struct http_conn *htc)
 	return (HTC_Complete(htc));
 }
 
+/*--------------------------------------------------------------------
+ * Read up to len bytes, returning pipelined data first.
+ */
+
 ssize_t
-HTC_Read(struct http_conn *htc, void *d, size_t len)
+HTC_Read(struct worker *w, struct http_conn *htc, void *d, size_t len)
 {
 	size_t l;
 	unsigned char *p;
 	ssize_t i;
 
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	l = 0;
 	p = d;
-	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	if (htc->pipeline.b) {
 		l = Tlen(htc->pipeline);
 		if (l > len)
@@ -204,9 +225,8 @@ HTC_Read(struct http_conn *htc, void *d, size_t len)
 		return (l);
 	i = read(htc->fd, p, len);
 	if (i < 0) {
-		htc->error = strerror(errno);
+		WSL(w, SLT_FetchError, htc->fd, "%s", strerror(errno));
 		return (i);
-	} else if (i == 0)
-		htc->error = "Remote closed connection";
+	}
 	return (i + l);
 }



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] 2a3a532 typos
Message-ID: <E1SJhAr-00056M-4h@project.varnish-software.com>

commit 2a3a532fed6a41c652f3367d1bfee042a4a0231b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 20 20:28:50 2011 +0000

    typos

diff --git a/doc/sphinx/phk/thetoolsweworkwith.rst b/doc/sphinx/phk/thetoolsweworkwith.rst
index cff365b..6921926 100644
--- a/doc/sphinx/phk/thetoolsweworkwith.rst
+++ b/doc/sphinx/phk/thetoolsweworkwith.rst
@@ -65,7 +65,7 @@ text actually means:
 	an uppercase letter or another underscore are always reserved
 	for any use.
 
-Feel free to guess.
+Feel free to guess, there's more such on pdf page 200 of the draft.
 
 Next, they broke the upper/lower rule, by adding special keywords
 in mixed case, probably because they thought it looked nicer::
@@ -103,8 +103,8 @@ aspects.
 
 For instance, neither pthreads nor C1X-threads offer a "assert I'm
 holding this mutex locked" facility.  I will posit that you cannot
-successfully develop real-world threaded programs and APIs that,
-or without wasting a lot of time debugging silly mistakes.
+successfully develop real-world threaded programs and APIs without
+that, or without wasting a lot of time debugging silly mistakes.
 
 If you look in the Varnish source code, which uses pthreads, you
 will see that I have wrapped pthread mutexes in my own little



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] 9fdaf89 s/reponse/response/
Message-ID: <E1SJhAq-000554-LR@project.varnish-software.com>

commit 9fdaf898e06623799a36e9a3a6fee159f6b4f319
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Wed Nov 30 15:23:37 2011 +0100

    s/reponse/response/

diff --git a/doc/sphinx/tutorial/increasing_your_hitrate.rst b/doc/sphinx/tutorial/increasing_your_hitrate.rst
index 437461a..31e32d0 100644
--- a/doc/sphinx/tutorial/increasing_your_hitrate.rst
+++ b/doc/sphinx/tutorial/increasing_your_hitrate.rst
@@ -94,7 +94,7 @@ googling "Live HTTP Headers".
 The role of HTTP Headers
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-Along with each HTTP request and reponse comes a bunch of headers
+Along with each HTTP request and response comes a bunch of headers
 carrying metadata. Varnish will look at these headers to determine if
 it is appropriate to cache the contents and how long Varnish can keep
 the content.



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] 237da8d Add a label, fix a typo and remove -o (now default)
Message-ID: <E1SJhAq-00055U-QZ@project.varnish-software.com>

commit 237da8ddae8907f5952c716ed8b3b3dc37c8ae0d
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Dec 19 11:25:39 2011 +0100

    Add a label, fix a typo and remove -o (now default)

diff --git a/doc/sphinx/tutorial/logging.rst b/doc/sphinx/tutorial/logging.rst
index 070dc3d..331742f 100644
--- a/doc/sphinx/tutorial/logging.rst
+++ b/doc/sphinx/tutorial/logging.rst
@@ -1,3 +1,4 @@
+.. _tutorial-logging:
 
 Logging in Varnish
 ------------------
@@ -8,7 +9,7 @@ memory segment. When the end of the segment is reached we start over,
 overwriting old data. This is much, much faster then logging to a file
 and it doesn't require disk space.
 
-The flip side is that is you forget to have program actually write the
+The flip side is that if you forget to have program actually write the
 logs to disk they will disappear.
 
 varnishlog is one of the programs you can use to look at what Varnish
@@ -69,9 +70,6 @@ want to know are:
  show all cookie headers coming from the clients:
  ``$ varnishlog -c -i RxHeader -I Cookie``
 
--o 
- Group log entries by request ID.
-
 
 Now that Varnish seem to work OK it's time to put Varnish on port 80
 while we tune it.



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] 6a0e8fd A minor preemptive cleanup before the ban-lurker gets
	remodelled.
Message-ID: <E1SJhAp-00052p-Qb@project.varnish-software.com>

commit 6a0e8fd44b6cdb89381daeb5c231148726aac0c6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 8 10:03:43 2011 +0000

    A minor preemptive cleanup before the ban-lurker gets remodelled.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 7177f9f..04ff2d0 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -437,6 +437,7 @@ struct objcore {
 #define OC_F_PASS		(1<<2)
 #define OC_F_LRUDONTMOVE	(1<<4)
 #define OC_F_PRIV		(1<<5)		/* Stevedore private flag */
+#define OC_F_LURK		(3<<6)		/* Ban-lurker-color */
 	unsigned		timer_idx;
 	VTAILQ_ENTRY(objcore)	list;
 	VTAILQ_ENTRY(objcore)	lru_list;
diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index 7fbeffd..3a6ac36 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -65,6 +65,7 @@ struct ban {
 	unsigned		flags;
 #define BAN_F_GONE		(1 << 0)
 #define BAN_F_REQ		(1 << 2)
+#define BAN_F_LURK		(3 << 3)	/* ban-lurker-color */
 	VTAILQ_HEAD(,objcore)	objcore;
 	struct vsb		*vsb;
 	uint8_t			*spec;
@@ -148,23 +149,6 @@ BAN_Free(struct ban *b)
 	FREE_OBJ(b);
 }
 
-static struct ban *
-ban_CheckLast(void)
-{
-	struct ban *b;
-
-	Lck_AssertHeld(&ban_mtx);
-	b = VTAILQ_LAST(&ban_head, banhead_s);
-	if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
-		VSC_C_main->n_ban--;
-		VSC_C_main->n_ban_retire++;
-		VTAILQ_REMOVE(&ban_head, b, list);
-	} else {
-		b = NULL;
-	}
-	return (b);
-}
-
 /*--------------------------------------------------------------------
  * Get & Release a tail reference, used to hold the list stable for
  * traversals etc.
@@ -457,7 +441,6 @@ BAN_NewObjCore(struct objcore *oc)
 void
 BAN_DestroyObj(struct objcore *oc)
 {
-	struct ban *b;
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	if (oc->ban == NULL)
@@ -468,12 +451,7 @@ BAN_DestroyObj(struct objcore *oc)
 	oc->ban->refcount--;
 	VTAILQ_REMOVE(&oc->ban->objcore, oc, ban_list);
 	oc->ban = NULL;
-
-	/* Attempt to purge last ban entry */
-	b = ban_CheckLast();
 	Lck_Unlock(&ban_mtx);
-	if (b != NULL)
-		BAN_Free(b);
 }
 
 /*--------------------------------------------------------------------
@@ -734,6 +712,23 @@ BAN_CheckObject(struct object *o, const struct sess *sp)
 	return (ban_check_object(o, sp, 1));
 }
 
+static struct ban *
+ban_CheckLast(void)
+{
+	struct ban *b;
+
+	Lck_AssertHeld(&ban_mtx);
+	b = VTAILQ_LAST(&ban_head, banhead_s);
+	if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
+		VSC_C_main->n_ban--;
+		VSC_C_main->n_ban_retire++;
+		VTAILQ_REMOVE(&ban_head, b, list);
+	} else {
+		b = NULL;
+	}
+	return (b);
+}
+
 /*--------------------------------------------------------------------
  * Ban tail lurker thread
  */
@@ -747,9 +742,6 @@ ban_lurker_work(const struct sess *sp)
 	struct object *o;
 	int i;
 
-	WSL_Flush(sp->wrk, 0);
-	WRK_SumStat(sp->wrk);
-
 	Lck_Lock(&ban_mtx);
 
 	/* First try to route the last ban */
@@ -815,21 +807,34 @@ ban_lurker_work(const struct sess *sp)
 static void * __match_proto__(bgthread_t)
 ban_lurker(struct sess *sp, void *priv)
 {
+	struct ban *bf;
+
 	int i = 0;
 	(void)priv;
 	while (1) {
-		if (params->ban_lurker_sleep == 0.0) {
-			/* Lurker is disabled.  */
-			TIM_sleep(1.0);
-			continue;
+
+		while (params->ban_lurker_sleep == 0.0) {
+			/*
+			 * Ban-lurker is disabled:
+			 * Clean the last ban, if possible, and sleep
+			 */
+			Lck_Lock(&ban_mtx);
+			bf = ban_CheckLast();
+			Lck_Unlock(&ban_mtx);
+			if (bf != NULL)
+				BAN_Free(bf);
+			else
+				TIM_sleep(1.0);
 		}
+
+		i = ban_lurker_work(sp);
+		WSL_Flush(sp->wrk, 0);
+		WRK_SumStat(sp->wrk);
+
 		if (i != 0)
 			TIM_sleep(params->ban_lurker_sleep);
 		else
 			TIM_sleep(1.0);
-		i = ban_lurker_work(sp);
-		WSL_Flush(sp->wrk, 0);
-		WRK_SumStat(sp->wrk);
 	}
 	NEEDLESS_RETURN(NULL);
 }
@@ -941,10 +946,11 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 	/* Get a reference so we are safe to traverse the list */
 	bl = BAN_TailRef();
 
+	VCLI_Out(cli, "Present bans:\n");
 	VTAILQ_FOREACH(b, &ban_head, list) {
 		if (b == bl)
 			break;
-		VCLI_Out(cli, "%p %10.6f %5u%s\t", b, ban_time(b->spec),
+		VCLI_Out(cli, "%10.6f %5u%s\t", ban_time(b->spec),
 		    bl == b ? b->refcount - 1 : b->refcount,
 		    b->flags & BAN_F_GONE ? "G" : " ");
 		ban_render(cli, b->spec);



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] 479107f All relevant BSD's have a testable kqueue now.
Message-ID: <E1SJhAr-00056o-90@project.varnish-software.com>

commit 479107f336268e4e538956d4ae0990e764c4e561
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 2 12:10:57 2012 +0000

    All relevant BSD's have a testable kqueue now.

diff --git a/configure.ac b/configure.ac
index 1938c1f..05397fd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -340,16 +340,7 @@ AC_ARG_ENABLE(kqueue,
     [enable_kqueue=yes])
 
 if test "$enable_kqueue" = yes; then
-	case $target in
-	*-*-freebsd* | *-*-darwin9* | *-*-netbsd* )
-		AC_CHECK_FUNCS([kqueue])
-		;;
-	*-*-bsd*)
-		# No other BSD has a sufficiently recent implementation
-		AC_MSG_WARN([won't look for kqueue() on $target])
-		ac_cv_func_kqueue=no
-		;;
-	esac
+	AC_CHECK_FUNCS([kqueue])
 else
 	ac_cv_func_kqueue=no
 fi



From tfheen at varnish-cache.org  Mon Apr 16 08:20:38 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:38 +0200
Subject: [3.0] bdc88fb Honor remove-flag also when processing comments in ESI
	parsing.
Message-ID: <E1SJhAs-0005AQ-Hc@project.varnish-software.com>

commit bdc88fbb082022c56ab1aa475474a33cb9552e25
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 14 15:02:32 2012 +0100

    Honor remove-flag also when processing comments in ESI parsing.
    
    Fixes: #1092

diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index 3f3ab2b..ae45008 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -705,7 +705,6 @@ VEP_parse(const struct sess *sp, const char *p, size_t l)
 					vep->until_p = vep->until = "-->";
 					vep->until_s = VEP_NEXTTAG;
 					vep->state = VEP_UNTIL;
-					vep_mark_verbatim(vep, p);
 					break;
 				}
 				p++;
diff --git a/bin/varnishtest/tests/r01092.vtc b/bin/varnishtest/tests/r01092.vtc
new file mode 100644
index 0000000..26b0b9c
--- /dev/null
+++ b/bin/varnishtest/tests/r01092.vtc
@@ -0,0 +1,34 @@
+varnishtest "Test case for #1092 - esi:remove and comments"
+
+server s1 {
+	rxreq
+	txresp -body {
+		<html>
+		Keep-1
+		<!--esX Keep-22 -->
+		<!--esi Keep-333 -->
+		<esi:remove>
+		Remove-1
+		<!-- Remove-22 -->
+		<!--esi Remove-333 -->
+		Remove-4444
+		</esi:remove>
+		Keep-4444
+		</html>
+	}
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_esi = true;
+	}
+} -start -cliok "param.set esi_syntax 4"
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.bodylen == 80
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 1



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] 1259255 FlexeLint spotted a memory leak in Tollefs
	"admin-health" code, so I started fixing that,
	and then I fixed another couple of nits and polished the naming a bit
	and ...
Message-ID: <E1SJhAp-00051G-DF@project.varnish-software.com>

commit 1259255959e01744b7c3e93b711fbb0a1a9e909f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 7 11:06:42 2011 +0000

    FlexeLint spotted a memory leak in Tollefs "admin-health" code, so I
    started fixing that, and then I fixed another couple of nits and polished
    the naming a bit and ...
    
    Well, you know how it is.
    
    Changed the backend matching function to take a call-back function
    to eliminate the need for the memory which got leaked.
    
    This meant that backend.list could also use it, if only the parsing
    code accepted a blank input.
    
    So I rewrote the parsing code to also allow empty names, so you can:
    	backend.list (:80)
    	backend.list (192.168)
    etc.
    
    And then I added a summary report of the probing, if it is happening,
    and explicitly say that no probing is happening if it isn't.

diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index b296761..756d659 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -259,10 +259,10 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp)
 	backend = vs->backend;
 	CHECK_OBJ_NOTNULL(backend, BACKEND_MAGIC);
 
-	if (backend->admin_health == from_probe && !backend->healthy)
+	if (backend->admin_health == ah_probe && !backend->healthy)
 		return (0);
 
-	if (backend->admin_health == sick)
+	if (backend->admin_health == ah_sick)
 		return (0);
 
 	/* VRT/VCC sets threshold to UINT_MAX to mark that it's not
@@ -273,7 +273,7 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp)
 	else
 		threshold = vs->vrt->saintmode_threshold;
 
-	if (backend->admin_health == healthy)
+	if (backend->admin_health == ah_healthy)
 		threshold = UINT_MAX;
 
 	/* Saintmode is disabled */
diff --git a/bin/varnishd/cache_backend.h b/bin/varnishd/cache_backend.h
index 25148ac..059b4e7 100644
--- a/bin/varnishd/cache_backend.h
+++ b/bin/varnishd/cache_backend.h
@@ -106,10 +106,11 @@ struct trouble {
  * An instance of a backend from a VCL program.
  */
 
-enum health_status {
-	healthy,
-	sick,
-	from_probe
+enum admin_health {
+	ah_invalid = 0,
+	ah_healthy,
+	ah_sick,
+	ah_probe
 };
 
 struct backend {
@@ -135,7 +136,7 @@ struct backend {
 
 	struct vbp_target	*probe;
 	unsigned		healthy;
-	enum health_status	admin_health;
+	enum admin_health	admin_health;
 	VTAILQ_HEAD(, trouble)	troublelist;
 
 	struct VSC_C_vbe	*vsc;
@@ -155,6 +156,7 @@ void VBE_DropRefLocked(struct backend *b);
 void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, const char *hosthdr);
 void VBP_Remove(struct backend *b, struct vrt_backend_probe const *p);
 void VBP_Use(const struct backend *b, const struct vrt_backend_probe const *p);
+void VBP_Summary(struct cli *cli, const struct vbp_target *vt);
 
 /* Init functions for directors */
 typedef void dir_init_f(struct cli *, struct director **, int , const void*);
diff --git a/bin/varnishd/cache_backend_cfg.c b/bin/varnishd/cache_backend_cfg.c
index 215a4dc..955ae26 100644
--- a/bin/varnishd/cache_backend_cfg.c
+++ b/bin/varnishd/cache_backend_cfg.c
@@ -32,6 +32,7 @@
 
 #include "config.h"
 
+#include <ctype.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -48,6 +49,7 @@
 
 struct lock VBE_mtx;
 
+
 /*
  * The list of backends is not locked, it is only ever accessed from
  * the CLI thread, so there is no need.
@@ -83,6 +85,11 @@ VBE_Poll(void)
 
 	ASSERT_CLI();
 	VTAILQ_FOREACH_SAFE(b, &backends, list, b2) {
+		assert(
+			b->admin_health == ah_healthy ||
+			b->admin_health == ah_sick ||
+			b->admin_health == ah_probe
+		);
 		if (b->refcount == 0 && b->probe == NULL)
 			VBE_Nuke(b);
 	}
@@ -233,7 +240,7 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 	assert(b->ipv4 != NULL || b->ipv6 != NULL);
 
 	b->healthy = 1;
-	b->admin_health = from_probe;
+	b->admin_health = ah_probe;
 
 	VTAILQ_INSERT_TAIL(&backends, b, list);
 	VSC_C_main->n_backend++;
@@ -277,172 +284,227 @@ VRT_fini_dir(struct cli *cli, struct director *b)
 	b->priv = NULL;
 }
 
-/*--------------------------------------------------------------------*/
+/*---------------------------------------------------------------------
+ * String to admin_health
+ */
+
+static enum admin_health
+vbe_str2adminhealth(const char *wstate)
+{
+
+	if (strcasecmp(wstate, "healthy") == 0)
+		return(ah_healthy);
+	if (strcasecmp(wstate, "sick") == 0)
+		return(ah_sick);
+	if (strcmp(wstate, "auto") == 0)
+		return(ah_probe);
+	return (ah_invalid);
+}
+
+/*---------------------------------------------------------------------
+ * A general function for finding backends and doing things with them.
+ *
+ * Return -1 on match-argument parse errors.
+ *
+ * If the call-back function returns non-zero, the search is terminated
+ * and we relay that return value.
+ *
+ * Otherwise we return the number of matches.
+ */
+
+typedef int bf_func(struct cli *cli, struct backend *b, void *priv);
 
 static int
-backend_find(const char *matcher, struct backend **r, int n)
+backend_find(struct cli *cli, const char *matcher, bf_func *func, void *priv)
 {
 	struct backend *b;
-	char *vcl_name;
-	char *s;
-	char *match_ip = NULL;
-	char *match_port = NULL;
+	const char *s;
+	const char *name_b;
+	ssize_t name_l = 0;
+	const char *ip_b = NULL;
+	ssize_t ip_l = 0;
+	const char *port_b = NULL;
+	ssize_t port_l = 0;
 	int found = 0;
+	int i;
 
-	s = strchr(matcher, '(');
+	name_b = matcher;
+	if (matcher != NULL) {
+		s = strchr(matcher,'(');
 
-	if (s == NULL) {
-		/* Simple match, max one hit */
-		VTAILQ_FOREACH(b, &backends, list) {
-			CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-			if (strcmp(b->vcl_name, matcher) == 0) {
-				if (r && found < n)
-					r[found] = b;
-				found++;
-			}
-		}
-		return found;
-	}
+		if (s != NULL)
+			name_l = s - name_b;
+		else
+			name_l = strlen(name_b);
 
-	vcl_name = strndup(matcher, s - matcher);
-	AN(vcl_name);
-	s++;
-	while (*s != ')') {
-		if (*s == ':') {
-			/* Port */
+		if (s != NULL) {
 			s++;
-			match_port = s;
-			if (!(s = strchr(match_port, ','))) {
-				s = strchr(match_port, ')');
+			while (isspace(*s))
+				s++;
+			ip_b = s;
+			while (*s != '\0' &&
+			    *s != ')' &&
+			    *s != ':' &&
+			    !isspace(*s))
+				s++;
+			ip_l = s - ip_b;
+			while (isspace(*s))
+				s++;
+			if (*s == ':') {
+				s++;
+				while (isspace(*s))
+					s++;
+				port_b = s;
+				while (*s != '\0' && *s != ')' && !isspace(*s))
+					s++;
+				port_l = s - port_b;
 			}
-			XXXAN(s);
-			match_port = strndup(match_port, s - match_port);
-			AN(match_port);
-			if (*s == ',')
+			while (isspace(*s))
 				s++;
-		} else {
-			/* IP */
-			match_ip = s;
-			if (!(s = strchr(match_ip, ','))) {
-				s = strchr(match_ip, ')');
+			if (*s != ')') {
+				VCLI_Out(cli,
+				    "Match string syntax error:"
+				    " ')' not found.");
+				VCLI_SetResult(cli, CLIS_CANT);
+				return (-1);
 			}
-			XXXAN(s);
-			match_ip = strndup(match_ip, s - match_ip);
-			AN(match_ip);
-			if (*s == ',')
+			s++;
+			while (isspace(*s))
 				s++;
+			if (*s != '\0') {
+				VCLI_Out(cli,
+				    "Match string syntax error:"
+				    " junk after ')'");
+				VCLI_SetResult(cli, CLIS_CANT);
+				return (-1);
+			}
 		}
 	}
 	VTAILQ_FOREACH(b, &backends, list) {
 		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-		if (match_port && strcmp(b->port, match_port) != 0)
+		if (port_b != NULL && strncmp(b->port, port_b, port_l) != 0)
 			continue;
-		if (match_ip &&
-		    (strcmp(b->ipv4_addr, match_ip) != 0) &&
-		    (strcmp(b->ipv6_addr, match_ip) != 0))
+		if (name_b != NULL && strncmp(b->vcl_name, name_b, name_l) != 0)
 			continue;
-		if (strcmp(b->vcl_name, vcl_name) == 0) {
-			if (r && found < n)
-				r[found] = b;
-			found++;
-		}
+		if (ip_b != NULL &&
+		    (b->ipv4_addr == NULL ||
+		      strncmp(b->ipv4_addr, ip_b, ip_l)) &&
+		    (b->ipv6_addr == NULL ||
+		      strncmp(b->ipv6_addr, ip_b, ip_l)))
+			continue;
+		found++;
+		i = func(cli, b, priv);
+		if (i)
+			return(i);
+	}
+	return (found);
+}
+
+/*---------------------------------------------------------------------*/
+
+static int __match_proto__()
+do_list(struct cli *cli, struct backend *b, void *priv)
+{
+	int *hdr;
+	char buf[128];
+
+	AN(priv);
+	hdr = priv;
+	if (!*hdr) {
+		VCLI_Out(cli, "%-30s %-6s %-10s %s",
+		    "Backend name", "Refs", "Admin", "Probe");
+		*hdr = 1;
+	}
+	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+
+	bprintf(buf, "%s(%s,%s,%s)",
+		b->vcl_name,
+		b->ipv4_addr == NULL ? "" : b->ipv4_addr,
+		b->ipv6_addr == NULL ? "" : b->ipv6_addr, b->port);
+	VCLI_Out(cli, "\n%-30s %-6d", buf, b->refcount);
+
+	if (b->admin_health == ah_probe)
+		VCLI_Out(cli, " %-10s", "probe");
+	else if (b->admin_health == ah_sick)
+		VCLI_Out(cli, " %-10s", "sick");
+	else if (b->admin_health == ah_healthy)
+		VCLI_Out(cli, " %-10s", "healthy");
+	else
+		VCLI_Out(cli, " %-10s", "invalid");
+
+	if (b->probe == NULL)
+		VCLI_Out(cli, " %s", "Healthy (no probe)");
+	else {
+		if (b->healthy)
+			VCLI_Out(cli, " %s", "Healthy ");
+		else
+			VCLI_Out(cli, " %s", "Sick ");
+		VBP_Summary(cli, b->probe);
 	}
-	return found;
+
+	return (0);
 }
 
 static void
 cli_backend_list(struct cli *cli, const char * const *av, void *priv)
 {
-	struct backend *b;
-	const char *ah;
+	int hdr = 0;
 
 	(void)av;
 	(void)priv;
 	ASSERT_CLI();
-	VCLI_Out(cli, "%-30s %10s %15s %15s", "Backend name",
-		 "Conns", "Probed healthy", "Admin health");
-	VTAILQ_FOREACH(b, &backends, list) {
-		char buf[128];
-		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+	(void)backend_find(cli, av[2], do_list, &hdr);
+}
 
-		bprintf(buf, "%s(%s,%s,%s)",
-			b->vcl_name,
-			b->ipv4_addr == NULL ? "" : b->ipv4_addr,
-			b->ipv6_addr == NULL ? "" : b->ipv6_addr, b->port);
-		if (b->admin_health == from_probe) {
-			ah = "Auto";
-		} else if (b->admin_health == sick) {
-			ah = "Sick";
-		} else {
-			ah = "Healthy";
-		}
-		VCLI_Out(cli, "%p %s(%s,%s,:%s) %d %d\n",
-		    b, b->vcl_name, b->ipv4_addr, b->ipv6_addr, b->port,
-		    b->refcount, b->n_conn);
-		VCLI_Out(cli, "\n%-30s %10d %15s %15s",
-			 buf,
-			 b->refcount,
-			 (b ->healthy ? "Yes" : "No"),
-			 ah);
-	}
+/*---------------------------------------------------------------------*/
+
+static int __match_proto__()
+do_set_health(struct cli *cli, struct backend *b, void *priv)
+{
+	enum admin_health state;
+
+	(void)cli;
+	state = *(enum admin_health*)priv;
+	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+	b->admin_health = state;
+	return (0);
 }
 
 static void
 cli_backend_set_health(struct cli *cli, const char * const *av, void *priv)
 {
-	struct backend **b;
-	enum health_status state;
+	enum admin_health state;
 	int n;
-	const char *wstate;
 
 	(void)av;
 	(void)priv;
 	ASSERT_CLI();
-	wstate = av[3];
-	if (strcmp(wstate, "healthy") == 0) {
-		state = healthy;
-	} else if (strcmp(wstate, "sick") == 0) {
-		state = sick;
-	} else if (strcmp(wstate, "auto") == 0) {
-		state = from_probe;
-	} else {
-		VCLI_Out(cli, "Invalid state %s", wstate);
-		VCLI_SetResult(cli, CLIS_CANT);
+	AN(av[2]);
+	AN(av[3]);
+	state = vbe_str2adminhealth(av[3]);
+	if (state == ah_invalid) {
+		VCLI_Out(cli, "Invalid state %s", av[3]);
+		VCLI_SetResult(cli, CLIS_PARAM);
 		return;
 	}
-	n = backend_find(av[2], NULL, 0);
+	n = backend_find(cli, av[2], do_set_health, &state);
 	if (n == 0) {
-		VCLI_Out(cli, "No matching backends");
-		VCLI_SetResult(cli, CLIS_CANT);
-		return;
-	}
-
-	b = calloc(n, sizeof(struct backend *));
-	AN(b);
-	n = backend_find(av[2], b, n);
-
-	VCLI_Out(cli, "Set state to %s for the following backends:", wstate);
-	for (int i = 0; i < n; i++) {
-		char buf[128];
-		bprintf(buf, "%s(%s,%s,%s)",
-			b[i]->vcl_name,
-			b[i]->ipv4_addr == NULL ? "" : b[i]->ipv4_addr,
-			b[i]->ipv6_addr == NULL ? "" : b[i]->ipv6_addr, b[i]->port);
-
-		b[i]->admin_health = state;
-		VCLI_Out(cli, "\n\t%s", buf);
+		VCLI_Out(cli, "No Backends matches");
+		VCLI_SetResult(cli, CLIS_PARAM);
 	}
 }
 
+/*---------------------------------------------------------------------*/
+
 static struct cli_proto backend_cmds[] = {
 	{ "backend.list", "backend.list",
-	    "\tList all backends\n", 0, 0, "d", cli_backend_list },
+	    "\tList all backends\n", 0, 1, "d", cli_backend_list },
 	{ "backend.set_health", "backend.set_health matcher state",
 	    "\tShow a backend\n", 2, 2, "d", cli_backend_set_health },
 	{ NULL }
 };
 
-/*--------------------------------------------------------------------*/
+/*---------------------------------------------------------------------*/
 
 void
 VBE_Init(void)
diff --git a/bin/varnishd/cache_backend_poll.c b/bin/varnishd/cache_backend_poll.c
index 0854dfd..ce00a67 100644
--- a/bin/varnishd/cache_backend_poll.c
+++ b/bin/varnishd/cache_backend_poll.c
@@ -362,6 +362,14 @@ vbp_wrk_poll_backend(void *priv)
  * Cli functions
  */
 
+void
+VBP_Summary(struct cli *cli, const struct vbp_target *vt)
+{
+
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+	VCLI_Out(cli, "%d/%d", vt->good, vt->probe.window);
+}
+
 static void
 vbp_bitmap(struct cli *cli, char c, uint64_t map, const char *lbl)
 {
diff --git a/bin/varnishtest/tests/c00048.vtc b/bin/varnishtest/tests/c00048.vtc
index ed8671f..a4d0847 100644
--- a/bin/varnishtest/tests/c00048.vtc
+++ b/bin/varnishtest/tests/c00048.vtc
@@ -49,7 +49,7 @@ client c1 {
 	expect resp.status == 200
 } -run
 
-varnish v1 -clierr 300 "backend.set_health s1 foo"
-varnish v1 -clierr 300 "backend.set_health s2 foo"
-varnish v1 -clierr 300 "backend.set_health s2 auto"
+varnish v1 -clierr 106 "backend.set_health s1 foo"
+varnish v1 -clierr 106 "backend.set_health s2 foo"
+varnish v1 -clierr 106 "backend.set_health s2 auto"
 



From tfheen at varnish-cache.org  Mon Apr 16 08:20:38 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:38 +0200
Subject: [3.0] 0854c5d Make it possible to limit the total transfer time.
Message-ID: <E1SJhAs-0005BB-O9@project.varnish-software.com>

commit 0854c5dc0874b0e5752fb0621fd7bee1c196eb88
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Nov 7 14:20:06 2011 +0100

    Make it possible to limit the total transfer time.
    
    Set the default total transfer time to 600s.

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 2b3f87e..007fe6e 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -234,9 +234,9 @@ vca_acct(void *arg)
 	t0 = TIM_real();
 	while (1) {
 #ifdef SO_SNDTIMEO_WORKS
-		if (params->send_timeout != send_timeout) {
+		if (params->idle_send_timeout != send_timeout) {
 			need_test = 1;
-			send_timeout = params->send_timeout;
+			send_timeout = params->idle_send_timeout;
 			tv_sndtimeo = TIM_timeval(send_timeout);
 			VTAILQ_FOREACH(ls, &heritage.socks, list) {
 				if (ls->sock < 0)
diff --git a/bin/varnishd/cache_wrw.c b/bin/varnishd/cache_wrw.c
index 286bc45..90af92a 100644
--- a/bin/varnishd/cache_wrw.c
+++ b/bin/varnishd/cache_wrw.c
@@ -133,6 +133,14 @@ WRW_Flush(struct worker *w)
 			*/
 			size_t used = 0;
 
+			if (TIM_real() - w->sp->t_resp > params->send_timeout) {
+				WSL(w, SLT_Debug, *wrw->wfd,
+				    "Hit total send timeout, wrote = %ld/%ld; not retrying",
+				    i, wrw->liov);
+				i = -1;
+				break;
+			}
+
 			WSL(w, SLT_Debug, *wrw->wfd,
 			    "Hit send timeout, wrote = %ld/%ld; retrying",
 			    i, wrw->liov);
diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index daf4a1c..3f8330e 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -110,6 +110,7 @@ struct params {
 	unsigned		sess_timeout;
 	unsigned		pipe_timeout;
 	unsigned		send_timeout;
+	unsigned		idle_send_timeout;
 
 	/* Management hints */
 	unsigned		auto_restart;
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index b3ec8b9..76d9fe2 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -612,6 +612,13 @@ static const struct parspec input_parspec[] = {
                 "seconds the session is closed. \n"
 		"See setsockopt(2) under SO_SNDTIMEO for more information.",
 		DELAYED_EFFECT,
+		"600", "seconds" },
+	{ "idle_send_timeout", tweak_timeout, &master.idle_send_timeout, 0, 0,
+		"Time to wait with no data sent. "
+		"If no data has been transmitted in this many\n"
+                "seconds the session is closed. \n"
+		"See setsockopt(2) under SO_SNDTIMEO for more information.",
+		DELAYED_EFFECT,
 		"60", "seconds" },
 	{ "auto_restart", tweak_bool, &master.auto_restart, 0, 0,
 		"Restart child process automatically if it dies.\n",



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] 999d163 Make the VFP calls symmetric and pair-wise visible,
	allow ->end() to fail, and act accordingly.
Message-ID: <E1SJhAo-0004yV-Mk@project.varnish-software.com>

commit 999d163d10706eba3cbb8018deebc6ef340b594e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 31 09:03:04 2011 +0000

    Make the VFP calls symmetric and pair-wise visible, allow ->end()
    to fail, and act accordingly.
    
    Conflicts:
    
    	bin/varnishd/cache_fetch.c

diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 73edaa1..e0d5667 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -196,15 +196,12 @@ fetch_number(const char *nbr, int radix)
 /*--------------------------------------------------------------------*/
 
 static int
-fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
+fetch_straight(struct sess *sp, struct http_conn *htc, ssize_t cl)
 {
 	int i;
-	ssize_t cl;
 
 	assert(sp->wrk->body_status == BS_LENGTH);
 
-	cl = fetch_number(b, 10);
-	sp->wrk->vfp->begin(sp, cl > 0 ? cl : 0);
 	if (cl < 0) {
 		WSP(sp, SLT_FetchError, "straight length field bogus");
 		return (-1);
@@ -240,7 +237,6 @@ fetch_chunked(struct sess *sp, struct http_conn *htc)
 	unsigned u;
 	ssize_t cl;
 
-	sp->wrk->vfp->begin(sp, 0);
 	assert(sp->wrk->body_status == BS_CHUNKED);
 	do {
 		/* Skip leading whitespace */
@@ -308,7 +304,6 @@ fetch_eof(struct sess *sp, struct http_conn *htc)
 	int i;
 
 	assert(sp->wrk->body_status == BS_EOF);
-	sp->wrk->vfp->begin(sp, 0);
 	i = sp->wrk->vfp->bytes(sp, htc, SSIZE_MAX);
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "eof read_error: %d (%s)",
@@ -484,6 +479,7 @@ FetchBody(struct sess *sp)
 	struct storage *st;
 	struct worker *w;
 	int mklen;
+	ssize_t cl;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
@@ -499,6 +495,9 @@ FetchBody(struct sess *sp)
 
 	AZ(w->vgz_rx);
 	AZ(VTAILQ_FIRST(&sp->obj->store));
+
+	/* XXX: pick up estimate from objdr ? */
+	cl = 0;
 	switch (w->body_status) {
 	case BS_NONE:
 		cls = 0;
@@ -509,20 +508,26 @@ FetchBody(struct sess *sp)
 		mklen = 1;
 		break;
 	case BS_LENGTH:
-		cls = fetch_straight(sp, w->htc,
-		    w->h_content_length);
+		cl = fetch_number(sp->wrk->h_content_length, 10);
+		w->vfp->begin(sp, cl > 0 ? cl : 0);
+		cls = fetch_straight(sp, w->htc, cl);
 		mklen = 1;
-		XXXAZ(w->vfp->end(sp));
+		if (w->vfp->end(sp))
+			cls = -1;
 		break;
 	case BS_CHUNKED:
+		w->vfp->begin(sp, cl);
 		cls = fetch_chunked(sp, w->htc);
 		mklen = 1;
-		XXXAZ(w->vfp->end(sp));
+		if (w->vfp->end(sp))
+			cls = -1;
 		break;
 	case BS_EOF:
+		w->vfp->begin(sp, cl);
 		cls = fetch_eof(sp, w->htc);
 		mklen = 1;
-		XXXAZ(w->vfp->end(sp));
+		if (w->vfp->end(sp))
+			cls = -1;
 		break;
 	case BS_ERROR:
 		cls = 1;



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] 5365096 add a label so we can ref this in the user guide /
	tutorial
Message-ID: <E1SJhAq-00055P-P5@project.varnish-software.com>

commit 5365096cdde6406af402228654f668b8764d7352
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Dec 19 11:24:42 2011 +0100

    add a label so we can ref this in the user guide / tutorial

diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index be9c9c5..edd8fbc 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -1,3 +1,5 @@
+_reference-varnishlog:
+
 ==========
 varnishlog
 ==========



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] b72c1e6 Remove outdated comment
Message-ID: <E1SJhAr-000573-BZ@project.varnish-software.com>

commit b72c1e6025deae70ab8407604d32607798e83a27
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 3 08:15:12 2012 +0000

    Remove outdated comment

diff --git a/configure.ac b/configure.ac
index 05397fd..41ddf24 100644
--- a/configure.ac
+++ b/configure.ac
@@ -326,12 +326,6 @@ AC_CHECK_FUNCS([clock_gettime])
 AC_CHECK_FUNCS([gethrtime]) 
 LIBS="${save_LIBS}"
 
-# Check which mechanism to use for the acceptor.  We look for kqueue
-# only on platforms on which we know that it works, because there are
-# platforms where a simple AC_CHECK_FUNCS([kqueue]) would succeed but
-# the build would fail.  We also allow the user to disable mechanisms
-# he doesn't want to use.
-
 # --enable-kqueue
 AC_ARG_ENABLE(kqueue,
     AS_HELP_STRING([--enable-kqueue],



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] a9f7b8a Correct ACL syntax
Message-ID: <E1SJhAq-00055A-Md@project.varnish-software.com>

commit a9f7b8aff442081318c21859d2a65296e3632bed
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Dec 13 15:35:03 2011 +0100

    Correct ACL syntax

diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst
index cea1ccd..ecf0e96 100644
--- a/doc/sphinx/tutorial/purging.rst
+++ b/doc/sphinx/tutorial/purging.rst
@@ -29,7 +29,7 @@ following VCL in place::
 
   acl purge {
 	  "localhost";
-	  "192.168.55.0/24";
+	  "192.168.55.0"/24;
   }
   
   sub vcl_recv {



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] 4d1c103 remove -i and -I,
	add -m and describe it. Simplify the docs. The user only needs to
	care about -m, really
Message-ID: <E1SJhAq-00055b-Rm@project.varnish-software.com>

commit 4d1c103184723ef6461f5be67dbe72a3e3ff7344
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Dec 19 11:30:26 2011 +0100

    remove -i and -I, add -m and describe it. Simplify the docs. The user only needs to care about -m, really

diff --git a/doc/sphinx/tutorial/logging.rst b/doc/sphinx/tutorial/logging.rst
index 331742f..68f1d87 100644
--- a/doc/sphinx/tutorial/logging.rst
+++ b/doc/sphinx/tutorial/logging.rst
@@ -60,16 +60,9 @@ want to know are:
 -c 
  Same as -b but for client side traffic.
 
--i tag
- Only show lines with a certain tag. "varnishlog -i SessionOpen" 
- will only give you new sessions. Note that the tags are case 
- insensitive. 
-
--I Regex
- Filter the data through a regex and only show the matching lines. To
- show all cookie headers coming from the clients:
- ``$ varnishlog -c -i RxHeader -I Cookie``
-
+-m tag:regex
+ Only list transactions where the tag matches a regular expression. If
+ it matches you will get the whole transaction.
 
 Now that Varnish seem to work OK it's time to put Varnish on port 80
 while we tune it.



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] b400281 Fix syntax for 3.0
Message-ID: <E1SJhAr-000568-1o@project.varnish-software.com>

commit b4002811cb56561cceedb8cc35528809658b3d9d
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Dec 20 16:18:12 2011 +0100

    Fix syntax for 3.0

diff --git a/doc/sphinx/tutorial/cookies.rst b/doc/sphinx/tutorial/cookies.rst
index ab59e21..0278712 100644
--- a/doc/sphinx/tutorial/cookies.rst
+++ b/doc/sphinx/tutorial/cookies.rst
@@ -50,7 +50,7 @@ cookies named COOKIE1 and COOKIE2 and you can marvel at it::
 
   sub vcl_recv {
     if (req.http.Cookie) {
-      set req.http.Cookie = ";" req.http.Cookie;
+      set req.http.Cookie = ";" + req.http.Cookie;
       set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
       set req.http.Cookie = regsuball(req.http.Cookie, ";(COOKIE1|COOKIE2)=", "; \1=");
       set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");



From tfheen at varnish-cache.org  Mon Apr 16 08:20:38 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:38 +0200
Subject: [3.0] 8332ca8 Drop struct workreq, we don't use it.
Message-ID: <E1SJhAs-0005Au-LU@project.varnish-software.com>

commit 8332ca8638f5c8bf91147044aad2500bf95e4af2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 16:45:07 2011 +0000

    Drop struct workreq, we don't use it.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 04ff2d0..31b19ef 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -108,7 +108,6 @@ struct objhead;
 struct objcore;
 struct busyobj;
 struct storage;
-struct workreq;
 struct vrt_backend;
 struct cli_proto;
 struct ban;
@@ -304,7 +303,7 @@ struct worker {
 	pthread_cond_t		cond;
 
 	VTAILQ_ENTRY(worker)	list;
-	struct workreq		*wrq;
+	struct sess		*sp;
 
 	struct VCL_conf		*vcl;
 
@@ -367,21 +366,6 @@ struct worker {
 	struct acct		acct_tmp;
 };
 
-/* Work Request for worker thread ------------------------------------*/
-
-/*
- * This is a worker-function.
- * XXX: typesafety is probably not worth fighting for
- */
-
-typedef void workfunc(struct worker *, void *priv);
-
-struct workreq {
-	VTAILQ_ENTRY(workreq)	list;
-	workfunc		*func;
-	void			*priv;
-};
-
 /* Storage -----------------------------------------------------------*/
 
 struct storage {
@@ -613,7 +597,7 @@ struct sess {
 	/* Various internal stuff */
 	struct sessmem		*mem;
 
-	struct workreq		workreq;
+	VTAILQ_ENTRY(sess)	poollist;
 	struct acct		acct_req;
 	struct acct		acct_ses;
 
diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index a3d1851..dd26303 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -131,7 +131,6 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv)
 	SZOF(struct http_conn);
 	SZOF(struct acct);
 	SZOF(struct worker);
-	SZOF(struct workreq);
 	SZOF(struct storage);
 	SZOF(struct object);
 	SZOF(struct objcore);
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 1260bd0..14679a5 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -67,7 +67,7 @@ struct wq {
 #define WQ_MAGIC		0x606658fa
 	struct lock		mtx;
 	struct workerhead	idle;
-	VTAILQ_HEAD(, workreq)	queue;
+	VTAILQ_HEAD(, sess)	queue;
 	unsigned		nthr;
 	unsigned		lqueue;
 	unsigned		last_lqueue;
@@ -159,9 +159,9 @@ wrk_thread_real(struct wq *qp, unsigned shm_workspace, unsigned sess_workspace,
 		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 
 		/* Process queued requests, if any */
-		w->wrq = VTAILQ_FIRST(&qp->queue);
-		if (w->wrq != NULL) {
-			VTAILQ_REMOVE(&qp->queue, w->wrq, list);
+		w->sp = VTAILQ_FIRST(&qp->queue);
+		if (w->sp != NULL) {
+			VTAILQ_REMOVE(&qp->queue, w->sp, poollist);
 			qp->lqueue--;
 		} else {
 			if (isnan(w->lastused))
@@ -171,17 +171,21 @@ wrk_thread_real(struct wq *qp, unsigned shm_workspace, unsigned sess_workspace,
 				WRK_SumStat(w);
 			Lck_CondWait(&w->cond, &qp->mtx);
 		}
-		if (w->wrq == NULL)
+		if (w->sp == NULL)
 			break;
 		Lck_Unlock(&qp->mtx);
 		stats_clean = 0;
-		AN(w->wrq);
-		AN(w->wrq->func);
 		w->lastused = NAN;
 		WS_Reset(w->ws, NULL);
 		w->storage_hint = NULL;
 
-		w->wrq->func(w, w->wrq->priv);
+		AZ(w->sp->wrk);
+		THR_SetSession(w->sp);
+		w->sp->wrk = w;
+		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+		CNT_Session(w->sp);
+		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+		THR_SetSession(NULL);
 
 		WS_Assert(w->ws);
 		AZ(w->bereq->ws);
@@ -190,7 +194,7 @@ wrk_thread_real(struct wq *qp, unsigned shm_workspace, unsigned sess_workspace,
 		AZ(w->wrw.wfd);
 		AZ(w->storage_hint);
 		assert(w->wlp == w->wlb);
-		w->wrq = NULL;
+		w->sp = NULL;
 		if (params->diag_bitmap & 0x00040000) {
 			if (w->vcl != NULL)
 				VCL_Rel(&w->vcl);
@@ -241,7 +245,7 @@ wrk_thread(void *priv)
  */
 
 static int
-WRK_Queue(struct workreq *wrq)
+WRK_Queue(struct sess *sp)
 {
 	struct worker *w;
 	struct wq *qp;
@@ -266,7 +270,7 @@ WRK_Queue(struct workreq *wrq)
 	if (w != NULL) {
 		VTAILQ_REMOVE(&qp->idle, w, list);
 		Lck_Unlock(&qp->mtx);
-		w->wrq = wrq;
+		w->sp = sp;
 		AZ(pthread_cond_signal(&w->cond));
 		return (0);
 	}
@@ -278,7 +282,7 @@ WRK_Queue(struct workreq *wrq)
 		return (-1);
 	}
 
-	VTAILQ_INSERT_TAIL(&qp->queue, wrq, list);
+	VTAILQ_INSERT_TAIL(&qp->queue, sp, poollist);
 	qp->nqueue++;
 	qp->lqueue++;
 	Lck_Unlock(&qp->mtx);
@@ -288,31 +292,12 @@ WRK_Queue(struct workreq *wrq)
 
 /*--------------------------------------------------------------------*/
 
-static void
-wrk_do_cnt_sess(struct worker *w, void *priv)
-{
-	struct sess *sess;
-
-	CAST_OBJ_NOTNULL(sess, priv, SESS_MAGIC);
-	AZ(sess->wrk);
-	THR_SetSession(sess);
-	sess->wrk = w;
-	CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
-	CNT_Session(sess);
-	CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
-	THR_SetSession(NULL);
-}
-
-/*--------------------------------------------------------------------*/
-
 int
 WRK_QueueSession(struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk);
-	sp->workreq.func = wrk_do_cnt_sess;
-	sp->workreq.priv = sp;
-	if (WRK_Queue(&sp->workreq) == 0)
+	if (WRK_Queue(sp) == 0)
 		return (0);
 
 	/*
@@ -389,7 +374,7 @@ wrk_decimate_flock(struct wq *qp, double t_idle, struct VSC_C_main *vs)
 
 	/* And give it a kiss on the cheek... */
 	if (w != NULL) {
-		AZ(w->wrq);
+		AZ(w->sp);
 		AZ(pthread_cond_signal(&w->cond));
 		TIM_sleep(params->wthread_purge_delay * 1e-3);
 	}



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] 29d126f Avoid infinite loop in regsuball
Message-ID: <E1SJhAp-000501-20@project.varnish-software.com>

commit 29d126fd6674cdfc12b1bf1378cca5dbb6d63c0a
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Nov 1 09:25:22 2011 +0100

    Avoid infinite loop in regsuball
    
    Also optimise out a few strlen calls in favour of just tracking the
    lengths.
    
    Fixes: #1047

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index 30ae5ca..fe6b814 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -95,13 +95,16 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 	char *b0;
 	const char *s;
 	unsigned u, x;
+	int options = 0;
+	size_t len;
 
 	AN(re);
 	if (str == NULL)
 		str = "";
 	t = re;
 	memset(ovector, 0, sizeof(ovector));
-	i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30,
+	len = strlen(str);
+	i = VRE_exec(t, str, len, 0, options, ovector, 30,
 	    &params->vre_limits);
 
 	/* If it didn't match, we can return the original string */
@@ -137,10 +140,12 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 			}
 		}
 		str += ovector[1];
+		len -= ovector[1];
 		if (!all)
 			break;
 		memset(&ovector, 0, sizeof(ovector));
-		i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30,
+		options |= VRE_NOTEMPTY_ATSTART;
+		i = VRE_exec(t, str, len, 0, options, ovector, 30,
 		    &params->vre_limits);
 		if (i < VRE_ERROR_NOMATCH ) {
 			WSP(sp, SLT_VCL_error,
@@ -150,8 +155,7 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 	} while (i != VRE_ERROR_NOMATCH);
 
 	/* Copy suffix to match */
-	l = strlen(str) + 1;
-	Tadd(&res, str, l);
+	Tadd(&res, str, len+1);
 	if (res.b >= res.e) {
 		WS_Release(sp->http->ws, 0);
 		return (str);
diff --git a/bin/varnishtest/tests/c00047.vtc b/bin/varnishtest/tests/c00047.vtc
new file mode 100644
index 0000000..4b23194
--- /dev/null
+++ b/bin/varnishtest/tests/c00047.vtc
@@ -0,0 +1,33 @@
+varnishtest "Test VCL regsuball()"
+
+server s1 {
+	rxreq 
+	txresp \
+		-hdr "foo: barbar" \
+		-hdr "bar: bbbar"
+} -start
+
+varnish v1 -vcl+backend { 
+	sub vcl_fetch {
+		set beresp.http.baz1 = regsuball(beresp.http.foo, "barb", "zz");
+		set beresp.http.baz2 = regsuball(beresp.http.foo, "ar", "zz");
+		set beresp.http.baz3 = regsuball(beresp.http.foo, "^", "baz");
+		set beresp.http.baz4 = regsuball(beresp.http.foo, "^[;]*", "baz");
+		set beresp.http.baz5 = regsuball(beresp.http.bar, "^b*", "b");
+		set beresp.http.baz6 = regsuball(beresp.http.foo, "^b*", "z");
+		set beresp.http.baz7 = regsuball(beresp.http.foo, "ping", "pong");
+	}
+} -start 
+
+client c1 {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.baz1 == "zzar"
+	expect resp.http.baz2 == "bzzbzz"
+	expect resp.http.baz3 == "bazbarbar"
+	expect resp.http.baz4 == "bazbarbar"
+	expect resp.http.baz5 == "bar"
+	expect resp.http.baz6 == "zarbar"
+	expect resp.http.baz7 == "barbar"
+} -run
diff --git a/include/vre.h b/include/vre.h
index 29c1b08..a1206e5 100644
--- a/include/vre.h
+++ b/include/vre.h
@@ -49,6 +49,7 @@ typedef struct vre vre_t;
 
 /* And those to PCRE options */
 #define VRE_CASELESS           0x00000001
+#define VRE_NOTEMPTY_ATSTART   0x10000000
 
 vre_t *VRE_compile(const char *, int, const char **, int *);
 int VRE_exec(const vre_t *code, const char *subject, int length,



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] 856bd82 Overhaul the detection and reporting of fetch errors,
	to properly catch trouble that materializes only when we destroy the
	VGZ instance.
Message-ID: <E1SJhAo-0004yl-PR@project.varnish-software.com>

commit 856bd826ca8d40f4d6459dc2fa7a8c40b00f2a96
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 31 14:20:34 2011 +0000

    Overhaul the detection and reporting of fetch errors, to properly
    catch trouble that materializes only when we destroy the VGZ instance.
    
    Fixes	#1037
    Fixes	#1043
    Fixes	#1044
    
    Conflicts:
    
    	bin/varnishd/cache.h
    	bin/varnishd/cache_center.c
    	bin/varnishd/cache_esi_deliver.c
    	bin/varnishd/cache_esi_fetch.c
    	bin/varnishd/cache_fetch.c
    	bin/varnishd/cache_gzip.c
    	bin/varnishd/cache_response.c

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 3cb0af9..4c228ba 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -331,6 +331,7 @@ struct worker {
 	struct vfp		*vfp;
 	struct vgz		*vgz_rx;
 	struct vef_priv		*vef_priv;
+	unsigned		fetch_failed;
 	unsigned		do_stream;
 	unsigned		do_esi;
 	unsigned		do_gzip;
@@ -718,6 +719,8 @@ int EXP_NukeOne(struct worker *w, struct lru *lru);
 
 /* cache_fetch.c */
 struct storage *FetchStorage(const struct sess *sp, ssize_t sz);
+int FetchError(struct sess *sp, const char *error);
+int FetchError2(struct sess *sp, const char *error, const char *more);
 int FetchHdr(struct sess *sp);
 int FetchBody(struct sess *sp);
 int FetchReqBody(struct sess *sp);
@@ -736,7 +739,7 @@ int VGZ_ObufFull(const struct vgz *vg);
 int VGZ_ObufStorage(const struct sess *sp, struct vgz *vg);
 int VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag);
 int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
-void VGZ_Destroy(struct vgz **);
+int VGZ_Destroy(struct vgz **);
 void VGZ_UpdateObj(const struct vgz*, struct object *);
 int VGZ_WrwGunzip(const struct sess *, struct vgz *, const void *ibuf,
     ssize_t ibufl, char *obuf, ssize_t obufl, ssize_t *obufp);
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 6cbd532..c75f93d 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -935,7 +935,7 @@ cnt_streambody(struct sess *sp)
 
 	RES_StreamEnd(sp);
 	if (sp->wrk->res_mode & RES_GUNZIP)
-		VGZ_Destroy(&sctx.vgz);
+		(void)VGZ_Destroy(&sctx.vgz);
 
 	sp->wrk->sctx = NULL;
 	assert(WRW_IsReleased(sp->wrk));
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index bbf51fd..cc3f569 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -408,7 +408,7 @@ ESI_Deliver(struct sess *sp)
 	if (vgz != NULL) {
 		if (obufl > 0)
 			(void)WRW_Write(sp->wrk, obuf, obufl);
-		VGZ_Destroy(&vgz);
+		(void)VGZ_Destroy(&vgz);
 	}
 	if (sp->wrk->gzip_resp && sp->esi_level == 0) {
 		/* Emit a gzip literal block with finish bit set */
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index c8df1b1..cb02737 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -114,7 +114,7 @@ vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			bytes -= w;
 		}
 		if (VGZ_ObufStorage(sp, vg))
-			return (-1);
+			return(FetchError(sp, "Could not get storage"));
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		xxxassert(i == VGZ_OK || i == VGZ_END);
 		VEP_parse(sp, dp, dl);
@@ -299,7 +299,6 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 	struct vef_priv *vef;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	/* XXX: snapshot WS's ? We'll need the space */
 
 	AZ(sp->wrk->vgz_rx);
 	if (sp->wrk->is_gzip && sp->wrk->do_gunzip) {
@@ -308,9 +307,6 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 	} else if (sp->wrk->is_gunzip && sp->wrk->do_gzip) {
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
-		//vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
-		//memset(vef, 0, sizeof *vef);
-		//vef->magic = VEF_MAGIC;
 		vef->vgz = VGZ_NewGzip(sp, "G F E");
 		AZ(sp->wrk->vef_priv);
 		sp->wrk->vef_priv = vef;
@@ -319,9 +315,6 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 		sp->wrk->vgz_rx = VGZ_NewUngzip(sp, "U F E");
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
-		//vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
-		//memset(vef, 0, sizeof *vef);
-		//vef->magic = VEF_MAGIC;
 		vef->vgz = VGZ_NewGzip(sp, "G F E");
 		AZ(sp->wrk->vef_priv);
 		sp->wrk->vef_priv = vef;
@@ -341,6 +334,7 @@ vfp_esi_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	AZ(sp->wrk->fetch_failed);
 	AN(sp->wrk->vep);
 	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
 		i = vfp_esi_bytes_gu(sp, htc, bytes);
@@ -360,35 +354,48 @@ vfp_esi_end(struct sess *sp)
 	struct vsb *vsb;
 	struct vef_priv *vef;
 	ssize_t l;
+	int retval;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AN(sp->wrk->vep);
 
+	retval = sp->wrk->fetch_failed;
+
+	if (sp->wrk->vgz_rx != NULL && VGZ_Destroy(&sp->wrk->vgz_rx) != VGZ_END)
+		retval = FetchError(sp,
+		    "Gunzip+ESI Failed at the very end");
+
 	vsb = VEP_Finish(sp);
 
 	if (vsb != NULL) {
-		l = VSB_len(vsb);
-		assert(l > 0);
-		/* XXX: This is a huge waste of storage... */
-		sp->obj->esidata = STV_alloc(sp, l);
-		XXXAN(sp->obj->esidata);
-		memcpy(sp->obj->esidata->ptr, VSB_data(vsb), l);
-		sp->obj->esidata->len = l;
+		if (!retval) {
+			l = VSB_len(vsb);
+			assert(l > 0);
+			/* XXX: This is a huge waste of storage... */
+			sp->obj->esidata = STV_alloc(sp, l);
+			if (sp->obj->esidata != NULL) {
+				memcpy(sp->obj->esidata->ptr,
+				       VSB_data(vsb), l);
+				sp->obj->esidata->len = l;
+			} else {
+				retval = FetchError(sp,
+				    "Could not allocate storage for esidata");
+			}
+		}
 		VSB_delete(vsb);
 	}
-	if (sp->wrk->vgz_rx != NULL)
-		VGZ_Destroy(&sp->wrk->vgz_rx);
 
 	if (sp->wrk->vef_priv != NULL) {
 		vef = sp->wrk->vef_priv;
 		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 		sp->wrk->vef_priv = NULL;
 		VGZ_UpdateObj(vef->vgz, sp->obj);
-		VGZ_Destroy(&vef->vgz);
-		XXXAZ(vef->error);
+		if (VGZ_Destroy(&vef->vgz) != VGZ_END)
+			retval = FetchError(sp, 
+			    "ESI+Gzip Failed at the very end");
 		FREE_OBJ(vef);
 	}
-	return (0);
+	return (retval);
 }
 
 struct vfp vfp_esi = {
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index e0d5667..14ec4f4 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -43,6 +43,40 @@
 static unsigned fetchfrag;
 
 /*--------------------------------------------------------------------
+ * We want to issue the first error we encounter on fetching and
+ * supress the rest.  This function does that.
+ *
+ * Other code is allowed to look at w->fetch_failed to bail out
+ *
+ * For convenience, always return -1
+ */
+
+int
+FetchError2(struct sess *sp, const char *error, const char *more)
+{
+	struct worker *w;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+	w = sp->wrk;
+
+	if (!w->fetch_failed) {
+		if (more == NULL)
+			WSP(sp, SLT_FetchError, "%s", error);
+		else
+			WSP(sp, SLT_FetchError, "%s: %s", error, more);
+	}
+	w->fetch_failed = 1;
+	return (-1);
+}
+
+int
+FetchError(struct sess *sp, const char *error)
+{
+	return(FetchError2(sp, error, NULL));
+}
+
+/*--------------------------------------------------------------------
  * VFP_NOP
  *
  * This fetch-processor does nothing but store the object.
@@ -75,7 +109,8 @@ vfp_nop_begin(struct sess *sp, size_t estimate)
  *
  * Process (up to) 'bytes' from the socket.
  *
- * Return -1 on error
+ * Return -1 on error, issue FetchError()
+ *	will not be called again, once error happens.
  * Return 0 on EOF on socket even if bytes not reached.
  * Return 1 when 'bytes' have been processed.
  */
@@ -86,17 +121,18 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	ssize_t l, w;
 	struct storage *st;
 
+	AZ(sp->wrk->fetch_failed);
 	while (bytes > 0) {
 		st = FetchStorage(sp, 0);
-		if (st == NULL) {
-			htc->error = "Could not get storage";
-			return (-1);
-		}
+		if (st == NULL)
+			return(FetchError(sp, "Could not get storage"));
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
 		w = HTC_Read(htc, st->ptr + st->len, l);
-		if (w <= 0)
+		if (w < 0)
+			return(FetchError(sp, htc->error));
+		if (w == 0)
 			return (w);
 		st->len += w;
 		sp->obj->len += w;
@@ -203,16 +239,13 @@ fetch_straight(struct sess *sp, struct http_conn *htc, ssize_t cl)
 	assert(sp->wrk->body_status == BS_LENGTH);
 
 	if (cl < 0) {
-		WSP(sp, SLT_FetchError, "straight length field bogus");
-		return (-1);
+		return (FetchError(sp, "straight length field bogus"));
 	} else if (cl == 0)
 		return (0);
 
 	i = sp->wrk->vfp->bytes(sp, htc, cl);
 	if (i <= 0) {
-		WSP(sp, SLT_FetchError, "straight read_error: %d %d (%s)",
-		    i, errno, htc->error);
-		return (-1);
+		return (FetchError(sp, "straight insufficient bytes"));
 	}
 	return (0);
 }
@@ -256,8 +289,7 @@ fetch_chunked(struct sess *sp, struct http_conn *htc)
 		}
 
 		if (u >= sizeof buf) {
-			WSP(sp, SLT_FetchError,	"chunked header too long");
-			return (-1);
+			return (FetchError(sp,"chunked header too long"));
 		}
 
 		/* Skip trailing white space */
@@ -267,15 +299,13 @@ fetch_chunked(struct sess *sp, struct http_conn *htc)
 		}
 
 		if (buf[u] != '\n') {
-			WSP(sp, SLT_FetchError,	"chunked header char syntax");
-			return (-1);
+			return (FetchError(sp,"chunked header no NL"));
 		}
 		buf[u] = '\0';
 
 		cl = fetch_number(buf, 16);
 		if (cl < 0) {
-			WSP(sp, SLT_FetchError,	"chunked header nbr syntax");
-			return (-1);
+			return (FetchError(sp,"chunked header number syntax"));
 		} else if (cl > 0) {
 			i = sp->wrk->vfp->bytes(sp, htc, cl);
 			CERR();
@@ -286,10 +316,8 @@ fetch_chunked(struct sess *sp, struct http_conn *htc)
 			i = HTC_Read(htc, buf, 1);
 			CERR();
 		}
-		if (buf[0] != '\n') {
-			WSP(sp, SLT_FetchError,	"chunked tail syntax");
-			return (-1);
-		}
+		if (buf[0] != '\n')
+			return (FetchError(sp,"chunked tail no NL"));
 	} while (cl > 0);
 	return (0);
 }
@@ -305,11 +333,8 @@ fetch_eof(struct sess *sp, struct http_conn *htc)
 
 	assert(sp->wrk->body_status == BS_EOF);
 	i = sp->wrk->vfp->bytes(sp, htc, SSIZE_MAX);
-	if (i < 0) {
-		WSP(sp, SLT_FetchError, "eof read_error: %d (%s)",
-		    errno, htc->error);
+	if (i < 0) 
 		return (-1);
-	}
 	return (0);
 }
 
@@ -498,6 +523,7 @@ FetchBody(struct sess *sp)
 
 	/* XXX: pick up estimate from objdr ? */
 	cl = 0;
+	w->fetch_failed = 0;
 	switch (w->body_status) {
 	case BS_NONE:
 		cls = 0;
@@ -568,6 +594,7 @@ FetchBody(struct sess *sp)
 		sp->obj->len = 0;
 		return (__LINE__);
 	}
+	AZ(w->fetch_failed);
 
 	if (cls == 0 && w->do_close)
 		cls = 1;
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 1eaa7fa..0bd158c 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -82,7 +82,7 @@ struct vgz {
 	const char		*id;
 	struct ws		*tmp;
 	char			*tmp_snapshot;
-	const char		*error;
+	int			last_i;
 
 	struct storage		*obuf;
 
@@ -206,8 +206,6 @@ VGZ_NewGzip(struct sess *sp, const char *id)
 	    16 + params->gzip_window,	/* Window bits (16=gzip + 15) */
 	    params->gzip_memlevel,	/* memLevel */
 	    Z_DEFAULT_STRATEGY);
-	if (i != Z_OK)
-		printf("deflateInit2() = %d\n", i);
 	assert(Z_OK == i);
 	return (vg);
 }
@@ -264,10 +262,8 @@ VGZ_ObufStorage(const struct sess *sp, struct vgz *vg)
 	struct storage *st;
 
 	st = FetchStorage(sp, 0);
-	if (st == NULL) {
-		vg->error = "Could not get ObufStorage";
+	if (st == NULL)
 		return (-1);
-	}
 
 	vg->obuf = st;
 	VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
@@ -299,6 +295,7 @@ VGZ_Gunzip(struct vgz *vg, const void **pptr, size_t *plen)
 		if (vg->obuf != NULL)
 			vg->obuf->len += l;
 	}
+	vg->last_i = i;
 	if (i == Z_OK)
 		return (VGZ_OK);
 	if (i == Z_STREAM_END)
@@ -341,6 +338,7 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, enum vgz_flag flags)
 		if (vg->obuf != NULL)
 			vg->obuf->len += l;
 	}
+	vg->last_i = i;
 	if (i == Z_OK)
 		return (0);
 	if (i == Z_STREAM_END)
@@ -408,11 +406,11 @@ VGZ_UpdateObj(const struct vgz *vg, struct object *obj)
 
 /*--------------------------------------------------------------------*/
 
-void
+int
 VGZ_Destroy(struct vgz **vgp)
 {
 	struct vgz *vg;
-	const char *err;
+	int i;
 
 	vg = *vgp;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
@@ -425,14 +423,22 @@ VGZ_Destroy(struct vgz **vgp)
 	    (intmax_t)vg->vz.start_bit,
 	    (intmax_t)vg->vz.last_bit,
 	    (intmax_t)vg->vz.stop_bit);
-	err = vg->error;
 	if (vg->tmp != NULL)
 		WS_Reset(vg->tmp, vg->tmp_snapshot);
 	if (vg->dir == VGZ_GZ)
-		assert(deflateEnd(&vg->vz) == 0 || err != NULL);
+		i = deflateEnd(&vg->vz);
 	else
-		assert(inflateEnd(&vg->vz) == 0 || err != NULL);
+		i = inflateEnd(&vg->vz);
+	if (vg->last_i == Z_STREAM_END && i == Z_OK)
+		i = Z_STREAM_END;
 	FREE_OBJ(vg);
+	if (i == Z_OK)
+		return (VGZ_OK);
+	if (i == Z_STREAM_END)
+		return (VGZ_END);
+	if (i == Z_BUF_ERROR)
+		return (VGZ_STUCK);
+	return (VGZ_ERROR);
 }
 
 /*--------------------------------------------------------------------
@@ -459,6 +465,7 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	size_t dl;
 	const void *dp;
 
+	AZ(sp->wrk->fetch_failed);
 	vg = sp->wrk->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -468,27 +475,25 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			if (l > bytes)
 				l = bytes;
 			w = HTC_Read(htc, ibuf, l);
-			if (w <= 0)
+			if (w < 0)
+				return(FetchError(sp, htc->error));
+			if (w == 0)
 				return (w);
 			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
 
-		if (VGZ_ObufStorage(sp, vg)) {
-			htc->error = "Could not get storage";
-			return (-1);
-		}
+		if (VGZ_ObufStorage(sp, vg))
+			return(FetchError(sp, "Could not get storage"));
 		i = VGZ_Gunzip(vg, &dp, &dl);
-		assert(i == VGZ_OK || i == VGZ_END);
+		if (i != VGZ_OK && i != VGZ_END)
+			return(FetchError(sp, "Gunzip data error"));
 		sp->obj->len += dl;
 		if (sp->wrk->do_stream)
 			RES_StreamPoll(sp);
 	}
-	if (i == Z_OK || i == Z_STREAM_END)
-		return (1);
-	htc->error = "See other message";
-	WSP(sp, SLT_FetchError, "Gunzip trouble (%d)", i);
-	return (-1);
+	assert(i == Z_OK || i == Z_STREAM_END);
+	return (1);
 }
 
 static int __match_proto__()
@@ -499,7 +504,12 @@ vfp_gunzip_end(struct sess *sp)
 	vg = sp->wrk->vgz_rx;
 	sp->wrk->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	VGZ_Destroy(&vg);
+	if (sp->wrk->fetch_failed) {
+		(void)VGZ_Destroy(&vg);
+		return(0);
+	}
+	if (VGZ_Destroy(&vg) != VGZ_END)
+		return(FetchError(sp, "Gunzip error at the very end"));
 	return (0);
 }
 
@@ -535,6 +545,7 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	size_t dl;
 	const void *dp;
 
+	AZ(sp->wrk->fetch_failed);
 	vg = sp->wrk->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -544,15 +555,15 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			if (l > bytes)
 				l = bytes;
 			w = HTC_Read(htc, ibuf, l);
-			if (w <= 0)
+			if (w < 0)
+				return(FetchError(sp, htc->error));
+			if (w == 0)
 				return (w);
 			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
-		if (VGZ_ObufStorage(sp, vg)) {
-			htc->error = "Could not get storage";
-			return (-1);
-		}
+		if (VGZ_ObufStorage(sp, vg))
+			return(FetchError(sp, "Could not get storage"));
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
 		sp->obj->len += dl;
@@ -574,19 +585,22 @@ vfp_gzip_end(struct sess *sp)
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	sp->wrk->vgz_rx = NULL;
 
-	if (vg->error == NULL) {
-		do {
-			VGZ_Ibuf(vg, "", 0);
-			if (VGZ_ObufStorage(sp, vg))
-				return (-1);
-			i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
-			sp->obj->len += dl;
-		} while (i != Z_STREAM_END);
-		if (sp->wrk->do_stream)
-			RES_StreamPoll(sp);
-		VGZ_UpdateObj(vg, sp->obj);
+	if (sp->wrk->fetch_failed) {
+		(void)VGZ_Destroy(&vg);
+		return(0);
 	}
-	VGZ_Destroy(&vg);
+	do {
+		VGZ_Ibuf(vg, "", 0);
+		if (VGZ_ObufStorage(sp, vg))
+			return(FetchError(sp, "Could not get storage"));
+		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
+		sp->obj->len += dl;
+	} while (i != Z_STREAM_END);
+	if (sp->wrk->do_stream)
+		RES_StreamPoll(sp);
+	VGZ_UpdateObj(vg, sp->obj);
+	if (VGZ_Destroy(&vg) != VGZ_END)
+		return(FetchError(sp, "Gzip error at the very end"));
 	return (0);
 }
 
@@ -622,21 +636,21 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	const void *dp;
 	struct storage *st;
 
+	AZ(sp->wrk->fetch_failed);
 	vg = sp->wrk->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0) {
 		st = FetchStorage(sp, 0);
-		if (st == NULL) {
-			htc->error = "Could not get storage";
-			vg->error = htc->error;
-			return (-1);
-		}
+		if (st == NULL)
+			return(FetchError(sp, "Could not get storage"));
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
 		w = HTC_Read(htc, st->ptr + st->len, l);
-		if (w <= 0)
+		if (w < 0)
+			return(FetchError(sp, htc->error));
+		if (w == 0)
 			return (w);
 		bytes -= w;
 		VGZ_Ibuf(vg, st->ptr + st->len, w);
@@ -648,23 +662,16 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		while (!VGZ_IbufEmpty(vg)) {
 			VGZ_Obuf(vg, obuf, sizeof obuf);
 			i = VGZ_Gunzip(vg, &dp, &dl);
-			if (i == VGZ_END && !VGZ_IbufEmpty(vg)) {
-				htc->error = "Junk after gzip data";
-				return (-1);
-			}
-			if (i != VGZ_OK && i != VGZ_END) {
-				htc->error = "See other message";
-				WSP(sp, SLT_FetchError,
-				    "Invalid Gzip data: %s", vg->vz.msg);
-				return (-1);
-			}
+			if (i == VGZ_END && !VGZ_IbufEmpty(vg))
+				return(FetchError(sp, "Junk after gzip data"));
+			if (i != VGZ_OK && i != VGZ_END)
+				return(FetchError2(sp,
+				   "Invalid Gzip data", vg->vz.msg));
+
 		}
 	}
-	if (i == VGZ_OK || i == VGZ_END)
-		return (1);
-	htc->error = "See other message";
-	WSP(sp, SLT_FetchError, "Gunzip trouble (%d)", i);
-	return (-1);
+	assert(i == VGZ_OK || i == VGZ_END);
+	return (1);
 }
 
 static int __match_proto__()
@@ -675,8 +682,13 @@ vfp_testgzip_end(struct sess *sp)
 	vg = sp->wrk->vgz_rx;
 	sp->wrk->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	if (sp->wrk->fetch_failed) {
+		(void)VGZ_Destroy(&vg);
+		return(0);
+	}
 	VGZ_UpdateObj(vg, sp->obj);
-	VGZ_Destroy(&vg);
+	if (VGZ_Destroy(&vg) != VGZ_END)
+		return(FetchError(sp, "TestGunzip error at the very end"));
 	return (0);
 }
 
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index b27c8af..0f7e435 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -187,7 +187,7 @@ res_WriteGunzipObj(struct sess *sp)
 		(void)WRW_Write(sp->wrk, obuf, obufl);
 		(void)WRW_Flush(sp->wrk);
 	}
-	VGZ_Destroy(&vg);
+	(void)VGZ_Destroy(&vg);
 	assert(u == sp->obj->len);
 }
 
diff --git a/bin/varnishtest/tests/g00004.vtc b/bin/varnishtest/tests/g00004.vtc
new file mode 100644
index 0000000..5d518e0
--- /dev/null
+++ b/bin/varnishtest/tests/g00004.vtc
@@ -0,0 +1,43 @@
+varnishtest "truncated gzip from backend"
+
+server s1 -repeat 2 {
+	rxreq
+	txresp -nolen \
+		-hdr "Content-Encoding: gzip" \
+		-hdr "Transfer-Encoding: Chunked"
+	send "18\r\n"
+		# A truncate gzip file
+		sendhex "1f8b"
+		sendhex "08"
+		sendhex "00"
+		sendhex "f5 64 ae 4e  02 03 f3 cd cf 53 f0 4f"
+		sendhex "2e 51 30 36 54 30 b0 b4"
+	send "\r\n"
+	chunkedlen 0
+
+} -start
+
+varnish v1 \
+	-arg {-p diag_bitmap=0x00010000} \
+	-vcl+backend {
+	sub vcl_fetch {
+		if (req.url == "/gunzip") {
+			set beresp.do_gunzip = true;
+		} 
+	}
+} 
+
+varnish v1 -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+} -run
+
+client c1 {
+	txreq -url /gunzip
+	rxresp
+	expect resp.status == 503
+} -run
+
diff --git a/bin/varnishtest/tests/r01037.vtc b/bin/varnishtest/tests/r01037.vtc
new file mode 100644
index 0000000..acb77d3
--- /dev/null
+++ b/bin/varnishtest/tests/r01037.vtc
@@ -0,0 +1,29 @@
+varnishtest "Test case for #1037"
+
+server s1 {
+	rxreq
+	# Send a bodylen of 1,5M, which will exceed cache space of 1M
+	non-fatal
+	txresp -nolen -hdr "Transfer-encoding: chunked"
+	chunked {<HTML>}
+	chunkedlen 500000
+	chunked {<esi:remove>}
+	chunkedlen 500000
+	chunked {</esi:remove>}
+	chunkedlen 500000
+	chunkedlen 0
+} -start
+
+varnish v1 -arg "-smalloc,1M" -arg "-pgzip_level=0" -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_esi = true;
+		set beresp.do_gzip = true;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == "503"
+} -run
+



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] c41f68d Curse WG14
Message-ID: <E1SJhAr-00056F-3S@project.varnish-software.com>

commit c41f68d331364d30f708b2e779fe7fb43703fd02
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 20 20:09:54 2011 +0000

    Curse WG14

diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst
index 6264f55..822434c 100644
--- a/doc/sphinx/phk/index.rst
+++ b/doc/sphinx/phk/index.rst
@@ -8,6 +8,7 @@ You may or may not want to know what Poul-Henning think.
 
 .. toctree::
 
+	thetoolsweworkwith.rst
 	three-zero.rst
 	ssl.rst
 	gzip.rst
diff --git a/doc/sphinx/phk/thetoolsweworkwith.rst b/doc/sphinx/phk/thetoolsweworkwith.rst
new file mode 100644
index 0000000..cff365b
--- /dev/null
+++ b/doc/sphinx/phk/thetoolsweworkwith.rst
@@ -0,0 +1,211 @@
+.. _phk_thetoolsweworkwith:
+
+======================
+The Tools We Work With
+======================
+
+"Only amateurs were limited by their tools" is an old wisdom, and
+the world is littered with art and architecture that very much
+proves this point.
+
+But as amazing as the Aquaeduct of Segovia is, tools are the reason
+why it looks nowhere near as fantastic as the Sydney Opera House.
+
+Concrete has been known since antiquity, but steel-reinforced
+concrete and massive numerical calculations of stress-distribution,
+is the tools that makes the difference between using concrete as a
+filler material between stones, and as gravity-defying curved but
+perfectly safe load-bearing wall.
+
+My tool for writing Varnish is the C-language which in many ways
+is unique amongst all of the computer programming languages for
+having no ambitions.
+
+The C language was invented as a portable assembler language, it
+doesn't do objects and garbage-collection, it does numbers and
+pointers, just like your CPU.
+
+Compared to the high ambitions, then as now, of new programming
+languages, that was almost ridiculous unambitious.  Other people
+were trying to make their programming languages provably correct,
+or safe for multiprogramming and quite an effort went into using
+natural languages as programming languages.
+
+But C was written to write programs, not to research computer science
+and that's exactly what made it useful and popular.
+
+Unfortunately C fell in bad company over the years, and the reason
+for this outburst is that I just browsed the latest draft from the
+ISO-C standardisation working-group 14.
+
+I won't claim that it is enough to make grown men cry, but it
+certainly was enough to make me angry.
+
+Let me give you an example of their utter sillyness:
+
+The book which defined the C langauge had a list af reserved
+identifiers, all of them lower-case words.  The UNIX libraries
+defined a lot of functions, all of them lower-case words.
+
+When compiled, the assembler saw all of these words prefixed
+with an underscore, which made it easy to mix assembler and
+C code.
+
+All the macros for the C-preprocessor on the other hand, were
+UPPERCASE, making them easy to spot.
+
+Which meant that if you mixed upper and lower case, in your
+identifiers, you were safe: That wouldn't collide with anything.
+
+First the ISO-C standards people got confused about the leading
+underscore, and I'll leave you guessing as to what the current
+text actually means:
+
+	All identifiers that begin with an underscore and either
+	an uppercase letter or another underscore are always reserved
+	for any use.
+
+Feel free to guess.
+
+Next, they broke the upper/lower rule, by adding special keywords
+in mixed case, probably because they thought it looked nicer::
+
+	_Atomic, _Bool, _Noreturn &c
+
+Then, presumably, somebody pointed out that this looked ugly::
+
+	void _Noreturn foo(int bar);
+
+So they have come up with a #include file called <stdnoreturn.h>
+so that instead you can write::
+
+	#include <nostdreturn.h>
+	void noreturn foo(int bar);
+
+The <nostdreturn.h> file according to the standard shall have
+exactly this content::
+
+	#define noreturn _Noreturn
+
+Are you crying or laughing yet ?   You should be.
+
+Another thing brought by the new draft is an entirely new thread
+API, which is incompatible with the POSIX 'pthread' API which have
+been used for about 20 years now.
+
+If they had improved on the shortcomings of the pthreads, I would
+have cheered them on, because there are some very annoying mistakes
+in pthreads.
+
+But they didn't, in fact, as far as I can tell, the C1X draft's
+threads are worse than the 20 years older pthreads in all relevant
+aspects.
+
+For instance, neither pthreads nor C1X-threads offer a "assert I'm
+holding this mutex locked" facility.  I will posit that you cannot
+successfully develop real-world threaded programs and APIs that,
+or without wasting a lot of time debugging silly mistakes.
+
+If you look in the Varnish source code, which uses pthreads, you
+will see that I have wrapped pthread mutexes in my own little
+datastructure, to be able to do those asserts, and to get some
+usable statistics on lock-contention.
+
+Another example where C1X did not improve on pthreads at all, was
+in timed sleeps, where you say "get me this lock, but give up if
+it takes longer than X time".
+
+The way both pthreads and C1X threads do this, is you specify a UTC
+wall clock time you want to sleep until.
+
+The only problem with that is that UTC wall clock time is not
+continuous when implemented on a computer, and it may not even be
+monotonously increasing, since NTPD or other timesync facilites may
+step the clock backwards, particularly in the first minutes after
+boot.
+
+If the practice of saying "get me this lock before 16:00Z" was
+widespread, I could see the point, but I have actually never seen
+that in any source code.  What I have seen are wrappers that take
+the general shape of::
+
+	int
+	get_lock_timed(lock, timeout)
+	{
+		while (timeout > 0) {
+			t0 = time();
+			i = get_lock_before(lock, t + timeout));
+			if (i == WASLOCKED)
+				return (i);
+			t1 = time();
+			timeout -= (t1 - t0);
+		}
+		return (TIMEDOUT);
+	}
+
+Because it's not like the call is actually guaranteed to return at
+16:00Z if you ask it to, you are only promised it will not return
+later than that, so you have to wrap the call in a loop.
+
+Whoever defined the select(2) and poll(2) systemcalls knew better
+than the POSIX and ISO-C group-think:  They specifed a maximum
+duration for the call, because then it doesn't matter what time
+it is, only how long time has transpired.
+
+Ohh, and setting the stack-size for a new thread ?
+That is appearantly "too dangerous" so there is no argument in the
+C1X API for doing so, a clear step backwards from pthreads.
+
+But guess what:  Thread stacks are like T-shirts:  There is no "one
+size fits all."
+
+I have no idea what the "danger" they perceived were, my best
+guess is that feared it might make the API useful ?
+
+This single idiocy will single-handedly doom the C1X thread API
+to uselessness.
+
+Now, don't get me wrong:  There are lot of ways to improve the C
+language that would make sense:  Bitmaps, defined structure packing
+(think: communication protocol packets), big/little endian variables
+(data sharing), sensible handling of linked lists etc.
+
+As ugly as it is, even the printf()/scanf() format strings could
+be improved, by offering a sensible plugin mechanism, which the
+compiler can understand and use to issue warnings.
+
+Heck, even a simple basic object facility would be good addition,
+now that C++ have become this huge bloated monster language.
+
+But none of that is appearantly as important as <stdnoreturn.h>
+and a new, crippled and therefore useless thread API.
+
+The neat thing about the C language, and the one feature that made
+it so popular, is that not even an ISO-C working group can prevent
+you from implementing all these things using macros and other tricks.
+
+But it would be better to have them in the language, so the compiler
+could issue sensible warnings and programmers won't have to write
+monsters like::
+
+    #define VTAILQ_INSERT_BEFORE(listelm, elm, field) do {              \
+        (elm)->field.vtqe_prev = (listelm)->field.vtqe_prev;            \
+        VTAILQ_NEXT((elm), field) = (listelm);                          \
+        *(listelm)->field.vtqe_prev = (elm);                            \
+        (listelm)->field.vtqe_prev = &VTAILQ_NEXT((elm), field);        \
+    } while (0)
+
+To put an element on a linked list.
+
+I could go on like this, but it would rapidly become boring for
+both you and me, because the current C1X draft is 701 pages, and
+it contains not a single explanatory example if how to use any of
+the verbiage in pratice.
+
+Compare this with The C Programming Language, a book of 274 pages
+which in addition to define the C language, taught people how to
+program through well-thought-out examples.
+
+From where I sit, ISO WG14 are destroying the C language I use and love.
+
+Poul-Henning, 2011-12-20



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] f1b3895 updated syntax, thanks to Chris Handy for spotting it
Message-ID: <E1SJhAq-00055h-Sb@project.varnish-software.com>

commit f1b3895d14feb0c4b949c79ea0b4afd730c75388
Author: Per Buer <perbu at varnish-software.com>
Date:   Sun Dec 18 10:16:16 2011 +0100

    updated syntax, thanks to Chris Handy for spotting it

diff --git a/doc/sphinx/tutorial/increasing_your_hitrate.rst b/doc/sphinx/tutorial/increasing_your_hitrate.rst
index 31e32d0..0f0cbe8 100644
--- a/doc/sphinx/tutorial/increasing_your_hitrate.rst
+++ b/doc/sphinx/tutorial/increasing_your_hitrate.rst
@@ -32,8 +32,8 @@ Tool: varnishlog
 
 When you have identified the an URL which is frequently sent to the
 backend you can use varnishlog to have a look at the whole request.
-``varnishlog -c -o /foo/bar`` will give the whole (-o) requests coming
-from the client (-c) matching /foo/bar. 
+``varnishlog -c -m 'RxURL:^/foo/bar`` will give the whole (-o)
+requests coming from the client (-c) matching /foo/bar.
 
 For extended diagnostics headers, see
 http://www.varnish-cache.org/trac/wiki/VCLExampleHitMissHeader



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] 4b1466c typo. Thanks to 张学彬
Message-ID: <E1SJhAq-00054e-Es@project.varnish-software.com>

commit 4b1466c0998e6a2f95969c5087b473b42d2f165c
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Nov 30 10:38:43 2011 +0100

    typo. Thanks to ???

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index d7b71c3..31e51c1 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -498,7 +498,7 @@ vcl_pass
   client, but is not entered into the cache.  Subsequent requests sub?
   mitted over the same client connection are handled normally.
   
-  The vcl_recv subroutine may terminate with calling return() with one of
+  The vcl_pass subroutine may terminate with calling return() with one of
   the following keywords:
   
   error code [reason]



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] ed3d41b Remove the ref to -o for varnishlog Add a link to the
	varnishlog man page and the logging chapter in the tutorial
Message-ID: <E1SJhAq-00055p-Ui@project.varnish-software.com>

commit ed3d41bb1c0549de53b87410717bfb24cbfe1c44
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Dec 19 11:31:44 2011 +0100

    Remove the ref to -o for varnishlog
    Add a link to the varnishlog man page and the logging chapter in the tutorial

diff --git a/doc/sphinx/tutorial/increasing_your_hitrate.rst b/doc/sphinx/tutorial/increasing_your_hitrate.rst
index 0f0cbe8..c5f0c6a 100644
--- a/doc/sphinx/tutorial/increasing_your_hitrate.rst
+++ b/doc/sphinx/tutorial/increasing_your_hitrate.rst
@@ -31,9 +31,11 @@ Tool: varnishlog
 ~~~~~~~~~~~~~~~~
 
 When you have identified the an URL which is frequently sent to the
-backend you can use varnishlog to have a look at the whole request.
-``varnishlog -c -m 'RxURL:^/foo/bar`` will give the whole (-o)
-requests coming from the client (-c) matching /foo/bar.
+backend you can use varnishlog to have a look at the request.
+``varnishlog -c -m 'RxURL:^/foo/bar`` will show you the requests
+coming from the client (-c) matching /foo/bar.
+
+For more information on how varnishlog works please see :ref:`tutorial-logging`.
 
 For extended diagnostics headers, see
 http://www.varnish-cache.org/trac/wiki/VCLExampleHitMissHeader



From tfheen at varnish-cache.org  Mon Apr 16 08:43:11 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:43:11 +0200
Subject: [3.0] 8306db9 Call SES_Delete on sessions found to be closed during
	vca_return_session.
Message-ID: <E1SJhWh-0002QO-OV@project.varnish-software.com>

commit 8306db9a95c7b3022fdeee038b1e6973f46382f9
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Mar 7 11:12:55 2012 +0100

    Call SES_Delete on sessions found to be closed during vca_return_session.

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 007fe6e..c513e1a 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -382,9 +382,10 @@ vca_return_session(struct sess *sp)
 	 * Set nonblocking in the worker-thread, before passing to the
 	 * acceptor thread, to reduce syscall density of the latter.
 	 */
-	if (VTCP_nonblocking(sp->fd))
+	if (VTCP_nonblocking(sp->fd)) {
 		vca_close_session(sp, "remote closed");
-	else if (vca_act->pass == NULL)
+		SES_Delete(sp);
+	} else if (vca_act->pass == NULL)
 		assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
 	else
 		vca_act->pass(sp);



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] e8fb459 One more backend.list test to get all of the code.
Message-ID: <E1SJhAp-00051z-JZ@project.varnish-software.com>

commit e8fb459ea0d6c872d89b514ef2d646da18f0ffe8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 7 12:51:14 2011 +0000

    One more backend.list test to get all of the code.

diff --git a/bin/varnishtest/tests/c00048.vtc b/bin/varnishtest/tests/c00048.vtc
index 4ddb888..2502275 100644
--- a/bin/varnishtest/tests/c00048.vtc
+++ b/bin/varnishtest/tests/c00048.vtc
@@ -61,4 +61,5 @@ varnish v1 -clierr 106 "backend.set_health s2 auto"
 varnish v1 -cliok "backend.list (foo)"
 varnish v1 -clierr 300 "backend.list ("
 varnish v1 -clierr 300 {backend.list " ( : ) -"}
+varnish v1 -cliok {backend.list "a ( b : c )"}
 



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] 9cd0acb Fix inconsistant man page, as reported by Chris Adams
Message-ID: <E1SJhAr-00056h-82@project.varnish-software.com>

commit 9cd0acbd9c583b9078720952209870290696e93c
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Wed Dec 28 16:17:57 2011 +0100

    Fix inconsistant man page, as reported by Chris Adams

diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 91622d8..0b3f97f 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -15,7 +15,7 @@ Display Varnish logs in Apache / NCSA combined log format
 SYNOPSIS
 ========
 
-varnishncsa [-a] [-b] [-C] [-c] [-D] [-d] [-f] [-F format] [-I regex]
+varnishncsa [-a] [-C] [-D] [-d] [-f] [-F format] [-I regex]
 [-i tag] [-n varnish_name] [-m tag:regex ...] [-P file] [-r file] [-V] [-w file] 
 [-X regex] [-x tag]
 
@@ -30,16 +30,8 @@ The following options are available:
 
 -a          When writing to a file, append to it rather than overwrite it.
 
--b          Include log entries which result from communication with a 
-	    backend server.  If neither -b nor -c is
-	    specified, varnishncsa acts as if they both were.
-
 -C          Ignore case when matching regular expressions.
 
--c          Include log entries which result from communication 
-	    with a client.  If neither -b nor -c is specified, 
-	    varnishncsa acts as if they both were.
-
 -D          Daemonize.
 
 -d          Process old log entries on startup.  Normally, varnishncsa 



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] 7fa1728 Add some backend.list commands to exercise that code
	also
Message-ID: <E1SJhAp-00051Y-FV@project.varnish-software.com>

commit 7fa1728d9e58a331c80ea85721eaeb100be8085a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 7 11:44:06 2011 +0000

    Add some backend.list commands to exercise that code also

diff --git a/bin/varnishtest/tests/c00048.vtc b/bin/varnishtest/tests/c00048.vtc
index a4d0847..bf1c3c9 100644
--- a/bin/varnishtest/tests/c00048.vtc
+++ b/bin/varnishtest/tests/c00048.vtc
@@ -25,7 +25,9 @@ varnish v1 -vcl {
 
 delay 1
 
+varnish v1 -cliok "backend.list"
 varnish v1 -cliok "backend.set_health s1 auto"
+varnish v1 -cliok "backend.list s"
 
 client c1 {
 	txreq
@@ -33,7 +35,9 @@ client c1 {
 	expect resp.status == 200
 } -run
 
+varnish v1 -cliok "backend.list (:)"
 varnish v1 -cliok "backend.set_health s1 sick"
+varnish v1 -cliok "backend.list (1:)"
 
 client c1 {
 	txreq
@@ -41,7 +45,9 @@ client c1 {
 	expect resp.status == 503
 } -run
 
+varnish v1 -cliok "backend.list"
 varnish v1 -cliok "backend.set_health s1 healthy"
+varnish v1 -cliok "backend.list"
 
 client c1 {
 	txreq
@@ -52,4 +58,5 @@ client c1 {
 varnish v1 -clierr 106 "backend.set_health s1 foo"
 varnish v1 -clierr 106 "backend.set_health s2 foo"
 varnish v1 -clierr 106 "backend.set_health s2 auto"
+varnish v1 -cliok "backend.list (foo)"
 



From tfheen at varnish-cache.org  Mon Apr 16 08:20:38 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:38 +0200
Subject: [3.0] 654a81d Documentation fixes from Federico G. Schwindt
	<fgsch@lodoss.net>
Message-ID: <E1SJhAs-0005Aj-K1@project.varnish-software.com>

commit 654a81d522a51f486457e26f77ed0aaa9d41be43
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Thu Feb 23 17:57:50 2012 +0100

    Documentation fixes from Federico G. Schwindt <fgsch at lodoss.net>
    
    - mention vmod_std(7)
    - HEADER is released and used by vmod_std(7) so put it in its own line.
      The explanation is still needed though.
    - for consistency replace leading tabs with spaces
    - *fallback* are not strings so avoid mentioning the type
    - use ascii characters in vmod_std(7)
    
    And a few style changes from me to vmod_std.rst

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 2b515c1..f00053e 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -22,7 +22,7 @@ For instance::
 The "std" vmod is one you get with Varnish, it will always be there
 and we will put "boutique" functions in it, such as the "toupper"
 function shown above.  The full contents of the "std" module is
-documented in XXX:TBW.
+documented in vmod_std(7).
 
 This part of the manual is about how you go about writing your own
 VMOD, how the language interface between C and VCC works etc.  This
@@ -185,7 +185,12 @@ VOID
 	Can only be used for return-value, which makes the function a VCL
 	procedure.
 
-IP, BOOL, HEADER
+HEADER
+	C-type: ``enum gethdr_e, const char *``
+
+	XXX: explain me
+
+IP, BOOL
 	XXX: these types are not released for use in vmods yet.
 
 
diff --git a/doc/sphinx/reference/vmod_std.rst b/doc/sphinx/reference/vmod_std.rst
index 1ea8193..9c28a3c 100644
--- a/doc/sphinx/reference/vmod_std.rst
+++ b/doc/sphinx/reference/vmod_std.rst
@@ -31,31 +31,31 @@ toupper
 -------
 
 Prototype
-	toupper(STRING S)
+	toupper(STRING s)
 Return value
-       String
+	String
 Description
-	Converts the STRING S to upper case.
+	Converts the string *s* to upper case.
 Example
 	set beresp.http.x-scream = std.toupper("yes!");
 
 tolower
 -------
 Prototype
-	tolower(STRING S)
+	tolower(STRING s)
 Return value
-       String
+	String
 Description
-	Converts the STRING to lower case.
+	Converts the string *s* to lower case.
 Example
-        set beresp.http.x-nice = std.tolower("VerY");
+	set beresp.http.x-nice = std.tolower("VerY");
 
 set_up_tos
 ----------
 Prototype
-	set_ip_tos(INT I)
+	set_ip_tos(INT i)
 Return value
-       Void
+	Void
 Description
 	Sets the Type-of-Service flag for the current session. Please
 	note that the TOS flag is not removed by the end of the
@@ -71,7 +71,7 @@ random
 Prototype
 	random(REAL a, REAL b)
 Return value
-       Real
+	Real
 Description
 	Returns a random REAL number between *a* and *b*.
 Example
@@ -82,9 +82,9 @@ log
 Prototype
 	log(STRING string)
 Return value
-       Void
+	Void
 Description
-	Logs string to the shared memory log.
+	Logs *string* to the shared memory log.
 Example
 	std.log("Something fishy is going on with the vhost " + req.host);
 
@@ -93,7 +93,7 @@ syslog
 Prototype
 	syslog(INT priority, STRING string)
 Return value
-        Void
+	Void
 Description
 	Logs *string* to syslog marked with *priority*.
 Example
@@ -104,7 +104,7 @@ fileread
 Prototype
 	fileread(STRING filename)
 Return value
-        String
+	String
 Description
 	Reads a file and returns a string with the content. Please
 	note that it is not recommended to send variables to this
@@ -118,12 +118,11 @@ duration
 Prototype
 	duration(STRING s, DURATION fallback)
 Return value
-       Duration
+	Duration
 Description
-	Converts the string s to seconds. s can be quantified with the
-	usual s (seconds), m (minutes), h (hours), d (days) and w
-	(weeks) units. If it fails to parse the string *fallback* 
-	will be used
+	Converts the string *s* to seconds. *s* can be quantified with
+	the usual s (seconds), m (minutes), h (hours), d (days) and w
+	(weeks) units. If *s* fails to parse, *fallback* will be used.
 Example
 	set beresp.ttl = std.duration("1w", 3600);
 
@@ -132,19 +131,19 @@ integer
 Prototype
 	integer(STRING s, INT fallback)
 Return value
-       Int
+	Int
 Description
-	Converts the string s to an integer.  If it fails to parse the
-	string *fallback* will be used
+	Converts the string *s* to an integer.  If *s* fails to parse,
+	*fallback* will be used
 Example
-	if (std.integer(beresp.http.x-foo, 0) > 5) { ? }
+	if (std.integer(beresp.http.x-foo, 0) > 5) { ... }
 
 collect
 -------
 Prototype
 	collect(HEADER header)
 Return value
-       Void
+	Void
 Description
 	Collapses the header, joining the headers into one.
 Example



From tfheen at varnish-cache.org  Mon Apr 16 08:20:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:37 +0200
Subject: [3.0] d243c2d Another typo
Message-ID: <E1SJhAr-00056T-5o@project.varnish-software.com>

commit d243c2dc3c46ffcf1c078c8c6aea96c6684043c4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 20 20:30:28 2011 +0000

    Another typo

diff --git a/doc/sphinx/phk/thetoolsweworkwith.rst b/doc/sphinx/phk/thetoolsweworkwith.rst
index 6921926..2631c73 100644
--- a/doc/sphinx/phk/thetoolsweworkwith.rst
+++ b/doc/sphinx/phk/thetoolsweworkwith.rst
@@ -200,7 +200,7 @@ To put an element on a linked list.
 I could go on like this, but it would rapidly become boring for
 both you and me, because the current C1X draft is 701 pages, and
 it contains not a single explanatory example if how to use any of
-the verbiage in pratice.
+the verbiage in practice.
 
 Compare this with The C Programming Language, a book of 274 pages
 which in addition to define the C language, taught people how to



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] 0beaedb add a ref to man varnishlog
Message-ID: <E1SJhAq-00055w-Vv@project.varnish-software.com>

commit 0beaedbc9e9f72d756341dd4969bd27b243ce032
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Dec 19 12:29:11 2011 +0100

    add a ref to man varnishlog

diff --git a/doc/sphinx/tutorial/increasing_your_hitrate.rst b/doc/sphinx/tutorial/increasing_your_hitrate.rst
index c5f0c6a..83236d6 100644
--- a/doc/sphinx/tutorial/increasing_your_hitrate.rst
+++ b/doc/sphinx/tutorial/increasing_your_hitrate.rst
@@ -35,7 +35,8 @@ backend you can use varnishlog to have a look at the request.
 ``varnishlog -c -m 'RxURL:^/foo/bar`` will show you the requests
 coming from the client (-c) matching /foo/bar.
 
-For more information on how varnishlog works please see :ref:`tutorial-logging`.
+For more information on how varnishlog works please see
+:ref:`tutorial-logging` or man :ref:`ref-varnishlog`.
 
 For extended diagnostics headers, see
 http://www.varnish-cache.org/trac/wiki/VCLExampleHitMissHeader



From tfheen at varnish-cache.org  Mon Apr 16 08:20:35 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:35 +0200
Subject: [3.0] 64a8c2a varnishstat: Add json output and continous mode
Message-ID: <E1SJhAp-00052I-Ly@project.varnish-software.com>

commit 64a8c2a283ef6e2a8992d99ffb263b8dbae3cfcd
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Mon Nov 7 13:14:35 2011 +0100

    varnishstat: Add json output and continous mode

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index b5b1e6f..d487b54 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -84,6 +84,69 @@ do_xml(struct VSM_data *vd)
 	printf("</varnishstat>\n");
 }
 
+
+/*--------------------------------------------------------------------*/
+
+static int
+do_json_cb(void *priv, const struct VSC_point * const pt)
+{
+	uint64_t val;
+	int *jp;
+	char jsonkey[255];
+	char jsontmp[255];
+
+	jp = priv;
+
+	assert(!strcmp(pt->fmt, "uint64_t"));
+	val = *(const volatile uint64_t*)pt->ptr;
+
+	if (*jp) *jp = 0; else printf(",\n");
+	jsonkey[0] = '\0';
+
+	/* build the JSON key name.  */
+	if (strcmp(pt->ident, "")  && strcmp(pt->class, ""))  sprintf(jsonkey, "%s.%s", pt->class, pt->ident);
+	if (strcmp(pt->ident, "")  && !strcmp(pt->class, "")) sprintf(jsonkey, "%s", pt->ident);
+	if (!strcmp(pt->ident, "") && strcmp(pt->class, ""))  sprintf(jsonkey, "%s", pt->class);
+
+	strcpy(jsontmp, jsonkey);
+	if (strcmp(jsonkey, "")) sprintf(jsonkey, "%s.%s", jsontmp, pt->name);
+	else strcpy(jsonkey, pt->name);
+
+	printf("\t\"%s\": {", jsonkey);
+
+	if (strcmp(pt->class, "")) printf("\"type\": \"%s\", ",  pt->class);
+	if (strcmp(pt->ident, "")) printf("\"ident\": \"%s\", ", pt->ident);
+
+	printf("\"value\": %ju, ", val);
+
+	printf("\"flag\": \"%c\", ", pt->flag);
+	printf("\"description\": \"%s\"", pt->desc);
+	printf("}");
+
+	if (*jp) printf("\n");
+	return (0);
+}
+
+static void
+do_json(struct VSM_data *vd)
+{
+	char time_stamp[20];
+	time_t now;
+	int jp;
+
+	jp = 1;
+
+	printf("{\n");
+	now = time(NULL);
+
+	(void)strftime(time_stamp, 20, "%Y-%m-%dT%H:%M:%S", localtime(&now));
+	printf("\t\"timestamp\": \"%s\",\n", time_stamp);
+	(void)VSC_Iter(vd, do_json_cb, &jp);
+	printf("\n}\n");
+	fflush(stdout);
+}
+
+
 /*--------------------------------------------------------------------*/
 
 struct once_priv {
@@ -169,7 +232,7 @@ usage(void)
 	    "[-1lV] [-f field_list] "
 	    VSC_n_USAGE " "
 	    "[-w delay]\n");
-	fprintf(stderr, FMT, "-1", "Print the statistics once and exit");
+	fprintf(stderr, FMT, "-1", "Print the statistics to stdout.");
 	fprintf(stderr, FMT, "-f field_list",
 	    "Comma separated list of fields to display. ");
 	fprintf(stderr, FMT, "",
@@ -180,9 +243,11 @@ usage(void)
 	    "The varnishd instance to get logs from");
 	fprintf(stderr, FMT, "-V", "Display the version number and exit");
 	fprintf(stderr, FMT, "-w delay",
-	    "Wait delay seconds between updates.  The default is 1.");
+	    "Wait delay seconds between updates.  Default is 1 second. Can also be be used with -1, -x or -j for repeated output.");
 	fprintf(stderr, FMT, "-x",
-	    "Print statistics once as XML and exit.");
+	    "Print statistics to stdout as XML.");
+	fprintf(stderr, FMT, "-j",
+	    "Print statistics to stdout as JSON.");
 #undef FMT
 	exit(1);
 }
@@ -193,12 +258,12 @@ main(int argc, char * const *argv)
 	int c;
 	struct VSM_data *vd;
 	const struct VSC_C_main *VSC_C_main;
-	int delay = 1, once = 0, xml = 0;
+	int delay = 1, once = 0, xml = 0, json = 0, do_repeat = 0;
 
 	vd = VSM_New();
 	VSC_Setup(vd);
 
-	while ((c = getopt(argc, argv, VSC_ARGS "1f:lVw:x")) != -1) {
+	while ((c = getopt(argc, argv, VSC_ARGS "1f:lVw:xjt:")) != -1) {
 		switch (c) {
 		case '1':
 			once = 1;
@@ -212,11 +277,15 @@ main(int argc, char * const *argv)
 			VCS_Message("varnishstat");
 			exit(0);
 		case 'w':
+			do_repeat = 1;
 			delay = atoi(optarg);
 			break;
 		case 'x':
 			xml = 1;
 			break;
+		case 'j':
+			json = 1;
+			break;
 		default:
 			if (VSC_Arg(vd, c, optarg) > 0)
 				break;
@@ -229,12 +298,27 @@ main(int argc, char * const *argv)
 
 	VSC_C_main = VSC_Main(vd);
 
-	if (xml)
-		do_xml(vd);
-	else if (once)
-		do_once(vd, VSC_C_main);
-	else
+	if (!(xml || json || once)) {
 		do_curses(vd, VSC_C_main, delay);
+		exit(0);
+	}
+
+	while (1) {
+		if (xml)
+			do_xml(vd);
+		else if (json)
+			do_json(vd);
+		else if (once)
+			do_once(vd, VSC_C_main);
+		else {
+			assert(0);
+		}
+		if (!do_repeat) break;
+
+		// end of output block marker.
+		printf("\n");
 
+		sleep(delay);
+	} 
 	exit(0);
 }
diff --git a/doc/sphinx/reference/varnishstat.rst b/doc/sphinx/reference/varnishstat.rst
index 09c4989..3c5d541 100644
--- a/doc/sphinx/reference/varnishstat.rst
+++ b/doc/sphinx/reference/varnishstat.rst
@@ -10,15 +10,16 @@ Varnish Cache statistics
 
 :Author: Dag-Erling Sm?rgrav
 :Author: Per Buer
-:Date:   2010-06-1
-:Version: 1.0
+:Author: Lasse Karstensen
+:Date:   2011-11-07
+:Version: 1.1
 :Manual section: 1
 
 
 SYNOPSIS
 ========
 
-varnishstat [-1] [-x] [-f field_list] [-l] [-n varnish_name] [-V] [-w delay]
+varnishstat [-1] [-x] [-j] [-f field_list] [-l] [-n varnish_name] [-V] [-w delay]
 
 DESCRIPTION
 ===========
@@ -27,7 +28,7 @@ The varnishstat utility displays statistics from a running varnishd(1) instance.
 
 The following options are available:
 
--1          Instead of presenting of a continuously updated display, print the statistics once and exit.
+-1          Instead of presenting of a continuously updated display, print the statistics to stdout.
 
 -f          A comma separated list of the fields to display.  If it starts with '^' it is used as an exclusion
 	    list.
@@ -39,9 +40,11 @@ The following options are available:
 
 -V          Display the version number and exit.
 
--w delay    Wait delay seconds between updates.  The default is 1.
+-w delay    Wait delay seconds between updates.  The default is 1. Can also be used with -1, -x or -j for repeated output.
 
--x          Displays the result as XML once.
+-x          Displays the result as XML.
+
+-j          Displays the result as JSON.
 
 The columns in the main display are, from left to right:
 
@@ -65,6 +68,29 @@ When using the -x option, the output is::
     <description>FIELD DESCRIPTION</description> 
   </stat> 
 
+With -j the output format is::
+
+  {
+    "timestamp": "YYYY-MM-DDTHH:mm:SS",
+    "client_conn": {
+	"value": 0, "flag": "a", 
+	"description": "Client connections accepted"
+    },
+    "client_drop": {
+	"value": 0, "flag": "a", 
+	"description": "Connection dropped, no sess/wrk"
+    },
+    "LCK.backend.creat": {
+	"type": "LCK", "ident": "backend", "value": 1, 
+	"flag": "a", "description": "Created locks"
+    },
+    [..]
+  }
+
+Timestamp is the time when the report was generated by varnishstat.
+
+Repeated output with -1, -x or -j will have a single empty line (\\n) between each block of output.
+
 
 SEE ALSO
 ========
@@ -91,4 +117,4 @@ This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
 * Copyright (c) 2006 Verdens Gang AS
-* Copyright (c) 2006-2008 Varnish Software AS
+* Copyright (c) 2006-2011 Varnish Software AS



From tfheen at varnish-cache.org  Mon Apr 16 08:20:38 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:38 +0200
Subject: [3.0] 835a072 Add proper handling of TCP timeouts on client side
Message-ID: <E1SJhAs-0005B4-Mi@project.varnish-software.com>

commit 835a0722eabe261ebcb9747aba7a445dea3120f0
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Nov 7 14:05:36 2011 +0100

    Add proper handling of TCP timeouts on client side
    
    Retry if we hit a TCP timeout if some data was sent and only error out
    when we get an error or no data was sent in the time period.

diff --git a/bin/varnishd/cache_wrw.c b/bin/varnishd/cache_wrw.c
index f25bafb..286bc45 100644
--- a/bin/varnishd/cache_wrw.c
+++ b/bin/varnishd/cache_wrw.c
@@ -123,7 +123,37 @@ WRW_Flush(struct worker *w)
 			wrw->iov[wrw->ciov].iov_len = 0;
 		}
 		i = writev(*wrw->wfd, wrw->iov, wrw->niov);
-		if (i != wrw->liov) {
+		while (i != wrw->liov && i > 0) {
+			/* Remove sent data from start of I/O vector,
+			 * then retry; we hit a timeout, but some data
+			 * was sent.
+
+			 XXX: Add a "minimum sent data per timeout
+			 counter to prevent slowlaris attacks
+			*/
+			size_t used = 0;
+
+			WSL(w, SLT_Debug, *wrw->wfd,
+			    "Hit send timeout, wrote = %ld/%ld; retrying",
+			    i, wrw->liov);
+
+			for (int j = 0; j < wrw->niov; j++) {
+				if (used + wrw->iov[j].iov_len > i) {
+					/* Cutoff is in this iov */
+					int used_here = i - used;
+					wrw->iov[j].iov_len -= used_here;
+					wrw->iov[j].iov_base = (char*)wrw->iov[j].iov_base + used_here;
+					memmove(wrw->iov, &wrw->iov[j],
+						(wrw->niov - j) * sizeof(struct iovec));
+					wrw->niov -= j;
+					wrw->liov -= i;
+					break;
+				}
+				used += wrw->iov[j].iov_len;
+			}
+			i = writev(*wrw->wfd, wrw->iov, wrw->niov);
+		}
+		if (i <= 0) {
 			wrw->werr++;
 			WSL(w, SLT_Debug, *wrw->wfd,
 			    "Write error, retval = %d, len = %d, errno = %s",



From tfheen at varnish-cache.org  Mon Apr 16 08:20:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:34 +0200
Subject: [3.0] e17acd8 Polish the chunked body fetcher. It still irks me that
	it does one byte reads for the hex-length headers.
Message-ID: <E1SJhAo-0004zQ-UA@project.varnish-software.com>

commit e17acd8f99179ff193ab5854a57445896ea7a72f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 31 22:19:45 2011 +0000

    Polish the chunked body fetcher.  It still irks me that it does
    one byte reads for the hex-length headers.
    
    Conflicts:
    
    	bin/varnishd/cache_fetch.c

diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 29d8b00..53acf25 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -251,9 +251,9 @@ fetch_straight(struct sess *sp, struct http_conn *htc, ssize_t cl)
 
 /*--------------------------------------------------------------------
  * Read a chunked HTTP object.
+ *
  * XXX: Reading one byte at a time is pretty pessimal.
  */
- 
 
 static int
 fetch_chunked(struct sess *sp, struct http_conn *htc)
@@ -267,17 +267,18 @@ fetch_chunked(struct sess *sp, struct http_conn *htc)
 	do {
 		/* Skip leading whitespace */
 		do {
-			i = HTC_Read(sp->wrk, htc, buf, 1);
-			if (i <= 0)
-				return (i);
+			if (HTC_Read(sp->wrk, htc, buf, 1) <= 0)
+				return (-1);
 		} while (vct_islws(buf[0]));
 
+		if (!vct_ishex(buf[0]))
+			return (FetchError(sp,"chunked header non-hex"));
+
 		/* Collect hex digits, skipping leading zeros */
 		for (u = 1; u < sizeof buf; u++) {
 			do {
-				i = HTC_Read(sp->wrk, htc, buf + u, 1);
-				if (i <= 0)
-					return (i);
+				if (HTC_Read(sp->wrk, htc, buf + u, 1) <= 0)
+					return (-1);
 			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
 			if (!vct_ishex(buf[u]))
 				break;
@@ -288,41 +289,32 @@ fetch_chunked(struct sess *sp, struct http_conn *htc)
 		}
 
 		/* Skip trailing white space */
-		while(vct_islws(buf[u]) && buf[u] != '\n') {
-			i = HTC_Read(sp->wrk, htc, buf + u, 1);
-			if (i <= 0)
-				return (i);
-		}
+		while(vct_islws(buf[u]) && buf[u] != '\n')
+			if (HTC_Read(sp->wrk, htc, buf + u, 1) <= 0)
+				return (-1);
 
-		if (buf[u] != '\n') {
+		if (buf[u] != '\n') 
 			return (FetchError(sp,"chunked header no NL"));
-		}
-		buf[u] = '\0';
 
+		buf[u] = '\0';
 		cl = fetch_number(buf, 16);
-		if (cl < 0) {
+		if (cl < 0)
 			return (FetchError(sp,"chunked header number syntax"));
-		} else if (cl > 0) {
-			i = sp->wrk->vfp->bytes(sp, htc, cl);
-			if (i <= 0)
-				return (-1);
-		}
+
+		if (cl > 0 && sp->wrk->vfp->bytes(sp, htc, cl) <= 0)
+			return (-1);
+
 		i = HTC_Read(sp->wrk, htc, buf, 1);
 		if (i <= 0)
 			return (-1);
-		if (buf[0] == '\r') {
-			i = HTC_Read(sp->wrk, htc, buf, 1);
-			if (i <= 0)
-				return (-1);
-		}
+		if (buf[0] == '\r' && HTC_Read(sp->wrk, htc, buf, 1) <= 0)
+			return (-1);
 		if (buf[0] != '\n')
 			return (FetchError(sp,"chunked tail no NL"));
 	} while (cl > 0);
 	return (0);
 }
 
-#undef CERR
-
 /*--------------------------------------------------------------------*/
 
 static int



From tfheen at varnish-cache.org  Mon Apr 16 08:20:36 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 10:20:36 +0200
Subject: [3.0] 5449220 Some clarification on how 'now' works I'm not sure if
	we should go into more detail about how the whole thing works
Message-ID: <E1SJhAq-00055H-O0@project.varnish-software.com>

commit 5449220e3025a7d62c49b65aa699b55a4f406aac
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Dec 14 16:04:39 2011 +0100

    Some clarification on how 'now' works
    I'm not sure if we should go into more detail about how the whole thing works

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 31e51c1..d39128b 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -673,7 +673,8 @@ made available to the handler subroutines through global variables.
 The following variables are always available:
 
 now   
-  The current time, in seconds since the epoch.
+  The current time, in seconds since the epoch. When used in string context
+  it returns a formatted string.
 
 The following variables are available in backend declarations:
 



From phk at varnish-cache.org  Mon Apr 16 10:21:14 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 16 Apr 2012 12:21:14 +0200
Subject: [master] 7175bb3 Remove the -w argument, -p should be used instead.
Message-ID: <E1SJj3a-0004x8-VG@project.varnish-software.com>

commit 7175bb38e9f8ea5df2daf3725b9d624ef81cface
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 16 10:20:55 2012 +0000

    Remove the -w argument, -p should be used instead.

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 97ac5da..7047fa9 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -100,20 +100,6 @@ pick(const struct choice *cp, const char *which, const char *kind)
 
 /*--------------------------------------------------------------------*/
 
-static unsigned long
-arg_ul(const char *p)
-{
-	char *q;
-	unsigned long ul;
-
-	ul = strtoul(p, &q, 0);
-	if (*q != '\0')
-		ARGV_ERR("Invalid number: \"%s\"\n", p);
-	return (ul);
-}
-
-/*--------------------------------------------------------------------*/
-
 static void
 usage(void)
 {
@@ -160,52 +146,11 @@ usage(void)
 	fprintf(stderr, FMT, "-T address:port",
 	    "Telnet listen address and port");
 	fprintf(stderr, FMT, "-V", "version");
-	fprintf(stderr, FMT, "-w int[,int[,int]]", "Number of worker threads");
-	fprintf(stderr, FMT, "", "  -w <fixed_count>");
-	fprintf(stderr, FMT, "", "  -w min,max");
-	fprintf(stderr, FMT, "", "  -w min,max,timeout [default: -w2,500,300]");
 	fprintf(stderr, FMT, "-u user", "Priviledge separation user id");
 #undef FMT
 	exit(1);
 }
 
-
-/*--------------------------------------------------------------------*/
-
-static void
-tackle_warg(const char *argv)
-{
-	char **av;
-	unsigned int u;
-
-	av = VAV_Parse(argv, NULL, ARGV_COMMA);
-	AN(av);
-
-	if (av[0] != NULL)
-		ARGV_ERR("%s\n", av[0]);
-
-	if (av[1] == NULL)
-		usage();
-
-	u = arg_ul(av[1]);
-	if (u < 1)
-		usage();
-	mgt_param.wthread_max = mgt_param.wthread_min = u;
-
-	if (av[2] != NULL) {
-		u = arg_ul(av[2]);
-		if (u < mgt_param.wthread_min)
-			usage();
-		mgt_param.wthread_max = u;
-
-		if (av[3] != NULL) {
-			u = arg_ul(av[3]);
-			mgt_param.wthread_timeout = u;
-		}
-	}
-	VAV_Free(av);
-}
-
 /*--------------------------------------------------------------------*/
 
 static void
@@ -516,8 +461,7 @@ main(int argc, char * const *argv)
 			usage();
 			break;
 		case 'w':
-			tackle_warg(optarg);
-			break;
+			ARGV_ERR("-w has been removed, use -p instead\n");
 		default:
 			usage();
 		}
diff --git a/bin/varnishtest/tests/v00006.vtc b/bin/varnishtest/tests/v00006.vtc
index f3faace..ca4c827 100644
--- a/bin/varnishtest/tests/v00006.vtc
+++ b/bin/varnishtest/tests/v00006.vtc
@@ -8,7 +8,7 @@ server s1 {
 } -start
 
 # Only one pool, to avoid getting more than one work thread
-varnish v1 -arg "-p thread_pools=1 -w1,1,300" -vcl+backend {
+varnish v1 -arg "-p thread_pools=1" -vcl+backend {
 } -start
 
 client c1 {



From perbu at varnish-cache.org  Mon Apr 16 11:20:41 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Mon, 16 Apr 2012 13:20:41 +0200
Subject: [master] 9a1b232 remove the -w option from man varnishd
Message-ID: <E1SJjz7-00040O-MZ@project.varnish-software.com>

commit 9a1b232b4a9ef1fbfa68d79499b598052149f2d1
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Apr 16 13:20:38 2012 +0200

    remove the -w option from man varnishd

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 351c7b3..f0647b3 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -23,7 +23,7 @@ varnishd [-a address[:port]] [-b host[:port]] [-d] [-F] [-f config]
 	 [-g group] [-h type[,options]] [-i identity]
 	 [-l shmlogsize] [-n name] [-P file] [-p param=value] 
 	 [-s type[,options]] [-T address[:port]] [-t ttl]
-	 [-u user] [-V] [-w min[,max[,timeout]]]
+	 [-u user] [-V] 
 
 DESCRIPTION
 ===========
@@ -120,20 +120,6 @@ OPTIONS
 
 -V          Display the version number and exit.
 
--w min[,max[,timeout]]
-
-            Start at least min but no more than max worker threads
-            with the specified idle timeout.  This is a shortcut for
-            specifying the thread_pool_min, thread_pool_max and
-            thread_pool_timeout run-time parameters.
-
-            If only one number is specified, thread_pool_min and
-            thread_pool_max are both set to this number, and
-            thread_pool_timeout has no effect.
-
-
-
-
 
 Hash Algorithms
 ---------------



From perbu at varnish-cache.org  Mon Apr 16 11:37:51 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Mon, 16 Apr 2012 13:37:51 +0200
Subject: [master] 8931bc0 :: need to be on a separate line in RST.
Message-ID: <E1SJkFj-0007Ba-0f@project.varnish-software.com>

commit 8931bc07513bdb4cbc111ebeff7dc60e24371197
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Apr 16 13:37:47 2012 +0200

    :: need to be on a separate line in RST.

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 5a5ca6e..b8f47c3 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -96,14 +96,16 @@ file as a quoted string.
 Backend declarations
 --------------------
 
-A backend declaration creates and initializes a named backend object::
+A backend declaration creates and initializes a named backend object:
+::
 
   backend www {
     .host = "www.example.com";
     .port = "http";
   }
 
-The backend object can later be used to select a backend at request time::
+The backend object can later be used to select a backend at request time:
+::
 
   if (req.http.host ~ "(?i)^(www.)?example.com$") {
     set req.backend = www;
@@ -118,7 +120,8 @@ backend connection, .first_byte_timeout for the time to wait for the
 first byte from the backend and .between_bytes_timeout for time to
 wait between each received byte.
 
-These can be set in the declaration like this::
+These can be set in the declaration like this:
+::
 
   backend www {
     .host = "www.example.com";
@@ -145,7 +148,8 @@ be used.
 There are several types of directors. The different director types
 use different algorithms to choose which backend to use.
 
-Configuring a director may look like this::
+Configuring a director may look like this:
+::
 
   director b2 random {
     .retries = 5;
@@ -226,7 +230,8 @@ The DNS director
 ~~~~~~~~~~~~~~~~
 
 The DNS director can use backends in two different ways. Either like the
-random or round-robin director or using .list::
+random or round-robin director or using .list:
+::
 
   director directorname dns {
           .list = {
@@ -266,7 +271,8 @@ considers them in the order in which they are listed in its definition.
 
 The fallback director does not take any options.
 
-An example of a fallback director::
+An example of a fallback director:
+::
 
   director b3 fallback {
     { .backend = www1; }
@@ -312,7 +318,8 @@ Probes take the following parameters:
   Default is 2 seconds.
 
 A backend with a probe can be defined like this, together with the
-backend or director::
+backend or director:
+::
 
   backend www {
     .host = "www.example.com";
@@ -326,7 +333,8 @@ backend or director::
     }
   }
 
-Or it can be defined separately and then referenced::
+Or it can be defined separately and then referenced:
+::
 
   probe healthcheck {
      .url = "/status.cgi";
@@ -347,7 +355,8 @@ Or it can be defined separately and then referenced::
 If you have many backends this can simplify the config a lot.
 
 
-It is also possible to specify the raw HTTP request::
+It is also possible to specify the raw HTTP request:
+::
 
   probe rawprobe {
       # NB: \r\n automatically inserted after each string!
@@ -361,7 +370,8 @@ ACLs
 ----
 
 An ACL declaration creates and initializes a named access control list
-which can later be used to match client addresses::
+which can later be used to match client addresses:
+::
 
   acl local {
     "localhost";         // myself
@@ -375,7 +385,8 @@ if it is preceded by a negation mark, it will reject any address it is
 compared to, which may not be what you intended.  If the entry is
 enclosed in parentheses, however, it will simply be ignored.
 
-To match an IP address against an ACL, simply use the match operator::
+To match an IP address against an ACL, simply use the match operator:
+::
 
   if (client.ip ~ local) {
     return (pipe);
@@ -390,7 +401,8 @@ PCRE(3) man page.
 
 To send flags to the PCRE engine, such as to turn on *case
 insensitivity* add the flag within parens following a question mark,
-like this::
+like this:
+::
 
   if (req.http.host ~ "(?i)example.com$") {
           ...
@@ -424,7 +436,8 @@ ban_url(regex)
 Subroutines
 ~~~~~~~~~~~
 
-A subroutine is used to group code for legibility or reusability::
+A subroutine is used to group code for legibility or reusability:
+::
   
   sub pipe_if_local {
     if (client.ip ~ local) {
@@ -646,7 +659,8 @@ appear in the source.
 The default versions distributed with Varnish will be implicitly
 concatenated as a last resort at the end.
 
-Example::
+Example:
+::
 
 	# in file "main.vcl"
 	include "backends.vcl";
@@ -893,7 +907,8 @@ resp.response
 resp.http.header
   The corresponding HTTP header.
 
-Values may be assigned to variables using the set keyword::
+Values may be assigned to variables using the set keyword:
+::
 
   sub vcl_recv {
     # Normalize the Host: header
@@ -902,7 +917,8 @@ Values may be assigned to variables using the set keyword::
     }
   }
 
-HTTP headers can be removed entirely using the remove keyword::
+HTTP headers can be removed entirely using the remove keyword:
+::
 
   sub vcl_fetch {
     # Don't cache cookies
@@ -919,7 +935,8 @@ fresh object is being generated by the backend.
 
 The following vcl code will make Varnish serve expired objects.  All
 object will be kept up to two minutes past their expiration time or a
-fresh object is generated.::
+fresh object is generated.
+::
 
   sub vcl_recv {
     set req.grace = 2m;
@@ -944,7 +961,8 @@ EXAMPLES
 
 The following code is the equivalent of the default configuration with
 the backend address set to "backend.example.com" and no backend port
-specified::
+specified:
+::
 
   backend default {
    .host = "backend.example.com";
@@ -956,7 +974,8 @@ specified::
 
 The following example shows how to support multiple sites running on
 separate backends in the same Varnish instance, by selecting backends
-based on the request URL::
+based on the request URL:
+::
 
   backend www {
     .host = "www.example.com";
@@ -982,7 +1001,8 @@ based on the request URL::
 The following snippet demonstrates how to force a minimum TTL for
 all documents.  Note that this is not the same as setting the
 default_ttl run-time parameter, as that only affects document for
-which the backend did not specify a TTL::
+which the backend did not specify a TTL:
+::
   
   import std; # needed for std.log
 
@@ -994,7 +1014,8 @@ which the backend did not specify a TTL::
   }
 
 The following snippet demonstrates how to force Varnish to cache
-documents even when cookies are present::
+documents even when cookies are present:
+::
 
   sub vcl_recv {
     if (req.request == "GET" && req.http.cookie) {
@@ -1009,7 +1030,8 @@ documents even when cookies are present::
   }
 
 The following code implements the HTTP PURGE method as used by Squid
-for object invalidation::
+for object invalidation:
+::
 
   acl purge {
     "localhost";



From tfheen at varnish-cache.org  Mon Apr 16 12:17:17 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Apr 2012 14:17:17 +0200
Subject: [master] 3a82559 Fix escaping in regsub docs
Message-ID: <E1SJkrt-0004Az-PS@project.varnish-software.com>

commit 3a82559d539246d3d3fedee8aa76b6e93a0b9b03
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Apr 16 14:11:30 2012 +0200

    Fix escaping in regsub docs

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index b8f47c3..ed970e2 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -420,9 +420,9 @@ hash_data(str)
 
 regsub(str, regex, sub)
   Returns a copy of str with the first occurrence of the regular 
-  expression regex replaced with sub. Within sub, \\0 (which can 
-  also be spelled &) is replaced with the entire matched string, 
-  and \n is replaced with the contents of subgroup n in the 
+  expression regex replaced with sub. Within sub, \\0 (which can
+  also be spelled \\&) is replaced with the entire matched string,
+  and \\n is replaced with the contents of subgroup n in the 
   matched string.
 
 regsuball(str, regex, sub)



From phk at varnish-cache.org  Mon Apr 16 13:55:03 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 16 Apr 2012 15:55:03 +0200
Subject: [master] 2612267 Remove unused #include
Message-ID: <E1SJmOV-0000SN-6H@project.varnish-software.com>

commit 26122673a7fc867f22e06ed4e7ff82a789463bfe
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 16 13:54:55 2012 +0000

    Remove unused #include

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 7047fa9..6ad7454 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -45,7 +45,6 @@
 
 #include "mgt/mgt.h"
 #include "common/heritage.h"
-#include "common/params.h"
 
 #include "hash/hash_slinger.h"
 #include "vav.h"



From phk at varnish-cache.org  Mon Apr 16 14:07:35 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 16 Apr 2012 16:07:35 +0200
Subject: [master] 0679d60 Be much more cautious about how much workspace we
	have to build predictive vary string.
Message-ID: <E1SJmad-0002Tv-O9@project.varnish-software.com>

commit 0679d603fe4dad3027206f0cb272cd9c4aedb557
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 16 14:06:55 2012 +0000

    Be much more cautious about how much workspace we have to build
    predictive vary string.
    
    Fixes #1120

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 073eb75..ee17d31 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -956,6 +956,7 @@ void RES_WriteObj(struct sess *sp);
 struct vsb *VRY_Create(struct req *sp, const struct http *hp);
 int VRY_Match(struct req *, const uint8_t *vary);
 void VRY_Validate(const uint8_t *vary);
+void VRY_Prep(struct req *);
 
 /* cache_vcl.c */
 void VCL_Init(void);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 5530f1b..4a04f5b 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1081,18 +1081,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	AZ(req->busyobj);
 
-	if (req->hash_objhead == NULL) {
-		/* Not a waiting list return */
-		AZ(req->vary_b);
-		AZ(req->vary_l);
-		AZ(req->vary_e);
-		(void)WS_Reserve(req->ws, 0);
-	} else {
-		AN(req->ws->r);
-	}
-	req->vary_b = (void*)req->ws->f;
-	req->vary_e = (void*)req->ws->r;
-	req->vary_b[2] = '\0';
+	VRY_Prep(req);
 
 	AZ(req->objcore);
 	oc = HSH_Lookup(sp);
@@ -1359,7 +1348,7 @@ cnt_restart(struct sess *sp, const struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	
+
 	req->director = NULL;
 	if (++req->restarts >= cache_param->max_restarts) {
 		req->err_code = 503;
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 138efda..bf6537f 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -176,22 +176,61 @@ vry_cmp(const uint8_t *v1, const uint8_t *v2)
 	return (retval);
 }
 
+/**********************************************************************
+ * Prepare predictive vary string
+ */
+
+void
+VRY_Prep(struct req *req)
+{
+	if (req->hash_objhead == NULL) {
+		/* Not a waiting list return */
+		AZ(req->vary_b);
+		AZ(req->vary_l);
+		AZ(req->vary_e);
+		(void)WS_Reserve(req->ws, 0);
+	} else {
+		AN(req->ws->r);
+	}
+	req->vary_b = (void*)req->ws->f;
+	req->vary_e = (void*)req->ws->r;
+	if (req->vary_b + 2 < req->vary_e)
+		req->vary_b[2] = '\0';
+}
+
+/**********************************************************************
+ * Match vary strings, and build a new cached string if possible.
+ *
+ * Return zero if there is certainly no match.
+ * Return non-zero if there could be a match or if we couldn't tell.
+ */
+
 int
 VRY_Match(struct req *req, const uint8_t *vary)
 {
 	uint8_t *vsp = req->vary_b;
 	char *h, *e;
 	unsigned lh, ln;
-	int i, retval = 1, oflo = 0;
+	int i, oflo = 0;
 
 	AN(vsp);
 	while (vary[2]) {
+		if (vsp + 2 >= req->vary_e) {
+			/*
+			 * Too little workspace to find out
+			 */
+			oflo = 1;
+			break;
+		}
 		i = vry_cmp(vary, vsp);
 		if (i == 1) {
-			/* Build a new entry */
+			/*
+			 * Different header, build a new entry,
+			 * then compare again with that new entry.
+			 */
 
-			i = http_GetHdr(req->http,
-			    (const char*)(vary+2), &h);
+			ln = 2 + vary[2] + 2;
+			i = http_GetHdr(req->http, (const char*)(vary+2), &h);
 			if (i) {
 				/* Trim trailing space */
 				e = strchr(h, '\0');
@@ -199,55 +238,55 @@ VRY_Match(struct req *req, const uint8_t *vary)
 					e--;
 				lh = e - h;
 				assert(lh < 0xffff);
+				ln += lh;
 			} else {
 				e = h = NULL;
 				lh = 0xffff;
 			}
 
-			/* Length of the entire new vary entry */
-			ln = 2 + vary[2] + 2 + (lh == 0xffff ? 0 : lh);
-			if (vsp + ln >= req->vary_e) {
-				vsp = req->vary_b;
+			if (vsp + ln + 2 >= req->vary_e) {
+				/*
+				 * Not enough space to build new entry
+				 * and put terminator behind it.
+				 */
 				oflo = 1;
+				break;
 			}
 
-			/*
-			 * We MUST have space for one entry and the end marker
-			 * after it, which prevents old junk from confusing us
-			 */
-			assert(vsp + ln + 2 < req->vary_e);
-
 			vbe16enc(vsp, (uint16_t)lh);
 			memcpy(vsp + 2, vary + 2, vary[2] + 2);
-			if (h != NULL && e != NULL) {
-				memcpy(vsp + 2 + vsp[2] + 2, h, e - h);
-				vsp[2 + vary[2] + 2 + (e - h) + 2] = '\0';
-			} else
-				vsp[2 + vary[2] + 2 + 2] = '\0';
+			if (h != NULL)
+				memcpy(vsp + 2 + vsp[2] + 2, h, lh);
+			vsp[ln + 0] = 0xff;
+			vsp[ln + 1] = 0xff;
+			vsp[ln + 2] = 0;
+			VRY_Validate(vsp);
+			req->vary_l = vsp + 3;
 
 			i = vry_cmp(vary, vsp);
-			assert(i != 1);	/* hdr must be the same now */
+			assert(i == 0 || i == 2);
+		}
+		if (i == 0) {
+			/* Same header, same contents*/
+			vsp += vry_len(vsp);
+			vary += vry_len(vary);
+		} else if (i == 2) {
+			/* Same header, different contents, cannot match */
+			return (0);
 		}
-		if (i != 0)
-			retval = 0;
-		vsp += vry_len(vsp);
-		vary += vry_len(vary);
 	}
-	if (vsp + 3 > req->vary_e)
-		oflo = 1;
-
 	if (oflo) {
-		/* XXX: Should log this */
 		vsp = req->vary_b;
-	}
-	vsp[0] = 0xff;
-	vsp[1] = 0xff;
-	vsp[2] = 0;
-	if (oflo)
 		req->vary_l = NULL;
-	else
-		req->vary_l = vsp + 3;
-	return (retval);
+		if (vsp + 2 < req->vary_e) {
+			vsp[0] = 0xff;
+			vsp[1] = 0xff;
+			vsp[2] = 0;
+		}
+		return (0);
+	} else {
+		return (1);
+	}
 }
 
 void
diff --git a/bin/varnishtest/tests/r01120.vtc b/bin/varnishtest/tests/r01120.vtc
new file mode 100644
index 0000000..97bc2e5
--- /dev/null
+++ b/bin/varnishtest/tests/r01120.vtc
@@ -0,0 +1,24 @@
+varnishtest "insanely long vary string"
+
+server s1 {
+	rxreq
+	txresp -hdr "Vary: Foo" -body "xxxx"
+	rxreq
+	txresp -hdr "Vary: Foo" -body "yyyyy"
+} -start
+
+varnish v1 \
+	-cliok "param.set workspace_client 8k" \
+	-cliok "param.set workspace_backend 200k" \
+	-vcl+backend {
+} -start
+
+client c1 {
+	txreq -hdr "Foo: blabla"
+	rxresp
+	expect resp.bodylen == 4
+	#txreq -hdr "Foo: blablaA"
+	txreq -hdr "Foo: blablaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaaaaaaaaaaaaaaaaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+	rxresp
+	expect resp.bodylen == 5
+} -run



From martin at varnish-cache.org  Mon Apr 16 14:19:46 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Mon, 16 Apr 2012 16:19:46 +0200
Subject: [master] f992125 Reset output buffer on VGZ_WrwFlush()
Message-ID: <E1SJmmQ-00040u-8k@project.varnish-software.com>

commit f992125b545fdd1d090941f2ee8b7c9cd6c29dc8
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Mar 27 17:05:49 2012 +0200

    Reset output buffer on VGZ_WrwFlush()
    
    Fixes: #1123

diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index ca40903..f7dba8e 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -366,6 +366,7 @@ VGZ_WrwFlush(const struct worker *wrk, struct vgz *vg)
 	(void)WRW_Write(wrk, vg->m_buf, vg->m_len);
 	(void)WRW_Flush(wrk);
 	vg->m_len = 0;
+	VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishtest/tests/r01123.vtc b/bin/varnishtest/tests/r01123.vtc
new file mode 100644
index 0000000..37a9254
--- /dev/null
+++ b/bin/varnishtest/tests/r01123.vtc
@@ -0,0 +1,25 @@
+varnishtest "Test case for #1123 - gunzip buffer reset"
+
+server s1 {
+	rxreq
+	txresp -body {<html>start <esi:include src="/included"/> end}
+	rxreq
+	expect req.url == "/included"
+	txresp -body {INCLUDED}
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		if (req.url == "/") {
+			set beresp.do_esi = true;
+		}
+		set beresp.do_gzip = true;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.bodylen == 24
+	expect resp.body == "<html>start INCLUDED end"
+} -run



From martin at varnish-cache.org  Mon Apr 16 14:19:46 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Mon, 16 Apr 2012 16:19:46 +0200
Subject: [master] 5625a9a Expose resp.body for inspection and testing in
	varnishtest
Message-ID: <E1SJmmQ-00040z-9Z@project.varnish-software.com>

commit 5625a9a532536f5c77ea9a1798b89127589e7fce
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Mar 27 14:12:33 2012 +0200

    Expose resp.body for inspection and testing in varnishtest
    
    Needed to test #1123

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 00be0be..0678efd 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -190,6 +190,8 @@ cmd_var_resolve(struct http *hp, char *spec)
 		return(hp->chunklen);
 	if (!strcmp(spec, "resp.bodylen"))
 		return(hp->bodylen);
+	if (!strcmp(spec, "resp.body"))
+		return(hp->body != NULL ? hp->body : spec);
 	if (!memcmp(spec, "req.http.", 9)) {
 		hh = hp->req;
 		hdr = spec + 9;



From apj at varnish-cache.org  Tue Apr 17 20:26:26 2012
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Tue, 17 Apr 2012 22:26:26 +0200
Subject: [master] 058fa0f stats are no longer in CLI
Message-ID: <E1SKEyo-00086j-6U@project.varnish-software.com>

commit 058fa0fac3581652600d3de317a7457bc3fcde32
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Apr 17 22:25:56 2012 +0200

    stats are no longer in CLI

diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index 668ceab..ad9cc9c 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -28,9 +28,6 @@ parameters
      available through the CLI. The individual parameters are
      documented in the varnishd(1) man page.
 
-statistics
-     Statistic counters are available from the CLI.
-
 bans 
      Bans are filters that are applied to keep Varnish from serving
      stale content. When you issue a ban Varnish will not serve any
@@ -149,13 +146,6 @@ quit
 start
       Start the Varnish cache process if it is not already running.
 
-stats
-      Show summary statistics.
-
-      All the numbers presented are totals since server startup; for a
-      better idea of the current situation, use the varnishstat(1)
-      utility.
-
 status
       Check the status of the Varnish cache process.
 



From phk at varnish-cache.org  Thu Apr 19 07:35:05 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 19 Apr 2012 09:35:05 +0200
Subject: [master] 0f7d9a3 Cave in to the microoptimization demands and remove
	a pointless unlock/lock sequence when using critbit,
	at the expense of a slightly more ornate API for hashers.
Message-ID: <E1SKltR-0004iK-2T@project.varnish-software.com>

commit 0f7d9a361da8436bf01670be6a86f0b4d17a09ac
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Apr 19 07:33:41 2012 +0000

    Cave in to the microoptimization demands and remove a pointless
    unlock/lock sequence when using critbit, at the expense of a
    slightly more ornate API for hashers.
    
    Also make the "noh" argument optional, as originally intended.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index e746813..6facf0a 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -259,7 +259,7 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc)
 	AN(wrk->nobjhead);
 	oh = hash->lookup(wrk, digest, &wrk->nobjhead);
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	Lck_Lock(&oh->mtx);
+	Lck_AssertHeld(&oh->mtx);
 	assert(oh->refcnt > 0);
 
 	/* Insert (precreated) objcore in objecthead */
@@ -310,6 +310,7 @@ HSH_Lookup(struct sess *sp)
 		 */
 		CHECK_OBJ_NOTNULL(req->hash_objhead, OBJHEAD_MAGIC);
 		oh = req->hash_objhead;
+		Lck_Lock(&oh->mtx);
 		req->hash_objhead = NULL;
 	} else {
 		AN(wrk->nobjhead);
@@ -317,7 +318,8 @@ HSH_Lookup(struct sess *sp)
 	}
 
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	Lck_Lock(&oh->mtx);
+	Lck_AssertHeld(&oh->mtx);
+
 	assert(oh->refcnt > 0);
 	busy_found = 0;
 	grace_oc = NULL;
diff --git a/bin/varnishd/hash/hash_classic.c b/bin/varnishd/hash/hash_classic.c
index 87e0561..9121ec1 100644
--- a/bin/varnishd/hash/hash_classic.c
+++ b/bin/varnishd/hash/hash_classic.c
@@ -120,8 +120,8 @@ hcl_lookup(struct worker *wrk, const void *digest, struct objhead **noh)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	AN(digest);
-	AN(noh);
-	CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC);
+	if (noh != NULL)
+		CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC);
 
 	assert(sizeof oh->digest >= sizeof hdigest);
 	memcpy(&hdigest, digest, sizeof hdigest);
@@ -130,6 +130,7 @@ hcl_lookup(struct worker *wrk, const void *digest, struct objhead **noh)
 
 	Lck_Lock(&hp->mtx);
 	VTAILQ_FOREACH(oh, &hp->head, hoh_list) {
+		CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 		i = memcmp(oh->digest, digest, sizeof oh->digest);
 		if (i < 0)
 			continue;
@@ -137,9 +138,15 @@ hcl_lookup(struct worker *wrk, const void *digest, struct objhead **noh)
 			break;
 		oh->refcnt++;
 		Lck_Unlock(&hp->mtx);
+		Lck_Lock(&oh->mtx);
 		return (oh);
 	}
 
+	if (noh == NULL) {
+		Lck_Unlock(&hp->mtx);
+		return (NULL);
+	}
+
 	if (oh != NULL)
 		VTAILQ_INSERT_BEFORE(oh, *noh, hoh_list);
 	else
@@ -152,6 +159,7 @@ hcl_lookup(struct worker *wrk, const void *digest, struct objhead **noh)
 	oh->hoh_head = hp;
 
 	Lck_Unlock(&hp->mtx);
+	Lck_Lock(&oh->mtx);
 	return (oh);
 }
 
diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
index 5bdab86..08be561 100644
--- a/bin/varnishd/hash/hash_critbit.c
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -425,9 +425,10 @@ hcb_lookup(struct worker *wrk, const void *digest, struct objhead **noh)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	AN(digest);
-	AN(noh);
-	CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC);
-	assert((*noh)->refcnt == 1);
+	if (noh != NULL) {
+		CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC);
+		assert((*noh)->refcnt == 1);
+	}
 
 	/* First try in read-only mode without holding a lock */
 
@@ -440,11 +441,11 @@ hcb_lookup(struct worker *wrk, const void *digest, struct objhead **noh)
 		 * under us, so fall through and try with the lock held.
 		 */
 		u = oh->refcnt;
-		if (u > 0)
+		if (u > 0) {
 			oh->refcnt++;
-		Lck_Unlock(&oh->mtx);
-		if (u > 0)
 			return (oh);
+		}
+		Lck_Unlock(&oh->mtx);
 	}
 
 	while (1) {
@@ -455,23 +456,27 @@ hcb_lookup(struct worker *wrk, const void *digest, struct objhead **noh)
 		oh = hcb_insert(wrk, &hcb_root, digest, noh);
 		Lck_Unlock(&hcb_mtx);
 
+		if (oh == NULL)
+			return (NULL);
+
+		Lck_Lock(&oh->mtx);
+
 		CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-		if (*noh == NULL) {
+		if (noh != NULL && *noh == NULL) {
 			assert(oh->refcnt > 0);
 			VSC_C_main->hcb_insert++;
 			return (oh);
 		}
-		Lck_Lock(&oh->mtx);
 		/*
 		 * A refcount of zero indicates that the tree changed
 		 * under us, so fall through and try with the lock held.
 		 */
 		u = oh->refcnt;
-		if (u > 0)
+		if (u > 0) {
 			oh->refcnt++;
-		Lck_Unlock(&oh->mtx);
-		if (u > 0)
 			return (oh);
+		}
+		Lck_Unlock(&oh->mtx);
 	}
 }
 
diff --git a/bin/varnishd/hash/hash_simple_list.c b/bin/varnishd/hash/hash_simple_list.c
index d70e76f..d99bb89 100644
--- a/bin/varnishd/hash/hash_simple_list.c
+++ b/bin/varnishd/hash/hash_simple_list.c
@@ -67,8 +67,8 @@ hsl_lookup(struct worker *wrk, const void *digest, struct objhead **noh)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	AN(digest);
-	AN(noh);
-	CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC);
+	if (noh != NULL)
+		CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC);
 
 	Lck_Lock(&hsl_mtx);
 	VTAILQ_FOREACH(oh, &hsl_head, hoh_list) {
@@ -79,9 +79,13 @@ hsl_lookup(struct worker *wrk, const void *digest, struct objhead **noh)
 			break;
 		oh->refcnt++;
 		Lck_Unlock(&hsl_mtx);
+		Lck_Lock(&oh->mtx);
 		return (oh);
 	}
 
+	if (noh == NULL)
+		return (NULL);
+
 	if (oh != NULL)
 		VTAILQ_INSERT_BEFORE(oh, *noh, hoh_list);
 	else
@@ -91,6 +95,7 @@ hsl_lookup(struct worker *wrk, const void *digest, struct objhead **noh)
 	*noh = NULL;
 	memcpy(oh->digest, digest, sizeof oh->digest);
 	Lck_Unlock(&hsl_mtx);
+	Lck_Lock(&oh->mtx);
 	return (oh);
 }
 



From phk at varnish-cache.org  Sun Apr 22 20:06:17 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 22 Apr 2012 22:06:17 +0200
Subject: [master] b8e04e6 Don't have mempool->free police the max parameter,
	leave that to the guard. This makes 'max' parameter mis-sizing much
	less problematic.
Message-ID: <E1SM333-0006hW-50@project.varnish-software.com>

commit b8e04e61410068ea69e5e291fabec233747072b0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Apr 22 20:05:22 2012 +0000

    Don't have mempool->free police the max parameter, leave that to
    the guard.  This makes 'max' parameter mis-sizing much less problematic.
    
    Spotted by:	scoof

diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
index 8906406..badb621 100644
--- a/bin/varnishd/cache/cache_mempool.c
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -329,9 +329,6 @@ MPL_Free(struct mempool *mpl, void *item)
 	if (mi->size < *mpl->cur_size) {
 		mpl->vsc->toosmall++;
 		VTAILQ_INSERT_HEAD(&mpl->surplus, mi, list);
-	} else if (mpl->n_pool >= mpl->param->max_pool) {
-		mpl->vsc->surplus++;
-		VTAILQ_INSERT_HEAD(&mpl->surplus, mi, list);
 	} else {
 		mpl->vsc->pool = ++mpl->n_pool;
 		mi->touched = mpl->t_now;



From phk at varnish-cache.org  Mon Apr 23 06:46:29 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Apr 2012 08:46:29 +0200
Subject: [master] 4601ddc Fix some stats-counter issues scoof pointed out
	yesterday:
Message-ID: <E1SMD2b-0007O2-7h@project.varnish-software.com>

commit 4601ddcd8db6926bd6315c642863a123e4c15b01
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 23 06:45:30 2012 +0000

    Fix some stats-counter issues scoof pointed out yesterday:
    
    n_objsendfile is of course no longer relevant.
    
    That makes n_objwrite pointless.
    
    n_objoverflow was unused.
    
    and threads_failed were not incremented where it should be.

diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 29d467c..bf3ab8e 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -340,7 +340,7 @@ pool_breed(struct pool *qp, const pthread_attr_t *tp_attr)
 			VSL(SLT_Debug, 0, "Create worker thread failed %d %s",
 			    errno, strerror(errno));
 			Lck_Lock(&pool_mtx);
-			VSC_C_main->threads_limited++;
+			VSC_C_main->threads_failed++;
 			Lck_Unlock(&pool_mtx);
 			VTIM_sleep(cache_param->wthread_fail_delay * 1e-3);
 		} else {
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index 8579e37..cd45945 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -168,8 +168,6 @@ res_WriteGunzipObj(const struct sess *sp)
 		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
 		u += st->len;
 
-		VSC_C_main->n_objwrite++;
-
 		i = VGZ_WrwGunzip(sp->wrk, vg, st->ptr, st->len);
 		/* XXX: error check */
 		(void)i;
@@ -215,7 +213,6 @@ res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high)
 		ptr += len;
 
 		sp->wrk->acct_tmp.bodybytes += len;
-		VSC_C_main->n_objwrite++;
 		(void)WRW_Write(sp->wrk, st->ptr + off, len);
 	}
 	assert(u == sp->req->obj->len);
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 36bb629..2938d7a 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -304,22 +304,6 @@ VSC_F(losthdr,			uint64_t, 0, 'a',
 	""
 )
 
-VSC_F(n_objsendfile,		uint64_t, 0, 'a',
-    "Objects sent with sendfile",
-	"The number of objects sent with the sendfile system call. If enabled "
-	"sendfile will be used on object larger than a certain size."
-)
-VSC_F(n_objwrite,		uint64_t, 0, 'a',
-    "Objects sent with write",
-	"The number of objects sent with regular write calls."
-	"Writes are used when the objects are too small for sendfile "
-	"or if the sendfile call has been disabled"
-)
-VSC_F(n_objoverflow,		uint64_t, 1, 'a',
-    "Objects overflowing workspace",
-	""
-)
-
 VSC_F(s_sess,			uint64_t, 1, 'a',
     "Total Sessions",
 	""



From phk at varnish-cache.org  Mon Apr 23 07:11:08 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Apr 2012 09:11:08 +0200
Subject: [master] 5961b65 Fix a flexelint nit
Message-ID: <E1SMDQS-0000x1-5R@project.varnish-software.com>

commit 5961b65fa57fb8eb02ac6630beec05f6a5fe28b1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 23 07:10:57 2012 +0000

    Fix a flexelint nit

diff --git a/lib/libvcl/vcc_symb.c b/lib/libvcl/vcc_symb.c
index 222680f..4066e56 100644
--- a/lib/libvcl/vcc_symb.c
+++ b/lib/libvcl/vcc_symb.c
@@ -70,7 +70,7 @@ vcc_AddSymbol(struct vcc *tl, const char *nb, int l, enum symkind kind)
 	}
 	ALLOC_OBJ(sym, SYMBOL_MAGIC);
 	AN(sym);
-	sym->name = malloc(l + 1);
+	sym->name = malloc(l + 1L);
 	AN(sym->name);
 	memcpy(sym->name, nb, l);
 	sym->name[l] = '\0';



From phk at varnish-cache.org  Mon Apr 23 08:00:42 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Apr 2012 10:00:42 +0200
Subject: [master] 353073c Make VSC->hcb_nolock a worker local counter for
	consistency, locking it would seriously miss the point.
Message-ID: <E1SMECQ-0005bo-KD@project.varnish-software.com>

commit 353073cf26dc461fdc0e5f11e40fdebc34c0c763
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 23 08:00:01 2012 +0000

    Make VSC->hcb_nolock a worker local counter for consistency, locking
    it would seriously miss the point.
    
    Spotted by:	scoof

diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
index 08be561..46f7f95 100644
--- a/bin/varnishd/hash/hash_critbit.c
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -432,7 +432,7 @@ hcb_lookup(struct worker *wrk, const void *digest, struct objhead **noh)
 
 	/* First try in read-only mode without holding a lock */
 
-	VSC_C_main->hcb_nolock++;
+	wrk->stats.hcb_nolock++;
 	oh = hcb_insert(wrk, &hcb_root, digest, NULL);
 	if (oh != NULL) {
 		Lck_Lock(&oh->mtx);
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 2938d7a..bc79d5b 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -454,7 +454,7 @@ VSC_F(bans_dups,		uint64_t, 0, 'c',
 
 /**********************************************************************/
 
-VSC_F(hcb_nolock,		uint64_t, 0, 'a',
+VSC_F(hcb_nolock,		uint64_t, 1, 'a',
     "HCB Lookups without lock",
 	""
 )



From martin at varnish-cache.org  Mon Apr 23 11:29:39 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Mon, 23 Apr 2012 13:29:39 +0200
Subject: [3.0] 9a27210 Set next state to STP_DONE also when an overflow error
	is encountered while parsing the HTTP request.
Message-ID: <E1SMHSd-0001z7-S6@project.varnish-software.com>

commit 9a27210a77fbaaeb1678cf8756f0e2a7228ecc46
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Apr 23 13:23:44 2012 +0200

    Set next state to STP_DONE also when an overflow error is encountered
    while parsing the HTTP request.
    
    Patch by: mark

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index c75f93d..b3d7237 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -115,11 +115,9 @@ cnt_wait(struct sess *sp)
 		sp->step = STP_START;
 		return (0);
 	}
-	if (i == -2) {
+	if (i == -2)
 		vca_close_session(sp, "overflow");
-		return (0);
-	}
-	if (i == -1 && Tlen(sp->htc->rxbuf) == 0 &&
+	else if (i == -1 && Tlen(sp->htc->rxbuf) == 0 &&
 	    (errno == 0 || errno == ECONNRESET))
 		vca_close_session(sp, "EOF");
 	else



From phk at varnish-cache.org  Mon Apr 23 11:36:30 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Apr 2012 13:36:30 +0200
Subject: [master] 017ba98 Add an explicit macro_undef() function so we don't
	pass a NULL argument to a printflike function.
Message-ID: <E1SMHZG-0002aS-Uw@project.varnish-software.com>

commit 017ba98d6bc565756ade72736c2e6e9a8592f0d1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 23 11:35:56 2012 +0000

    Add an explicit macro_undef() function so we don't pass a NULL
    argument to a printflike function.

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index b8556ce..99cf840 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -87,6 +87,8 @@ macro_def(struct vtclog *vl, const char *instance, const char *name,
 	struct macro *m;
 	va_list ap;
 
+	AN(fmt);
+
 	if (instance != NULL) {
 		bprintf(buf1, "%s_%s", instance, name);
 		name = buf1;
@@ -96,23 +98,40 @@ macro_def(struct vtclog *vl, const char *instance, const char *name,
 	VTAILQ_FOREACH(m, &macro_list, list)
 		if (!strcmp(name, m->name))
 			break;
-	if (m == NULL && fmt != NULL) {
+	if (m == NULL) {
 		m = calloc(sizeof *m, 1);
 		AN(m);
 		REPLACE(m->name, name);
 		VTAILQ_INSERT_TAIL(&macro_list, m, list);
 	}
-	if (fmt != NULL) {
-		AN(m);
-		va_start(ap, fmt);
-		free(m->val);
-		m->val = NULL;
-		vbprintf(buf2, fmt, ap);
-		va_end(ap);
-		m->val = strdup(buf2);
-		AN(m->val);
-		vtc_log(vl, 4, "macro def %s=%s", name, m->val);
-	} else if (m != NULL) {
+	AN(m);
+	va_start(ap, fmt);
+	free(m->val);
+	m->val = NULL;
+	vbprintf(buf2, fmt, ap);
+	va_end(ap);
+	m->val = strdup(buf2);
+	AN(m->val);
+	vtc_log(vl, 4, "macro def %s=%s", name, m->val);
+	AZ(pthread_mutex_unlock(&macro_mtx));
+}
+
+void
+macro_undef(struct vtclog *vl, const char *instance, const char *name)
+{
+	char buf1[256];
+	struct macro *m;
+
+	if (instance != NULL) {
+		bprintf(buf1, "%s_%s", instance, name);
+		name = buf1;
+	}
+
+	AZ(pthread_mutex_lock(&macro_mtx));
+	VTAILQ_FOREACH(m, &macro_list, list)
+		if (!strcmp(name, m->name))
+			break;
+	if (m != NULL) {
 		vtc_log(vl, 4, "macro undef %s", name);
 		VTAILQ_REMOVE(&macro_list, m, list);
 		free(m->name);
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index be20cb9..333a1df 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -87,6 +87,7 @@ void vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx,
 int exec_file(const char *fn, const char *script, const char *tmpdir,
     char *logbuf, unsigned loglen);
 
+void macro_undef(struct vtclog *vl, const char *instance, const char *name);
 void macro_def(struct vtclog *vl, const char *instance, const char *name,
     const char *fmt, ...)
     __printflike(4, 5);
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 75c4f30..eaa680f 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -143,9 +143,9 @@ server_delete(struct server *s)
 {
 
 	CHECK_OBJ_NOTNULL(s, SERVER_MAGIC);
-	macro_def(s->vl, s->name, "addr", NULL);
-	macro_def(s->vl, s->name, "port", NULL);
-	macro_def(s->vl, s->name, "sock", NULL);
+	macro_undef(s->vl, s->name, "addr");
+	macro_undef(s->vl, s->name, "port");
+	macro_undef(s->vl, s->name, "sock");
 	vtc_logclose(s->vl);
 	free(s->name);
 	/* XXX: MEMLEAK (?) (VSS ??) */
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index c00a143..e493ae7 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -522,9 +522,9 @@ varnish_stop(struct varnish *v)
 		varnish_launch(v);
 	if (vtc_error)
 		return;
-	macro_def(v->vl, v->name, "addr", NULL);
-	macro_def(v->vl, v->name, "port", NULL);
-	macro_def(v->vl, v->name, "sock", NULL);
+	macro_undef(v->vl, v->name, "addr");
+	macro_undef(v->vl, v->name, "port");
+	macro_undef(v->vl, v->name, "sock");
 	vtc_log(v->vl, 2, "Stop");
 	(void)varnish_ask_cli(v, "stop", NULL);
 	while (1) {



From phk at varnish-cache.org  Mon Apr 23 12:15:44 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Apr 2012 14:15:44 +0200
Subject: [master] cab6375 Extra asserts to try to catch a weird panic on
	tinderbox::osx
Message-ID: <E1SMIBE-0006x5-BR@project.varnish-software.com>

commit cab6375cab80b2220c9eb0de8b08a304100aaac6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 23 12:15:23 2012 +0000

    Extra asserts to try to catch a weird panic on tinderbox::osx

diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index bf3ab8e..a723a9e 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -194,6 +194,7 @@ pool_accept(struct worker *wrk, void *arg)
 			return;
 		}
 		VTAILQ_REMOVE(&pp->idle_queue, &wrk2->task, list);
+		AZ(wrk2->task.func);
 		Lck_Unlock(&pp->mtx);
 		assert(sizeof *wa2 == WS_Reserve(wrk2->aws, sizeof *wa2));
 		wa2 = (void*)wrk2->aws->f;
@@ -227,6 +228,7 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 	wrk = pool_getidleworker(pp, 0);
 	if (wrk != NULL) {
 		VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list);
+		AZ(wrk->task.func);
 		Lck_Unlock(&pp->mtx);
 		wrk->task.func = task->func;
 		wrk->task.priv = task->priv;
@@ -298,6 +300,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 				wrk->lastused = VTIM_real();
 			wrk->task.func = NULL;
 			wrk->task.priv = wrk;
+			AZ(wrk->task.func);
 			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
 			if (!stats_clean)
 				WRK_SumStat(wrk);
@@ -427,6 +430,7 @@ pool_herder(void *priv)
 		if (wrk != NULL && (wrk->lastused < t_idle ||
 		    pp->nthr > cache_param->wthread_max)) {
 			VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list);
+			AZ(wrk->task.func);
 		} else
 			wrk = NULL;
 		Lck_Unlock(&pp->mtx);



From phk at varnish-cache.org  Mon Apr 23 16:17:44 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Apr 2012 18:17:44 +0200
Subject: [master] 8bf6c6d Also reflect the VCC exit code through if -C is
	specified.
Message-ID: <E1SMLxQ-0000Qm-32@project.varnish-software.com>

commit 8bf6c6d2891b50b04af463791fc380c359b1d02c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 23 16:17:14 2012 +0000

    Also reflect the VCC exit code through if -C is specified.
    
    Fixes #1069

diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index 12433ad..98dc4c4 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -394,18 +394,16 @@ mgt_vcc_default(const char *b_arg, const char *f_arg, char *vcl, int C_flag)
 	if (VSB_len(sb) > 0)
 		fprintf(stderr, "%s", VSB_data(sb));
 	VSB_delete(sb);
-	if (C_flag) {
-		if (vf != NULL)
-			AZ(unlink(vf));
-		return (0);
-	}
+	if (C_flag && vf != NULL)
+		AZ(unlink(vf));
 	if (vf == NULL) {
 		fprintf(stderr, "\nVCL compilation failed\n");
 		return (1);
+	} else {
+		vp = mgt_vcc_add(buf, vf);
+		vp->active = 1;
+		return (0);
 	}
-	vp = mgt_vcc_add(buf, vf);
-	vp->active = 1;
-	return (0);
 }
 
 /*--------------------------------------------------------------------*/



From apj at varnish-cache.org  Mon Apr 23 17:07:57 2012
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Mon, 23 Apr 2012 19:07:57 +0200
Subject: [master] ad9356a Clean up docs about esi:remove and <!--esi
Message-ID: <E1SMMk1-0005HA-UU@project.varnish-software.com>

commit ad9356a952617bf97a539dfcc686142b7120d7f0
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Apr 23 19:07:13 2012 +0200

    Clean up docs about esi:remove and <!--esi
    
    Fixes #1122

diff --git a/doc/sphinx/tutorial/esi.rst b/doc/sphinx/tutorial/esi.rst
index 1badd5d..720b790 100644
--- a/doc/sphinx/tutorial/esi.rst
+++ b/doc/sphinx/tutorial/esi.rst
@@ -22,7 +22,9 @@ have three ESI statements:
 Content substitution based on variables and cookies is not implemented
 but is on the roadmap. 
 
-Example: esi include
+Varnish will not process ESI instructions in HTML comments.
+
+Example: esi:include
 ~~~~~~~~~~~~~~~~~~~~
 
 Lets see an example how this could be used. This simple cgi script
@@ -55,32 +57,23 @@ For ESI to work you need to activate ESI processing in VCL, like this::
         }
     }
 
-Example: esi remove
-~~~~~~~~~~~~~~~~~~~
-
-The *remove* keyword allows you to remove output. You can use this to make
-a fall back of sorts, when ESI is not available, like this::
+Example: esi:remove and <!--esi ... -->
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The <esi:remove> and <!--esi ... --> constructs can be used to present
+appropriate content whether or not ESI is available, for example you can
+include content when ESI is available or link to it when it is not.
+ESI processors will remove the start ("<!--esi") and end ("-->") when
+the page is processed, while still processing the contents. If the page
+is not processed, it will remain, becoming an HTML/XML comment tag.
+ESI processors will remove <esi:remove> tags and all content contained
+in them, allowing you to only render the content when the page is not
+being ESI-processed.
+For example::
 
-  <esi:include src="http://www.example.com/ad.html"/> 
   <esi:remove> 
-    <a href="http://www.example.com">www.example.com</a>
+    <a href="http://www.example.com/LICENSE">The license</a>
   </esi:remove>
-
-Example: <!--esi ... -->
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-
-This is a special construct to allow HTML marked up with ESI to render
-without processing. ESI Processors will remove the start ("<!--esi")
-and end ("-->") when the page is processed, while still processing the
-contents. If the page is not processed, it will remain, becoming an
-HTML/XML comment tag. For example::
-
   <!--esi  
-  <p>Warning: ESI Disabled!</p>
-  </p>  -->
-
-This assures that the ESI markup will not interfere with the rendering
-of the final HTML if not processed.
-
-
+  <p>The full text of the license:</p>
+  <esi:include src="http://example.com/LICENSE" />
+  -->



From ingvar at varnish-cache.org  Tue Apr 24 13:02:38 2012
From: ingvar at varnish-cache.org (Ingvar)
Date: Tue, 24 Apr 2012 15:02:38 +0200
Subject: [master] 5056ea1 Stopped using macros for make and install,
	according to Fedora's packaging guidelines
Message-ID: <E1SMfOA-0006ju-Sq@project.varnish-software.com>

commit 5056ea143dcd659abe89c1c92de72bba17e88894
Author: Ingvar Hagelund <ingvar at redpill-linpro.com>
Date:   Mon Apr 23 10:43:20 2012 +0200

    Stopped using macros for make and install, according to Fedora's packaging guidelines

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index ead5745..d892f78 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -101,7 +101,7 @@ cp bin/varnishd/default.vcl etc/zope-plone.vcl examples
 #sed -i 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|g;
 #	s|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool
 
-%{__make} %{?_smp_mflags}
+make %{?_smp_mflags}
 
 head -6 etc/default.vcl > redhat/default.vcl
 
@@ -141,7 +141,7 @@ cp -r doc/sphinx/\=build/html doc
 	%endif
 %endif
 
-%{__make} check LD_LIBRARY_PATH="../../lib/libvarnish/.libs:../../lib/libvarnishcompat/.libs:../../lib/libvarnishapi/.libs:../../lib/libvcl/.libs:../../lib/libvgz/.libs"
+make check LD_LIBRARY_PATH="../../lib/libvarnish/.libs:../../lib/libvarnishcompat/.libs:../../lib/libvarnishapi/.libs:../../lib/libvcl/.libs:../../lib/libvgz/.libs"
 
 %install
 rm -rf %{buildroot}



From ingvar at varnish-cache.org  Tue Apr 24 13:02:38 2012
From: ingvar at varnish-cache.org (Ingvar)
Date: Tue, 24 Apr 2012 15:02:38 +0200
Subject: [master] a43175e Removed source install instructions from the built
	package, as requested by rpmlint
Message-ID: <E1SMfOA-0006jz-Ue@project.varnish-software.com>

commit a43175e2a8637c73962aa0b95ac2d650114da18c
Author: Ingvar Hagelund <ingvar at redpill-linpro.com>
Date:   Mon Apr 23 10:46:00 2012 +0200

    Removed source install instructions from the built package, as requested by rpmlint

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index d892f78..9f45c2f 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -177,7 +177,7 @@ rm -rf %{buildroot}
 %{_mandir}/man1/*.1*
 %{_mandir}/man3/*.3*
 %{_mandir}/man7/*.7*
-%doc INSTALL LICENSE README redhat/README.redhat ChangeLog
+%doc LICENSE README redhat/README.redhat ChangeLog
 %doc examples
 %dir %{_sysconfdir}/varnish/
 %config(noreplace) %{_sysconfdir}/varnish/default.vcl



From ingvar at varnish-cache.org  Tue Apr 24 13:02:38 2012
From: ingvar at varnish-cache.org (Ingvar)
Date: Tue, 24 Apr 2012 15:02:38 +0200
Subject: [master] 0897b8b Stopped using macros for make and install,
	according to Fedora's packaging guidelines
Message-ID: <E1SMfOA-0006k5-W7@project.varnish-software.com>

commit 0897b8b925659fcfa7d24ca6a980cc1a23540523
Author: Ingvar Hagelund <ingvar at redpill-linpro.com>
Date:   Mon Apr 23 10:50:41 2012 +0200

    Stopped using macros for make and install, according to Fedora's packaging guidelines

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 9f45c2f..5ab53c5 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -156,13 +156,13 @@ find %{buildroot}/%{_libdir}/ -name '*.la' -exec rm -f {} ';'
 mkdir -p %{buildroot}/var/lib/varnish
 mkdir -p %{buildroot}/var/log/varnish
 mkdir -p %{buildroot}/var/run/varnish
-%{__install} -D -m 0644 redhat/default.vcl %{buildroot}%{_sysconfdir}/varnish/default.vcl
-%{__install} -D -m 0644 redhat/varnish.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/varnish
-%{__install} -D -m 0644 redhat/varnish.logrotate %{buildroot}%{_sysconfdir}/logrotate.d/varnish
-%{__install} -D -m 0755 redhat/varnish.initrc %{buildroot}%{_initrddir}/varnish
-%{__install} -D -m 0755 redhat/varnishlog.initrc %{buildroot}%{_initrddir}/varnishlog
-%{__install} -D -m 0755 redhat/varnishncsa.initrc %{buildroot}%{_initrddir}/varnishncsa
-%{__install} -D -m 0755 redhat/varnish_reload_vcl %{buildroot}%{_bindir}/varnish_reload_vcl
+install -D -m 0644 redhat/default.vcl %{buildroot}%{_sysconfdir}/varnish/default.vcl
+install -D -m 0644 redhat/varnish.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/varnish
+install -D -m 0644 redhat/varnish.logrotate %{buildroot}%{_sysconfdir}/logrotate.d/varnish
+install -D -m 0755 redhat/varnish.initrc %{buildroot}%{_initrddir}/varnish
+install -D -m 0755 redhat/varnishlog.initrc %{buildroot}%{_initrddir}/varnishlog
+install -D -m 0755 redhat/varnishncsa.initrc %{buildroot}%{_initrddir}/varnishncsa
+install -D -m 0755 redhat/varnish_reload_vcl %{buildroot}%{_bindir}/varnish_reload_vcl
 
 %clean
 rm -rf %{buildroot}
@@ -177,7 +177,7 @@ rm -rf %{buildroot}
 %{_mandir}/man1/*.1*
 %{_mandir}/man3/*.3*
 %{_mandir}/man7/*.7*
-%doc LICENSE README redhat/README.redhat ChangeLog
+%doc INSTALL LICENSE README redhat/README.redhat ChangeLog
 %doc examples
 %dir %{_sysconfdir}/varnish/
 %config(noreplace) %{_sysconfdir}/varnish/default.vcl



From ingvar at varnish-cache.org  Tue Apr 24 13:02:39 2012
From: ingvar at varnish-cache.org (Ingvar)
Date: Tue, 24 Apr 2012 15:02:39 +0200
Subject: [master] d31f331 Removed source installation instructions INSTALL,
	as requested by rpmlint
Message-ID: <E1SMfOB-0006kC-1Q@project.varnish-software.com>

commit d31f331e4c93a0751a785d20be6abb23f0bcf678
Author: Ingvar Hagelund <ingvar at redpill-linpro.com>
Date:   Mon Apr 23 10:52:40 2012 +0200

    Removed source installation instructions INSTALL, as requested by rpmlint

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 5ab53c5..5364637 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -177,7 +177,7 @@ rm -rf %{buildroot}
 %{_mandir}/man1/*.1*
 %{_mandir}/man3/*.3*
 %{_mandir}/man7/*.7*
-%doc INSTALL LICENSE README redhat/README.redhat ChangeLog
+%doc LICENSE README redhat/README.redhat ChangeLog
 %doc examples
 %dir %{_sysconfdir}/varnish/
 %config(noreplace) %{_sysconfdir}/varnish/default.vcl



From ingvar at varnish-cache.org  Tue Apr 24 13:02:39 2012
From: ingvar at varnish-cache.org (Ingvar)
Date: Tue, 24 Apr 2012 15:02:39 +0200
Subject: [master] 1607091 Moved libdir/varnish to the varnish-libs subpackage
	where it belongs,
	and added an ld.so.conf.d fragment file listing libdir/varnish
Message-ID: <E1SMfOB-0006kL-4E@project.varnish-software.com>

commit 1607091cdf7ff965a3b6f080087db6372c1a25bc
Author: Ingvar Hagelund <ingvar at redpill-linpro.com>
Date:   Mon Apr 23 11:01:24 2012 +0200

    Moved libdir/varnish to the varnish-libs subpackage where it belongs, and added an ld.so.conf.d fragment file listing libdir/varnish

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 5364637..a2d3ae9 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -156,6 +156,7 @@ find %{buildroot}/%{_libdir}/ -name '*.la' -exec rm -f {} ';'
 mkdir -p %{buildroot}/var/lib/varnish
 mkdir -p %{buildroot}/var/log/varnish
 mkdir -p %{buildroot}/var/run/varnish
+mkdir -p %{buildroot}%{_sysconfdir}/ld.so.conf.d/
 install -D -m 0644 redhat/default.vcl %{buildroot}%{_sysconfdir}/varnish/default.vcl
 install -D -m 0644 redhat/varnish.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/varnish
 install -D -m 0644 redhat/varnish.logrotate %{buildroot}%{_sysconfdir}/logrotate.d/varnish
@@ -164,6 +165,8 @@ install -D -m 0755 redhat/varnishlog.initrc %{buildroot}%{_initrddir}/varnishlog
 install -D -m 0755 redhat/varnishncsa.initrc %{buildroot}%{_initrddir}/varnishncsa
 install -D -m 0755 redhat/varnish_reload_vcl %{buildroot}%{_bindir}/varnish_reload_vcl
 
+echo %{_libdir}/varnish > %{buildroot}%{_sysconfdir}/ld.so.conf.d/varnish-%{_arch}.conf
+
 %clean
 rm -rf %{buildroot}
 
@@ -171,7 +174,6 @@ rm -rf %{buildroot}
 %defattr(-,root,root,-)
 %{_sbindir}/*
 %{_bindir}/*
-%{_libdir}/varnish
 %{_var}/lib/varnish
 %{_var}/log/varnish
 %{_mandir}/man1/*.1*
@@ -190,7 +192,9 @@ rm -rf %{buildroot}
 %files libs
 %defattr(-,root,root,-)
 %{_libdir}/*.so.*
+%{_libdir}/varnish
 %doc LICENSE
+%config %{_sysconfdir}/ld.so.conf.d/varnish-%{_arch}.conf
 
 %files libs-devel
 %defattr(-,root,root,-)



From ingvar at varnish-cache.org  Tue Apr 24 13:02:39 2012
From: ingvar at varnish-cache.org (Ingvar)
Date: Tue, 24 Apr 2012 15:02:39 +0200
Subject: [master] c562b7c No need to keep the sphinx doc =build dir. If a user
	wants them, she can recreate them.
Message-ID: <E1SMfOB-0006ka-6T@project.varnish-software.com>

commit c562b7c46513204859fbe1816ce61eaf4ee39e06
Author: Ingvar Hagelund <ingvar at redpill-linpro.com>
Date:   Mon Apr 23 11:17:57 2012 +0200

    No need to keep the sphinx doc =build dir. If a user wants them, she can recreate them.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index a2d3ae9..091e15b 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -122,7 +122,9 @@ tail -n +11 etc/default.vcl >> redhat/default.vcl
 	redhat/varnish.initrc redhat/varnishlog.initrc redhat/varnishncsa.initrc
 %endif
 
-cp -r doc/sphinx/\=build/html doc
+rm -rf doc/sphinx/\=build/html/_sources
+mv doc/sphinx/\=build/html doc
+rm -rf doc/sphinx/\=build
 
 %check
 # rhel5 on ppc64 is just too strange



From ingvar at varnish-cache.org  Tue Apr 24 13:02:39 2012
From: ingvar at varnish-cache.org (Ingvar)
Date: Tue, 24 Apr 2012 15:02:39 +0200
Subject: [master] 3dbab85 Added missing changelog entries from fedora
Message-ID: <E1SMfOB-0006kk-9C@project.varnish-software.com>

commit 3dbab85b2755e38c268c8a9ed9a50e699fb7492c
Author: Ingvar Hagelund <ingvar at redpill-linpro.com>
Date:   Mon Apr 23 13:13:03 2012 +0200

    Added missing changelog entries from fedora

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 091e15b..a66e0b9 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -248,6 +248,22 @@ fi
 %postun libs -p /sbin/ldconfig
 
 %changelog
+* Fri Feb 10 2012 Petr Pisar <ppisar at redhat.com> - 2.1.5-4
+- Rebuild against PCRE 8.30
+
+* Sat Jan 14 2012 Fedora Release Engineering <rel-eng at lists.fedoraproject.org> - 2.1.5-3
+- Rebuilt for https://fedoraproject.org/wiki/Fedora_17_Mass_Rebuild
+
+* Mon Feb 07 2011 Fedora Release Engineering <rel-eng at lists.fedoraproject.org> - 2.1.5-2
+- Rebuilt for https://fedoraproject.org/wiki/Fedora_15_Mass_Rebuild
+
+* Tue Feb 01 2011 Ingvar Hagelund <ingvar at redpill-linpro.com> - 2.1.5-1
+- New upstream release
+- New download location
+- Moved varnish_reload_vcl to sbin
+- Removed patches included upstream
+- Use jemalloc as system installed library
+
 * Mon Nov 15 2010 Ingvar Hagelund <ingvar at redpill-linpro.com> - 3.0.0-0.svn20101115r5543
 - Merged some changes from fedora
 - Upped general version to 3.0 prerelease in trunk



From ingvar at varnish-cache.org  Tue Apr 24 13:02:39 2012
From: ingvar at varnish-cache.org (Ingvar)
Date: Tue, 24 Apr 2012 15:02:39 +0200
Subject: [master] e82877e Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1SMfOB-0006kx-Cc@project.varnish-software.com>

commit e82877eaa1cb6ca8fe90a55a9134bb7fd04ed50a
Merge: 3dbab85 ad9356a
Author: Ingvar Hagelund <ingvar at redpill-linpro.com>
Date:   Tue Apr 24 14:55:59 2012 +0200

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From ingvar at varnish-cache.org  Tue Apr 24 13:02:39 2012
From: ingvar at varnish-cache.org (Ingvar)
Date: Tue, 24 Apr 2012 15:02:39 +0200
Subject: [master] d52a72e merged changelog entries from fedora
Message-ID: <E1SMfOB-0006lO-J2@project.varnish-software.com>

commit d52a72e6e012cea4cafe595895cbc901fc1281c6
Author: Ingvar Hagelund <ingvar at redpill-linpro.com>
Date:   Tue Apr 24 15:02:21 2012 +0200

    merged changelog entries from fedora

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index f71fc9e..ea0fc24 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -301,6 +301,31 @@ fi
 %postun libs -p /sbin/ldconfig
 
 %changelog
+* Mon Mar 12 2012 Ingvar Hagelund <ingvar at redpill-linpro.com> - 3.0.2-2
+- Added PrivateTmp=true to varnishd unit file, closing #782539
+- Fixed comment typos in varnish unit file
+
+* Tue Mar 06 2012 Ingvar Hagelund <ingvar at redpill-linpro.com> - 3.0.2-1
+- New upstream version 3.0.2
+- Removed INSTALL as requested by rpmlint
+- Added a ld.so.conf.d fragment file listing libdir/varnish 
+- Removed redundant doc/html/_sources
+- systemd support from fedora 17
+- Stopped using macros for make and install, according to 
+  Fedora's packaging guidelines
+- Changes merged from upstream:
+  - Added suse_version macro
+  - Added comments on building from a git checkout
+  - mkpasswd -> uuidgen for fewer dependencies
+  - Fixed missing quotes around cflags for pcre
+  - Removed unnecessary 32/64 bit parallell build hack as this is fixed upstream
+  - Fixed typo in configure call, disable -> without
+  - Added lib/libvgz/.libs to LD_LIBRARY_PATH in make check
+  - Added section 3 manpages
+  - Configure with --without-rst2man --without-rst2html
+  - changelog entries
+- Removed unnecessary patch for system jemalloc, upstream now supports this
+
 * Fri Feb 10 2012 Petr Pisar <ppisar at redhat.com> - 2.1.5-4
 - Rebuild against PCRE 8.30
 



From ingvar at varnish-cache.org  Tue Apr 24 13:02:39 2012
From: ingvar at varnish-cache.org (Ingvar)
Date: Tue, 24 Apr 2012 15:02:39 +0200
Subject: [master] 9ec4417 systemd support for fedora
Message-ID: <E1SMfOB-0006l9-G5@project.varnish-software.com>

commit 9ec4417a3aa3c85690815ce86b74a190c148b4d1
Author: Ingvar Hagelund <ingvar at redpill-linpro.com>
Date:   Tue Apr 24 15:01:10 2012 +0200

    systemd support for fedora

diff --git a/redhat/varnish.params b/redhat/varnish.params
new file mode 100644
index 0000000..d889ee4
--- /dev/null
+++ b/redhat/varnish.params
@@ -0,0 +1,40 @@
+# Varnish environment configuration description. This was derived from
+# the old style sysconfig/defaults settings
+
+# Set this to 1 to make systemd reload try to switch vcl without restart. 
+RELOAD_VCL=1
+
+# Main configuration file. You probably want to change it.
+VARNISH_VCL_CONF=/etc/varnish/default.vcl 
+
+# Default address and port to bind to. Blank address means all IPv4
+# and IPv6 interfaces, otherwise specify a host name, an IPv4 dotted
+# quad, or an IPv6 address in brackets.
+# VARNISH_LISTEN_ADDRESS=192.168.1.5
+VARNISH_LISTEN_PORT=6081                  
+
+# Admin interface listen address and port
+VARNISH_ADMIN_LISTEN_ADDRESS=127.0.0.1    
+VARNISH_ADMIN_LISTEN_PORT=6082            
+
+# Shared secret file for admin interface
+VARNISH_SECRET_FILE=/etc/varnish/secret   
+
+# The minimum and maximum number of worker threads to start
+VARNISH_MIN_THREADS=1                     
+VARNISH_MAX_THREADS=1000                  
+
+# Idle timeout for worker threads
+VARNISH_THREAD_TIMEOUT=120                
+
+# Backend storage specification, see Storage Types in the varnishd(5)
+# man page for details.
+VARNISH_STORAGE="file,/var/lib/varnish/varnish_storage.bin,1G"
+
+# Default TTL used when the backend does not specify one
+VARNISH_TTL=120                           
+
+# User and group for the varnishd worker processes
+VARNISH_USER=varnish                      
+VARNISH_GROUP=varnish                     
+
diff --git a/redhat/varnish.service b/redhat/varnish.service
new file mode 100644
index 0000000..6f7b107
--- /dev/null
+++ b/redhat/varnish.service
@@ -0,0 +1,47 @@
+[Unit]
+Description=Varnish a high-perfomance HTTP accelerator
+After=syslog.target network.target
+
+[Service]
+
+#
+# If you want to make changes to this file, please copy it to 
+# /etc/systemd/system/varnish.service and make your changes there.
+# This will override the file kept at /lib/systemd/system/varnish.service
+#
+# Enviroment variables may be found in /etc/varnish/varnish.params
+#
+
+# Maximum number of open files (for ulimit -n)
+LimitNOFILE=131072
+
+# Locked shared memory (for ulimit -l)
+# Default log size is 82MB + header
+LimitMEMLOCK=82000
+
+# Maximum size of the corefile. 
+LimitCORE=infinity
+
+EnvironmentFile=/etc/varnish/varnish.params
+
+Type=forking
+PIDFile=/var/run/varnish.pid
+PrivateTmp=true
+ExecStart=/usr/sbin/varnishd \
+	-P /var/run/varnish.pid \
+	-f $VARNISH_VCL_CONF \
+	-a ${VARNISH_LISTEN_ADDRESS}:${VARNISH_LISTEN_PORT} \
+	-T ${VARNISH_ADMIN_LISTEN_ADDRESS}:${VARNISH_ADMIN_LISTEN_PORT} \
+	-t $VARNISH_TTL \
+	-w ${VARNISH_MIN_THREADS},${VARNISH_MAX_THREADS},${VARNISH_THREAD_TIMEOUT} \
+	-u $VARNISH_USER \
+	-g $VARNISH_GROUP \
+	-S $VARNISH_SECRET_FILE \
+	-s $VARNISH_STORAGE \
+	$DAEMON_OPTS
+
+ExecReload=/usr/sbin/varnish_reload_vcl
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index a66e0b9..f71fc9e 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -27,6 +27,13 @@ Requires(preun): /sbin/service
 %if %{undefined suse_version}
 Requires(preun): initscripts
 %endif
+%if 0%{?fedora} >= 17
+Requires(post): systemd-units
+Requires(post): systemd-sysv
+Requires(preun): systemd-units
+Requires(postun): systemd-units
+BuildRequires: systemd-units
+%endif
 
 # Varnish actually needs gcc installed to work. It uses the C compiler 
 # at runtime to compile the VCL configuration files. This is by design.
@@ -160,12 +167,24 @@ mkdir -p %{buildroot}/var/log/varnish
 mkdir -p %{buildroot}/var/run/varnish
 mkdir -p %{buildroot}%{_sysconfdir}/ld.so.conf.d/
 install -D -m 0644 redhat/default.vcl %{buildroot}%{_sysconfdir}/varnish/default.vcl
-install -D -m 0644 redhat/varnish.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/varnish
 install -D -m 0644 redhat/varnish.logrotate %{buildroot}%{_sysconfdir}/logrotate.d/varnish
+
+# systemd support
+%if 0%{?fedora} >= 17
+mkdir -p %{buildroot}%{_unitdir}
+install -D -m 0644 redhat/varnish.service %{buildroot}%{_unitdir}/varnish.service
+install -D -m 0644 redhat/varnish.params %{buildroot}%{_sysconfdir}/varnish/varnish.params
+install -D -m 0644 redhat/varnishncsa.service %{buildroot}%{_unitdir}/varnishncsa.service
+install -D -m 0644 redhat/varnishlog.service %{buildroot}%{_unitdir}/varnishlog.service
+sed -i 's,sysconfig/varnish,varnish/varnish.params,' redhat/varnish_reload_vcl
+# default is standard sysvinit
+%else
+install -D -m 0644 redhat/varnish.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/varnish
 install -D -m 0755 redhat/varnish.initrc %{buildroot}%{_initrddir}/varnish
 install -D -m 0755 redhat/varnishlog.initrc %{buildroot}%{_initrddir}/varnishlog
 install -D -m 0755 redhat/varnishncsa.initrc %{buildroot}%{_initrddir}/varnishncsa
-install -D -m 0755 redhat/varnish_reload_vcl %{buildroot}%{_bindir}/varnish_reload_vcl
+%endif
+install -D -m 0755 redhat/varnish_reload_vcl %{buildroot}%{_sbindir}/varnish_reload_vcl
 
 echo %{_libdir}/varnish > %{buildroot}%{_sysconfdir}/ld.so.conf.d/varnish-%{_arch}.conf
 
@@ -185,11 +204,22 @@ rm -rf %{buildroot}
 %doc examples
 %dir %{_sysconfdir}/varnish/
 %config(noreplace) %{_sysconfdir}/varnish/default.vcl
-%config(noreplace) %{_sysconfdir}/sysconfig/varnish
 %config(noreplace) %{_sysconfdir}/logrotate.d/varnish
+
+# systemd from fedora 17
+%if 0%{?fedora} >= 17
+%{_unitdir}/varnish.service
+%{_unitdir}/varnishncsa.service
+%{_unitdir}/varnishlog.service
+%config(noreplace)%{_sysconfdir}/varnish/varnish.params
+
+# default is standard sysvinit
+%else
+%config(noreplace) %{_sysconfdir}/sysconfig/varnish
 %{_initrddir}/varnish
 %{_initrddir}/varnishlog
 %{_initrddir}/varnishncsa
+%endif
 
 %files libs
 %defattr(-,root,root,-)
@@ -228,19 +258,42 @@ getent passwd varnish >/dev/null || \
 exit 0
 
 %post
+%if 0%{?fedora} >= 17
+/bin/systemctl daemon-reload >/dev/null 2>&1 || :
+%else
 /sbin/chkconfig --add varnish
 /sbin/chkconfig --add varnishlog
 /sbin/chkconfig --add varnishncsa 
+%endif
 test -f /etc/varnish/secret || (uuidgen > /etc/varnish/secret && chmod 0600 /etc/varnish/secret)
 
+%triggerun -- varnish < 3.0.2-1
+# Save the current service runlevel info
+# User must manually run systemd-sysv-convert --apply varnish 
+# to migrate them to systemd targets
+%{_bindir}/systemd-sysv-convert --save varnish >/dev/null 2>&1 ||:
+
+# If the package is allowed to autostart:
+#/bin/systemctl --no-reload enable varnish.service >/dev/null 2>&1 ||:
+
+# Run these because the SysV package being removed won't do them
+/sbin/chkconfig --del varnish >/dev/null 2>&1 || :
+#/bin/systemctl try-restart varnish.service >/dev/null 2>&1 || :
+
 %preun
 if [ $1 -lt 1 ]; then
+  # Package removal, not upgrade
+  %if 0%{?fedora} >= 17
+  /bin/systemctl --no-reload disable varnish.service > /dev/null 2>&1 || :
+  /bin/systemctl stop varnish.service > /dev/null 2>&1 || :
+  %else
   /sbin/service varnish stop > /dev/null 2>&1
   /sbin/service varnishlog stop > /dev/null 2>&1
   /sbin/service varnishncsa stop > /dev/null 2>%1
   /sbin/chkconfig --del varnish
   /sbin/chkconfig --del varnishlog
   /sbin/chkconfig --del varnishncsa 
+  %endif
 fi
 
 %post libs -p /sbin/ldconfig
diff --git a/redhat/varnishlog.service b/redhat/varnishlog.service
new file mode 100644
index 0000000..1e3e274
--- /dev/null
+++ b/redhat/varnishlog.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Varnish HTTP accelerator logging daemon
+After=network.target
+
+[Service]
+Type=forking
+PIDFile=/run/varnishlog.pid
+ExecStart=/usr/bin/varnishlog -a -w /var/log/varnish/varnish.log -D -P /run/varnishlog.pid
+
+[Install]
+WantedBy=multi-user.target
diff --git a/redhat/varnishncsa.service b/redhat/varnishncsa.service
new file mode 100644
index 0000000..df5f19f
--- /dev/null
+++ b/redhat/varnishncsa.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Varnish NCSA logging
+After=network.target
+
+[Service]
+Type=forking
+PIDFile=/run/varnishncsa.pid
+ExecStart=/usr/bin/varnishncsa -a -w /var/log/varnish/varnishncsa.log -D -P /run/varnishncsa.pid
+
+[Install]
+WantedBy=multi-user.target



From ingvar at varnish-cache.org  Tue Apr 24 14:04:24 2012
From: ingvar at varnish-cache.org (Ingvar)
Date: Tue, 24 Apr 2012 16:04:24 +0200
Subject: [master] 7678c0f files for systemd
Message-ID: <E1SMgLw-0003Zv-Ri@project.varnish-software.com>

commit 7678c0f84bd422fa169b0b335e8485d73165834e
Author: Ingvar Hagelund <ingvar at redpill-linpro.com>
Date:   Tue Apr 24 16:04:18 2012 +0200

    files for systemd

diff --git a/redhat/Makefile.am b/redhat/Makefile.am
index 86ddfd4..16ff9d6 100644
--- a/redhat/Makefile.am
+++ b/redhat/Makefile.am
@@ -9,4 +9,8 @@ EXTRA_DIST =			\
 	varnish.spec		\
 	varnish.sysconfig	\
 	varnishlog.initrc	\
-	varnishncsa.initrc
+	varnishncsa.initrc	\
+	varnish.params		\
+	varnish.service		\
+	varnishncsa.service	\
+	varnishlog.service



From ingvar at varnish-cache.org  Tue Apr 24 14:04:24 2012
From: ingvar at varnish-cache.org (Ingvar)
Date: Tue, 24 Apr 2012 16:04:24 +0200
Subject: [master] 469aa69 update systemd params and unit file to match trunk
Message-ID: <E1SMgLw-0003Zq-Q8@project.varnish-software.com>

commit 469aa691ff6d2fa91646b9d755a7d5b8e3e3d8ec
Author: Ingvar Hagelund <ingvar at redpill-linpro.com>
Date:   Tue Apr 24 16:04:05 2012 +0200

    update systemd params and unit file to match trunk

diff --git a/redhat/varnish.params b/redhat/varnish.params
index d889ee4..636e975 100644
--- a/redhat/varnish.params
+++ b/redhat/varnish.params
@@ -20,13 +20,6 @@ VARNISH_ADMIN_LISTEN_PORT=6082
 # Shared secret file for admin interface
 VARNISH_SECRET_FILE=/etc/varnish/secret   
 
-# The minimum and maximum number of worker threads to start
-VARNISH_MIN_THREADS=1                     
-VARNISH_MAX_THREADS=1000                  
-
-# Idle timeout for worker threads
-VARNISH_THREAD_TIMEOUT=120                
-
 # Backend storage specification, see Storage Types in the varnishd(5)
 # man page for details.
 VARNISH_STORAGE="file,/var/lib/varnish/varnish_storage.bin,1G"
@@ -38,3 +31,5 @@ VARNISH_TTL=120
 VARNISH_USER=varnish                      
 VARNISH_GROUP=varnish                     
 
+# Other options, see the man page varnishd(1)
+#DAEMON_OPTS="-p thread_pool_min=5 -p thread_pool_max=500 -p thread_pool_timeout=300"
diff --git a/redhat/varnish.service b/redhat/varnish.service
index 6f7b107..659dba2 100644
--- a/redhat/varnish.service
+++ b/redhat/varnish.service
@@ -33,7 +33,6 @@ ExecStart=/usr/sbin/varnishd \
 	-a ${VARNISH_LISTEN_ADDRESS}:${VARNISH_LISTEN_PORT} \
 	-T ${VARNISH_ADMIN_LISTEN_ADDRESS}:${VARNISH_ADMIN_LISTEN_PORT} \
 	-t $VARNISH_TTL \
-	-w ${VARNISH_MIN_THREADS},${VARNISH_MAX_THREADS},${VARNISH_THREAD_TIMEOUT} \
 	-u $VARNISH_USER \
 	-g $VARNISH_GROUP \
 	-S $VARNISH_SECRET_FILE \



From phk at varnish-cache.org  Thu Apr 26 16:10:48 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 26 Apr 2012 18:10:48 +0200
Subject: [master] 562c67d Make it possible for Open/Net/DragonFly-BSD to find
	readline.h
Message-ID: <E1SNRHM-00056u-QM@project.varnish-software.com>

commit 562c67db250ed6decea1015edb5b6170096c123a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Apr 26 16:10:17 2012 +0000

    Make it possible for Open/Net/DragonFly-BSD to find readline.h

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index f3b8b69..aff9336 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -35,6 +35,8 @@
 #  include <stdio.h>
 #  ifdef HAVE_EDIT_READLINE_READLINE_H
 #    include <edit/readline/readline.h>
+#  elif HAVE_READLINE_READLINE_H
+#    include <readline/readline.h>
 #  else
 #    include <editline/readline.h>
 #  endif



From tfheen at varnish-cache.org  Fri Apr 27 10:54:37 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Fri, 27 Apr 2012 12:54:37 +0200
Subject: [master] a8ac09b Add missing + in docs
Message-ID: <E1SNiov-0001Nr-H0@project.varnish-software.com>

commit a8ac09bb97bc2903c0696b0af8c659bb57d851dd
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Fri Apr 27 12:53:44 2012 +0200

    Add missing + in docs
    
    Thanks to Jeff Williams for pointing this out.

diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst
index 846ca82..22056a7 100644
--- a/doc/sphinx/tutorial/purging.rst
+++ b/doc/sphinx/tutorial/purging.rst
@@ -146,7 +146,7 @@ You can use the following template to write ban lurker friendly bans::
       if (client.ip !~ purge) {
         error 401 "Not allowed";
       }
-      ban("obj.http.x-url ~ " req.url); # Assumes req.url is a regex. This might be a bit too simple
+      ban("obj.http.x-url ~ " + req.url); # Assumes req.url is a regex. This might be a bit too simple
     }
   }
 



From tfheen at varnish-cache.org  Fri Apr 27 10:57:12 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Fri, 27 Apr 2012 12:57:12 +0200
Subject: [3.0] 8dd2a4c Add missing + in docs
Message-ID: <E1SNirQ-0002qI-Gz@project.varnish-software.com>

commit 8dd2a4c436fc9931ba9deb1709e16e659f0da666
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Fri Apr 27 12:53:44 2012 +0200

    Add missing + in docs
    
    Thanks to Jeff Williams for pointing this out.

diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst
index ecf0e96..5e5768e 100644
--- a/doc/sphinx/tutorial/purging.rst
+++ b/doc/sphinx/tutorial/purging.rst
@@ -146,7 +146,7 @@ You can use the following template to write ban lurker friendly bans::
       if (client.ip !~ purge) {
         error 401 "Not allowed";
       }
-      ban("obj.http.x-url ~ " req.url); # Assumes req.url is a regex. This might be a bit too simple
+      ban("obj.http.x-url ~ " + req.url); # Assumes req.url is a regex. This might be a bit too simple
     }
   }
 



From phk at varnish-cache.org  Fri Apr 27 13:44:06 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 27 Apr 2012 15:44:06 +0200
Subject: [master] 9c449d0 Make things compile on OpenBSD
Message-ID: <E1SNlSw-0006Vp-TJ@project.varnish-software.com>

commit 9c449d0417df9e1268f81271dbfd11f700aa24bd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Apr 27 13:43:51 2012 +0000

    Make things compile on OpenBSD
    
    Submitted by:	Brad Smith <brad at comstyle.com>

diff --git a/bin/varnishd/cache/cache_dir_dns.c b/bin/varnishd/cache/cache_dir_dns.c
index 9cad3f6..ee411dc 100644
--- a/bin/varnishd/cache/cache_dir_dns.c
+++ b/bin/varnishd/cache/cache_dir_dns.c
@@ -29,6 +29,8 @@
 
 #include "config.h"
 
+#include <sys/types.h>
+#include <sys/socket.h>
 #include <netinet/in.h>
 
 #include <netdb.h>
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index cbe5637..10658cd 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -31,6 +31,8 @@
 
 #include "config.h"
 
+#include <sys/types.h>
+#include <sys/socket.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
 
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index cbea81f..3b0c1a7 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -31,6 +31,7 @@
 
 #include "config.h"
 
+#include <sys/types.h>
 #include <sys/socket.h>
 
 #include <fcntl.h>
diff --git a/bin/varnishd/storage/stevedore_utils.c b/bin/varnishd/storage/stevedore_utils.c
index 89c6027..2364289 100644
--- a/bin/varnishd/storage/stevedore_utils.c
+++ b/bin/varnishd/storage/stevedore_utils.c
@@ -34,6 +34,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #ifdef HAVE_SYS_MOUNT_H
+#  include <sys/param.h>
 #  include <sys/mount.h>
 #endif
 #ifdef HAVE_SYS_STATVFS_H



From phk at varnish-cache.org  Sat Apr 28 07:16:56 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 28 Apr 2012 09:16:56 +0200
Subject: [master] 563b21e Missed these two in yesterdays OpenBSD compat commit.
Message-ID: <E1SO1to-0005H0-7k@project.varnish-software.com>

commit 563b21e1c6f4ea9891b527877cf36d0df8380cc4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Apr 28 07:16:13 2012 +0000

    Missed these two in yesterdays OpenBSD compat commit.

diff --git a/lib/libvarnish/vsha256.c b/lib/libvarnish/vsha256.c
index 10b7a6a..0a521b2 100644
--- a/lib/libvarnish/vsha256.c
+++ b/lib/libvarnish/vsha256.c
@@ -29,6 +29,7 @@
 #include "config.h"
 
 #ifdef HAVE_SYS_ENDIAN_H
+#include <sys/types.h>
 #include <sys/endian.h>
 #define VBYTE_ORDER	_BYTE_ORDER
 #define VBIG_ENDIAN	_BIG_ENDIAN
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index d1f6771..5b5c0aa 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -28,6 +28,8 @@
 
 #include "config.h"
 
+#include <sys/types.h>
+#include <sys/socket.h>
 #include <netinet/in.h>
 
 #include <ctype.h>



From phk at varnish-cache.org  Sat Apr 28 07:16:56 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 28 Apr 2012 09:16:56 +0200
Subject: [master] f564428 Be consistent about what environment we test the
	compiled VCL in.
Message-ID: <E1SO1to-0005H5-8u@project.varnish-software.com>

commit f564428bb480edf3c3d170087a227dee2797bdbb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Apr 28 07:16:33 2012 +0000

    Be consistent about what environment we test the compiled VCL in.

diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index 98dc4c4..e419261 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -186,7 +186,9 @@ run_dlopen(void *priv)
 
 	of = priv;
 
-	/* Try to load the object into the management process */
+	mgt_sandbox();
+
+	/* Try to load the object into this sub-process */
 	if ((dlh = dlopen(of, RTLD_NOW | RTLD_LOCAL)) == NULL) {
 		fprintf(stderr,
 		    "Compiled VCL program failed to load:\n  %s\n",



From phk at varnish-cache.org  Sat Apr 28 09:19:24 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 28 Apr 2012 11:19:24 +0200
Subject: [master] 6d5c733 Variable rename for easier searching.
Message-ID: <E1SO3oK-0003sB-C5@project.varnish-software.com>

commit 6d5c7338eddf7f1fdeef11193db1519244319380
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Apr 28 08:44:11 2012 +0000

    Variable rename for easier searching.

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index f6493ad..7f63e2f 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -56,7 +56,7 @@
 #define MAGIC_INIT_STRING	"\001"
 struct params mgt_param;
 static int nparspec;
-static struct parspec const ** parspec;
+static struct parspec const ** parspecs;
 static int margin;
 
 /*--------------------------------------------------------------------*/
@@ -67,8 +67,8 @@ mcf_findpar(const char *name)
 	int i;
 
 	for (i = 0; i < nparspec; i++)
-		if (!strcmp(parspec[i]->name, name))
-			return (parspec[i]);
+		if (!strcmp(parspecs[i]->name, name))
+			return (parspecs[i]);
 	return (NULL);
 }
 
@@ -1289,7 +1289,7 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv)
 	else
 		lfmt = 1;
 	for (i = 0; i < nparspec; i++) {
-		pp = parspec[i];
+		pp = parspecs[i];
 		if (av[2] != NULL && !lfmt && strcmp(pp->name, av[2]))
 			continue;
 		VCLI_Out(cli, "%-*s ", margin, pp->name);
@@ -1398,12 +1398,12 @@ MCF_AddParams(const struct parspec *ps)
 			margin = strlen(pp->name) + 1;
 		n++;
 	}
-	parspec = realloc(parspec, (1L + nparspec + n) * sizeof *parspec);
-	XXXAN(parspec);
+	parspecs = realloc(parspecs, (1L + nparspec + n) * sizeof *parspecs);
+	XXXAN(parspecs);
 	for (pp = ps; pp->name != NULL; pp++)
-		parspec[nparspec++] = pp;
-	parspec[nparspec] = NULL;
-	qsort (parspec, nparspec, sizeof parspec[0], parspec_cmp);
+		parspecs[nparspec++] = pp;
+	parspecs[nparspec] = NULL;
+	qsort (parspecs, nparspec, sizeof parspecs[0], parspec_cmp);
 }
 
 /*--------------------------------------------------------------------
@@ -1417,7 +1417,7 @@ MCF_SetDefaults(struct cli *cli)
 	int i;
 
 	for (i = 0; i < nparspec; i++) {
-		pp = parspec[i];
+		pp = parspecs[i];
 		if (cli != NULL)
 			VCLI_Out(cli,
 			    "Set Default for %s = %s\n", pp->name, pp->def);
@@ -1453,7 +1453,7 @@ MCF_DumpRst(void)
 	printf("\n.. The following is the autogenerated "
 	    "output from varnishd -x dumprst\n\n");
 	for (i = 0; i < nparspec; i++) {
-		pp = parspec[i];
+		pp = parspecs[i];
 		printf("%s\n", pp->name);
 		if (pp->units != NULL && *pp->units != '\0')
 			printf("\t- Units: %s\n", pp->units);



From phk at varnish-cache.org  Sat Apr 28 09:19:24 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 28 Apr 2012 11:19:24 +0200
Subject: [master] 2361947 Another bit of the OpenBSD commit I missed.
Message-ID: <E1SO3oK-0003sK-DO@project.varnish-software.com>

commit 2361947fca3b004a87c08fdefa74231520881e7b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Apr 28 08:44:33 2012 +0000

    Another bit of the OpenBSD commit I missed.

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index aff9336..f4c262c 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -29,6 +29,7 @@
 
 #include "config.h"
 
+#include <sys/types.h>
 #include <sys/socket.h>
 
 #ifdef HAVE_LIBEDIT



From phk at varnish-cache.org  Sat Apr 28 09:20:07 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 28 Apr 2012 11:20:07 +0200
Subject: [master] 0b00877 Add a -r ("read-only") argument which can protect
	parameters from subsequent changes.
Message-ID: <E1SO3p1-0004As-U8@project.varnish-software.com>

commit 0b00877030e426e9ed3b867772074fc2e0b9df63
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Apr 28 09:19:35 2012 +0000

    Add a -r ("read-only") argument which can protect parameters from
    subsequent changes.

diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index ffaef56..c319e57 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -69,6 +69,7 @@ const void *pick(const struct choice *cp, const char *which, const char *kind);
 /* mgt_param.c */
 void MCF_ParamInit(struct cli *);
 void MCF_ParamSet(struct cli *, const char *param, const char *val);
+void MCF_ParamProtect(struct cli *, const char *arg);
 void MCF_DumpRst(void);
 extern struct params mgt_param;
 
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 6ad7454..13799f9 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -127,6 +127,7 @@ usage(void)
 	fprintf(stderr, FMT, "-n dir", "varnishd working directory");
 	fprintf(stderr, FMT, "-P file", "PID file");
 	fprintf(stderr, FMT, "-p param=value", "set parameter");
+	fprintf(stderr, FMT, "-r param[,param...]", "make parameter read-only");
 	fprintf(stderr, FMT,
 	    "-s kind[,storageoptions]", "Backend storage specification");
 	fprintf(stderr, FMT, "", "  -s malloc");
@@ -369,7 +370,7 @@ main(int argc, char * const *argv)
 	cli_check(cli);
 
 	while ((o = getopt(argc, argv,
-	    "a:b:Cdf:Fg:h:i:l:L:M:n:P:p:S:s:T:t:u:Vx:w:")) != -1)
+	    "a:b:Cdf:Fg:h:i:l:L:M:n:P:p:r:S:s:T:t:u:Vx:w:")) != -1)
 		switch (o) {
 		case 'a':
 			MCF_ParamSet(cli, "listen_address", optarg);
@@ -432,6 +433,10 @@ main(int argc, char * const *argv)
 			MCF_ParamSet(cli, optarg, p);
 			cli_check(cli);
 			break;
+		case 'r':
+			MCF_ParamProtect(cli, optarg);
+			cli_check(cli);
+			break;
 		case 's':
 			s_arg_given = 1;
 			STV_Config(optarg);
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 7f63e2f..e79b7e9 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -668,6 +668,9 @@ tweak_poolparam(struct cli *cli, const struct parspec *par, const char *arg)
 	"\nNB: Do not change this parameter, unless a developer tell " \
 	"you to do so."
 
+#define PROTECTED_TEXT \
+	"\nNB: This parameter is protected and can not be changed."
+
 #define MEMPOOL_TEXT							\
 	"The three numbers are:\n"					\
 	"   min_pool -- minimum size of free pool.\n"			\
@@ -1319,6 +1322,8 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv)
 				mcf_wrap(cli, MUST_RESTART_TEXT);
 			if (pp->flags & WIZARD)
 				mcf_wrap(cli, WIZARD_TEXT);
+			if (pp->flags & PROTECTED)
+				mcf_wrap(cli, PROTECTED_TEXT);
 			if (!lfmt)
 				return;
 			else
@@ -1331,6 +1336,43 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv)
 	}
 }
 
+/*--------------------------------------------------------------------
+ * Mark paramters as protected
+ */
+
+void
+MCF_ParamProtect(struct cli *cli, const char *args)
+{
+	char **av;
+	struct parspec *pp;
+	int i, j;
+
+	av = VAV_Parse(args, NULL, ARGV_COMMA);
+	if (av[0] != NULL) {
+		VCLI_Out(cli, "Parse error: %s", av[0]);
+		VCLI_SetResult(cli, CLIS_PARAM);
+		VAV_Free(av);
+		return;
+	}
+	for (i = 1; av[i] != NULL; i++) {
+		for (j = 0; j < nparspec; j++)
+			if (!strcmp(parspecs[j]->name, av[i]))
+				break;
+		if (j == nparspec) {
+			VCLI_Out(cli, "Unknown parameter %s", av[i]);
+			VCLI_SetResult(cli, CLIS_PARAM);
+			VAV_Free(av);
+			return;
+		}
+		pp = calloc(sizeof *pp, 1L);
+		XXXAN(pp);
+		memcpy(pp, parspecs[j], sizeof *pp);
+		pp->flags |= PROTECTED;
+		parspecs[j] = pp;
+	}
+	VAV_Free(av);
+}
+
 /*--------------------------------------------------------------------*/
 
 void
@@ -1344,6 +1386,11 @@ MCF_ParamSet(struct cli *cli, const char *param, const char *val)
 		VCLI_Out(cli, "Unknown parameter \"%s\".", param);
 		return;
 	}
+	if (pp->flags & PROTECTED) {
+		VCLI_SetResult(cli, CLIS_AUTH);
+		VCLI_Out(cli, "parameter \"%s\" is protected.", param);
+		return;
+	}
 	pp->func(cli, pp, val);
 
 	if (cli->result == CLIS_OK && heritage.param != NULL)
diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h
index a5d4d75..b4538d2 100644
--- a/bin/varnishd/mgt/mgt_param.h
+++ b/bin/varnishd/mgt/mgt_param.h
@@ -45,6 +45,7 @@ struct parspec {
 #define MUST_RESTART	(1<<2)
 #define MUST_RELOAD	(1<<3)
 #define WIZARD		(1<<4)
+#define PROTECTED	(1<<5)
 	const char	*def;
 	const char	*units;
 };
diff --git a/bin/varnishtest/tests/c00051.vtc b/bin/varnishtest/tests/c00051.vtc
new file mode 100644
index 0000000..78fdc81
--- /dev/null
+++ b/bin/varnishtest/tests/c00051.vtc
@@ -0,0 +1,6 @@
+varnishtest "test parameter protection"
+
+varnish v1 -arg "-r cli_timeout" 
+
+varnish v1 -cliok "param.show cli_timeout"
+varnish v1 -clierr 107 "param.set cli_timeout 1m"



From phk at varnish-cache.org  Sat Apr 28 16:41:39 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 28 Apr 2012 18:41:39 +0200
Subject: [master] 2b623ea Add a parameter to disable inline-C code in VCL
Message-ID: <E1SOAiJ-0003pM-3R@project.varnish-software.com>

commit 2b623eafd3de8f9dfdc814c30d264e9dc9090e33
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Apr 28 15:06:14 2012 +0000

    Add a parameter to disable inline-C code in VCL

diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index c319e57..0ad01de 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -105,6 +105,7 @@ extern char *mgt_cc_cmd;
 extern const char *mgt_vcl_dir;
 extern const char *mgt_vmod_dir;
 extern unsigned mgt_vcc_err_unref;
+extern unsigned mgt_vcc_allow_inline_c;
 
 #define REPORT0(pri, fmt)				\
 	do {						\
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index e79b7e9..b72b0df 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -1177,11 +1177,17 @@ static const struct parspec input_parspec[] = {
 		".",
 #endif
 		NULL },
+
 	{ "vcc_err_unref", tweak_bool, &mgt_vcc_err_unref, 0, 0,
 		"Unreferenced VCL objects result in error.\n",
 		0,
 		"on", "bool" },
 
+	{ "vcc_allow_inline_c", tweak_bool, &mgt_vcc_allow_inline_c, 0, 0,
+		"Allow inline C code in VCL.\n",
+		0,
+		"on", "bool" },
+
 
 	{ "pcre_match_limit", tweak_uint,
 		&mgt_param.vre_limits.match,
diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index e419261..269a453 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -63,6 +63,7 @@ char *mgt_cc_cmd;
 const char *mgt_vcl_dir;
 const char *mgt_vmod_dir;
 unsigned mgt_vcc_err_unref;
+unsigned mgt_vcc_allow_inline_c;
 
 static struct vcc *vcc;
 
@@ -139,6 +140,7 @@ run_vcc(void *priv)
 	VCC_VCL_dir(vcc, mgt_vcl_dir);
 	VCC_VMOD_dir(vcc, mgt_vmod_dir);
 	VCC_Err_Unref(vcc, mgt_vcc_err_unref);
+	VCC_Allow_InlineC(vcc, mgt_vcc_allow_inline_c);
 	csrc = VCC_Compile(vcc, sb, vp->vcl);
 	AZ(VSB_finish(sb));
 	if (VSB_len(sb))



From phk at varnish-cache.org  Sat Apr 28 16:41:39 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 28 Apr 2012 18:41:39 +0200
Subject: [master] fb6f3c5 Add parameter vcc_unsafe_path which allows '/' in
	include "..." and 'import ... from ...'. Default is on (= no change)
Message-ID: <E1SOAiJ-0003pS-5K@project.varnish-software.com>

commit fb6f3c54177d7aa85a59d10da3bb9b549c0b9aa8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Apr 28 16:03:17 2012 +0000

    Add parameter vcc_unsafe_path which allows '/' in include "..."
    and 'import ... from ...'.  Default is on (= no change)

diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 0ad01de..65dbc02 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -106,6 +106,7 @@ extern const char *mgt_vcl_dir;
 extern const char *mgt_vmod_dir;
 extern unsigned mgt_vcc_err_unref;
 extern unsigned mgt_vcc_allow_inline_c;
+extern unsigned mgt_vcc_unsafe_path;
 
 #define REPORT0(pri, fmt)				\
 	do {						\
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index b72b0df..885d2ce 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -1188,6 +1188,11 @@ static const struct parspec input_parspec[] = {
 		0,
 		"on", "bool" },
 
+	{ "vcc_unsafe_path", tweak_bool, &mgt_vcc_unsafe_path, 0, 0,
+		"Allow '/' in vmod & include paths.\n"
+		"Allow 'import ... from ...'.\n",
+		0,
+		"on", "bool" },
 
 	{ "pcre_match_limit", tweak_uint,
 		&mgt_param.vre_limits.match,
diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index 269a453..9b46da3 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -64,6 +64,7 @@ const char *mgt_vcl_dir;
 const char *mgt_vmod_dir;
 unsigned mgt_vcc_err_unref;
 unsigned mgt_vcc_allow_inline_c;
+unsigned mgt_vcc_unsafe_path;
 
 static struct vcc *vcc;
 
@@ -141,6 +142,7 @@ run_vcc(void *priv)
 	VCC_VMOD_dir(vcc, mgt_vmod_dir);
 	VCC_Err_Unref(vcc, mgt_vcc_err_unref);
 	VCC_Allow_InlineC(vcc, mgt_vcc_allow_inline_c);
+	VCC_Unsafe_Path(vcc, mgt_vcc_unsafe_path);
 	csrc = VCC_Compile(vcc, sb, vp->vcl);
 	AZ(VSB_finish(sb));
 	if (VSB_len(sb))
diff --git a/bin/varnishtest/tests/c00052.vtc b/bin/varnishtest/tests/c00052.vtc
new file mode 100644
index 0000000..e640230
--- /dev/null
+++ b/bin/varnishtest/tests/c00052.vtc
@@ -0,0 +1,51 @@
+varnishtest "Test disabling inline C code"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 
+
+varnish v1 -cliok "param.show vcc_allow_inline_c"
+
+varnish v1 -vcl+backend {
+	C{ getpid(); }C
+} 
+
+varnish v1 -cliok "param.set vcc_allow_inline_c false"
+
+varnish v1 -badvcl {
+	backend default {
+		.host = "${s1_sock}";
+	}
+	C{ getpid(); }C
+}
+
+varnish v1 -badvcl {
+	backend default {
+		.host = "${s1_sock}";
+	}
+	sub vcl_recv {
+		C{ getpid(); }C
+	}
+}
+
+varnish v1 -cliok "param.set vcc_allow_inline_c true"
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		C{ getpid(); }C
+	}
+}
+
+varnish v1 -vcl+backend {
+	C{ extern int getpid(); }C
+}
+
+varnish v1 -start
+
+client c1 {
+	txreq
+	rxresp
+} -run
diff --git a/bin/varnishtest/tests/c00053.vtc b/bin/varnishtest/tests/c00053.vtc
new file mode 100644
index 0000000..c326293
--- /dev/null
+++ b/bin/varnishtest/tests/c00053.vtc
@@ -0,0 +1,29 @@
+varnishtest "Test inclide vs. unsafe_path"
+
+server s1 {
+	rxreq 
+	txresp -hdr "foo: bAr" -hdr "bar: fOo" -bodylen 4
+} -start
+
+shell "echo > ${pwd}/_.c00053"
+
+varnish v1 -vcl+backend {
+	include "${pwd}/_.c00053";
+}
+
+varnish v1 -cliok "param.set vcc_unsafe_path off"
+
+varnish v1 -badvcl {
+	backend default {
+		.host = "${s1_sock}";
+	}
+	include "${pwd}/_.c00053";
+}
+
+varnish v1 -cliok "param.set vcl_dir ${pwd}"
+
+varnish v1 -vcl+backend {
+	include "_.c00053";
+}
+
+shell "rm -f ${pwd}/_.c00053"
diff --git a/bin/varnishtest/tests/m00008.vtc b/bin/varnishtest/tests/m00008.vtc
new file mode 100644
index 0000000..3d1d665
--- /dev/null
+++ b/bin/varnishtest/tests/m00008.vtc
@@ -0,0 +1,23 @@
+varnishtest "Test std vmod vs. unsafe_path"
+
+server s1 {
+	rxreq 
+	txresp -hdr "foo: bAr" -hdr "bar: fOo" -bodylen 4
+} -start
+
+varnish v1 -vcl+backend {
+	import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ;
+}
+
+varnish v1 -cliok "param.set vcc_unsafe_path off"
+
+varnish v1 -badvcl {
+	backend default { .host = "${s1_sock}"; }
+	import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ;
+}
+
+varnish v1 -cliok "param.set vmod_dir ${topbuild}/lib/libvmod_std/.libs/"
+
+varnish v1 -vcl+backend {
+	import std;
+}
diff --git a/include/libvcl.h b/include/libvcl.h
index e046db9..5c50c34 100644
--- a/include/libvcl.h
+++ b/include/libvcl.h
@@ -35,5 +35,7 @@ void VCC_Default_VCL(struct vcc *, const char *str);
 void VCC_VCL_dir(struct vcc *, const char *str);
 void VCC_VMOD_dir(struct vcc *, const char *str);
 void VCC_Err_Unref(struct vcc *tl, unsigned u);
+void VCC_Allow_InlineC(struct vcc *tl, unsigned u);
+void VCC_Unsafe_Path(struct vcc *tl, unsigned u);
 
 char *VCC_Compile(const struct vcc *, struct vsb *sb, const char *b);
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index 66d89f5..c9fe573 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -411,6 +411,10 @@ vcc_file_source(const struct vcc *tl, struct vsb *sb, const char *fn)
 	char *f;
 	struct source *sp;
 
+	if (!tl->unsafe_path && strchr(fn, '/') != NULL) {
+		VSB_printf(sb, "Include path is unsafe '%s'\n", fn);
+		return (NULL);
+	}
 	f = VFIL_readfile(tl->vcl_dir, fn, NULL);
 	if (f == NULL) {
 		VSB_printf(sb, "Cannot read file '%s': %s\n",
@@ -487,6 +491,8 @@ vcc_NewVcc(const struct vcc *tl0)
 		REPLACE(tl->vmod_dir, tl0->vmod_dir);
 		tl->vars = tl0->vars;
 		tl->err_unref = tl0->err_unref;
+		tl->allow_inline_c = tl0->allow_inline_c;
+		tl->unsafe_path = tl0->unsafe_path;
 	} else {
 		tl->err_unref = 1;
 	}
@@ -763,7 +769,7 @@ VCC_VMOD_dir(struct vcc *tl, const char *str)
 }
 
 /*--------------------------------------------------------------------
- * Configure default
+ * Configure settings
  */
 
 void
@@ -773,3 +779,19 @@ VCC_Err_Unref(struct vcc *tl, unsigned u)
 	CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
 	tl->err_unref = u;
 }
+
+void
+VCC_Allow_InlineC(struct vcc *tl, unsigned u)
+{
+
+	CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
+	tl->allow_inline_c = u;
+}
+
+void
+VCC_Unsafe_Path(struct vcc *tl, unsigned u)
+{
+
+	CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
+	tl->unsafe_path = u;
+}
diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h
index b64564e..c85af5b 100644
--- a/lib/libvcl/vcc_compile.h
+++ b/lib/libvcl/vcc_compile.h
@@ -193,6 +193,8 @@ struct vcc {
 	unsigned		nvmodpriv;
 
 	unsigned		err_unref;
+	unsigned		allow_inline_c;
+	unsigned		unsafe_path;
 };
 
 struct var {
diff --git a/lib/libvcl/vcc_parse.c b/lib/libvcl/vcc_parse.c
index d8e74a4..1b7ee88 100644
--- a/lib/libvcl/vcc_parse.c
+++ b/lib/libvcl/vcc_parse.c
@@ -153,10 +153,16 @@ vcc_Compound(struct vcc *tl)
 			Fb(tl, 1, "}\n");
 			return;
 		case CSRC:
-			Fb(tl, 1, "%.*s\n",
-			    (int) (tl->t->e - (tl->t->b + 2)),
-			    tl->t->b + 1);
-			vcc_NextToken(tl);
+			if (tl->allow_inline_c) {
+				Fb(tl, 1, "%.*s\n",
+				    (int) (tl->t->e - (tl->t->b + 2)),
+				    tl->t->b + 1);
+				vcc_NextToken(tl);
+			} else {
+				VSB_printf(tl->sb,
+				    "Inline-C not allowed");
+				vcc_ErrWhere(tl, tl->t);
+			}
 			break;
 		case EOI:
 			VSB_printf(tl->sb,
@@ -273,9 +279,16 @@ vcc_Parse(struct vcc *tl)
 		ERRCHK(tl);
 		switch (tl->t->tok) {
 		case CSRC:
-			Fc(tl, 0, "%.*s\n",
-			    (int) (tl->t->e - (tl->t->b + 4)), tl->t->b + 2);
-			vcc_NextToken(tl);
+			if (tl->allow_inline_c) {
+				Fc(tl, 0, "%.*s\n",
+				    (int) (tl->t->e - (tl->t->b + 4)),
+				    tl->t->b + 2);
+				vcc_NextToken(tl);
+			} else {
+				VSB_printf(tl->sb,
+				    "Inline-C not allowed");
+				vcc_ErrWhere(tl, tl->t);
+			}
 			break;
 		case EOI:
 			break;
diff --git a/lib/libvcl/vcc_vmod.c b/lib/libvcl/vcc_vmod.c
index bd9f366..4ce4f95 100644
--- a/lib/libvcl/vcc_vmod.c
+++ b/lib/libvcl/vcc_vmod.c
@@ -56,7 +56,6 @@ vcc_ParseImport(struct vcc *tl)
 
 	ExpectErr(tl, ID);
 	mod = tl->t;
-
 	vcc_NextToken(tl);
 
 	osym = VCC_FindSymbol(tl, mod, SYM_NONE);
@@ -83,6 +82,14 @@ vcc_ParseImport(struct vcc *tl)
 	sym->def_e = tl->t;
 
 	if (tl->t->tok == ID) {
+		if (!tl->unsafe_path) {
+			VSB_printf(tl->sb,
+			    "'import ... from path...'"
+			    " not allowed.\nAt:");
+			vcc_ErrToken(tl, tl->t);
+			vcc_ErrWhere(tl, tl->t);
+			return;
+		}
 		if (!vcc_IdIs(tl->t, "from")) {
 			VSB_printf(tl->sb, "Expected 'from path...' at ");
 			vcc_ErrToken(tl, tl->t);



From phk at varnish-cache.org  Sun Apr 29 18:22:28 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 29 Apr 2012 20:22:28 +0200
Subject: [master] 1d588b3 Silence an annoying message when running varnishd -C
Message-ID: <E1SOYlQ-0004Eu-E6@project.varnish-software.com>

commit 1d588b3f35772d55b4f7ea7ee471ccd3a018d48b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Apr 29 18:22:08 2012 +0000

    Silence an annoying message when running varnishd -C

diff --git a/bin/varnishtest/tests/a00009.vtc b/bin/varnishtest/tests/a00009.vtc
index 90569cc..4d35f85 100644
--- a/bin/varnishtest/tests/a00009.vtc
+++ b/bin/varnishtest/tests/a00009.vtc
@@ -1,3 +1,3 @@
 varnishtest "See that the VCL compiler works"
 
-shell "cd ${topbuild}/bin/varnishd && ./varnishd -b 127.0.0.1:80 -C -n ${tmpdir} > /dev/null"
+shell "cd ${topbuild}/bin/varnishd && ./varnishd -b 127.0.0.1:80 -C -n ${tmpdir} > /dev/null 2>&1"



From phk at varnish-cache.org  Mon Apr 30 06:31:31 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 30 Apr 2012 08:31:31 +0200
Subject: [master] ee1f86d Writing an article about the utility of asserts,
	I noticed some inconsistent whitespace with asserts.
Message-ID: <E1SOk8x-0005LW-M5@project.varnish-software.com>

commit ee1f86d208ae2c3e4e72d956419bae1b4a20dc3c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 30 06:30:57 2012 +0000

    Writing an article about the utility of asserts, I noticed some inconsistent
    whitespace with asserts.

diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index 85ad69e..ab405f9 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -500,7 +500,7 @@ VBP_Insert(struct backend *b, const struct vrt_backend_probe *p,
 	}
 
 	VTAILQ_FOREACH(vcl, &vt->vcls, list)
-		assert (vcl->probep != p);
+		assert(vcl->probep != p);
 
 	vcl = vbp_new_vcl(p, hosthdr);
 	Lck_Lock(&vbp_mtx);
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 6e6eafb..c53d945 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -525,7 +525,7 @@ BAN_Reload(const uint8_t *ban, unsigned len)
 
 	VTAILQ_FOREACH(b, &ban_head, list) {
 		t1 = ban_time(b->spec);
-		assert (t1 < t2);
+		assert(t1 < t2);
 		t2 = t1;
 		if (t1 == t0) {
 			Lck_Unlock(&ban_mtx);
diff --git a/bin/varnishd/cache/cache_dir_round_robin.c b/bin/varnishd/cache/cache_dir_round_robin.c
index 7d75473..350f18c 100644
--- a/bin/varnishd/cache/cache_dir_round_robin.c
+++ b/bin/varnishd/cache/cache_dir_round_robin.c
@@ -152,7 +152,7 @@ vrt_init_dir(struct cli *cli, struct director **bp, int idx,
 	te = t->members;
 	for (i = 0; i < t->nmember; i++, vh++, te++) {
 		vh->backend = bp[te->host];
-		AN (vh->backend);
+		AN(vh->backend);
 	}
 	vs->nhosts = t->nmember;
 	vs->next_host = 0;
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 0e8b11a..73aac72 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -326,7 +326,7 @@ ESI_Deliver(struct sess *sp)
 						p = e;
 						break;
 					}
-					assert (i == VGZ_OK || i == VGZ_END);
+					assert(i == VGZ_OK || i == VGZ_END);
 				} else {
 					/*
 					 * Ungzip'ed VEC, ungzip'ed ESI response
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 39ded77..5a41875 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -424,7 +424,7 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 	Lck_Lock(&exp_mtx);
 	VTAILQ_FOREACH(oc, &lru->lru_head, lru_list) {
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-		assert (oc->timer_idx != BINHEAP_NOIDX);
+		assert(oc->timer_idx != BINHEAP_NOIDX);
 		/*
 		 * It wont release any space if we cannot release the last
 		 * reference, besides, if somebody else has a reference,
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index e019702..ee9147d 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -666,7 +666,7 @@ FetchBody(struct worker *wrk, void *priv)
 				uu += st->len;
 			if (bo->do_stream)
 				/* Streaming might have started freeing stuff */
-				assert (uu <= obj->len);
+				assert(uu <= obj->len);
 
 			else
 				assert(uu == obj->len);
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index bcdfa6d..f977400 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -224,7 +224,7 @@ wslr(struct vsl_log *vsl, enum VSL_tag_e tag, int id, txt t)
 	/* Wrap if necessary */
 	if (VSL_END(vsl->wlp, l) >= vsl->wle)
 		VSL_Flush(vsl, 1);
-	assert (VSL_END(vsl->wlp, l) < vsl->wle);
+	assert(VSL_END(vsl->wlp, l) < vsl->wle);
 	memcpy(VSL_DATA(vsl->wlp), t.b, l);
 	vsl_hdr(tag, vsl->wlp, l, id);
 	vsl->wlp = VSL_END(vsl->wlp, l);
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index d206a86..6118fbf 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -216,7 +216,7 @@ smp_open_segs(struct smp_sc *sc, struct smp_signctx *ctx)
 		}
 	}
 
-	assert (l >= sc->free_reserve);
+	assert(l >= sc->free_reserve);
 
 
 	sg1 = NULL;
diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 8bb19d4..2c6dd3f 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -268,7 +268,7 @@ VTCP_close(int *s)
 
 	i = close(*s);
 
-	assert (VTCP_Check(i));
+	assert(VTCP_Check(i));
 	*s = -1;
 }
 
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index 044d12d..b4713e6 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -352,7 +352,7 @@ vcc_ParseAction(struct vcc *tl)
 	const struct symbol *sym;
 
 	at = tl->t;
-	assert (at->tok == ID);
+	assert(at->tok == ID);
 	for(atp = action_table; atp->name != NULL; atp++) {
 		if (vcc_IdIs(at, atp->name)) {
 			if (atp->bitmask != 0)
diff --git a/lib/libvcl/vcc_symb.c b/lib/libvcl/vcc_symb.c
index 4066e56..691f6b8 100644
--- a/lib/libvcl/vcc_symb.c
+++ b/lib/libvcl/vcc_symb.c
@@ -118,7 +118,7 @@ VCC_FindSymbol(struct vcc *tl, const struct token *t, enum symkind kind)
 		if (sym->kind == SYM_WILDCARD &&
 		   (t->e - t->b > sym->nlen) &&
 		   !memcmp(sym->name, t->b, sym->nlen)) {
-			AN (sym->wildcard);
+			AN(sym->wildcard);
 			return (sym->wildcard(tl, t, sym));
 		}
 		if (kind != SYM_NONE && kind != sym->kind)



From phk at varnish-cache.org  Mon Apr 30 07:35:46 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 30 Apr 2012 09:35:46 +0200
Subject: [master] f222778 Eliminate the prepfetch state
Message-ID: <E1SOl98-0007Ou-Gw@project.varnish-software.com>

commit f2227781d49de76a56955e69c13dde699dbbf4e0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 30 07:35:27 2012 +0000

    Eliminate the prepfetch state

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 4a04f5b..f963bbd 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -550,8 +550,8 @@ DOT		shape=record
 DOT		label="{cnt_fetch:|fetch hdr\nfrom backend|(find obj.ttl)|{vcl_fetch\{\}|{req.|bereq.|beresp.}}|{<err>error?|<rst>restart?}}"
 DOT	]
 DOT }
-DOT fetch -> prepfetch [style=bold,color=red]
-DOT fetch -> prepfetch [style=bold,color=blue]
+DOT fetch -> fetchbody [style=bold,color=red]
+DOT fetch -> fetchbody [style=bold,color=blue]
  */
 
 static int
@@ -630,7 +630,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 
 		switch (req->handling) {
 		case VCL_RET_DELIVER:
-			sp->step = STP_PREPFETCH;
+			sp->step = STP_FETCHBODY;
 			return (0);
 		default:
 			break;
@@ -670,18 +670,17 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
  * Prepare to fetch body from backend
  *
 DOT subgraph xcluster_body {
-DOT	prepfetch [
+DOT	fetchbody [
 DOT		shape=record
-DOT		label="{cnt_prepfetch:|error?|<out>stream ?}"
+DOT		label="{cnt_fetchbody:|start fetch_thread}"
 DOT	]
 DOT }
-DOT prepfetch:out -> fetchbody [style=bold,color=red]
-DOT prepfetch:out -> fetchbody [style=bold,color=blue]
-DOT prepfetch:out -> prepresp [label=yes,style=bold,color=cyan]
+DOT fetchbody:out -> prepresp [style=bold,color=red]
+DOT fetchbody:out -> prepresp [style=bold,color=blue]
  */
 
 static int
-cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req)
+cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 {
 	struct http *hp, *hp2;
 	char *b;
@@ -868,34 +867,9 @@ cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req)
 	    RFC2616_Do_Cond(sp))
 		bo->do_stream = 0;
 
-	sp->step = STP_FETCHBODY;
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Actually fetch body from backend
- *
-DOT subgraph xcluster_fetchbody {
-DOT	fetchbody [
-DOT		shape=record
-DOT		label="{cnt_fetchbody:|error ?|<out>success ?}"
-DOT	]
-DOT }
-DOT fetchbody:out -> prepresp [style=bold,color=red]
-DOT fetchbody:out -> prepresp [style=bold,color=blue]
- */
-
-static int
-cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
-{
-	struct busyobj *bo;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	bo = req->busyobj;
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-
+	/*
+	 * Ready to fetch the body
+	 */
 	bo->fetch_task.func = FetchBody;
 	bo->fetch_task.priv = bo;
 
@@ -1707,5 +1681,3 @@ CNT_Init(void)
 	xids = random();
 	CLI_AddFuncs(debug_cmds);
 }
-
-
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index ee70874..15fd811 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -40,7 +40,6 @@ STEP(lookup,		LOOKUP,		(sp, sp->wrk, sp->req))
 STEP(miss,		MISS,		(sp, sp->wrk, sp->req))
 STEP(hit,		HIT,		(sp, sp->wrk, sp->req))
 STEP(fetch,		FETCH,		(sp, sp->wrk, sp->req))
-STEP(prepfetch,		PREPFETCH,	(sp, sp->wrk, sp->req))
 STEP(fetchbody,		FETCHBODY,	(sp, sp->wrk, sp->req))
 STEP(prepresp,		PREPRESP,	(sp, sp->wrk, sp->req))
 STEP(deliver,		DELIVER,	(sp, sp->wrk, sp->req))



From perbu at varnish-cache.org  Mon Apr 30 07:49:03 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Mon, 30 Apr 2012 09:49:03 +0200
Subject: [master] 19a9743 -r option for read only parameters
Message-ID: <E1SOlLz-00008Y-HF@project.varnish-software.com>

commit 19a97432827c29a6fdc63101494ca72b109c8df2
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Apr 30 09:48:56 2012 +0200

    -r option for read only parameters

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index f0647b3..b75bbb0 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -23,7 +23,7 @@ varnishd [-a address[:port]] [-b host[:port]] [-d] [-F] [-f config]
 	 [-g group] [-h type[,options]] [-i identity]
 	 [-l shmlogsize] [-n name] [-P file] [-p param=value] 
 	 [-s type[,options]] [-T address[:port]] [-t ttl]
-	 [-u user] [-V] 
+	 [-r param[,param...]] [-u user] [-V] 
 
 DESCRIPTION
 ===========
@@ -110,6 +110,13 @@ OPTIONS
             documents.  This is a shortcut for specifying the
             default_ttl run-time parameter.
 
+-r param[,param...]  
+            Specifies a list of parameters that are read only. In a
+            very secure environment you want to consider setting
+            parameters such as *user*, *group*, *cc_command*,
+            *vcc_allow_inline_c* to read only as these can potentially
+            be used to escalate privileges.
+
 -u user     Specifies the name of an unprivileged user to which the child
             process should switch before it starts accepting
             connections.  This is a shortcut for specifying the user



From perbu at varnish-cache.org  Mon Apr 30 08:05:39 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Mon, 30 Apr 2012 10:05:39 +0200
Subject: [master] 170d084 explain the reason behind -r
Message-ID: <E1SOlc3-0003l1-Pg@project.varnish-software.com>

commit 170d0842e5218393e9fa2e5c3025e2aedabd16e0
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Apr 30 10:05:34 2012 +0200

    explain the reason behind -r

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index b75bbb0..805e012 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -111,11 +111,13 @@ OPTIONS
             default_ttl run-time parameter.
 
 -r param[,param...]  
-            Specifies a list of parameters that are read only. In a
-            very secure environment you want to consider setting
-            parameters such as *user*, *group*, *cc_command*,
-            *vcc_allow_inline_c* to read only as these can potentially
-            be used to escalate privileges.
+            Specifies a list of parameters that are read only. This
+            gives the system administrator a way to limit what someone
+            with access to the Varnish CLI can do. In a very secure
+            environment you want to consider setting parameters such
+            as *user*, *group*, *cc_command*, *vcc_allow_inline_c* to
+            read only as these can potentially be used to escalate
+            privileges.
 
 -u user     Specifies the name of an unprivileged user to which the child
             process should switch before it starts accepting



From tfheen at varnish-cache.org  Mon Apr 30 13:00:56 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 30 Apr 2012 15:00:56 +0200
Subject: [master] 816c1c3 Enable PCRE JIT-compiled regular expressions
Message-ID: <E1SOqDo-0004U9-9c@project.varnish-software.com>

commit 816c1c34e9afacbb439c24490284573dd2fbc065
Author: Dan McGee <dan at archlinux.org>
Date:   Fri Dec 30 15:14:56 2011 -0600

    Enable PCRE JIT-compiled regular expressions
    
    Implemented more or less as described in the pcrejit(3) manpage, and
    adds some compatibility defines for use with older pre-8.20 libraries
    that do not have this functionality.
    
    Fixes: #1080

diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index fb2dcda..ee462c7 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -40,9 +40,14 @@
 struct vre {
 	unsigned		magic;
 #define VRE_MAGIC		0xe83097dc
-	pcre *re;
+	pcre			*re;
+	pcre_extra		*re_extra;
 };
 
+#ifndef PCRE_STUDY_JIT_COMPILE
+#define PCRE_STUDY_JIT_COMPILE 0
+#endif
+
 /*
  * We don't want to spread or even expose the majority of PCRE options
  * so we establish our own options and implement hard linkage to PCRE
@@ -66,6 +71,20 @@ VRE_compile(const char *pattern, int options,
 		VRE_free(&v);
 		return (NULL);
 	}
+	v->re_extra = pcre_study(v->re, PCRE_STUDY_JIT_COMPILE, errptr);
+	if (v->re_extra == NULL) {
+		if (*errptr != NULL) {
+			VRE_free(&v);
+			return (NULL);
+		}
+		/* allocate our own, pcre_study can return NULL without it
+		 * being an error */
+		v->re_extra = calloc(1, sizeof(pcre_extra));
+		if (v->re_extra == NULL) {
+			VRE_free(&v);
+			return (NULL);
+		}
+	}
 	return (v);
 }
 
@@ -76,22 +95,23 @@ VRE_exec(const vre_t *code, const char *subject, int length,
 {
 	CHECK_OBJ_NOTNULL(code, VRE_MAGIC);
 	int ov[30];
-	pcre_extra extra;
 
 	if (ovector == NULL) {
 		ovector = ov;
 		ovecsize = sizeof(ov)/sizeof(ov[0]);
 	}
 
-	memset(&extra, 0, sizeof extra);
 	if (lim != NULL) {
-		extra.match_limit = lim->match;
-		extra.flags |= PCRE_EXTRA_MATCH_LIMIT;
-		extra.match_limit_recursion = lim->match_recursion;
-		extra.flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION;
+		code->re_extra->match_limit = lim->match;
+		code->re_extra->flags |= PCRE_EXTRA_MATCH_LIMIT;
+		code->re_extra->match_limit_recursion = lim->match_recursion;
+		code->re_extra->flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION;
+	} else {
+		code->re_extra->flags &= ~PCRE_EXTRA_MATCH_LIMIT;
+		code->re_extra->flags &= ~PCRE_EXTRA_MATCH_LIMIT_RECURSION;
 	}
 
-	return (pcre_exec(code->re, &extra, subject, length,
+	return (pcre_exec(code->re, code->re_extra, subject, length,
 	    startoffset, options, ovector, ovecsize));
 }
 
@@ -102,6 +122,11 @@ VRE_free(vre_t **vv)
 
 	*vv = NULL;
 	CHECK_OBJ(v, VRE_MAGIC);
+#ifdef PCRE_CONFIG_JIT
+	pcre_free_study(v->re_extra);
+#else
+	free(v->re_extra);
+#endif
 	pcre_free(v->re);
 	FREE_OBJ(v);
 }