From lkarsten at varnish-software.com  Fri May  2 09:20:58 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 02 May 2014 11:20:58 +0200
Subject: [master] 292e70e RPM: find-provides should not call find-requires.
Message-ID: <E1Wg9eM-0002tK-2x@project.varnish-software.com>


commit 292e70eb6f5bc91299419b63f25b8714ac119f60
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri May 2 11:17:23 2014 +0200

    RPM: find-provides should not call find-requires.
    
    Also remove the libvarnishapi "workaround" we put in place
    earlier.
    
    Found by: Ingvar Hagelund
    
    Fixes: #1484

diff --git a/redhat/find-provides b/redhat/find-provides
index bf3ed59..8c7dce5 100755
--- a/redhat/find-provides
+++ b/redhat/find-provides
@@ -4,8 +4,10 @@
 
 set -x
 
-if [ -x /usr/lib/rpm/find-requires ]; then
-    /usr/lib/rpm/find-requires "$@"
+if [ -x /usr/lib/rpm/redhat/find-provides ]; then
+    /usr/lib/rpm/redhat/find-provides "$@"
+elif [ -x /usr/lib/rpm/find-provides ]; then
+    /usr/lib/rpm/find-provides "$@"
 fi
 
 cd $(dirname $0)/..
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index c1f02cf..09752b8 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -51,11 +51,6 @@ available on the following web site: http://www.varnish-cache.org/
 Summary: Libraries for %{name}
 Group: System Environment/Libraries
 BuildRequires: ncurses-devel
-Provides: libvarnishapi.so.1
-Provides: libvarnishapi.so.1(LIBVARNISHAPI_1.0)(64bit)
-Provides: libvarnishapi.so.1(LIBVARNISHAPI_1.1)(64bit)
-Provides: libvarnishapi.so.1(LIBVARNISHAPI_1.2)(64bit)
-Provides: libvarnishapi.so.1(LIBVARNISHAPI_1.3)(64bit)
 #Obsoletes: libvarnish1
 
 %description libs



From phk at FreeBSD.org  Mon May  5 07:02:12 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 05 May 2014 09:02:12 +0200
Subject: [master] 278d90f Update the documentation for obj.hits to reflect
 code reality.
Message-ID: <E1WhCui-0003QK-Am@project.varnish-software.com>


commit 278d90fdc0008a55405550482ccf9bc4d74f18dc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 5 07:00:56 2014 +0000

    Update the documentation for obj.hits to reflect code reality.
    
    In 4.x we changed obj.hits to be read-only, and to count all variants
    under the same hash-key (= same objhdr).
    
    Since we cannot trivially tell if a ban hits all Vary: values or
    just some of them, there is no sane heuristic to fiddle obj.hits
    on bans that would Do The Right Thing.
    
    Fixes #1492

diff --git a/bin/varnishtest/tests/v00039.vtc b/bin/varnishtest/tests/v00039.vtc
new file mode 100644
index 0000000..90bc865
--- /dev/null
+++ b/bin/varnishtest/tests/v00039.vtc
@@ -0,0 +1,58 @@
+varnishtest "obj.hits vs Vary"
+
+server s1 {
+	rxreq
+	txresp -hdr "Vary: bar" -body "foobar"
+	rxreq
+	txresp -hdr "Vary: bar" -body "barf"
+} -start
+
+varnish v1 \
+	-arg "-p ban_lurker_sleep=0.01" \
+	-arg "-p ban_lurker_age=0.01" \
+	-vcl+backend {
+		sub vcl_deliver {
+			set resp.http.hits = obj.hits;
+		}
+	} -start
+
+client c1 {
+	# This is a miss -> hits == 0
+	txreq -url "/" -hdr "Bar: 1"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+	expect resp.http.hits == 0
+
+	# This is a hit -> hits == 1
+	txreq -url "/" -hdr "Bar: 1"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+	expect resp.http.hits == 1
+
+	# This is a miss on different vary -> hits still == 1
+	txreq -url "/" -hdr "Bar: 2"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 4
+	expect resp.http.hits == 1
+
+	# This is a hit -> hits == 2
+	txreq -url "/" -hdr "Bar: 2"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 4
+	expect resp.http.hits == 2
+
+} -run
+
+# Ban everything on this hash-key
+varnish v1 -cliok "ban obj.http.vary ~ ."
+delay 1
+
+# And run the entire test again to see that obj.hits got reset.
+
+server s1 -start
+
+client c1 -run
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 57f22ca..1e523f1 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -511,9 +511,12 @@ sp_variables = [
 		'INT',
 		( 'hit', 'deliver',),
 		( ), """
-		The approximate number of times the object has been
-		delivered. A value of 0 indicates a cache miss.
-		This variable is also available in vcl_deliver.
+		The count of cache-hits on this hash-key since it was
+		last instantiated.  This counts cache-hits across all
+		Vary:-ants on this hash-key.
+		The counter will only be reset to zero if/when all objects
+		with this hash-key have disappeared from cache.
+		NB: obj.hits == 0 does *not* indicate a cache miss.
 		"""
 	),
 	('obj.http.',



From perbu at varnish-software.com  Mon May  5 07:15:06 2014
From: perbu at varnish-software.com (Per Buer)
Date: Mon, 05 May 2014 09:15:06 +0200
Subject: [master] 697a1c3 Import std before use
Message-ID: <E1WhD7C-0004u9-FQ@project.varnish-software.com>


commit 697a1c373b7f4a294a857fb6e9d773ae5de0e4c2
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon May 5 09:15:04 2014 +0200

    Import std before use

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index 788764a..dbd8a8a 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -127,6 +127,8 @@ is reserved for builtin subs.
 req.backend.healthy replaced by std.healthy(req.backend)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+Remeber to import the std module if you're not doing so already.
+
 client.port, and server.port replaced by respectively std.port(client.ip) and std.port(server.ip)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 



From phk at FreeBSD.org  Mon May  5 08:20:48 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 05 May 2014 10:20:48 +0200
Subject: [master] bc2a411 Check pthread_join() with an AZ()
Message-ID: <E1WhE8m-0004xC-B0@project.varnish-software.com>


commit bc2a4111b3e464b4c2b61a6eb7b446b6408d8f8d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 5 08:19:20 2014 +0000

    Check pthread_join() with an AZ()

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index e8eb1ff..1656f0e 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -463,7 +463,7 @@ main(int argc, char **argv)
 	VUT.dispatch_priv = NULL;
 	VUT_Main();
 	end_of_file = 1;
-	pthread_join(thr, NULL);
+	AZ(pthread_join(thr, NULL));
 	VUT_Fini();
 	exit(0);
 }



From phk at FreeBSD.org  Mon May  5 08:24:08 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 05 May 2014 10:24:08 +0200
Subject: [master] 6eaf812 Type error in sizeof.
Message-ID: <E1WhEC0-0005Gg-Vq@project.varnish-software.com>


commit 6eaf812b4f573cdb2b07d751a1e219ac04fe4515
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 5 08:23:54 2014 +0000

    Type error in sizeof.
    
    Spotted by:	Coverity

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 1656f0e..7a7693e 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -442,8 +442,8 @@ main(int argc, char **argv)
 
 	hist_range = hist_high - hist_low;
 	hist_buckets = hist_range * HIST_RES;
-	bucket_hit = calloc(sizeof bucket_hit, hist_buckets);
-	bucket_miss = calloc(sizeof bucket_miss, hist_buckets);
+	bucket_hit = calloc(sizeof *bucket_hit, hist_buckets);
+	bucket_miss = calloc(sizeof *bucket_miss, hist_buckets);
 
 	format = malloc(4 * fnum);
 	for (i = 0; i < fnum-1; i++) {



From phk at FreeBSD.org  Mon May  5 08:26:21 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 05 May 2014 10:26:21 +0200
Subject: [master] 7c8451d Check VUT_Arg() return with AN for consistency.
Message-ID: <E1WhEE9-0005dl-LN@project.varnish-software.com>


commit 7c8451dd3f4a70a64d4933399bf355a4af1ea160
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 5 08:26:01 2014 +0000

    Check VUT_Arg() return with AN for consistency.

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 761c3f3..80b19f2 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -329,7 +329,7 @@ main(int argc, char **argv)
 	while ((o = getopt(argc, argv, vopt_optstring)) != -1) {
 		switch (o) {
 		case '1':
-			VUT_Arg('d', NULL);
+			AN(VUT_Arg('d', NULL));
 			once = 1;
 			break;
 		case 'f':



From phk at FreeBSD.org  Mon May  5 08:30:00 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 05 May 2014 10:30:00 +0200
Subject: [master] 8c36361 Insignificant memory leak.
Message-ID: <E1WhEHg-00063d-EP@project.varnish-software.com>


commit 8c363610f79f0460a55e2b7f1017921574f30450
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 5 08:29:42 2014 +0000

    Insignificant memory leak.
    
    Spotted by:	Coverity

diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c
index 2c1d8d6..85b3d67 100644
--- a/lib/libvarnish/vss.c
+++ b/lib/libvarnish/vss.c
@@ -149,9 +149,12 @@ VSS_resolve(const char *addr, const char *port, struct vss_addr ***vap)
 	if (adp == NULL)
 		ret = getaddrinfo(addr, port, &hints, &res0);
 	else {
-		ptst = strtol(adp,NULL,10);
-		if (ptst < 0 || ptst > 65535)
+		ptst = strtol(adp, NULL, 10);
+		if (ptst < 0 || ptst > 65535) {
+			free(hop);
+			free(adp);
 			return(0);
+		}
 		ret = getaddrinfo(hop, adp, &hints, &res0);
 	}
 



From martin at varnish-software.com  Mon May  5 08:42:25 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 05 May 2014 10:42:25 +0200
Subject: [master] 5c12ec5 Remember to signal the thread to exit when
 decimating the flock
Message-ID: <E1WhETh-0007ML-V5@project.varnish-software.com>


commit 5c12ec5f2568b43fc1f4cc68d76e9c9889462595
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Apr 30 13:26:11 2014 +0200

    Remember to signal the thread to exit when decimating the flock
    
    The pool herder didn't signal the thread when decimating the flock,
    causing the thread to be leaked.
    
    Spotted by: xcir
    Fixes: #1490
    
    Also close a couple of other races:
    
    When pthread_cond_timedwait returns ETIMEDOUT, the predicate
    (wrk->task.func == NULL) could still have changed while reacquiring
    the mutex. If so, the signal would've been lost and the thread
    leaked. Changed the idle wait loop in Pool_Work_Thread() to always
    check the predicate before resuming the cond_wait.
    
    The update of the predicate (wrk->task.func) from e.g. Pool_Task() is
    done without holding the mutex. This opens a race on the predicate,
    and the possibility of the worker going back on cond_wait loosing the
    signal. Changed to update the predicate while holding the mutex.

diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 75fac47..ce8d526 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -163,12 +163,12 @@ pool_accept(struct worker *wrk, void *arg)
 		}
 		VTAILQ_REMOVE(&pp->idle_queue, &wrk2->task, list);
 		AZ(wrk2->task.func);
-		Lck_Unlock(&pp->mtx);
 		assert(sizeof *wa2 == WS_Reserve(wrk2->aws, sizeof *wa2));
 		wa2 = (void*)wrk2->aws->f;
 		memcpy(wa2, wa, sizeof *wa);
 		wrk2->task.func = SES_pool_accept_task;
 		wrk2->task.priv = pp->sesspool;
+		Lck_Unlock(&pp->mtx);
 		AZ(pthread_cond_signal(&wrk2->cond));
 
 		/*
@@ -204,9 +204,9 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 	if (wrk != NULL) {
 		VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list);
 		AZ(wrk->task.func);
-		Lck_Unlock(&pp->mtx);
 		wrk->task.func = task->func;
 		wrk->task.priv = task->priv;
+		Lck_Unlock(&pp->mtx);
 		AZ(pthread_cond_signal(&wrk->cond));
 		return (0);
 	}
@@ -237,6 +237,17 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 }
 
 /*--------------------------------------------------------------------
+ * Empty function used as a pointer value for the thread exit condition.
+ */
+
+static void
+pool_kiss_of_death(struct worker *wrk, void *priv)
+{
+	(void)wrk;
+	(void)priv;
+}
+
+/*--------------------------------------------------------------------
  * This is the work function for worker threads in the pool.
  */
 
@@ -274,7 +285,6 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 				wrk->lastused = VTIM_real();
 			wrk->task.func = NULL;
 			wrk->task.priv = wrk;
-			AZ(wrk->task.func);
 			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
 			if (!stats_clean)
 				WRK_SumStat(wrk);
@@ -283,12 +293,12 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 				    wrk->vcl == NULL ?  0 : wrk->lastused+60.);
 				if (i == ETIMEDOUT)
 					VCL_Rel(&wrk->vcl);
-			} while (i);
+			} while (wrk->task.func == NULL);
 			tp = &wrk->task;
 		}
 		Lck_Unlock(&pp->mtx);
 
-		if (tp->func == NULL)
+		if (tp->func == pool_kiss_of_death)
 			break;
 
 		assert(wrk->pool == pp);
@@ -387,23 +397,23 @@ pool_herder(void *priv)
 				CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC);
 
 				if (wrk->lastused < t_idle ||
-				    pp->nthr > cache_param->wthread_max)
+				    pp->nthr > cache_param->wthread_max) {
+					/* Give it a kiss on the cheek... */
 					VTAILQ_REMOVE(&pp->idle_queue,
 					    &wrk->task, list);
-				else
+					wrk->task.func = pool_kiss_of_death;
+					AZ(pthread_cond_signal(&wrk->cond));
+				} else
 					wrk = NULL;
 			}
 			Lck_Unlock(&pp->mtx);
 
-			/* And give it a kiss on the cheek... */
 			if (wrk != NULL) {
 				pp->nthr--;
 				Lck_Lock(&pool_mtx);
 				VSC_C_main->threads--;
 				VSC_C_main->threads_destroyed++;
 				Lck_Unlock(&pool_mtx);
-				wrk->task.func = NULL;
-				wrk->task.priv = NULL;
 				VTIM_sleep(cache_param->wthread_destroy_delay);
 				continue;
 			}
diff --git a/bin/varnishtest/tests/r01490.vtc b/bin/varnishtest/tests/r01490.vtc
new file mode 100644
index 0000000..82ffdb4
--- /dev/null
+++ b/bin/varnishtest/tests/r01490.vtc
@@ -0,0 +1,38 @@
+varnishtest "#1490 - thread destruction"
+
+server s1 {
+} -start
+
+varnish v1 \
+	-arg "-p debug=+syncvsl" \
+	-arg "-p vsl_mask=+WorkThread" \
+	-arg "-p thread_pool_min=2" \
+	-arg "-p thread_pool_max=3" \
+	-arg "-p thread_pools=1" \
+	-vcl+backend {}
+varnish v1 -start
+
+varnish v1 -expect threads == 2
+
+logexpect l1 -v v1 -g raw {
+	expect * 0 WorkThread {^\S+ start$}
+	expect * 0 WorkThread {^\S+ end$}
+} -start
+
+varnish v1 -cliok "param.set thread_pool_min 3"
+
+# Herder thread sleeps 5 seconds. Have to wait longer than that.
+delay 6
+
+varnish v1 -expect threads == 3
+
+varnish v1 -cliok "param.set thread_pool_min 2"
+varnish v1 -cliok "param.set thread_pool_max 2"
+
+# Herder thread sleeps 5 seconds. Have to wait longer than that.
+delay 6
+
+varnish v1 -expect threads == 2
+
+# Use logexpect to see that the thread actually exited
+logexpect l1 -wait



From nils.goroll at uplex.de  Mon May  5 10:51:18 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 05 May 2014 12:51:18 +0200
Subject: [master] 6d71c1f set RST2MAN=no if no rst2man found to make it a hard
 dependency for real
Message-ID: <E1WhGUQ-0003Gi-5U@project.varnish-software.com>


commit 6d71c1f96e6ab2fe682cc5c7779b56d6e9f0b1da
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 5 12:50:24 2014 +0200

    set RST2MAN=no if no rst2man found to make it a hard dependency for real

diff --git a/configure.ac b/configure.ac
index 82bd906..36cc542 100644
--- a/configure.ac
+++ b/configure.ac
@@ -54,8 +54,8 @@ AC_PROG_MAKE_SET
 AC_ARG_WITH([rst2man],
   AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]),
   [RST2MAN="$withval"],
-  AC_CHECK_PROGS(RST2MAN, [rst2man rst2man.py], []))
-if test "$RST2MAN" = "no"; then
+  AC_CHECK_PROGS(RST2MAN, [rst2man rst2man.py], [no]))
+if test "x$RST2MAN" = "xno"; then
   AC_MSG_ERROR(
     [rst2man is needed to build Varnish, please install python-docutils.])
 fi



From daghf at varnish-software.com  Mon May  5 11:34:27 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Mon, 05 May 2014 13:34:27 +0200
Subject: [master] b2a6c79 Fix tiny memory leak in vdir.c
Message-ID: <E1WhHAB-00084H-IC@project.varnish-software.com>


commit b2a6c79f751e0899fff96c80e30ecb0bd9b164e8
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon May 5 13:34:09 2014 +0200

    Fix tiny memory leak in vdir.c

diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c
index c8a8c84..12268eb 100644
--- a/lib/libvmod_directors/vdir.c
+++ b/lib/libvmod_directors/vdir.c
@@ -88,6 +88,7 @@ vdir_delete(struct vdir **vdp)
 	free(vd->backend);
 	free(vd->weight);
 	AZ(pthread_mutex_destroy(&vd->mtx));
+	free(vd->dir->vcl_name);
 	FREE_OBJ(vd->dir);
 	vbit_destroy(vd->vbm);
 	FREE_OBJ(vd);



From lkarsten at varnish-software.com  Mon May  5 11:49:58 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 05 May 2014 13:49:58 +0200
Subject: [master] 9a2ac5b We accept docfixes on github.
Message-ID: <E1WhHPC-0001O8-Gq@project.varnish-software.com>


commit 9a2ac5bf18be9de80fcf3d980f56c9c29e97a994
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 5 13:48:53 2014 +0200

    We accept docfixes on github.
    
    As discussed on bugwash today. Lowering the bar may help us improve the
    documentation.

diff --git a/CONTRIBUTING b/CONTRIBUTING
index 8199602..94a668f 100644
--- a/CONTRIBUTING
+++ b/CONTRIBUTING
@@ -13,7 +13,6 @@ Official development tree is here:
 
 Patches can be sent to varnish-dev at varnish-cache.org.
 
-There is a READ-ONLY git-mirror on Github.  Please do not send
-contributions there, we need them by email to varnish-dev@ for now.
-
+Documentation fixes and other small items for the documentation team may be
+sent as Github pull requests.
 



From phk at FreeBSD.org  Mon May  5 11:51:29 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 05 May 2014 13:51:29 +0200
Subject: [master] f6ef923 Add a 'y'ear conversion factor to std.duration().
Message-ID: <E1WhHQf-0001lH-Bv@project.varnish-software.com>


commit f6ef923b2155b323f5182eeae2d4423796cd35b9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 5 11:50:39 2014 +0000

    Add a 'y'ear conversion factor to std.duration().
    
    Submitted by:	github::brianjarita

diff --git a/bin/varnishtest/tests/m00005.vtc b/bin/varnishtest/tests/m00005.vtc
index 53f5ae3..5ef9dde 100644
--- a/bin/varnishtest/tests/m00005.vtc
+++ b/bin/varnishtest/tests/m00005.vtc
@@ -97,4 +97,10 @@ client c1 {
 	expect resp.http.ttl == 1000002.000
 	expect resp.bodylen == 1
 
+	txreq -url "/1"  -hdr "ttl: 1y"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.ttl == 32536001.000
+	expect resp.bodylen == 1
+
 } -run
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index b9020d1..f98b5fb 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -86,6 +86,7 @@ vmod_duration(const struct vrt_ctx *ctx, const char *p, VCL_DURATION d)
 	case 'h': r *= 60.*60.; break;
 	case 'd': r *= 60.*60.*24.; break;
 	case 'w': r *= 60.*60.*24.*7.; break;
+	case 'y': r *= 60.*60.*24.*365.; break;
 	default:
 		return (d);
 	}



From lkarsten at varnish-software.com  Mon May  5 12:04:54 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 05 May 2014 14:04:54 +0200
Subject: [master] 18b687e Add travis build script
Message-ID: <E1WhHde-0003Pt-CS@project.varnish-software.com>


commit 18b687e45d986a18241eee92e8413efe872b259f
Author: Joshua Bussdieker <jbussdieker at gmail.com>
Date:   Fri Aug 2 09:13:10 2013 -0700

    Add travis build script

diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..129e43a
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,6 @@
+---
+script: 'make check'
+before_install:
+  - sudo apt-get install python-docutils
+  - ./autogen.sh
+  - ./configure



From nils.goroll at uplex.de  Mon May  5 12:33:46 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 05 May 2014 14:33:46 +0200
Subject: [master] b6c3a8e move curses includes to one file
Message-ID: <E1WhI5a-0001vx-KA@project.varnish-software.com>


commit b6c3a8eb48d61a776bb450fc3929636c8a2c01e5
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 5 14:29:24 2014 +0200

    move curses includes to one file

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 7a7693e..d01595c 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -35,7 +35,6 @@
 
 #include <sys/types.h>
 
-#include <curses.h>
 #include <errno.h>
 #include <limits.h>
 #include <math.h>
@@ -47,6 +46,7 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "vcurses.h"
 #include "vapi/vsl.h"
 #include "vapi/vsm.h"
 #include "vapi/voptget.h"
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 373cf09..fd80580 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -53,20 +53,7 @@
 #include "vtim.h"
 
 #include "varnishstat.h"
-
-#if defined HAVE_NCURSESW_CURSES_H
-#  include <ncursesw/curses.h>
-#elif defined HAVE_NCURSESW_H
-#  include <ncursesw.h>
-#elif defined HAVE_NCURSES_CURSES_H
-#  include <ncurses/curses.h>
-#elif defined HAVE_NCURSES_H
-#  include <ncurses.h>
-#elif defined HAVE_CURSES_H
-#  include <curses.h>
-#else
-#  error "SysV or X/Open-compatible Curses header file required"
-#endif
+#include "vcurses.h"
 
 #define LINES_STATUS		3
 #define LINES_BAR_T		1
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 80b19f2..2c195ef 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -34,7 +34,6 @@
 #include "config.h"
 
 #include <ctype.h>
-#include <curses.h>
 #include <errno.h>
 #include <limits.h>
 #include <pthread.h>
@@ -45,6 +44,7 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "vcurses.h"
 #include "vapi/vsm.h"
 #include "vapi/vsl.h"
 #include "vapi/voptget.h"
diff --git a/include/Makefile.am b/include/Makefile.am
index 8291a50..50fed78 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -50,6 +50,7 @@ nobase_noinst_HEADERS = \
 	vcli_serve.h \
 	vcs_version.h \
 	vct.h \
+	vcurses.h \
 	vend.h \
 	vev.h \
 	vfil.h \
diff --git a/include/vcurses.h b/include/vcurses.h
new file mode 100644
index 0000000..d24b556
--- /dev/null
+++ b/include/vcurses.h
@@ -0,0 +1,41 @@
+/*-
+ * Copyright (c) 2014 Varnish Software AS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * include the right curses headers
+ */
+
+#if defined HAVE_NCURSESW_CURSES_H
+#  include <ncursesw/curses.h>
+#elif defined HAVE_NCURSESW_H
+#  include <ncursesw.h>
+#elif defined HAVE_NCURSES_CURSES_H
+#  include <ncurses/curses.h>
+#elif defined HAVE_NCURSES_H
+#  include <ncurses.h>
+#elif defined HAVE_CURSES_H
+#  include <curses.h>
+#else
+#  error "SysV or X/Open-compatible Curses header file required"
+#endif



From lkarsten at varnish-software.com  Mon May  5 12:49:21 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 05 May 2014 14:49:21 +0200
Subject: [master] 57a9a71 [travis] Run tests in parallell.
Message-ID: <E1WhIKf-0004nA-Fn@project.varnish-software.com>


commit 57a9a7136343b2acea27347180dbc0a23989b132
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 5 14:48:59 2014 +0200

    [travis] Run tests in parallell.

diff --git a/.travis.yml b/.travis.yml
index 129e43a..f55a37b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,5 @@
 ---
-script: 'make check'
+script: 'make -j3 check'
 before_install:
   - sudo apt-get install python-docutils
   - ./autogen.sh



From fgsch at lodoss.net  Mon May  5 15:27:58 2014
From: fgsch at lodoss.net (Federico Schwindt)
Date: Mon, 5 May 2014 16:27:58 +0100
Subject: [master] f6ef923 Add a 'y'ear conversion factor to std.duration().
In-Reply-To: <E1WhHQf-0001lH-Bv@project.varnish-software.com>
References: <E1WhHQf-0001lH-Bv@project.varnish-software.com>
Message-ID: <CAJV_h0aa4X8kfpufSzp8wTvy=4raC_OSZYCfM0UbDgLbJwhqaw@mail.gmail.com>

The doc bit is missing and vcc_TimeUnit() should be updated as well.

f.-


On Mon, May 5, 2014 at 12:51 PM, Poul-Henning Kamp <phk at freebsd.org> wrote:

>
> commit f6ef923b2155b323f5182eeae2d4423796cd35b9
> Author: Poul-Henning Kamp <phk at FreeBSD.org>
> Date:   Mon May 5 11:50:39 2014 +0000
>
>     Add a 'y'ear conversion factor to std.duration().
>
>     Submitted by:       github::brianjarita
>
> diff --git a/bin/varnishtest/tests/m00005.vtc
> b/bin/varnishtest/tests/m00005.vtc
> index 53f5ae3..5ef9dde 100644
> --- a/bin/varnishtest/tests/m00005.vtc
> +++ b/bin/varnishtest/tests/m00005.vtc
> @@ -97,4 +97,10 @@ client c1 {
>         expect resp.http.ttl == 1000002.000
>         expect resp.bodylen == 1
>
> +       txreq -url "/1"  -hdr "ttl: 1y"
> +       rxresp
> +       expect resp.status == 200
> +       expect resp.http.ttl == 32536001.000
> +       expect resp.bodylen == 1
> +
>  } -run
> diff --git a/lib/libvmod_std/vmod_std_conversions.c
> b/lib/libvmod_std/vmod_std_conversions.c
> index b9020d1..f98b5fb 100644
> --- a/lib/libvmod_std/vmod_std_conversions.c
> +++ b/lib/libvmod_std/vmod_std_conversions.c
> @@ -86,6 +86,7 @@ vmod_duration(const struct vrt_ctx *ctx, const char *p,
> VCL_DURATION d)
>         case 'h': r *= 60.*60.; break;
>         case 'd': r *= 60.*60.*24.; break;
>         case 'w': r *= 60.*60.*24.*7.; break;
> +       case 'y': r *= 60.*60.*24.*365.; break;
>         default:
>                 return (d);
>         }
>
> _______________________________________________
> varnish-commit mailing list
> varnish-commit at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-commit/attachments/20140505/e1bfb462/attachment.html>

From nils.goroll at uplex.de  Mon May  5 15:56:09 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 05 May 2014 17:56:09 +0200
Subject: [master] 5321145 these .c files live in $(srcdir), not in $(builddir)
Message-ID: <E1WhLFR-0004NP-Cx@project.varnish-software.com>


commit 5321145882f6edd65bbeae24a7ff23cc6985f57e
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 5 17:55:39 2014 +0200

    these .c files live in $(srcdir), not in $(builddir)

diff --git a/bin/varnishadm/Makefile.am b/bin/varnishadm/Makefile.am
index f3050eb..ba476d6 100644
--- a/bin/varnishadm/Makefile.am
+++ b/bin/varnishadm/Makefile.am
@@ -6,10 +6,10 @@ bin_PROGRAMS = varnishadm
 
 varnishadm_SOURCES = \
 	varnishadm.c \
-	$(top_builddir)/lib/libvarnish/vas.c \
-	$(top_builddir)/lib/libvarnish/vsa.c \
-	$(top_builddir)/lib/libvarnish/vtcp.c \
-	$(top_builddir)/lib/libvarnish/vss.c
+	$(top_srcdir)/lib/libvarnish/vas.c \
+	$(top_srcdir)/lib/libvarnish/vsa.c \
+	$(top_srcdir)/lib/libvarnish/vtcp.c \
+	$(top_srcdir)/lib/libvarnish/vss.c
 
 varnishadm_CFLAGS = @LIBEDIT_CFLAGS@
 
diff --git a/bin/varnishreplay/Makefile.am b/bin/varnishreplay/Makefile.am
index 72a293d..78fed42 100644
--- a/bin/varnishreplay/Makefile.am
+++ b/bin/varnishreplay/Makefile.am
@@ -6,9 +6,9 @@ bin_PROGRAMS = varnishreplay
 
 varnishreplay_SOURCES = \
 	varnishreplay.c \
-	$(top_builddir)/lib/libvarnish/vas.c \
-	$(top_builddir)/lib/libvarnish/vtcp.c \
-	$(top_builddir)/lib/libvarnish/vss.c
+	$(top_srcdir)/lib/libvarnish/vas.c \
+	$(top_srcdir)/lib/libvarnish/vtcp.c \
+	$(top_srcdir)/lib/libvarnish/vss.c
 
 varnishreplay_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
diff --git a/bin/varnishstat/Makefile.am b/bin/varnishstat/Makefile.am
index fe17250..5c55bab 100644
--- a/bin/varnishstat/Makefile.am
+++ b/bin/varnishstat/Makefile.am
@@ -9,9 +9,9 @@ varnishstat_SOURCES = \
 	\
 	varnishstat.c \
 	varnishstat_curses.c \
-	$(top_builddir)/lib/libvarnish/vas.c \
-	$(top_builddir)/lib/libvarnish/version.c \
-	$(top_builddir)/lib/libvarnish/vtim.c
+	$(top_srcdir)/lib/libvarnish/vas.c \
+	$(top_srcdir)/lib/libvarnish/version.c \
+	$(top_srcdir)/lib/libvarnish/vtim.c
 
 varnishstat_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \



From nils.goroll at uplex.de  Mon May  5 18:11:08 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 05 May 2014 20:11:08 +0200
Subject: [master] 98a52d4 also include from builddir
Message-ID: <E1WhNM4-00046k-A1@project.varnish-software.com>


commit 98a52d43782ea1530c03a95eeb30e42e472d982c
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 5 18:17:43 2014 +0200

    also include from builddir

diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index 4e09fb9..5503f86 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -1,4 +1,4 @@
-AM_CPPFLAGS = -I$(top_srcdir)/include @PCRE_CFLAGS@
+AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include @PCRE_CFLAGS@
 
 AM_LDFLAGS  = $(AM_LT_LDFLAGS)
 



From nils.goroll at uplex.de  Mon May  5 18:11:08 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 05 May 2014 20:11:08 +0200
Subject: [master] b1d777d also include from builddir
Message-ID: <E1WhNM4-00046q-BI@project.varnish-software.com>


commit b1d777da5843cab65c7c3abd150a69e8e6439070
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 5 18:18:24 2014 +0200

    also include from builddir

diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index f0a23fa..1ee44dd 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -2,7 +2,7 @@
 
 AM_LDFLAGS  = $(AM_LT_LDFLAGS)
 
-AM_CPPFLAGS = -I$(top_srcdir)/include @PCRE_CFLAGS@
+AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include @PCRE_CFLAGS@
 
 lib_LTLIBRARIES = libvarnishapi.la
 



From nils.goroll at uplex.de  Mon May  5 18:11:08 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 05 May 2014 20:11:08 +0200
Subject: [master] 97023ef streamline AM_CPPFLAGS to include from the builddir
Message-ID: <E1WhNM4-00046z-DW@project.varnish-software.com>


commit 97023ef179a09d5680312d6c46607cde55aac7a1
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 5 18:31:18 2014 +0200

    streamline AM_CPPFLAGS to include from the builddir

diff --git a/bin/varnishadm/Makefile.am b/bin/varnishadm/Makefile.am
index ba476d6..9033e08 100644
--- a/bin/varnishadm/Makefile.am
+++ b/bin/varnishadm/Makefile.am
@@ -1,6 +1,8 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 bin_PROGRAMS = varnishadm
 
diff --git a/bin/varnishhist/Makefile.am b/bin/varnishhist/Makefile.am
index 748721b..5904430 100644
--- a/bin/varnishhist/Makefile.am
+++ b/bin/varnishhist/Makefile.am
@@ -1,6 +1,8 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 bin_PROGRAMS = varnishhist
 
diff --git a/bin/varnishlog/Makefile.am b/bin/varnishlog/Makefile.am
index 1e6fa79..2df9415 100644
--- a/bin/varnishlog/Makefile.am
+++ b/bin/varnishlog/Makefile.am
@@ -1,6 +1,8 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 bin_PROGRAMS = varnishlog
 
diff --git a/bin/varnishncsa/Makefile.am b/bin/varnishncsa/Makefile.am
index 12a3f4d..d63df61 100644
--- a/bin/varnishncsa/Makefile.am
+++ b/bin/varnishncsa/Makefile.am
@@ -1,6 +1,8 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 bin_PROGRAMS = varnishncsa
 
diff --git a/bin/varnishreplay/Makefile.am b/bin/varnishreplay/Makefile.am
index 78fed42..38d1c30 100644
--- a/bin/varnishreplay/Makefile.am
+++ b/bin/varnishreplay/Makefile.am
@@ -1,6 +1,8 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 bin_PROGRAMS = varnishreplay
 
diff --git a/bin/varnishstat/Makefile.am b/bin/varnishstat/Makefile.am
index 5c55bab..664131d 100644
--- a/bin/varnishstat/Makefile.am
+++ b/bin/varnishstat/Makefile.am
@@ -1,6 +1,8 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 bin_PROGRAMS = varnishstat
 
diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am
index 5af8068..fe599d4 100644
--- a/bin/varnishtest/Makefile.am
+++ b/bin/varnishtest/Makefile.am
@@ -15,7 +15,10 @@ check-local:
 
 DISTCLEANFILES = _.ok
 
-AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/lib/libvgz
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include \
+	-I$(top_srcdir)/lib/libvgz
 
 bin_PROGRAMS =	varnishtest
 
diff --git a/bin/varnishtop/Makefile.am b/bin/varnishtop/Makefile.am
index b919dba..f48e972 100644
--- a/bin/varnishtop/Makefile.am
+++ b/bin/varnishtop/Makefile.am
@@ -1,6 +1,8 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 bin_PROGRAMS = varnishtop
 
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index 5503f86..9275fd2 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -1,4 +1,7 @@
-AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include @PCRE_CFLAGS@
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include \
+	@PCRE_CFLAGS@
 
 AM_LDFLAGS  = $(AM_LT_LDFLAGS)
 
diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index 1ee44dd..287472b 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -2,7 +2,10 @@
 
 AM_LDFLAGS  = $(AM_LT_LDFLAGS)
 
-AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include @PCRE_CFLAGS@
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include \
+	@PCRE_CFLAGS@
 
 lib_LTLIBRARIES = libvarnishapi.la
 
diff --git a/lib/libvarnishcompat/Makefile.am b/lib/libvarnishcompat/Makefile.am
index 543fbb2..78683c0 100644
--- a/lib/libvarnishcompat/Makefile.am
+++ b/lib/libvarnishcompat/Makefile.am
@@ -1,6 +1,9 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
+
 AM_LDFLAGS  = $(AM_LT_LDFLAGS)
 
 pkglib_LTLIBRARIES = libvarnishcompat.la
diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am
index b030dc4..c7e183d 100644
--- a/lib/libvcc/Makefile.am
+++ b/lib/libvcc/Makefile.am
@@ -2,7 +2,9 @@
 
 AM_LDFLAGS  = $(AM_LT_LDFLAGS)
 
-AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 pkglib_LTLIBRARIES = libvcc.la
 



From phk at FreeBSD.org  Tue May  6 08:31:31 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 06 May 2014 10:31:31 +0200
Subject: [master] cec1281 Don't close and reopen(+bind) the acceptor socket in
 the server instances whenever we give the server instance more work,
 doing so runs into a dark and murky corner somewhere in Stevens and Solaris
 sometimes doesn't like that.
Message-ID: <E1Whami-0003NJ-02@project.varnish-software.com>


commit cec12811e38df53ea1e9a3307788b7741496c80a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 6 08:27:44 2014 +0000

    Don't close and reopen(+bind) the acceptor socket in the server
    instances whenever we give the server instance more work, doing
    so runs into a dark and murky corner somewhere in Stevens and
    Solaris sometimes doesn't like that.
    
    The trouble is that the initial bind(2) asks the kernel to assign
    a port, but the subsequent insist on reusing that port, and that,
    in some cases, could mean toruble which cannot be allowed.
    
    Add a "-break" instruction to server so that test case c00035 can
    still work.
    
    Spotted by:	Nils Goroll / UPLEX

diff --git a/bin/varnishtest/tests/c00035.vtc b/bin/varnishtest/tests/c00035.vtc
index 83400f1..9463548 100644
--- a/bin/varnishtest/tests/c00035.vtc
+++ b/bin/varnishtest/tests/c00035.vtc
@@ -1,6 +1,6 @@
 varnishtest "Dropping polling of a backend"
 
-server s1 -repeat 1 {
+server s1 -repeat 20 {
 	rxreq
 	txresp
 } -start
@@ -32,7 +32,7 @@ delay 1
 varnish v1 -cliok "vcl.list"
 varnish v1 -cliok "debug.health"
 
-server s1 {
+server s1 -break {
 	rxreq
 	expect req.url == /foo
 	txresp -bodylen 4
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 86cf293..396a869 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -187,6 +187,23 @@ server_start(struct server *s)
 }
 
 /**********************************************************************
+ * Force stop the server thread
+ */
+
+static void
+server_break(struct server *s)
+{
+	void *res;
+
+	CHECK_OBJ_NOTNULL(s, SERVER_MAGIC);
+	vtc_log(s->vl, 2, "Breaking for server");
+	(void)pthread_cancel(s->tp);
+	AZ(pthread_join(s->tp, &res));
+	s->tp = 0;
+	s->run = 0;
+}
+
+/**********************************************************************
  * Wait for server thread to stop
  */
 
@@ -202,8 +219,6 @@ server_wait(struct server *s)
 		vtc_log(s->vl, 0, "Server returned \"%p\"",
 		    (char *)res);
 	s->tp = 0;
-	VTCP_close(&s->sock);
-	s->sock = -1;
 	s->run = 0;
 }
 
@@ -246,6 +261,10 @@ cmd_server(CMD_ARGS)
 				(void)pthread_cancel(s->tp);
 				server_wait(s);
 			}
+			if (s->sock >= 0) {
+				VTCP_close(&s->sock);
+				s->sock = -1;
+			}
 			server_delete(s);
 		}
 		return;
@@ -271,6 +290,12 @@ cmd_server(CMD_ARGS)
 			server_wait(s);
 			continue;
 		}
+
+		if (!strcmp(*av, "-break")) {
+			server_break(s);
+			continue;
+		}
+
 		/*
 		 * We do an implict -wait if people muck about with a
 		 * running server.
@@ -285,6 +310,10 @@ cmd_server(CMD_ARGS)
 			continue;
 		}
 		if (!strcmp(*av, "-listen")) {
+			if (s->sock >= 0) {
+				VTCP_close(&s->sock);
+				s->sock = -1;
+			}
 			bprintf(s->listen, "%s", av[1]);
 			AZ(VSS_parse(s->listen, &s->addr, &s->port));
 			av++;



From martin at varnish-software.com  Tue May  6 08:46:02 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 06 May 2014 10:46:02 +0200
Subject: [master] 2baa875 Recycle backend connection before setting
 BOS_FINISHED (take two)
Message-ID: <E1Whb0k-0005dm-5J@project.varnish-software.com>


commit 2baa8754fe4a0fc3e4efc9865bddc265fae13690
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon May 5 14:06:40 2014 +0200

    Recycle backend connection before setting BOS_FINISHED (take two)
    
    This to give predictable backend reuse behavior for varnishtest,
    making the test cases non-racy.
    
    2nd version of this patch. This version will only deal with recyclable
    backend connections before BOS_FINISHED, leaving the slower close path
    to the clean up code that is run after the BOS_FINISHED state.
    
    Fixes: #1491

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index dded999..ac4c478 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -548,6 +548,14 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		HSH_Unbusy(&wrk->stats, obj->objcore);
 	}
 	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
+
+	/* Recycle the backend connection before setting BOS_FINISHED to
+	   give predictable backend reuse behavior for varnishtest */
+	if (bo->vbc != NULL && !(bo->should_close)) {
+		VDI_RecycleFd(&bo->vbc, &bo->acct);
+		AZ(bo->vbc);
+	}
+
 	VBO_setstate(bo, BOS_FINISHED);
 	return (F_STP_DONE);
 }
@@ -639,6 +647,14 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	assert(al == bo->ims_obj->len);
 	assert(obj->len == al);
 	EXP_Rearm(bo->ims_obj, bo->ims_obj->exp.t_origin, 0, 0, 0);
+
+	/* Recycle the backend connection before setting BOS_FINISHED to
+	   give predictable backend reuse behavior for varnishtest */
+	if (bo->vbc != NULL && !(bo->should_close)) {
+		VDI_RecycleFd(&bo->vbc, &bo->acct);
+		AZ(bo->vbc);
+	}
+
 	VBO_setstate(bo, BOS_FINISHED);
 	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
 	return (F_STP_DONE);



From martin at varnish-software.com  Tue May  6 08:46:02 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 06 May 2014 10:46:02 +0200
Subject: [master] dabd65a Be consistent with when the BerespBody timestamp is
 taken.
Message-ID: <E1Whb0k-0005ds-6W@project.varnish-software.com>


commit dabd65a91b8737c0d71311f68464ed33c249e5f0
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue May 6 10:43:44 2014 +0200

    Be consistent with when the BerespBody timestamp is taken.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index ac4c478..f8e2848 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -547,7 +547,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		assert(bo->state == BOS_REQ_DONE);
 		HSH_Unbusy(&wrk->stats, obj->objcore);
 	}
-	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
 
 	/* Recycle the backend connection before setting BOS_FINISHED to
 	   give predictable backend reuse behavior for varnishtest */
@@ -557,6 +556,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	}
 
 	VBO_setstate(bo, BOS_FINISHED);
+	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
 	return (F_STP_DONE);
 }
 



From phk at FreeBSD.org  Tue May  6 08:57:34 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 06 May 2014 10:57:34 +0200
Subject: [master] a0c9c7e Add two counters, one counts purge operations,
 the other objects purged.
Message-ID: <E1WhbBu-0007Ou-Gq@project.varnish-software.com>


commit a0c9c7ec12596df022e1546fb21b88acc9c65340
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 6 08:56:41 2014 +0000

    Add two counters, one counts purge operations, the other objects purged.
    
    Suggested by:	Steven Engelhardt via github

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b0740c1..75a64ac 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1219,6 +1219,7 @@ void WAIT_Write_Session(struct sess *sp, int fd);
 void WRK_Init(void);
 int WRK_TrySumStat(struct worker *w);
 void WRK_SumStat(struct worker *w);
+void WRK_PurgeStat(unsigned nobj);
 void *WRK_thread(void *priv);
 typedef void *bgthread_t(struct worker *, void *priv);
 void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func,
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index f4bbae0..7aa6d24 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -602,6 +602,7 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
 		(void)HSH_DerefObj(&wrk->stats, &o);
 	}
 	WS_Release(wrk->aws, 0);
+	WRK_PurgeStat(nobj);
 }
 
 
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index ec18df9..6496a3a 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -77,6 +77,19 @@ WRK_TrySumStat(struct worker *w)
 }
 
 /*--------------------------------------------------------------------
+ * Helper function to update stats for purges under lock
+ */
+
+void
+WRK_PurgeStat(unsigned nobj)
+{
+	Lck_Lock(&wstat_mtx);
+	VSC_C_main->n_purges++;
+	VSC_C_main->n_obj_purged += nobj;
+	Lck_Unlock(&wstat_mtx);
+}
+
+/*--------------------------------------------------------------------
  * Create and starte a back-ground thread which as its own worker and
  * session data structures;
  */
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 4040b0d..8f25d39 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -583,6 +583,17 @@ VSC_F(bans_persisted_fragmentation,	uint64_t, 0, 'g', diag,
 
 /*--------------------------------------------------------------------*/
 
+VSC_F(n_purges,			uint64_t, 0, 'i', info,
+    "Number of purge operations",
+	""
+)
+VSC_F(n_obj_purged,		uint64_t, 0, 'i', info,
+    "number of purged objects",
+	""
+)
+
+/*--------------------------------------------------------------------*/
+
 VSC_F(exp_mailed,			uint64_t, 0, 'c', diag,
     "Number of objects mailed to expiry thread",
 	"Number of objects mailed to expiry thread for handling."



From lkarsten at varnish-software.com  Tue May  6 10:44:32 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 06 May 2014 12:44:32 +0200
Subject: [master] 4a3735e Remove extra whitespace.
Message-ID: <E1WhcrQ-00041c-Hk@project.varnish-software.com>


commit 4a3735ebe28335c36b9336dcaa96690ee7aff631
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 6 12:34:54 2014 +0200

    Remove extra whitespace.

diff --git a/redhat/varnish.initrc b/redhat/varnish.initrc
index c2faa25..dbaf673 100755
--- a/redhat/varnish.initrc
+++ b/redhat/varnish.initrc
@@ -53,7 +53,7 @@ start() {
 	# Open files (usually 1024, which is way too small for varnish)
 	ulimit -n ${NFILES:-131072}
 
-	# Varnish wants to lock shared memory log in memory. 
+	# Varnish wants to lock shared memory log in memory.
 	ulimit -l ${MEMLOCK:-82000}
 
 	# Maximum number of threads (default in CentOS is 1024, which



From lkarsten at varnish-software.com  Tue May  6 10:44:32 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 06 May 2014 12:44:32 +0200
Subject: [master] 1bbbc97 Add support for setting coresize from sysconfig.
Message-ID: <E1WhcrQ-00041i-Iz@project.varnish-software.com>


commit 1bbbc97deb6a201d36b8ec191f205acd13f3b8bf
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 6 12:36:51 2014 +0200

    Add support for setting coresize from sysconfig.
    
    Add support for using this if it is defined by the administrator
    in the /etc/sysconfig/varnish file.
    
    From way back we've had a commented-out DAEMON_COREFILE_LIMIT in
    our /etc/sysconfig/varnish file, which wasn't referenced anywhere.
    
    Suggested by: gaoyongwei/itxx00 on github.

diff --git a/redhat/varnish.initrc b/redhat/varnish.initrc
index dbaf673..0db3ab6 100755
--- a/redhat/varnish.initrc
+++ b/redhat/varnish.initrc
@@ -60,6 +60,12 @@ start() {
 	# is often too small for varnish)
 	ulimit -u ${NPROCS:-unlimited}
 
+	# If defined, set maximum core size.
+	if [ -n?"${DAEMON_COREFILE_LIMIT}" ]
+	then
+		ulimit -c ${DAEMON_COREFILE_LIMIT}
+	fi
+
         # $DAEMON_OPTS is set in /etc/sysconfig/varnish. At least, one
         # has to set up a backend, or /tmp will be used, which is a bad idea.
 	if [ "$DAEMON_OPTS" = "" ]; then



From phk at FreeBSD.org  Tue May  6 10:54:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 06 May 2014 12:54:27 +0200
Subject: [master] c0463aa Up the repeat count for slow test-enviroments
Message-ID: <E1Whd11-0005sI-My@project.varnish-software.com>


commit c0463aa3d91d86e71a628e93657f770ee625b0d3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 6 10:54:02 2014 +0000

    Up the repeat count for slow test-enviroments

diff --git a/bin/varnishtest/tests/c00035.vtc b/bin/varnishtest/tests/c00035.vtc
index 9463548..8b6bbea 100644
--- a/bin/varnishtest/tests/c00035.vtc
+++ b/bin/varnishtest/tests/c00035.vtc
@@ -1,6 +1,6 @@
 varnishtest "Dropping polling of a backend"
 
-server s1 -repeat 20 {
+server s1 -repeat 40 {
 	rxreq
 	txresp
 } -start



From perbu at varnish-software.com  Tue May  6 12:40:32 2014
From: perbu at varnish-software.com (Per Buer)
Date: Tue, 06 May 2014 14:40:32 +0200
Subject: [master] 928943c Update the grace docs to 4.0
Message-ID: <E1Whefg-0007Mr-2C@project.varnish-software.com>


commit 928943c78ca0051cfd8f5baac5fd341530d64a30
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue May 6 14:40:28 2014 +0200

    Update the grace docs to 4.0

diff --git a/doc/sphinx/users-guide/vcl-grace.rst b/doc/sphinx/users-guide/vcl-grace.rst
index c3a122c..16c1d11 100644
--- a/doc/sphinx/users-guide/vcl-grace.rst
+++ b/doc/sphinx/users-guide/vcl-grace.rst
@@ -24,34 +24,47 @@ the objects in cache beyond their TTL and to serve the waiting
 requests somewhat stale content.
 
 So, in order to serve stale content we must first have some content to
-serve. So to make Varnish keep all objects for 30 minutes beyond their
+serve. So to make Varnish keep all objects for 2 minutes beyond their
 TTL use the following VCL::
 
   sub vcl_backend_response {
-    set beresp.grace = 30m;
+    set beresp.grace = 2m;
   }
 
-Varnish still won't serve the stale objects. In order to enable
-Varnish to actually serve the stale object we must enable this on the
-request. Lets us say that we accept serving 15s old object.::
+Now Varnish will be allowed to serve objects that are up to two
+minutes out of date. When it does it will also schedule a refresh of
+the object. This will happen asynchronously and the moment the new
+object is in it will replace the one we've already got.
 
-  sub vcl_recv {
-    set req.grace = 15s;
+You can influence how this logic works by adding code in vcl_hit. The
+default looks like this:::
+
+  sub vcl_hit {
+     if (obj.ttl >= 0s) {
+         // A pure unadultered hit, deliver it
+         return (deliver);
+     }
+     if (obj.ttl + obj.grace > 0s) {
+         // Object is in grace, deliver it
+         // Automatically triggers a background fetch
+         return (deliver);
+     }
+     // fetch & deliver once we get the result
+     return (fetch);
   }
 
-You might wonder why we should keep the objects in the cache for 30
-minutes if we are unable to serve them? Well, if you have enabled
-:ref:`users-guide-advanced_backend_servers-health` you can check if the
-backend is sick and if it is we can serve the stale content for a bit
-longer.::
+The grace logic is pretty obvious here. If you have enabled
+:ref:`users-guide-advanced_backend_servers-health` you can check if
+the backend is sick and only serve graced object then. Replace the
+second if-clause with something like this:::
 
-   if (!std.healthy(backend)) {
-      set req.grace = 5m;
+   if (!std.healthy(backend) && (obj.ttl + obj.grace > 0s) {
+         return (deliver);
    } else {
-      set req.grace = 15s;
+         return (fetch);
    }
 
 So, to sum up, grace mode solves two problems:
  * it serves stale content to avoid request pile-up.
- * it serves stale content if the backend is not healthy.
+ * it serves stale content if you allow it.
 



From perbu at varnish-software.com  Tue May  6 14:09:35 2014
From: perbu at varnish-software.com (Per Buer)
Date: Tue, 06 May 2014 16:09:35 +0200
Subject: [master] 817ecee Syntax fixed
Message-ID: <E1Whg3r-0002Iu-KG@project.varnish-software.com>


commit 817eceee934072faa442cd831541d8ff235f8955
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue May 6 16:09:32 2014 +0200

    Syntax fixed

diff --git a/doc/sphinx/users-guide/vcl-grace.rst b/doc/sphinx/users-guide/vcl-grace.rst
index 16c1d11..e226ab6 100644
--- a/doc/sphinx/users-guide/vcl-grace.rst
+++ b/doc/sphinx/users-guide/vcl-grace.rst
@@ -58,7 +58,7 @@ The grace logic is pretty obvious here. If you have enabled
 the backend is sick and only serve graced object then. Replace the
 second if-clause with something like this:::
 
-   if (!std.healthy(backend) && (obj.ttl + obj.grace > 0s) {
+   if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace > 0s)) {
          return (deliver);
    } else {
          return (fetch);



From fgsch at lodoss.net  Thu May  8 01:27:44 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 08 May 2014 03:27:44 +0200
Subject: [master] 0a06bad Fix syntax
Message-ID: <E1WiD7g-0002GR-Na@project.varnish-software.com>


commit 0a06badefd94a3a1a69711b9b01371a7fe3a9371
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu May 8 02:26:21 2014 +0100

    Fix syntax
    
    Submitted by:	github::matsuu
    
    While here add missing semicolon.

diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index a51e4f6..be50e1c 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -64,7 +64,7 @@ Now we need tell Varnish where to send the difference URL. Lets look at `vcl_rec
         if (req.url ~ "^/java/") {
             set req.backend_hint = java;
         } else {
-            set req.backend_hint = default.
+            set req.backend_hint = default;
         }
     }
 
@@ -138,7 +138,7 @@ call certain actions in `vcl_init`.::
 
     sub vcl_recv {
         # send all traffic to the bar director:
-        req.backend_hint = bar.backend();
+        set req.backend_hint = bar.backend();
     }
 
 This director is a round-robin director. This means the director will



From phk at FreeBSD.org  Mon May 12 07:27:36 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 12 May 2014 09:27:36 +0200
Subject: [master] 2c2eb9b Fix a race i noticed while pondering #1500: We must
 snapshot the object length while processing the Range header to ensure
 consistent results when streaming from the backend.
Message-ID: <E1Wjke8-0005vP-0c@project.varnish-software.com>


commit 2c2eb9bcf45f75676f91eb6de2a4febd680db716
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 12 07:26:18 2014 +0000

    Fix a race i noticed while pondering #1500:  We must snapshot the
    object length while processing the Range header to ensure consistent
    results when streaming from the backend.

diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 765fd83..0b0e138 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -89,10 +89,18 @@ v1d_range_bytes(struct req *req, enum vdp_action act, const void *ptr,
 static void
 v1d_dorange(struct req *req, const char *r)
 {
-	ssize_t low, high, has_low;
+	ssize_t len, low, high, has_low;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 	assert(http_GetStatus(req->obj->http) == 200);
+
+	/* We must snapshot the length if we're streaming from the backend */
+	if (req->obj->objcore->busyobj != NULL)
+		len = VBO_waitlen(req->obj->objcore->busyobj, -1);
+	else
+		len = req->obj->len;
+
 	if (strncmp(r, "bytes=", 6))
 		return;
 	r += 6;
@@ -108,7 +116,7 @@ v1d_dorange(struct req *req, const char *r)
 		r++;
 	}
 
-	if (low >= req->obj->len)
+	if (low >= len)
 		return;
 
 	if (*r != '-')
@@ -124,24 +132,24 @@ v1d_dorange(struct req *req, const char *r)
 			r++;
 		}
 		if (!has_low) {
-			low = req->obj->len - high;
+			low = len - high;
 			if (low < 0)
 				low = 0;
-			high = req->obj->len - 1;
+			high = len - 1;
 		}
 	} else
-		high = req->obj->len - 1;
+		high = len - 1;
 	if (*r != '\0')
 		return;
 
-	if (high >= req->obj->len)
-		high = req->obj->len - 1;
+	if (high >= len)
+		high = len - 1;
 
 	if (low > high)
 		return;
 
 	http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/%jd",
-	    (intmax_t)low, (intmax_t)high, (intmax_t)req->obj->len);
+	    (intmax_t)low, (intmax_t)high, (intmax_t)len);
 	http_Unset(req->resp, H_Content_Length);
 	if (req->res_mode & RES_LEN)
 		http_PrintfHeader(req->resp, "Content-Length: %jd",



From phk at FreeBSD.org  Mon May 12 07:55:10 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 12 May 2014 09:55:10 +0200
Subject: [master] 6b27b8c Prefix the C-symbol of VCL objects with "vo_" to
 avoid issues with reserved C keywords.
Message-ID: <E1Wjl4o-0008Ml-DU@project.varnish-software.com>


commit 6b27b8c35c090df1dd61eead77184ac98e402f1f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 12 07:54:30 2014 +0000

    Prefix the C-symbol of VCL objects with "vo_" to avoid issues with
    reserved C keywords.
    
    Fixes #1498

diff --git a/bin/varnishtest/tests/r01498.vtc b/bin/varnishtest/tests/r01498.vtc
new file mode 100644
index 0000000..d321448
--- /dev/null
+++ b/bin/varnishtest/tests/r01498.vtc
@@ -0,0 +1,18 @@
+varnishtest "backend name VCC crash"
+
+varnish v1 -vcl {
+	vcl 4.0;
+	import ${vmod_directors};
+	backend s1 {
+	     .host = "127.0.0.1";
+	     .port = "80";
+	}
+	sub vcl_init {
+	    new static = directors.random();
+	    static.add_backend(s1, 100.0);
+	}
+
+	sub vcl_backend_fetch {
+		set bereq.backend = static.backend();
+	}
+}
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 84724c1..b06ff3d 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -200,17 +200,17 @@ parse_new(struct vcc *tl)
 		p++;
 	p += 2;
 
-	Fh(tl, 0, "static %s *%s;\n\n", s_struct, sy1->name);
+	Fh(tl, 0, "static %s *vo_%s;\n\n", s_struct, sy1->name);
 
 	vcc_NextToken(tl);
 
-	bprintf(buf1, ", &%s, \"%s\"", sy1->name, sy1->name);
+	bprintf(buf1, ", &vo_%s, \"%s\"", sy1->name, sy1->name);
 	vcc_Eval_Func(tl, s_init, buf1, "ASDF", s_init + strlen(s_init) + 1);
 	ifp = New_IniFin(tl);
-	VSB_printf(ifp->fin, "\t%s(&%s);", s_fini, sy1->name);
+	VSB_printf(ifp->fin, "\t%s(&vo_%s);", s_fini, sy1->name);
 	ExpectErr(tl, ';');
 
-	bprintf(buf1, ", %s", sy1->name);
+	bprintf(buf1, ", vo_%s", sy1->name);
 	/* Split the methods from the args */
 	while (*p != '\0') {
 		p += strlen(s_obj);



From phk at FreeBSD.org  Mon May 12 08:14:35 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 12 May 2014 10:14:35 +0200
Subject: [master] 8f036a0 Make our Via header RFC compliant.
Message-ID: <E1WjlNb-00022M-Rs@project.varnish-software.com>


commit 8f036a0b86221d6dcbf29021ccfdae1cbda7d04b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 12 08:14:09 2014 +0000

    Make our Via header RFC compliant.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index d465ba4..fea943e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -124,7 +124,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	http_PrintfHeader(req->resp, "Age: %.0f",
 	    fmax(0., req->t_prev - req->obj->exp.t_origin));
 
-	http_SetHeader(req->resp, "Via: 1.1 varnish (v4)");
+	http_SetHeader(req->resp, "Via: 1.1 varnish-v4");
 
 	if (cache_param->http_gzip_support && req->obj->gziped &&
 	    !RFC2616_Req_Gzip(req->http))



From martin at varnish-software.com  Mon May 12 08:56:45 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 12 May 2014 10:56:45 +0200
Subject: [master] 36c5d3a Deref the old object after finishing with it in
 cnt_miss()
Message-ID: <E1Wjm2P-0005IY-GH@project.varnish-software.com>


commit 36c5d3a7daee2a4088f9408ad92e3eb9c4468463
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu May 8 17:43:38 2014 +0200

    Deref the old object after finishing with it in cnt_miss()
    
    Fixes: #1499

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index fea943e..d60c136 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -498,6 +498,8 @@ cnt_miss(struct worker *wrk, struct req *req)
 		wrk->stats.cache_miss++;
 		VBF_Fetch(wrk, req, req->objcore, o, VBF_NORMAL);
 		req->req_step = R_STP_FETCH;
+		if (o != NULL)
+			(void)HSH_DerefObj(&wrk->stats, &o);
 		return (REQ_FSM_MORE);
 	case VCL_RET_SYNTH:
 		req->req_step = R_STP_SYNTH;
diff --git a/bin/varnishtest/tests/r01499.vtc b/bin/varnishtest/tests/r01499.vtc
new file mode 100644
index 0000000..7b438b2
--- /dev/null
+++ b/bin/varnishtest/tests/r01499.vtc
@@ -0,0 +1,32 @@
+varnishtest "#1499 - objcore ref leak on IMS update"
+
+server s1 {
+	rxreq
+	txresp -hdr "Etag: foo"
+	rxreq
+	expect req.http.if-none-match == "foo"
+	txresp -hdr "X-Resp: 2"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.ttl = 0.0001s;
+		set beresp.grace = 0s;
+		set beresp.keep = 2s;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+
+	delay 0.5
+
+	txreq
+	rxresp
+	expect resp.http.x-resp == "2"
+} -run
+
+delay 3
+
+varnish v1 -expect n_object == 0



From phk at FreeBSD.org  Mon May 12 08:57:15 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 12 May 2014 10:57:15 +0200
Subject: [master] 1dc303a The vcl_purge{} method can return synth or restart.
Message-ID: <E1Wjm2t-0005VX-I9@project.varnish-software.com>


commit 1dc303aac59deb4ec1ed0e9b491f4016aba2bcfb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 12 08:55:50 2014 +0000

    The vcl_purge{} method can return synth or restart.
    
    (Remember to change the "PURGE" to "GET" if you do this.)
    
    Fixes #1493

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index d60c136..9ebad3f 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -830,7 +830,16 @@ cnt_purge(struct worker *wrk, struct req *req)
 	AZ(HSH_DerefObjCore(&wrk->stats, &boc));
 
 	VCL_purge_method(req->vcl, wrk, req, NULL, req->http->ws);
-	req->req_step = R_STP_SYNTH;
+	switch (wrk->handling) {
+	case VCL_RET_RESTART:
+		req->req_step = R_STP_RESTART;
+		break;
+	case VCL_RET_SYNTH:
+		req->req_step = R_STP_SYNTH;
+		break;
+	default:
+		WRONG("Illegal return from vcl_purge{}");
+	}
 	return (REQ_FSM_MORE);
 }
 
diff --git a/bin/varnishtest/tests/r01493.vtc b/bin/varnishtest/tests/r01493.vtc
new file mode 100644
index 0000000..f27d899
--- /dev/null
+++ b/bin/varnishtest/tests/r01493.vtc
@@ -0,0 +1,24 @@
+varnishtest "restart in vcl_purge"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		if (req.method == "PURGE") {
+			return (purge);
+		}
+	}
+	sub vcl_purge {
+		set req.method = "GET";
+		return (restart);
+	}
+} -start
+
+client c1 {
+	txreq -req PURGE
+	rxresp
+	expect resp.status == 200
+} -run
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 1e523f1..c3b51ff 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -100,7 +100,7 @@ returns =(
 	),
 	('purge',
 		"C",
-		('synth', 'fetch',)
+		('synth', 'restart',)
 	),
 	('miss',
 		"C",



From lkarsten at varnish-software.com  Mon May 12 11:12:14 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 12 May 2014 13:12:14 +0200
Subject: [master] c3b1378 Fix typo.
Message-ID: <E1Wjo9W-0002bG-MP@project.varnish-software.com>


commit c3b1378870a85371ca9c24dcca883516b68de617
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 12 13:11:35 2014 +0200

    Fix typo.

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index dbd8a8a..3170a85 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -127,7 +127,7 @@ is reserved for builtin subs.
 req.backend.healthy replaced by std.healthy(req.backend)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Remeber to import the std module if you're not doing so already.
+Remember to import the std module if you're not doing so already.
 
 client.port, and server.port replaced by respectively std.port(client.ip) and std.port(server.ip)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~



From lkarsten at varnish-software.com  Mon May 12 11:12:14 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 12 May 2014 13:12:14 +0200
Subject: [master] 47cb9b9 Document that obj.hits may behave differently.
Message-ID: <E1Wjo9W-0002bM-Nm@project.varnish-software.com>


commit 47cb9b987508d710d9932d6a5ae03ad58c5d5574
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 12 13:11:53 2014 +0200

    Document that obj.hits may behave differently.

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index 3170a85..c882093 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -142,6 +142,12 @@ obj is now read-only
 `obj` is now read-only. `obj.hits`, if enabled in VCL, now counts per
 objecthead, not per object. `obj.last_use` has been retired.
 
+Note that obj.hits may not be reset in some cases where bans are in use. See
+bug 1492_ for details.
+
+.. _1492: https://www.varnish-cache.org/trac/ticket/1492
+
+
 Some return values have been replaced
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 



From lkarsten at varnish-software.com  Mon May 12 11:12:14 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 12 May 2014 13:12:14 +0200
Subject: [master] 28a9e89 Backtrack advice that bit during testing.
Message-ID: <E1Wjo9W-0002bV-Op@project.varnish-software.com>


commit 28a9e8947a924ca7d0eb1e970b024a71388cc824
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 12 13:12:11 2014 +0200

    Backtrack advice that bit during testing.

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index c882093..c8ed011 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -192,11 +192,13 @@ sess_workspace
 In 3.0 it was often necessary to increase `sess_workspace` if a
 lot of VMODs, complex header operations or ESI were in use.
 
-This memory segment has been split into two in 4.0;
-`workspace_backend` and `workspace_client`.
+This is no longer necessary, because ESI scratch space happens
+elsewhere in 4.0.
+
+If you are using a lot of VMODs,  you may need to increase
+either `workspace_backend` and `workspace_client` based on where
+your VMOD is doing its work.
 
-In most cases where you increased `sess_workspace` before, you
-want to increase `workspace_client` now.
 
 New parameters since 3.0
 ========================



From phk at FreeBSD.org  Mon May 12 11:30:13 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 12 May 2014 13:30:13 +0200
Subject: [master] 2fbac3f Cope with systems having both <sys/endian.h> and
 <endian.h>
Message-ID: <E1WjoQv-0006La-SU@project.varnish-software.com>


commit 2fbac3f212d858a55fe3ad11848fb2bfbee65671
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 12 11:29:46 2014 +0000

    Cope with systems having both <sys/endian.h> and <endian.h>
    
    Fixes:	1476

diff --git a/lib/libvarnish/vsha256.c b/lib/libvarnish/vsha256.c
index 6642d9f..90ac3ed 100644
--- a/lib/libvarnish/vsha256.c
+++ b/lib/libvarnish/vsha256.c
@@ -28,18 +28,17 @@
 
 #include "config.h"
 
-#ifdef HAVE_SYS_ENDIAN_H
-#include <sys/types.h>
-#include <sys/endian.h>
-#define VBYTE_ORDER	_BYTE_ORDER
-#define VBIG_ENDIAN	_BIG_ENDIAN
+#if defined(HAVE_SYS_ENDIAN_H)
+#  include <sys/types.h>
+#  include <sys/endian.h>
+#  define VBYTE_ORDER	_BYTE_ORDER
+#  define VBIG_ENDIAN	_BIG_ENDIAN
+#elif defined(HAVE_ENDIAN_H)
+#  include <endian.h>
+#  define VBYTE_ORDER	__BYTE_ORDER
+#  define VBIG_ENDIAN	__BIG_ENDIAN
 #endif
 
-#ifdef HAVE_ENDIAN_H
-#include <endian.h>
-#define VBYTE_ORDER	__BYTE_ORDER
-#define VBIG_ENDIAN	__BIG_ENDIAN
-#endif
 #include <errno.h>
 #include <stdint.h>
 #include <string.h>



From phk at FreeBSD.org  Mon May 12 12:07:08 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 12 May 2014 14:07:08 +0200
Subject: [master] 1af9e04 By default linux::getopt(3) mangles the argv order,
 such that
Message-ID: <E1Wjp0e-0004ax-2r@project.varnish-software.com>


commit 1af9e045df49393eba0951c67b65c924ab128f8f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 12 12:04:27 2014 +0000

    By default linux::getopt(3) mangles the argv order, such that
    
    	varnishadm -n bla param.set foo -bar
    
    gets interpreted as
    
    	varnishadm -n bla -bar param.set foo
    
    This is counterintuitive and invites random script breaking if people
    don't know they have to add an ornamental '--' in front of the CLI command
    in Linux systems.
    
    This commit makes Linux work the same way as every other POSIX system.
    
    Fixes	#1496

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index 90016a9..4b219de 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -439,7 +439,15 @@ main(int argc, char * const *argv)
 	const char *n_arg = NULL;
 	int opt, sock;
 
-	while ((opt = getopt(argc, argv, "n:S:T:t:")) != -1) {
+	/*
+	 * By default linux::getopt(3) mangles the argv order, such that
+	 * 	varnishadm -n bla param.set foo -bar
+	 * gets interpreted as
+	 * 	varnishadm -n bla -bar param.set foo 
+	 * The '+' stops that from happening
+	 * See #1496
+	 */
+	while ((opt = getopt(argc, argv, "+n:S:T:t:")) != -1) {
 		switch (opt) {
 		case 'n':
 			n_arg = optarg;



From phk at FreeBSD.org  Mon May 12 16:01:33 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 12 May 2014 18:01:33 +0200
Subject: [master] 32d0751 Reset the "must close" flag when we close the
 backend connection before a retry.
Message-ID: <E1WjsfV-0001zA-IB@project.varnish-software.com>


commit 32d0751bc19f3725e291e46b95ff33e1882a291e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 12 16:00:59 2014 +0000

    Reset the "must close" flag when we close the backend connection
    before a retry.
    
    Fixes #1494
    
    Testcase by:	scoof

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index f8e2848..0f49a3b 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -371,6 +371,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	if (wrk->handling == VCL_RET_RETRY) {
 		AN (bo->vbc);
 		VDI_CloseFd(&bo->vbc, &bo->acct);
+		bo->should_close = 0;
 		bo->retries++;
 		if (bo->retries <= cache_param->max_retries)
 			return (F_STP_RETRY);
diff --git a/bin/varnishtest/tests/r01494.vtc b/bin/varnishtest/tests/r01494.vtc
new file mode 100644
index 0000000..38880c5
--- /dev/null
+++ b/bin/varnishtest/tests/r01494.vtc
@@ -0,0 +1,27 @@
+varnishtest "Test retry in be_resp w/conn: close"
+
+server s1 {
+	rxreq
+	txresp -hdr "Connection: close" -bodylen 3
+	expect_close
+	accept
+
+	rxreq
+	txresp -hdr "Connection: close" -bodylen 5
+	expect_close
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		if(bereq.retries == 0) {
+			return(retry);
+		}
+	}
+} -start
+
+client c1 {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 5
+} -run



From phk at FreeBSD.org  Tue May 13 06:56:21 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 13 May 2014 08:56:21 +0200
Subject: [master] 578366e Also zap keep when purging.
Message-ID: <E1Wk6dR-0003XB-8F@project.varnish-software.com>


commit 578366ecf3fbc1159ea5ed8543f3e81412bbe43d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 13 06:56:04 2014 +0000

    Also zap keep when purging.
    
    Fixes #1139

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 7aa6d24..9acf893 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -555,7 +555,8 @@ hsh_rush(struct dstat *ds, struct objhead *oh)
  */
 
 void
-HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
+HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace,
+double keep)
 {
 	struct objcore *oc, **ocp;
 	unsigned spc, nobj, n;
@@ -598,7 +599,7 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
 		if (o == NULL)
 			continue;
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-		EXP_Rearm(o, now, ttl, grace, NAN);	// XXX: Keep ?
+		EXP_Rearm(o, now, ttl, grace, keep);
 		(void)HSH_DerefObj(&wrk->stats, &o);
 	}
 	WS_Release(wrk->aws, 0);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 9ebad3f..86fed3e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -825,7 +825,7 @@ cnt_purge(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(boc, OBJCORE_MAGIC);
 	VRY_Finish(req, DISCARD);
 
-	HSH_Purge(wrk, boc->objhead, 0, 0);
+	HSH_Purge(wrk, boc->objhead, 0, 0, 0);
 
 	AZ(HSH_DerefObjCore(&wrk->stats, &boc));
 
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index c423eb2..4e1bcbf 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -492,10 +492,10 @@ VRT_purge(const struct vrt_ctx *ctx, double ttl, double grace)
 	CHECK_OBJ_NOTNULL(ctx->req->wrk, WORKER_MAGIC);
 	if (ctx->method == VCL_MET_HIT)
 		HSH_Purge(ctx->req->wrk, ctx->req->obj->objcore->objhead,
-		    ttl, grace);
+		    ttl, grace, NAN);
 	else if (ctx->method == VCL_MET_MISS)
 		HSH_Purge(ctx->req->wrk, ctx->req->objcore->objhead,
-		    ttl, grace);
+		    ttl, grace, NAN);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index 40d3aef..3c6f815 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -70,7 +70,8 @@ void HSH_Drop(struct worker *, struct object **);
 void HSH_Init(const struct hash_slinger *slinger);
 void HSH_AddString(const struct req *, const char *str);
 void HSH_Insert(struct worker *, const void *hash, struct objcore *);
-void HSH_Purge(struct worker *, struct objhead *, double ttl, double grace);
+void HSH_Purge(struct worker *, struct objhead *, double ttl, double grace,
+    double keep);
 void HSH_config(const char *h_arg);
 struct busyobj *HSH_RefBusy(const struct objcore *oc);
 struct objcore *HSH_Private(struct worker *wrk);



From phk at FreeBSD.org  Tue May 13 07:10:47 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 13 May 2014 09:10:47 +0200
Subject: [master] ee92500 Remove the stand-alone "purge" action from VCL,
 now that we have return(purge) from vcl_recv.
Message-ID: <E1Wk6rP-0005EV-V5@project.varnish-software.com>


commit ee9250095af04b74c31ca769a55da3d18a6eeb7d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 13 07:10:11 2014 +0000

    Remove the stand-alone "purge" action from VCL, now that we have
    return(purge) from vcl_recv.
    
    Retain the VRT_purge() for VMOD and inline-C use.

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 4e1bcbf..08e3b77 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -480,11 +480,11 @@ VRT_CacheReqBody(const struct vrt_ctx *ctx, long long maxsize)
 }
 
 /*--------------------------------------------------------------------
- * "real" purges
+ * purges
  */
 
 void
-VRT_purge(const struct vrt_ctx *ctx, double ttl, double grace)
+VRT_purge(const struct vrt_ctx *ctx, double ttl, double grace, double keep)
 {
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
@@ -492,10 +492,10 @@ VRT_purge(const struct vrt_ctx *ctx, double ttl, double grace)
 	CHECK_OBJ_NOTNULL(ctx->req->wrk, WORKER_MAGIC);
 	if (ctx->method == VCL_MET_HIT)
 		HSH_Purge(ctx->req->wrk, ctx->req->obj->objcore->objhead,
-		    ttl, grace, NAN);
+		    ttl, grace, keep);
 	else if (ctx->method == VCL_MET_MISS)
 		HSH_Purge(ctx->req->wrk, ctx->req->objcore->objhead,
-		    ttl, grace, NAN);
+		    ttl, grace, keep);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishtest/tests/c00041.vtc b/bin/varnishtest/tests/c00041.vtc
index 2b7efc2..bd6abdb 100644
--- a/bin/varnishtest/tests/c00041.vtc
+++ b/bin/varnishtest/tests/c00041.vtc
@@ -39,8 +39,12 @@ server s1 {
 } -start
 
 varnish v1 -vcl+backend {
-	sub vcl_miss { if (req.http.purge == "yes") { purge; } }
-	sub vcl_hit { if (req.http.purge == "yes") { purge; return(restart);} }
+	sub vcl_recv { if (req.http.purge == "yes") { return(purge); } }
+	sub vcl_purge { if (req.http.restart == "yes") {
+		unset req.http.purge;
+		unset req.http.restart;
+		return(restart);} 
+	}
 } -start
 
 client c1 {
@@ -49,6 +53,7 @@ client c1 {
 	expect resp.status == 200
 	expect resp.http.x-varnish == 1001
 	expect resp.bodylen == 1
+	delay .1
 
 	txreq -url "/1" -hdr "Foo: foo2"
 	rxresp
@@ -62,42 +67,44 @@ client c1 {
 	expect resp.status == 200
 	expect resp.http.x-varnish == "1005 1002"
 	expect resp.bodylen == 1
+	delay .1
 
 	txreq -url "/1" -hdr "Foo: foo2"
 	rxresp
 	expect resp.status == 200
 	expect resp.http.x-varnish == "1006 1004"
 	expect resp.bodylen == 2
+	delay .1
 
 	# Purge on hit
-	txreq -url "/1" -hdr "Foo: foo2" -hdr "purge: yes"
+	txreq -url "/1" -hdr "Foo: foo2" -hdr "purge: yes" -hdr "restart: yes"
 	rxresp
 	expect resp.status == 200
 	expect resp.bodylen == 12
+	delay .1
 
 	txreq -url "/1" -hdr "foo: foo1"
 	rxresp
 	expect resp.status == 200
 	expect resp.bodylen == 11
+	delay .1
 
 	# Purge on miss
-	txreq -url "/1" -hdr "Foo: foo3" -hdr "purge: yes"
+	txreq -url "/1" -hdr "Foo: foo3" -hdr "purge: yes" -hdr "restart: yes"
 	rxresp
 	expect resp.status == 200
 	expect resp.bodylen == 23
+	delay .1
 
 	txreq -url "/1" -hdr "foo: foo1"
 	rxresp
 	expect resp.status == 200
 	expect resp.bodylen == 21
+	delay .1
 
 	txreq -url "/1" -hdr "Foo: foo2"
 	rxresp
 	expect resp.status == 200
 	expect resp.bodylen == 22
+	delay .1
 } -run
-
-varnish v1 -errvcl {'purge': not a valid action in method 'vcl_recv'.} {
-	backend s1 { .host = "${s1_addr}"; }
-	sub vcl_recv { if (req.http.purge == "yes") { purge; } }
-}
diff --git a/include/vrt.h b/include/vrt.h
index 8e5acba..bfbbb2d 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -201,7 +201,7 @@ const char *VRT_regsub(const struct vrt_ctx *, int all, const char *,
     void *, const char *);
 
 void VRT_ban_string(const struct vrt_ctx *, const char *);
-void VRT_purge(const struct vrt_ctx *, double ttl, double grace);
+void VRT_purge(const struct vrt_ctx *, double ttl, double grace, double keep);
 
 void VRT_count(const struct vrt_ctx *, unsigned);
 int VRT_rewrite(const char *, const char *);
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index b06ff3d..910a89c 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -367,16 +367,6 @@ parse_rollback(struct vcc *tl)
 /*--------------------------------------------------------------------*/
 
 static void
-parse_purge(struct vcc *tl)
-{
-
-	vcc_NextToken(tl);
-	Fb(tl, 1, "VRT_purge(ctx, 0, 0);\n");
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
 parse_synthetic(struct vcc *tl)
 {
 	vcc_NextToken(tl);
@@ -409,7 +399,6 @@ static struct action_table {
 	{ "call",		parse_call },
 	{ "hash_data",		parse_hash_data, VCL_MET_HASH },
 	{ "new",		parse_new, VCL_MET_INIT},
-	{ "purge",		parse_purge, VCL_MET_MISS | VCL_MET_HIT },
 	{ "return",		parse_return },
 	{ "rollback",		parse_rollback },
 	{ "set",		parse_set },



From phk at FreeBSD.org  Tue May 13 07:36:53 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 13 May 2014 09:36:53 +0200
Subject: [master] 39a3455 Insert a newline before the actual panic message in
 order to avoid line wrapping
Message-ID: <E1Wk7Gf-000750-IN@project.varnish-software.com>


commit 39a3455f894ed68ff1d6e55721e05aca5c2fab20
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 13 07:36:23 2014 +0000

    Insert a newline before the actual panic message in order to
    avoid line wrapping

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 3c739f1..1ae0c0e 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -121,7 +121,7 @@ mgt_panic_record(pid_t r)
 	char time_str[30];
 
 	AN(heritage.panic_str[0]);
-	REPORT(LOG_ERR, "Child (%jd) Panic message: %s",
+	REPORT(LOG_ERR, "Child (%jd) Panic message:\n%s",
 	    (intmax_t)r, heritage.panic_str);
 
 	if (child_panic != NULL)



From phk at FreeBSD.org  Tue May 13 08:44:58 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 13 May 2014 10:44:58 +0200
Subject: [master] 20d3eca Move a misplaced assert.
Message-ID: <E1Wk8KY-0004NR-2M@project.varnish-software.com>


commit 20d3ecae26e8b9f522c04b8759e8e5b90df5789a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 13 08:44:32 2014 +0000

    Move a misplaced assert.
    
    Fixes	#1478

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 9acf893..cd48671 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -458,10 +458,10 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 		exp_oc->refcnt++;
 
 		if (!busy_found) {
-			AZ(req->hash_ignore_busy);
 			*bocp = hsh_insert_busyobj(wrk, oh);
 			retval = HSH_EXPBUSY;
 		} else {
+			AZ(req->hash_ignore_busy);
 			retval = HSH_EXP;
 		}
 		if (oh->hits < LONG_MAX)



From lkarsten at varnish-software.com  Tue May 13 09:55:59 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 11:55:59 +0200
Subject: [master] 284c933 Add documentation for the fallback director.
Message-ID: <E1Wk9RH-0002C6-73@project.varnish-software.com>


commit 284c933d07337bbdd7530b8fd0745c93e91cc0c5
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 11:12:38 2014 +0200

    Add documentation for the fallback director.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 9b5732a..c360837 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -59,8 +59,9 @@ Note that directors can use other directors as backends.
 $Object round_robin()
 
 Description
-       Create a round robin director. This director will pick backends
-       in a round robin fashion.
+        Create a round robin director.
+
+	This director will pick backends in a round robin fashion.
 Example
        new bar = directors.round_robin();
 
@@ -78,30 +79,37 @@ Description
 Example
        set req.backend_hint = rrdir.backend();
 
+####################################################################
 $Object fallback()
 
 Description
-        Creates a fallback director.
+        Create a fallback director.
 
-#	XXX: Add description of fallback director.
+        A fallback director will try each of the added backends in turn,
+        and return the first one that is healthy.
 
 Example
-        new foo = directors.fallback();
+        new vdir = directors.fallback();
 
 $Method VOID .add_backend(BACKEND)
 
 Description
-       Adds a backend to the director.
+        Add a backend to the director.
+
+        Note that the order in which this is done matters in the for the
+        fallback director.
+
 Example
-       bar.add_backend(backend1);
+       vdir.add_backend(backend1);
 
 $Method BACKEND .backend()
 
 Description
-       Picks a backend from the director.
+       Pick a backend from the director.
 Example
-       set req.backend_hint = rrdir.backend();
+       set req.backend_hint = vdir.backend();
 
+####################################################################
 $Object random()
 
 Description



From lkarsten at varnish-software.com  Tue May 13 09:55:59 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 11:55:59 +0200
Subject: [master] 7ac9230 Rewrite first paragraph to make it easier to read.
Message-ID: <E1Wk9RH-0002CC-8S@project.varnish-software.com>


commit 7ac92307370a6f2ddce8cba892f6a87251b71ac6
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 11:18:14 2014 +0200

    Rewrite first paragraph to make it easier to read.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index c360837..d18151f 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -30,9 +30,11 @@ $Module directors 3 Backend traffic directors
 DESCRIPTION
 ===========
 
-Vmod_directors enables load balancing in Varnish. The module serves
-as an example on how one could extend the load balancing capabilities
-of Varnish.
+`vmod_directors` enables backend load balancing in Varnish.
+
+The module implements a set of basic load balancing techniques, and
+also serves as an example on how one could extend the load balancing
+capabilities of Varnish.
 
 To enable load balancing you must import this vmod (directors) in your
 VCL:::



From lkarsten at varnish-software.com  Tue May 13 09:55:59 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 11:55:59 +0200
Subject: [master] c7c7089 Add dependeny to build both rst files from vcc.
Message-ID: <E1Wk9RH-0002CL-9V@project.varnish-software.com>


commit c7c70895a4cc0213f9ec0f043eb4e2456599a6b0
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 11:24:06 2014 +0200

    Add dependeny to build both rst files from vcc.
    
    Fix spotty dependency where none or just one of the
    .rst files output by vmodtool.py was known by the Makefile.

diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am
index 89df1b2..f856dde 100644
--- a/lib/libvmod_debug/Makefile.am
+++ b/lib/libvmod_debug/Makefile.am
@@ -24,7 +24,7 @@ nodist_libvmod_debug_la_SOURCES = \
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
 vmod_debug.lo vmod_debug_obj.lo: vcc_if.h
 
-vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc
+vcc_if.c vcc_if.h vmod_debug.rst vmod_debug.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
 
 EXTRA_DIST = vmod.vcc
diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am
index 05912e0..583b1f7 100644
--- a/lib/libvmod_directors/Makefile.am
+++ b/lib/libvmod_directors/Makefile.am
@@ -28,7 +28,7 @@ nodist_libvmod_directors_la_SOURCES = \
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
 vdir.lo fall_back.lo hash.lo random.lo round_robin.lo: vcc_if.h
 
-vcc_if.c vcc_if.h vmod_directors.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
+vcc_if.c vcc_if.h vmod_directors.rst vmod_directors.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
 
 EXTRA_DIST = vmod.vcc
diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index 2e9832a..ff8466e 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -26,7 +26,7 @@ nodist_libvmod_std_la_SOURCES = \
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
 vmod_std.lo vmod_std_fileread.lo vmod_std_conversions.lo: vcc_if.h
 
-vcc_if.c vcc_if.h vmod_std.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
+vcc_if.c vcc_if.h vmod_std.rst vmod_std.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
 
 EXTRA_DIST = vmod.vcc



From lkarsten at varnish-software.com  Tue May 13 09:55:59 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 11:55:59 +0200
Subject: [master] 45ee24a Tiny lint; file is called vmod.vcc.
Message-ID: <E1Wk9RH-0002CU-Ac@project.varnish-software.com>


commit 45ee24af32f2a3742966efdf9c0aff95783a863a
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 11:44:31 2014 +0200

    Tiny lint; file is called vmod.vcc.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index be6cff6..b9d7c95 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -26,7 +26,7 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
-# Read the vmod.spec file and produce the vmod.h and vmod.c files.
+# Read the vmod.vcc file and produce the vmod.h and vmod.c files.
 #
 # vmod.h contains the prototypes for the published functions, the module
 # C-code should include this file to ensure type-consistency.



From lkarsten at varnish-software.com  Tue May 13 09:55:59 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 11:55:59 +0200
Subject: [master] 7a2a983 Output correct tool name.
Message-ID: <E1Wk9RH-0002Cd-Bz@project.varnish-software.com>


commit 7a2a9836424092dbd2807cd81751f1b9de19c26e
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 11:45:14 2014 +0200

    Output correct tool name.
    
    And avoid hardcoding it, in case we change it in the future
    some time.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index b9d7c95..9dc2c4b 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -39,6 +39,7 @@
 
 import sys
 import re
+from os.path import basename
 
 if len(sys.argv) == 2:
 	specfile = sys.argv[1]
@@ -69,10 +70,10 @@ def file_header(fo):
         fo.write("""/*
  * NB:  This file is machine generated, DO NOT EDIT!
  *
- * Edit vmod.vcc and run vmod.py instead
+ * Edit vmod.vcc and run %s instead
  */
 
-""")
+""" % basename(__file__))
 
 #######################################################################
 



From lkarsten at varnish-software.com  Tue May 13 09:55:59 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 11:55:59 +0200
Subject: [master] 2180ad2 Output autogen-msg to .rst; pythonify a bit.
Message-ID: <E1Wk9RH-0002Cp-DZ@project.varnish-software.com>


commit 2180ad2b748996d7550748199db46f4924765bf0
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 11:46:18 2014 +0200

    Output autogen-msg to .rst; pythonify a bit.
    
    Write a header in generated .rst files that it was generated from the
    .vcc file.
    
    Also softly sprinkled with some python formatting:
    * use a python 2.5 context manager for the filepointer
    * and make the output easier to read by avoiding concatination.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 9dc2c4b..74fd633 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -799,16 +799,19 @@ fc.close()
 fh.close()
 
 for suf in ("", ".man"):
-	fr = open("vmod_" + vx[0].nam + suf + ".rst", "w")
-	vx[0].doc_dump(fr, suf)
-
-	if len(copyright) > 0:
-		fr.write("\n")
-		fr.write("COPYRIGHT\n")
-		fr.write("=========\n")
-		fr.write("\n::\n\n")
-		for i in copyright:
-			fr.write("  " + i + "\n")
-		fr.write("\n")
-
-	fr.close()
+	with open("vmod_%s%s.rst" % (vx[0].nam, suf), "w") as fp:
+		fp.write("..\n")
+		fp.write(".. This file was autogenerated by %s. DO NOT EDIT!\n" %
+			 basename(__file__))
+		fp.write("..\n")
+
+		vx[0].doc_dump(fp, suf)
+
+		if len(copyright) > 0:
+			fp.write("\n")
+			fp.write("COPYRIGHT\n")
+			fp.write("=========\n")
+			fp.write("\n::\n\n")
+			for i in copyright:
+				fp.write("  %s\n" % i)
+			fp.write("\n")



From lkarsten at varnish-software.com  Tue May 13 09:55:59 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 11:55:59 +0200
Subject: [master] fb1e33d Remove text block dividers just added.
Message-ID: <E1Wk9RH-0002Cv-Eq@project.varnish-software.com>


commit fb1e33d83d70a3f822315815a31bb0f5c9936e6c
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 11:51:33 2014 +0200

    Remove text block dividers just added.
    
    This made it easier to see where objects started/stopped, but
    vmodtool.py don't believe in this comment format. Especially rst2man
    wasn't impressed, so roll back for now.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index d18151f..f0d9424 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -81,7 +81,7 @@ Description
 Example
        set req.backend_hint = rrdir.backend();
 
-####################################################################
+
 $Object fallback()
 
 Description
@@ -111,7 +111,7 @@ Description
 Example
        set req.backend_hint = vdir.backend();
 
-####################################################################
+
 $Object random()
 
 Description



From lkarsten at varnish-software.com  Tue May 13 12:08:41 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 14:08:41 +0200
Subject: [master] 8fb3ea6 Output RST comment with enough linefeeds.
Message-ID: <E1WkBVh-00082i-IU@project.varnish-software.com>


commit 8fb3ea686c03fe64bc66da54bbccf4d645343415
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 13:19:25 2014 +0200

    Output RST comment with enough linefeeds.
    
    Empty RST comments need an extra empty line afterwards, re this output
    from rst2man:
    
        (WARNING/2) Explicit markup ends without a blank line; unexpected unindent

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 74fd633..4d34d07 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -803,7 +803,7 @@ for suf in ("", ".man"):
 		fp.write("..\n")
 		fp.write(".. This file was autogenerated by %s. DO NOT EDIT!\n" %
 			 basename(__file__))
-		fp.write("..\n")
+		fp.write("..\n\n")
 
 		vx[0].doc_dump(fp, suf)
 



From lkarsten at varnish-software.com  Tue May 13 12:08:41 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 14:08:41 +0200
Subject: [master] 4141e7a Halt rst2man/rst2html on warnings.
Message-ID: <E1WkBVh-00082p-Jc@project.varnish-software.com>


commit 4141e7a47629ca2dfb6e1adbbc918a29767a7c9a
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 13:21:49 2014 +0200

    Halt rst2man/rst2html on warnings.
    
    We don't allow compiler warnings in the C code, so I don't see any
    reason why we should allow warnings in the documentation.
    
    Warnings in the documentation are now build errors.

diff --git a/man/Makefile.am b/man/Makefile.am
index e4a0bfe..a3d3077 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -25,67 +25,69 @@ dist_man_MANS = \
 
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 
+RST2ANY_FLAGS = --halt=2
+
 varnish-cli.7: $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst $@
 
 varnish-counters.7: vsc2rst
-	./vsc2rst | ${RST2MAN} - $@
+	./vsc2rst | ${RST2MAN} $(RST2ANY_FLAGS) - $@
 
 vcl.7: $(top_srcdir)/doc/sphinx/reference/vcl.rst \
 	$(top_srcdir)/bin/varnishd/builtin.vcl
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vcl.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/vcl.rst $@
 
 vsl.7: $(top_srcdir)/doc/sphinx/reference/vsl.rst \
 	$(top_srcdir)/lib/libvarnishapi/vsl-tags.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vsl.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/vsl.rst $@
 
 vsl-query.7: $(top_srcdir)/doc/sphinx/reference/vsl-query.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vsl-query.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/vsl-query.rst $@
 
 varnishadm.1: $(top_srcdir)/doc/sphinx/reference/varnishadm.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishadm.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishadm.rst $@
 
 varnishd.1: \
 	$(top_srcdir)/doc/sphinx/reference/varnishd.rst \
 	$(top_srcdir)/doc/sphinx/include/params.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishd.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishd.rst $@
 
 varnishncsa.1: \
 	$(top_srcdir)/doc/sphinx/reference/varnishncsa.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishncsa_options.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishncsa_synopsis.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishncsa.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishncsa.rst $@
 
 varnishlog.1: \
 	$(top_srcdir)/doc/sphinx/reference/varnishlog.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishlog_options.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishlog_synopsis.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishlog.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishlog.rst $@
 
 varnishreplay.1: $(top_srcdir)/doc/sphinx/reference/varnishreplay.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishreplay.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishreplay.rst $@
 
 # XXX add _options.rst and _synopsis.rst here when it's been _opt2rst'ed
 varnishstat.1: $(top_srcdir)/doc/sphinx/reference/varnishstat.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishstat.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishstat.rst $@
 
 varnishtest.1: $(top_srcdir)/doc/sphinx/reference/varnishtest.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishtest.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishtest.rst $@
 
 varnishtop.1: \
 	$(top_srcdir)/doc/sphinx/reference/varnishtop.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishtop_options.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishtop_synopsis.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishtop.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishtop.rst $@
 
 varnishhist.1: \
 	$(top_srcdir)/doc/sphinx/reference/varnishhist.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishhist_options.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishhist_synopsis.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishhist.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishhist.rst $@
 
 vmod_std.3: $(top_srcdir)/lib/libvmod_std/vmod_std.man.rst
-	${RST2MAN} $? $@
+	${RST2MAN} $(RST2ANY_FLAGS) $? $@
 
 vmod_directors.3: $(top_srcdir)/lib/libvmod_directors/vmod_directors.man.rst
-	${RST2MAN} $? $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(RST2ANY_FLAGS) $? $@



From lkarsten at varnish-software.com  Tue May 13 12:08:41 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 14:08:41 +0200
Subject: [master] ebbd828 Delete unreferenced file vcl-intro.rst.
Message-ID: <E1WkBVh-00082z-Ke@project.varnish-software.com>


commit ebbd828839bd4668382d83acc2004b3e370c445f
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 13:39:38 2014 +0200

    Delete unreferenced file vcl-intro.rst.
    
    Sphinx complains that this file isn't referenced
    anywhere, and a quick glance indicate that the contents
    have been merged into other rst files.

diff --git a/doc/sphinx/users-guide/vcl-intro.rst b/doc/sphinx/users-guide/vcl-intro.rst
deleted file mode 100644
index 095896c..0000000
--- a/doc/sphinx/users-guide/vcl-intro.rst
+++ /dev/null
@@ -1,28 +0,0 @@
-Varnish Configuration Language - VCL
--------------------------------------
-
-Varnish has a great configuration system. Most other systems use
-configuration directives, where you basically turn on and off lots of
-switches. Varnish uses a domain specific language called Varnish
-Configuration Language, or VCL for short. Varnish translates this
-configuration into binary code which is then executed when requests
-arrive.
-
-The VCL files are divided into subroutines. The different subroutines
-are executed at different times. One is executed when we get the
-request, another when files are fetched from the backend server.
-
-Varnish will execute these subroutines of code at different stages of
-its work. At some point you call an action in this subroutine and then
-the execution of that subroutine stops.
-
-If you don't call an action in your subroutine and it reaches the end
-Varnish will execute the built in VCL code. You will see this VCL
-code commented out in default.vcl.
-
-99% of all the changes you'll need to do will be done in two of these
-subroutines. *vcl_recv* and *vcl_backend_response*.
-
-.. _users-guide-vcl_fetch_actions:
-
-



From lkarsten at varnish-software.com  Tue May 13 12:08:41 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 14:08:41 +0200
Subject: [master] 57a46f0 Put vmod docs back into the documentation.
Message-ID: <E1WkBVh-000838-Lm@project.varnish-software.com>


commit 57a46f0229a4fbbb328a9cbbcb76da750e394c66
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 13:54:15 2014 +0200

    Put vmod docs back into the documentation.
    
    Squashing Sphinx warnings revealed that the vmod_std and
    vmod_directors includes wasn't working.
    
    Move the generated files a bit so Sphinx is happy to load them.
    
    I'm not using include (where relative paths seem to work)
    since that works differently in a TOC listing.

diff --git a/.gitignore b/.gitignore
index ba64e39..b4ae99b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -80,6 +80,7 @@ cscope.*out
 /man/*.3
 /man/*.7
 /doc/sphinx/include
+/doc/sphinx/*/*generated.rst
 /bin/varnish*/varnish*_opt2rst
 /bin/varnishadm/varnishadm
 /bin/varnishd/varnishd
diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index ab8d021..0219fac 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -213,13 +213,13 @@ include/varnishhist_synopsis.rst: $(top_builddir)/bin/varnishhist/varnishhist_op
 BUILT_SOURCES += include/varnishhist_options.rst \
 	 include/varnishhist_synopsis.rst
 
-include/vmod_std.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst
+reference/vmod_std.generated.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst
 	cp $? $@
-BUILT_SOURCES += include/vmod_std.rst
+BUILT_SOURCES += reference/vmod_std.generated.rst
 
-include/vmod_directors.rst: $(top_builddir)/lib/libvmod_directors/vmod_directors.rst
+reference/vmod_directors.generated.rst: $(top_builddir)/lib/libvmod_directors/vmod_directors.rst
 	cp $? $@
-BUILT_SOURCES += include/vmod_directors.rst
+BUILT_SOURCES += reference/vmod_directors.generated.rst
 
 EXTRA_DIST += $(BUILT_SOURCES)
 MAINTAINERCLEANFILES = $(EXTRA_DIST)
diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index 20ac975..32458a6 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -21,8 +21,8 @@ The Varnish Reference Manual
 	varnishtop.rst
 	vsm.rst
 	vmod.rst
-	../include/vmod_std.rst
-	../include/vmod_directors.rst
+	vmod_std.generated.rst
+	vmod_directors.generated.rst
 	vsl.rst
 	vsl-query.rst
 



From lkarsten at varnish-software.com  Tue May 13 12:08:41 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 14:08:41 +0200
Subject: [master] 9acf7dc Increase Sphinx strictness.
Message-ID: <E1WkBVh-00083H-Mk@project.varnish-software.com>


commit 9acf7dccf0f768ee265454364fc3a66e8beb5011
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 13:59:01 2014 +0200

    Increase Sphinx strictness.
    
    Warnings are now considered build errors.
    
    We've been shipping documentation with sections missing
    because we didn't take warnings seriously.
    
    Bonus change: build in parallel, save 82 centiseconds per build!
    (html target on my laptop, ymmv)

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 0219fac..a5b9718 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
+SPHINXBUILD   = sphinx-build -j4 -W -N -n
 PAPER         = a4
 BUILDDIR      = build
 



From lkarsten at varnish-software.com  Tue May 13 12:15:08 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 14:15:08 +0200
Subject: [master] 57136c2 Disable parallel build due to old sphinx.
Message-ID: <E1WkBbw-0002bl-MD@project.varnish-software.com>


commit 57136c2322315ed88e0bacbf6df6759ab8a10287
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 14:10:41 2014 +0200

    Disable parallel build due to old sphinx.
    
    The sphinx version on git.varnish-cache.org is v1.1.8 which
    hasn't got -j.
    
    In sphinx v1.2.2 (debian jessie) it is in place.
    
    Guess we'll have to live with those extra 820 milliseconds
    per build for now.

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index a5b9718..27f8523 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build -j4 -W -N -n
+SPHINXBUILD   = sphinx-build -W -N -n
 PAPER         = a4
 BUILDDIR      = build
 



From lkarsten at varnish-software.com  Tue May 13 12:44:43 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 14:44:43 +0200
Subject: [master] d2358fd Remove remaining references to vcl-intro.rst.
Message-ID: <E1WkC4Z-00074Z-Pd@project.varnish-software.com>


commit d2358fd4b7925129f2003c415e86fc24a45b6694
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 14:42:41 2014 +0200

    Remove remaining references to vcl-intro.rst.
    
    These hard-code references were forgotten when I
    removed this file in ebbd828.

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 27f8523..e714383 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -166,7 +166,6 @@ EXTRA_DIST = \
 	users-guide/vcl-examples.rst \
 	users-guide/vcl-hashing.rst \
 	users-guide/vcl-inline-c.rst \
-	users-guide/vcl-intro.rst \
 	users-guide/vcl-grace.rst \
 	users-guide/vcl-syntax.rst \
 	users-guide/vcl-variables.rst \
diff --git a/doc/sphinx/Makefile.phk b/doc/sphinx/Makefile.phk
index fbfc35f..7e43fef 100644
--- a/doc/sphinx/Makefile.phk
+++ b/doc/sphinx/Makefile.phk
@@ -176,7 +176,6 @@ EXTRA_DIST = \
 	users-guide/vcl-examples.rst \
 	users-guide/vcl-hashing.rst \
 	users-guide/vcl-inline-c.rst \
-	users-guide/vcl-intro.rst \
 	users-guide/vcl-saint-and-grace.rst \
 	users-guide/vcl-syntax.rst \
 	users-guide/vcl-variables.rst \



From lkarsten at varnish-software.com  Tue May 13 16:08:11 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 18:08:11 +0200
Subject: [master] 96c0ae4 Add option parser, error handling and strictness.
Message-ID: <E1WkFFT-0008Bg-Qb@project.varnish-software.com>


commit 96c0ae47a0cd5d5325fae83f3b3cb64b8b5553de
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 16:30:33 2014 +0200

    Add option parser, error handling and strictness.
    
    This is one of those tiny jobs you start on (add --strict),
    and then it just goes downhill from there.
    
    Main changes:
    * Use optparse to parse command line arguments.
    * With --strict it aborts if a parse or format error is seen.
    * A basic test case for the parser has been added.
    
    No changes were made to the parser or output format.
    
    Adding the test case meant reordering the execution flow.
    
    Since Varnish code standard is using tabs, I've kept this to my
    best ability. Pylint is not happy about this and give the code
    a score of -4 out of 10 points. :-)

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 4d34d07..3319199 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #-
 # Copyright (c) 2010-2014 Varnish Software AS
 # All rights reserved.
@@ -26,7 +26,7 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
-# Read the vmod.vcc file and produce the vmod.h and vmod.c files.
+# Read the vmod.vcc file (inputvcc) and produce the vmod.h and vmod.c files.
 #
 # vmod.h contains the prototypes for the published functions, the module
 # C-code should include this file to ensure type-consistency.
@@ -39,12 +39,12 @@
 
 import sys
 import re
-from os.path import basename
+import optparse
+import unittest
+from os import unlink
+from os.path import dirname, basename, realpath, exists
+from pprint import pprint, pformat
 
-if len(sys.argv) == 2:
-	specfile = sys.argv[1]
-else:
-	specfile = "vmod.vcc"
 
 ctypes = {
 	'BACKEND':	"VCL_BACKEND",
@@ -66,8 +66,8 @@ ctypes = {
 
 #######################################################################
 
-def file_header(fo):
-        fo.write("""/*
+def write_file_header(fo):
+	fo.write("""/*
  * NB:  This file is machine generated, DO NOT EDIT!
  *
  * Edit vmod.vcc and run %s instead
@@ -80,13 +80,29 @@ def file_header(fo):
 def is_c_name(s):
 	return None != re.match("^[a-z][a-z0-9_]*$", s)
 
+
+class ParseError(Exception):
+	"An error reading the input file."
+	pass
+
+class FormatError(Exception):
+	"""
+		Raised if the content of the (otherwise well-formed) input file
+		is invalid.
+  """
+	def __init__(self, msg, details):
+		self.msg = msg
+		self.details = details
+		Exception.__init__(self)
+
+
 #######################################################################
 
 class token(object):
-	def __init__(self, ln, ch, str):
+	def __init__(self, ln, ch, tokstr):
 		self.ln = ln
 		self.ch = ch
-		self.str = str
+		self.str = tokstr
 
 	def __repr__(self):
 		return "<@%d \"%s\">" % (self.ln, self.str)
@@ -96,7 +112,7 @@ class token(object):
 class vmod(object):
 	def __init__(self, nam, dnam, sec):
 		if not is_c_name(nam):
-			raise Exception("Module name '%s' is illegal" % nam)
+			raise ParseError("Module name '%s' is illegal", nam)
 		self.nam = nam
 		self.dnam = dnam
 		self.sec = sec
@@ -108,9 +124,9 @@ class vmod(object):
 
 	def set_init(self, nam):
 		if self.init != None:
-			raise Exception("Module %s already has Init" % self.nam)
+			raise ParseError("Module %s already has Init", self.nam)
 		if not is_c_name(nam):
-			raise Exception("Init name '%s' is illegal" % nam)
+			raise ParseError("Init name '%s' is illegal", nam)
 		self.init = nam
 
 	def add_func(self, fn):
@@ -159,7 +175,7 @@ class vmod(object):
 		cs = self.c_struct()
 		fo.write(cs + ';\n')
 
-		vfn = 'Vmod_' + self.nam + '_Func';
+		vfn = 'Vmod_' + self.nam + '_Func'
 
 		fo.write("extern const struct " + vfn + " " + vfn + ';\n')
 		fo.write("const struct " + vfn + " " + vfn + ' =')
@@ -286,8 +302,8 @@ class func(object):
 		#if not is_c_name(nam):
 		#	raise Exception("Func name '%s' is illegal" % nam)
 		if retval not in ctypes:
-			raise Exception(
-			    "Return type '%s' not a valid type" % retval)
+			raise TypeError(
+			    "Return type '%s' not a valid type", retval)
 		self.nam = nam
 		self.cnam = nam.replace(".", "_")
 		self.al = al
@@ -400,6 +416,7 @@ class obj(object):
 		self.fini = None
 		self.methods = list()
 		self.doc_str = []
+		self.st = None
 
 	def fixup(self, modnam):
 		assert self.nam != None
@@ -431,7 +448,7 @@ class obj(object):
 	def c_proto(self, fo):
 		fo.write(self.st + ";\n")
 		self.init.c_proto(fo)
-		self.fini.c_proto(fo, fini = True)
+		self.fini.c_proto(fo, fini=True)
 		for m in self.methods:
 			m.c_proto(fo)
 
@@ -491,7 +508,7 @@ class obj(object):
 #######################################################################
 
 class arg(object):
-	def __init__(self, typ, nam = None, det = None):
+	def __init__(self, typ, nam=None, det=None):
 		self.nam = nam
 		self.typ = typ
 		self.det = det
@@ -512,7 +529,7 @@ class arg(object):
 def parse_enum2(tl):
 	t = tl.get_token()
 	if t.str != "{":
-		raise Exception("expected \"{\"")
+		raise ParseError("expected \"{\"")
 	s = "ENUM\\0"
 	t = None
 	while True:
@@ -527,7 +544,7 @@ def parse_enum2(tl):
 		elif t.str == "}":
 			break
 		else:
-			raise Exception(
+			raise ParseError(
 			    "Expected \"}\" or \",\" not \"%s\"" % t.str)
 	s += "\\0"
 	return arg("ENUM", det=s)
@@ -549,13 +566,13 @@ def parse_module(tl):
 #
 #
 
-def parse_func(tl, rt_type = None, obj=None):
+def parse_func(tl, rt_type=None, obj=None):
 	al = list()
 	if rt_type == None:
 		t = tl.get_token()
 		rt_type = t.str
 	if rt_type not in ctypes:
-		raise Exception(
+		raise TypeError(
 		    "Return type '%s' not a valid type" % rt_type)
 
 	t = tl.get_token()
@@ -563,11 +580,11 @@ def parse_func(tl, rt_type = None, obj=None):
 	if obj != None and fname[0] == "." and is_c_name(fname[1:]):
 		fname = obj + fname
 	elif not is_c_name(fname):
-		raise Exception("Function name '%s' is illegal" % fname)
+		raise ParseError("Function name '%s' is illegal", fname)
 
 	t = tl.get_token()
 	if t.str != "(":
-		raise Exception("Expected \"(\" got \"%s\"", t.str)
+		raise ParseError("Expected \"(\" got \"%s\"", t.str)
 
 	t = None
 	while True:
@@ -582,7 +599,7 @@ def parse_func(tl, rt_type = None, obj=None):
 		elif t.str == ")":
 			break
 		else:
-			raise Exception("ARG? %s" % t.str)
+			raise Exception("ARG? %s", t.str)
 		t = tl.get_token()
 		if is_c_name(t.str):
 			al[-1].nam = t.str
@@ -592,10 +609,10 @@ def parse_func(tl, rt_type = None, obj=None):
 		elif t.str == ")":
 			break
 		else:
-			raise Exception(
-			    "Expceted \")\" or \",\" not \"%s\"" % t.str)
+			raise ParseError(
+			    "Expected \")\" or \",\" not \"%s\"" % t.str)
 	if t.str != ")":
-		raise Exception("End Of Input looking for ')'")
+		raise ParseError("End Of Input looking for ')'")
 	f = func(fname, rt_type, al)
 
 	return f
@@ -612,7 +629,7 @@ def parse_obj(tl):
 
 
 #######################################################################
-# A section of the specfile, starting at a keyword
+# A section of the inputvcc, starting at a keyword
 
 class file_section(object):
 	def __init__(self):
@@ -633,7 +650,7 @@ class file_section(object):
 		return None
 
 	def more_tokens(self):
-		ln,l = self.l.pop(0)
+		ln, l = self.l.pop(0)
 		if l == "":
 			return
 		l = re.sub("[ \t]*#.*$", "", l)
@@ -669,19 +686,23 @@ class file_section(object):
 			vx.append(o)
 		elif t.str == "$Method":
 			if len(vx) != 2:
-				raise Exception("$Method outside $Object")
-			o = parse_func(self, obj = vx[1].nam)
+				raise FormatError("$Method outside $Object", "")
+			o = parse_func(self, obj=vx[1].nam)
 			vx[1].add_method(o)
 		else:
-			raise Exception("Unknown keyword: " + t.str)
+			raise FormatError("Unknown keyword: %s" % t.str, "")
+
 		assert len(self.tl) == 0
-		if o == None:
-			print("Warning:")
-			print("%s description is not included in .rst:" %t0)
-			for ln,i in self.l:
-				print("\t", i)
+		if o is None and len(self.l) > 0:
+			m = "%s description is not included in .rst" % t0
+			details = pformat(self.l)
+			if opts.strict:
+				raise FormatError(m, details)
+			else:
+				print("WARNING: %s:" % m, file=sys.stderr)
+				print(details, file=sys.stderr)
 		else:
-			for ln,i in self.l:
+			for ln, i in self.l:
 				o.doc(i)
 
 #######################################################################
@@ -704,86 +725,104 @@ def polish(l):
 		return True
 	return False
 
-#######################################################################
-# Read the file in
-
-f = open(specfile, "r")
-lines = []
-for i in f:
-	lines.append(i.rstrip())
-f.close()
-ln = 0
 
-#######################################################################
-# First collect the copyright:  All initial lines starting with '#'
+class SimpleTestCase(unittest.TestCase):
+	def test_included_vccs(self):
+		from tempfile import mktemp
+		from glob import glob
+		tmpfile = mktemp()
+		for inputfile in glob(dirname(realpath(__file__)) + "../libvmod_*/vmod.vcc"):
+			runmain(inputvcc, outputname=tmpfile)
+			for suffix in [".c", ".h"]:
+				unlink(tmpfile + suffix)
 
-copyright = []
-while len(lines[0]) > 0 and lines[0][0] == "#":
-	ln += 1
-	copyright.append(lines.pop(0))
-
-if len(copyright) > 0:
-	if copyright[0] == "#-":
-		copyright = [ ]
-	else:
-		while polish(copyright):
-			continue
-
-if False:
-	for i in copyright:
-		print("(C)\t", i)
 
 #######################################################################
-# Break into sections
-
-keywords = {
-	"$Module":	True,
-	"$Function":	True,
-	"$Object":	True,
-	"$Method":	True,
-	"$Init":	True,
-}
-
-sl = []
-sc = file_section()
-sl.append(sc)
-while len(lines) > 0:
-	ln += 1
-	l = lines.pop(0)
-	j = l.split()
-	if len(j) > 0 and j[0] in keywords:
-		sc = file_section()
-		sl.append(sc)
-	sc.add_line(ln,l)
-
-#######################################################################
-# Parse each section
-
-first = True
-
-vx = []
-for i in sl:
-	i.parse(vx)
-	assert len(i.tl) == 0
-
-#######################################################################
-# Parsing done, now process
-#
-
-fc = open("vcc_if.c", "w")
-fh = open("vcc_if.h", "w")
-
-file_header(fc)
-file_header(fh)
-
-fh.write('struct vrt_ctx;\n')
-fh.write('struct VCL_conf;\n')
-fh.write('struct vmod_priv;\n')
-fh.write("\n");
-
-vx[0].c_proto(fh)
-
-fc.write("""#include "config.h"
+def runmain(inputvcc, outputname="vcc_if"):
+	# Read the file in
+	lines = []
+	with open(inputvcc, "r") as fp:
+		for i in fp:
+			lines.append(i.rstrip())
+	ln = 0
+
+	#######################################################################
+	# First collect the copyright:  All initial lines starting with '#'
+
+	copyright = []
+	while len(lines[0]) > 0 and lines[0][0] == "#":
+		ln += 1
+		copyright.append(lines.pop(0))
+
+	if len(copyright) > 0:
+		if copyright[0] == "#-":
+			copyright = []
+		else:
+			while polish(copyright):
+				continue
+
+	if False:
+		for i in copyright:
+			print("(C)\t", i)
+
+	#######################################################################
+	# Break into sections
+
+	keywords = {
+		"$Module":	True,
+		"$Function":	True,
+		"$Object":	True,
+		"$Method":	True,
+		"$Init":	True,
+	}
+
+	sl = []
+	sc = file_section()
+	sl.append(sc)
+	while len(lines) > 0:
+		ln += 1
+		l = lines.pop(0)
+		j = l.split()
+		if len(j) > 0 and j[0] in keywords:
+			sc = file_section()
+			sl.append(sc)
+		sc.add_line(ln, l)
+
+	#######################################################################
+	# Parse each section
+
+	try:
+		vx = []
+		for i in sl:
+			i.parse(vx)
+			assert len(i.tl) == 0
+	except ParseError as e:
+		print("ERROR: Parse error reading \"%s\":" % inputvcc)
+		pprint(str(e))
+		exit(-1)
+	except FormatError as e:
+		print("ERROR: Format error reading \"%s\": %s" % (inputvcc, pformat(e.msg)))
+		print(e.details)
+		exit(-2)
+
+	#######################################################################
+	# Parsing done, now process
+	#
+
+	fc = open("%s.c" % outputname, "w")
+	fh = open("%s.h" % outputname, "w")
+
+	write_file_header(fc)
+	write_file_header(fh)
+
+	fh.write('struct vrt_ctx;\n')
+	fh.write('struct VCL_conf;\n')
+	fh.write('struct vmod_priv;\n')
+	fh.write("\n")
+
+	vx[0].c_proto(fh)
+
+	fc.write("""#include "config.h"
 
 #include "vrt.h"
 #include "vcc_if.h"
@@ -792,26 +831,55 @@ fc.write("""#include "config.h"
 
 """)
 
-vx[0].c_typedefs(fc)
-vx[0].c_vmod(fc)
-
-fc.close()
-fh.close()
-
-for suf in ("", ".man"):
-	with open("vmod_%s%s.rst" % (vx[0].nam, suf), "w") as fp:
-		fp.write("..\n")
-		fp.write(".. This file was autogenerated by %s. DO NOT EDIT!\n" %
-			 basename(__file__))
-		fp.write("..\n\n")
-
-		vx[0].doc_dump(fp, suf)
-
-		if len(copyright) > 0:
-			fp.write("\n")
-			fp.write("COPYRIGHT\n")
-			fp.write("=========\n")
-			fp.write("\n::\n\n")
-			for i in copyright:
-				fp.write("  %s\n" % i)
-			fp.write("\n")
+	vx[0].c_typedefs(fc)
+	vx[0].c_vmod(fc)
+
+	fc.close()
+	fh.close()
+
+	for suf in ("", ".man"):
+		with open("vmod_%s%s.rst" % (vx[0].nam, suf), "w") as fp:
+			fp.write("..\n")
+			fp.write(".. This file was autogenerated by %s. DO NOT EDIT!\n" %
+				 basename(__file__))
+			fp.write("..\n\n")
+
+			vx[0].doc_dump(fp, suf)
+
+			if len(copyright) > 0:
+				fp.write("\n")
+				fp.write("COPYRIGHT\n")
+				fp.write("=========\n")
+				fp.write("\n::\n\n")
+				for i in copyright:
+					fp.write("  %s\n" % i)
+				fp.write("\n")
+
+
+if __name__ == "__main__":
+	usagetext = "Usage: %prog [options] <vmod.vcc>"
+	oparser = optparse.OptionParser(usage=usagetext)
+
+	oparser.add_option('-N', '--strict', action='store_true', default=False,
+			    help="Be strict when parsing input file. (vmod.vcc)")
+	oparser.add_option('', '--runtests', action='store_true', default=False,
+			    dest="runtests", help=optparse.SUPPRESS_HELP)
+	(opts, args) = oparser.parse_args()
+
+	if opts.runtests:
+		del sys.argv[1]  # Pop off --runtests, pass remaining to unittest.
+		unittest.main()
+		exit()
+
+	inputvcc = None
+	if len(args) == 1 and exists(args[0]):
+		inputvcc = args[0]
+	elif exists("vmod.vcc"):
+		if not inputvcc:
+		    inputvcc = "vmod.vcc"
+	else:
+		print("ERROR: No vmod.vcc file supplied or found.", file=sys.stderr)
+		oparser.print_help()
+		exit(-1)
+
+	runmain(inputvcc)



From lkarsten at varnish-software.com  Tue May 13 16:08:11 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 18:08:11 +0200
Subject: [master] 92c9466 Remove unused description.
Message-ID: <E1WkFFT-0008Bn-Ru@project.varnish-software.com>


commit 92c94665e8d63ff15b97dcd5940c2c9009596f1d
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 18:04:21 2014 +0200

    Remove unused description.
    
    This description isn't used anywhere, and it doesn't contain
    much useful information.

diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 71a98d6..6872775 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -33,10 +33,8 @@ DESCRIPTION
 This vmod is used to develop, test and debug the various aspects
 of VMOD handling in Varnish.
 
-$Init init_function
-
-Call this whenever a VCL is loaded which imports this vmod.
 
+$Init init_function
 $Function VOID panic(STRING_LIST)
 
 Don't.



From lkarsten at varnish-software.com  Tue May 13 16:08:11 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 13 May 2014 18:08:11 +0200
Subject: [master] 30f6671 Use vmodtool.py in strict mode by default.
Message-ID: <E1WkFFT-0008Bw-TF@project.varnish-software.com>


commit 30f66716781e8af2c45ad06505fc0ca3823fed5a
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 18:06:06 2014 +0200

    Use vmodtool.py in strict mode by default.
    
    Use the strict parsing mode (still infant) to be sure
    that our own vmod.vcc files are valid.

diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am
index f856dde..d189102 100644
--- a/lib/libvmod_debug/Makefile.am
+++ b/lib/libvmod_debug/Makefile.am
@@ -9,6 +9,8 @@ AM_CPPFLAGS = \
 vmoddir = $(pkglibdir)/vmods
 vmod_srcdir = $(top_srcdir)/lib/libvmod_debug
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
+vmodtoolargs = --strict
+
 noinst_LTLIBRARIES = libvmod_debug.la
 
 libvmod_debug_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -shared -rpath /nowhere
@@ -25,7 +27,7 @@ nodist_libvmod_debug_la_SOURCES = \
 vmod_debug.lo vmod_debug_obj.lo: vcc_if.h
 
 vcc_if.c vcc_if.h vmod_debug.rst vmod_debug.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(vmod_srcdir)/vmod.vcc
 
 EXTRA_DIST = vmod.vcc
 
diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am
index 583b1f7..64b72a8 100644
--- a/lib/libvmod_directors/Makefile.am
+++ b/lib/libvmod_directors/Makefile.am
@@ -9,6 +9,7 @@ AM_CPPFLAGS = \
 vmoddir = $(pkglibdir)/vmods
 vmod_srcdir = $(top_srcdir)/lib/libvmod_directors
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
+vmodtoolargs = --strict
 vmod_LTLIBRARIES = libvmod_directors.la
 
 libvmod_directors_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -shared
@@ -29,7 +30,8 @@ nodist_libvmod_directors_la_SOURCES = \
 vdir.lo fall_back.lo hash.lo random.lo round_robin.lo: vcc_if.h
 
 vcc_if.c vcc_if.h vmod_directors.rst vmod_directors.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(vmod_srcdir)/vmod.vcc
+
 
 EXTRA_DIST = vmod.vcc
 
diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index ff8466e..c005b85 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -10,6 +10,7 @@ AM_CPPFLAGS = \
 vmoddir = $(pkglibdir)/vmods
 vmod_srcdir = $(top_srcdir)/lib/libvmod_std
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
+vmodtoolargs = --strict
 vmod_LTLIBRARIES = libvmod_std.la
 
 libvmod_std_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -shared
@@ -27,7 +28,8 @@ nodist_libvmod_std_la_SOURCES = \
 vmod_std.lo vmod_std_fileread.lo vmod_std_conversions.lo: vcc_if.h
 
 vcc_if.c vcc_if.h vmod_std.rst vmod_std.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(vmod_srcdir)/vmod.vcc
+
 
 EXTRA_DIST = vmod.vcc
 



From lkarsten at varnish-software.com  Wed May 14 08:21:09 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Wed, 14 May 2014 10:21:09 +0200
Subject: [master] bbf9d33 Walk back Python 3 changes, use python2 explicitly.
Message-ID: <E1WkUR3-0000MJ-9A@project.varnish-software.com>


commit bbf9d33084fef3070b56e207582714baef924fcf
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed May 14 10:15:09 2014 +0200

    Walk back Python 3 changes, use python2 explicitly.
    
    None of our automatic build environments have Python 3 currently.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 3319199..4cc13e1 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python2
 #-
 # Copyright (c) 2010-2014 Varnish Software AS
 # All rights reserved.
@@ -699,8 +699,8 @@ class file_section(object):
 			if opts.strict:
 				raise FormatError(m, details)
 			else:
-				print("WARNING: %s:" % m, file=sys.stderr)
-				print(details, file=sys.stderr)
+				print >>sys.stderr, "WARNING: %s:" % m
+				print >>sys.stderr, details
 		else:
 			for ln, i in self.l:
 				o.doc(i)
@@ -797,12 +797,12 @@ def runmain(inputvcc, outputname="vcc_if"):
 			i.parse(vx)
 			assert len(i.tl) == 0
 	except ParseError as e:
-		print("ERROR: Parse error reading \"%s\":" % inputvcc)
+		print "ERROR: Parse error reading \"%s\":" % inputvcc
 		pprint(str(e))
 		exit(-1)
 	except FormatError as e:
-		print("ERROR: Format error reading \"%s\": %s" % (inputvcc, pformat(e.msg)))
-		print(e.details)
+		print "ERROR: Format error reading \"%s\": %s" % (inputvcc, pformat(e.msg))
+		print e.details
 		exit(-2)
 
 	#######################################################################
@@ -878,7 +878,7 @@ if __name__ == "__main__":
 		if not inputvcc:
 		    inputvcc = "vmod.vcc"
 	else:
-		print("ERROR: No vmod.vcc file supplied or found.", file=sys.stderr)
+		print >>sys.stderr, "ERROR: No vmod.vcc file supplied or found."
 		oparser.print_help()
 		exit(-1)
 



From lkarsten at varnish-software.com  Wed May 14 08:21:09 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Wed, 14 May 2014 10:21:09 +0200
Subject: [master] babbaa5 Fix glob that found no input files.
Message-ID: <E1WkUR3-0000MS-Ag@project.varnish-software.com>


commit babbaa5318027849e423ed4ed2d9f62fcc0bd8fd
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed May 14 10:19:34 2014 +0200

    Fix glob that found no input files.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 4cc13e1..6e875bf 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -731,8 +731,9 @@ class SimpleTestCase(unittest.TestCase):
 		from tempfile import mktemp
 		from glob import glob
 		tmpfile = mktemp()
-		for inputfile in glob(dirname(realpath(__file__)) + "../libvmod_*/vmod.vcc"):
-			runmain(inputvcc, outputname=tmpfile)
+		bdir = dirname(realpath(__file__))
+		for inputfile in glob(bdir + "/../libvmod_*/vmod.vcc"):
+			runmain(inputfile, outputname=tmpfile)
 			for suffix in [".c", ".h"]:
 				unlink(tmpfile + suffix)
 



From lkarsten at varnish-software.com  Wed May 14 08:31:21 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Wed, 14 May 2014 10:31:21 +0200
Subject: [master] 45362c9 Support both python2 and python3.
Message-ID: <E1WkUav-0001GY-9E@project.varnish-software.com>


commit 45362c9c1ad0aa2b949d5aa49c2c8fa39d363565
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed May 14 10:26:12 2014 +0200

    Support both python2 and python3.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 6e875bf..25e0624 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 #-
 # Copyright (c) 2010-2014 Varnish Software AS
 # All rights reserved.
@@ -37,6 +37,9 @@
 # as a string, suitable for inclusion in the C-source of the compile VCL
 # program.
 
+# This script should work with both Python 2 and Python 3.
+from __future__ import print_function
+
 import sys
 import re
 import optparse
@@ -45,7 +48,6 @@ from os import unlink
 from os.path import dirname, basename, realpath, exists
 from pprint import pprint, pformat
 
-
 ctypes = {
 	'BACKEND':	"VCL_BACKEND",
 	'BOOL':		"VCL_BOOL",
@@ -89,7 +91,7 @@ class FormatError(Exception):
 	"""
 		Raised if the content of the (otherwise well-formed) input file
 		is invalid.
-  """
+	"""
 	def __init__(self, msg, details):
 		self.msg = msg
 		self.details = details
@@ -699,8 +701,8 @@ class file_section(object):
 			if opts.strict:
 				raise FormatError(m, details)
 			else:
-				print >>sys.stderr, "WARNING: %s:" % m
-				print >>sys.stderr, details
+				print("WARNING: %s:" % m, file=sys.stderr)
+				print(details, file=sys.stderr)
 		else:
 			for ln, i in self.l:
 				o.doc(i)
@@ -798,12 +800,12 @@ def runmain(inputvcc, outputname="vcc_if"):
 			i.parse(vx)
 			assert len(i.tl) == 0
 	except ParseError as e:
-		print "ERROR: Parse error reading \"%s\":" % inputvcc
+		print("ERROR: Parse error reading \"%s\":" % inputvcc)
 		pprint(str(e))
 		exit(-1)
 	except FormatError as e:
-		print "ERROR: Format error reading \"%s\": %s" % (inputvcc, pformat(e.msg))
-		print e.details
+		print("ERROR: Format error reading \"%s\": %s" % (inputvcc, pformat(e.msg)))
+		print(e.details)
 		exit(-2)
 
 	#######################################################################
@@ -879,7 +881,7 @@ if __name__ == "__main__":
 		if not inputvcc:
 		    inputvcc = "vmod.vcc"
 	else:
-		print >>sys.stderr, "ERROR: No vmod.vcc file supplied or found."
+		print("ERROR: No vmod.vcc file supplied or found.", file=sys.stderr)
 		oparser.print_help()
 		exit(-1)
 



From lkarsten at varnish-software.com  Thu May 15 14:37:17 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 15 May 2014 16:37:17 +0200
Subject: [master] 1929299 Load balancing example should use new syntax.
Message-ID: <E1Wkwmb-0006Bz-OC@project.varnish-software.com>


commit 192929994b4b4d72de3ba5c3085fc196c2bb0c1b
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu May 15 16:34:03 2014 +0200

    Load balancing example should use new syntax.
    
    One of the examples was still referring to the old way of defining
    directors, which I am sure is confusing for the new users this
    document is meant for.
    
    Noticed by: coredump on irc.
    
    Used the occation to do some minor scrubbing of the text.

diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index be50e1c..4a83b75 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -71,7 +71,7 @@ Now we need tell Varnish where to send the difference URL. Lets look at `vcl_rec
 It's quite simple, really. Lets stop and think about this for a
 moment. As you can see you can define how you choose backends based on
 really arbitrary data. You want to send mobile devices to a different
-backend? No problem. ``if (req.User-agent ~ /mobile/) ..`` should do the
+backend? No problem. ``if (req.http.User-agent ~ /mobile/) ..`` should do the
 trick.
 
 
@@ -79,7 +79,7 @@ Backends and virtual hosts in Varnish
 -------------------------------------
 
 Varnish fully supports virtual hosts. They might however work in a somewhat
-counter intuitive fashion since they are never declared
+counter-intuitive fashion since they are never declared
 explicitly. You set up the routing of incoming HTTP requests in
 `vcl_recv`. If you want this routing to be done on the basis of virtual
 hosts you just need to inspect `req.http.host`.
@@ -155,8 +155,8 @@ Checks come into play.
 Health checks
 -------------
 
-Lets set up a director with two backends and health checks. First lets
-define the backends.::
+Lets set up a director with two backends and health checks. First let
+us define the backends.::
 
     backend server1 {
         .host = "server1.example.com";
@@ -204,29 +204,28 @@ initial
     Defaults to the same amount as the threshold.
 
 
-.. XXX: Where and why? benc
-
 Now we define the 'director'::
 
+
     import directors;
 
-    director example_director round-robin {
-        { .backend = server1; }
-        { .backend = server2; }
+    sub vcl_init {
+        new vdir = directors.round_robin();
+        vdir.add_backend(backend1);
+        vdir.add_backend(backend2);
     }
 
 
-You use this director just as you would use any other director or
-backend. Varnish will not send traffic to hosts that are marked as
-unhealthy. Varnish can also serve stale content if all the backends are
+You use this director as a backend_hint for requests, just like you would
+with a simple backend. Varnish will not send traffic to hosts that are marked as
+unhealthy.
+
+Varnish can also serve stale content if all the backends are
 down. See :ref:`users-guide-handling_misbehaving_servers` for more
 information on how to enable this.
 
-Please note that Varnish will keep probes active for all loaded VCLs. Varnish
+Please note that Varnish will keep health probes running for all loaded VCLs. Varnish
 will coalesce probes that seem identical - so be careful not to change the
 probe config if you do a lot of VCL loading. Unloading the VCL will discard the
-probes.
-
-For more information on how to do this please see
-ref:`reference-vcl-director`.
+probes. For more information on how to do this please see ref:`reference-vcl-director`.
 



From lkarsten at varnish-software.com  Thu May 15 14:47:46 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 15 May 2014 16:47:46 +0200
Subject: [master] 52a6e58 Use consistent naming through the example.
Message-ID: <E1Wkwwk-0002eU-JZ@project.varnish-software.com>


commit 52a6e58600f31378251a250fe1954769587b4fc7
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu May 15 16:47:09 2014 +0200

    Use consistent naming through the example.

diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index 4a83b75..ecb8b31 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -211,12 +211,12 @@ Now we define the 'director'::
 
     sub vcl_init {
         new vdir = directors.round_robin();
-        vdir.add_backend(backend1);
-        vdir.add_backend(backend2);
+        vdir.add_backend(server1);
+        vdir.add_backend(server2);
     }
 
 
-You use this director as a backend_hint for requests, just like you would
+You use this `vdir` director as a backend_hint for requests, just like you would
 with a simple backend. Varnish will not send traffic to hosts that are marked as
 unhealthy.
 



From martin at varnish-software.com  Fri May 16 09:21:50 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 16 May 2014 11:21:50 +0200
Subject: [3.0] af56dab Fix up function declarations to fix compile errors on
 clang
Message-ID: <E1WlEKs-0006Ll-On@project.varnish-software.com>


commit af56daba68c028ac59550eb5314287f2188a2137
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri May 16 11:20:58 2014 +0200

    Fix up function declarations to fix compile errors on clang

diff --git a/bin/varnishd/cache_backend.h b/bin/varnishd/cache_backend.h
index b16d51e..a3041d0 100644
--- a/bin/varnishd/cache_backend.h
+++ b/bin/varnishd/cache_backend.h
@@ -153,9 +153,10 @@ void VBE_DropRefVcl(struct backend *);
 void VBE_DropRefLocked(struct backend *b);
 
 /* cache_backend_poll.c */
-void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, const char *hosthdr);
-void VBP_Remove(struct backend *b, struct vrt_backend_probe const *p);
-void VBP_Use(const struct backend *b, const struct vrt_backend_probe const *p);
+void VBP_Insert(struct backend *b, const struct vrt_backend_probe *p,
+    const char *hosthdr);
+void VBP_Remove(struct backend *b, const struct vrt_backend_probe *p);
+void VBP_Use(const struct backend *b, const struct vrt_backend_probe *p);
 void VBP_Summary(struct cli *cli, const struct vbp_target *vt);
 
 /* Init functions for directors */
diff --git a/bin/varnishd/cache_backend_poll.c b/bin/varnishd/cache_backend_poll.c
index ce00a67..4995a57 100644
--- a/bin/varnishd/cache_backend_poll.c
+++ b/bin/varnishd/cache_backend_poll.c
@@ -475,7 +475,8 @@ vbp_new_vcl(const struct vrt_backend_probe *p, const char *hosthdr)
  */
 
 void
-VBP_Insert(struct backend *b, const struct vrt_backend_probe *p, const char *hosthdr)
+VBP_Insert(struct backend *b, const struct vrt_backend_probe *p,
+    const char *hosthdr)
 {
 	struct vbp_target *vt;
 	struct vbp_vcl *vcl;
@@ -542,7 +543,7 @@ VBP_Use(const struct backend *b, const struct vrt_backend_probe *p)
 }
 
 void
-VBP_Remove(struct backend *b, struct vrt_backend_probe const *p)
+VBP_Remove(struct backend *b, const struct vrt_backend_probe *p)
 {
 	struct vbp_target *vt;
 	struct vbp_vcl *vcl;



From lkarsten at varnish-software.com  Fri May 16 09:30:57 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 16 May 2014 11:30:57 +0200
Subject: [3.0] 2e4b131 Fix two minor typographical errors.
Message-ID: <E1WlETh-0007Rn-CP@project.varnish-software.com>


commit 2e4b131371ac8bbf88ece2e41b42077a9b98de7a
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri May 16 11:30:25 2014 +0200

    Fix two minor typographical errors.

diff --git a/doc/sphinx/tutorial/handling_misbehaving_servers.rst b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
index 406b4b3..88eb423 100644
--- a/doc/sphinx/tutorial/handling_misbehaving_servers.rst
+++ b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
@@ -88,13 +88,13 @@ Known limitations on grace- and saint mode
 If your request fails while it is being fetched you're thrown into
 vcl_error. vcl_error has access to a rather limited set of data so you
 can't enable saint mode or grace mode here. This will be addressed in a
-future release but a work-around available.
+future release but a workaround available.
 
 * Declare a backend that is always sick.
 * Set a magic marker in vcl_error
 * Restart the transaction
 * Note the magic marker in vcl_recv and set the backend to the one mentioned
-* Varnish will now serve stale data is any is available
+* Varnish will now serve stale data if any is available
 
 
 God mode



From martin at varnish-software.com  Fri May 16 12:23:54 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 16 May 2014 14:23:54 +0200
Subject: [master] bba689d Truncate %D varnishncsa format specifier to an
 integer value.
Message-ID: <E1WlHB4-0004uf-NF@project.varnish-software.com>


commit bba689d2fa11223f765ac773d36fd7e0b7c63dc3
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri May 16 14:23:28 2014 +0200

    Truncate %D varnishncsa format specifier to an integer value.
    
    This follows the format Apache logs in.

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index e3f2432..2162118 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -276,7 +276,7 @@ format_time(const struct format *format)
 
 	switch (format->time_type) {
 	case 'D':
-		AZ(VSB_printf(CTX.vsb, "%f", (t_end - t_start) * 1e6));
+		AZ(VSB_printf(CTX.vsb, "%d", (int)((t_end - t_start) * 1e6)));
 		break;
 	case 't':
 		AN(format->time_fmt);



From martin at varnish-software.com  Fri May 16 12:24:27 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 16 May 2014 14:24:27 +0200
Subject: [3.0] 12f835b Format %D as microseconds in integer value
Message-ID: <E1WlHBb-000566-62@project.varnish-software.com>


commit 12f835be3772b1598b7e15b6c799f2cd13d784d3
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri May 16 14:19:47 2014 +0200

    Format %D as microseconds in integer value
    
    Do as we have documented we do (and Apache does), printing the %D as
    request time in microseconds, truncated to an integer value.

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 9bd2423..d941aae 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -665,7 +665,7 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 
 		case 'D':
 			/* %D */
-			VSB_printf(os, "%f", lp->df_D);
+			VSB_printf(os, "%d", (int)(lp->df_D * 1e6));
 			break;
 
 		case 'H':



From martin at varnish-software.com  Fri May 16 12:55:32 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 16 May 2014 14:55:32 +0200
Subject: [3.0] 8213a0b Don't medd up the t_open and t_resp when handling ESI
 subrequests.
Message-ID: <E1WlHfg-0008Q1-P0@project.varnish-software.com>


commit 8213a0b3c7a779ee505db7467ed7f2dadfd70012
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri May 16 14:53:26 2014 +0200

    Don't medd up the t_open and t_resp when handling ESI subrequests.
    
    ESI subrequests would mangle the t_open and t_resp timestamps of the
    original requests, causing negative time and NAN being logged in
    ReqEnd.
    
    Fixes: #1297

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index d0f3bf2..22edb9b 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -350,8 +350,6 @@ cnt_done(struct sess *sp)
 		    sp->xid, sp->t_req, sp->t_end, dh, dp, da);
 	}
 	sp->xid = 0;
-	sp->t_open = sp->t_end;
-	sp->t_resp = NAN;
 	WSL_Flush(sp->wrk, 0);
 
 	/* If we did an ESI include, don't mess up our state */
@@ -360,7 +358,9 @@ cnt_done(struct sess *sp)
 
 	memset(&sp->acct_req, 0, sizeof sp->acct_req);
 
+	sp->t_open = sp->t_end;
 	sp->t_req = NAN;
+	sp->t_resp = NAN;
 	sp->hash_always_miss = 0;
 	sp->hash_ignore_busy = 0;
 



From apj at mutt.dk  Fri May 16 15:53:36 2014
From: apj at mutt.dk (Andreas Plesner)
Date: Fri, 16 May 2014 17:53:36 +0200
Subject: [master] bc73592 Exclude vcl_var since it is included in vcl.rst.
Message-ID: <E1WlKS0-0007Pu-CS@project.varnish-software.com>


commit bc73592130bcbad5f30ad351809f11770320ac2c
Author: Andreas Plesner <apj at mutt.dk>
Date:   Fri May 16 17:51:34 2014 +0200

    Exclude vcl_var since it is included in vcl.rst.

diff --git a/doc/sphinx/conf.py.in b/doc/sphinx/conf.py.in
index 8104bb3..5e411b6 100644
--- a/doc/sphinx/conf.py.in
+++ b/doc/sphinx/conf.py.in
@@ -64,7 +64,7 @@ release = '@VERSION@'
 
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
-exclude_patterns = ['build','include/*.rst']
+exclude_patterns = ['build','include/*.rst','reference/vcl_var.rst']
 
 # The reST default role (used for this markup: `text`) to use for all documents.
 #default_role = None



From apj at mutt.dk  Fri May 16 15:53:36 2014
From: apj at mutt.dk (Andreas Plesner)
Date: Fri, 16 May 2014 17:53:36 +0200
Subject: [master] 419d15f Add missing returns, and do a bit of cleanup
Message-ID: <E1WlKS0-0007Q0-DY@project.varnish-software.com>


commit 419d15fadd06470ce46c7e4c9883469ba7f748eb
Author: Andreas Plesner <apj at mutt.dk>
Date:   Fri May 16 17:53:15 2014 +0200

    Add missing returns, and do a bit of cleanup

diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst
index 0a90a34..cb6396e 100644
--- a/doc/sphinx/users-guide/vcl-built-in-subs.rst
+++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst
@@ -22,8 +22,8 @@ yourself doing frequently.
 The `vcl_recv` subroutine may terminate with calling ``return()`` on one
 of the following keywords:
 
-  synth 
-    Return a synthetic object with the specified error code to the
+  synth(status code, reason)
+    Return a synthetic object with the specified status code to the
     client and abandon the request.
 
   pass
@@ -52,8 +52,8 @@ shuffling bytes back and forth.
 The `vcl_pipe` subroutine may terminate with calling ``return()`` with one
 of the following keywords:
 
-  synth(error code, reason)
-    Return the specified error code to the client and abandon the request.
+  synth(status code, reason)
+    Return the specified status code to the client and abandon the request.
 
   pipe
     Proceed with pipe mode.
@@ -69,8 +69,8 @@ submitted over the same client connection are handled normally.
 The `vcl_pass` subroutine may terminate with calling ``return()`` with one
 of the following keywords:
 
-  synth(error code, reason)
-    Return the specified error code to the client and abandon the request.
+  synth(status code, reason)
+    Return the specified status code to the client and abandon the request.
 
   pass
     Proceed with pass mode.
@@ -86,7 +86,7 @@ vcl_hit
 
 Called when a cache lookup is successful. 
 
-.. XXX: missing the "The `vcl_hit` subroutine may terminate with calling ``return()`` with one of the following keywords:" thing. benc
+The `vcl_hit` subroutine may terminate with calling ``return()`` with one of the following keywords:
 
 
   restart
@@ -97,8 +97,8 @@ Called when a cache lookup is successful.
   deliver
     Deliver the object. Control passes to `vcl_deliver`.
 
-  synth(error code, reason)
-    Return the specified error code to the client and abandon the request.
+  synth(status code, reason)
+    Return the specified status code to the client and abandon the request.
 
 
 vcl_miss
@@ -111,8 +111,8 @@ retrieve the document from the backend, and which backend to use.
 The `vcl_miss` subroutine may terminate with calling ``return()`` with one
 of the following keywords:
 
-  synth(error code, reason)
-    Return the specified error code to the client and abandon the request.
+  synth(status code, reason)
+    Return the specified status code to the client and abandon the request.
 
   pass
     Switch to pass mode. Control will eventually pass to `vcl_pass`.
@@ -121,38 +121,51 @@ of the following keywords:
     Retrieve the requested object from the backend. Control will
     eventually pass to `vcl_backend_fetch`.
 
+  restart
+    Restart the transaction. Increases the restart counter. If the number
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation
+    error.
+
+
 vcl_hash
 ~~~~~~~~
 
 Called after `vcl_recv` to create a hash value for the request. This is
 used as a key to look up the object in Varnish.
 
+The `vcl_hash` subroutine may terminate with calling ``return()`` with one
+of the following keywords:
+
   lookup
     Look up the object in cache. Control passes to vcl_miss, vcl_hit
     or vcl_purge.
 
 
-
-
 vcl_purge
 ~~~~~~~~~
 
 Called after the purge has been executed and all its variants have been evited.
 
+The `vcl_purge` subroutine may terminate with calling ``return()`` with one
+of the following keywords:
+
   synth
     Produce a response.
 
+  restart
+    Restart the transaction. Increases the restart counter. If the number
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation
+    error.
+
 
 vcl_deliver
 ~~~~~~~~~~~
 
 Called before a cached object is delivered to the client.
 
-The ``vcl_deliver`` subroutine may terminate calling ``return()`` with one
+The `vcl_deliver` subroutine may terminate with calling ``return()`` with one
 of the following keywords:
 
-.. XXX: Should perhaps be return as above? benc
-
   deliver
     Deliver the object to the client.
 
@@ -168,7 +181,8 @@ vcl_backend_fetch
 Called before sending the backend request. In this subroutine you
 typically alter the request before it gets to the backend.
 
-.. XXX: Missing terminate..keywords sentence? benc
+The `vcl_backend_fetch` subroutine may terminate with calling
+``return()`` with one of the following keywords:
 
   fetch
     Fetch the object from the backend.
@@ -180,71 +194,40 @@ typically alter the request before it gets to the backend.
 vcl_backend_response
 ~~~~~~~~~~~~~~~~~~~~
 
-Called after a response has been successfully retrieved from the
-backend. The response is available as `beresp`. 
-
-.. XXX: beresp comes out of the blue here. maybe a short description? benc
-
-Note that Varnish might
-not be talking to an actual client, so operations that require a
-client to be present are not allowed. Specifically there is no `req
-object` and restarts are not allowed.
+Called after the response headers has been successfully retrieved from
+the backend.
 
-.. XXX: I do not follow sentence above. benc
-
-The `vcl_backend_response` subroutine may terminate with calling ``return()`` with one
-of the following keywords:
+The `vcl_backend_response` subroutine may terminate with calling
+``return()`` with one of the following keywords:
 
   deliver
     Possibly insert the object into the cache, then deliver it to the
     Control will eventually pass to `vcl_deliver`. Caching is dependant
     on 'beresp.cacheable'.
 
-.. XXX:A parameter? that is set how? benc
-    
-
-  error(error code, reason)
-    Return the specified error code to the client and abandon the request.
+  abandon
+    Abandon the backend request and generates an error.
 
   retry
-    Retry the backend transaction. Increases the `retries` counter. If the number
-    of retries is higher than *max_retries* Varnish emits a guru meditation
-    error.
+    Retry the backend transaction. Increases the `retries` counter.
+    If the number of retries is higher than *max_retries* Varnish
+    emits a guru meditation error.
 
 vcl_backend_error
 ~~~~~~~~~~~~~~~~~
 
 This subroutine is called if we fail the backend fetch. 
 
-.. XXX:Missing the terminate return structure? benc
+The `vcl_backend_error` subroutine may terminate with calling ``return()``
+with one of the following keywords:
 
   deliver
     Deliver the error.
 
   retry
-    Retry the backend transaction. Increases the `retries` counter. If the number
-    of retries is higher than *max_retries* Varnish emits a guru meditation
-    error.
-
-
-vcl_backend_error
-~~~~~~~~~~~~~~~~~
-
-.. XXX: Same name as section above? benc
-
-Called when we hit an error, either explicitly or implicitly due to
-backend or internal errors.
-
-The `vcl_backend_error` subroutine may terminate by calling ``return()`` with one of
-the following keywords:
-
-  deliver
-    Deliver the error object to the client.
-
-  retry
-    Retry the backend transaction. Increases the retries counter. If the number
-    of retries is higher than *max_retries* Varnish emits a guru meditation
-    error.
+    Retry the backend transaction. Increases the `retries` counter. If
+    the number of retries is higher than *max_retries* Varnish emits a
+    guru meditation error.
 
 
 vcl_init
@@ -253,9 +236,8 @@ vcl_init
 Called when VCL is loaded, before any requests pass through it.
 Typically used to initialize VMODs.
 
-.. XXX: Missing the terminate return structure? benc
-
-  ``return()`` values:
+The `vcl_init` subroutine may terminate with calling ``return()``
+with one of the following keywords:
 
   ok
     Normal return, VCL continues loading.
@@ -267,13 +249,8 @@ vcl_fini
 Called when VCL is discarded only after all requests have exited the VCL.
 Typically used to clean up VMODs.
 
-
-.. XXX: Missing the terminate return structure? benc
-
-  ``return()`` values:
+The `vcl_fini` subroutine may terminate with calling ``return()``
+with one of the following keywords:
 
   ok
     Normal return, VCL will be discarded.
-
-
-.. XXX: Maybe end here with the detailed flowchart from the book together with a reference to the book? benc



From daghf at varnish-software.com  Fri May 16 18:37:25 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Fri, 16 May 2014 20:37:25 +0200
Subject: [master] f599976 req.backend -> req.backend_hint
Message-ID: <E1WlN0X-0008OX-Qc@project.varnish-software.com>


commit f5999765e77edc182846c99c59286aa443b87bf6
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Fri May 16 20:37:14 2014 +0200

    req.backend -> req.backend_hint

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index c8ed011..f48548f 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -124,8 +124,8 @@ vcl_* reserved
 Any custom-made subs cannot be named 'vcl_*' anymore. This namespace
 is reserved for builtin subs.
 
-req.backend.healthy replaced by std.healthy(req.backend)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+req.backend.healthy replaced by std.healthy(req.backend_hint)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Remember to import the std module if you're not doing so already.
 



From phk at FreeBSD.org  Mon May 19 06:47:00 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 19 May 2014 08:47:00 +0200
Subject: [master] d3b8f34 Forgot to commit the test-case
Message-ID: <E1WmHLg-0001FO-GQ@project.varnish-software.com>


commit d3b8f34ba42db8772ac7ec468d6ffbcaa6d97b9b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 19 06:45:31 2014 +0000

    Forgot to commit the test-case

diff --git a/bin/varnishtest/tests/r01478.vtc b/bin/varnishtest/tests/r01478.vtc
new file mode 100644
index 0000000..1df49a6
--- /dev/null
+++ b/bin/varnishtest/tests/r01478.vtc
@@ -0,0 +1,60 @@
+varnishtest "panic due to hash_ignore_busy"
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	delay .5
+	sema r1 sync 2
+	delay .5
+	chunkedlen 10
+	delay .5
+	sema r2 sync 2
+	delay .5
+	chunkedlen 10
+	delay .5
+	sema r3 sync 2
+	delay .5
+	chunkedlen 10
+	delay .5
+	sema r4 sync 2
+	delay .5
+	chunkedlen 10
+	delay .5
+	chunkedlen 0
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		set req.hash_ignore_busy         = true;
+		return(hash);
+	}
+	sub vcl_backend_response {
+		# we assume streaming for all objects as default:
+		set beresp.do_stream             = true;
+		set beresp.ttl = 2s;
+	}
+} -start
+
+client c1 {
+	txreq -hdr "client: c1"
+	rxresp
+} -start
+
+sema r1 sync 2
+client c2 {
+	txreq -hdr "client: c2"
+	sema r2 sync 2
+	rxresp
+} -start
+
+sema r3 sync 2
+client c3 {
+	txreq -hdr "client: c3"
+	sema r4 sync 2
+	rxresp
+} -start
+
+client c1 -wait
+client c2 -wait
+client c3 -wait



From phk at FreeBSD.org  Mon May 19 06:47:00 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 19 May 2014 08:47:00 +0200
Subject: [master] a1b74f7 Allow ACLs to be unreferenced when vcc_err_unref=off.
Message-ID: <E1WmHLg-0001FW-HW@project.varnish-software.com>


commit a1b74f7b57d8a95abbbc0fa74217a09f8f06a39f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 19 06:45:44 2014 +0000

    Allow ACLs to be unreferenced when vcc_err_unref=off.
    
    Add a reference from the init function to any non-anonymous match_acl_*()
    functions to ensure the compiler does not barf.
    
    Fixes	#1504

diff --git a/bin/varnishtest/tests/r01504.vtc b/bin/varnishtest/tests/r01504.vtc
new file mode 100644
index 0000000..0277c65
--- /dev/null
+++ b/bin/varnishtest/tests/r01504.vtc
@@ -0,0 +1,10 @@
+varnishtest "unreferenced acls"
+
+varnish v1 -arg "-p vcc_err_unref=off" -vcl {
+	backend s1 {
+		.host = "${bad_ip}";
+	}
+	acl foo {
+		"127.0.0.1";
+	}
+}
diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index 1ab670a..997ae3c 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -343,13 +343,14 @@ vcc_acl_entry(struct vcc *tl)
  */
 
 static void
-vcc_acl_emit(const struct vcc *tl, const char *acln, int anon)
+vcc_acl_emit(struct vcc *tl, const char *acln, int anon)
 {
 	struct acl_e *ae;
 	int depth, l, m, i;
 	unsigned at[VRT_ACL_MAXADDR + 1];
 	const char *oc;
 	struct token *t;
+	struct inifin *ifp;
 
 	Fh(tl, 0, "\nstatic int\n");
 	Fh(tl, 0,
@@ -364,6 +365,11 @@ vcc_acl_emit(const struct vcc *tl, const char *acln, int anon)
 	Fh(tl, 0, "\t\tVRT_acl_log(ctx, \"NO_FAM %s\");\n", acln);
 	Fh(tl, 0, "\t\treturn(0);\n");
 	Fh(tl, 0, "\t}\n\n");
+	if (!tl->err_unref && !anon ) {
+		ifp = New_IniFin(tl);
+		VSB_printf(ifp->ini,
+			"\tif (0) match_acl_named_%s(0, 0);\n", acln);
+	}
 	depth = -1;
 	oc = 0;
 	at[0] = 256;



From phk at FreeBSD.org  Mon May 19 07:15:42 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 19 May 2014 09:15:42 +0200
Subject: [master] 75802e2 Add a debug.no_backend() function to simulate a
 director failing to pick a backend.
Message-ID: <E1WmHnS-0004kj-GX@project.varnish-software.com>


commit 75802e279457d209e59bdd3d26aceb59bb214715
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 19 07:14:50 2014 +0000

    Add a debug.no_backend() function to simulate a director failing to
    pick a backend.

diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 6872775..8667298 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -59,6 +59,10 @@ $Function STRING blob2hex(BLOB)
 
 Hexdump a blob
 
+$Function BACKEND no_backend()
+
+Fails at backend selection
+
 $Object obj(STRING) 
 
 Test object
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index 146f46b..b3e8334 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -129,3 +129,12 @@ vmod_blob2hex(const struct vrt_ctx *ctx, VCL_BLOB b)
 	VRT_priv_fini(b);
 	return (s);
 }
+
+VCL_BACKEND
+vmod_no_backend(const struct vrt_ctx *ctx)
+{
+	
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	return (NULL);
+}
+



From phk at FreeBSD.org  Mon May 19 07:19:43 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 19 May 2014 09:19:43 +0200
Subject: [master] 01f0e71 Directors are allowed to fail to pick a backend.
Message-ID: <E1WmHrL-000522-6O@project.varnish-software.com>


commit 01f0e719f90bba1219426fa25abcd6cf3b68eebd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 19 07:19:17 2014 +0000

    Directors are allowed to fail to pick a backend.
    
    Fixes	#1501

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 0f49a3b..7ad709d 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -181,7 +181,6 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
 
 	assert(bo->state == BOS_INVALID);
-	AN(bo->director);
 	AZ(bo->vbc);
 	AZ(bo->should_close);
 	AZ(bo->storage_hint);
@@ -255,7 +254,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
-	AN(bo->director);
 	AZ(bo->vbc);
 	AZ(bo->should_close);
 	AZ(bo->storage_hint);
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index 4ddacb1..11c712e 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -300,6 +300,10 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	htc = &bo->htc;
 
+	if (bo->director == NULL) {
+		VSLb(bo->vsl, SLT_FetchError, "No backend");
+		return (-1);
+	}
 	AN(bo->director);
 
 	hp = bo->bereq;
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index b68031c..65c72db 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -270,9 +270,8 @@ VRT_r_beresp_backend_name(const struct vrt_ctx *ctx)
 		CHECK_OBJ_NOTNULL(ctx->bo->vbc, VBC_MAGIC);
 		return (ctx->bo->vbc->backend->vcl_name);
 	}
-	if (ctx->bo->director != NULL) {
+	if (ctx->bo->director != NULL)
 		return (ctx->bo->director->vcl_name);
-	}
 	return (NULL);
 }
 
@@ -350,7 +349,6 @@ VRT_l_bereq_backend(const struct vrt_ctx *ctx, struct director *be)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
-	AN(ctx->bo->director);
 	ctx->bo->director = be;
 }
 
@@ -360,7 +358,6 @@ VRT_r_bereq_backend(const struct vrt_ctx *ctx)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
-	AN(ctx->bo->director);
 	return (ctx->bo->director);
 }
 
diff --git a/bin/varnishtest/tests/r01501.vtc b/bin/varnishtest/tests/r01501.vtc
new file mode 100644
index 0000000..c7ce3b8
--- /dev/null
+++ b/bin/varnishtest/tests/r01501.vtc
@@ -0,0 +1,21 @@
+varnishtest "director fails to pick backend"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+
+	import ${vmod_debug};
+
+	sub vcl_recv {
+		set req.backend_hint = debug.no_backend();
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+} -run



From phk at FreeBSD.org  Mon May 19 07:21:31 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 19 May 2014 09:21:31 +0200
Subject: [master] 7084fb1 White-space fix
Message-ID: <E1WmHt5-0005Kk-6y@project.varnish-software.com>


commit 7084fb10dca811c86c6c39b5a42cb71c413422bb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 19 07:20:30 2014 +0000

    White-space fix

diff --git a/bin/varnishtest/tests/c00041.vtc b/bin/varnishtest/tests/c00041.vtc
index bd6abdb..c46e95b 100644
--- a/bin/varnishtest/tests/c00041.vtc
+++ b/bin/varnishtest/tests/c00041.vtc
@@ -40,10 +40,12 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_recv { if (req.http.purge == "yes") { return(purge); } }
-	sub vcl_purge { if (req.http.restart == "yes") {
-		unset req.http.purge;
-		unset req.http.restart;
-		return(restart);} 
+	sub vcl_purge {
+		if (req.http.restart == "yes") {
+			unset req.http.purge;
+			unset req.http.restart;
+			return(restart);
+		}
 	}
 } -start
 



From phk at FreeBSD.org  Mon May 19 07:21:31 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 19 May 2014 09:21:31 +0200
Subject: [master] 147e71d White space fixes
Message-ID: <E1WmHt5-0005Kq-83@project.varnish-software.com>


commit 147e71d7d378173f81a62c3f861e03981b70ed66
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 19 07:21:20 2014 +0000

    White space fixes

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index 4b219de..60f36ba 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -441,9 +441,9 @@ main(int argc, char * const *argv)
 
 	/*
 	 * By default linux::getopt(3) mangles the argv order, such that
-	 * 	varnishadm -n bla param.set foo -bar
+	 *	varnishadm -n bla param.set foo -bar
 	 * gets interpreted as
-	 * 	varnishadm -n bla -bar param.set foo 
+	 *	varnishadm -n bla -bar param.set foo
 	 * The '+' stops that from happening
 	 * See #1496
 	 */
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index b3e8334..b2f3313 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -133,7 +133,7 @@ vmod_blob2hex(const struct vrt_ctx *ctx, VCL_BLOB b)
 VCL_BACKEND
 vmod_no_backend(const struct vrt_ctx *ctx)
 {
-	
+
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	return (NULL);
 }



From lkarsten at varnish-software.com  Mon May 19 09:30:16 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 19 May 2014 11:30:16 +0200
Subject: [master] 0993266 Remove duplicate command line arguments.
Message-ID: <E1WmJtg-0006F6-7W@project.varnish-software.com>


commit 099326633d6ce324dc356ab09e4d9bcf11dcc28a
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 19 10:26:49 2014 +0200

    Remove duplicate command line arguments.
    
    We don't need to add the RST2ANY_FLAGS arguments twice.

diff --git a/man/Makefile.am b/man/Makefile.am
index a3d3077..c7dd1bf 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -90,4 +90,4 @@ vmod_std.3: $(top_srcdir)/lib/libvmod_std/vmod_std.man.rst
 	${RST2MAN} $(RST2ANY_FLAGS) $? $@
 
 vmod_directors.3: $(top_srcdir)/lib/libvmod_directors/vmod_directors.man.rst
-	${RST2MAN} $(RST2ANY_FLAGS) $(RST2ANY_FLAGS) $? $@
+	${RST2MAN} $(RST2ANY_FLAGS) $? $@



From lkarsten at varnish-software.com  Mon May 19 09:30:16 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 19 May 2014 11:30:16 +0200
Subject: [master] 30f32b6 Copy from source where the file exists.
Message-ID: <E1WmJtg-0006FC-8p@project.varnish-software.com>


commit 30f32b65ee3135b101c57621a2aa483f651fc73f
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 19 11:27:46 2014 +0200

    Copy from source where the file exists.
    
    With parallell builds, building this file has usually not happened yet
    in the build directory.
    
    Suggested by: Martin

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index e714383..dbed031 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -212,11 +212,11 @@ include/varnishhist_synopsis.rst: $(top_builddir)/bin/varnishhist/varnishhist_op
 BUILT_SOURCES += include/varnishhist_options.rst \
 	 include/varnishhist_synopsis.rst
 
-reference/vmod_std.generated.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst
+reference/vmod_std.generated.rst: $(top_srcdir)/lib/libvmod_std/vmod_std.rst
 	cp $? $@
 BUILT_SOURCES += reference/vmod_std.generated.rst
 
-reference/vmod_directors.generated.rst: $(top_builddir)/lib/libvmod_directors/vmod_directors.rst
+reference/vmod_directors.generated.rst: $(top_srcdir)/lib/libvmod_directors/vmod_directors.rst
 	cp $? $@
 BUILT_SOURCES += reference/vmod_directors.generated.rst
 



From lkarsten at varnish-software.com  Mon May 19 11:06:31 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 19 May 2014 13:06:31 +0200
Subject: [master] 4482199 Don't package build/doctrees in tarball.
Message-ID: <E1WmLOp-000161-Kb@project.varnish-software.com>


commit 448219905424d04f3a77cb3af64fcca4bfebdc17
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 19 12:59:04 2014 +0200

    Don't package build/doctrees in tarball.
    
    These are just intermediate files written by Sphinx to
    build the HTML documentation. We don't need to distribute
    them.

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index dbed031..22dd894 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -91,89 +91,21 @@ doctest:
 EXTRA_DIST = \
 	conf.py \
 	index.rst \
-	glossary/index.rst \
-	installation/bugs.rst \
-	installation/help.rst \
-	installation/index.rst \
-	installation/install.rst \
-	installation/platformnotes.rst \
-	installation/prerequisites.rst \
-	phk/autocrap.rst \
-	phk/backends.rst \
-	phk/barriers.rst \
-	phk/gzip.rst \
-	phk/http20.rst \
-	phk/index.rst \
-	phk/ipv6suckage.rst \
-	phk/platforms.rst \
-	phk/spdy.rst \
-	phk/sphinx.rst \
-	phk/ssl.rst \
-	phk/thetoolsweworkwith.rst \
-	phk/thoughts.rst \
-	phk/three-zero.rst \
-	phk/varnish_does_not_hash.rst \
-	phk/vcl_expr.rst \
-	phk/wanton_destruction.rst \
-	reference/index.rst \
-	reference/varnish-cli.rst \
-	reference/varnishadm.rst \
-	reference/varnishd.rst \
-	reference/varnishhist.rst \
-	reference/varnishlog.rst \
-	reference/varnishncsa.rst \
-	reference/varnishstat.rst \
-	reference/varnishtest.rst \
-	reference/varnishtop.rst \
-	reference/vcl.rst \
-	reference/vmod.rst \
-	reference/vsl-query.rst \
-	reference/vsl.rst \
-	reference/vsm.rst \
-	tutorial/backend_servers.rst \
-	tutorial/index.rst \
-	tutorial/introduction.rst \
-	tutorial/now_what.rst \
-	tutorial/peculiarities.rst \
-	tutorial/putting_varnish_on_port_80.rst \
-	tutorial/starting_varnish.rst \
-	users-guide/command-line.rst \
-	users-guide/compression.rst \
-	users-guide/devicedetection.rst \
-	users-guide/esi.rst \
-	users-guide/increasing-your-hitrate.rst \
-	users-guide/index.rst \
-	users-guide/intro.rst \
-	users-guide/operation-logging.rst \
-	users-guide/operation-statistics.rst \
-	users-guide/params.rst \
-	users-guide/performance.rst \
-	users-guide/purging.rst \
-	users-guide/report.rst \
-	users-guide/run_cli.rst \
-	users-guide/run_security.rst \
-	users-guide/running.rst \
-	users-guide/sizing-your-cache.rst \
-	users-guide/storage-backends.rst \
-	users-guide/troubleshooting.rst \
-	users-guide/vcl-actions.rst \
-	users-guide/vcl-backends.rst \
-	users-guide/vcl-built-in-subs.rst \
-	users-guide/vcl-example-acls.rst \
-	users-guide/vcl-example-manipulating-headers.rst \
-	users-guide/vcl-example-manipulating-responses.rst \
-	users-guide/vcl-example-websockets.rst \
-	users-guide/vcl-examples.rst \
-	users-guide/vcl-hashing.rst \
-	users-guide/vcl-inline-c.rst \
-	users-guide/vcl-grace.rst \
-	users-guide/vcl-syntax.rst \
-	users-guide/vcl-variables.rst \
-	users-guide/vcl.rst
+	include \
+	glossary \
+	installation \
+	phk \
+	reference \
+	tutorial \
+	users-guide \
+	whats-new
 
 dist-hook:
 	$(MAKE) html
+	rm -rf $(BUILDDIR)/doctrees
 	cp -r $(BUILDDIR) $(distdir)
+# Remove any extra non-rst files we may have copied over.
+	find $(distdir) ! -type f -a -name \*.rst -ls
 
 distclean-local:
 	rm -rf $(BUILDDIR)



From lkarsten at varnish-software.com  Mon May 19 11:31:42 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 19 May 2014 13:31:42 +0200
Subject: [master] 010aac8 Remove any superflous non rst/html from tarball.
Message-ID: <E1WmLnC-0006Cp-Pd@project.varnish-software.com>


commit 010aac8abadfff3ebe227d1011bd8ba8d8acc289
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 19 13:28:44 2014 +0200

    Remove any superflous non rst/html from tarball.
    
    When we in the future introduce something new and shiny here, don't
    automatically start putting it into the tarball release.
    
    This may seem a bit backward, the main idea is to avoid hardcoding
    the section/subdirectory names yet another place.

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 22dd894..f1b9bfc 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -105,7 +105,7 @@ dist-hook:
 	rm -rf $(BUILDDIR)/doctrees
 	cp -r $(BUILDDIR) $(distdir)
 # Remove any extra non-rst files we may have copied over.
-	find $(distdir) ! -type f -a -name \*.rst -ls
+	find $(distdir) -mindepth 2 -type f ! -name \*.rst | grep -v '/build/' | xargs rm -f
 
 distclean-local:
 	rm -rf $(BUILDDIR)



From phk at FreeBSD.org  Tue May 20 07:12:34 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 May 2014 09:12:34 +0200
Subject: [master] afe89d5 Reduce contention on the wstat_mtx marginally while
 we figure out what the real solution is going to be.
Message-ID: <E1WmeDy-0003Fz-4T@project.varnish-software.com>


commit afe89d5520114dc297a3d3c4cec3c4373fefadf3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 20 07:11:57 2014 +0000

    Reduce contention on the wstat_mtx marginally while we figure out
    what the real solution is going to be.

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 6496a3a..7f9d410 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -54,7 +54,6 @@ wrk_sumstat(struct worker *w)
 #undef VSC_F
 #undef L0
 #undef L1
-	memset(&w->stats, 0, sizeof w->stats);
 }
 
 void
@@ -64,6 +63,7 @@ WRK_SumStat(struct worker *w)
 	Lck_Lock(&wstat_mtx);
 	wrk_sumstat(w);
 	Lck_Unlock(&wstat_mtx);
+	memset(&w->stats, 0, sizeof w->stats);
 }
 
 int
@@ -73,6 +73,7 @@ WRK_TrySumStat(struct worker *w)
 		return (0);
 	wrk_sumstat(w);
 	Lck_Unlock(&wstat_mtx);
+	memset(&w->stats, 0, sizeof w->stats);
 	return (1);
 }
 



From phk at FreeBSD.org  Tue May 20 08:02:31 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 May 2014 10:02:31 +0200
Subject: [master] 504c885 Stabilize a race in this test with some small delay
Message-ID: <E1Wmf0J-000624-Sg@project.varnish-software.com>


commit 504c885847f2d0ed8ae350b11c28fb00b05e21cd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 20 07:48:56 2014 +0000

    Stabilize a race in this test with some small delay

diff --git a/bin/varnishtest/tests/c00020.vtc b/bin/varnishtest/tests/c00020.vtc
index 509d6ee..ec54fe3 100644
--- a/bin/varnishtest/tests/c00020.vtc
+++ b/bin/varnishtest/tests/c00020.vtc
@@ -3,7 +3,7 @@ varnishtest "Test -h critbit a bit"
 server s1 {
         rxreq
 	expect req.url == "/"
-        txresp -hdr "Connection: close" -body "012345\n"
+        txresp -hdr "ID: slash" -hdr "Connection: close" -body "012345\n"
 } -start
 
 varnish v1 -arg "-hcritbit" -vcl+backend { } -start
@@ -13,41 +13,55 @@ client c1 {
         rxresp
         expect resp.status == 200
         expect resp.http.X-Varnish == "1001"
+        expect resp.http.ID == "slash"
 } -run
+delay .1
 
 client c2 {
         txreq -url "/"
         rxresp
         expect resp.status == 200
         expect resp.http.X-Varnish == "1004 1002"
+        expect resp.http.ID == "slash"
 } -run
+delay .1
 
 server s1 {
         rxreq
 	expect req.url == "/foo"
-        txresp -body "012345\n"
+        txresp -hdr "ID: foo" -body "012345\n"
         rxreq
 	expect req.url == "/bar"
-        txresp -body "012345\n"
+        txresp -hdr "ID: bar" -body "012345\n"
 } -start
 
-client c2 {
+client c1 {
         txreq -url "/foo"
         rxresp
         expect resp.status == 200
         expect resp.http.X-Varnish == "1006"
+        expect resp.http.ID == "foo"
+	delay .1
+
         txreq -url "/"
         rxresp
         expect resp.status == 200
         expect resp.http.X-Varnish == "1008 1002"
+        expect resp.http.ID == "slash"
+	delay .1
+
         txreq -url "/bar"
         rxresp
         expect resp.status == 200
         expect resp.http.X-Varnish == "1009"
+        expect resp.http.ID == "bar"
+	delay .1
+
         txreq -url "/foo"
         rxresp
         expect resp.status == 200
         expect resp.http.X-Varnish == "1011 1007"
+        expect resp.http.ID == "foo"
 } -run
 
 varnish v1 -expect sess_conn == 3



From phk at FreeBSD.org  Tue May 20 08:02:31 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 May 2014 10:02:31 +0200
Subject: [master] b1ad9cd Move the wthread_stats_rate check into the
 threadpool, so that it also takes account of fetch jobs.
Message-ID: <E1Wmf0J-00062A-U1@project.varnish-software.com>


commit b1ad9cd6488ecbd7d969542372c9f665125a5f8d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 20 08:01:53 2014 +0000

    Move the wthread_stats_rate check into the threadpool, so that it
    also takes account of fetch jobs.

diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index 3702827..9191d1c 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -240,9 +240,6 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 		return (SESS_DONE_RET_GONE);
 	}
 
-	if (wrk->stats.client_req >= cache_param->wthread_stats_rate)
-		WRK_SumStat(wrk);
-
 	WS_Reset(req->ws, NULL);
 	WS_Reset(wrk->aws, NULL);
 
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index ce8d526..fd5a43c 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -255,13 +255,13 @@ void
 Pool_Work_Thread(void *priv, struct worker *wrk)
 {
 	struct pool *pp;
-	int stats_clean;
+	int stats_dirty;
 	struct pool_task *tp;
 	int i;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 	wrk->pool = pp;
-	stats_clean = 1;
+	stats_dirty = 0;
 	while (1) {
 		Lck_Lock(&pp->mtx);
 
@@ -286,8 +286,10 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			wrk->task.func = NULL;
 			wrk->task.priv = wrk;
 			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
-			if (!stats_clean)
+			if (stats_dirty) {
 				WRK_SumStat(wrk);
+				stats_dirty = 0;
+			}
 			do {
 				i = Lck_CondWait(&wrk->cond, &pp->mtx,
 				    wrk->vcl == NULL ?  0 : wrk->lastused+60.);
@@ -303,7 +305,12 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 
 		assert(wrk->pool == pp);
 		tp->func(wrk, tp->priv);
-		stats_clean = WRK_TrySumStat(wrk);
+
+		if (++stats_dirty >= cache_param->wthread_stats_rate) {
+			WRK_SumStat(wrk);
+			stats_dirty = 0;
+		} else if (WRK_TrySumStat(wrk))
+			stats_dirty = 0;
 	}
 	wrk->pool = NULL;
 }
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 7f9d410..2c266ce 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -43,12 +43,12 @@ static struct lock		wstat_mtx;
 /*--------------------------------------------------------------------*/
 
 static void
-wrk_sumstat(struct worker *w)
+wrk_sumstat(const struct dstat *ds)
 {
 
 	Lck_AssertHeld(&wstat_mtx);
 #define L0(n)
-#define L1(n) (VSC_C_main->n += w->stats.n)
+#define L1(n) (VSC_C_main->n += ds->n)
 #define VSC_F(n, t, l, f, v, d, e) L##l(n);
 #include "tbl/vsc_f_main.h"
 #undef VSC_F
@@ -61,7 +61,7 @@ WRK_SumStat(struct worker *w)
 {
 
 	Lck_Lock(&wstat_mtx);
-	wrk_sumstat(w);
+	wrk_sumstat(&w->stats);
 	Lck_Unlock(&wstat_mtx);
 	memset(&w->stats, 0, sizeof w->stats);
 }
@@ -71,7 +71,7 @@ WRK_TrySumStat(struct worker *w)
 {
 	if (Lck_Trylock(&wstat_mtx))
 		return (0);
-	wrk_sumstat(w);
+	wrk_sumstat(&w->stats);
 	Lck_Unlock(&wstat_mtx);
 	memset(&w->stats, 0, sizeof w->stats);
 	return (1);
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index b3294f7..f43f019 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -183,8 +183,8 @@ struct parspec WRK_parspec[] = {
 		"0", NULL,
 		"Worker threads accumulate statistics, and dump these into "
 		"the global stats counters if the lock is free when they "
-		"finish a request.\n"
-		"This parameters defines the maximum number of requests "
+		"finish a job (request/fetch etc.)\n"
+		"This parameters defines the maximum number of jobs "
 		"a worker thread may handle, before it is forced to dump "
 		"its accumulated stats into the global counters.",
 		EXPERIMENTAL,



From phk at FreeBSD.org  Tue May 20 08:44:27 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 May 2014 10:44:27 +0200
Subject: [master] 46dd961 Move the stat-summing from WRK to Pool where it
 mostly happens.
Message-ID: <E1Wmfet-00008O-To@project.varnish-software.com>


commit 46dd9617c7d031aa82ca420af95e6579e84d62ad
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 20 08:44:05 2014 +0000

    Move the stat-summing from WRK to Pool where it mostly happens.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 75a64ac..cab075b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1107,6 +1107,8 @@ void Pool_Init(void);
 void Pool_Accept(void);
 void Pool_Work_Thread(void *priv, struct worker *w);
 int Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how);
+void Pool_Sumstat(struct worker *w);
+void Pool_PurgeStat(unsigned nobj);
 
 #define WRW_IsReleased(w)	((w)->wrw == NULL)
 int WRW_Error(const struct worker *w);
@@ -1216,10 +1218,6 @@ void WAIT_Write_Session(struct sess *sp, int fd);
 
 /* cache_wrk.c */
 
-void WRK_Init(void);
-int WRK_TrySumStat(struct worker *w);
-void WRK_SumStat(struct worker *w);
-void WRK_PurgeStat(unsigned nobj);
 void *WRK_thread(void *priv);
 typedef void *bgthread_t(struct worker *, void *priv);
 void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func,
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index b5ecfbe..a893ecc 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -404,7 +404,7 @@ EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru)
 	}
 	Lck_Unlock(&lru->mtx);
 
-	WRK_SumStat(wrk);
+	Pool_Sumstat(wrk);
 }
 
 #endif
@@ -589,7 +589,7 @@ exp_thread(struct worker *wrk, void *priv)
 			tnext = 0;
 		} else if (tnext > t) {
 			VSL_Flush(&ep->vsl, 0);
-			WRK_SumStat(wrk);
+			Pool_Sumstat(wrk);
 			(void)Lck_CondWait(&ep->condvar, &ep->mtx, tnext);
 		}
 		Lck_Unlock(&ep->mtx);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index cd48671..73acf01 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -603,7 +603,7 @@ double keep)
 		(void)HSH_DerefObj(&wrk->stats, &o);
 	}
 	WS_Release(wrk->aws, 0);
-	WRK_PurgeStat(nobj);
+	Pool_PurgeStat(nobj);
 }
 
 
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index c6a2599..31d167d 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -217,7 +217,6 @@ child_main(void)
 	VBO_Init();
 	VBE_InitCfg();
 	VBP_Init();
-	WRK_Init();
 	Pool_Init();
 	Pipe_Init();
 
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index fd5a43c..f438b26 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -78,6 +78,59 @@ static struct lock		pool_mtx;
 static pthread_t		thr_pool_herder;
 static unsigned			pool_accepting = 0;
 
+static struct lock		wstat_mtx;
+
+/*--------------------------------------------------------------------*/
+
+static void
+wrk_sumstat(const struct dstat *ds)
+{
+
+	Lck_AssertHeld(&wstat_mtx);
+#define L0(n)
+#define L1(n) (VSC_C_main->n += ds->n)
+#define VSC_F(n, t, l, f, v, d, e) L##l(n);
+#include "tbl/vsc_f_main.h"
+#undef VSC_F
+#undef L0
+#undef L1
+}
+
+void
+Pool_Sumstat(struct worker *w)
+{
+
+	Lck_Lock(&wstat_mtx);
+	wrk_sumstat(&w->stats);
+	Lck_Unlock(&wstat_mtx);
+	memset(&w->stats, 0, sizeof w->stats);
+}
+
+static int
+Pool_TrySumstat(struct worker *w)
+{
+	if (Lck_Trylock(&wstat_mtx))
+		return (0);
+	wrk_sumstat(&w->stats);
+	Lck_Unlock(&wstat_mtx);
+	memset(&w->stats, 0, sizeof w->stats);
+	return (1);
+}
+
+/*--------------------------------------------------------------------
+ * Helper function to update stats for purges under lock
+ */
+
+void
+Pool_PurgeStat(unsigned nobj)
+{
+	Lck_Lock(&wstat_mtx);
+	VSC_C_main->n_purges++;
+	VSC_C_main->n_obj_purged += nobj;
+	Lck_Unlock(&wstat_mtx);
+}
+
+
 /*--------------------------------------------------------------------
  */
 
@@ -148,7 +201,7 @@ pool_accept(struct worker *wrk, void *arg)
 		if (VCA_Accept(ps->lsock, wa) < 0) {
 			wrk->stats.sess_fail++;
 			/* We're going to pace in vca anyway... */
-			(void)WRK_TrySumStat(wrk);
+			(void)Pool_TrySumstat(wrk);
 			continue;
 		}
 
@@ -287,7 +340,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			wrk->task.priv = wrk;
 			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
 			if (stats_dirty) {
-				WRK_SumStat(wrk);
+				Pool_Sumstat(wrk);
 				stats_dirty = 0;
 			}
 			do {
@@ -307,9 +360,9 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 		tp->func(wrk, tp->priv);
 
 		if (++stats_dirty >= cache_param->wthread_stats_rate) {
-			WRK_SumStat(wrk);
+			Pool_Sumstat(wrk);
 			stats_dirty = 0;
-		} else if (WRK_TrySumStat(wrk))
+		} else if (Pool_TrySumstat(wrk))
 			stats_dirty = 0;
 	}
 	wrk->pool = NULL;
@@ -531,6 +584,7 @@ void
 Pool_Init(void)
 {
 
+	Lck_New(&wstat_mtx, lck_wstat);
 	Lck_New(&pool_mtx, lck_wq);
 	AZ(pthread_create(&thr_pool_herder, NULL, pool_poolherder, NULL));
 }
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 2c266ce..38a662b 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -37,59 +37,6 @@
 #include "cache.h"
 
 #include "hash/hash_slinger.h"
-
-static struct lock		wstat_mtx;
-
-/*--------------------------------------------------------------------*/
-
-static void
-wrk_sumstat(const struct dstat *ds)
-{
-
-	Lck_AssertHeld(&wstat_mtx);
-#define L0(n)
-#define L1(n) (VSC_C_main->n += ds->n)
-#define VSC_F(n, t, l, f, v, d, e) L##l(n);
-#include "tbl/vsc_f_main.h"
-#undef VSC_F
-#undef L0
-#undef L1
-}
-
-void
-WRK_SumStat(struct worker *w)
-{
-
-	Lck_Lock(&wstat_mtx);
-	wrk_sumstat(&w->stats);
-	Lck_Unlock(&wstat_mtx);
-	memset(&w->stats, 0, sizeof w->stats);
-}
-
-int
-WRK_TrySumStat(struct worker *w)
-{
-	if (Lck_Trylock(&wstat_mtx))
-		return (0);
-	wrk_sumstat(&w->stats);
-	Lck_Unlock(&wstat_mtx);
-	memset(&w->stats, 0, sizeof w->stats);
-	return (1);
-}
-
-/*--------------------------------------------------------------------
- * Helper function to update stats for purges under lock
- */
-
-void
-WRK_PurgeStat(unsigned nobj)
-{
-	Lck_Lock(&wstat_mtx);
-	VSC_C_main->n_purges++;
-	VSC_C_main->n_obj_purged += nobj;
-	Lck_Unlock(&wstat_mtx);
-}
-
 /*--------------------------------------------------------------------
  * Create and starte a back-ground thread which as its own worker and
  * session data structures;
@@ -164,7 +111,7 @@ wrk_thread_real(void *priv, unsigned thread_workspace)
 	if (w->nbo != NULL)
 		VBO_Free(&w->nbo);
 	HSH_Cleanup(w);
-	WRK_SumStat(w);
+	Pool_Sumstat(w);
 	return (NULL);
 }
 
@@ -174,9 +121,3 @@ WRK_thread(void *priv)
 
 	return (wrk_thread_real(priv, cache_param->workspace_thread));
 }
-
-void
-WRK_Init(void)
-{
-	Lck_New(&wstat_mtx, lck_wstat);
-}
diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
index 4e14bdd..4c7e4b5 100644
--- a/bin/varnishd/hash/hash_critbit.c
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -369,7 +369,7 @@ hcb_cleaner(struct worker *wrk, void *priv)
 		VSTAILQ_CONCAT(&dead_y, &cool_y);
 		VTAILQ_CONCAT(&dead_h, &cool_h, hoh_list);
 		Lck_Unlock(&hcb_mtx);
-		WRK_SumStat(wrk);
+		Pool_Sumstat(wrk);
 		VTIM_sleep(cache_param->critbit_cooloff);
 	}
 	NEEDLESS_RETURN(NULL);
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index f27ccd9..f66cd52 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -161,7 +161,7 @@ smp_load_seg(struct worker *wrk, const struct smp_sc *sc,
 		EXP_Inject(oc, sg->lru, so->ttl);
 		sg->nobj++;
 	}
-	WRK_SumStat(wrk);
+	Pool_Sumstat(wrk);
 	sg->flags |= SMP_SEG_LOADED;
 }
 



From phk at FreeBSD.org  Tue May 20 10:34:48 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 May 2014 12:34:48 +0200
Subject: [master] 38ee783 The mutex protecting summing of stats into the
 global VSC counters is a contention point and scales negatively with number
 of thread-pools.
Message-ID: <E1WmhNg-000797-8w@project.varnish-software.com>


commit 38ee7837bb5129276e4c75d45a554fb56ddbeb37
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 20 10:31:48 2014 +0000

    The mutex protecting summing of stats into the global VSC counters
    is a contention point and scales negatively with number of thread-pools.
    
    Sum into a per-pool dstat structure instead, and have the first
    idle thread summ that into the global counters without blocking the
    pool mutex.
    
    Fixes	#1495
    
    (I hope)

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index cab075b..da77ecb 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -248,6 +248,7 @@ struct acct_bereq {
 #define L1(t, n)		t n;
 #define VSC_F(n,t,l,f,v,e,d)	L##l(t, n)
 struct dstat {
+	unsigned		summs;
 #include "tbl/vsc_f_main.h"
 };
 #undef VSC_F
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index f438b26..754438a 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -72,6 +72,8 @@ struct pool {
 	uintmax_t			ndropped;
 	uintmax_t			nqueued;
 	struct sesspool			*sesspool;
+	struct dstat			*a_stat;
+	struct dstat			*b_stat;
 };
 
 static struct lock		pool_mtx;
@@ -80,15 +82,17 @@ static unsigned			pool_accepting = 0;
 
 static struct lock		wstat_mtx;
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Summing of stats into global stats counters
+ */
 
 static void
-wrk_sumstat(const struct dstat *ds)
+pool_sumstat(const struct dstat *src)
 {
 
 	Lck_AssertHeld(&wstat_mtx);
 #define L0(n)
-#define L1(n) (VSC_C_main->n += ds->n)
+#define L1(n) (VSC_C_main->n += src->n)
 #define VSC_F(n, t, l, f, v, d, e) L##l(n);
 #include "tbl/vsc_f_main.h"
 #undef VSC_F
@@ -101,7 +105,7 @@ Pool_Sumstat(struct worker *w)
 {
 
 	Lck_Lock(&wstat_mtx);
-	wrk_sumstat(&w->stats);
+	pool_sumstat(&w->stats);
 	Lck_Unlock(&wstat_mtx);
 	memset(&w->stats, 0, sizeof w->stats);
 }
@@ -111,13 +115,32 @@ Pool_TrySumstat(struct worker *w)
 {
 	if (Lck_Trylock(&wstat_mtx))
 		return (0);
-	wrk_sumstat(&w->stats);
+	pool_sumstat(&w->stats);
 	Lck_Unlock(&wstat_mtx);
 	memset(&w->stats, 0, sizeof w->stats);
 	return (1);
 }
 
 /*--------------------------------------------------------------------
+ * Summing of stats into pool counters
+ */
+
+static void
+pool_addstat(struct dstat *dst, struct dstat *src)
+{
+
+	dst->summs++;
+#define L0(n)
+#define L1(n) (dst->n += src->n)
+#define VSC_F(n, t, l, f, v, d, e) L##l(n);
+#include "tbl/vsc_f_main.h"
+#undef VSC_F
+#undef L0
+#undef L1
+	memset(src, 0, sizeof *src);
+}
+
+/*--------------------------------------------------------------------
  * Helper function to update stats for purges under lock
  */
 
@@ -301,6 +324,27 @@ pool_kiss_of_death(struct worker *wrk, void *priv)
 }
 
 /*--------------------------------------------------------------------
+ * Special function to summ stats
+ */
+
+static void __match_proto__(pool_func_t)
+pool_stat_summ(struct worker *wrk, void *priv)
+{
+	struct dstat *src;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->pool, POOL_MAGIC);
+	VSL(SLT_Debug, 0, "STATSUMM");
+	AN(priv);
+	src = priv;
+	Lck_Lock(&wstat_mtx);
+	pool_sumstat(src);
+	Lck_Unlock(&wstat_mtx);
+	memset(src, 0, sizeof *src);
+	wrk->pool->b_stat = src;
+}
+
+/*--------------------------------------------------------------------
  * This is the work function for worker threads in the pool.
  */
 
@@ -308,13 +352,12 @@ void
 Pool_Work_Thread(void *priv, struct worker *wrk)
 {
 	struct pool *pp;
-	int stats_dirty;
 	struct pool_task *tp;
+	struct pool_task tps;
 	int i;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 	wrk->pool = pp;
-	stats_dirty = 0;
 	while (1) {
 		Lck_Lock(&pp->mtx);
 
@@ -332,17 +375,26 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 				VTAILQ_REMOVE(&pp->back_queue, tp, list);
 		}
 
-		if (tp == NULL) {
+		if ((tp == NULL && wrk->stats.summs > 0) ||
+		    (wrk->stats.summs >= cache_param->wthread_stats_rate))
+			pool_addstat(pp->a_stat, &wrk->stats);
+
+		if (tp != NULL) {
+			wrk->stats.summs++;
+		} else if (pp->b_stat != NULL && pp->a_stat->summs) {
+			/* Nothing to do, push pool stats into global pool */
+			tps.func = pool_stat_summ;
+			tps.priv = pp->a_stat;
+			pp->a_stat = pp->b_stat;
+			pp->b_stat = NULL;
+			tp = &tps;
+		} else {
 			/* Nothing to do: To sleep, perchance to dream ... */
 			if (isnan(wrk->lastused))
 				wrk->lastused = VTIM_real();
 			wrk->task.func = NULL;
 			wrk->task.priv = wrk;
 			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
-			if (stats_dirty) {
-				Pool_Sumstat(wrk);
-				stats_dirty = 0;
-			}
 			do {
 				i = Lck_CondWait(&wrk->cond, &pp->mtx,
 				    wrk->vcl == NULL ?  0 : wrk->lastused+60.);
@@ -350,6 +402,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 					VCL_Rel(&wrk->vcl);
 			} while (wrk->task.func == NULL);
 			tp = &wrk->task;
+			wrk->stats.summs++;
 		}
 		Lck_Unlock(&pp->mtx);
 
@@ -358,12 +411,6 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 
 		assert(wrk->pool == pp);
 		tp->func(wrk, tp->priv);
-
-		if (++stats_dirty >= cache_param->wthread_stats_rate) {
-			Pool_Sumstat(wrk);
-			stats_dirty = 0;
-		} else if (Pool_TrySumstat(wrk))
-			stats_dirty = 0;
 	}
 	wrk->pool = NULL;
 }
@@ -507,6 +554,10 @@ pool_mkpool(unsigned pool_no)
 	ALLOC_OBJ(pp, POOL_MAGIC);
 	if (pp == NULL)
 		return (NULL);
+	pp->a_stat = calloc(1, sizeof *pp->a_stat);
+	AN(pp->a_stat);
+	pp->b_stat = calloc(1, sizeof *pp->b_stat);
+	AN(pp->b_stat);
 	Lck_New(&pp->mtx, lck_wq);
 
 	VTAILQ_INIT(&pp->idle_queue);



From phk at FreeBSD.org  Mon May 26 08:40:16 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 26 May 2014 10:40:16 +0200
Subject: [master] 60b5e71 Properly complain if people try to reuse symbols.
Message-ID: <E1WoqS8-0001ot-Ha@project.varnish-software.com>


commit 60b5e71e8a59a53f2575802c30a2f6eb2b7d2336
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 26 08:39:50 2014 +0000

    Properly complain if people try to reuse symbols.
    
    Fixes #1510

diff --git a/bin/varnishtest/tests/r01510.vtc b/bin/varnishtest/tests/r01510.vtc
new file mode 100644
index 0000000..e8933be
--- /dev/null
+++ b/bin/varnishtest/tests/r01510.vtc
@@ -0,0 +1,19 @@
+varnishtest "Duplicate object names"
+
+varnish v1 -errvcl {Object name 'first' already used.} {
+	import ${vmod_debug};
+	sub vcl_init {
+		new first = debug.obj("FOO");
+		new first = debug.obj("BAH");
+	}
+} 
+
+varnish v1 -errvcl {Object name 'first' already used.} {
+	import ${vmod_debug};
+
+	backend first { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new first = debug.obj("FOO");
+	}
+} 
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 910a89c..745f5b4 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -164,10 +164,25 @@ parse_new(struct vcc *tl)
 		return;
 	}
 	sy1 = VCC_FindSymbol(tl, tl->t, SYM_NONE);
+	if (sy1 != NULL) {
+		VSB_printf(tl->sb, "Object name '%.*s' already used.\n",
+		    PF(tl->t));
+
+		VSB_printf(tl->sb, "First usage:\n");
+		AN(sy1->def_b);
+		if (sy1->def_e != NULL)
+			vcc_ErrWhere2(tl, sy1->def_b, sy1->def_e);
+		else
+			vcc_ErrWhere(tl, sy1->def_b);
+		VSB_printf(tl->sb, "Redefinition:\n");
+		vcc_ErrWhere(tl, tl->t);
+		return;
+	}
 	XXXAZ(sy1);
 
 	sy1 = VCC_AddSymbolTok(tl, tl->t, SYM_NONE);	// XXX: NONE ?
 	XXXAN(sy1);
+	sy1->def_b = tl->t;
 	vcc_NextToken(tl);
 
 	ExpectErr(tl, '=');
@@ -243,6 +258,7 @@ parse_new(struct vcc *tl)
 		}
 		p += 2;
 	}
+	sy1->def_e = tl->t;
 	/*lint -restore */
 }
 



From phk at FreeBSD.org  Mon May 26 08:56:41 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 26 May 2014 10:56:41 +0200
Subject: [master] b8a53e5 Remove a now pointless assert.
Message-ID: <E1Woqi1-0002kD-5S@project.varnish-software.com>


commit b8a53e57afc6e688c81c3e581cc8ae4226c41f92
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 26 08:56:00 2014 +0000

    Remove a now pointless assert.
    
    Pointed out by:	Dridi Boukelmoune

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 745f5b4..080f50e 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -178,7 +178,6 @@ parse_new(struct vcc *tl)
 		vcc_ErrWhere(tl, tl->t);
 		return;
 	}
-	XXXAZ(sy1);
 
 	sy1 = VCC_AddSymbolTok(tl, tl->t, SYM_NONE);	// XXX: NONE ?
 	XXXAN(sy1);



From daghf at varnish-software.com  Mon May 26 13:07:14 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Mon, 26 May 2014 15:07:14 +0200
Subject: [master] 2e48fba Verify that s_resp_bodybytes is indeed incremented
 the correct number of bytes on partial content responses.
Message-ID: <E1WoucU-0000XK-UQ@project.varnish-software.com>


commit 2e48fbaf3b02caaae6b7f51b58e70dd07afc3238
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon May 26 15:06:26 2014 +0200

    Verify that s_resp_bodybytes is indeed incremented the correct number of bytes on partial content responses.

diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index ede05b2..513038c 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -90,3 +90,5 @@ client c1 {
 	expect resp.status == 206
 	expect resp.bodylen == 100
 } -run
+
+varnish v1 -expect s_resp_bodybytes == 1040



From phk at FreeBSD.org  Mon May 26 20:53:54 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 26 May 2014 22:53:54 +0200
Subject: [master] 46b6b56 Add a message about persistent storage being
 deprecated now.
Message-ID: <E1Wp1u6-00080P-FJ@project.varnish-software.com>


commit 46b6b56da4fdbea9cb86c2e69fa20e6370447c0d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 26 20:53:02 2014 +0000

    Add a message about persistent storage being deprecated now.

diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst
index 40a4830..3cef4af 100644
--- a/doc/sphinx/phk/index.rst
+++ b/doc/sphinx/phk/index.rst
@@ -8,6 +8,7 @@ You may or may not want to know what Poul-Henning thinks.
 .. toctree::
 	:maxdepth: 1
 
+	persistent.rst
 	dough.rst
 	wanton_destruction.rst
 	spdy.rst
diff --git a/doc/sphinx/phk/persistent.rst b/doc/sphinx/phk/persistent.rst
new file mode 100644
index 0000000..33af07c
--- /dev/null
+++ b/doc/sphinx/phk/persistent.rst
@@ -0,0 +1,98 @@
+.. _phk_pesistent:
+
+====================
+A persistent message
+====================
+
+This message is about -spersistent and why you should not use it,
+even though it is still present in Varnish 4.x.
+
+TL;DR:
+------
+
+Under narrow and ill defined circumstances, -spersistent works well,
+but in general it is more trouble than it is worth for you to run
+it, and we don't presently have the development resources to fix that.
+
+If you think you have these circumstances, you need to specify
+
+	-sdeprecated_persistence
+
+in order to use it.
+
+The long story
+--------------
+
+When we added -spersistent, to Varnish, it was in response to, and
+sponsored by a specific set of customers who really wanted this.
+
+A persistent storage module is an entirely different kettle of vax
+than a non-persistent module, because of all the ugly consistency
+issues it raises.
+
+Let me give you an example.
+
+Imagine a cluster of some Varnish servers on which bans are used.
+
+Without persistent storage, if one of them goes down and comes back
+up, all the old cached objects are gone, and so are, by definition
+all the banned objects.
+
+With persistent storage, we not only have to store the still live
+bans with the cached objects, and keep the two painfully in sync,
+so the bans gets revived with the objects, we also have to worry
+about missing bans duing the downtime, since those might ban objects
+we will recover on startup.
+
+Ouch:  Straight into database/filesystem consistency territory.
+
+But we knew that, and I thought I had a good strategy to deal with
+this.
+
+And in a sense I did.
+
+Varnish has the advantage over databases and filesystems that we
+can actually loose objects without it being a catastrophy.  It would
+be better if we didn't, but we can simply ditch stuff which doesn't
+look consistent and we'll be safe.
+
+The strategy was to do a "Log Structured Filesystem", a once promising
+concept which soon proved very troublesome to implement well.
+
+Interestingly, today the ARM chip in your SSD most likely implements
+a LFS for wear-levelling, but with a vastly reduced feature set:
+All "files" are one sector long, filenames are integers and there
+are no subdirectories or rename operations.  On the other hand,
+there is extra book-keeping about the state of the flash array.
+
+A LFS consists of two major components:  The bit that reads and
+writes, which is pretty trivial, and the bit which makes space
+available which isn't.
+
+Initially we didn't even do the second part, because in varnish
+objects expire, and provided they do so fast enough, the space will
+magically make itself available.  This worked well enough for our
+initial users, and they only used bans sporadically so that was
+cool too.
+
+In other words, a classic 20% effort, 80% benefit.
+
+Unfortunately we have not been able to find time and money for the
+other 80% effort which gives the last 20% benefit, and therefor
+-spersistent has ended up in limbo.
+
+Today we decided to officially deprecate -spersistent, and start
+warning people against using it, but we will leave it in the source
+code for now, in order to keep the interfaces necessary for a
+persistent storage working, in the hope that we will get to use
+them again later.
+
+So you can still use persistent storage, if you really want to,
+and if you know what you're doing, by using:
+
+	-sdeprecated_persistent
+
+You've been warned.
+
+
+Poul-Henning, 2014-05-26



From phk at FreeBSD.org  Mon May 26 22:22:50 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 May 2014 00:22:50 +0200
Subject: [master] 3b4efd0 Rename -spersistent to make it clear that it is
 deprecated.
Message-ID: <E1Wp3IA-0001G3-LZ@project.varnish-software.com>


commit 3b4efd06d2a687c3a32e561cf5ec0b64f3e8cd30
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 26 22:22:27 2014 +0000

    Rename -spersistent to make it clear that it is deprecated.

diff --git a/bin/varnishd/storage/stevedore_mgt.c b/bin/varnishd/storage/stevedore_mgt.c
index 53bfb07..7a38171 100644
--- a/bin/varnishd/storage/stevedore_mgt.c
+++ b/bin/varnishd/storage/stevedore_mgt.c
@@ -74,17 +74,44 @@ struct cli_proto cli_stv[] = {
 	    0, 0, "", stv_cli_list },
 	{ NULL}
 };
+
+/*--------------------------------------------------------------------
+ */
+
+static void
+smp_fake_init(struct stevedore *parent, int ac, char * const *av)
+{
+
+	(void)parent;
+	(void)ac;
+	(void)av;
+	ARGV_ERR(
+	    "-spersistent has been deprecated, please see:\n"
+	    "  https://www.varnish-cache.org/docs/trunk/phk/persistent.html\n"
+	    "for details.\n"
+	);
+}
+
+
+static const struct stevedore smp_fake_stevedore = {
+	.magic = STEVEDORE_MAGIC,
+	.name = "deprecated_persistent",
+	.init = smp_fake_init,
+};
+
+
 /*--------------------------------------------------------------------
  * Parse a stevedore argument on the form:
  *	[ name '=' ] strategy [ ',' arg ] *
  */
 
 static const struct choice STV_choice[] = {
-	{ "file",	&smf_stevedore },
-	{ "malloc",	&sma_stevedore },
-	{ "persistent",	&smp_stevedore },
+	{ "file",			&smf_stevedore },
+	{ "malloc",			&sma_stevedore },
+	{ "deprecated_persistent",	&smp_stevedore },
+	{ "persistent",			&smp_fake_stevedore },
 #ifdef HAVE_LIBUMEM
-	{ "umem",	&smu_stevedore },
+	{ "umem",			&smu_stevedore },
 #endif
 	{ NULL,		NULL }
 };
@@ -133,9 +160,6 @@ STV_Config(const char *spec)
 
 	*stv = *stv2;
 	AN(stv->name);
-	AN(stv->alloc);
-	if (stv->allocobj == NULL)
-		stv->allocobj = stv_default_allocobj;
 
 	if (p == NULL)
 		bprintf(stv->ident, "s%u", seq++);
@@ -158,6 +182,10 @@ STV_Config(const char *spec)
 	else if (ac != 0)
 		ARGV_ERR("(-s%s) too many arguments\n", stv->name);
 
+	AN(stv->alloc);
+	if (stv->allocobj == NULL)
+		stv->allocobj = stv_default_allocobj;
+
 	if (!strcmp(stv->ident, TRANSIENT_STORAGE)) {
 		stv->transient = 1;
 		AZ(stv_transient);
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index d7506f0..6633799 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -588,7 +588,7 @@ smp_free(struct storage *st)
 
 const struct stevedore smp_stevedore = {
 	.magic	=	STEVEDORE_MAGIC,
-	.name	=	"persistent",
+	.name	=	"deprecated_persistent",
 	.init	=	smp_mgt_init,
 	.open	=	smp_open,
 	.close	=	smp_close,
diff --git a/bin/varnishtest/tests/a00009.vtc b/bin/varnishtest/tests/a00009.vtc
index 34a56f0..73021ba 100644
--- a/bin/varnishtest/tests/a00009.vtc
+++ b/bin/varnishtest/tests/a00009.vtc
@@ -1,5 +1,6 @@
-varnishtest "Code coverage of VCL compiler and RSTdump"
+varnishtest "Code coverage of VCL compiler and RSTdump etc"
 
 shell "${varnishd} -b 127.0.0.1:80 -C -n ${tmpdir} > /dev/null 2>&1"
 shell "${varnishd} -x dumprstparam > /dev/null 2>&1"
 shell "${varnishd} -x dumprstvsl > /dev/null 2>&1"
+shell "! ${varnishd} -spersistent > /dev/null 2>&1"
diff --git a/bin/varnishtest/tests/p00000.vtc b/bin/varnishtest/tests/p00000.vtc
index ab59ccf..eb9d29d 100644
--- a/bin/varnishtest/tests/p00000.vtc
+++ b/bin/varnishtest/tests/p00000.vtc
@@ -9,7 +9,7 @@ shell "rm -f ${tmpdir}/_.per"
 
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-vcl+backend { } -start
 
 varnish v1 -stop
diff --git a/bin/varnishtest/tests/p00002.vtc b/bin/varnishtest/tests/p00002.vtc
index 1e04ebc..e60c8a6 100644
--- a/bin/varnishtest/tests/p00002.vtc
+++ b/bin/varnishtest/tests/p00002.vtc
@@ -10,8 +10,8 @@ server s1 {
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0" \
-	-arg "-spersistent,${tmpdir}/_.per1,10m" \
-	-arg "-spersistent,${tmpdir}/_.per2,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \
 	-vcl+backend { } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/p00003.vtc b/bin/varnishtest/tests/p00003.vtc
index 0e8eec7..98984df 100644
--- a/bin/varnishtest/tests/p00003.vtc
+++ b/bin/varnishtest/tests/p00003.vtc
@@ -9,7 +9,7 @@ server s1 {
 
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-arg "-pban_lurker_sleep=0" \
 	-vcl+backend { } -start
 
diff --git a/bin/varnishtest/tests/p00004.vtc b/bin/varnishtest/tests/p00004.vtc
index c3aea98..a908913 100644
--- a/bin/varnishtest/tests/p00004.vtc
+++ b/bin/varnishtest/tests/p00004.vtc
@@ -11,7 +11,7 @@ server s1 {
 
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-arg "-pban_lurker_sleep=0" \
 	-vcl+backend { } -start
 
diff --git a/bin/varnishtest/tests/p00005.vtc b/bin/varnishtest/tests/p00005.vtc
index 1ac8e08..55bc6c9 100644
--- a/bin/varnishtest/tests/p00005.vtc
+++ b/bin/varnishtest/tests/p00005.vtc
@@ -8,7 +8,7 @@ server s1 {
 } -start
 
 varnish v1 \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-arg "-pban_lurker_sleep=0" \
 	-arg "-pshortlived=0" \
 	-vcl+backend {
diff --git a/bin/varnishtest/tests/p00006.vtc b/bin/varnishtest/tests/p00006.vtc
index 96f4d17..fb5fcca 100644
--- a/bin/varnishtest/tests/p00006.vtc
+++ b/bin/varnishtest/tests/p00006.vtc
@@ -11,7 +11,7 @@ server s1 {
 
 
 varnish v1 \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-arg "-pban_lurker_sleep=0" \
 	-vcl+backend { } -start
 
diff --git a/bin/varnishtest/tests/p00007.vtc b/bin/varnishtest/tests/p00007.vtc
index 5fd9e9e..1daf4e5 100644
--- a/bin/varnishtest/tests/p00007.vtc
+++ b/bin/varnishtest/tests/p00007.vtc
@@ -24,7 +24,7 @@ server s1 {
 	txresp -bodylen 48
 } -start
 
-varnish v1 -arg "-spersistent,${tmpdir}/_.per,10m" \
+varnish v1 -arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-vcl+backend {} -start
 
 varnish v1 -cliok "debug.fragfetch 32"
diff --git a/bin/varnishtest/tests/p00008.vtc b/bin/varnishtest/tests/p00008.vtc
index b839a0c..c7a144d 100644
--- a/bin/varnishtest/tests/p00008.vtc
+++ b/bin/varnishtest/tests/p00008.vtc
@@ -16,8 +16,8 @@ server s1 {
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0" \
-	-arg "-sper1=persistent,${tmpdir}/_.per1,10m" \
-	-arg "-sper2=persistent,${tmpdir}/_.per2,10m" \
+	-arg "-sper1=deprecated_persistent,${tmpdir}/_.per1,10m" \
+	-arg "-sper2=deprecated_persistent,${tmpdir}/_.per2,10m" \
 	-vcl+backend {
 		sub vcl_backend_response {
 			set beresp.storage_hint = "per1";
@@ -45,7 +45,7 @@ server s1 -wait
 varnish v2 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0" \
-	-arg "-spersistent,${tmpdir}/_.per1,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \
 	-vcl+backend { } -start
 varnish v2 -cliok "ban obj.http.x-foo == foo"
 varnish v2 -cliok "ban.list"
@@ -56,8 +56,8 @@ varnish v2 -stop
 varnish v3 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0" \
-	-arg "-spersistent,${tmpdir}/_.per1,10m" \
-	-arg "-spersistent,${tmpdir}/_.per2,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \
 	-vcl+backend { } -start
 varnish v3 -cliok "ban.list"
 varnish v3 -stop
@@ -72,7 +72,7 @@ server s1 {
 varnish v4 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0" \
-	-arg "-spersistent,${tmpdir}/_.per2,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \
 	-vcl+backend { } -start
 client c1 -connect ${v4_sock} {
 	txreq -url "/silo2"
diff --git a/bin/varnishtest/tests/p00009.vtc b/bin/varnishtest/tests/p00009.vtc
index 66b1e2f..6ec15d4 100644
--- a/bin/varnishtest/tests/p00009.vtc
+++ b/bin/varnishtest/tests/p00009.vtc
@@ -14,8 +14,8 @@ server s1 {
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0" \
-	-arg "-sper1=persistent,${tmpdir}/_.per1,10m" \
-	-arg "-sper2=persistent,${tmpdir}/_.per2,10m" \
+	-arg "-sper1=deprecated_persistent,${tmpdir}/_.per1,10m" \
+	-arg "-sper2=deprecated_persistent,${tmpdir}/_.per2,10m" \
 	-vcl+backend {
 	}
 varnish v1 -start
diff --git a/bin/varnishtest/tests/r00915.vtc b/bin/varnishtest/tests/r00915.vtc
index 607d6db..3e1528c 100644
--- a/bin/varnishtest/tests/r00915.vtc
+++ b/bin/varnishtest/tests/r00915.vtc
@@ -9,7 +9,7 @@ shell "rm -f ${tmpdir}/_.per"
 
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-vcl+backend {
 
 	sub vcl_backend_response {
diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc
index 9cae4cf..6d750cb 100644
--- a/bin/varnishtest/tests/r00962.vtc
+++ b/bin/varnishtest/tests/r00962.vtc
@@ -12,8 +12,8 @@ shell "rm -f ${tmpdir}/_.per?"
 
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
-	-arg "-spersistent,${tmpdir}/_.per1,10m" \
-	-arg "-spersistent,${tmpdir}/_.per2,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \
 	-vcl+backend {
 	sub vcl_backend_response {
 		set beresp.storage = "s0";
@@ -44,8 +44,8 @@ server s1 {
 
 varnish v2 \
 	-arg "-pfeature=+wait_silo" \
-	-arg "-spersistent,${tmpdir}/_.per2,10m" \
-	-arg "-spersistent,${tmpdir}/_.per1,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \
 	-vcl+backend { } -start
 
 client c1 -connect ${v2_sock} {
diff --git a/bin/varnishtest/tests/r01225.vtc b/bin/varnishtest/tests/r01225.vtc
index 23d0253..85cc418 100644
--- a/bin/varnishtest/tests/r01225.vtc
+++ b/bin/varnishtest/tests/r01225.vtc
@@ -10,7 +10,7 @@ server s1 {
 
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-arg "-pban_lurker_sleep=0.01" \
 	-vcl+backend { } -start
 
diff --git a/bin/varnishtest/tests/r01266.vtc b/bin/varnishtest/tests/r01266.vtc
index 76352c3..bf4bf42 100644
--- a/bin/varnishtest/tests/r01266.vtc
+++ b/bin/varnishtest/tests/r01266.vtc
@@ -12,7 +12,7 @@ server s1 {
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0.01" \
-	-arg "-sper1=persistent,${tmpdir}/_.per1,10m" \
+	-arg "-sper1=deprecated_persistent,${tmpdir}/_.per1,10m" \
 	-vcl+backend {
 	}
 varnish v1 -start



From phk at FreeBSD.org  Tue May 27 10:42:16 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 May 2014 12:42:16 +0200
Subject: [master] 5dfd0c0 Minor cleanup of how we write the warning at the top
 of generated files.
Message-ID: <E1WpEpk-0007Hm-DP@project.varnish-software.com>


commit 5dfd0c099569282164e4f03c64e172205cb2c17d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 27 10:18:22 2014 +0000

    Minor cleanup of how we write the warning at the top of generated files.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 25e0624..42e2fd3 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -68,14 +68,18 @@ ctypes = {
 
 #######################################################################
 
-def write_file_header(fo):
-	fo.write("""/*
- * NB:  This file is machine generated, DO NOT EDIT!
- *
- * Edit vmod.vcc and run %s instead
- */
+def write_file_warning(fo, a, b, c):
+	fo.write(a + "\n")
+	fo.write(b + "NB:  This file is machine generated, DO NOT EDIT!\n")
+	fo.write(b + "\n")
+	fo.write(b + "Edit vmod.vcc and run make instead\n")
+	fo.write(c + "\n\n")
 
-""" % basename(__file__))
+def write_c_file_warning(fo):
+	write_file_warning(fo, "/*", " * ", " */")
+
+def write_rst_file_warning(fo):
+	write_file_warning(fo, "..", ".. ", "..")
 
 #######################################################################
 
@@ -815,8 +819,8 @@ def runmain(inputvcc, outputname="vcc_if"):
 	fc = open("%s.c" % outputname, "w")
 	fh = open("%s.h" % outputname, "w")
 
-	write_file_header(fc)
-	write_file_header(fh)
+	write_c_file_warning(fc)
+	write_c_file_warning(fh)
 
 	fh.write('struct vrt_ctx;\n')
 	fh.write('struct VCL_conf;\n')
@@ -841,22 +845,19 @@ def runmain(inputvcc, outputname="vcc_if"):
 	fh.close()
 
 	for suf in ("", ".man"):
-		with open("vmod_%s%s.rst" % (vx[0].nam, suf), "w") as fp:
-			fp.write("..\n")
-			fp.write(".. This file was autogenerated by %s. DO NOT EDIT!\n" %
-				 basename(__file__))
-			fp.write("..\n\n")
-
-			vx[0].doc_dump(fp, suf)
-
-			if len(copyright) > 0:
-				fp.write("\n")
-				fp.write("COPYRIGHT\n")
-				fp.write("=========\n")
-				fp.write("\n::\n\n")
-				for i in copyright:
-					fp.write("  %s\n" % i)
-				fp.write("\n")
+		fp = open("vmod_%s%s.rst" % (vx[0].nam, suf), "w")
+		write_rst_file_warning(fp)
+
+		vx[0].doc_dump(fp, suf)
+
+		if len(copyright) > 0:
+			fp.write("\n")
+			fp.write("COPYRIGHT\n")
+			fp.write("=========\n")
+			fp.write("\n::\n\n")
+			for i in copyright:
+				fp.write("  %s\n" % i)
+			fp.write("\n")
 
 
 if __name__ == "__main__":



From martin at varnish-software.com  Tue May 27 11:06:17 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 27 May 2014 13:06:17 +0200
Subject: [master] 4a18350 Added %I and %O formatters to varnishncsa
Message-ID: <E1WpFCz-0001Ag-R2@project.varnish-software.com>


commit 4a18350b045b945dfc0b22eab2585c8c8e11ba92
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue May 27 12:37:10 2014 +0200

    Added %I and %O formatters to varnishncsa
    
    This adds %I and %O formatters to varnishncsa, which are total bytes
    received and total bytes sent. These are similar to the same
    formatters in apache (with mod_logio). Useful for bandwidth accounting.
    
    Patch by: David Robertson

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 2162118..5aa0361 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -45,6 +45,8 @@
  *	%b		Length of reply body, or "-"
  *	%{Referer}i	Contents of "Referer" request header
  *	%{User-agent}i	Contents of "User-agent" request header
+ *  %I		Total bytes recieved from client
+ *  %O		Total bytes sent to client
  *
  * Actually, we cheat a little and replace "%r" with something close to
  * "%m http://%{Host}i%U%q %H", where the additional fields are:
@@ -95,6 +97,8 @@ enum e_frag {
 	F_h,			/* %h Host name / IP Address */
 	F_m,			/* %m Method */
 	F_s,			/* %s Status */
+	F_I,			/* %I Bytes recieved */
+	F_O,			/* %O Bytes sent */
 	F_tstart,		/* Time start */
 	F_tend,			/* Time end */
 	F_ttfb,			/* %{Varnish:time_firstbyte}x */
@@ -574,6 +578,12 @@ parse_format(const char *format)
 		case 'U':	/* URL */
 			addf_fragment(&CTX.frag[F_U], "-");
 			break;
+		case 'I':	/* Bytes recieved */
+			addf_fragment(&CTX.frag[F_I], "-");
+			break;
+		case 'O':	/* Bytes sent */
+			addf_fragment(&CTX.frag[F_O], "-");
+			break;
 		case '{':
 			p++;
 			q = p;
@@ -783,6 +793,8 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				break;
 			case SLT_ReqAcct:
 				frag_fields(b, e, 5, &CTX.frag[F_b], 0, NULL);
+				frag_fields(b, e, 3, &CTX.frag[F_I], 0, NULL);
+				frag_fields(b, e, 6, &CTX.frag[F_O], 0, NULL);
 				break;
 			case SLT_Timestamp:
 				if (isprefix(b, "Start:", e, &p)) {



From martin at varnish-software.com  Tue May 27 11:06:17 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 27 May 2014 13:06:17 +0200
Subject: [master] d4baf14 Document %I and %O ncsa format specifiers
Message-ID: <E1WpFCz-0001An-SP@project.varnish-software.com>


commit d4baf146d1f412d6ab36c21cd7ca45b5357fe77b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue May 27 13:01:45 2014 +0200

    Document %I and %O ncsa format specifiers

diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 77086ac..ffdb7f4 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -57,6 +57,9 @@ Supported formatters are:
 %h
   Remote host. Defaults to '-' if not known.
 
+%I
+  Total bytes received from client.
+
 %{X}i
   The contents of request header X.
 
@@ -72,6 +75,9 @@ Supported formatters are:
 %{X}o
   The contents of response header X.
 
+%O
+  Total bytes sent to client.
+
 %r
   The first line of the request. Synthesized from other fields, so it
   may not be the request verbatim.



From martin at varnish-software.com  Tue May 27 11:06:17 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 27 May 2014 13:06:17 +0200
Subject: [master] f53ff46 Only parse ReqAcct once
Message-ID: <E1WpFCz-0001Az-Tn@project.varnish-software.com>


commit f53ff463561ca6e4c79a820852550453fb0dac5b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue May 27 13:02:55 2014 +0200

    Only parse ReqAcct once

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 5aa0361..16e9e30 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -792,9 +792,11 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				frag_line(b, e, &CTX.frag[F_s]);
 				break;
 			case SLT_ReqAcct:
-				frag_fields(b, e, 5, &CTX.frag[F_b], 0, NULL);
-				frag_fields(b, e, 3, &CTX.frag[F_I], 0, NULL);
-				frag_fields(b, e, 6, &CTX.frag[F_O], 0, NULL);
+				frag_fields(b, e,
+				    3, &CTX.frag[F_I],
+				    5, &CTX.frag[F_b],
+				    6, &CTX.frag[F_O],
+				    0, NULL);
 				break;
 			case SLT_Timestamp:
 				if (isprefix(b, "Start:", e, &p)) {



From martin at varnish-software.com  Tue May 27 11:06:17 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 27 May 2014 13:06:17 +0200
Subject: [master] f2cf2ea Keep the list of ncsa format specifiers in one
 location only
Message-ID: <E1WpFCz-0001B8-Up@project.varnish-software.com>


commit f2cf2eaf5c3ea5a304454b9d265028024f4889b9
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue May 27 13:03:52 2014 +0200

    Keep the list of ncsa format specifiers in one location only
    
    Other minor cleanups

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 16e9e30..2ff1fc3 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -30,32 +30,12 @@
  * SUCH DAMAGE.
  *
  * Obtain log data from the shared memory log, order it by session ID, and
- * display it in Apache / NCSA combined log format:
+ * display it in Apache / NCSA combined log format.
  *
- *	%h %l %u %t "%r" %s %b "%{Referer}i" "%{User-agent}i"
+ * See doc/sphinx/reference/varnishncsa.rst for the supported format
+ * specifiers.
  *
- * where the fields are defined as follows:
- *
- *	%h		Client host name or IP address (always the latter)
- *	%l		Client user ID as reported by identd (always "-")
- *	%u		User ID if using HTTP authentication, or "-"
- *	%t		Date and time of request
- *	%r		Request line
- *	%s		Status code
- *	%b		Length of reply body, or "-"
- *	%{Referer}i	Contents of "Referer" request header
- *	%{User-agent}i	Contents of "User-agent" request header
- *  %I		Total bytes recieved from client
- *  %O		Total bytes sent to client
- *
- * Actually, we cheat a little and replace "%r" with something close to
- * "%m http://%{Host}i%U%q %H", where the additional fields are:
- *
- *	%m		Request method
- *	%{Host}i	Contents of "Host" request header
- *	%U		URL path
- *	%q		Query string
- *	%H		Protocol version
+ * Note: %r is "%m http://%{Host}i%U%q %H"
  *
  */
 
@@ -551,12 +531,18 @@ parse_format(const char *format)
 		case 'H':	/* Protocol */
 			addf_fragment(&CTX.frag[F_H], "HTTP/1.0");
 			break;
+		case 'I':	/* Bytes recieved */
+			addf_fragment(&CTX.frag[F_I], "-");
+			break;
 		case 'l':	/* Client user ID (identd) always '-' */
 			AZ(VSB_putc(vsb, '-'));
 			break;
 		case 'm':	/* Method */
 			addf_fragment(&CTX.frag[F_m], "-");
 			break;
+		case 'O':	/* Bytes sent */
+			addf_fragment(&CTX.frag[F_O], "-");
+			break;
 		case 'q':	/* Query string */
 			addf_fragment(&CTX.frag[F_q], "");
 			break;
@@ -572,18 +558,12 @@ parse_format(const char *format)
 		case 'T':	/* Int request time */
 			addf_time(*p, NULL, NULL);
 			break;
-		case 'u':
+		case 'u':	/* Remote user from auth */
 			addf_auth("-");
 			break;
 		case 'U':	/* URL */
 			addf_fragment(&CTX.frag[F_U], "-");
 			break;
-		case 'I':	/* Bytes recieved */
-			addf_fragment(&CTX.frag[F_I], "-");
-			break;
-		case 'O':	/* Bytes sent */
-			addf_fragment(&CTX.frag[F_O], "-");
-			break;
 		case '{':
 			p++;
 			q = p;
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index ffdb7f4..85906fc 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -69,15 +69,15 @@ Supported formatters are:
 %m
    Request method. Defaults to '-' if not known.
 
-%q
-  The query string, if no query string exists, an empty string.
-
 %{X}o
   The contents of response header X.
 
 %O
   Total bytes sent to client.
 
+%q
+  The query string, if no query string exists, an empty string.
+
 %r
   The first line of the request. Synthesized from other fields, so it
   may not be the request verbatim.



From phk at FreeBSD.org  Tue May 27 14:10:59 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 May 2014 16:10:59 +0200
Subject: [master] 9c3a747 Beautify the output of the vmodtool to make it
 (also) human readable.
Message-ID: <E1WpI5j-0006dg-Ja@project.varnish-software.com>


commit 9c3a747d2ae9352458337316e90c06693e960832
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 27 14:10:29 2014 +0000

    Beautify the output of the vmodtool to make it (also) human readable.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 42e2fd3..ea01391 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -83,6 +83,27 @@ def write_rst_file_warning(fo):
 
 #######################################################################
 
+def lwrap(s, w=72):
+	"""
+	Wrap a c-prototype like string into a number of lines
+	"""
+	l = []
+	p=""
+	while len(s) > w:
+		y = s[:w].rfind(',')
+		if y == -1:
+			y = s[:w].rfind('(')
+		if y == -1:
+			break
+		l.append(p + s[:y + 1])
+		s = s[y + 1:].lstrip()
+		p = "    "
+	if len(s) > 0:
+		l.append(p + s)
+	return l
+		
+#######################################################################
+
 def is_c_name(s):
 	return None != re.match("^[a-z][a-z0-9_]*$", s)
 
@@ -145,11 +166,15 @@ class vmod(object):
 
 	def c_proto(self, fo):
 		for o in self.objs:
+			fo.write("/* Object %s */\n" % o.nam)
 			o.fixup(self.nam)
 			o.c_proto(fo)
 			fo.write("\n")
+		if len(self.funcs) > 0:
+			fo.write("/* Functions */\n")
 		for f in self.funcs:
-			f.c_proto(fo)
+			for i in lwrap(f.c_proto()):
+				fo.write(i + "\n")
 		if self.init != None:
 			fo.write("\n")
 			fo.write("int " + self.init)
@@ -162,7 +187,8 @@ class vmod(object):
 			for t in o.c_typedefs(self.nam):
 				l.append(t)
 			l.append("")
-		l.append("/* Functions */")
+		if len(self.funcs) > 0:
+			l.append("/* Functions */")
 		for f in self.funcs:
 			l.append(f.c_typedef(self.nam))
 		l.append("")
@@ -170,7 +196,8 @@ class vmod(object):
 
 	def c_typedefs(self, fo):
 		for i in self.c_typedefs_():
-			fo.write(i + "\n")
+			for j in lwrap(i):
+				fo.write(j + "\n")
 
 	def c_vmod(self, fo):
 		fo.write('extern const char Vmod_' + self.nam + '_Name[];\n')
@@ -197,7 +224,8 @@ class vmod(object):
 		fo.write("extern const char Vmod_" + self.nam + "_Proto[];\n")
 		fo.write("const char Vmod_" + self.nam + "_Proto[] =\n")
 		for t in self.c_typedefs_():
-			fo.write('\t"' + t + '\\n"\n')
+			for i in lwrap(t, w=64):
+				fo.write('\t"' + i + '\\n"\n')
 		fo.write('\t"\\n"\n')
 		for i in (cs + ";").split("\n"):
 			fo.write('\n\t"' + i + '\\n"')
@@ -249,14 +277,15 @@ class vmod(object):
 		s = "extern " + s + ";\n" + s + " = {\n"
 
 		for o in self.objs:
-			s += o.c_strspec(self.nam)
+			s += o.c_strspec(self.nam) + ",\n\n"
 
-		s += "\n\t/* Functions */\n"
+		if len(self.funcs) > 0:
+			s += "\t/* Functions */\n"
 		for f in self.funcs:
-			s += '\t"' + f.c_strspec(self.nam) + '",\n'
+			s += f.c_strspec(self.nam) + ',\n\n'
 
-		s += "\n\t/* Init/Fini */\n"
 		if self.init != None:
+			s += "\t/* Init/Fini */\n"
 			s += '\t"INIT\\0Vmod_' + self.nam + '_Func._init",\n'
 
 		s += "\t0\n"
@@ -323,22 +352,22 @@ class func(object):
 	def set_pfx(self, s):
 		self.pfx = s
 
-	def c_proto(self, fo, fini=False):
-		fo.write(ctypes[self.retval])
-		fo.write(" vmod_" + self.cnam + "(")
+	def c_proto(self, fini=False):
+		s = ctypes[self.retval] + " vmod_" + self.cnam + "("
 		p = ""
 		if not fini:
-			fo.write("const struct vrt_ctx *")
+			s += "const struct vrt_ctx *"
 			p = ", "
 		if self.pfx != None:
-			fo.write(p + self.pfx)
+			s += p + self.pfx
 			p = ", "
 		for a in self.al:
-			fo.write(p + ctypes[a.typ])
+			s += p + ctypes[a.typ]
 			p = ", "
 			if a.nam != None:
-				fo.write(" " + a.nam)
-		fo.write(");\n")
+				s += " " + a.nam
+		s += ");"
+		return s
 
 	def c_typedef(self, modname, fini=False):
 		s = "typedef "
@@ -359,21 +388,25 @@ class func(object):
 
 	def c_struct(self, modname):
 		s = '\ttd_' + modname + "_" + self.cnam
-		while len(s.expandtabs()) < 40:
-			s += "\t"
+		if len(s.expandtabs()) >= 40:
+			s += "\n\t\t\t\t\t"
+		else:
+			while len(s.expandtabs()) < 40:
+				s += "\t"
 		s += "*" + self.cnam + ";\n"
 		return s
 
 	def c_initializer(self):
 		return "\tvmod_" + self.cnam + ",\n"
 
-	def c_strspec(self, modnam):
-		s = modnam + "." + self.nam
-		s += "\\0"
-		s += "Vmod_" + modnam + "_Func." + self.cnam + "\\0"
-		s += self.retval + "\\0"
+	def c_strspec(self, modnam, pfx = "\t"):
+		s = pfx + '"' + modnam + "." + self.nam + '\\0"\n'
+		s += pfx + '"'
+		s += "Vmod_" + modnam + "_Func." + self.cnam + '\\0"\n'
+		s += pfx + '    "' + self.retval + '\\0"\n'
 		for a in self.al:
-			s += a.c_strspec()
+			s += pfx + '\t"' + a.c_strspec() + '"\n'
+		s += pfx + '"\\0"'
 		return s
 
 	def doc(self, l):
@@ -453,10 +486,13 @@ class obj(object):
 
 	def c_proto(self, fo):
 		fo.write(self.st + ";\n")
-		self.init.c_proto(fo)
-		self.fini.c_proto(fo, fini=True)
+		l = []
+		l += lwrap(self.init.c_proto())
+		l += lwrap(self.fini.c_proto(fini=True))
 		for m in self.methods:
-			m.c_proto(fo)
+			l += lwrap(m.c_proto())
+		for i in l:
+			fo.write(i + "\n")
 
 	def c_struct(self, modnam):
 		s = "\t/* Object " + self.nam + " */\n"
@@ -477,12 +513,12 @@ class obj(object):
 	def c_strspec(self, modnam):
 		s = "\t/* Object " + self.nam + " */\n"
 		s += '\t"OBJ\\0"\n'
-		s += '\t\t"' + self.init.c_strspec(modnam) + '\\0"\n'
+		s += self.init.c_strspec(modnam, pfx="\t\t") + '\n'
 		s += '\t\t"' + self.st + '\\0"\n'
-		s += '\t\t"' + self.fini.c_strspec(modnam) + '\\0"\n'
+		s += self.fini.c_strspec(modnam, pfx="\t\t") + '\n'
 		for m in self.methods:
-			s += '\t\t"' + m.c_strspec(modnam) + '\\0"\n'
-		s += '\t\t"\\0",\n'
+			s += m.c_strspec(modnam, pfx="\t\t") + '\n'
+		s += '\t"\\0"'
 		return s
 
 	def doc(self, l):