From phk at varnish-cache.org  Mon Dec  3 09:12:56 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 03 Dec 2012 10:12:56 +0100
Subject: [master] 8c113ff Make sure that the ban-lurker cleans the tail at
 least once every 10 loops. Simplify the structure of the code some more.
Message-ID: <E1TfS5A-0006Cy-9w@project.varnish-software.com>

commit 8c113ff64b78554f6219f14774fed418c466318c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 3 09:12:21 2012 +0000

    Make sure that the ban-lurker cleans the tail at least once every
    10 loops.  Simplify the structure of the code some more.
    
    Prodded by:	martin

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 688842b..985057f 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -818,25 +818,30 @@ BAN_CheckObject(struct object *o, struct req *req)
 	return (ban_check_object(o, req->vsl, req->http) > 0);
 }
 
-static struct ban *
-ban_CheckLast(void)
+static void
+ban_cleantail(void)
 {
 	struct ban *b;
 
-	Lck_AssertHeld(&ban_mtx);
-	b = VTAILQ_LAST(&ban_head, banhead_s);
-	if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
-		if (b->flags & BAN_F_GONE)
-			VSC_C_main->bans_gone--;
-		if (b->flags & BAN_F_REQ)
-			VSC_C_main->bans_req--;
-		VSC_C_main->bans--;
-		VSC_C_main->bans_deleted++;
-		VTAILQ_REMOVE(&ban_head, b, list);
-	} else {
-		b = NULL;
-	}
-	return (b);
+	do {
+		Lck_Lock(&ban_mtx);
+		b = VTAILQ_LAST(&ban_head, banhead_s);
+		if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
+			if (b->flags & BAN_F_GONE)
+				VSC_C_main->bans_gone--;
+			if (b->flags & BAN_F_REQ)
+				VSC_C_main->bans_req--;
+			VSC_C_main->bans--;
+			VSC_C_main->bans_deleted++;
+			VTAILQ_REMOVE(&ban_head, b, list);
+			STV_BanInfo(BI_DROP, b->spec, ban_len(b->spec));
+		} else {
+			b = NULL;
+		}
+		Lck_Unlock(&ban_mtx);
+		if (b != NULL)
+			BAN_Free(b);
+	} while (b != NULL);
 }
 
 /*--------------------------------------------------------------------
@@ -983,41 +988,29 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 static void * __match_proto__(bgthread_t)
 ban_lurker(struct worker *wrk, void *priv)
 {
-	struct ban *bf;
 	struct vsl_log vsl;
+	volatile double d;
+	int i = 0, n = 0;
 
-	int i = 0;
 	VSL_Setup(&vsl, NULL, 0);
 
 	(void)priv;
 	while (1) {
-
-		do {
-			/*
-			 * Ban-lurker is disabled:
-			 * Clean the last ban, if possible, and sleep
-			 */
-			Lck_Lock(&ban_mtx);
-			bf = ban_CheckLast();
-			if (bf != NULL)
-				/* Notify stevedores */
-				STV_BanInfo(BI_DROP, bf->spec,
-					    ban_len(bf->spec));
-			Lck_Unlock(&ban_mtx);
-			if (bf != NULL)
-				BAN_Free(bf);
-		} while (bf != NULL);
-
-		if (cache_param->ban_lurker_sleep != 0.0) {
-			do {
-				i = ban_lurker_work(wrk, &vsl);
-				VSL_Flush(&vsl, 0);
-				WRK_SumStat(wrk);
-				if (i)
-					VTIM_sleep(
-					    cache_param->ban_lurker_sleep);
-			} while (i);
+		d = cache_param->ban_lurker_sleep;
+		if (d > 0.0) {
+			i = ban_lurker_work(wrk, &vsl);
+			VSL_Flush(&vsl, 0);
+			WRK_SumStat(wrk);
+			if (i) {
+				VTIM_sleep(d);
+				if (++n > 10) {
+					ban_cleantail();
+					n = 0;
+				}
+				continue;
+			}
 		}
+		ban_cleantail();
 		VTIM_sleep(0.609);	// Random, non-magic
 	}
 	NEEDLESS_RETURN(NULL);



From tfheen at varnish-cache.org  Mon Dec  3 10:58:45 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 03 Dec 2012 11:58:45 +0100
Subject: [master] f118ecf Get rid of ban_url/ban.url
Message-ID: <E1TfTjZ-000707-T7@project.varnish-software.com>

commit f118ecf114ce729c77910cc8ed985dca28aaee38
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Fri Nov 23 11:04:48 2012 +0100

    Get rid of ban_url/ban.url
    
    ban.url is confusing as it takes a regular expression rather than a
    fixed string, so get rid of it in favour of people being explicit and
    using ban req.url ~ /foo

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 985057f..95c655e 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -1059,21 +1059,6 @@ ccf_ban(struct cli *cli, const char * const *av, void *priv)
 }
 
 static void
-ccf_ban_url(struct cli *cli, const char * const *av, void *priv)
-{
-	const char *aav[6];
-
-	(void)priv;
-	aav[0] = NULL;
-	aav[1] = "ban";
-	aav[2] = "req.url";
-	aav[3] = "~";
-	aav[4] = av[2];
-	aav[5] = NULL;
-	ccf_ban(cli, aav, priv);
-}
-
-static void
 ban_render(struct cli *cli, const uint8_t *bs)
 {
 	struct ban_test bt;
@@ -1145,7 +1130,6 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 }
 
 static struct cli_proto ban_cmds[] = {
-	{ CLI_BAN_URL,				"", ccf_ban_url },
 	{ CLI_BAN,				"", ccf_ban },
 	{ CLI_BAN_LIST,				"", ccf_ban_list },
 	{ NULL }
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 9359800..0e4440d 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -683,7 +683,7 @@ static const struct parspec input_parspec[] = {
 		"Objects already cached will not be affected by changes "
 		"made until they are fetched from the backend again.\n"
 		"To force an immediate effect at the expense of a total "
-		"flush of the cache use \"ban.url .\"",
+		"flush of the cache use \"ban obj.http.date ~ .\"",
 		0,
 		"120", "seconds" },
 	{ "workspace_client",
diff --git a/bin/varnishtest/tests/c00006.vtc b/bin/varnishtest/tests/c00006.vtc
index 9342886..83c3ea2 100644
--- a/bin/varnishtest/tests/c00006.vtc
+++ b/bin/varnishtest/tests/c00006.vtc
@@ -20,7 +20,7 @@ client c1 {
 
 client c1 -run
 
-varnish v1 -cli "ban.url foo"
+varnish v1 -cli "ban req.url ~ foo"
 
 client c1 {
 	txreq -url "/foo"
diff --git a/bin/varnishtest/tests/c00019.vtc b/bin/varnishtest/tests/c00019.vtc
index 5cc5973..22b5ca1 100644
--- a/bin/varnishtest/tests/c00019.vtc
+++ b/bin/varnishtest/tests/c00019.vtc
@@ -13,7 +13,7 @@ server s1 {
 
 varnish v1 -vcl+backend {} -start
 
-varnish v1 -cliok "ban.url FOO"
+varnish v1 -cliok "ban req.url ~ FOO"
 
 # There is one "magic" ban from boot
 varnish v1 -expect bans_added == 2
@@ -35,7 +35,7 @@ varnish v1 -expect bans_tested == 0
 varnish v1 -expect bans_tests_tested == 0
 
 # Add another ban
-varnish v1 -cliok "ban.url FOO"
+varnish v1 -cliok "ban req.url ~ FOO"
 varnish v1 -expect bans_added == 3
 varnish v1 -cliok "ban.list"
 
@@ -60,15 +60,15 @@ client c1 {
 
 
 # Now add another two bans, Kilroy should not be hit
-varnish v1 -cliok "ban.url KILROY"
-varnish v1 -cliok "ban.url FOO"
+varnish v1 -cliok "ban req.url ~ KILROY"
+varnish v1 -cliok "ban req.url ~ FOO"
 varnish v1 -expect bans_added == 5
 
 # Enable dup removal of bans
 varnish v1 -cliok "param.set ban_dups on"
 
 # This should incapacitate the two previous FOO bans.
-varnish v1 -cliok "ban.url FOO"
+varnish v1 -cliok "ban req.url ~ FOO"
 varnish v1 -expect bans_added == 6
 varnish v1 -expect bans_dups == 3
 varnish v1 -cliok "ban.list"
@@ -87,4 +87,4 @@ varnish v1 -cliok "ban.list"
 
 # Test a bogus regexp
 
-varnish v1 -clierr 106 "ban.url [[["
+varnish v1 -clierr 106 "ban req.url ~ [[["
diff --git a/bin/varnishtest/tests/v00011.vtc b/bin/varnishtest/tests/v00011.vtc
index 43ea8c2..dc8bd18 100644
--- a/bin/varnishtest/tests/v00011.vtc
+++ b/bin/varnishtest/tests/v00011.vtc
@@ -11,7 +11,7 @@ server s1 {
 varnish v1 -vcl+backend { 
 	sub vcl_recv {
 		if (req.request == "PURGE") {
-			ban_url("^/$");
+			ban("req.url ~ ^/$");
 			error 209 "foo";
 		}
 	}
diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index 7a9ccbc..6fe35fb 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -86,7 +86,7 @@ varnish v1 -errvcl {Only http header variables can be unset.} {
 
 varnish v1 -errvcl {Unknown token 'if' when looking for STRING} {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_recv { ban_url (if); }
+	sub vcl_recv { ban (if); }
 }
 
 varnish v1 -errvcl {Expected an action, 'if', '{' or '}'} {
@@ -96,7 +96,7 @@ varnish v1 -errvcl {Expected an action, 'if', '{' or '}'} {
 
 varnish v1 -vcl {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_recv { ban_url ("foo"); }
+	sub vcl_recv { ban ("req.url ~ foo"); }
 }
 
 varnish v1 -errvcl {Expected an action, 'if', '{' or '}'} {
diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index 43fd590..787c09d 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -118,13 +118,6 @@ ban.list
 
       Then follows the actual ban it self.
 
-ban.url regexp
-      Immediately invalidate all documents whose URL matches the
-      specified regular expression. Please note that the Host part of
-      the URL is ignored, so if you have several virtual hosts all of
-      them will be banned. Use *ban* to specify a complete ban if you
-      need to narrow it down.
-
 help [command]
       Display a list of available commands.
       If the command is specified, display help for this command.
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 50c339b..98c1c47 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -327,9 +327,6 @@ regsuball(str, regex, sub)
 ban(ban expression)
   Bans all objects in cache that match the expression.
 
-ban_url(regex)
-  Bans all objects in cache whose URLs match regex.
-
 Subroutines
 ~~~~~~~~~~~
 
@@ -574,7 +571,7 @@ Example:
 	sub vcl_recv {
 	  if (client.ip ~ admin_network) {
 	    if (req.http.Cache-Control ~ "no-cache") {
-	      ban_url(req.url);
+	      ban("req.url ~ " + req.url);
 	    }
 	  }
 	}
diff --git a/include/vcli.h b/include/vcli.h
index 04ac11c..3294dd7 100644
--- a/include/vcli.h
+++ b/include/vcli.h
@@ -58,13 +58,6 @@
 	    "\tReturns the TTL, size and checksum of the object.",	\
 	1, 1
 
-#define CLI_BAN_URL							\
-	"ban.url",							\
-	"ban.url <regexp>",						\
-	"\tAll objects where the urls matches regexp will be "		\
-	    "marked obsolete.",						\
-	1, 1
-
 #define CLI_BAN								\
 	"ban",								\
 	"ban <field> <operator> <arg> [&& <field> <oper> <arg>]...",	\
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index eae6231..1eec1c4 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -195,24 +195,6 @@ parse_ban(struct vcc *tl)
 /*--------------------------------------------------------------------*/
 
 static void
-parse_ban_url(struct vcc *tl)
-{
-
-	vcc_NextToken(tl);
-	ExpectErr(tl, '(');
-	vcc_NextToken(tl);
-
-	Fb(tl, 1, "VRT_ban(req, \"req.url\", \"~\", ");
-	vcc_Expr(tl, STRING);
-	ERRCHK(tl);
-	ExpectErr(tl, ')');
-	vcc_NextToken(tl);
-	Fb(tl, 0, ", 0);\n");
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
 parse_new_syntax(struct vcc *tl)
 {
 
@@ -324,7 +306,6 @@ static struct action_table {
 	{ "call",		parse_call },
 	{ "hash_data",		parse_hash_data, VCL_MET_HASH },
 	{ "ban",		parse_ban },
-	{ "ban_url",		parse_ban_url },
 	{ "remove",		parse_unset }, /* backward compatibility */
 	{ "return",		parse_return },
 	{ "rollback",		parse_rollback },



From tfheen at varnish-cache.org  Mon Dec  3 10:58:45 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 03 Dec 2012 11:58:45 +0100
Subject: [master] 264bc4c s/obj.cookie/obj.http.set-cookie/
Message-ID: <E1TfTjZ-00070D-UE@project.varnish-software.com>

commit 264bc4c704618475f6170625a17d72d7e0fde40b
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Fri Nov 23 14:54:12 2012 +0100

    s/obj.cookie/obj.http.set-cookie/

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 95c655e..6e532fa 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -33,7 +33,7 @@
  *
  *	req.url == "/foo"
  *	req.url ~ ".iso" && obj.size > 10MB
- *	req.http.host ~ "web1.com" && obj.set-cookie ~ "USER=29293"
+ *	req.http.host ~ "web1.com" && obj.http.set-cookie ~ "USER=29293"
  *
  * We make the "&&" mandatory from the start, leaving the syntax space
  * for latter handling of "||" as well.
diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index 787c09d..88b05fe 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -306,7 +306,7 @@ Example: Ban all documents where the serving host is "example.com"
 or "www.example.com", and where the Set-Cookie header received from
 the backend contains "USERID=1663"::
 
-    req.http.host ~ "^(?i)(www\.)example.com$" && obj.set-cookie ~ "USERID=1663"
+    req.http.host ~ "^(?i)(www\.)example.com$" && obj.http.set-cookie ~ "USERID=1663"
 
 SEE ALSO
 ========



From tfheen at varnish-cache.org  Mon Dec  3 10:58:45 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 03 Dec 2012 11:58:45 +0100
Subject: [master] d4f456f Stop referencing the module index,
 since no such thing exists
Message-ID: <E1TfTjZ-00070J-VA@project.varnish-software.com>

commit d4f456f0326d02a94aed5c0b8143e77f6d180dc9
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Dec 3 11:25:08 2012 +0100

    Stop referencing the module index, since no such thing exists
    
    Fixes #1233

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index a663170..9d948b4 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -29,7 +29,6 @@ Indices and tables
 ==================
 
 * :ref:`genindex`
-* :ref:`modindex`
 * :ref:`search`
 
 
diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index 88b05fe..8fc4205 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -83,10 +83,11 @@ backend.set_health matcher state
       Sets the health state on a specific backend. This is useful if
       you want to take a certain backend out of sirculations.
 
-ban   *field operator argument* [&& field operator argument [...]]
-      Immediately invalidate all documents matching the ban
-      expression.  See *Ban Expressions* for more documentation and
-      examples.
+ban [-t ttl] [-g grace] [-k keep] *field operator argument* [&& field operator argument [...]]
+      Sets the given ttl, grace and keep for objects matching the ban
+      expression.  If none of ttl, grace or keep are given, they all
+      get set to -1, definitively removing the object from cache.  See
+      *Ban Expressions* for more documentation and examples.
 
 ban.list
       All requests for objects from the cache are matched against
@@ -199,7 +200,6 @@ The argument could be a quoted string, a regexp, or an integer.
 Integers can have "KB", "MB", "GB" or "TB" appended for size related
 fields.
 
-
 Scripting
 ---------
 
@@ -295,18 +295,23 @@ EXAMPLES
 Simple example: All requests where req.url exactly matches the string
 /news are banned from the cache::
 
-    req.url == "/news"
+    ban req.url == "/news"
+
+Example: Same as above, but explicit in what the TTL, grace and keep
+values are set to.
+
+    ban -t -1s -g -1s -k -1s req.url == "/news"
 
-Example: Ban all documents where the name does not end with ".ogg",
-and where the size of the object is greater than 10 megabytes::
+Example: Set grace for all objects whose URL end with `.ogg` to 60
+seconds.  Does not change the TTL or the keep of the objects.
 
-    req.url !~ "\.ogg$" && obj.size > 10MB
+    ban -g 60s req.url !~ "\.ogg$"
 
 Example: Ban all documents where the serving host is "example.com"
 or "www.example.com", and where the Set-Cookie header received from
 the backend contains "USERID=1663"::
 
-    req.http.host ~ "^(?i)(www\.)example.com$" && obj.http.set-cookie ~ "USERID=1663"
+    ban req.http.host ~ "^(?i)(www\.)example.com$" && obj.http.set-cookie ~ "USERID=1663"
 
 SEE ALSO
 ========
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 98c1c47..9327ff9 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -324,8 +324,11 @@ regsub(str, regex, sub)
 regsuball(str, regex, sub)
   As regsub() but this replaces all occurrences.
 
-ban(ban expression)
-  Bans all objects in cache that match the expression.
+ban(ban expression [, ttl= $ttl][, grace= $grace][, keep= $keep])
+  Sets the given ttl, grace and keep for objects matching the ban
+  expression.  If none of ttl, grace or keep are given, they all get
+  set to -1, definitively removing the object from cache.  See *Ban
+  Expressions* in varnish-cli(7) for more documentation and examples.
 
 Subroutines
 ~~~~~~~~~~~
@@ -956,11 +959,34 @@ for object invalidation:
     }
   }
 
+TTL, grace and keep ? when are objects removed from the cache?
+--------------------------------------------------------------
+
+There are two mechanisms that remove objects from the cache: LRU,
+which happens when the cache is full, and expiry.
+
+Objects exist in one of multiple states:
+
+ * Valid: The TTL has not yet expired and the object is served as normal
+ * Graced: The TTL has expired, but its grace period has not expired.
+     It is a grace candidate and might be delivered to clients.  See grace.
+ * Kept: The TTL has expired, but the keep period has not yet
+     expired. The object is a candidate to be used for conditional
+     requests to the backend.
+
+The keep and grace intervals run in parallel, so an object will be
+kept in the cache until the time period specified by the maximum of
+TTL + grace and TTL + keep has expired.  Once all the timers have
+expired, the object will be removed from the cache and any requests
+for it will cause a completely new object to be fetched from the
+backend.
+
 SEE ALSO
 ========
 
 * varnishd(1)
 * vmod_std(7)
+* varnish-cli(7)
 
 HISTORY
 =======



From tfheen at varnish-cache.org  Mon Dec  3 11:01:44 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 03 Dec 2012 12:01:44 +0100
Subject: [master] 665318c Revert "Stop referencing the module index,
 since no such thing exists"
Message-ID: <E1TfTmS-0000PF-KG@project.varnish-software.com>

commit 665318c2b7f5e0a0fe6d84ecf4e271ea5abb0e0f
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Dec 3 12:00:46 2012 +0100

    Revert "Stop referencing the module index, since no such thing exists"
    
    This commit included som unfinished work, so revert it.
    
    This reverts commit d4f456f0326d02a94aed5c0b8143e77f6d180dc9.

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index 9d948b4..a663170 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -29,6 +29,7 @@ Indices and tables
 ==================
 
 * :ref:`genindex`
+* :ref:`modindex`
 * :ref:`search`
 
 
diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index 8fc4205..88b05fe 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -83,11 +83,10 @@ backend.set_health matcher state
       Sets the health state on a specific backend. This is useful if
       you want to take a certain backend out of sirculations.
 
-ban [-t ttl] [-g grace] [-k keep] *field operator argument* [&& field operator argument [...]]
-      Sets the given ttl, grace and keep for objects matching the ban
-      expression.  If none of ttl, grace or keep are given, they all
-      get set to -1, definitively removing the object from cache.  See
-      *Ban Expressions* for more documentation and examples.
+ban   *field operator argument* [&& field operator argument [...]]
+      Immediately invalidate all documents matching the ban
+      expression.  See *Ban Expressions* for more documentation and
+      examples.
 
 ban.list
       All requests for objects from the cache are matched against
@@ -200,6 +199,7 @@ The argument could be a quoted string, a regexp, or an integer.
 Integers can have "KB", "MB", "GB" or "TB" appended for size related
 fields.
 
+
 Scripting
 ---------
 
@@ -295,23 +295,18 @@ EXAMPLES
 Simple example: All requests where req.url exactly matches the string
 /news are banned from the cache::
 
-    ban req.url == "/news"
-
-Example: Same as above, but explicit in what the TTL, grace and keep
-values are set to.
-
-    ban -t -1s -g -1s -k -1s req.url == "/news"
+    req.url == "/news"
 
-Example: Set grace for all objects whose URL end with `.ogg` to 60
-seconds.  Does not change the TTL or the keep of the objects.
+Example: Ban all documents where the name does not end with ".ogg",
+and where the size of the object is greater than 10 megabytes::
 
-    ban -g 60s req.url !~ "\.ogg$"
+    req.url !~ "\.ogg$" && obj.size > 10MB
 
 Example: Ban all documents where the serving host is "example.com"
 or "www.example.com", and where the Set-Cookie header received from
 the backend contains "USERID=1663"::
 
-    ban req.http.host ~ "^(?i)(www\.)example.com$" && obj.http.set-cookie ~ "USERID=1663"
+    req.http.host ~ "^(?i)(www\.)example.com$" && obj.http.set-cookie ~ "USERID=1663"
 
 SEE ALSO
 ========
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 9327ff9..98c1c47 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -324,11 +324,8 @@ regsub(str, regex, sub)
 regsuball(str, regex, sub)
   As regsub() but this replaces all occurrences.
 
-ban(ban expression [, ttl= $ttl][, grace= $grace][, keep= $keep])
-  Sets the given ttl, grace and keep for objects matching the ban
-  expression.  If none of ttl, grace or keep are given, they all get
-  set to -1, definitively removing the object from cache.  See *Ban
-  Expressions* in varnish-cli(7) for more documentation and examples.
+ban(ban expression)
+  Bans all objects in cache that match the expression.
 
 Subroutines
 ~~~~~~~~~~~
@@ -959,34 +956,11 @@ for object invalidation:
     }
   }
 
-TTL, grace and keep ? when are objects removed from the cache?
---------------------------------------------------------------
-
-There are two mechanisms that remove objects from the cache: LRU,
-which happens when the cache is full, and expiry.
-
-Objects exist in one of multiple states:
-
- * Valid: The TTL has not yet expired and the object is served as normal
- * Graced: The TTL has expired, but its grace period has not expired.
-     It is a grace candidate and might be delivered to clients.  See grace.
- * Kept: The TTL has expired, but the keep period has not yet
-     expired. The object is a candidate to be used for conditional
-     requests to the backend.
-
-The keep and grace intervals run in parallel, so an object will be
-kept in the cache until the time period specified by the maximum of
-TTL + grace and TTL + keep has expired.  Once all the timers have
-expired, the object will be removed from the cache and any requests
-for it will cause a completely new object to be fetched from the
-backend.
-
 SEE ALSO
 ========
 
 * varnishd(1)
 * vmod_std(7)
-* varnish-cli(7)
 
 HISTORY
 =======



From tfheen at varnish-cache.org  Mon Dec  3 11:01:44 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 03 Dec 2012 12:01:44 +0100
Subject: [master] 8762363 Stop referencing the module index,
 since no such thing exists
Message-ID: <E1TfTmS-0000PL-LK@project.varnish-software.com>

commit 8762363b9ec1eeca715f97eae0189759075a8c2a
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Dec 3 11:25:08 2012 +0100

    Stop referencing the module index, since no such thing exists
    
    Fixes #1233

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index a663170..9d948b4 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -29,7 +29,6 @@ Indices and tables
 ==================
 
 * :ref:`genindex`
-* :ref:`modindex`
 * :ref:`search`
 
 



From tfheen at varnish-cache.org  Mon Dec  3 11:27:38 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 03 Dec 2012 12:27:38 +0100
Subject: [3.0] 41c153b Stop referencing the module index,
 since no such thing exists
Message-ID: <E1TfUBW-00047z-Pe@project.varnish-software.com>

commit 41c153b599bfd41f3757563d42f610f441761474
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Dec 3 11:25:08 2012 +0100

    Stop referencing the module index, since no such thing exists
    
    Fixes #1233

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index 2558413..2dbd605 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -29,7 +29,6 @@ Indices and tables
 ==================
 
 * :ref:`genindex`
-* :ref:`modindex`
 * :ref:`search`
 
 



From tfheen at varnish-cache.org  Mon Dec  3 11:52:56 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 03 Dec 2012 12:52:56 +0100
Subject: [master] ca8dda4 Make vcl.use slightly more chatty
Message-ID: <E1TfUa0-0007FU-B0@project.varnish-software.com>

commit ca8dda44cf4c7a6460093c37c7b9129f60effdc6
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Dec 3 12:52:15 2012 +0100

    Make vcl.use slightly more chatty
    
    Output that we've switched VCLs and which one we have switched to when
    using vcl.use.
    
    Fixes #1232

diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index d2d077f..28ac476 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -607,6 +607,7 @@ mcf_config_use(struct cli *cli, const char * const *av, void *priv)
 		VCLI_SetResult(cli, status);
 		VCLI_Out(cli, "%s", p);
 	} else {
+		VCLI_Out(cli, "VCL '%s' now active", av[2]);
 		vp->active = 2;
 		VTAILQ_FOREACH(vp, &vclhead, list) {
 			if (vp->active == 1)



From phk at varnish-cache.org  Mon Dec 10 08:34:07 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 10 Dec 2012 09:34:07 +0100
Subject: [master] 5121efb A couple of style nits
Message-ID: <E1ThyoR-0008LR-3g@project.varnish-software.com>

commit 5121efb32fc90e5ba09b22133a89d1cfdc9421be
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 10 08:33:54 2012 +0000

    A couple of style nits

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 0a209a8..e50a9a5 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -215,7 +215,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		assert(bo->state >= BOS_FAILED);
 
 		if (bo->state == BOS_FAILED) {
-			HSH_Deref(&wrk->stats, NULL, &req->obj);
+			(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
 			VBO_DerefBusyObj(wrk, &req->busyobj);
 			req->err_code = 503;
 			req->req_step = R_STP_ERROR;
@@ -679,7 +679,7 @@ cnt_fetchbody(struct worker *wrk, struct req *req)
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 	} else if (bo->state == BOS_FAILED) {
 		/* handle early failures */
-		HSH_Deref(&wrk->stats, NULL, &req->obj);
+		(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 		req->err_code = 503;
 		req->req_step = R_STP_ERROR;
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index b54717c..5c19997 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -282,6 +282,7 @@ ccf_config_discard(struct cli *cli, const char * const *av, void *priv)
 
 	ASSERT_CLI();
 	AZ(priv);
+	(void)priv;
 	vcl = vcl_find(av[2]);
 	if (vcl == NULL) {
 		VCLI_SetResult(cli, CLIS_PARAM);



From phk at varnish-cache.org  Mon Dec 10 10:54:24 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 10 Dec 2012 11:54:24 +0100
Subject: [master] 881dcc3 Remove a pseudo-bogus test for pthreads in the
 manager process.
Message-ID: <E1Ti10C-0006qm-35@project.varnish-software.com>

commit 881dcc3d7e2384ccf6f56de162ceebbe75d68384
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 10 10:53:03 2012 +0000

    Remove a pseudo-bogus test for pthreads in the manager process.
    
    The subsequent cleanup/separation of .h files makes this test
    much less important, and it's not like it is a conclusive or
    comprehensive test in the first place.
    
    Fixes	#1230

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 1b7f1e3..fe6e489 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -655,7 +655,3 @@ main(int argc, char * const *argv)
 		(void)VPF_Remove(pfh);
 	exit(exit_status);
 }
-
-#if defined(PTHREAD_CANCELED) || defined(PTHREAD_MUTEX_DEFAULT)
-#error "Keep pthreads out of in manager process"
-#endif



From martin at varnish-cache.org  Thu Dec 13 14:17:15 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 13 Dec 2012 15:17:15 +0100
Subject: [master] fef35ed Add a BAN_Shutdown() routine that is called before
 STV_Close(), and makes sure that the ban lists will not change until we exit.
Message-ID: <E1Tj9b9-0008Ry-Fx@project.varnish-software.com>

commit fef35edcb553d0f1ebbd3dca1d02a57cfb6aff9c
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Nov 19 17:21:53 2012 +0100

    Add a BAN_Shutdown() routine that is called before STV_Close(), and
    makes sure that the ban lists will not change until we exit.
    
    This is to give persistent stevedores a chance to clean up their ban
    lists without having to worry about the lists changing under them.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index bc07923..4b29167 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -746,8 +746,9 @@ struct ban *BAN_New(void);
 int BAN_AddTest(struct cli *, struct ban *, const char *, const char *,
     const char *);
 void BAN_Free(struct ban *b);
-void BAN_Insert(struct ban *b);
+int BAN_Insert(struct ban *b);
 void BAN_Init(void);
+void BAN_Shutdown(void);
 void BAN_NewObjCore(struct objcore *oc);
 void BAN_DestroyObj(struct objcore *oc);
 int BAN_CheckObject(struct object *o, struct req *sp);
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 6e532fa..7b9adf5 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -105,6 +105,7 @@ static struct ban *ban_magic;
 static pthread_t ban_thread;
 static struct ban * volatile ban_start;
 static bgthread_t ban_lurker;
+static int ban_shutdown = 0;
 
 /*--------------------------------------------------------------------
  * BAN string defines & magic markers
@@ -417,9 +418,14 @@ BAN_AddTest(struct cli *cli, struct ban *b, const char *a1, const char *a2,
  * as a separate variable from the VTAILQ, to avoid depending on the
  * internals of the VTAILQ macros.  We tacitly assume that a pointer
  * write is always atomic in doing so.
+ *
+ * Returns:
+ *   0: Ban successfully inserted
+ *  -1: Ban not inserted due to shutdown in progress. The ban has been
+ *      deleted.
  */
 
-void
+int
 BAN_Insert(struct ban *b)
 {
 	struct ban  *bi, *be;
@@ -428,6 +434,11 @@ BAN_Insert(struct ban *b)
 
 	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
 
+	if (ban_shutdown) {
+		BAN_Free(b);
+		return (-1);
+	}
+
 	AZ(VSB_finish(b->vsb));
 	ln = VSB_len(b->vsb);
 	assert(ln >= 0);
@@ -446,6 +457,12 @@ BAN_Insert(struct ban *b)
 	b->vsb = NULL;
 
 	Lck_Lock(&ban_mtx);
+	if (ban_shutdown) {
+		/* Check again, we might have raced */
+		Lck_Unlock(&ban_mtx);
+		BAN_Free(b);
+		return (-1);
+	}
 	VTAILQ_INSERT_HEAD(&ban_head, b, list);
 	ban_start = b;
 	VSC_C_main->bans++;
@@ -463,12 +480,12 @@ BAN_Insert(struct ban *b)
 	Lck_Unlock(&ban_mtx);
 
 	if (be == NULL)
-		return;
+		return (0);
 
 	/* Hunt down duplicates, and mark them as gone */
 	bi = b;
 	Lck_Lock(&ban_mtx);
-	while(bi != be) {
+	while(!ban_shutdown && bi != be) {
 		bi = VTAILQ_NEXT(bi, list);
 		if (bi->flags & BAN_F_GONE)
 			continue;
@@ -479,6 +496,8 @@ BAN_Insert(struct ban *b)
 	}
 	be->refcount--;
 	Lck_Unlock(&ban_mtx);
+
+	return (0);
 }
 
 /*--------------------------------------------------------------------
@@ -562,6 +581,7 @@ BAN_Reload(const uint8_t *ban, unsigned len)
 	double t0, t1, t2 = 9e99;
 
 	ASSERT_CLI();
+	AZ(ban_shutdown);
 
 	t0 = ban_time(ban);
 	assert(len == ban_len(ban));
@@ -639,6 +659,7 @@ BAN_Compile(void)
 {
 
 	ASSERT_CLI();
+	AZ(ban_shutdown);
 
 	/* Notify stevedores */
 	STV_BanInfo(BI_NEW, ban_magic->spec, ban_len(ban_magic->spec));
@@ -891,6 +912,8 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 			    ban_time(b->spec), b->refcount);
 		while (1) {
 			Lck_Lock(&ban_mtx);
+			if (ban_shutdown)
+				break;
 			oc = VTAILQ_FIRST(&b->objcore);
 			if (oc == NULL)
 				break;
@@ -974,6 +997,8 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 				    ban_time(b->spec));
 		}
 		Lck_Unlock(&ban_mtx);
+		if (ban_shutdown)
+			break;
 		VTIM_sleep(cache_param->ban_lurker_sleep);
 		if (b == b0)
 			break;
@@ -995,13 +1020,13 @@ ban_lurker(struct worker *wrk, void *priv)
 	VSL_Setup(&vsl, NULL, 0);
 
 	(void)priv;
-	while (1) {
+	while (!ban_shutdown) {
 		d = cache_param->ban_lurker_sleep;
 		if (d > 0.0) {
 			i = ban_lurker_work(wrk, &vsl);
 			VSL_Flush(&vsl, 0);
 			WRK_SumStat(wrk);
-			if (i) {
+			if (i && !ban_shutdown) {
 				VTIM_sleep(d);
 				if (++n > 10) {
 					ban_cleantail();
@@ -1011,8 +1036,13 @@ ban_lurker(struct worker *wrk, void *priv)
 			}
 		}
 		ban_cleantail();
+		if (ban_shutdown)
+			break;
 		VTIM_sleep(0.609);	// Random, non-magic
 	}
+
+	pthread_exit(0);
+
 	NEEDLESS_RETURN(NULL);
 }
 
@@ -1055,7 +1085,11 @@ ccf_ban(struct cli *cli, const char * const *av, void *priv)
 			BAN_Free(b);
 			return;
 		}
-	BAN_Insert(b);
+	if (BAN_Insert(b) < 0) {
+		VCLI_Out(cli, "Shutdown in progress");
+		VCLI_SetResult(cli, CLIS_CANT);
+		return;
+	}
 }
 
 static void
@@ -1151,3 +1185,24 @@ BAN_Init(void)
 	VSC_C_main->bans_gone++;
 	BAN_Insert(ban_magic);
 }
+
+/*--------------------------------------------------------------------
+ * Shutdown of the ban system.
+ *
+ * When this function returns, no new bans will be accepted, and no
+ * bans will be dropped (ban lurker thread stopped), so that no
+ * STV_BanInfo calls will be executed.
+ */
+
+void
+BAN_Shutdown(void)
+{
+	void *status;
+
+	Lck_Lock(&ban_mtx);
+	ban_shutdown = 1;
+	Lck_Unlock(&ban_mtx);
+
+	AZ(pthread_join(ban_thread, &status));
+	AZ(status);
+}
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 468401b..13c6612 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -232,6 +232,7 @@ child_main(void)
 
 	CLI_Run();
 
+	BAN_Shutdown();
 	STV_close();
 
 	printf("Child dies\n");
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index ed7ae59..914531e 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -459,10 +459,9 @@ VRT_ban(const struct req *req, char *cmds, ...)
 		good = 1;
 	}
 	if (!good)
-		/* XXX: report error how ? */
-		BAN_Free(b);
+		BAN_Free(b);		/* XXX: report error how ? */
 	else
-		BAN_Insert(b);
+		(void)BAN_Insert(b);	/* XXX: report error how ? */
 }
 
 /*--------------------------------------------------------------------*/
@@ -506,10 +505,9 @@ VRT_ban_string(const struct req *req, const char *str)
 			break;
 	}
 	if (!good)
-		/* XXX: report error how ? */
-		BAN_Free(b);
+		BAN_Free(b);		/* XXX: report error how ? */
 	else
-		BAN_Insert(b);
+		(void)BAN_Insert(b);	/* XXX: report error how ? */
 	VAV_Free(av);
 }
 



From martin at varnish-cache.org  Thu Dec 13 14:17:15 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 13 Dec 2012 15:17:15 +0100
Subject: [master] dbd7ddc Allow reload of a series of persisted ban specs in
 one go.
Message-ID: <E1Tj9b9-0008S4-Hj@project.varnish-software.com>

commit dbd7ddc0c6158121b8511272bc76fe891f856f96
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Nov 19 13:46:56 2012 +0100

    Allow reload of a series of persisted ban specs in one go.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 7b9adf5..ef03ab2 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -573,29 +573,25 @@ BAN_RefBan(struct objcore *oc, double t0, const struct ban *tail)
  * mark any older bans, with the same condition, GONE as well.
  */
 
-void
-BAN_Reload(const uint8_t *ban, unsigned len)
+static void
+ban_reload(const uint8_t *ban, unsigned len)
 {
 	struct ban *b, *b2;
 	int gone = 0;
 	double t0, t1, t2 = 9e99;
 
 	ASSERT_CLI();
-	AZ(ban_shutdown);
+	Lck_AssertHeld(&ban_mtx);
 
 	t0 = ban_time(ban);
 	assert(len == ban_len(ban));
 
-	Lck_Lock(&ban_mtx);
-
 	VTAILQ_FOREACH(b, &ban_head, list) {
 		t1 = ban_time(b->spec);
 		assert(t1 < t2);
 		t2 = t1;
-		if (t1 == t0) {
-			Lck_Unlock(&ban_mtx);
+		if (t1 == t0)
 			return;
-		}
 		if (t1 < t0)
 			break;
 		if (ban_equal(b->spec, ban)) {
@@ -632,6 +628,30 @@ BAN_Reload(const uint8_t *ban, unsigned len)
 			VSC_C_main->bans_dups++;
 		}
 	}
+}
+
+/*--------------------------------------------------------------------
+ * Reload a series of persisted ban specs
+ */
+
+void
+BAN_Reload(const uint8_t *ptr, unsigned len)
+{
+	const uint8_t *pe;
+	unsigned l;
+
+	AZ(ban_shutdown);
+	pe = ptr + len;
+	Lck_Lock(&ban_mtx);
+	while (ptr < pe) {
+		/* XXX: This can be optimized by traversing the live
+		 * ban list together with the reload list (combining
+		 * the loops in BAN_Reload and ban_reload). */
+		l = ban_len(ptr);
+		assert(ptr + l <= pe);
+		ban_reload(ptr, l);
+		ptr += l;
+	}
 	Lck_Unlock(&ban_mtx);
 }
 



From martin at varnish-cache.org  Thu Dec 13 14:17:15 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 13 Dec 2012 15:17:15 +0100
Subject: [master] 535fbe7 Drop the internal ban delimiters in the persistent
 ban lists, and reload all the statements in one go.
Message-ID: <E1Tj9b9-0008SA-J3@project.varnish-software.com>

commit 535fbe74c8940f7c1b1723bbf3478475b3def30c
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Nov 19 14:09:41 2012 +0100

    Drop the internal ban delimiters in the persistent ban lists, and
    reload all the statements in one go.

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 24a4d08..1efa24b 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -72,22 +72,12 @@ static void
 smp_appendban(struct smp_sc *sc, struct smp_signspace *spc,
     uint32_t len, const uint8_t *ban)
 {
-	uint8_t *ptr, *ptr2;
 
 	(void)sc;
-	ptr = ptr2 = SIGNSPACE_FRONT(spc);
-	assert(SIGNSPACE_FREE(spc) >= 4L + 4 + len);
+	assert(SIGNSPACE_FREE(spc) >=  len);
 
-	memcpy(ptr, "BAN", 4);
-	ptr += 4;
-
-	vbe32enc(ptr, len);
-	ptr += 4;
-
-	memcpy(ptr, ban, len);
-	ptr += len;
-
-	smp_append_signspace(spc, ptr - ptr2);
+	memcpy(SIGNSPACE_FRONT(spc), ban, len);
+	smp_append_signspace(spc, len);
 }
 
 /* Trust that cache_ban.c takes care of locking */
@@ -120,38 +110,19 @@ static int
 smp_open_bans(struct smp_sc *sc, struct smp_signspace *spc)
 {
 	uint8_t *ptr, *pe;
-	uint32_t length;
-	int i, retval = 0;
+	int i;
 
 	ASSERT_CLI();
 	(void)sc;
 	i = smp_chk_signspace(spc);
 	if (i)
 		return (i);
+
 	ptr = SIGNSPACE_DATA(spc);
 	pe = SIGNSPACE_FRONT(spc);
+	BAN_Reload(ptr, pe - ptr);
 
-	while (ptr < pe) {
-		if (memcmp(ptr, "BAN", 4)) {
-			retval = 1001;
-			break;
-		}
-		ptr += 4;
-
-		length = vbe32dec(ptr);
-		ptr += 4;
-
-		if (ptr + length > pe) {
-			retval = 1003;
-			break;
-		}
-
-		BAN_Reload(ptr, length);
-
-		ptr += length;
-	}
-	assert(ptr <= pe);
-	return (retval);
+	return (0);
 }
 
 /*--------------------------------------------------------------------



From martin at varnish-cache.org  Thu Dec 13 14:17:15 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 13 Dec 2012 15:17:15 +0100
Subject: [master] 3d406aa Don't report ban_magic to the stevedores on
 insertion, but delay it until ban compile.
Message-ID: <E1Tj9b9-0008SI-KU@project.varnish-software.com>

commit 3d406aa85cfd3caf7f458b03a899b037d5398339
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Dec 13 14:18:44 2012 +0100

    Don't report ban_magic to the stevedores on insertion, but delay it
    until ban compile.
    
    Document the reason.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index ef03ab2..09958a2 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -476,7 +476,15 @@ BAN_Insert(struct ban *b)
 	else
 		be = NULL;
 
-	STV_BanInfo(BI_NEW, b->spec, ln);	/* Notify stevedores */
+	/* ban_magic is magic, and needs to be inserted early to give
+	 * a handle to grab a ref on. We don't report it here as the
+	 * stevedores will not be opened and ready to accept it
+	 * yet. Instead it is reported on BAN_Compile, which is after
+	 * the stevedores has been opened, but before any new objects
+	 * can have entered the cache (thus no objects in the mean
+	 * time depending on ban_magic in the list) */
+	if (b != ban_magic)
+		STV_BanInfo(BI_NEW, b->spec, ln); /* Notify stevedores */
 	Lck_Unlock(&ban_mtx);
 
 	if (be == NULL)
@@ -681,7 +689,7 @@ BAN_Compile(void)
 	ASSERT_CLI();
 	AZ(ban_shutdown);
 
-	/* Notify stevedores */
+	/* Do late reporting of ban_magic */
 	STV_BanInfo(BI_NEW, ban_magic->spec, ban_len(ban_magic->spec));
 
 	ban_start = VTAILQ_FIRST(&ban_head);



From martin at varnish-cache.org  Thu Dec 13 14:17:15 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 13 Dec 2012 15:17:15 +0100
Subject: [master] 8e646cf Fix bug in smp_baninfo causing duplicate ban entries.
Message-ID: <E1Tj9b9-0008SU-MZ@project.varnish-software.com>

commit 8e646cfb1704ce63e1879642e002eb64854191d5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Nov 6 14:52:48 2012 +0100

    Fix bug in smp_baninfo causing duplicate ban entries.

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 1efa24b..07ca29c 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -88,13 +88,12 @@ smp_baninfo(struct stevedore *stv, enum baninfo event,
 {
 	struct smp_sc *sc;
 
-	(void)stv;
+	CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC);
+
 	switch (event) {
 	case BI_NEW:
-		VTAILQ_FOREACH(sc, &silos, list) {
-			smp_appendban(sc, &sc->ban1, len, ban);
-			smp_appendban(sc, &sc->ban2, len, ban);
-		}
+		smp_appendban(sc, &sc->ban1, len, ban);
+		smp_appendban(sc, &sc->ban2, len, ban);
 		break;
 	default:
 		/* Ignored */



From martin at varnish-cache.org  Thu Dec 13 14:17:15 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 13 Dec 2012 15:17:15 +0100
Subject: [master] 53e74e4 Report BAN event persistence failures back to
 cache_ban.c
Message-ID: <E1Tj9b9-0008Sf-O4@project.varnish-software.com>

commit 53e74e4dea157d6d36d040d5882be3e3841f5fef
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Nov 30 10:01:37 2012 +0100

    Report BAN event persistence failures back to cache_ban.c

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 4b29167..6a0d1a4 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1060,7 +1060,7 @@ void STV_free(struct storage *st);
 void STV_open(void);
 void STV_close(void);
 void STV_Freestore(struct object *o);
-void STV_BanInfo(enum baninfo event, const uint8_t *ban, unsigned len);
+int STV_BanInfo(enum baninfo event, const uint8_t *ban, unsigned len);
 
 /* storage_synth.c */
 struct vsb *SMS_Makesynth(struct object *obj);
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 09958a2..92499b0 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -484,7 +484,7 @@ BAN_Insert(struct ban *b)
 	 * can have entered the cache (thus no objects in the mean
 	 * time depending on ban_magic in the list) */
 	if (b != ban_magic)
-		STV_BanInfo(BI_NEW, b->spec, ln); /* Notify stevedores */
+		AZ(STV_BanInfo(BI_NEW, b->spec, ln)); /* Notify stevedores */
 	Lck_Unlock(&ban_mtx);
 
 	if (be == NULL)
@@ -690,7 +690,7 @@ BAN_Compile(void)
 	AZ(ban_shutdown);
 
 	/* Do late reporting of ban_magic */
-	STV_BanInfo(BI_NEW, ban_magic->spec, ban_len(ban_magic->spec));
+	AZ(STV_BanInfo(BI_NEW, ban_magic->spec, ban_len(ban_magic->spec)));
 
 	ban_start = VTAILQ_FIRST(&ban_head);
 	WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL);
@@ -883,7 +883,7 @@ ban_cleantail(void)
 			VSC_C_main->bans--;
 			VSC_C_main->bans_deleted++;
 			VTAILQ_REMOVE(&ban_head, b, list);
-			STV_BanInfo(BI_DROP, b->spec, ban_len(b->spec));
+			AZ(STV_BanInfo(BI_DROP, b->spec, ban_len(b->spec)));
 		} else {
 			b = NULL;
 		}
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 5262b70..be16057 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -453,14 +453,23 @@ STV_close(void)
 		stv->close(stv);
 }
 
-void
+/*-------------------------------------------------------------------
+ * Notify the stevedores of BAN related events. A non-zero return
+ * value indicates that the stevedore is unable to persist the
+ * event.
+ */
+
+int
 STV_BanInfo(enum baninfo event, const uint8_t *ban, unsigned len)
 {
 	struct stevedore *stv;
+	int r = 0;
 
 	VTAILQ_FOREACH(stv, &stv_stevedores, list)
 		if (stv->baninfo != NULL)
-			stv->baninfo(stv, event, ban, len);
+			r |= stv->baninfo(stv, event, ban, len);
+
+	return (r);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index e9971e7..cddf6ef 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -48,7 +48,7 @@ typedef struct object *storage_allocobj_f(struct stevedore *, struct busyobj *,
     struct objcore **, unsigned ltot, const struct stv_objsecrets *);
 typedef void storage_close_f(const struct stevedore *);
 typedef void storage_signal_close_f(const struct stevedore *);
-typedef void storage_baninfo_f(struct stevedore *, enum baninfo event,
+typedef int storage_baninfo_f(struct stevedore *, enum baninfo event,
     const uint8_t *ban, unsigned len);
 
 /* Prototypes for VCL variable responders */
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 07ca29c..89b60cc 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -68,37 +68,43 @@ static VTAILQ_HEAD(,smp_sc)	silos = VTAILQ_HEAD_INITIALIZER(silos);
  * Add bans to silos
  */
 
-static void
+static int
 smp_appendban(struct smp_sc *sc, struct smp_signspace *spc,
     uint32_t len, const uint8_t *ban)
 {
 
 	(void)sc;
-	assert(SIGNSPACE_FREE(spc) >=  len);
+	if (SIGNSPACE_FREE(spc) < len)
+		return (-1);
 
 	memcpy(SIGNSPACE_FRONT(spc), ban, len);
 	smp_append_signspace(spc, len);
+
+	return (0);
 }
 
 /* Trust that cache_ban.c takes care of locking */
 
-static void
+static int
 smp_baninfo(struct stevedore *stv, enum baninfo event,
 	    const uint8_t *ban, unsigned len)
 {
 	struct smp_sc *sc;
+	int r = 0;
 
 	CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC);
 
 	switch (event) {
 	case BI_NEW:
-		smp_appendban(sc, &sc->ban1, len, ban);
-		smp_appendban(sc, &sc->ban2, len, ban);
+		r |= smp_appendban(sc, &sc->ban1, len, ban);
+		r |= smp_appendban(sc, &sc->ban2, len, ban);
 		break;
 	default:
 		/* Ignored */
 		break;
 	}
+
+	return (r);
 }
 
 /*--------------------------------------------------------------------



From martin at varnish-cache.org  Thu Dec 13 14:17:15 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 13 Dec 2012 15:17:15 +0100
Subject: [master] 4c2344b Add a facility to export a complete ban list to the
 stevedores. Call this on startup, after reading the persisted ban lists,
 to make sure all stevedores has all the bans from every stevedore stored.
Message-ID: <E1Tj9b9-0008So-Pp@project.varnish-software.com>

commit 4c2344ba0b3728c5b68f46f6c147c44a3d8bf143
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Nov 30 11:28:24 2012 +0100

    Add a facility to export a complete ban list to the stevedores. Call
    this on startup, after reading the persisted ban lists, to make sure
    all stevedores has all the bans from every stevedore stored.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 6a0d1a4..f97f8a7 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1061,6 +1061,7 @@ void STV_open(void);
 void STV_close(void);
 void STV_Freestore(struct object *o);
 int STV_BanInfo(enum baninfo event, const uint8_t *ban, unsigned len);
+void STV_BanExport(const uint8_t *bans, unsigned len);
 
 /* storage_synth.c */
 struct vsb *SMS_Makesynth(struct object *obj);
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 92499b0..ec288d9 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -574,6 +574,29 @@ BAN_RefBan(struct objcore *oc, double t0, const struct ban *tail)
 }
 
 /*--------------------------------------------------------------------
+ * Compile a full ban list and export this area to the stevedores for
+ * persistence.
+ */
+
+static void
+ban_export(void)
+{
+	struct ban *b;
+	struct vsb vsb;
+
+	Lck_AssertHeld(&ban_mtx);
+	/* XXX: Use the ban entry size measurements to hit the target
+	 * and avoid multiple allocations */
+	AN(VSB_new(&vsb, NULL, 64 * VSC_C_main->bans, VSB_AUTOEXTEND));
+	VTAILQ_FOREACH_REVERSE(b, &ban_head, banhead_s, list) {
+		AZ(VSB_bcat(&vsb, b->spec, ban_len(b->spec)));
+	}
+	AZ(VSB_finish(&vsb));
+	STV_BanExport((const uint8_t *)VSB_data(&vsb), VSB_len(&vsb));
+	VSB_delete(&vsb);
+}
+
+/*--------------------------------------------------------------------
  * Put a skeleton ban in the list, unless there is an identical,
  * time & condition, ban already in place.
  *
@@ -689,9 +712,17 @@ BAN_Compile(void)
 	ASSERT_CLI();
 	AZ(ban_shutdown);
 
+	Lck_Lock(&ban_mtx);
+
 	/* Do late reporting of ban_magic */
 	AZ(STV_BanInfo(BI_NEW, ban_magic->spec, ban_len(ban_magic->spec)));
 
+	/* All bans have been read from all persistent stevedores. Export
+	   the compiled list */
+	ban_export();
+
+	Lck_Unlock(&ban_mtx);
+
 	ban_start = VTAILQ_FIRST(&ban_head);
 	WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL);
 }
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index be16057..63400eb 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -472,6 +472,22 @@ STV_BanInfo(enum baninfo event, const uint8_t *ban, unsigned len)
 	return (r);
 }
 
+/*-------------------------------------------------------------------
+ * Export a complete ban list to the stevedores for persistence.
+ * The stevedores should clear any previous ban lists and replace
+ * them with this list.
+ */
+
+void
+STV_BanExport(const uint8_t *bans, unsigned len)
+{
+	struct stevedore *stv;
+
+	VTAILQ_FOREACH(stv, &stv_stevedores, list)
+		if (stv->banexport != NULL)
+			stv->banexport(stv, bans, len);
+}
+
 /*--------------------------------------------------------------------
  * VRT functions for stevedores
  */
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index cddf6ef..4a69bb0 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -50,6 +50,8 @@ typedef void storage_close_f(const struct stevedore *);
 typedef void storage_signal_close_f(const struct stevedore *);
 typedef int storage_baninfo_f(struct stevedore *, enum baninfo event,
     const uint8_t *ban, unsigned len);
+typedef void storage_banexport_f(struct stevedore *, const uint8_t *bans,
+    unsigned len);
 
 /* Prototypes for VCL variable responders */
 #define VRTSTVTYPE(ct) typedef ct storage_var_##ct(const struct stevedore *);
@@ -74,6 +76,7 @@ struct stevedore {
 	storage_allocobj_f	*allocobj;	/* --//-- */
 	storage_signal_close_f	*signal_close;	/* --//-- */
 	storage_baninfo_f	*baninfo;	/* --//-- */
+	storage_banexport_f	*banexport;	/* --//-- */
 
 	struct lru		*lru;
 
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 89b60cc..3d0c3fa 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -107,6 +107,26 @@ smp_baninfo(struct stevedore *stv, enum baninfo event,
 	return (r);
 }
 
+static void
+smp_banexport_spc(struct smp_signspace *spc, const uint8_t *bans, unsigned len)
+{
+	smp_reset_signspace(spc);
+	assert(SIGNSPACE_FREE(spc) >= len);
+	memcpy(SIGNSPACE_DATA(spc), bans, len);
+	smp_append_signspace(spc, len);
+	smp_sync_sign(&spc->ctx);
+}
+
+static void
+smp_banexport(struct stevedore *stv, const uint8_t *bans, unsigned len)
+{
+	struct smp_sc *sc;
+
+	CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC);
+	smp_banexport_spc(&sc->ban1, bans, len);
+	smp_banexport_spc(&sc->ban2, bans, len);
+}
+
 /*--------------------------------------------------------------------
  * Attempt to open and read in a ban list
  */
@@ -579,6 +599,7 @@ const struct stevedore smp_stevedore = {
 	.free	=	smp_free,
 	.signal_close = smp_signal_close,
 	.baninfo =	smp_baninfo,
+	.banexport =	smp_banexport,
 };
 
 /*--------------------------------------------------------------------



From martin at varnish-cache.org  Thu Dec 13 14:17:15 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 13 Dec 2012 15:17:15 +0100
Subject: [master] e4d28b8 Try to export the ban list on STV_BanInfo failures.
Message-ID: <E1Tj9b9-0008T1-RM@project.varnish-software.com>

commit e4d28b8ea95c02e7c5fac193b8cc2bbebff16597
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Nov 30 12:05:06 2012 +0100

    Try to export the ban list on STV_BanInfo failures.
    
    When STV_BanInfo fails, it means the stevedore has run out of space to
    persist the list. An export should descrease the size.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index ec288d9..09cd7d9 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -99,6 +99,7 @@ struct ban_test {
 	const void		*arg2_spec;
 };
 
+static void ban_info(enum baninfo event, const uint8_t *ban, unsigned len);
 static VTAILQ_HEAD(banhead_s,ban) ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
 static struct lock ban_mtx;
 static struct ban *ban_magic;
@@ -484,7 +485,7 @@ BAN_Insert(struct ban *b)
 	 * can have entered the cache (thus no objects in the mean
 	 * time depending on ban_magic in the list) */
 	if (b != ban_magic)
-		AZ(STV_BanInfo(BI_NEW, b->spec, ln)); /* Notify stevedores */
+		ban_info(BI_NEW, b->spec, ln); /* Notify stevedores */
 	Lck_Unlock(&ban_mtx);
 
 	if (be == NULL)
@@ -596,6 +597,21 @@ ban_export(void)
 	VSB_delete(&vsb);
 }
 
+static void
+ban_info(enum baninfo event, const uint8_t *ban, unsigned len)
+{
+	if (STV_BanInfo(event, ban, len)) {
+		/* One or more stevedores reported failure. Export the
+		 * list instead. The exported list should take up less
+		 * space due to drops being purged and gones being
+		 * truncated. */
+		/* XXX: Keep some measure of how much space can be
+		 * saved, and only export if it's worth it. Assert if
+		 * not */
+		ban_export();
+	}
+}
+
 /*--------------------------------------------------------------------
  * Put a skeleton ban in the list, unless there is an identical,
  * time & condition, ban already in place.
@@ -914,7 +930,7 @@ ban_cleantail(void)
 			VSC_C_main->bans--;
 			VSC_C_main->bans_deleted++;
 			VTAILQ_REMOVE(&ban_head, b, list);
-			AZ(STV_BanInfo(BI_DROP, b->spec, ban_len(b->spec)));
+			ban_info(BI_DROP, b->spec, ban_len(b->spec));
 		} else {
 			b = NULL;
 		}



From martin at varnish-cache.org  Thu Dec 13 14:17:15 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 13 Dec 2012 15:17:15 +0100
Subject: [master] 679fe13 Export the ban list on shutdown to compact it
Message-ID: <E1Tj9b9-0008TC-Sl@project.varnish-software.com>

commit 679fe13a854d109e01e60aa7a47154e0c09b3b3f
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Nov 30 12:14:39 2012 +0100

    Export the ban list on shutdown to compact it

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 09cd7d9..47e4f70 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -1280,4 +1280,9 @@ BAN_Shutdown(void)
 
 	AZ(pthread_join(ban_thread, &status));
 	AZ(status);
+
+	Lck_Lock(&ban_mtx);
+	/* Export the ban list to compact it */
+	ban_export();
+	Lck_Unlock(&ban_mtx);
 }



From martin at varnish-cache.org  Thu Dec 13 14:17:15 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 13 Dec 2012 15:17:15 +0100
Subject: [master] 0296c84 Add a test case for ban list syncing across silos
Message-ID: <E1Tj9b9-0008TN-UM@project.varnish-software.com>

commit 0296c8441ebe8e36766b89b5651343bfe9e42241
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Nov 30 13:18:54 2012 +0100

    Add a test case for ban list syncing across silos

diff --git a/bin/varnishtest/tests/p00008.vtc b/bin/varnishtest/tests/p00008.vtc
new file mode 100644
index 0000000..e2ab62f
--- /dev/null
+++ b/bin/varnishtest/tests/p00008.vtc
@@ -0,0 +1,80 @@
+varnishtest "Ban list sync across silos"
+
+shell "rm -f ${tmpdir}/_.per[12]"
+
+# Silo 1 & 2
+# Prime each with an object with X-Foo: foo
+server s1 {
+	rxreq
+	expect req.url == "/silo1"
+	txresp -hdr "X-Foo: foo"
+
+	rxreq
+	expect req.url == "/silo2"
+	txresp -hdr "X-Foo: foo"
+} -start
+varnish v1 \
+	-arg "-pfeature=+wait_silo" \
+	-arg "-pban_lurker_sleep=0" \
+	-storage "-sper1=persistent,${tmpdir}/_.per1,10m -sper2=persistent,${tmpdir}/_.per2,10m" \
+	-vcl+backend {
+		sub vcl_fetch {
+			set beresp.storage = "per1";
+			if (req.url ~ "silo2") {
+				set beresp.storage = "per2";
+			}
+		}
+	} -start 
+client c1 {
+	txreq -url "/silo1"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.x-foo == "foo"
+
+	txreq -url "/silo2"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.x-foo == "foo"
+} -run
+varnish v1 -stop
+server s1 -wait
+
+# Only silo 1
+# Ban on obj.http.x-foo == foo
+varnish v2 \
+	-arg "-pfeature=+wait_silo" \
+	-arg "-pban_lurker_sleep=0" \
+	-storage "-spersistent,${tmpdir}/_.per1,10m" \
+	-vcl+backend { } -start
+varnish v2 -cliok "ban obj.http.x-foo == foo"
+varnish v2 -cliok "ban.list"
+varnish v2 -stop
+
+# Silo 1 & 2
+# Bans should be transferred
+varnish v3 \
+	-arg "-pfeature=+wait_silo" \
+	-arg "-pban_lurker_sleep=0" \
+	-storage "-spersistent,${tmpdir}/_.per1,10m -spersistent,${tmpdir}/_.per2,10m" \
+	-vcl+backend { } -start
+varnish v3 -cliok "ban.list"
+varnish v3 -stop
+
+# Only silo 2
+# Check that /silo2 is banned
+server s1 {
+	rxreq 
+	expect req.url == "/silo2"
+	txresp -hdr "X-Foo: bar"
+} -start
+varnish v4 \
+	-arg "-pfeature=+wait_silo" \
+	-arg "-pban_lurker_sleep=0" \
+	-storage "-spersistent,${tmpdir}/_.per2,10m" \
+	-vcl+backend { } -start
+client c1 -connect ${v4_sock} {
+	txreq -url "/silo2"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.x-foo == "bar"
+} -run



From martin at varnish-cache.org  Thu Dec 13 14:17:15 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 13 Dec 2012 15:17:15 +0100
Subject: [master] ae090cf Fix test case
Message-ID: <E1Tj9b9-0008TY-W8@project.varnish-software.com>

commit ae090cf1ac8aab0b9339c32c289f370c0ec485c9
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Nov 30 13:25:07 2012 +0100

    Fix test case
    
    Varnishtest -storage option can not be specified multiple times to
    append. Fix the test case to list all the silos in one -storage
    option.

diff --git a/bin/varnishtest/tests/p00002.vtc b/bin/varnishtest/tests/p00002.vtc
index dc6e584..53bf892 100644
--- a/bin/varnishtest/tests/p00002.vtc
+++ b/bin/varnishtest/tests/p00002.vtc
@@ -10,8 +10,7 @@ server s1 {
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0" \
-	-storage "-spersistent,${tmpdir}/_.per1,10m" \
-	-storage "-spersistent,${tmpdir}/_.per2,10m" \
+	-storage "-spersistent,${tmpdir}/_.per1,10m -spersistent,${tmpdir}/_.per2,10m" \
 	-vcl+backend { } -start 
 
 client c1 {



From martin at varnish-cache.org  Fri Dec 14 11:18:56 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Fri, 14 Dec 2012 12:18:56 +0100
Subject: [master] 801f3a6 Set the split character back to '=' after applying
 the runtime parameter to preserve the arguments as seen e.g. ps.
Message-ID: <E1TjTI8-0002ps-Mu@project.varnish-software.com>

commit 801f3a62f00eea3f02023650d2f3ac144240d538
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Dec 13 13:13:38 2012 +0100

    Set the split character back to '=' after applying the runtime
    parameter to preserve the arguments as seen e.g. ps.
    
    Fixes: #1238

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index fe6e489..2b3c709 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -465,6 +465,7 @@ main(int argc, char * const *argv)
 			AN(p);
 			*p++ = '\0';
 			MCF_ParamSet(cli, optarg, p);
+			*--p = '=';
 			cli_check(cli);
 			break;
 		case 'r':



From phk at varnish-cache.org  Sat Dec 15 10:16:14 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 15 Dec 2012 11:16:14 +0100
Subject: [master] e74a5a0 Constify VSB_len() and VSB_data()
Message-ID: <E1Tjon0-00039E-5W@project.varnish-software.com>

commit e74a5a03f537b5715647345425a27ed279230314
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Dec 15 10:16:00 2012 +0000

    Constify VSB_len() and VSB_data()

diff --git a/include/vsb.h b/include/vsb.h
index e17fe5b..555b17f 100644
--- a/include/vsb.h
+++ b/include/vsb.h
@@ -76,8 +76,8 @@ int		 VSB_putc(struct vsb *, int);
 int		 VSB_trim(struct vsb *);
 int		 VSB_error(const struct vsb *);
 int		 VSB_finish(struct vsb *);
-char		*VSB_data(struct vsb *);
-ssize_t		 VSB_len(struct vsb *);
+char		*VSB_data(const struct vsb *);
+ssize_t		 VSB_len(const struct vsb *);
 int		 VSB_done(const struct vsb *);
 void		 VSB_delete(struct vsb *);
 void		 VSB_quote(struct vsb *s, const char *p, int len, int how);
diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c
index 16f6d4b..a751046 100644
--- a/lib/libvarnish/vsb.c
+++ b/lib/libvarnish/vsb.c
@@ -77,7 +77,7 @@ __FBSDID("$FreeBSD: head/sys/kern/subr_vsb.c 222004 2011-05-17 06:36:32Z phk $")
  */
 #if !defined(NDEBUG)
 static void
-_assert_VSB_integrity(const char *fun, struct vsb *s)
+_assert_VSB_integrity(const char *fun, const struct vsb *s)
 {
 
 	(void)fun;
@@ -93,7 +93,7 @@ _assert_VSB_integrity(const char *fun, struct vsb *s)
 }
 
 static void
-_assert_VSB_state(const char *fun, struct vsb *s, int state)
+_assert_VSB_state(const char *fun, const struct vsb *s, int state)
 {
 
 	(void)fun;
@@ -490,7 +490,7 @@ VSB_finish(struct vsb *s)
  * Return a pointer to the vsb data.
  */
 char *
-VSB_data(struct vsb *s)
+VSB_data(const struct vsb *s)
 {
 
 	assert_VSB_integrity(s);
@@ -503,7 +503,7 @@ VSB_data(struct vsb *s)
  * Return the length of the vsb data.
  */
 ssize_t
-VSB_len(struct vsb *s)
+VSB_len(const struct vsb *s)
 {
 
 	assert_VSB_integrity(s);



From phk at varnish-cache.org  Sat Dec 15 10:18:19 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 15 Dec 2012 11:18:19 +0100
Subject: [master] f81b8e5 Assert BAN_Insert() success.
Message-ID: <E1Tjop1-0003Hy-5D@project.varnish-software.com>

commit f81b8e57fc46188b0e109792bd862dfb54767820
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Dec 15 10:17:07 2012 +0000

    Assert BAN_Insert() success.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 47e4f70..35c520b 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -1258,7 +1258,7 @@ BAN_Init(void)
 	AN(ban_magic);
 	ban_magic->flags |= BAN_F_GONE;
 	VSC_C_main->bans_gone++;
-	BAN_Insert(ban_magic);
+	AZ(BAN_Insert(ban_magic));
 }
 
 /*--------------------------------------------------------------------



From phk at varnish-cache.org  Sat Dec 15 10:18:19 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 15 Dec 2012 11:18:19 +0100
Subject: [master] df0f0fd Avoid "const const" duplication
Message-ID: <E1Tjop1-0003I4-7A@project.varnish-software.com>

commit df0f0fd1dff0c9c2a62375048ff388d46aa88aeb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Dec 15 10:17:36 2012 +0000

    Avoid "const const" duplication

diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index dbf31af..45f015e 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -843,7 +843,10 @@ for i in sp_variables:
 		fo.write('",\n')
 	else:
 		fo.write('\t    "VRT_r_%s(req)",\n' % cnam)
-		fh.write(ctyp + " VRT_r_%s(const %s);\n" % (cnam, i[4]))
+		if i[4][:5] != "const":
+			fh.write(ctyp + " VRT_r_%s(const %s);\n" % (cnam, i[4]))
+		else:
+			fh.write(ctyp + " VRT_r_%s(%s);\n" % (cnam, i[4]))
 	restrict(fo, i[2])
 
 	if len(i[3]) == 0:



From phk at varnish-cache.org  Sat Dec 15 10:18:19 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 15 Dec 2012 11:18:19 +0100
Subject: [master] 3c67950 Constification
Message-ID: <E1Tjop1-0003IA-8n@project.varnish-software.com>

commit 3c67950c6c416c4cdd40ac3f46d734dfb660356c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Dec 15 10:18:05 2012 +0000

    Constification

diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index b84e702..fb88308 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -178,7 +178,7 @@ void VBE_DropRefLocked(struct backend *b);
 void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p,
     const char *hosthdr);
 void VBP_Remove(struct backend *b, struct vrt_backend_probe const *p);
-void VBP_Use(const struct backend *b, const struct vrt_backend_probe const *p);
+void VBP_Use(const struct backend *b, const struct vrt_backend_probe *p);
 void VBP_Summary(struct cli *cli, const struct vbp_target *vt);
 
 /* Init functions for directors */
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index 4a69bb0..be6078b 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -48,9 +48,9 @@ typedef struct object *storage_allocobj_f(struct stevedore *, struct busyobj *,
     struct objcore **, unsigned ltot, const struct stv_objsecrets *);
 typedef void storage_close_f(const struct stevedore *);
 typedef void storage_signal_close_f(const struct stevedore *);
-typedef int storage_baninfo_f(struct stevedore *, enum baninfo event,
+typedef int storage_baninfo_f(const struct stevedore *, enum baninfo event,
     const uint8_t *ban, unsigned len);
-typedef void storage_banexport_f(struct stevedore *, const uint8_t *bans,
+typedef void storage_banexport_f(const struct stevedore *, const uint8_t *bans,
     unsigned len);
 
 /* Prototypes for VCL variable responders */
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 3d0c3fa..55279c8 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -49,7 +49,6 @@
 #include "hash/hash_slinger.h"
 #include "vcli.h"
 #include "vcli_priv.h"
-#include "vend.h"
 #include "vsha256.h"
 #include "vtim.h"
 
@@ -86,7 +85,7 @@ smp_appendban(struct smp_sc *sc, struct smp_signspace *spc,
 /* Trust that cache_ban.c takes care of locking */
 
 static int
-smp_baninfo(struct stevedore *stv, enum baninfo event,
+smp_baninfo(const struct stevedore *stv, enum baninfo event,
 	    const uint8_t *ban, unsigned len)
 {
 	struct smp_sc *sc;
@@ -118,7 +117,7 @@ smp_banexport_spc(struct smp_signspace *spc, const uint8_t *bans, unsigned len)
 }
 
 static void
-smp_banexport(struct stevedore *stv, const uint8_t *bans, unsigned len)
+smp_banexport(const struct stevedore *stv, const uint8_t *bans, unsigned len)
 {
 	struct smp_sc *sc;