From phk at projects.linpro.no  Mon May 26 08:36:18 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 26 May 2008 10:36:18 +0200 (CEST)
Subject: r2636 - trunk/varnish-cache/bin/varnishd
Message-ID: <20080526083618.AB97A1EC106@projects.linpro.no>

Author: phk
Date: 2008-05-26 10:36:18 +0200 (Mon, 26 May 2008)
New Revision: 2636

Modified:
   trunk/varnish-cache/bin/varnishd/mgt_param.c
Log:
Add compiler command for Solaris.

Closes ticket #239


Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-04-24 06:46:39 UTC (rev 2635)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-05-26 08:36:18 UTC (rev 2636)
@@ -616,7 +616,9 @@
 		MUST_RELOAD,
 #ifdef __APPLE__
 		"exec cc -dynamiclib -Wl,-undefined,dynamic_lookup -o %o %s"
-#else
+#elseif defined(__SOLARIS__)
+		"exec cc -shared -fpic -c %o %s"
+#else /* default: GCC on Linux & FreeBSD */
 		"exec cc -fpic -shared -Wl,-x -o %o %s"
 #endif
 		, NULL },



From phk at projects.linpro.no  Mon May 26 08:38:00 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 26 May 2008 10:38:00 +0200 (CEST)
Subject: r2637 - trunk/varnish-cache/bin/varnishd
Message-ID: <20080526083800.C0DE61EC106@projects.linpro.no>

Author: phk
Date: 2008-05-26 10:38:00 +0200 (Mon, 26 May 2008)
New Revision: 2637

Modified:
   trunk/varnish-cache/bin/varnishd/cache_fetch.c
   trunk/varnish-cache/bin/varnishd/common.h
   trunk/varnish-cache/bin/varnishd/tcp.c
Log:
Add TCP_blocking() and TCP_nonblocking() and use them instead of
fondling fcntl(2) directly.


Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c	2008-05-26 08:36:18 UTC (rev 2636)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c	2008-05-26 08:38:00 UTC (rev 2637)
@@ -65,9 +65,7 @@
 	sp->obj->len = cl;
 	p = st->ptr;
 
-	i = fcntl(htc->fd, F_GETFL);		/* XXX ? */
-	i &= ~O_NONBLOCK;
-	i = fcntl(htc->fd, F_SETFL, i);
+	TCP_blocking(htc->fd);
 
 	while (cl > 0) {
 		i = HTC_Read(htc, p, cl);
@@ -211,9 +209,7 @@
 	struct storage *st;
 	unsigned v;
 
-	i = fcntl(htc->fd, F_GETFL);		/* XXX ? */
-	i &= ~O_NONBLOCK;
-	i = fcntl(htc->fd, F_SETFL, i);
+	TCP_blocking(htc->fd);
 
 	p = NULL;
 	v = 0;

Modified: trunk/varnish-cache/bin/varnishd/common.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/common.h	2008-05-26 08:36:18 UTC (rev 2636)
+++ trunk/varnish-cache/bin/varnishd/common.h	2008-05-26 08:38:00 UTC (rev 2637)
@@ -44,5 +44,7 @@
 void TCP_name(const struct sockaddr *addr, unsigned l, char *abuf, unsigned alen, char *pbuf, unsigned plen);
 void TCP_myname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen);
 int TCP_filter_http(int sock);
+void TCP_blocking(int sock);
+void TCP_nonblocking(int sock);
 
 #define TRUST_ME(ptr)	((void*)(uintptr_t)(ptr))

Modified: trunk/varnish-cache/bin/varnishd/tcp.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/tcp.c	2008-05-26 08:36:18 UTC (rev 2636)
+++ trunk/varnish-cache/bin/varnishd/tcp.c	2008-05-26 08:38:00 UTC (rev 2637)
@@ -37,6 +37,7 @@
 #include <netinet/in.h>
 
 #include <errno.h>
+#include <fcntl.h>
 #include <netdb.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -110,3 +111,29 @@
 	return (0);
 #endif
 }
+
+/*--------------------------------------------------------------------*/
+
+void
+TCP_blocking(int sock)
+{
+	int i;
+
+	i = fcntl(sock, F_GETFL);
+	assert(i != -1);
+	i &= ~O_NONBLOCK;
+	i = fcntl(sock, F_SETFL, i);
+	assert(i != -1);
+}
+
+void
+TCP_nonblocking(int sock)
+{
+	int i;
+
+	i = fcntl(sock, F_GETFL);
+	assert(i != -1);
+	i |= O_NONBLOCK;
+	i = fcntl(sock, F_SETFL, i);
+	assert(i != -1);
+}



From phk at projects.linpro.no  Mon May 26 09:14:24 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 26 May 2008 11:14:24 +0200 (CEST)
Subject: r2638 - trunk/varnish-cache/bin/varnishd
Message-ID: <20080526091424.38E251EC0FD@projects.linpro.no>

Author: phk
Date: 2008-05-26 11:14:23 +0200 (Mon, 26 May 2008)
New Revision: 2638

Modified:
   trunk/varnish-cache/bin/varnishd/cache_fetch.c
Log:
Backend sockets are always in blocking mode.


Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c	2008-05-26 08:38:00 UTC (rev 2637)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c	2008-05-26 09:14:23 UTC (rev 2638)
@@ -65,8 +65,6 @@
 	sp->obj->len = cl;
 	p = st->ptr;
 
-	TCP_blocking(htc->fd);
-
 	while (cl > 0) {
 		i = HTC_Read(htc, p, cl);
 		if (i <= 0)
@@ -209,8 +207,6 @@
 	struct storage *st;
 	unsigned v;
 
-	TCP_blocking(htc->fd);
-
 	p = NULL;
 	v = 0;
 	st = NULL;



From phk at projects.linpro.no  Mon May 26 10:13:52 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 26 May 2008 12:13:52 +0200 (CEST)
Subject: r2639 - trunk/varnish-cache/bin/varnishd
Message-ID: <20080526101352.7A0DB1EC106@projects.linpro.no>

Author: phk
Date: 2008-05-26 12:13:52 +0200 (Mon, 26 May 2008)
New Revision: 2639

Modified:
   trunk/varnish-cache/bin/varnishd/cache_acceptor.c
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/mgt_child.c
   trunk/varnish-cache/bin/varnishd/tcp.c
Log:
Be more consistent about sockets and blocking/non-blocking mode:

Accept sockets are non-blocking, to avoid races where the client closes
before we get to accept it.  (Spotted by: "chen xiaoyong")

Unfortunately, that means that session sockets inherit non-blocking mode,
which is the opposite of what we want in the worker thread but correct
for the acceptor thread.

We prefer to have the extra syscalls in the worker thread, that complicates
things a little bit.

Use ioctl(FIONBIO) instead of fcntl(2) which is surprisingly expensive.


Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_acceptor.c	2008-05-26 09:14:23 UTC (rev 2638)
+++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c	2008-05-26 10:13:52 UTC (rev 2639)
@@ -270,6 +270,11 @@
 	AZ(sp->obj);
 	AZ(sp->vcl);
 	assert(sp->fd >= 0);
+	/*
+	 * Set nonblocking in the worker-thread, before passing to the
+	 * acceptor thread, to reduce syscall density of the latter.
+	 */
+	TCP_nonblocking(sp->fd);
 	if (vca_act->pass == NULL)
 		assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
 	else

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2008-05-26 09:14:23 UTC (rev 2638)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2008-05-26 10:13:52 UTC (rev 2639)
@@ -906,6 +906,16 @@
 	w = sp->wrk;
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 
+	/*
+	 * Whenever we come in from the acceptor we need to set blocking
+	 * mode, but there is no point in setting it when we come from
+	 * ESI or when a parked sessions returns.
+	 * It would be simpler to do this in the acceptor, but we'd rather
+	 * do the syscall in the worker thread.
+ 	 */
+	if (sp->step == STP_FIRST || sp->step == STP_START)
+		TCP_blocking(sp->fd);
+
 	for (done = 0; !done; ) {
 		assert(sp->wrk == w);
 		/*

Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_child.c	2008-05-26 09:14:23 UTC (rev 2638)
+++ trunk/varnish-cache/bin/varnishd/mgt_child.c	2008-05-26 10:13:52 UTC (rev 2639)
@@ -150,6 +150,12 @@
 			free(ls);
 			continue;
 		}
+		/*
+		 * Set nonblocking mode to avoid a race where a client
+		 * closes before we call accept(2) and nobody else are in
+		 * the listen queue to release us.
+		 */
+		TCP_nonblocking(ls->sock);
 		TCP_filter_http(ls->sock);
 		good++;
 	}

Modified: trunk/varnish-cache/bin/varnishd/tcp.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/tcp.c	2008-05-26 09:14:23 UTC (rev 2638)
+++ trunk/varnish-cache/bin/varnishd/tcp.c	2008-05-26 10:13:52 UTC (rev 2639)
@@ -37,7 +37,7 @@
 #include <netinet/in.h>
 
 #include <errno.h>
-#include <fcntl.h>
+#include <sys/ioctl.h>
 #include <netdb.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -112,18 +112,22 @@
 #endif
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Functions for controlling NONBLOCK mode.
+ * 
+ * We use FIONBIO because it is cheaper than fcntl(2), which requires
+ * us to do two syscalls, one to get and one to set, the latter of
+ * which mucks about a bit before it ends up calling ioctl(FIONBIO),
+ * at least on FreeBSD.
+ */
 
 void
 TCP_blocking(int sock)
 {
 	int i;
 
-	i = fcntl(sock, F_GETFL);
-	assert(i != -1);
-	i &= ~O_NONBLOCK;
-	i = fcntl(sock, F_SETFL, i);
-	assert(i != -1);
+	i = 0;
+	AZ(ioctl(sock, FIONBIO, &i));
 }
 
 void
@@ -131,9 +135,6 @@
 {
 	int i;
 
-	i = fcntl(sock, F_GETFL);
-	assert(i != -1);
-	i |= O_NONBLOCK;
-	i = fcntl(sock, F_SETFL, i);
-	assert(i != -1);
+	i = 1;
+	AZ(ioctl(sock, FIONBIO, &i));
 }



From phk at projects.linpro.no  Tue May 27 07:10:23 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 27 May 2008 09:10:23 +0200 (CEST)
Subject: r2640 - in trunk/varnish-cache: bin/varnishd include lib/libvcl man
Message-ID: <20080527071023.646341EC118@projects.linpro.no>

Author: phk
Date: 2008-05-27 09:10:22 +0200 (Tue, 27 May 2008)
New Revision: 2640

Modified:
   trunk/varnish-cache/bin/varnishd/cache_vrt_re.c
   trunk/varnish-cache/include/vrt.h
   trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c
   trunk/varnish-cache/lib/libvcl/vcc_string.c
   trunk/varnish-cache/man/vcl.7
Log:
Add a new vcl function regsuball()

It works just like regsub(), but replaces all occurrences of the regexp.

	regsub("1foofoofoo2", "foo", "bar") -> "1barfoofoo2"

	regsuball("1foofoofoo2", "foo", "bar") -> "1barbarbar2"

Fixes ticket 238



Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_re.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt_re.c	2008-05-26 10:13:52 UTC (rev 2639)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt_re.c	2008-05-27 07:10:22 UTC (rev 2640)
@@ -104,12 +104,13 @@
 }
 
 const char *
-VRT_regsub(const struct sess *sp, const char *str, void *re, const char *sub)
+VRT_regsub(const struct sess *sp, int all, const char *str, void *re, const char *sub)
 {
 	regmatch_t pm[10];
 	regex_t *t;
 	int i, l;
 	char *b, *p, *e;
+	const char *s;
 	unsigned u, x;
 
 	AN(re);
@@ -124,42 +125,48 @@
 	e = p = b = sp->http->ws->f;
 	e += u;
 
-	/* Copy prefix to match */
-	if (pm[0].rm_so > 0) {
-		if (p + pm[0].rm_so < e)
-			memcpy(p, str, pm[0].rm_so);
-		p += pm[0].rm_so;
-	}
+	do {
+		/* Copy prefix to match */
+		if (pm[0].rm_so > 0) {
+			if (p + pm[0].rm_so < e)
+				memcpy(p, str, pm[0].rm_so);
+			p += pm[0].rm_so;
+		}
 
-	for ( ; *sub != '\0'; sub++ ) {
-		if (*sub == '&') {
-			l = pm[0].rm_eo - pm[0].rm_so;
-			if (l > 0) {
-				if (p + l < e)
-					memcpy(p, str + pm[0].rm_so, l);
-				p += l;
+		for (s = sub ; *s != '\0'; s++ ) {
+			if (*s == '&') {
+				l = pm[0].rm_eo - pm[0].rm_so;
+				if (l > 0) {
+					if (p + l < e)
+						memcpy(p, str + pm[0].rm_so, l);
+					p += l;
+				}
+			} else if (*s == '$' && isdigit(s[1])) {
+				x = sub[1] - '0';
+				sub++;
+				l = pm[x].rm_eo - pm[x].rm_so;
+				if (l > 0) {
+					if (p + l < e)
+						memcpy(p, str + pm[x].rm_so, l);
+					p += l;
+				}
+			} else {
+				if (p + 1 < e)
+					*p = *s;
+				p++;
 			}
-		} else if (*sub == '$' && isdigit(sub[1])) {
-			x = sub[1] - '0';
-			sub++;
-			l = pm[x].rm_eo - pm[x].rm_so;
-			if (l > 0) {
-				if (p + l < e)
-					memcpy(p, str + pm[x].rm_so, l);
-				p += l;
-			}
-		} else {
-			if (p + 1 < e)
-				*p = *sub;
-			p++;
 		}
-	}
+		str += pm[0].rm_eo;
+		if (!all)
+			break;
+		i = regexec(t, str, 10, pm, 0);
+	} while (i != REG_NOMATCH);
 
 	/* Copy suffix to match */
-	l = strlen(str + pm[0].rm_eo);
+	l = strlen(str);
 	if (l > 0) {
 		if (p + l < e)
-			memcpy(p, str + pm[0].rm_eo, l);
+			memcpy(p, str, l);
 		p += l;
 	}
 	if (p + 1 < e)

Modified: trunk/varnish-cache/include/vrt.h
===================================================================
--- trunk/varnish-cache/include/vrt.h	2008-05-26 10:13:52 UTC (rev 2639)
+++ trunk/varnish-cache/include/vrt.h	2008-05-27 07:10:22 UTC (rev 2640)
@@ -108,7 +108,7 @@
 void VRT_re_fini(void *);
 int VRT_re_match(const char *, void *re);
 int VRT_re_test(struct vsb *, const char *, int sub);
-const char *VRT_regsub(const struct sess *sp, const char *, void *, const char *);
+const char *VRT_regsub(const struct sess *sp, int all, const char *, void *, const char *);
 
 void VRT_purge(const char *, int hash);
 

Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c	2008-05-26 10:13:52 UTC (rev 2639)
+++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c	2008-05-27 07:10:22 UTC (rev 2640)
@@ -462,7 +462,7 @@
 	vsb_cat(sb, "void VRT_re_fini(void *);\n");
 	vsb_cat(sb, "int VRT_re_match(const char *, void *re);\n");
 	vsb_cat(sb, "int VRT_re_test(struct vsb *, const char *, int sub);\n");
-	vsb_cat(sb, "const char *VRT_regsub(const struct sess *sp, const char *, void *, const char *);\n");
+	vsb_cat(sb, "const char *VRT_regsub(const struct sess *sp, int all, const char *, void *, const char *);\n");
 	vsb_cat(sb, "\n");
 	vsb_cat(sb, "void VRT_purge(const char *, int hash);\n");
 	vsb_cat(sb, "\n");
@@ -541,7 +541,7 @@
 	vsb_cat(sb, "double VRT_r_obj_prefetch(const struct sess *);\n");
 	vsb_cat(sb, "void VRT_l_obj_prefetch(const struct sess *, double);\n");
 	vsb_cat(sb, "double VRT_r_obj_lastuse(const struct sess *);\n");
-	vsb_cat(sb, "const char *VRT_r_obj_hash(struct sess *sp);\n");
+	vsb_cat(sb, "const char * VRT_r_obj_hash(const struct sess *);\n");
 	vsb_cat(sb, "const char * VRT_r_resp_proto(const struct sess *);\n");
 	vsb_cat(sb, "void VRT_l_resp_proto(const struct sess *, const char *, ...);\n");
 	vsb_cat(sb, "int VRT_r_resp_status(const struct sess *);\n");

Modified: trunk/varnish-cache/lib/libvcl/vcc_string.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_string.c	2008-05-26 10:13:52 UTC (rev 2639)
+++ trunk/varnish-cache/lib/libvcl/vcc_string.c	2008-05-27 07:10:22 UTC (rev 2640)
@@ -69,13 +69,13 @@
 /*--------------------------------------------------------------------*/
 
 static int
-vcc_regsub(struct tokenlist *tl)
+vcc_regsub(struct tokenlist *tl, int all)
 {
 	char *p;
 
 	vcc_NextToken(tl);
 
-	Fb(tl, 0, "VRT_regsub(sp, ");
+	Fb(tl, 0, "VRT_regsub(sp, %d, ", all);
 
 	Expect(tl, '(');
 	vcc_NextToken(tl);
@@ -96,7 +96,6 @@
 	Expect(tl, ',');
 	vcc_NextToken(tl);
 	
-	Expect(tl, CSTR);
 	if (!vcc_StringVal(tl)) {
 		vcc_ExpectedStringval(tl);
 		return (0);
@@ -129,7 +128,9 @@
 		return (1);
 	}
 	if (tl->t->tok == ID && vcc_IdIs(tl->t, "regsub"))
-		return (vcc_regsub(tl));
+		return (vcc_regsub(tl, 0));
+	if (tl->t->tok == ID && vcc_IdIs(tl->t, "regsuball"))
+		return (vcc_regsub(tl, 1));
 	if (tl->t->tok == VAR) {
 		vp = vcc_FindVar(tl, tl->t, vcc_vars);
 		if (tl->err)

Modified: trunk/varnish-cache/man/vcl.7
===================================================================
--- trunk/varnish-cache/man/vcl.7	2008-05-26 10:13:52 UTC (rev 2639)
+++ trunk/varnish-cache/man/vcl.7	2008-05-27 07:10:22 UTC (rev 2640)
@@ -122,7 +122,7 @@
 .It Fn regsub "str" "regex" "sub"
 Returns a copy of
 .Fa str
-with all occurrences of the regular expression
+with the first occurrence of the regular expression
 .Fa regex
 replaced with
 .Fa sub .
@@ -136,6 +136,10 @@
 is replaced with the contents of subgroup
 .Ar n
 in the matched string.
+.It Fn regsuball "str" "regex" "sub"
+As
+.Fn regsuball
+but this replaces all occurrences.
 .It Fn purge_hash "regex"
 Purge all objects in cache whose hash strings match
 .Fa regex .



From phk at projects.linpro.no  Fri May 30 21:39:57 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Fri, 30 May 2008 23:39:57 +0200 (CEST)
Subject: r2641 - trunk/varnish-cache/bin/varnishd
Message-ID: <20080530213957.113E11EC0DA@projects.linpro.no>

Author: phk
Date: 2008-05-30 23:39:56 +0200 (Fri, 30 May 2008)
New Revision: 2641

Modified:
   trunk/varnish-cache/bin/varnishd/common.h
   trunk/varnish-cache/bin/varnishd/tcp.c
Log:
Back in the mists of time, the SocketWizards were so happy to get
any connections at all, that they didn't even consider that maybe
connect(2) should have a timeout argument.

Add TCP_connect() which addresses this shortcoming, using what I
belive is a widely supported workaround.

Not to POSIX: please fix.



Modified: trunk/varnish-cache/bin/varnishd/common.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/common.h	2008-05-27 07:10:22 UTC (rev 2640)
+++ trunk/varnish-cache/bin/varnishd/common.h	2008-05-30 21:39:56 UTC (rev 2641)
@@ -46,5 +46,8 @@
 int TCP_filter_http(int sock);
 void TCP_blocking(int sock);
 void TCP_nonblocking(int sock);
+#ifdef SOL_SOCKET
+int TCP_connect(int s, const struct sockaddr *name, socklen_t namelen, int msec);
+#endif
 
 #define TRUST_ME(ptr)	((void*)(uintptr_t)(ptr))

Modified: trunk/varnish-cache/bin/varnishd/tcp.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/tcp.c	2008-05-27 07:10:22 UTC (rev 2640)
+++ trunk/varnish-cache/bin/varnishd/tcp.c	2008-05-30 21:39:56 UTC (rev 2641)
@@ -39,6 +39,7 @@
 #include <errno.h>
 #include <sys/ioctl.h>
 #include <netdb.h>
+#include <poll.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -138,3 +139,55 @@
 	i = 1;
 	AZ(ioctl(sock, FIONBIO, &i));
 }
+
+/*--------------------------------------------------------------------
+ * On TCP a connect(2) can block for a looong time, and we don't want that.
+ * Unfortunately, the SocketWizards back in those days were happy to wait
+ * any amount of time for a connection, so the connect(2) syscall does not
+ * take an argument for patience.
+ *
+ * There is a little used work-around, and we employ it at our peril.
+ *
+ */
+
+int
+TCP_connect(int s, const struct sockaddr *name, socklen_t namelen, int msec)
+{
+	int i, k;
+	socklen_t l;
+	struct pollfd fds[1];
+
+	assert(s >= 0);
+
+	/* Set the socket non-blocking */
+	TCP_nonblocking(s);
+
+	/* Attempt the connect */
+	i = connect(s, name, namelen);
+	if (i == 0 || errno != EINPROGRESS)
+		return (i);
+
+	/* Exercise our patience, polling for write */
+	fds[0].fd = s;
+	fds[0].events = POLLWRNORM;
+	fds[0].revents = 0;
+	i = poll(fds, 1, msec);
+
+	if (i == 0) {
+		/* Timeout, close and give up */
+		errno = ETIMEDOUT;
+		return (-1);
+	}
+
+	/* Find out if we got a connection */
+	l = sizeof k;
+	AZ(getsockopt(s, SOL_SOCKET, SO_ERROR, &k, &l));
+
+	/* An error means no connection established */
+	errno = k;
+	if (k)
+		return (-1);
+
+	TCP_blocking(s);
+	return (0);
+}



From phk at projects.linpro.no  Fri May 30 22:18:26 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Sat, 31 May 2008 00:18:26 +0200 (CEST)
Subject: r2642 - in trunk/varnish-cache: bin/varnishd include lib/libvcl
Message-ID: <20080530221826.642021EC206@projects.linpro.no>

Author: phk
Date: 2008-05-31 00:18:26 +0200 (Sat, 31 May 2008)
New Revision: 2642

Modified:
   trunk/varnish-cache/bin/varnishd/cache_backend.c
   trunk/varnish-cache/bin/varnishd/heritage.h
   trunk/varnish-cache/bin/varnishd/mgt_param.c
   trunk/varnish-cache/include/vrt.h
   trunk/varnish-cache/lib/libvcl/vcc_backend.c
   trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c
Log:
Add a new backend attribute in VCL: "connect_timeout".

This is how long time we wait for a TCP connection to the backend to
become established.

Typical usage:

		backend b1 {
			.host = "hex";
			.port = "80";
			.connect_timeout = 500 ms;
		}

It can also be used in backends in director declarations.

Also add a parameter called "connect_timeout" which sets the default
to 400 msec (a number pulled out of my old black magicians hat).




Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_backend.c	2008-05-30 21:39:56 UTC (rev 2641)
+++ trunk/varnish-cache/bin/varnishd/cache_backend.c	2008-05-30 22:18:26 UTC (rev 2642)
@@ -150,7 +150,7 @@
 	struct sockaddr_storage ss;
 	int fam, sockt, proto;
 	socklen_t alen;
-	int s;
+	int s, i, tmo;
 	char abuf1[TCP_ADDRBUFSIZE], abuf2[TCP_ADDRBUFSIZE];
 	char pbuf1[TCP_PORTBUFSIZE], pbuf2[TCP_PORTBUFSIZE];
 
@@ -177,7 +177,16 @@
 		return (s);
 	}
 
-	if (connect(s, (void *)&ss, alen) != 0) {
+	tmo = params->connect_timeout;
+	if (sp->backend->vrt->connect_timeout > 10e-3)
+		tmo = sp->backend->vrt->connect_timeout * 1000;
+
+	if (tmo > 0)
+		i = TCP_connect(s, (void *)&ss, alen, tmo);
+	else
+		i = connect(s, (void *)&ss, alen);
+
+	if (i != 0) {
 		AZ(close(s));
 		LOCK(&sp->backend->mtx);
 		return (-1);

Modified: trunk/varnish-cache/bin/varnishd/heritage.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/heritage.h	2008-05-30 21:39:56 UTC (rev 2641)
+++ trunk/varnish-cache/bin/varnishd/heritage.h	2008-05-30 22:18:26 UTC (rev 2642)
@@ -140,6 +140,9 @@
 	/* Cache vbe_conns */
 	unsigned		cache_vbe_conns;
 
+	/* Default connection_timeout */
+	unsigned		connect_timeout;
+
 	/* CLI buffer size */
 	unsigned		cli_buffer;
 

Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-05-30 21:39:56 UTC (rev 2641)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-05-30 22:18:26 UTC (rev 2642)
@@ -639,6 +639,14 @@
 		"Cache vbe_conn's or rely on malloc, that's the question.",
 		EXPERIMENTAL,
 		"off", "bool" },
+	{ "connect_timeout", tweak_uint,
+		&master.connect_timeout,0, UINT_MAX,
+		"Default connection timeout for backend connections.  "
+		"We only try to connect to the backend for this many "
+		"milliseconds before giving up.  "
+		"VCL can override this default value for each backend.",
+		0,
+		"400", "ms" },
 	{ "cli_buffer", tweak_uint, &master.cli_buffer, 4096, UINT_MAX,
 		"Size of buffer for CLI input."
 		"\nYou may need to increase this if you have big VCL files "

Modified: trunk/varnish-cache/include/vrt.h
===================================================================
--- trunk/varnish-cache/include/vrt.h	2008-05-30 21:39:56 UTC (rev 2641)
+++ trunk/varnish-cache/include/vrt.h	2008-05-30 22:18:26 UTC (rev 2642)
@@ -49,6 +49,7 @@
 	const char	*hostname;
 	const char	*vcl_name;
 	const char	*ident;
+	double		connect_timeout;
 };
 
 /*

Modified: trunk/varnish-cache/lib/libvcl/vcc_backend.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_backend.c	2008-05-30 21:39:56 UTC (rev 2641)
+++ trunk/varnish-cache/lib/libvcl/vcc_backend.c	2008-05-30 22:18:26 UTC (rev 2642)
@@ -223,7 +223,7 @@
 	struct host *h;
 	struct fld_spec *fs;
 
-	fs = vcc_FldSpec(tl, "!host", "?port", NULL);
+	fs = vcc_FldSpec(tl, "!host", "?port", "?connect_timeout", NULL);
 	t_first = tl->t;
 
 	if (tl->t->tok == ID) {
@@ -273,10 +273,16 @@
 			t_host = tl->t;
 			vcc_NextToken(tl);
 		} else if (vcc_IdIs(t_field, "port")) {
-			ExpectErr(tl, CSTR);
 			assert(tl->t->dec != NULL);
 			t_port = tl->t;
 			vcc_NextToken(tl);
+		} else if (vcc_IdIs(t_field, "connect_timeout")) {
+			Fh(tl, 0, "\t.connect_timeout = ");
+			tl->fb = tl->fh;
+			vcc_TimeVal(tl);
+			tl->fb = NULL;
+			ERRCHK(tl);
+			Fh(tl, 0, ",\n");
 		} else {
 			ErrInternal(tl);
 			return;

Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c	2008-05-30 21:39:56 UTC (rev 2641)
+++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c	2008-05-30 22:18:26 UTC (rev 2642)
@@ -403,6 +403,7 @@
 	vsb_cat(sb, "	const char	*hostname;\n");
 	vsb_cat(sb, "	const char	*vcl_name;\n");
 	vsb_cat(sb, "	const char	*ident;\n");
+	vsb_cat(sb, "	double		connect_timeout;\n");
 	vsb_cat(sb, "};\n");
 	vsb_cat(sb, "\n");
 	vsb_cat(sb, "/*\n");



From phk at projects.linpro.no  Fri May 30 22:25:58 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Sat, 31 May 2008 00:25:58 +0200 (CEST)
Subject: r2643 - trunk/varnish-cache/bin/varnishd
Message-ID: <20080530222558.E43FF1EC0DA@projects.linpro.no>

Author: phk
Date: 2008-05-31 00:25:58 +0200 (Sat, 31 May 2008)
New Revision: 2643

Modified:
   trunk/varnish-cache/bin/varnishd/cache_center.c
Log:
I belive this fixes the ESI issue that JT Justman has reported.

(no ticket)



Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2008-05-30 22:18:26 UTC (rev 2642)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2008-05-30 22:25:58 UTC (rev 2643)
@@ -234,7 +234,6 @@
 	sp->xid = 0;
 	SES_Charge(sp);
 	sp->t_open = sp->t_end;
-	sp->t_req = NAN;
 	sp->t_resp = NAN;
 	WSL_Flush(sp->wrk, 0);
 
@@ -242,6 +241,8 @@
 	if (sp->esis > 0)
 		return (1);
 
+	sp->t_req = NAN;
+
 	if (sp->fd >= 0 && sp->doclose != NULL)
 		vca_close_session(sp, sp->doclose);
 	if (sp->fd < 0) {



From phk at projects.linpro.no  Sat May 31 09:34:28 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Sat, 31 May 2008 11:34:28 +0200 (CEST)
Subject: r2644 - in trunk/varnish-cache: bin/varnishd include
Message-ID: <20080531093428.B42FA1EC1FC@projects.linpro.no>

Author: phk
Date: 2008-05-31 11:34:28 +0200 (Sat, 31 May 2008)
New Revision: 2644

Modified:
   trunk/varnish-cache/bin/varnishd/mgt_cli.c
   trunk/varnish-cache/bin/varnishd/mgt_event.c
   trunk/varnish-cache/include/cli.h
Log:
Go over the Telnet/CLI code in the manager process.

Implement "quit" (ticket 125) and drop the planned aliases.

Add CLIS_CLOSE return status, to communicate that close is happening.

Fix logical bug in fd-table allocation bug in event-engine.

Plug memory leak in CLI session destruction.

Increase listen depth of telnet socket to 10.

Ignore trailing whitespace and empty CLI input lines.

Log CLI sessions and commands into syslog.



Modified: trunk/varnish-cache/bin/varnishd/mgt_cli.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_cli.c	2008-05-30 22:25:58 UTC (rev 2643)
+++ trunk/varnish-cache/bin/varnishd/mgt_cli.c	2008-05-31 09:34:28 UTC (rev 2644)
@@ -37,8 +37,10 @@
 
 #include <stdarg.h>
 #include <stdio.h>
+#include <ctype.h>
 #include <fcntl.h>
 #include <stdlib.h>
+#include <syslog.h>
 #include <string.h>
 #include <unistd.h>
 #include <sys/socket.h>
@@ -78,6 +80,8 @@
 #undef MAC_STAT
 }
 
+/*--------------------------------------------------------------------*/
+
 static void
 mcf_help(struct cli *cli, const char * const *av, void *priv)
 {
@@ -98,7 +102,18 @@
 
 /*--------------------------------------------------------------------*/
 
+static void
+mcf_close(struct cli *cli, const char *const *av, void *priv)
+{
 
+	(void)av;
+	(void)priv;
+	cli_out(cli, "Closing CLI connection");
+	cli_result(cli, CLIS_CLOSE);
+}
+
+/*--------------------------------------------------------------------*/
+
 /* XXX: what order should this list be in ? */
 static struct cli_proto cli_proto[] = {
 	{ CLI_HELP,		mcf_help, cli_proto },
@@ -115,13 +130,12 @@
 	{ CLI_VCL_SHOW,		mcf_config_show, NULL },
 	{ CLI_PARAM_SHOW,	mcf_param_show, NULL },
 	{ CLI_PARAM_SET,	mcf_param_set, NULL },
+
+	{ CLI_QUIT, 		mcf_close, NULL},
 #if 0
 	{ CLI_SERVER_RESTART },
 	{ CLI_ZERO },
 	{ CLI_VERBOSE,		m_cli_func_verbose, NULL },
-	{ CLI_EXIT, 		m_cli_func_exit, NULL},
-	{ CLI_QUIT },
-	{ CLI_BYE },
 #endif
 	{ NULL }
 };
@@ -205,27 +219,6 @@
 };
 
 static int
-mgt_cli_close(const struct cli_port *cp)
-{
-
-	CHECK_OBJ_NOTNULL(cp, CLI_PORT_MAGIC);
-	fprintf(stderr, "CLI closing: %s\n", cp->name);
-	vsb_delete(cp->cli->sb);
-	VLU_Destroy(cp->vlu);
-	free(cp->name);
-	(void)close(cp->fdi);
-	if (cp->fdi == 0)
-		assert(open("/dev/null", O_RDONLY) == 0);
-	(void)close(cp->fdo);
-	if (cp->fdo == 1) {
-		assert(open("/dev/null", O_WRONLY) == 1);
-		(void)close(2);
-		assert(open("/dev/null", O_WRONLY) == 2);
-	}
-	return (1);
-}
-
-static int
 mgt_cli_vlu(void *priv, const char *p)
 {
 	struct cli_port *cp;
@@ -235,6 +228,17 @@
 
 	CAST_OBJ_NOTNULL(cp, priv, CLI_PORT_MAGIC);
 	vsb_clear(cp->cli->sb);
+
+	/* Remove trailing whitespace */
+	q = strchr(p, '\0');
+	while (q > p && isspace(q[-1]))
+		q--;
+	*q = '\0';
+
+	/* Ignore empty lines */
+	if (*p == 0)
+		return (0);
+
 	cli_dispatch(cp->cli, cli_proto, p);
 	vsb_finish(cp->cli->sb);
 	AZ(vsb_overflowed(cp->cli->sb));
@@ -266,30 +270,71 @@
 	}
 
 	/* send the result back */
-	if (cli_writeres(cp->fdo, cp->cli))
-		return (mgt_cli_close(cp));
+	syslog(LOG_INFO, "CLI %d result %d \"%s\"",
+	    cp->fdi, cp->cli->result, p);
+	if (cli_writeres(cp->fdo, cp->cli) || cp->cli->result == CLIS_CLOSE)
+		return (1);
 	return (0);
 }
 
+/*--------------------------------------------------------------------
+ * Get rid of a CLI session.
+ *
+ * Always and only called from mgt_cli_callback().
+ *
+ * We must get rid of everything but the event, which gets GC'ed by
+ * ev_schdule_one() when mgt_cli_callback, through our return value
+ * returns non-zero.
+ */
+
 static int
+mgt_cli_close(struct cli_port *cp)
+{
+
+	CHECK_OBJ_NOTNULL(cp, CLI_PORT_MAGIC);
+	syslog(LOG_NOTICE, "CLI %d closed", cp->fdi);
+	free(cp->name);
+	vsb_delete(cp->cli->sb);
+	VLU_Destroy(cp->vlu);
+
+	(void)close(cp->fdi);
+	if (cp->fdo != cp->fdi)
+		(void)close(cp->fdo);
+
+	/* Special case for stdin/out/err */
+	if (cp->fdi == 0)
+		assert(open("/dev/null", O_RDONLY) == 0);
+	if (cp->fdo == 1) {
+		assert(open("/dev/null", O_WRONLY) == 1);
+		(void)close(2);
+		assert(open("/dev/null", O_WRONLY) == 2);
+	}
+
+	free(cp);
+	return (1);
+}
+
+/*--------------------------------------------------------------------
+ * Callback whenever something happens to the input fd of the session.
+ */
+
+static int
 mgt_cli_callback(const struct ev *e, int what)
 {
 	struct cli_port *cp;
-	int i;
 
 	CAST_OBJ_NOTNULL(cp, e->priv, CLI_PORT_MAGIC);
 
 	if (what & (EV_ERR | EV_HUP | EV_GONE))
 		return (mgt_cli_close(cp));
 
-	i = VLU_Fd(cp->fdi, cp->vlu);
-	if (i != 0) {
-		mgt_cli_close(cp);
-		return (1);
-	}
+	if (VLU_Fd(cp->fdi, cp->vlu))
+		return (mgt_cli_close(cp));
 	return (0);
 }
 
+/*--------------------------------------------------------------------*/
+
 void
 mgt_cli_setup(int fdi, int fdo, int verbose, const char *ident)
 {
@@ -299,9 +344,9 @@
 	XXXAN(cp);
 	cp->vlu = VLU_New(cp, mgt_cli_vlu, params->cli_buffer);
 
-	asprintf(&cp->name, "cli %s fds{%d,%d}", ident, fdi, fdo);
+	cp->name = strdup(ident);
 	XXXAN(cp->name);
-	fprintf(stderr, "CLI opened: %s\n", cp->name);
+	syslog(LOG_NOTICE, "CLI %d open from %s", fdi, cp->name);
 	cp->magic = CLI_PORT_MAGIC;
 
 	cp->fdi = fdi;
@@ -339,7 +384,7 @@
 	TCP_myname(ev->fd, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
 	TCP_name((void*)&addr, addrlen, abuf2, sizeof abuf2,
 	    pbuf2, sizeof pbuf2);
-	asprintf(&p, "telnet{%s:%s,%s:%s}", abuf2, pbuf2, abuf1, pbuf1);
+	asprintf(&p, "telnet %s:%s %s:%s", abuf2, pbuf2, abuf1, pbuf1);
 	XXXAN(p);
 
 	mgt_cli_setup(i, i, 0, p);
@@ -363,7 +408,7 @@
 		exit(2);
 	}
 	for (i = 0; i < n; ++i) {
-		int sock = VSS_listen(ta[i], 1);
+		int sock = VSS_listen(ta[i], 10);
 		struct ev *ev = ev_new();
 		XXXAN(ev);
 		ev->fd = sock;

Modified: trunk/varnish-cache/bin/varnishd/mgt_event.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_event.c	2008-05-30 22:25:58 UTC (rev 2643)
+++ trunk/varnish-cache/bin/varnishd/mgt_event.c	2008-05-31 09:34:28 UTC (rev 2644)
@@ -107,13 +107,13 @@
 	unsigned u;
 	void *p;
 
-	if (evb->lpfd < evb->npfd)
+	if (evb->lpfd + 1 < evb->npfd)
 		return (0);
 
-	if (evb->npfd > 256)
+	if (evb->npfd < 8)
+		u = 8;
+	else if (evb->npfd > 256)
 		u = evb->npfd + 256;
-	else if (evb->npfd < 8)
-		u = 8;
 	else
 		u = evb->npfd * 2;
 	p = realloc(evb->pfd, sizeof *evb->pfd * u);

Modified: trunk/varnish-cache/include/cli.h
===================================================================
--- trunk/varnish-cache/include/cli.h	2008-05-30 22:25:58 UTC (rev 2643)
+++ trunk/varnish-cache/include/cli.h	2008-05-31 09:34:28 UTC (rev 2644)
@@ -200,24 +200,12 @@
 	"\tEnable/Disable verbosity",					\
 	0, 0
 
-#define CLI_EXIT							\
-	"exit",								\
-	"exit",								\
-	"\tClose connection",						\
-	0, 0
-
 #define CLI_QUIT							\
 	"quit",								\
 	"quit",								\
 	"\tClose connection",						\
 	0, 0
 
-#define CLI_BYE								\
-	"bye",								\
-	"bye",								\
-	"\tClose connection",						\
-	0, 0
-
 # define CLI_SERVER_STATUS						\
 	"status",							\
 	"status",							\
@@ -237,7 +225,8 @@
 	CLIS_PARAM	= 106,
 	CLIS_OK		= 200,
 	CLIS_CANT	= 300,
-	CLIS_COMMS	= 400
+	CLIS_COMMS	= 400,
+	CLIS_CLOSE	= 500
 };
 
 /* Length of first line of response */



From phk at projects.linpro.no  Sat May 31 21:26:22 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Sat, 31 May 2008 23:26:22 +0200 (CEST)
Subject: r2645 - in trunk/varnish-cache: bin/varnishd include lib/libvarnish
Message-ID: <20080531212622.443551EC0FD@projects.linpro.no>

Author: phk
Date: 2008-05-31 23:26:20 +0200 (Sat, 31 May 2008)
New Revision: 2645

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_ban.c
   trunk/varnish-cache/bin/varnishd/cache_hash.c
   trunk/varnish-cache/include/cli.h
   trunk/varnish-cache/lib/libvarnish/cli.c
Log:

Overhaul the regexp purge/ban code, with a detour around the CLI code.


CLI code:

	In CLI help, don't list commands with no syntax description.

	Add a CLI_HIDDEN macro to construct such entries.  They are
	useful as backwards compatibility entries which we do not want
	to show in the help.

CLI interface to BAN (purge) code:

	Get the CLI names right for purging so they are purge.FOO instead
	of FOO.purge.

	This means that you should use "purge.url" and "purge.hash"
	instead of "url.purge" and "hash.purge".

	Add compat entries for the old, and keep them through the 2.x
	release series.
	

	Add purge.list command to list purges currently in effect.
	NB: This is not 100% locking safe, so don't abuse it.


BAN (purge) code:

	Add reference counting and GC to bans.

	Since we now have full reference counting, drop the sequence
	number based soft references and go to "hard" pointer
	references from the object to the purges.

	Give the "ban" structure the miniobj treatment while we are
	at it.

	The cost of this is a lock operation to update refcounts
	once all applicable bans have been checked on an object.

	There is no locking cost if there is no bans to check.

	Add explicit call to new BAN_DestroyObj() when objects are
	destroyed to release the refcount.

	When we release an object refcount in BAN_DestroyObj(),
	check if we can destroy the last purge in the list also.

	We only destroy one ban per BAN_DestroyObj() call, to avoid
	getting stuck too long time, (tacitly assuming that we will
	destroy more objects than bans.)



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2008-05-31 09:34:28 UTC (rev 2644)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2008-05-31 21:26:20 UTC (rev 2645)
@@ -83,6 +83,7 @@
 struct esi_bit;
 struct vrt_backend;
 struct cli_proto;
+struct ban;
 
 /*--------------------------------------------------------------------*/
 
@@ -244,7 +245,7 @@
 	struct ws		ws_o[1];
 	unsigned char		*vary;
 
-	unsigned		ban_seq;
+	struct ban		*ban;
 
 	unsigned		pass;
 
@@ -420,6 +421,7 @@
 void AddBan(const char *, int hash);
 void BAN_Init(void);
 void BAN_NewObj(struct object *o);
+void BAN_DestroyObj(struct object *o);
 int BAN_CheckObject(struct object *o, const char *url, const char *hash);
 
 /* cache_center.c [CNT] */

Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_ban.c	2008-05-31 09:34:28 UTC (rev 2644)
+++ trunk/varnish-cache/bin/varnishd/cache_ban.c	2008-05-31 21:26:20 UTC (rev 2645)
@@ -28,7 +28,7 @@
  *
  * $Id$
  *
- * Ban processing
+ * Ban ("purge") processing
  */
 
 #include "config.h"
@@ -45,24 +45,33 @@
 #include "cache.h"
 
 struct ban {
+	unsigned		magic;
+#define BAN_MAGIC		0x700b08ea
 	VTAILQ_ENTRY(ban)	list;
-	unsigned		gen;
+	unsigned		refcount;
 	regex_t			regexp;
 	char			*ban;
 	int			hash;
 };
 
-static VTAILQ_HEAD(,ban) ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
-static unsigned ban_next;
-static struct ban *ban_start;
+static VTAILQ_HEAD(banhead,ban) ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
+static MTX ban_mtx;
 
+/*
+ * We maintain ban_start as a pointer to the first element of the list
+ * as a separate variable from the VTAILQ, to avoid depending on the
+ * internals of the VTAILQ macros.  We tacitly assume that a pointer
+ * write is always atomic in doing so.
+ */
+static struct ban * volatile ban_start;
+
 void
 AddBan(const char *regexp, int hash)
 {
 	struct ban *b;
 	int i;
 
-	b = calloc(sizeof *b, 1);
+	ALLOC_OBJ(b, BAN_MAGIC);
 	XXXAN(b);
 
 	i = regcomp(&b->regexp, regexp, REG_EXTENDED | REG_ICASE | REG_NOSUB);
@@ -71,60 +80,152 @@
 
 		(void)regerror(i, &b->regexp, buf, sizeof buf);
 		VSL(SLT_Debug, 0, "REGEX: <%s>", buf);
+		return;
 	}
 	b->hash = hash;
-	b->gen = ++ban_next;
 	b->ban = strdup(regexp);
+	AN(b->ban);
+	LOCK(&ban_mtx);
 	VTAILQ_INSERT_HEAD(&ban_head, b, list);
 	ban_start = b;
+	UNLOCK(&ban_mtx);
 }
 
 void
 BAN_NewObj(struct object *o)
 {
 
-	o->ban_seq = ban_next;
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	AZ(o->ban);
+	LOCK(&ban_mtx);
+	o->ban = ban_start;
+	ban_start->refcount++;
+	UNLOCK(&ban_mtx);
 }
 
+void
+BAN_DestroyObj(struct object *o)
+{
+	struct ban *b;
+
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	if (o->ban == NULL)
+		return;
+	CHECK_OBJ_NOTNULL(o->ban, BAN_MAGIC);
+	LOCK(&ban_mtx);
+	o->ban->refcount--;
+	o->ban = NULL;
+
+	/* Check if we can purge the last ban entry */
+	b = VTAILQ_LAST(&ban_head, banhead);
+	if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) 
+		VTAILQ_REMOVE(&ban_head, b, list);
+	else
+		b = NULL;
+	UNLOCK(&ban_mtx);
+	if (b != NULL) {
+		free(b->ban);
+		regfree(&b->regexp);
+		FREE_OBJ(b);
+	}
+
+}
+
 int
 BAN_CheckObject(struct object *o, const char *url, const char *hash)
 {
-	struct ban *b, *b0;
-	int i;
+	struct ban *b;
+	struct ban * volatile b0;
 
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(o->ban, BAN_MAGIC);
+
 	b0 = ban_start;
-	for (b = b0;
-	    b != NULL && b->gen > o->ban_seq;
-	    b = VTAILQ_NEXT(b, list)) {
-		i = regexec(&b->regexp, b->hash ? hash : url, 0, NULL, 0);
-		if (!i)
-			return (1);
+
+	if (b0 == o->ban)
+		return (0);
+
+	/*
+	 * This loop is safe without locks, because we know we hold
+	 * a refcount on a ban somewhere in the list and we do not
+	 * inspect the list past that ban.
+	 */
+	for (b = b0; b != o->ban; b = VTAILQ_NEXT(b, list)) {
+		if (!regexec(&b->regexp, b->hash ? hash : url, 0, NULL, 0))
+			break;
 	}
-	o->ban_seq = b0->gen;
-	return (0);
+
+	LOCK(&ban_mtx);
+	o->ban->refcount--;
+	if (b == o->ban)	/* not banned */
+		b0->refcount++;
+	UNLOCK(&ban_mtx);
+
+	if (b == o->ban) {	/* not banned */
+		o->ban = b0;
+		return (0);
+	} else {
+		o->ban = NULL;
+		return (1);
+	}
 }
 
+/*--------------------------------------------------------------------
+ * CLI functions to add bans
+ */
+
 static void
-ccf_url_purge(struct cli *cli, const char * const *av, void *priv)
+ccf_purge_url(struct cli *cli, const char * const *av, void *priv)
 {
 
 	(void)priv;
 	AddBan(av[2], 0);
-	cli_out(cli, "PURGE %s\n", av[2]);
+	cli_out(cli, "URL_PURGE %s\n", av[2]);
 }
 
 static void
-ccf_hash_purge(struct cli *cli, const char * const *av, void *priv)
+ccf_purge_hash(struct cli *cli, const char * const *av, void *priv)
 {
 
 	(void)priv;
 	AddBan(av[2], 1);
-	cli_out(cli, "PURGE %s\n", av[2]);
+	cli_out(cli, "HASH_PURGE %s\n", av[2]);
 }
 
+static void
+ccf_purge_list(struct cli *cli, const char * const *av, void *priv)
+{
+	struct ban * volatile b0;
+
+	(void)av;
+	(void)priv;
+	/*
+ 	 * XXX: Strictly speaking, this loop traversal is not lock-safe
+	 * XXX: because we might inspect the last ban while it gets
+	 * XXX: destroyed.  To properly fix this, we would need to either
+ 	 * XXX: hold the lock over the entire loop, or grab refcounts
+	 * XXX: under lock for each element of the list.
+	 * XXX: We do neither, and hope for the best.
+	 */
+	for (b0 = ban_start; b0 != NULL; b0 = VTAILQ_NEXT(b0, list)) {
+		if (b0->refcount == 0 && VTAILQ_NEXT(b0, list) == NULL)
+			break;
+		cli_out(cli, "%5u %s \"%s\"\n",
+		    b0->refcount, b0->hash ? "hash" : "url ", b0->ban);
+	}
+}
+
 static struct cli_proto ban_cmds[] = {
-	{ CLI_URL_PURGE,	ccf_url_purge },
-	{ CLI_HASH_PURGE,	ccf_hash_purge },
+	/*
+	 * XXX: COMPAT: Retain these two entries for entire 2.x series
+	 * XXX: COMPAT: to stay compatible with 1.x series syntax.
+	 */
+	{ CLI_HIDDEN("url.purge", 1, 1)		ccf_purge_url },
+	{ CLI_HIDDEN("hash.purge", 1, 1)	ccf_purge_hash },
+
+	{ CLI_PURGE_URL,			ccf_purge_url },
+	{ CLI_PURGE_HASH,			ccf_purge_hash },
+	{ CLI_PURGE_LIST,			ccf_purge_list },
 	{ NULL }
 };
 
@@ -132,6 +233,7 @@
 BAN_Init(void)
 {
 
+	MTX_INIT(&ban_mtx);
 	CLI_AddFuncs(PUBLIC_CLI, ban_cmds);
-	AddBan("\001", 0);
+	AddBan("^\001$", 0);
 }

Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c	2008-05-31 09:34:28 UTC (rev 2644)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c	2008-05-31 21:26:20 UTC (rev 2645)
@@ -263,6 +263,10 @@
 		grace_o->refcnt++;
 	}
 	UNLOCK(&oh->mtx);
+	/*
+	 * XXX: This may be too early, relative to pass objects.
+	 * XXX: possibly move to when we commit to have it in the cache.
+	 */
 	BAN_NewObj(o);
 	return (o);
 }
@@ -364,6 +368,7 @@
 	if (r != 0)
 		return;
 
+	BAN_DestroyObj(o);
 	DSL(0x40, SLT_Debug, 0, "Object %u workspace min free %u",
 	    o->xid, WS_Free(o->ws_o));
 

Modified: trunk/varnish-cache/include/cli.h
===================================================================
--- trunk/varnish-cache/include/cli.h	2008-05-31 09:34:28 UTC (rev 2644)
+++ trunk/varnish-cache/include/cli.h	2008-05-31 21:26:20 UTC (rev 2645)
@@ -60,20 +60,26 @@
 	    "\tReturns the TTL, size and checksum of the object.", 	\
 	1, 1
 
-#define CLI_URL_PURGE							\
-	"url.purge",							\
-	"url.purge <regexp>",						\
+#define CLI_PURGE_URL							\
+	"purge.url",							\
+	"purge.url <regexp>",						\
 	"\tAll objects where the urls matches regexp will be "		\
 	    "marked obsolete.",						\
 	1, 1
 
-#define CLI_HASH_PURGE							\
-	"hash.purge",							\
-	"hash.purge <regexp>",						\
+#define CLI_PURGE_HASH							\
+	"purge.hash",							\
+	"purge.hash <regexp>",						\
 	"\tAll objects where the hash string matches regexp will be "	\
 	    "marked obsolete.",						\
 	1, 1
 
+#define CLI_PURGE_LIST							\
+	"purge.list",							\
+	"purge.list",							\
+	"\tList the active purges.",					\
+	0, 0
+
 #define CLI_URL_STATUS							\
 	"url.status",							\
 	"url.status <url>",						\
@@ -206,12 +212,15 @@
 	"\tClose connection",						\
 	0, 0
 
-# define CLI_SERVER_STATUS						\
+#define CLI_SERVER_STATUS						\
 	"status",							\
 	"status",							\
 	"\tCheck status of Varnish cache process.",			\
 	0, 0
 
+#define CLI_HIDDEN(foo, min_arg, max_arg)				\
+	foo, NULL, NULL, min_arg, max_arg,
+
 /*
  * Status/return codes in the CLI protocol
  */

Modified: trunk/varnish-cache/lib/libvarnish/cli.c
===================================================================
--- trunk/varnish-cache/lib/libvarnish/cli.c	2008-05-31 09:34:28 UTC (rev 2644)
+++ trunk/varnish-cache/lib/libvarnish/cli.c	2008-05-31 21:26:20 UTC (rev 2645)
@@ -55,10 +55,13 @@
 
 	if (av[2] == NULL || *av[2] == '-') {
 		for (cp = priv; cp->request != NULL; cp++)
-			cli_out(cli, "%s\n", cp->syntax);
+			if (cp->syntax != NULL)
+				cli_out(cli, "%s\n", cp->syntax);
 		return;
 	}
 	for (cp = priv; cp->request != NULL; cp++) {
+		if (cp->syntax == NULL)
+			continue;
 		if (!strcmp(cp->request, av[2])) {
 			cli_out(cli, "%s\n%s\n", cp->syntax, cp->help);
 			return;