From tfheen at varnish-cache.org  Mon Aug  5 08:49:32 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 05 Aug 2013 10:49:32 +0200
Subject: [master] 8fa5b82 Define a few names in the pkg-config data
Message-ID: <E1V6GTs-0004qC-Ar@project.varnish-software.com>

commit 8fa5b829ceadcdd812a82e44b97409cf30ef68ce
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Jul 30 14:48:56 2013 +0200

    Define a few names in the pkg-config data

diff --git a/varnishapi.pc.in b/varnishapi.pc.in
index 7f5036a..059725e 100644
--- a/varnishapi.pc.in
+++ b/varnishapi.pc.in
@@ -9,7 +9,9 @@ datarootdir=@datarootdir@
 datadir=@datadir@
 pkgdatadir=${datadir}/@PACKAGE@
 pkgdataincludedir=${pkgdatadir}/include
+vmodincludedir=${pkgdataincludedir}
 vmoddir=${libdir}/@PACKAGE@/vmods
+vmodtool=${pkgdatadir}/vmodtool.py
 
 Name: VarnishAPI
 Description: Varnish API



From tfheen at varnish-cache.org  Mon Aug  5 08:49:32 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 05 Aug 2013 10:49:32 +0200
Subject: [master] ae5a35f Use /usr/bin/env instead of hard coding python
 location
Message-ID: <E1V6GTs-0004qH-By@project.varnish-software.com>

commit ae5a35f0c77f31c518d071551b5847d50a33510b
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Jul 31 10:54:26 2013 +0200

    Use /usr/bin/env instead of hard coding python location

diff --git a/lib/libvcl/vmodtool.py b/lib/libvcl/vmodtool.py
index 259f97f..d99d8c6 100755
--- a/lib/libvcl/vmodtool.py
+++ b/lib/libvcl/vmodtool.py
@@ -1,4 +1,4 @@
-#!/usr/local/bin/python
+#!/usr/bin/env python
 #-
 # Copyright (c) 2010-2011 Varnish Software AS
 # All rights reserved.



From tfheen at varnish-cache.org  Mon Aug  5 08:49:32 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 05 Aug 2013 10:49:32 +0200
Subject: [master] b42beeb Install autoconf macros
Message-ID: <E1V6GTs-0004qN-Cu@project.varnish-software.com>

commit b42beebc229f07db863e7209399c2acf22e38507
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Jul 31 10:55:01 2013 +0200

    Install autoconf macros
    
    Add a few macros that are useful when writing vmods.

diff --git a/Makefile.am b/Makefile.am
index 87bbe7a..dabe7c8 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -7,8 +7,11 @@ SUBDIRS += redhat
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = varnishapi.pc
 
+m4dir = $(datadir)/aclocal
+m4_DATA = varnish.m4
+
 CLEANFILES = cscope.in.out cscope.out cscope.po.out
-EXTRA_DIST = LICENSE autogen.sh varnishapi.pc.in
+EXTRA_DIST = LICENSE autogen.sh varnishapi.pc.in varnish.m4
 
 DISTCHECK_CONFIGURE_FLAGS = \
     --enable-developer-warnings \
diff --git a/varnish.m4 b/varnish.m4
new file mode 100644
index 0000000..16e27c5
--- /dev/null
+++ b/varnish.m4
@@ -0,0 +1,96 @@
+# varnish.m4 - Macros to locate Varnish header files.            -*- Autoconf -*-
+# serial 1 (varnish-4.0)
+
+# Copyright (c) 2013 Varnish Software AS
+# All rights reserved.
+#
+# Author: Tollef Fog Heen <tfheen at varnish-software.com>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+# VARNISH_VMOD_INCLUDE_DIR([])
+# ----------------------------
+
+AC_DEFUN([VARNISH_VMOD_INCLUDES],
+[
+m4_pattern_forbid([^_?VARNISH[A-Z_]+$])
+m4_pattern_allow([^VARNISH_VMOD(_INCLUDE_DIR|TOOL)$])
+# Check for pkg-config
+PKG_CHECK_EXISTS([varnishapi],[],[
+	if test -n "$PKG_CONFIG"; then
+		AC_MSG_FAILURE(
+[The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/>.])
+	else
+		AC_MSG_FAILURE(
+[pkg-config was unable to locate the varnishapi configuration data.
+
+Please check config.log or adjust the PKG_CONFIG_PATH environment
+variable if you installed software in a non-standard prefix.])
+	fi
+])
+
+VARNISH_PKG_GET_VAR([VMOD_INCLUDE_DIR], [vmodincludedir])
+VARNISH_PKG_GET_VAR([VAPI_INCLUDE_DIR], [pkgincludedir])
+_CPPFLAGS="$CPPFLAGS"
+VMOD_INCLUDES="-I$VMOD_INCLUDE_DIR -I$VAPI_INCLUDE_DIR"
+CPPFLAGS="$VMOD_INCLUDES $CPPFLAGS"
+AC_CHECK_HEADERS([vsha256.h cache/cache.h])
+CPPFLAGS="$_CPPFLAGS"
+AC_SUBST([VMOD_INCLUDES])
+])# VARNISH_VMOD_INCLUDE_DIR
+
+# VARNISH_VMOD_DIR([])
+# --------------------
+
+AC_DEFUN([VARNISH_VMOD_DIR],
+[
+VARNISH_PKG_GET_VAR([VMOD_DIR], [vmoddir])
+AC_SUBST([VMOD_DIR])
+])
+
+# VARNISH_VMODTOOL([])
+# --------------------
+
+AC_DEFUN([VARNISH_VMODTOOL],
+[
+VARNISH_PKG_GET_VAR([VMODTOOL], [vmodtool])
+AC_SUBST([VMODTOOL])
+])
+
+# VARNISH_PKG_GET_VAR([VARIABLE, PC_VAR_NAME])
+# -------------------------------
+
+AC_DEFUN([VARNISH_PKG_GET_VAR],
+[
+# Uses internal function for now..
+pkg_failed=no
+_PKG_CONFIG([$1], [variable=][$2], [varnishapi])
+if test "$pkg_failed" = "yes"; then
+   AC_MSG_FAILURE([$2][ not defined, too old Varnish?])
+fi
+AS_VAR_COPY([$1], [pkg_cv_][$1])
+])



From phk at varnish-cache.org  Mon Aug  5 09:13:01 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 05 Aug 2013 11:13:01 +0200
Subject: [master] f66b84e Add a new VTLA:  VSA.
Message-ID: <E1V6Gqb-00023E-TT@project.varnish-software.com>

commit f66b84e3374778aef8db4bcb579e0c27f6efea31
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 5 09:10:45 2013 +0000

    Add a new VTLA:  VSA.
    
    This does absolutely not stand for "suckaddr", and but it tries to
    sanitize the sockaddr_* mess enough that we can simply pass around
    struct sockaddr_storage without also tracking its length.
    
    The cost is that we do know something about address families (IPv4, IPv6),
    which strictly speaking should not be necessary, had the sockaddr
    not been messed up by the IPv6 people.

diff --git a/bin/varnishadm/Makefile.am b/bin/varnishadm/Makefile.am
index 22825cd..8fb28d7 100644
--- a/bin/varnishadm/Makefile.am
+++ b/bin/varnishadm/Makefile.am
@@ -9,6 +9,7 @@ dist_man_MANS = varnishadm.1
 varnishadm_SOURCES = \
 	varnishadm.c \
 	$(top_builddir)/lib/libvarnish/vas.c \
+	$(top_builddir)/lib/libvarnish/vsa.c \
 	$(top_builddir)/lib/libvarnish/vtcp.c \
 	$(top_builddir)/lib/libvarnish/vss.c
 
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index f3ae536..e802446 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -109,8 +109,7 @@ VBE_ReleaseConn(struct vbc *vc)
 
 static int
 vbe_TryConnect(const struct busyobj *bo, int pf,
-    const struct sockaddr_storage *sa, socklen_t salen,
-    const struct vdi_simple *vs)
+    const struct sockaddr_storage *sa, const struct vdi_simple *vs)
 {
 	int s, i, tmo;
 	double tmod;
@@ -126,7 +125,7 @@ vbe_TryConnect(const struct busyobj *bo, int pf,
 
 	tmo = (int)(tmod * 1000.0);
 
-	i = VTCP_connect(s, sa, salen, tmo);
+	i = VTCP_connect(s, sa, tmo);
 
 	if (i != 0) {
 		AZ(close(s));
@@ -160,19 +159,16 @@ bes_conn_try(struct busyobj *bo, struct vbc *vc, const struct vdi_simple *vs)
 	/* release lock during stuff that can take a long time */
 
 	if (cache_param->prefer_ipv6 && bp->ipv6 != NULL) {
-		s = vbe_TryConnect(bo, PF_INET6, bp->ipv6, bp->ipv6len, vs);
+		s = vbe_TryConnect(bo, PF_INET6, bp->ipv6, vs);
 		vc->addr = bp->ipv6;
-		vc->addrlen = bp->ipv6len;
 	}
 	if (s == -1 && bp->ipv4 != NULL) {
-		s = vbe_TryConnect(bo, PF_INET, bp->ipv4, bp->ipv4len, vs);
+		s = vbe_TryConnect(bo, PF_INET, bp->ipv4, vs);
 		vc->addr = bp->ipv4;
-		vc->addrlen = bp->ipv4len;
 	}
 	if (s == -1 && !cache_param->prefer_ipv6 && bp->ipv6 != NULL) {
-		s = vbe_TryConnect(bo, PF_INET6, bp->ipv6, bp->ipv6len, vs);
+		s = vbe_TryConnect(bo, PF_INET6, bp->ipv6, vs);
 		vc->addr = bp->ipv6;
-		vc->addrlen = bp->ipv6len;
 	}
 
 	vc->fd = s;
@@ -182,7 +178,6 @@ bes_conn_try(struct busyobj *bo, struct vbc *vc, const struct vdi_simple *vs)
 		bp->refcount--;		/* Only keep ref on success */
 		Lck_Unlock(&bp->mtx);
 		vc->addr = NULL;
-		vc->addrlen = 0;
 	} else {
 		VTCP_myname(s, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
 		VSLb(bo->vsl, SLT_BackendOpen, "%d %s %s %s ",
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 97f7898..8103af5 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -128,9 +128,7 @@ struct backend {
 	char			*port;
 
 	struct sockaddr_storage	*ipv4;
-	socklen_t		ipv4len;
 	struct sockaddr_storage	*ipv6;
-	socklen_t		ipv6len;
 
 	unsigned		n_conn;
 	VTAILQ_HEAD(, vbc)	connlist;
@@ -158,7 +156,6 @@ struct vbc {
 	int			fd;
 
 	struct sockaddr_storage	*addr;
-	socklen_t		addrlen;
 
 	uint8_t			recycled;
 
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index 5b68bfb..b07252e 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -41,6 +41,7 @@
 #include "cache_backend.h"
 #include "vcli.h"
 #include "vcli_priv.h"
+#include "vsa.h"
 #include "vrt.h"
 
 /*
@@ -149,15 +150,14 @@ VBE_DropRefConn(struct backend *b)
  */
 
 static void
-copy_sockaddr(struct sockaddr_storage **sa, socklen_t *len,
-    const unsigned char *src)
+copy_sockaddr(struct sockaddr_storage **sa, const unsigned char *src)
 {
 
 	assert(*src > 0);
 	*sa = calloc(sizeof **sa, 1);
 	XXXAN(*sa);
 	memcpy(*sa, src + 1, *src);
-	*len = *src;
+	assert(VSA_Sane(*sa));
 }
 
 /*--------------------------------------------------------------------
@@ -182,13 +182,11 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
 		if (strcmp(b->vcl_name, vb->vcl_name))
 			continue;
-		if (vb->ipv4_sockaddr != NULL && (
-		    b->ipv4len != vb->ipv4_sockaddr[0] ||
-		    memcmp(b->ipv4, vb->ipv4_sockaddr + 1, b->ipv4len)))
+		if (vb->ipv4_sockaddr != NULL &&
+		    VSA_Compare(b->ipv4, vb->ipv4_sockaddr + 1))
 			continue;
-		if (vb->ipv6_sockaddr != NULL && (
-		    b->ipv6len != vb->ipv6_sockaddr[0] ||
-		    memcmp(b->ipv6, vb->ipv6_sockaddr + 1, b->ipv6len)))
+		if (vb->ipv6_sockaddr != NULL &&
+		    VSA_Compare(b->ipv6, vb->ipv6_sockaddr + 1))
 			continue;
 		b->refcount++;
 		b->vsc->vcls++;
@@ -227,9 +225,9 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 	 * Copy over the sockaddrs
 	 */
 	if (vb->ipv4_sockaddr != NULL)
-		copy_sockaddr(&b->ipv4, &b->ipv4len, vb->ipv4_sockaddr);
+		copy_sockaddr(&b->ipv4, vb->ipv4_sockaddr);
 	if (vb->ipv6_sockaddr != NULL)
-		copy_sockaddr(&b->ipv6, &b->ipv6len, vb->ipv6_sockaddr);
+		copy_sockaddr(&b->ipv6, vb->ipv6_sockaddr);
 
 	assert(b->ipv4 != NULL || b->ipv6 != NULL);
 
diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index cd9a215..f93fe2f 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -105,7 +105,7 @@ static struct lock			vbp_mtx;
  */
 
 static int
-vbp_connect(int pf, const struct sockaddr_storage *sa, socklen_t salen, int tmo)
+vbp_connect(int pf, const struct sockaddr_storage *sa, int tmo)
 {
 	int s, i;
 
@@ -113,7 +113,7 @@ vbp_connect(int pf, const struct sockaddr_storage *sa, socklen_t salen, int tmo)
 	if (s < 0)
 		return (s);
 
-	i = VTCP_connect(s, sa, salen, tmo);
+	i = VTCP_connect(s, sa, tmo);
 	if (i == 0)
 		return (s);
 	VTCP_close(&s);
@@ -139,21 +139,21 @@ vbp_poke(struct vbp_target *vt)
 
 	s = -1;
 	if (cache_param->prefer_ipv6 && bp->ipv6 != NULL) {
-		s = vbp_connect(PF_INET6, bp->ipv6, bp->ipv6len, tmo);
+		s = vbp_connect(PF_INET6, bp->ipv6, tmo);
 		t_now = VTIM_real();
 		tmo = (int)round((t_end - t_now) * 1e3);
 		if (s >= 0)
 			vt->good_ipv6 |= 1;
 	}
 	if (tmo > 0 && s < 0 && bp->ipv4 != NULL) {
-		s = vbp_connect(PF_INET, bp->ipv4, bp->ipv4len, tmo);
+		s = vbp_connect(PF_INET, bp->ipv4, tmo);
 		t_now = VTIM_real();
 		tmo = (int)round((t_end - t_now) * 1e3);
 		if (s >= 0)
 			vt->good_ipv4 |= 1;
 	}
 	if (tmo > 0 && s < 0 && bp->ipv6 != NULL) {
-		s = vbp_connect(PF_INET6, bp->ipv6, bp->ipv6len, tmo);
+		s = vbp_connect(PF_INET6, bp->ipv6, tmo);
 		t_now = VTIM_real();
 		tmo = (int)round((t_end - t_now) * 1e3);
 		if (s >= 0)
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 1429602..d3b212f 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -39,6 +39,7 @@
 #include "cache_backend.h"
 #include "vrt.h"
 #include "vrt_obj.h"
+#include "vsa.h"
 #include "vtcp.h"
 #include "vtim.h"
 
@@ -274,7 +275,7 @@ VRT_r_beresp_backend_port(const struct vrt_ctx *ctx)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->bo->vbc, VBC_MAGIC);
-	return (VTCP_port(ctx->bo->vbc->addr));
+	return (VSA_Port(ctx->bo->vbc->addr));
 }
 
 const char *
@@ -609,7 +610,7 @@ VRT_r_server_port(const struct vrt_ctx *ctx)
 		    &ctx->req->sp->mysockaddrlen);
 		assert(VTCP_Check(i));
 	}
-	return (VTCP_port(&ctx->req->sp->mysockaddr));
+	return (VSA_Port(&ctx->req->sp->mysockaddr));
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/include/Makefile.am b/include/Makefile.am
index 92cc1bd..605a71e 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -80,6 +80,7 @@ nobase_pkgdatainclude_HEADERS = \
 	vdef.h \
 	vrt.h \
 	vrt_obj.h \
+	vsa.h \
 	vsb.h \
 	vsha256.h
 
diff --git a/include/vsa.h b/include/vsa.h
new file mode 100644
index 0000000..c9d521a
--- /dev/null
+++ b/include/vsa.h
@@ -0,0 +1,38 @@
+/*-
+ * Copyright (c) 2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#ifndef VSA_H_INCLUDED
+#define VSA_H_INCLUDED
+
+int VSA_Sane(const struct sockaddr_storage *ss);
+socklen_t VSA_Len(const struct sockaddr_storage *ss);
+unsigned VSA_Port(const struct sockaddr_storage *ss);
+int VSA_Compare(const struct sockaddr_storage *ss1, const void *ss2);
+
+#endif
diff --git a/include/vtcp.h b/include/vtcp.h
index 1594a4d..205d9e1 100644
--- a/include/vtcp.h
+++ b/include/vtcp.h
@@ -65,11 +65,9 @@ int VTCP_linger(int sock, int linger);
 int VTCP_check_hup(int sock);
 
 #ifdef SOL_SOCKET
-int VTCP_port(const struct sockaddr_storage *addr);
 void VTCP_name(const struct sockaddr_storage *addr, unsigned l, char *abuf,
     unsigned alen, char *pbuf, unsigned plen);
-int VTCP_connect(int s, const struct sockaddr_storage *name, socklen_t namelen,
-    int msec);
+int VTCP_connect(int s, const struct sockaddr_storage *name, int msec);
 void VTCP_close(int *s);
 void VTCP_set_read_timeout(int s, double seconds);
 #endif
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index 60f6a25..c471c9c 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -27,6 +27,7 @@ libvarnish_la_SOURCES = \
 	vmb.c \
 	vpf.c \
 	vre.c \
+	vsa.c \
 	vsb.c \
 	vsha256.c \
 	vss.c
diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c
new file mode 100644
index 0000000..51f214a
--- /dev/null
+++ b/lib/libvarnish/vsa.c
@@ -0,0 +1,98 @@
+/*-
+ * Copyright (c) 2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Struct sockaddr_* is not even close to a convenient API.
+ *
+ * These functions try to mitigate the madness, at the cost of actually
+ * knowing something about address families.
+ */
+
+#include "config.h"
+
+#include <errno.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+
+#include "vas.h"
+#include "vsa.h"
+
+int
+VSA_Sane(const struct sockaddr_storage *ss)
+{
+	switch(ss->ss_family) {
+		case PF_INET:
+		case PF_INET6:
+			return (1);
+		default:
+			return (0);
+	}
+}
+
+socklen_t
+VSA_Len(const struct sockaddr_storage *ss)
+{
+	switch(ss->ss_family) {
+		case PF_INET:
+			return (sizeof(struct sockaddr_in));
+		case PF_INET6:
+			return (sizeof(struct sockaddr_in6));
+		default:
+			WRONG("Illegal socket family");
+	}
+}
+
+int
+VSA_Compare(const struct sockaddr_storage *ss1, const void *ss2)
+{
+	switch(ss1->ss_family) {
+		case PF_INET:
+		case PF_INET6:
+			return (memcmp(ss1, ss2, VSA_Len(ss1)));
+		default:
+			return (-1);
+	}
+}
+
+unsigned
+VSA_Port(const struct sockaddr_storage *ss)
+{
+	switch(ss->ss_family) {
+		case PF_INET:
+			{
+			const struct sockaddr_in *ain = (const void *)ss;
+			return (ntohs((ain->sin_port)));
+			}
+		case PF_INET6:
+			{
+			const struct sockaddr_in6 *ain = (const void *)ss;
+			return (ntohs((ain->sin6_port)));
+			}
+		default:
+			WRONG("Illegal socket family");
+	}
+}
diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 7227fb3..6b9df11 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -51,28 +51,11 @@
 #include <unistd.h>
 
 #include "vas.h"
+#include "vsa.h"
 #include "vtcp.h"
 
 /*--------------------------------------------------------------------*/
 
-int
-VTCP_port(const struct sockaddr_storage *addr)
-{
-
-	if (addr->ss_family == AF_INET) {
-		const struct sockaddr_in *ain = (const void *)addr;
-		return (ntohs((ain->sin_port)));
-	}
-	if (addr->ss_family == AF_INET6) {
-		const struct sockaddr_in6 *ain = (const void *)addr;
-		return (ntohs((ain->sin6_port)));
-	}
-	return (-1);
-}
-
-
-/*--------------------------------------------------------------------*/
-
 void
 VTCP_name(const struct sockaddr_storage *addr, unsigned l,
     char *abuf, unsigned alen, char *pbuf, unsigned plen)
@@ -212,8 +195,7 @@ VTCP_nonblocking(int sock)
  */
 
 int
-VTCP_connect(int s, const struct sockaddr_storage *name, socklen_t namelen,
-    int msec)
+VTCP_connect(int s, const struct sockaddr_storage *name, int msec)
 {
 	int i, k;
 	socklen_t l;
@@ -226,7 +208,8 @@ VTCP_connect(int s, const struct sockaddr_storage *name, socklen_t namelen,
 		(void)VTCP_nonblocking(s);
 
 	/* Attempt the connect */
-	i = connect(s, (const void *)name, namelen);
+	assert(VSA_Sane(name));
+	i = connect(s, (const void *)name, VSA_Len(name));
 	if (i == 0 || errno != EINPROGRESS)
 		return (i);
 



From tfheen at varnish-cache.org  Mon Aug  5 09:15:59 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 05 Aug 2013 11:15:59 +0200
Subject: [3.0] d187c58 Update graph
Message-ID: <E1V6GtT-0002hL-9Q@project.varnish-software.com>

commit d187c58353281e28483ddab75ed088826f339e6d
Author: Federico Schwindt <fgsch at lodoss.net>
Date:   Mon Aug 5 11:15:40 2013 +0200

    Update graph
    
    Make it clearer that vcl_hash is called on lookup, pass and pipe.

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 599d0e7..d11d600 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -1077,7 +1077,9 @@ DOT	lookup2 [
 DOT		shape=diamond
 DOT		label="obj.f.pass ?"
 DOT	]
-DOT	hash -> lookup [label="hash",style=bold,color=green]
+DOT	hash -> lookup [style=bold,color=green]
+DOT	hash -> pipe [style=bold,color=orange]
+DOT	hash -> pass2 [style=bold,color=red]
 DOT	lookup -> lookup2 [label="yes",style=bold,color=green]
 DOT }
 DOT lookup2 -> hit [label="no", style=bold,color=green]
@@ -1364,11 +1366,11 @@ DOT		label="vcl_recv()|req."
 DOT	]
 DOT }
 DOT RESTART -> recv
-DOT recv -> pipe [label="pipe",style=bold,color=orange]
-DOT recv -> pass2 [label="pass",style=bold,color=red]
+DOT recv -> hash [label="lookup",style=bold,color=green]
+DOT recv -> hash [label="pass",style=bold,color=red]
+DOT recv -> hash [label="pipe",style=bold,color=orange]
 DOT recv -> err_recv [label="error"]
 DOT err_recv [label="ERROR",shape=plaintext]
-DOT recv -> hash [label="lookup",style=bold,color=green]
  */
 
 static int



From tfheen at varnish-cache.org  Mon Aug  5 09:16:19 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 05 Aug 2013 11:16:19 +0200
Subject: [master] b3a2823 Update graph
Message-ID: <E1V6Gtn-0002nU-4E@project.varnish-software.com>

commit b3a2823cfbb6c8eb326ed891b4a39075cd05ef0b
Author: Federico Schwindt <fgsch at lodoss.net>
Date:   Mon Aug 5 11:10:35 2013 +0200

    Update graph
    
    Make it clearer that vcl_hash is called on lookup, pass and pipe.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 5ee6d14..ec7505c 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -762,7 +762,7 @@ cnt_restart(const struct worker *wrk, struct req *req)
 DOT subgraph xcluster_recv {
 DOT	recv [
 DOT		shape=record
-DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}|{<pipe>pipe?|<pass>pass?|<error>error?|<lookup>lookup?}}"
+DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}|{<pass>pass?|<lookup>lookup?|<pipe>pipe?|<error>error?}}"
 DOT	]
 DOT }
 DOT subgraph xcluster_hash {
@@ -771,12 +771,14 @@ DOT		shape=record
 DOT		label="{cnt_recv:|{vcl_hash\{\}|req.*}}"
 DOT	]
 DOT }
-DOT recv:pipe -> pipe [style=bold,color=orange]
-DOT recv:pass -> pass [style=bold,color=red]
+DOT recv:pipe -> hash [style=bold,color=orange]
+DOT recv:pass -> hash [style=bold,color=red]
 #DOT recv:error -> err_recv
 #DOT err_recv [label="ERROR",shape=plaintext]
 DOT recv:lookup -> hash [style=bold,color=green]
-DOT hash -> lookup [label="hash",style=bold,color=green]
+DOT hash -> lookup [style=bold,color=green]
+DOT hash -> pipe [style=bold,color=orange]
+DOT hash -> pass [style=bold,color=red]
  */
 
 static enum req_fsm_nxt



From phk at varnish-cache.org  Tue Aug  6 09:31:28 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 06 Aug 2013 11:31:28 +0200
Subject: [master] 2314365 Change the way VCC emit sockaddr's, explain why.
Message-ID: <E1V6dc0-000624-6L@project.varnish-software.com>

commit 2314365f4b2bb70d24f82ee3b1d0bde7804606f5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 6 09:30:38 2013 +0000

    Change the way VCC emit sockaddr's, explain why.

diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index b07252e..c3e7c8c 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -150,13 +150,13 @@ VBE_DropRefConn(struct backend *b)
  */
 
 static void
-copy_sockaddr(struct sockaddr_storage **sa, const unsigned char *src)
+copy_sockaddr(struct sockaddr_storage **sa, const void *src)
 {
 
-	assert(*src > 0);
+	assert(VSA_Sane(src));
 	*sa = calloc(sizeof **sa, 1);
 	XXXAN(*sa);
-	memcpy(*sa, src + 1, *src);
+	memcpy(*sa, src, VSA_Len(src));
 	assert(VSA_Sane(*sa));
 }
 
@@ -183,10 +183,10 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 		if (strcmp(b->vcl_name, vb->vcl_name))
 			continue;
 		if (vb->ipv4_sockaddr != NULL &&
-		    VSA_Compare(b->ipv4, vb->ipv4_sockaddr + 1))
+		    VSA_Compare(b->ipv4, vb->ipv4_sockaddr))
 			continue;
 		if (vb->ipv6_sockaddr != NULL &&
-		    VSA_Compare(b->ipv6, vb->ipv6_sockaddr + 1))
+		    VSA_Compare(b->ipv6, vb->ipv6_sockaddr))
 			continue;
 		b->refcount++;
 		b->vsc->vcls++;
diff --git a/include/vrt.h b/include/vrt.h
index 1394668..8b6e8a8 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -124,8 +124,8 @@ struct vrt_backend {
 	const char			*ipv6_addr;
 	const char			*port;
 
-	const unsigned char		*ipv4_sockaddr;
-	const unsigned char		*ipv6_sockaddr;
+	const void			*ipv4_sockaddr;
+	const void 			*ipv6_sockaddr;
 
 	const char			*hosthdr;
 
diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c
index 09604cb..9bab3ce 100644
--- a/lib/libvcl/vcc_backend.c
+++ b/lib/libvcl/vcc_backend.c
@@ -61,6 +61,7 @@
 #include "vcc_compile.h"
 
 #include "vss.h"
+#include "vsa.h"
 
 struct host {
 	VTAILQ_ENTRY(host)      list;
@@ -68,27 +69,38 @@ struct host {
 	char			*vgcname;
 };
 
+/*
+ * The IPv6 crew royally screwed up the entire idea behind
+ * struct sockaddr, and combined with various other incomptency
+ * in the OS business, that means that there is no sane or even
+ * remotely portable way to initialize a sockaddr at compile time.
+ *
+ * In our case it is slightly more tricky than that, because we don't
+ * even want to #include the struct sockaddr* definitions.
+ *
+ * Instead we make sure the sockaddr is sane (for our values of sane)
+ * and dump it in binary, using a 64 bit integertype, hoping that this
+ * will ensure good enough alignment.
+ */
+
 static int
-emit_sockaddr(struct vcc *tl, void *sa, unsigned sal)
+emit_sockaddr(struct vcc *tl, const void *sa, unsigned sal)
 {
-	unsigned len;
-	uint8_t *u;
+	unsigned n = (sal + 7) / 8, len;
+	uint64_t b[n];
 
+	assert(VSA_Sane(sa));
 	AN(sa);
 	AN(sal);
 	assert(sal < 256);
-	Fh(tl, 0, "\nstatic const unsigned char sockaddr_%u[%d] = {\n",
-	    tl->unique, sal + 1);
-	Fh(tl, 0, "    %3u, /* Length */\n",  sal);
-	u = sa;
-	for (len = 0; len <sal; len++) {
-		if ((len % 8) == 0)
-			Fh(tl, 0, "   ");
-		Fh(tl, 0, " %3u", u[len]);
-		if (len + 1 < sal)
-			Fh(tl, 0, ",");
-		if ((len % 8) == 7)
-			Fh(tl, 0, "\n");
+	assert(sizeof(unsigned long long) == 8);
+	Fh(tl, 0, "\nstatic const unsigned long long");
+	Fh(tl, 0, " sockaddr_%u[%d] = {\n", tl->unique, n);
+	memcpy(b, sa, sal);
+	for (len = 0; len <n; len++) {
+		Fh(tl, 0, "%s    0x%016jx",
+		    len ? ",\n" : "",
+		    (uintmax_t)b[len]);
 	}
 	Fh(tl, 0, "\n};\n");
 	return (tl->unique++);



From tfheen at varnish-cache.org  Tue Aug  6 09:56:16 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 06 Aug 2013 11:56:16 +0200
Subject: [3.0] 412c4f7 Emit BackendXID and check c_flag/b_flag properly
Message-ID: <E1V6e00-0001QB-HI@project.varnish-software.com>

commit 412c4f754dd805e06803efa170e5d62afd23dd0d
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Aug 6 11:15:15 2013 +0200

    Emit BackendXID and check c_flag/b_flag properly
    
    07d488cb96c703dd7c0ccf7049448c6291f1f0cf partially broke varnishlog,
    since libvarnishapi did not properly check if both -c and -b were
    given.
    
    In addition, varnishd did not emit a tag at the start of a backend
    transaction.  Emit BackendXID since that seems appropriate.
    
    Fixes #1325

diff --git a/bin/varnishd/cache_dir.c b/bin/varnishd/cache_dir.c
index 2463aa8..dd44bf0 100644
--- a/bin/varnishd/cache_dir.c
+++ b/bin/varnishd/cache_dir.c
@@ -94,12 +94,15 @@ VDI_RecycleFd(struct sess *sp)
 struct vbc *
 VDI_GetFd(const struct director *d, struct sess *sp)
 {
-
+	struct vbc *vbc;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	if (d == NULL)
 		d = sp->director;
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	return (d->getfd(d, sp));
+	vbc = d->getfd(d, sp);
+	if (vbc)
+		WSL(sp->wrk, SLT_BackendXID, vbc->fd, "%d", sp->xid);
+	return (vbc);
 }
 
 /* Check health ------------------------------------------------------
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 8ffff30..8ae2856 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -252,9 +252,9 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 		}
 		if (vbit_test(vsl->vbm_supress, t))
 			continue;
-		if (vsl->b_opt && !vbit_test(vsl->vbm_backend, u))
+		if (vsl->b_opt && !vbit_test(vsl->vbm_backend, u) && !vsl->c_opt)
 			continue;
-		if (vsl->c_opt && !vbit_test(vsl->vbm_client, u))
+		if (vsl->c_opt && !vbit_test(vsl->vbm_client, u) && !vsl->b_opt)
 			continue;
 		if (vsl->regincl != NULL) {
 			i = VRE_exec(vsl->regincl, VSL_DATA(p), VSL_LEN(p),



From tfheen at varnish-cache.org  Tue Aug  6 10:32:42 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 06 Aug 2013 12:32:42 +0200
Subject: [master] ffc4166 Increase delay before looking for panic in c00057
Message-ID: <E1V6eZG-0000yD-4X@project.varnish-software.com>

commit ffc4166d824dff1a1c9cecce10d26292b9bb21b0
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Aug 6 12:18:40 2013 +0200

    Increase delay before looking for panic in c00057

diff --git a/bin/varnishtest/tests/c00057.vtc b/bin/varnishtest/tests/c00057.vtc
index c8d3bc9..1db243d 100644
--- a/bin/varnishtest/tests/c00057.vtc
+++ b/bin/varnishtest/tests/c00057.vtc
@@ -16,6 +16,6 @@ client c1 {
 	txreq
 } -run
 
-delay 1
+delay 5
 
 varnish v1 -cliok "panic.show"



From phk at varnish-cache.org  Tue Aug  6 14:05:18 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 06 Aug 2013 16:05:18 +0200
Subject: [master] a200cd2 Move the sockaddr emitting function to vcc_utils.c
 (ne: vcc_string.c) where the rest of the compiler can get at it.
Message-ID: <E1V6ht0-00078B-0J@project.varnish-software.com>

commit a200cd2469ef31bcd1bf32e760be0e10e3f9fb6e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 6 14:04:35 2013 +0000

    Move the sockaddr emitting function to vcc_utils.c (ne: vcc_string.c)
    where the rest of the compiler can get at it.

diff --git a/lib/libvcl/Makefile.am b/lib/libvcl/Makefile.am
index ebe0526..91bc0d3 100644
--- a/lib/libvcl/Makefile.am
+++ b/lib/libvcl/Makefile.am
@@ -22,7 +22,7 @@ libvcl_la_SOURCES = \
 	vcc_fixed_token.c \
 	vcc_obj.c \
 	vcc_storage.c \
-	vcc_string.c \
+	vcc_utils.c \
 	vcc_symb.c \
 	vcc_token.c \
 	vcc_var.c \
diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c
index 9bab3ce..1c69ae8 100644
--- a/lib/libvcl/vcc_backend.c
+++ b/lib/libvcl/vcc_backend.c
@@ -61,7 +61,6 @@
 #include "vcc_compile.h"
 
 #include "vss.h"
-#include "vsa.h"
 
 struct host {
 	VTAILQ_ENTRY(host)      list;
@@ -69,42 +68,6 @@ struct host {
 	char			*vgcname;
 };
 
-/*
- * The IPv6 crew royally screwed up the entire idea behind
- * struct sockaddr, and combined with various other incomptency
- * in the OS business, that means that there is no sane or even
- * remotely portable way to initialize a sockaddr at compile time.
- *
- * In our case it is slightly more tricky than that, because we don't
- * even want to #include the struct sockaddr* definitions.
- *
- * Instead we make sure the sockaddr is sane (for our values of sane)
- * and dump it in binary, using a 64 bit integertype, hoping that this
- * will ensure good enough alignment.
- */
-
-static int
-emit_sockaddr(struct vcc *tl, const void *sa, unsigned sal)
-{
-	unsigned n = (sal + 7) / 8, len;
-	uint64_t b[n];
-
-	assert(VSA_Sane(sa));
-	AN(sa);
-	AN(sal);
-	assert(sal < 256);
-	assert(sizeof(unsigned long long) == 8);
-	Fh(tl, 0, "\nstatic const unsigned long long");
-	Fh(tl, 0, " sockaddr_%u[%d] = {\n", tl->unique, n);
-	memcpy(b, sa, sal);
-	for (len = 0; len <n; len++) {
-		Fh(tl, 0, "%s    0x%016jx",
-		    len ? ",\n" : "",
-		    (uintmax_t)b[len]);
-	}
-	Fh(tl, 0, "\n};\n");
-	return (tl->unique++);
-}
 
 /*--------------------------------------------------------------------
  * Struct sockaddr is not really designed to be a compile time
@@ -124,9 +87,10 @@ Emit_Sockaddr(struct vcc *tl, const struct token *t_host, const char *port)
 {
 	struct foo_proto protos[3], *pp;
 	struct addrinfo *res, *res0, *res1, hint;
-	int error, retval, x;
+	int error, retval;
 	char hbuf[NI_MAXHOST];
 	char *hop, *pop;
+	const char *sa;
 
 	AN(t_host->dec);
 
@@ -204,8 +168,8 @@ Emit_Sockaddr(struct vcc *tl, const struct token *t_host, const char *port)
 		pp->l =  res->ai_addrlen;
 		memcpy(&pp->sa, res->ai_addr, pp->l);
 
-		x = emit_sockaddr(tl, res->ai_addr, res->ai_addrlen);
-		Fb(tl, 0, "\t.%s_sockaddr = sockaddr_%u,\n", pp->name, x);
+		sa = vcc_sockaddr(tl, res->ai_addr, res->ai_addrlen);
+		Fb(tl, 0, "\t.%s_sockaddr = %s,\n", pp->name, sa);
 		error = getnameinfo(res->ai_addr,
 		    res->ai_addrlen, hbuf, sizeof hbuf,
 		    NULL, 0, NI_NUMERICHOST);
diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h
index feb993a..11bd782 100644
--- a/lib/libvcl/vcc_compile.h
+++ b/lib/libvcl/vcc_compile.h
@@ -286,7 +286,8 @@ void vcc_Parse(struct vcc *tl);
 sym_wildcard_t vcc_Stv_Wildcard;
 
 /* vcc_string.c */
-char *vcc_regexp(struct vcc *tl);
+const char *vcc_regexp(struct vcc *tl);
+const char *vcc_sockaddr(struct vcc *tl, const void *sa, unsigned sal);
 
 /* vcc_symb.c */
 struct symbol *VCC_AddSymbolStr(struct vcc *tl, const char *name, enum symkind);
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 88d7ccb..95eceba 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -441,7 +441,7 @@ vcc_Eval_Regsub(struct vcc *tl, struct expr **e, const struct symbol *sym)
 {
 	struct expr *e2;
 	int all = sym->eval_priv == NULL ? 0 : 1;
-	char *p;
+	const char *p;
 	char buf[128];
 
 	vcc_delete_expr(*e);
@@ -963,7 +963,7 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, enum var_type fmt)
 	struct expr *e2;
 	const struct cmps *cp;
 	char buf[256];
-	char *re;
+	const char *re;
 	const char *not;
 	struct token *tk;
 
diff --git a/lib/libvcl/vcc_string.c b/lib/libvcl/vcc_string.c
deleted file mode 100644
index 4e4762e..0000000
--- a/lib/libvcl/vcc_string.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include <stdio.h>
-#include <string.h>
-
-#include "vcc_compile.h"
-
-#include "vre.h"
-#include "vrt.h"
-
-/*--------------------------------------------------------------------*/
-
-char *
-vcc_regexp(struct vcc *tl)
-{
-	char buf[BUFSIZ], *p;
-	vre_t *t;
-	const char *error;
-	int erroroffset;
-
-	Expect(tl, CSTR);
-	if (tl->err)
-		return (NULL);
-	memset(&t, 0, sizeof t);
-	t = VRE_compile(tl->t->dec, 0, &error, &erroroffset);
-	if (t == NULL) {
-		VSB_printf(tl->sb,
-		    "Regexp compilation error:\n\n%s\n\n", error);
-		vcc_ErrWhere(tl, tl->t);
-		return (NULL);
-	}
-	VRE_free(&t);
-	sprintf(buf, "VGC_re_%u", tl->unique++);
-	p = TlAlloc(tl, strlen(buf) + 1);
-	strcpy(p, buf);
-
-	Fh(tl, 0, "static void *%s;\n", buf);
-	Fi(tl, 0, "\tVRT_re_init(&%s, ",buf);
-	EncToken(tl->fi, tl->t);
-	Fi(tl, 0, ");\n");
-	Ff(tl, 0, "\tVRT_re_fini(%s);\n", buf);
-	return (p);
-}
diff --git a/lib/libvcl/vcc_utils.c b/lib/libvcl/vcc_utils.c
new file mode 100644
index 0000000..bfb3c05
--- /dev/null
+++ b/lib/libvcl/vcc_utils.c
@@ -0,0 +1,116 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <sys/socket.h>
+
+#include "vcc_compile.h"
+
+#include "vre.h"
+#include "vrt.h"
+#include "vsa.h"
+
+/*--------------------------------------------------------------------*/
+
+const char *
+vcc_regexp(struct vcc *tl)
+{
+	char buf[BUFSIZ], *p;
+	vre_t *t;
+	const char *error;
+	int erroroffset;
+
+	Expect(tl, CSTR);
+	if (tl->err)
+		return (NULL);
+	memset(&t, 0, sizeof t);
+	t = VRE_compile(tl->t->dec, 0, &error, &erroroffset);
+	if (t == NULL) {
+		VSB_printf(tl->sb,
+		    "Regexp compilation error:\n\n%s\n\n", error);
+		vcc_ErrWhere(tl, tl->t);
+		return (NULL);
+	}
+	VRE_free(&t);
+	sprintf(buf, "VGC_re_%u", tl->unique++);
+	p = TlAlloc(tl, strlen(buf) + 1);
+	strcpy(p, buf);
+
+	Fh(tl, 0, "static void *%s;\n", buf);
+	Fi(tl, 0, "\tVRT_re_init(&%s, ",buf);
+	EncToken(tl->fi, tl->t);
+	Fi(tl, 0, ");\n");
+	Ff(tl, 0, "\tVRT_re_fini(%s);\n", buf);
+	return (p);
+}
+
+/*
+ * The IPv6 crew royally screwed up the entire idea behind
+ * struct sockaddr, and combined with various other incomptency
+ * in the OS business, that means that there is no sane or even
+ * remotely portable way to initialize a sockaddr at compile time.
+ *
+ * In our case it is slightly more tricky than that, because we don't
+ * even want to #include the struct sockaddr* definitions.
+ *
+ * Instead we make sure the sockaddr is sane (for our values of sane)
+ * and dump it in binary, using a 64 bit integertype, hoping that this
+ * will ensure good enough alignment.
+ */
+
+const char *
+vcc_sockaddr(struct vcc *tl, const void *sa, unsigned sal)
+{
+	unsigned n = (sal + 7) / 8, len;
+	uint64_t b[n];
+	char *p;
+
+	assert(VSA_Sane(sa));
+	AN(sa);
+	AN(sal);
+	assert(sal < 256);
+	assert(sizeof(unsigned long long) == 8);
+
+	p = TlAlloc(tl, 20);
+	sprintf(p, "sockaddr_%u", tl->unique++);
+
+	Fh(tl, 0, "\nstatic const unsigned long long");
+	Fh(tl, 0, " %s[%d] = {\n", p, n);
+	memcpy(b, sa, sal);
+	for (len = 0; len <n; len++) {
+		Fh(tl, 0, "%s    0x%016jx",
+		    len ? ",\n" : "",
+		    (uintmax_t)b[len]);
+	}
+	Fh(tl, 0, "\n};\n");
+	return (p);
+}



From phk at varnish-cache.org  Tue Aug  6 17:26:23 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 06 Aug 2013 19:26:23 +0200
Subject: [master] 7e4f3b3 Further mucking about with IP data type,
 this should make it work.
Message-ID: <E1V6l1b-0000pQ-Hu@project.varnish-software.com>

commit 7e4f3b30825c939c2de5355f457f19e8609ba3e1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 6 17:26:01 2013 +0000

    Further mucking about with IP data type, this should make it work.

diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index d3b212f..763e6ff 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -258,7 +258,7 @@ VRT_r_beresp_backend_name(const struct vrt_ctx *ctx)
 	return(ctx->bo->vbc->backend->vcl_name);
 }
 
-struct sockaddr_storage *
+const void *
 VRT_r_beresp_backend_ip(const struct vrt_ctx *ctx)
 {
 
@@ -546,7 +546,7 @@ REQ_BOOL(hash_always_miss)
 
 /*--------------------------------------------------------------------*/
 
-struct sockaddr_storage *
+const void *
 VRT_r_client_ip(const struct vrt_ctx *ctx)
 {
 
@@ -555,7 +555,7 @@ VRT_r_client_ip(const struct vrt_ctx *ctx)
 	return (&ctx->req->sp->sockaddr);
 }
 
-struct sockaddr_storage *
+const void *
 VRT_r_server_ip(const struct vrt_ctx *ctx)
 {
 	int i;
diff --git a/include/vrt.h b/include/vrt.h
index 8b6e8a8..701e131 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -55,7 +55,7 @@ typedef double				VCL_DURATION;
 typedef const char *			VCL_ENUM;
 typedef const char *			VCL_HEADER;
 typedef long				VCL_INT;
-typedef struct sockaddr_storage *	VCL_IP;
+typedef const void *			VCL_IP;
 typedef double				VCL_REAL;
 typedef const char *			VCL_STRING;
 typedef double				VCL_TIME;
@@ -125,7 +125,7 @@ struct vrt_backend {
 	const char			*port;
 
 	const void			*ipv4_sockaddr;
-	const void 			*ipv6_sockaddr;
+	const void			*ipv6_sockaddr;
 
 	const char			*hosthdr;
 
diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c
index 1c69ae8..a8773d0 100644
--- a/lib/libvcl/vcc_backend.c
+++ b/lib/libvcl/vcc_backend.c
@@ -50,11 +50,7 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-#include <sys/socket.h>
-
 #include <limits.h>
-#include <netdb.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -75,34 +71,14 @@ struct host {
  * and put it in an official sockaddr when we load the VCL.
  */
 
-struct foo_proto {
-	const char		*name;
-	int			family;
-	struct sockaddr_storage	sa;
-	socklen_t		l;
-};
-
 static void
 Emit_Sockaddr(struct vcc *tl, const struct token *t_host, const char *port)
 {
-	struct foo_proto protos[3], *pp;
-	struct addrinfo *res, *res0, *res1, hint;
-	int error, retval;
-	char hbuf[NI_MAXHOST];
+	const char *ipv4, *ipv4a, *ipv6, *ipv6a, *pa;
 	char *hop, *pop;
-	const char *sa;
 
 	AN(t_host->dec);
 
-	memset(protos, 0, sizeof protos);
-	protos[0].name = "ipv4"; protos[0].family = PF_INET;
-	protos[1].name = "ipv6"; protos[1].family = PF_INET6;
-
-	retval = 0;
-	memset(&hint, 0, sizeof hint);
-	hint.ai_family = PF_UNSPEC;
-	hint.ai_socktype = SOCK_STREAM;
-
 	if (VSS_parse(t_host->dec, &hop, &pop)) {
 		VSB_printf(tl->sb,
 		    "Backend host '%.*s': wrong syntax (unbalanced [...] ?)\n",
@@ -110,88 +86,20 @@ Emit_Sockaddr(struct vcc *tl, const struct token *t_host, const char *port)
 		vcc_ErrWhere(tl, t_host);
 		return;
 	}
-	error = getaddrinfo(
+	Resolve_Sockaddr(tl,
 	    hop != NULL ? hop : t_host->dec,
 	    pop != NULL ? pop : port,
-	    &hint, &res0);
-	free(hop);
-	free(pop);
-	if (error) {
-		VSB_printf(tl->sb,
-		    "Backend host '%.*s'"
-		    " could not be resolved to an IP address:\n", PF(t_host));
-		VSB_printf(tl->sb,
-		    "\t%s\n"
-		    "(Sorry if that error message is gibberish.)\n",
-		    gai_strerror(error));
-		vcc_ErrWhere(tl, t_host);
-		return;
-	}
-
-	for (res = res0; res; res = res->ai_next) {
-		for (pp = protos; pp->name != NULL; pp++)
-			if (res->ai_family == pp->family)
-				break;
-		if (pp->name == NULL) {
-			/* Unknown proto, ignore */
-			continue;
-		}
-		if (pp->l == res->ai_addrlen &&
-		    !memcmp(&pp->sa, res->ai_addr, pp->l)) {
-			/*
-			 * Same address we already emitted.
-			 * This can happen using /etc/hosts
-			 */
-			continue;
-		}
-
-		if (pp->l > 0) {
-			VSB_printf(tl->sb,
-			    "Backend host %.*s: resolves to "
-			    "multiple %s addresses.\n"
-			    "Only one address is allowed.\n"
-			    "Please specify which exact address "
-			    "you want to use, we found these:\n",
-			    PF(t_host), pp->name);
-			for (res1 = res0; res1 != NULL; res1 = res1->ai_next) {
-				error = getnameinfo(res1->ai_addr,
-				    res1->ai_addrlen, hbuf, sizeof hbuf,
-				    NULL, 0, NI_NUMERICHOST);
-				AZ(error);
-				VSB_printf(tl->sb, "\t%s\n", hbuf);
-			}
-			freeaddrinfo(res0);
-			vcc_ErrWhere(tl, t_host);
-			return;
-		}
-
-		pp->l =  res->ai_addrlen;
-		memcpy(&pp->sa, res->ai_addr, pp->l);
-
-		sa = vcc_sockaddr(tl, res->ai_addr, res->ai_addrlen);
-		Fb(tl, 0, "\t.%s_sockaddr = %s,\n", pp->name, sa);
-		error = getnameinfo(res->ai_addr,
-		    res->ai_addrlen, hbuf, sizeof hbuf,
-		    NULL, 0, NI_NUMERICHOST);
-		AZ(error);
-		Fb(tl, 0, "\t.%s_addr = \"%s\",\n", pp->name, hbuf);
-		retval++;
-	}
-	if (res0 != NULL) {
-		error = getnameinfo(res0->ai_addr,
-		    res0->ai_addrlen, NULL, 0, hbuf, sizeof hbuf,
-		    NI_NUMERICSERV);
-		AZ(error);
-		Fb(tl, 0, "\t.port = \"%s\",\n", hbuf);
+	    &ipv4, &ipv4a, &ipv6, &ipv6a, &pa, 2, t_host, "Backend host");
+	ERRCHK(tl);
+	if (ipv4 != NULL) {
+		Fb(tl, 0, "\t.ipv4_sockaddr = %s,\n", ipv4);
+		Fb(tl, 0, "\t.ipv4_addr = \"%s\",\n", ipv4a);
 	}
-	freeaddrinfo(res0);
-	if (retval == 0) {
-		VSB_printf(tl->sb,
-		    "Backend host '%.*s': resolves to "
-		    "neither IPv4 nor IPv6 addresses.\n",
-		    PF(t_host) );
-		vcc_ErrWhere(tl, t_host);
+	if (ipv6 != NULL) {
+		Fb(tl, 0, "\t.ipv6_sockaddr = %s,\n", ipv6);
+		Fb(tl, 0, "\t.ipv6_addr = \"%s\",\n", ipv6a);
 	}
+	Fb(tl, 0, "\t.port = \"%s\",\n", pa);
 }
 
 /*--------------------------------------------------------------------
diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h
index 11bd782..1b6da33 100644
--- a/lib/libvcl/vcc_compile.h
+++ b/lib/libvcl/vcc_compile.h
@@ -50,6 +50,7 @@
 
 struct vsb;
 struct token;
+struct sockaddr_storage;
 
 #define isident1(c) (isalpha(c))
 #define isident(c) (isalpha(c) || isdigit(c) || (c) == '_' || (c) == '-')
@@ -285,9 +286,12 @@ void vcc_Parse(struct vcc *tl);
 /* vcc_storage.c */
 sym_wildcard_t vcc_Stv_Wildcard;
 
-/* vcc_string.c */
+/* vcc_utils.c */
 const char *vcc_regexp(struct vcc *tl);
-const char *vcc_sockaddr(struct vcc *tl, const void *sa, unsigned sal);
+void Resolve_Sockaddr(struct vcc *tl, const char *host, const char *port, \
+    const char **ipv4, const char **ipv4_ascii, const char **ipv6, \
+    const char **ipv6_ascii, const char **p_ascii, int maxips,
+    const struct token *t_err, const char *errid);
 
 /* vcc_symb.c */
 struct symbol *VCC_AddSymbolStr(struct vcc *tl, const char *name, enum symkind);
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 95eceba..164a8d6 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -652,6 +652,7 @@ static void
 vcc_expr4(struct vcc *tl, struct expr **e, enum var_type fmt)
 {
 	struct expr *e1, *e2;
+	const char *ip;
 	const struct symbol *sym;
 	double d;
 
@@ -698,13 +699,22 @@ vcc_expr4(struct vcc *tl, struct expr **e, enum var_type fmt)
 		return;
 	case CSTR:
 		assert(fmt != VOID);
-		e1 = vcc_new_expr();
-		EncToken(e1->vsb, tl->t);
-		e1->fmt = STRING;
+		if (fmt == IP) {
+			Resolve_Sockaddr(tl, tl->t->dec, "80",
+			    &ip, NULL, &ip, NULL, NULL, 1,
+			    tl->t, "IP constant");
+			ERRCHK(tl);
+			e1 = vcc_mk_expr(IP, "%s", ip);
+			ERRCHK(tl);
+		} else {
+			e1 = vcc_new_expr();
+			EncToken(e1->vsb, tl->t);
+			e1->fmt = STRING;
+			AZ(VSB_finish(e1->vsb));
+		}
 		e1->t1 = tl->t;
 		e1->constant = EXPR_CONST;
 		vcc_NextToken(tl);
-		AZ(VSB_finish(e1->vsb));
 		*e = e1;
 		break;
 	case CNUM:
diff --git a/lib/libvcl/vcc_utils.c b/lib/libvcl/vcc_utils.c
index bfb3c05..e75cf25 100644
--- a/lib/libvcl/vcc_utils.c
+++ b/lib/libvcl/vcc_utils.c
@@ -31,7 +31,9 @@
 
 #include <stdio.h>
 #include <string.h>
+#include <sys/types.h>
 #include <sys/socket.h>
+#include <netdb.h>
 
 #include "vcc_compile.h"
 
@@ -87,7 +89,7 @@ vcc_regexp(struct vcc *tl)
  * will ensure good enough alignment.
  */
 
-const char *
+static const char *
 vcc_sockaddr(struct vcc *tl, const void *sa, unsigned sal)
 {
 	unsigned n = (sal + 7) / 8, len;
@@ -97,13 +99,13 @@ vcc_sockaddr(struct vcc *tl, const void *sa, unsigned sal)
 	assert(VSA_Sane(sa));
 	AN(sa);
 	AN(sal);
-	assert(sal < 256);
+	assert(sal < sizeof(struct sockaddr_storage));
 	assert(sizeof(unsigned long long) == 8);
 
 	p = TlAlloc(tl, 20);
 	sprintf(p, "sockaddr_%u", tl->unique++);
 
-	Fh(tl, 0, "\nstatic const unsigned long long");
+	Fh(tl, 0, "static const unsigned long long");
 	Fh(tl, 0, " %s[%d] = {\n", p, n);
 	memcpy(b, sa, sal);
 	for (len = 0; len <n; len++) {
@@ -114,3 +116,140 @@ vcc_sockaddr(struct vcc *tl, const void *sa, unsigned sal)
 	Fh(tl, 0, "\n};\n");
 	return (p);
 }
+
+/*--------------------------------------------------------------------
+ * This routine is a monster, but at least we only have one such monster.
+ * Look up a IP number, and return IPv4/IPv6 address as VGC produced names
+ * and optionally ascii strings.
+ *
+ * For IP compile time constants we only want one IP#, but it can be
+ * IPv4 or IPv6.
+ *
+ * For backends, we accept up to one IPv4 and one IPv6.
+ */
+
+struct foo_proto {
+	const char		*name;
+	int			family;
+	struct sockaddr_storage	sa;
+	socklen_t		l;
+	const char		**dst;
+	const char		**dst_ascii;
+};
+
+void
+Resolve_Sockaddr(struct vcc *tl,
+    const char *host,
+    const char *port,
+    const char **ipv4,
+    const char **ipv4_ascii,
+    const char **ipv6,
+    const char **ipv6_ascii,
+    const char **p_ascii,
+    int maxips,
+    const struct token *t_err,
+    const char *errid)
+{
+	struct foo_proto protos[3], *pp;
+	struct addrinfo *res, *res0, *res1, hint;
+	int error, retval;
+	char hbuf[NI_MAXHOST];
+
+	memset(protos, 0, sizeof protos);
+	protos[0].name = "ipv4";
+	protos[0].family = PF_INET;
+	protos[0].dst = ipv4;
+	protos[0].dst_ascii = ipv4_ascii;
+	*ipv4 = NULL;
+
+	protos[1].name = "ipv6";
+	protos[1].family = PF_INET6;
+	protos[1].dst = ipv6;
+	protos[1].dst_ascii = ipv6_ascii;
+	*ipv6 = NULL;
+
+	retval = 0;
+	memset(&hint, 0, sizeof hint);
+	hint.ai_family = PF_UNSPEC;
+	hint.ai_socktype = SOCK_STREAM;
+
+	error = getaddrinfo(host, port, &hint, &res0);
+	if (error) {
+		VSB_printf(tl->sb,
+		    "%s '%.*s' could not be resolved to an IP address:\n",
+		    errid, PF(t_err));
+		VSB_printf(tl->sb,
+		    "\t%s\n"
+		    "(Sorry if that error message is gibberish.)\n",
+		    gai_strerror(error));
+		vcc_ErrWhere(tl, t_err);
+		return;
+	}
+
+	for (res = res0; res; res = res->ai_next) {
+		for (pp = protos; pp->name != NULL; pp++)
+			if (res->ai_family == pp->family)
+				break;
+		if (pp->name == NULL) {
+			/* Unknown proto, ignore */
+			continue;
+		}
+		if (pp->l == res->ai_addrlen &&
+		    !memcmp(&pp->sa, res->ai_addr, pp->l)) {
+			/*
+			 * Same address we already emitted.
+			 * This can happen using /etc/hosts
+			 */
+			continue;
+		}
+
+		if (pp->l != 0 || retval == maxips) {
+			VSB_printf(tl->sb,
+			    "%s %.*s: resolves to too many addresses.\n"
+			    "Only one IPv4 %s IPv6 are allowed.\n"
+			    "Please specify which exact address "
+			    "you want to use, we found all of these:\n",
+			    errid, PF(t_err),
+			    maxips > 1 ? "and one" :  "or");
+			for (res1 = res0; res1 != NULL; res1 = res1->ai_next) {
+				error = getnameinfo(res1->ai_addr,
+				    res1->ai_addrlen, hbuf, sizeof hbuf,
+				    NULL, 0, NI_NUMERICHOST);
+				AZ(error);
+				VSB_printf(tl->sb, "\t%s\n", hbuf);
+			}
+			freeaddrinfo(res0);
+			vcc_ErrWhere(tl, t_err);
+			return;
+		}
+
+		pp->l =  res->ai_addrlen;
+		assert(pp->l < sizeof(struct sockaddr_storage));
+		memcpy(&pp->sa, res->ai_addr, pp->l);
+
+		error = getnameinfo(res->ai_addr, res->ai_addrlen,
+		    hbuf, sizeof hbuf, NULL, 0, NI_NUMERICHOST);
+		AZ(error);
+
+		Fh(tl, 0, "\n/* \"%s\" -> %s */\n", host, hbuf);
+		*(pp->dst) = vcc_sockaddr(tl, &pp->sa, pp->l);
+		if (pp->dst_ascii != NULL) {
+			*pp->dst_ascii = TlDup(tl, hbuf);
+		}
+		retval++;
+	}
+	if (p_ascii != NULL) {
+		error = getnameinfo(res0->ai_addr,
+		    res0->ai_addrlen, NULL, 0, hbuf, sizeof hbuf,
+		    NI_NUMERICSERV);
+		AZ(error);
+		*p_ascii = TlDup(tl, hbuf);
+	}
+	if (retval == 0) {
+		VSB_printf(tl->sb,
+		    "%s '%.*s': resolves to "
+		    "neither IPv4 nor IPv6 addresses.\n",
+		    errid, PF(t_err) );
+		vcc_ErrWhere(tl, t_err);
+	}
+}



From ssm at varnish-cache.org  Fri Aug  9 08:49:31 2013
From: ssm at varnish-cache.org (Stig Sandbeck Mathisen)
Date: Fri, 09 Aug 2013 10:49:31 +0200
Subject: [master] dc16e3c Correct typo in vmod_std example
Message-ID: <E1V7iO3-0004JT-0r@project.varnish-software.com>

commit dc16e3c75584bcd0bfa4ce3a868c747a58cd64d3
Author: Stig Sandbeck Mathisen <ssm at redpill-linpro.com>
Date:   Fri Aug 9 10:47:16 2013 +0200

    Correct typo in vmod_std example

diff --git a/doc/sphinx/reference/vmod_std.rst b/doc/sphinx/reference/vmod_std.rst
index f8c4456..d310986 100644
--- a/doc/sphinx/reference/vmod_std.rst
+++ b/doc/sphinx/reference/vmod_std.rst
@@ -63,7 +63,7 @@ Description
 	utilize it.
 Example
 	| if (req.url ~ ^/slow/) {
-	|    std.set_up_tos(0x0);
+	|    std.set_ip_tos(0x0);
 	| }
 
 random



From phk at varnish-cache.org  Fri Aug  9 10:24:41 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 09 Aug 2013 12:24:41 +0200
Subject: [master] cb94438 Add std.ip() conversion function
Message-ID: <E1V7js9-00022S-U0@project.varnish-software.com>

commit cb94438d7e393dcf4d45db5a66030cb7eb167695
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Aug 9 10:24:32 2013 +0000

    Add std.ip() conversion function

diff --git a/bin/varnishtest/tests/m00011.vtc b/bin/varnishtest/tests/m00011.vtc
new file mode 100644
index 0000000..0a7e517
--- /dev/null
+++ b/bin/varnishtest/tests/m00011.vtc
@@ -0,0 +1,28 @@
+varnishtest "Test std.ip"
+
+
+server s1 {
+	rxreq
+	txresp -body "1"
+} -start
+
+
+varnish v1 -vcl+backend {
+
+	import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ;
+	sub vcl_deliver {
+		set resp.http.foo0 = std.ip("8.8.8.*", client.ip);
+		set resp.http.foo1 = std.ip("9.9.9.*", server.ip);
+		set resp.http.foo2 = std.ip("1.2.3.*", "127.0.0.2");
+		set resp.http.foo3 = std.ip("1.2.3.5", "127.0.0.3");
+	}
+} -start
+
+client c1 {
+	txreq -url "/foo1"
+	rxresp
+	expect resp.http.foo0 == "127.0.0.1"
+	expect resp.http.foo1 == "127.0.0.1"
+	expect resp.http.foo2 == "127.0.0.2"
+	expect resp.http.foo3 == "1.2.3.5"
+} -run
diff --git a/include/vsa.h b/include/vsa.h
index c9d521a..3ac8eb2 100644
--- a/include/vsa.h
+++ b/include/vsa.h
@@ -30,9 +30,9 @@
 #ifndef VSA_H_INCLUDED
 #define VSA_H_INCLUDED
 
-int VSA_Sane(const struct sockaddr_storage *ss);
-socklen_t VSA_Len(const struct sockaddr_storage *ss);
-unsigned VSA_Port(const struct sockaddr_storage *ss);
-int VSA_Compare(const struct sockaddr_storage *ss1, const void *ss2);
+int VSA_Sane(const void *);
+socklen_t VSA_Len(const void *);
+unsigned VSA_Port(const void *);
+int VSA_Compare(const void *, const void *);
 
 #endif
diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c
index 51f214a..1f2a92f 100644
--- a/lib/libvarnish/vsa.c
+++ b/lib/libvarnish/vsa.c
@@ -42,9 +42,11 @@
 #include "vsa.h"
 
 int
-VSA_Sane(const struct sockaddr_storage *ss)
+VSA_Sane(const void *s)
 {
-	switch(ss->ss_family) {
+	const struct sockaddr *sa = s;
+
+	switch(sa->sa_family) {
 		case PF_INET:
 		case PF_INET6:
 			return (1);
@@ -54,9 +56,11 @@ VSA_Sane(const struct sockaddr_storage *ss)
 }
 
 socklen_t
-VSA_Len(const struct sockaddr_storage *ss)
+VSA_Len(const void *s)
 {
-	switch(ss->ss_family) {
+	const struct sockaddr *sa = s;
+
+	switch(sa->sa_family) {
 		case PF_INET:
 			return (sizeof(struct sockaddr_in));
 		case PF_INET6:
@@ -67,29 +71,33 @@ VSA_Len(const struct sockaddr_storage *ss)
 }
 
 int
-VSA_Compare(const struct sockaddr_storage *ss1, const void *ss2)
+VSA_Compare(const void *s1, const void *s2)
 {
-	switch(ss1->ss_family) {
+	const struct sockaddr *sa = s1;
+
+	switch(sa->sa_family) {
 		case PF_INET:
 		case PF_INET6:
-			return (memcmp(ss1, ss2, VSA_Len(ss1)));
+			return (memcmp(s1, s2, VSA_Len(s1)));
 		default:
 			return (-1);
 	}
 }
 
 unsigned
-VSA_Port(const struct sockaddr_storage *ss)
+VSA_Port(const void *s)
 {
-	switch(ss->ss_family) {
+	const struct sockaddr *sa = s;
+
+	switch(sa->sa_family) {
 		case PF_INET:
 			{
-			const struct sockaddr_in *ain = (const void *)ss;
+			const struct sockaddr_in *ain = s;
 			return (ntohs((ain->sin_port)));
 			}
 		case PF_INET6:
 			{
-			const struct sockaddr_in6 *ain = (const void *)ss;
+			const struct sockaddr_in6 *ain = s;
 			return (ntohs((ain->sin6_port)));
 			}
 		default:
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index 3238164..1f4e33e 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -36,3 +36,4 @@ Function STRING fileread(PRIV_CALL, STRING)
 Function DURATION duration(STRING, DURATION)
 Function INT integer(STRING, INT)
 Function VOID collect(HEADER)
+Function IP ip(STRING, IP)
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index c763aa3..b5649b6 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -33,9 +33,14 @@
 #include <math.h>
 #include <stdlib.h>
 
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+
 #include "cache/cache.h"
 
 #include "vrt.h"
+#include "vsa.h"
 #include "vcc_if.h"
 
 VCL_DURATION __match_proto__()
@@ -117,3 +122,37 @@ vmod_integer(const struct vrt_ctx *ctx, const char *p, VCL_INT i)
 
 	return (r);
 }
+
+VCL_IP
+vmod_ip(const struct vrt_ctx *ctx, VCL_STRING s, VCL_IP d)
+{
+	struct addrinfo hints, *res0;
+	const struct addrinfo *res;
+	int error;
+	char *p;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	AN(d);
+	assert(VSA_Sane(d));
+
+	if (s != NULL) {
+		memset(&hints, 0, sizeof(hints));
+		hints.ai_family = PF_UNSPEC;
+		hints.ai_socktype = SOCK_STREAM;
+		error = getaddrinfo(s, "80", &hints, &res0);
+		if (!error) {
+			for (res = res0; res != NULL; res = res->ai_next) {
+				if (VSA_Sane(res->ai_addr) &&
+				    res->ai_addrlen >= VSA_Len(res->ai_addr)) {
+					d = res->ai_addr;
+					break;
+				}
+			}
+		}
+	}
+	AN(d);
+	p = WS_Alloc(ctx->ws, VSA_Len(d));
+	AN(p);
+	memcpy(p, d, VSA_Len(d));
+	return (p);
+}



From phk at varnish-cache.org  Fri Aug  9 11:55:17 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 09 Aug 2013 13:55:17 +0200
Subject: [master] ad1eb2f Fix memory leak I just introduced.
Message-ID: <E1V7lHp-0007Bv-Qk@project.varnish-software.com>

commit ad1eb2fd1877cd5f6d444faae8d9730dc5e7eb55
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Aug 9 11:54:49 2013 +0000

    Fix memory leak I just introduced.
    
    Spotted by:	Federico Schwindt

diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index b5649b6..651a77c 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -126,7 +126,7 @@ vmod_integer(const struct vrt_ctx *ctx, const char *p, VCL_INT i)
 VCL_IP
 vmod_ip(const struct vrt_ctx *ctx, VCL_STRING s, VCL_IP d)
 {
-	struct addrinfo hints, *res0;
+	struct addrinfo hints, *res0 = NULL;
 	const struct addrinfo *res;
 	int error;
 	char *p;
@@ -154,5 +154,7 @@ vmod_ip(const struct vrt_ctx *ctx, VCL_STRING s, VCL_IP d)
 	p = WS_Alloc(ctx->ws, VSA_Len(d));
 	AN(p);
 	memcpy(p, d, VSA_Len(d));
+	if (res0 != NULL)
+		freeaddrinfo(res0);
 	return (p);
 }



From phk at varnish-cache.org  Mon Aug 12 06:53:19 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Aug 2013 08:53:19 +0200
Subject: [master] 8988301 Start kicking the state-machine into final form.
Message-ID: <E1V8m0F-0004w4-F4@project.varnish-software.com>

commit 8988301e9e342709b21ecafa164538ca16e014ed
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 12 06:52:59 2013 +0000

    Start kicking the state-machine into final form.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index ec7505c..f8ab8aa 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -339,13 +339,13 @@ cnt_error(struct worker *wrk, struct req *req)
  * Prepare to fetch body from backend
  *
 DOT subgraph xcluster_body {
-DOT	fetchbody [
+DOT	fetch [
 DOT		shape=record
-DOT		label="{cnt_fetchbody:|start fetch_thread}"
+DOT		label="{cnt_fetch:|start fetch_thread}"
 DOT	]
 DOT }
-DOT fetchbody:out -> prepresp [style=bold,color=red]
-DOT fetchbody:out -> prepresp [style=bold,color=blue]
+DOT fetch:out -> prepresp [style=bold,color=red]
+DOT fetch:out -> prepresp [style=bold,color=blue]
  */
 
 static enum req_fsm_nxt
@@ -393,10 +393,10 @@ cnt_fetch(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_lookup {
 DOT	lookup [
 DOT		shape=record
-DOT		label="{<top>cnt_lookup:|hash lookup|{<busy>busy ?|<miss>miss ?}|{<no>no|obj.f.pass?|<yes>yes}}"
+DOT		label="{<top>cnt_lookup:|hash lookup|{<busy>busy?|exp|expbusy|hit|miss}|{vcl_lookup\{\}|{xx|xx}}|{<no>no|obj.f.pass?|<yes>yes}}"
 DOT	]
 DOT }
-DOT lookup:busy -> lookup:top [label="(waitinglist)"]
+DOT lookup:busy:w -> lookup:top:w [label="(waitinglist)"]
 DOT lookup:miss -> miss [style=bold,color=blue]
 DOT lookup:no -> hit [style=bold,color=green]
 DOT lookup:yes -> pass [style=bold,color=red]
@@ -547,7 +547,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_miss {
 DOT	miss [
 DOT		shape=record
-DOT		label="{cnt_miss:|filter req.-\>bereq.|{vcl_miss\{\}|{req.*|bereq.*}}|{<err>error?|<rst>restart?}|{<pass>pass?|<fetch>fetch?}}"
+DOT		label="{cnt_miss:|{vcl_miss\{\}|req.*}|{<err>error?|<rst>restart?}|{<pass>pass?|<fetch>fetch?}}"
 DOT	]
 DOT }
 DOT miss:fetch -> fetch [label="fetch",style=bold,color=blue]
@@ -591,10 +591,11 @@ cnt_miss(struct worker *wrk, struct req *req)
 		req->req_step = R_STP_RESTART;
 		return (REQ_FSM_MORE);
 	case VCL_RET_PASS:
+		VBO_DerefBusyObj(wrk, &req->busyobj);
 		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
-		req->objcore = HSH_NewObjCore(wrk);
-		bo->do_pass = 1;
-		break;
+		req->objcore = NULL;
+		req->req_step = R_STP_PASS;
+		return (REQ_FSM_MORE);
 	case VCL_RET_FETCH:
 		break;
 	default:
@@ -602,13 +603,12 @@ cnt_miss(struct worker *wrk, struct req *req)
 	}
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	/* One ref for req, one for vbf_fetch_body */
-	bo->refcount = 2;
 
 	AN (req->objcore);
-	req->objcore->busyobj = bo;
 	wrk->stats.cache_miss++;
 
+	req->objcore->busyobj = bo;
+	bo->refcount = 2;	/* One ref for req, one for vbf_fetch_body */
 	req->req_step = R_STP_FETCH;
 	return (REQ_FSM_MORE);
 }
@@ -620,7 +620,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_pass {
 DOT	pass [
 DOT		shape=record
-DOT		label="{cnt_pass:|(XXX: deref obj.)|filter req.*-\>bereq.|{vcl_pass\{\}|{req.*|bereq.*}}|{<err>error?|<rst>restart?}|<pass>create anon obj}"
+DOT		label="{cnt_pass:|{vcl_pass\{\}|req.*}|{<err>error?|<rst>restart?}|<pass>create anon obj}"
 DOT	]
 DOT }
 DOT pass:pass -> fetch [style=bold, color=red]
@@ -658,9 +658,9 @@ cnt_pass(struct worker *wrk, struct req *req)
 	req->busyobj = bo;
 	bo->do_pass = 1;
 	req->objcore = HSH_NewObjCore(wrk);
-	req->objcore->busyobj = bo;
-	bo->refcount = 2;
 
+	req->objcore->busyobj = bo;
+	bo->refcount = 2;	/* One ref for req, one for vbf_fetch_body */
 	req->req_step = R_STP_FETCH;
 	return (REQ_FSM_MORE);
 }
@@ -671,23 +671,16 @@ cnt_pass(struct worker *wrk, struct req *req)
  *
 DOT subgraph xcluster_pipe {
 DOT	pipe [
-DOT		shape=ellipse
-DOT		label="Filter req.->bereq."
-DOT	]
-DOT	vcl_pipe [
 DOT		shape=record
-DOT		label="vcl_pipe()|req.\nbereq\."
+DOT		label="{cnt_pipe:|filter req.*-\>bereq.*|{vcl_pipe()|req.*\nbereq\.*}|{<pipe>pipe?|<error>error?}}"
 DOT	]
 DOT	pipe_do [
 DOT		shape=ellipse
 DOT		label="send bereq.\npipe until close"
 DOT	]
-DOT	vcl_pipe -> pipe_do [label="pipe",style=bold,color=orange]
-DOT	pipe -> vcl_pipe [style=bold,color=orange]
+DOT	pipe:pipe -> pipe_do [label="pipe",style=bold,color=orange]
 DOT }
 DOT pipe_do -> DONE [style=bold,color=orange]
-DOT vcl_pipe -> err_pipe [label="error"]
-DOT err_pipe [label="ERROR",shape=plaintext]
  */
 
 static enum req_fsm_nxt
@@ -762,23 +755,15 @@ cnt_restart(const struct worker *wrk, struct req *req)
 DOT subgraph xcluster_recv {
 DOT	recv [
 DOT		shape=record
-DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}|{<pass>pass?|<lookup>lookup?|<pipe>pipe?|<error>error?}}"
-DOT	]
-DOT }
-DOT subgraph xcluster_hash {
-DOT	hash [
-DOT		shape=record
-DOT		label="{cnt_recv:|{vcl_hash\{\}|req.*}}"
+DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}|{vcl_hash\{\}|req.*}|{<pass>pass?|<lookup>lookup?|<pipe>pipe?|<error>error?|<purge>purge?}}"
 DOT	]
 DOT }
-DOT recv:pipe -> hash [style=bold,color=orange]
-DOT recv:pass -> hash [style=bold,color=red]
+DOT recv:pipe -> pipe [style=bold,color=orange]
+DOT recv:pass -> pass [style=bold,color=red]
+DOT recv:lookup -> lookup [style=bold,color=green]
+DOT recv:purge -> purge [style=bold,color=purple]
 #DOT recv:error -> err_recv
 #DOT err_recv [label="ERROR",shape=plaintext]
-DOT recv:lookup -> hash [style=bold,color=green]
-DOT hash -> lookup [style=bold,color=green]
-DOT hash -> pipe [style=bold,color=orange]
-DOT hash -> pass [style=bold,color=red]
  */
 
 static enum req_fsm_nxt
diff --git a/bin/varnishtest/tests/c00012.vtc b/bin/varnishtest/tests/c00012.vtc
index 570fc93..79185e1 100644
--- a/bin/varnishtest/tests/c00012.vtc
+++ b/bin/varnishtest/tests/c00012.vtc
@@ -25,7 +25,7 @@ client c1 {
 	rxresp
 	expect resp.status == 200
 	expect resp.bodylen == 7
-	expect resp.http.x-varnish == "1003"
+	expect resp.http.x-varnish == "1004"
 } 
 
 client c1 -run
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 1226b1e..5250cc5 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -234,28 +234,28 @@ sp_variables = (
 	),
 	('bereq.method',
 		'STRING',
-		( 'pipe', 'backend_fetch', 'pass', 'miss', 'backend_response',),
-		( 'pipe', 'backend_fetch', 'pass', 'miss', 'backend_response',),
+		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
+		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
 	),
 	('bereq.request',
 		'STRING',
-		( 'pipe', 'backend_fetch', 'pass', 'miss', 'backend_response',),
-		( 'pipe', 'backend_fetch', 'pass', 'miss', 'backend_response',),
+		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
+		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
 	),
 	('bereq.url',
 		'STRING',
-		( 'pipe', 'backend_fetch', 'pass', 'miss', 'backend_response',),
-		( 'pipe', 'backend_fetch', 'pass', 'miss', 'backend_response',),
+		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
+		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
 	),
 	('bereq.proto',
 		'STRING',
-		( 'pipe', 'backend_fetch', 'pass', 'miss', 'backend_response',),
-		( 'pipe', 'backend_fetch', 'pass', 'miss', 'backend_response',),
+		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
+		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
 	),
 	('bereq.http.',
 		'HEADER',
-		( 'pipe', 'backend_fetch', 'pass', 'miss', 'backend_response',),
-		( 'pipe', 'backend_fetch', 'pass', 'miss', 'backend_response',),
+		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
+		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
 	),
 	('bereq.uncacheable',
 		'BOOL',
@@ -264,18 +264,18 @@ sp_variables = (
 	),
 	('bereq.connect_timeout',
 		'DURATION',
-		( 'pipe', 'backend_fetch', 'pass', 'miss',),
-		( 'pipe', 'backend_fetch', 'pass', 'miss',),
+		( 'pipe', 'backend_fetch', 'miss',),
+		( 'pipe', 'backend_fetch', 'miss',),
 	),
 	('bereq.first_byte_timeout',
 		'DURATION',
-		( 'backend_fetch', 'pass', 'miss',),
-		( 'backend_fetch', 'pass', 'miss',),
+		( 'backend_fetch', 'miss',),
+		( 'backend_fetch', 'miss',),
 	),
 	('bereq.between_bytes_timeout',
 		'DURATION',
-		( 'backend_fetch', 'pass', 'miss',),
-		( 'backend_fetch', 'pass', 'miss',),
+		( 'backend_fetch', 'miss',),
+		( 'backend_fetch', 'miss',),
 	),
 	('beresp.proto',
 		'STRING',



From phk at varnish-cache.org  Mon Aug 12 08:25:37 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Aug 2013 10:25:37 +0200
Subject: [master] 0c189d8 Ignore graced hit-for-pass objects.
Message-ID: <E1V8nRZ-000853-7z@project.varnish-software.com>

commit 0c189d8cef05014ef82d2bcf6eea57fe615c68bc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 12 08:25:25 2013 +0000

    Ignore graced hit-for-pass objects.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 3d3ef81..ffd1087 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -404,7 +404,8 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 			*ocp = oc;
 			return (HSH_HIT);
 		}
-		if (o->exp.entered > exp_entered) {
+		if (o->exp.entered > exp_entered &&
+		    !(oc->flags & OC_F_PASS)) {
 			/* record the newest object */
 			exp_oc = oc;
 			exp_o = o;
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index f8ab8aa..c9974fd 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -433,58 +433,72 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		 */
 		return (REQ_FSM_DISEMBARK);
 	}
+
 	AZ(req->objcore);
+	if (lr == HSH_MISS) {
+		/* Found nothing */
+		VSLb(req->vsl, SLT_Debug, "XXXX MISS\n");
+		AZ(oc);
+		AN(boc);
+		AN(boc->flags & OC_F_BUSY);
+		req->objcore = boc;
+		req->req_step = R_STP_MISS;
+		return (REQ_FSM_MORE);
+	}
+
+	AN(oc);
+	AZ (oc->flags & OC_F_BUSY);
+
+	if (oc->flags & OC_F_PASS) {
+		/* Found a hit-for-pass */
+		VSLb(req->vsl, SLT_Debug, "XXXX HIT-FOR-PASS\n");
+		AZ(boc);
+		(void)HSH_Deref(&wrk->stats, oc, NULL);
+		req->objcore = NULL;
+		VRY_Finish(req, NULL);
+		wrk->stats.cache_hitpass++;
+		req->req_step = R_STP_PASS;
+		return (REQ_FSM_MORE);
+	}
 
+	if (boc != NULL)
+		AN(boc->flags & OC_F_BUSY);
 
 	switch (lr) {
 	case HSH_EXP:
+		/* Found expired object, and a busy already exists too */
 		VSLb(req->vsl, SLT_Debug, "XXXX EXP\n");
-		AN(oc);
 		AZ(boc);
 		break;
 	case HSH_EXPBUSY:
+		/* Found expired object, inserted busy objcore */
 		VSLb(req->vsl, SLT_Debug, "XXXX EXPBUSY\n");
-		AN(oc);
 		AN(boc);
 		if (VDI_Healthy(req->director, req->digest)) {
 			VSLb(req->vsl, SLT_Debug, "XXX EXPBUSY deref oc\n");
 			(void)HSH_Deref(&wrk->stats, oc, NULL);
-			oc = boc;
-			boc = NULL;
-		} else {
-			VSLb(req->vsl, SLT_Debug, "XXX EXPBUSY drop boc\n");
-			(void)HSH_Deref(&wrk->stats, boc, NULL);
-			boc = NULL;
+			req->objcore = boc;
+			req->req_step = R_STP_MISS;
+			return (REQ_FSM_MORE);
 		}
-		break;
-	case HSH_MISS:
-		VSLb(req->vsl, SLT_Debug, "XXXX MISS\n");
-		AZ(oc);
-		AN(boc);
-		oc = boc;
+		VSLb(req->vsl, SLT_Debug, "XXX EXPBUSY drop boc\n");
+		(void)HSH_Deref(&wrk->stats, boc, NULL);
 		boc = NULL;
-		AN(oc->flags & OC_F_BUSY);
 		break;
 	case HSH_HIT:
+		/* Found hit */
 		VSLb(req->vsl, SLT_Debug, "XXXX HIT\n");
-		AN(oc);
 		AZ(boc);
 		break;
 	default:
 		INCOMPL();
 	}
 
+
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	oh = oc->objhead;
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 
-	/* If we inserted a new object it's a miss */
-	if (oc->flags & OC_F_BUSY) {
-		req->objcore = oc;
-		req->req_step = R_STP_MISS;
-		return (REQ_FSM_MORE);
-	}
-
 	/* We are not prepared to do streaming yet */
 	XXXAZ(req->busyobj);
 
@@ -505,14 +519,6 @@ cnt_lookup(struct worker *wrk, struct req *req)
 
 	VCL_lookup_method(req->vcl, wrk, req, NULL, req->http->ws);
 
-	if ((req->obj->objcore->flags & OC_F_PASS) &&
-	    wrk->handling == VCL_RET_DELIVER) {
-		VSLb(req->vsl, SLT_VCL_Error,
-		    "obj.uncacheable set, but vcl_lookup{} returned 'deliver'"
-		    ", changing to 'pass'");
-		wrk->handling = VCL_RET_PASS;
-	}
-
 	if (wrk->handling == VCL_RET_DELIVER) {
 		//AZ(req->busyobj->bereq->ws);
 		//AZ(req->busyobj->beresp->ws);



From phk at varnish-cache.org  Mon Aug 12 08:48:22 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Aug 2013 10:48:22 +0200
Subject: [master] 5b6eb4e Polish the state machine some more
Message-ID: <E1V8nna-00026x-NX@project.varnish-software.com>

commit 5b6eb4e7277a67d7fba0ba93351b9001442ff7b3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 12 08:48:09 2013 +0000

    Polish the state machine some more

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c9974fd..cd55a7a 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -393,13 +393,19 @@ cnt_fetch(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_lookup {
 DOT	lookup [
 DOT		shape=record
-DOT		label="{<top>cnt_lookup:|hash lookup|{<busy>busy?|exp|expbusy|hit|miss}|{vcl_lookup\{\}|{xx|xx}}|{<no>no|obj.f.pass?|<yes>yes}}"
+DOT		label="{<top>cnt_lookup:|hash lookup|{<busy>busy?|<e>exp?|<eb>expbusy?|<h>hit?|<miss>miss?|<hfp>hit-for-pass?}}"
+DOT	]
+DOT	lookup2 [
+DOT		shape=record
+DOT		label="{<top>cnt_lookup:|{vcl_lookup\{\}|{xx|xx}}}"
 DOT	]
 DOT }
 DOT lookup:busy:w -> lookup:top:w [label="(waitinglist)"]
-DOT lookup:miss -> miss [style=bold,color=blue]
-DOT lookup:no -> hit [style=bold,color=green]
-DOT lookup:yes -> pass [style=bold,color=red]
+DOT lookup:miss:s -> miss [style=bold,color=blue]
+DOT lookup:hfp:s -> pass [style=bold,color=red]
+DOT lookup:e:s -> lookup2 [style=bold,color=green]
+DOT lookup:eb:s -> lookup2 [style=bold,color=green]
+DOT lookup:h:s -> lookup2 [style=bold,color=green]
  */
 
 static enum req_fsm_nxt
@@ -446,8 +452,10 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		return (REQ_FSM_MORE);
 	}
 
-	AN(oc);
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	AZ (oc->flags & OC_F_BUSY);
+	AZ(req->objcore);
+	AZ(req->busyobj);
 
 	if (oc->flags & OC_F_PASS) {
 		/* Found a hit-for-pass */
@@ -494,8 +502,6 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		INCOMPL();
 	}
 
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	oh = oc->objhead;
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 
@@ -508,15 +514,9 @@ cnt_lookup(struct worker *wrk, struct req *req)
 
 	VRY_Finish(req, NULL);
 
-	if (oc->flags & OC_F_PASS)
-		wrk->stats.cache_hitpass++;
-	else
-		wrk->stats.cache_hit++;
+	wrk->stats.cache_hit++;
 	VSLb(req->vsl, SLT_Hit, "%u", req->obj->vxid);
 
-	AZ(req->objcore);
-	AZ(req->busyobj);
-
 	VCL_lookup_method(req->vcl, wrk, req, NULL, req->http->ws);
 
 	if (wrk->handling == VCL_RET_DELIVER) {
@@ -553,7 +553,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_miss {
 DOT	miss [
 DOT		shape=record
-DOT		label="{cnt_miss:|{vcl_miss\{\}|req.*}|{<err>error?|<rst>restart?}|{<pass>pass?|<fetch>fetch?}}"
+DOT		label="{cnt_miss:|{vcl_miss\{\}|req.*}|{<err>error?|<rst>restart?}|{<fetch>fetch?|<pass>pass?}}"
 DOT	]
 DOT }
 DOT miss:fetch -> fetch [label="fetch",style=bold,color=blue]
@@ -761,7 +761,7 @@ cnt_restart(const struct worker *wrk, struct req *req)
 DOT subgraph xcluster_recv {
 DOT	recv [
 DOT		shape=record
-DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}|{vcl_hash\{\}|req.*}|{<pass>pass?|<lookup>lookup?|<pipe>pipe?|<error>error?|<purge>purge?}}"
+DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}|{vcl_hash\{\}|req.*}|{<lookup>lookup?|<pass>pass?|<pipe>pipe?|<error>error?|<purge>purge?}}"
 DOT	]
 DOT }
 DOT recv:pipe -> pipe [style=bold,color=orange]



From martin at varnish-cache.org  Mon Aug 12 10:26:32 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Mon, 12 Aug 2013 12:26:32 +0200
Subject: [master] 7ec9d11 Fix seg fault in VRT_synth_page when the string list
 has a NULL pointer as the first element.
Message-ID: <E1V8pKa-00041B-Hq@project.varnish-software.com>

commit 7ec9d11ee2e9d8f8e3084e6a0e95b9b97c10a93a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jul 22 14:13:49 2013 +0200

    Fix seg fault in VRT_synth_page when the string list has a NULL
    pointer as the first element.
    
    Fixes: #1287

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index c63be80..3c86a6b 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -422,9 +422,8 @@ VRT_synth_page(const struct vrt_ctx *ctx, unsigned flags, const char *str, ...)
 	vsb = SMS_Makesynth(ctx->req->obj);
 	AN(vsb);
 
-	VSB_cat(vsb, str);
 	va_start(ap, str);
-	p = va_arg(ap, const char *);
+	p = str;
 	while (p != vrt_magic_string_end) {
 		if (p == NULL)
 			p = "(null)";
diff --git a/bin/varnishtest/tests/r01287.vtc b/bin/varnishtest/tests/r01287.vtc
new file mode 100644
index 0000000..8560ce3
--- /dev/null
+++ b/bin/varnishtest/tests/r01287.vtc
@@ -0,0 +1,21 @@
+varnishtest "#1287 - check NULL as first pointer to VRT_synth_page"
+
+server s1 {
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		return (error(200, "OK"));
+	}
+	sub vcl_error {
+		synthetic obj.http.blank;
+		return (deliver);
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "(null)"
+} -run



From martin at varnish-cache.org  Mon Aug 12 10:29:46 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Mon, 12 Aug 2013 12:29:46 +0200
Subject: [3.0] 88d785b Fix seg fault in VRT_synth_page when the string list
 has a NULL pointer as the first element.
Message-ID: <E1V8pNi-0004Hf-8y@project.varnish-software.com>

commit 88d785b3a963105dd76c49ba2e4303afabc5e1db
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jul 22 14:13:49 2013 +0200

    Fix seg fault in VRT_synth_page when the string list has a NULL
    pointer as the first element.
    
    Fixes: #1287

diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c
index ca57b95..1b68107 100644
--- a/bin/varnishd/cache_vrt.c
+++ b/bin/varnishd/cache_vrt.c
@@ -407,9 +407,8 @@ VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...)
 	vsb = SMS_Makesynth(sp->obj);
 	AN(vsb);
 
-	VSB_cat(vsb, str);
 	va_start(ap, str);
-	p = va_arg(ap, const char *);
+	p = str;
 	while (p != vrt_magic_string_end) {
 		if (p == NULL)
 			p = "(null)";
diff --git a/bin/varnishtest/tests/r01287.vtc b/bin/varnishtest/tests/r01287.vtc
new file mode 100644
index 0000000..bfa5ccb
--- /dev/null
+++ b/bin/varnishtest/tests/r01287.vtc
@@ -0,0 +1,21 @@
+varnishtest "#1287 - check NULL as first pointer to VRT_synth_page"
+
+server s1 {
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		error 200 "OK";
+	}
+	sub vcl_error {
+		synthetic obj.http.blank;
+		return (deliver);
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "(null)"
+} -run



From lkarsten at varnish-cache.org  Mon Aug 12 14:24:16 2013
From: lkarsten at varnish-cache.org (Lasse Karstensen)
Date: Mon, 12 Aug 2013 16:24:16 +0200
Subject: [master] 3725eb9 Import latest version from
 github.com/varnish/varnish-devicedetect/
Message-ID: <E1V8t2e-0001qj-A9@project.varnish-software.com>

commit 3725eb968fbe31b582590c19023b8bb113e802db
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Aug 12 16:24:02 2013 +0200

    Import latest version from github.com/varnish/varnish-devicedetect/

diff --git a/etc/devicedetect.vcl b/etc/devicedetect.vcl
index ed5b39d..e7aecbf 100644
--- a/etc/devicedetect.vcl
+++ b/etc/devicedetect.vcl
@@ -5,65 +5,68 @@
 # Author: Lasse Karstensen <lasse at varnish-software.com>
 
 sub devicedetect {
-    unset req.http.X-UA-Device;
-    set req.http.X-UA-Device = "pc";
+	unset req.http.X-UA-Device;
+	set req.http.X-UA-Device = "pc";
 
-    if (req.http.User-Agent ~ "(?i)(ads|google|bing|msn|yandex|baidu|ro|career|)bot" ||
-	req.http.User-Agent ~ "(?i)(baidu|symantec)spider" ||
-	req.http.User-Agent ~ "(?i)scanner" ||
-	req.http.User-Agent ~ "(?i)(web)crawler") {
-	set req.http.X-UA-Device = "bot"; }
-    elsif (req.http.User-Agent ~ "(?i)ip(hone|od)") { set req.http.X-UA-Device = "mobile-iphone"; }
-    elsif (req.http.User-Agent ~ "(?i)ipad")        { set req.http.X-UA-Device = "tablet-ipad"; }
-    # how do we differ between an android phone and an android tablet?
-    # http://stackoverflow.com/questions/5341637/how-do-detect-android-tablets-in-general-useragent
-    elsif (req.http.User-Agent ~ "(?i)android.*(mobile|mini)") { set req.http.X-UA-Device = "mobile-android"; }       
-    # android 3/honeycomb was just about tablet-only, and any phones will probably handle a bigger page layout.
-    elsif (req.http.User-Agent ~ "(?i)android 3")              { set req.http.X-UA-Device = "tablet-android"; }
-    # may very well give false positives towards android tablets. Suggestions welcome.
-    elsif (req.http.User-Agent ~ "(?i)android")         { set req.http.X-UA-Device = "tablet-android"; }
-
-    elsif (req.http.User-Agent ~ "^HTC" ||
-        req.http.User-Agent ~ "Fennec" || 
-        req.http.User-Agent ~ "IEMobile" ||
-        req.http.User-Agent ~ "BlackBerry" ||
-        req.http.User-Agent ~ "SymbianOS.*AppleWebKit" ||
-        req.http.User-Agent ~ "Opera Mobi") {
-        set req.http.X-UA-Device = "mobile-smartphone";
-    }
-    elsif (req.http.User-Agent ~ "(?i)symbian" ||
-        req.http.User-Agent ~ "(?i)^sonyericsson" ||
-        req.http.User-Agent ~ "(?i)^nokia" ||
-        req.http.User-Agent ~ "(?i)^samsung" ||
-        req.http.User-Agent ~ "(?i)^lg" ||
-	req.http.User-Agent ~ "(?i)bada" ||
-	req.http.User-Agent ~ "(?i)blazer" ||
-	req.http.User-Agent ~ "(?i)cellphone" ||
-	req.http.User-Agent ~ "(?i)iemobile" ||
-	req.http.User-Agent ~ "(?i)midp-2.0" ||
-	req.http.User-Agent ~ "(?i)u990" ||
-	req.http.User-Agent ~ "(?i)netfront" ||
-	req.http.User-Agent ~ "(?i)opera mini" ||
-	req.http.User-Agent ~ "(?i)palm" ||
-	req.http.User-Agent ~ "(?i)nintendo wii" ||
-	req.http.User-Agent ~ "(?i)playstation portable" ||
-	req.http.User-Agent ~ "(?i)portalmmm" ||
-	req.http.User-Agent ~ "(?i)proxinet" ||
-	req.http.User-Agent ~ "(?i)sonyericsson" ||
-	req.http.User-Agent ~ "(?i)symbian" ||
-	req.http.User-Agent ~ "(?i)windows\ ?ce" ||
-	req.http.User-Agent ~ "(?i)winwap" ||
-	req.http.User-Agent ~ "(?i)eudoraweb" ||
-	req.http.User-Agent ~ "(?i)htc" ||
-	req.http.User-Agent ~ "(?i)240x320" ||
-	req.http.User-Agent ~ "(?i)avantgo") { 
-        set req.http.X-UA-Device = "mobile-generic";
-    } 
-    # handle overrides
-    if (req.http.Cookie ~ "(i?)X-UA-Device-force") {
-        # ;?? means zero or one ;, non-greedy to match the first.
-        set req.http.X-UA-Device = regsub(req.http.Cookie, "(?i).*X-UA-Device-force=([^;]+);??.*", "\1");
-    }
+	# Handle that a cookie may override the detection alltogether.
+	if (req.http.Cookie ~ "(?i)X-UA-Device-force") {
+		/* ;?? means zero or one ;, non-greedy to match the first. */
+		set req.http.X-UA-Device = regsub(req.http.Cookie, "(?i).*X-UA-Device-force=([^;]+);??.*", "\1");
+		/* Clean up our mess in the cookie header */
+		set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *X-UA-Device-force=[^;]+;? *", "\1");
+		/* If the cookie header is now empty, or just whitespace, unset it. */
+		if (req.http.Cookie ~ "^ *$") { unset req.http.Cookie; }
+	} else {
+		if (req.http.User-Agent ~ "(?i)(ads|google|bing|msn|yandex|baidu|ro|career|)bot" ||
+		    req.http.User-Agent ~ "(?i)(baidu|jike|symantec)spider" ||
+		    req.http.User-Agent ~ "(?i)scanner" ||
+		    req.http.User-Agent ~ "(?i)(web)crawler") {
+			set req.http.X-UA-Device = "bot"; }
+		elsif (req.http.User-Agent ~ "(?i)ipad")        { set req.http.X-UA-Device = "tablet-ipad"; }
+		elsif (req.http.User-Agent ~ "(?i)ip(hone|od)") { set req.http.X-UA-Device = "mobile-iphone"; }
+		/* how do we differ between an android phone and an android tablet?
+		   http://stackoverflow.com/questions/5341637/how-do-detect-android-tablets-in-general-useragent */
+		elsif (req.http.User-Agent ~ "(?i)android.*(mobile|mini)") { set req.http.X-UA-Device = "mobile-android"; }
+		// android 3/honeycomb was just about tablet-only, and any phones will probably handle a bigger page layout.
+		elsif (req.http.User-Agent ~ "(?i)android 3")              { set req.http.X-UA-Device = "tablet-android"; }
+		// May very well give false positives towards android tablets. Suggestions welcome.
+		elsif (req.http.User-Agent ~ "(?i)android")         { set req.http.X-UA-Device = "tablet-android"; }
+		elsif (req.http.User-Agent ~ "Mobile.+Firefox")     { set req.http.X-UA-Device = "mobile-firefoxos"; }
+		elsif (req.http.User-Agent ~ "^HTC" ||
+		    req.http.User-Agent ~ "Fennec" ||
+		    req.http.User-Agent ~ "IEMobile" ||
+		    req.http.User-Agent ~ "BlackBerry" ||
+		    req.http.User-Agent ~ "SymbianOS.*AppleWebKit" ||
+		    req.http.User-Agent ~ "Opera Mobi") {
+			set req.http.X-UA-Device = "mobile-smartphone";
+		}
+		elsif (req.http.User-Agent ~ "(?i)symbian" ||
+		    req.http.User-Agent ~ "(?i)^sonyericsson" ||
+		    req.http.User-Agent ~ "(?i)^nokia" ||
+		    req.http.User-Agent ~ "(?i)^samsung" ||
+		    req.http.User-Agent ~ "(?i)^lg" ||
+		    req.http.User-Agent ~ "(?i)bada" ||
+		    req.http.User-Agent ~ "(?i)blazer" ||
+		    req.http.User-Agent ~ "(?i)cellphone" ||
+		    req.http.User-Agent ~ "(?i)iemobile" ||
+		    req.http.User-Agent ~ "(?i)midp-2.0" ||
+		    req.http.User-Agent ~ "(?i)u990" ||
+		    req.http.User-Agent ~ "(?i)netfront" ||
+		    req.http.User-Agent ~ "(?i)opera mini" ||
+		    req.http.User-Agent ~ "(?i)palm" ||
+		    req.http.User-Agent ~ "(?i)nintendo wii" ||
+		    req.http.User-Agent ~ "(?i)playstation portable" ||
+		    req.http.User-Agent ~ "(?i)portalmmm" ||
+		    req.http.User-Agent ~ "(?i)proxinet" ||
+		    req.http.User-Agent ~ "(?i)sonyericsson" ||
+		    req.http.User-Agent ~ "(?i)symbian" ||
+		    req.http.User-Agent ~ "(?i)windows\ ?ce" ||
+		    req.http.User-Agent ~ "(?i)winwap" ||
+		    req.http.User-Agent ~ "(?i)eudoraweb" ||
+		    req.http.User-Agent ~ "(?i)htc" ||
+		    req.http.User-Agent ~ "(?i)240x320" ||
+		    req.http.User-Agent ~ "(?i)avantgo") {
+			set req.http.X-UA-Device = "mobile-generic";
+		}
+	}
 }
-
-# vim: sw=4:tw=120 # meh



From martin at varnish-cache.org  Tue Aug 13 08:37:59 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 13 Aug 2013 10:37:59 +0200
Subject: [master] df22bd2 Add special parsing of ENUM in vmod object method
 argument spec
Message-ID: <E1V9A75-00065q-59@project.varnish-software.com>

commit df22bd225a437abdd4e9742f8593a81128852407
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Aug 12 15:37:46 2013 +0200

    Add special parsing of ENUM in vmod object method argument spec
    
    The missing handler would end the object spec prematurily, hiding the
    rest of the methods.
    
    Fixes: #1332

diff --git a/bin/varnishtest/tests/r01332.vtc b/bin/varnishtest/tests/r01332.vtc
new file mode 100644
index 0000000..062e334
--- /dev/null
+++ b/bin/varnishtest/tests/r01332.vtc
@@ -0,0 +1,26 @@
+varnishtest "#1332 - Check enum as argument in vmod object"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import debug from "${topbuild}/lib/libvmod_debug/.libs/libvmod_debug.so";
+
+	sub vcl_init {
+		new obj = debug.obj("don't care");
+	}
+
+	sub vcl_deliver {
+		obj.enum(martin);
+		set resp.http.foo = obj.foo("");
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.http.foo == "BOO"
+} -run
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index e6a3f6c..6b08b34 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -234,8 +234,19 @@ parse_new(struct vcc *tl)
 
 		sy3->args = p;
 		sy3->extra = TlDup(tl, buf1);
-		while (p[0] != '\0' || p[1] != '\0')
+		while (p[0] != '\0' || p[1] != '\0') {
+			if (!memcmp(p, "ENUM\0", 5)) {
+				/* XXX: Special case for ENUM that has
+				   it's own \0\0 end marker. Not exactly
+				   elegant, we should consider
+				   alternatives here. Maybe runlength
+				   encode the entire block? */
+				p += strlen(p) + 1;
+				while (p[0] != '\0' || p[1] != '\0')
+					p++;
+			}
 			p++;
+		}
 		p += 2;
 	}
 	/*lint -restore */
diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 57d8721..a27e998 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -32,6 +32,8 @@ Function STRING author(ENUM { phk, des, kristian, mithrandir })
 Function VOID test_priv_call(PRIV_CALL)
 Function VOID test_priv_vcl(PRIV_VCL)
 Object obj(STRING) {
+	# NOTE: .enum before .foo as part of test r01332.vtc
+	Method VOID .enum(ENUM { phk, des, kristian, mithrandir, martin })
 	Method STRING .foo(STRING why)
 	Method TIME .date()
 }
diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/lib/libvmod_debug/vmod_debug_obj.c
index c27b6db..ad24578 100644
--- a/lib/libvmod_debug/vmod_debug_obj.c
+++ b/lib/libvmod_debug/vmod_debug_obj.c
@@ -69,6 +69,15 @@ vmod_obj__fini(struct vmod_debug_obj **op)
 	*op = NULL;
 }
 
+VCL_VOID __match_proto__()
+vmod_obj_enum(const struct vrt_ctx *ctx, struct vmod_debug_obj *o, VCL_ENUM e)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(o, VMOD_DEBUG_OBJ_MAGIC);
+	assert(!strcmp(e, "martin"));
+}
+
 VCL_STRING __match_proto__()
 vmod_obj_foo(const struct vrt_ctx *ctx, struct vmod_debug_obj *o, VCL_STRING s)
 {



From phk at varnish-cache.org  Tue Aug 13 12:05:29 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 13 Aug 2013 14:05:29 +0200
Subject: [master] eac60c9 Stash the predictive Vary: string in malloc space,
 we need to get the req workspace un-reserved before we can call
 vcl_lookup{} and we can't tell if we need the predictive Vary: until
 vcl_lookup{} tells us.
Message-ID: <E1V9DLt-0005pm-Tt@project.varnish-software.com>

commit eac60c98bcbe1bfcfa9f2cb425144f685a95ea51
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 13 12:04:38 2013 +0000

    Stash the predictive Vary: string in malloc space, we need to get
    the req workspace un-reserved before we can call vcl_lookup{} and
    we can't tell if we need the predictive Vary: until vcl_lookup{}
    tells us.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index eaaffb7..02d5b21 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1043,7 +1043,8 @@ int VRY_Create(struct busyobj *bo, struct vsb **psb);
 int VRY_Match(struct req *, const uint8_t *vary);
 void VRY_Validate(const uint8_t *vary);
 void VRY_Prep(struct req *);
-void VRY_Finish(struct req *req, struct busyobj *bo);
+enum vry_finish_flag { KEEP, DISCARD };
+void VRY_Finish(struct req *req, enum vry_finish_flag);
 
 /* cache_vcl.c */
 void VCL_Init(void);
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index a23c791..962cc3c 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -197,6 +197,9 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 
 	VCL_Rel(&bo->vcl);
 
+	if (bo->vary != NULL)
+		free(bo->vary);
+
 	memset(&bo->refcount, 0,
 	    sizeof *bo - offsetof(struct busyobj, refcount));
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index cd55a7a..be3b147 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -440,6 +440,11 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		return (REQ_FSM_DISEMBARK);
 	}
 
+	if (boc == NULL)
+		VRY_Finish(req, DISCARD);
+	else
+		VRY_Finish(req, KEEP);
+
 	AZ(req->objcore);
 	if (lr == HSH_MISS) {
 		/* Found nothing */
@@ -463,7 +468,6 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		AZ(boc);
 		(void)HSH_Deref(&wrk->stats, oc, NULL);
 		req->objcore = NULL;
-		VRY_Finish(req, NULL);
 		wrk->stats.cache_hitpass++;
 		req->req_step = R_STP_PASS;
 		return (REQ_FSM_MORE);
@@ -492,6 +496,8 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		VSLb(req->vsl, SLT_Debug, "XXX EXPBUSY drop boc\n");
 		(void)HSH_Deref(&wrk->stats, boc, NULL);
 		boc = NULL;
+		free(req->vary_b);
+		req->vary_b = NULL;
 		break;
 	case HSH_HIT:
 		/* Found hit */
@@ -512,8 +518,6 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	req->obj = o;
 
-	VRY_Finish(req, NULL);
-
 	wrk->stats.cache_hit++;
 	VSLb(req->vsl, SLT_Hit, "%u", req->obj->vxid);
 
@@ -580,7 +584,8 @@ cnt_miss(struct worker *wrk, struct req *req)
 	bo = VBO_GetBusyObj(wrk, req);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	req->busyobj = bo;
-	VRY_Finish(req, bo);
+	bo->vary = req->vary_b;
+	req->vary_b = NULL;
 
 	VCL_miss_method(req->vcl, wrk, req, NULL, req->http->ws);
 	switch (wrk->handling) {
@@ -886,7 +891,7 @@ cnt_purge(struct worker *wrk, struct req *req)
 	assert (lr == HSH_MISS);
 	AZ(oc);
 	CHECK_OBJ_NOTNULL(boc, OBJCORE_MAGIC);
-	VRY_Finish(req, NULL);
+	VRY_Finish(req, DISCARD);
 
 	HSH_Purge(wrk, boc->objhead, 0, 0);
 
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 1ec8548..08736c6 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -56,6 +56,8 @@
 
 #include "config.h"
 
+#include <stdlib.h>
+
 #include "cache.h"
 
 #include "vct.h"
@@ -213,6 +215,9 @@ vry_cmp(const uint8_t *v1, const uint8_t *v2)
 
 /**********************************************************************
  * Prepare predictive vary string
+ *
+ * XXX: Strictly speaking vary_b and vary_e could be replaced with
+ * XXX: req->ws->{f,r}.   Space in struct req vs. code-readability...
  */
 
 void
@@ -238,24 +243,22 @@ VRY_Prep(struct req *req)
  */
 
 void
-VRY_Finish(struct req *req, struct busyobj *bo)
+VRY_Finish(struct req *req, enum vry_finish_flag flg)
 {
-
-	if (bo != NULL) {
-		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-		VRY_Validate(req->vary_b);
-		if (req->vary_l != NULL) {
-			bo->vary = WS_Copy(bo->ws,
-			    req->vary_b, req->vary_l - req->vary_b);
-			AN(bo->vary);
-			VRY_Validate(bo->vary);
-		} else
-			bo->vary = NULL;
+	uint8_t *p = NULL;
+
+	VRY_Validate(req->vary_b);
+	if (flg == KEEP && req->vary_l != NULL) {
+		p = malloc(req->vary_l - req->vary_b);
+		if (p != NULL) {
+			memcpy(p, req->vary_b, req->vary_l - req->vary_b);
+			VRY_Validate(p);
+		}
 	}
 	WS_Release(req->ws, 0);
-	req->vary_b = NULL;
 	req->vary_l = NULL;
 	req->vary_e = NULL;
+	req->vary_b = p;
 }
 
 /**********************************************************************



From phk at varnish-cache.org  Tue Aug 13 13:42:19 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 13 Aug 2013 15:42:19 +0200
Subject: [master] 06df39d Ever so slightly closer to The New World Order
Message-ID: <E1V9Erb-0004Ns-MM@project.varnish-software.com>

commit 06df39d48695b31e6ff2610f5fe687dd525f5671
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 13 13:42:02 2013 +0000

    Ever so slightly closer to The New World Order

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index be3b147..8010016 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -397,7 +397,7 @@ DOT		label="{<top>cnt_lookup:|hash lookup|{<busy>busy?|<e>exp?|<eb>expbusy?|<h>h
 DOT	]
 DOT	lookup2 [
 DOT		shape=record
-DOT		label="{<top>cnt_lookup:|{vcl_lookup\{\}|{xx|xx}}}"
+DOT		label="{<top>cnt_lookup:|{vcl_lookup\{\}|{req.*|obj.*|obj_stale.*}}}"
 DOT	]
 DOT }
 DOT lookup:busy:w -> lookup:top:w [label="(waitinglist)"]
@@ -440,10 +440,12 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		return (REQ_FSM_DISEMBARK);
 	}
 
-	if (boc == NULL)
+	if (boc == NULL) {
 		VRY_Finish(req, DISCARD);
-	else
+	} else {
+		AN(boc->flags & OC_F_BUSY);
 		VRY_Finish(req, KEEP);
+	}
 
 	AZ(req->objcore);
 	if (lr == HSH_MISS) {
@@ -473,41 +475,6 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		return (REQ_FSM_MORE);
 	}
 
-	if (boc != NULL)
-		AN(boc->flags & OC_F_BUSY);
-
-	switch (lr) {
-	case HSH_EXP:
-		/* Found expired object, and a busy already exists too */
-		VSLb(req->vsl, SLT_Debug, "XXXX EXP\n");
-		AZ(boc);
-		break;
-	case HSH_EXPBUSY:
-		/* Found expired object, inserted busy objcore */
-		VSLb(req->vsl, SLT_Debug, "XXXX EXPBUSY\n");
-		AN(boc);
-		if (VDI_Healthy(req->director, req->digest)) {
-			VSLb(req->vsl, SLT_Debug, "XXX EXPBUSY deref oc\n");
-			(void)HSH_Deref(&wrk->stats, oc, NULL);
-			req->objcore = boc;
-			req->req_step = R_STP_MISS;
-			return (REQ_FSM_MORE);
-		}
-		VSLb(req->vsl, SLT_Debug, "XXX EXPBUSY drop boc\n");
-		(void)HSH_Deref(&wrk->stats, boc, NULL);
-		boc = NULL;
-		free(req->vary_b);
-		req->vary_b = NULL;
-		break;
-	case HSH_HIT:
-		/* Found hit */
-		VSLb(req->vsl, SLT_Debug, "XXXX HIT\n");
-		AZ(boc);
-		break;
-	default:
-		INCOMPL();
-	}
-
 	oh = oc->objhead;
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 
@@ -518,36 +485,56 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	req->obj = o;
 
-	wrk->stats.cache_hit++;
 	VSLb(req->vsl, SLT_Hit, "%u", req->obj->vxid);
 
 	VCL_lookup_method(req->vcl, wrk, req, NULL, req->http->ws);
 
-	if (wrk->handling == VCL_RET_DELIVER) {
-		//AZ(req->busyobj->bereq->ws);
-		//AZ(req->busyobj->beresp->ws);
+	switch (wrk->handling) {
+	case VCL_RET_DELIVER:
+		if (boc != NULL && VDI_Healthy(req->director, req->digest)) {
+			// XXX: Start bg-fetch */
+			(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
+			req->objcore = boc;
+			req->req_step = R_STP_MISS;
+			return (REQ_FSM_MORE);
+		} else if (boc != NULL) {
+			(void)HSH_Deref(&wrk->stats, boc, NULL);
+			free(req->vary_b);
+			req->vary_b = NULL;
+		}
+		wrk->stats.cache_hit++;
 		(void)HTTP1_DiscardReqBody(req);	// XXX: handle err
 		req->req_step = R_STP_PREPRESP;
 		return (REQ_FSM_MORE);
-	}
-
-	/* Drop our object, we won't need it */
-	(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
-	req->objcore = NULL;
-
-	switch(wrk->handling) {
-	case VCL_RET_PASS:
-		req->req_step = R_STP_PASS;
+	case VCL_RET_FETCH:
+		(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
+		req->objcore = boc;
+		req->req_step = R_STP_MISS;
+		return (REQ_FSM_MORE);
+	case VCL_RET_RESTART:
+		req->req_step = R_STP_RESTART;
 		break;
 	case VCL_RET_ERROR:
 		req->req_step = R_STP_ERROR;
 		break;
-	case VCL_RET_RESTART:
-		req->req_step = R_STP_RESTART;
+	case VCL_RET_PASS:
+		wrk->stats.cache_hit++;
+		req->req_step = R_STP_PASS;
 		break;
 	default:
-		WRONG("Illegal action in vcl_lookup{}");
+		INCOMPL();
+	}
+
+	/* Drop our object, we won't need it */
+	(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
+	req->objcore = NULL;
+
+	if (boc != NULL) {
+		(void)HSH_Deref(&wrk->stats, boc, NULL);
+		free(req->vary_b);
+		req->vary_b = NULL;
 	}
+
 	return (REQ_FSM_MORE);
 }
 
diff --git a/bin/varnishd/default.vcl b/bin/varnishd/default.vcl
index 267b3c8..569cea2 100644
--- a/bin/varnishd/default.vcl
+++ b/bin/varnishd/default.vcl
@@ -98,21 +98,17 @@ sub vcl_purge {
 }
 
 sub vcl_lookup {
-/*
-    if (!obj) {
-	    return (deliver);
-    }
-    if (obj.uncacheable) {
-	return (pass);
-    }
     if (obj.ttl >= 0s) {
+	// A pure unadultered hit, deliver it
 	return (deliver);
     }
     if (obj.ttl + obj.grace > 0s) {
-	return (deliver_stale);
+	// Object is in grace, delive it
+	// Automatically triggers a background fetch
+	return (deliver);
     }
-*/
-    return (deliver);
+    // fetch & deliver once we get the result
+    return (fetch);
 }
 
 sub vcl_miss {
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 5250cc5..225108a 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -84,7 +84,7 @@ returns =(
 	('hash',		"C", ('lookup',)),
 	('purge',		"C", ('error', 'fetch',)),
 	('miss',		"C", ('error', 'restart', 'pass', 'fetch',)),
-	('lookup',		"C", ('error', 'restart', 'pass', 'deliver',)),
+	('lookup',		"C", ('error', 'restart', 'pass', 'fetch', 'deliver',)),
 	('backend_fetch',	"B", ('fetch', 'abandon')),
 	('backend_response',	"B", ('deliver', 'retry', 'abandon')),
 	('deliver',		"C", ('restart', 'deliver',)),



From phk at varnish-cache.org  Thu Aug 15 10:58:09 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 15 Aug 2013 12:58:09 +0200
Subject: [master] 4a918fa Move busyobj allocation into the VBF code.
Message-ID: <E1V9vFp-0007lC-UE@project.varnish-software.com>

commit 4a918fa1647bfdd2871f86b038567b0d843e3359
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 15 10:52:10 2013 +0000

    Move busyobj allocation into the VBF code.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 02d5b21..5066281 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -838,7 +838,7 @@ int EXP_NukeOne(struct busyobj *, struct lru *lru);
 void EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru);
 
 /* cache_fetch.c */
-void VBF_Fetch(struct worker *wrk, struct req *req);
+struct busyobj *VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, int pass);
 
 /* cache_fetch_proc.c */
 struct storage *VFP_GetStorage(struct busyobj *, ssize_t sz);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index e814522..19ccc30 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -506,23 +506,31 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 /*--------------------------------------------------------------------
  */
 
-void
-VBF_Fetch(struct worker *wrk, struct req *req)
+struct busyobj *
+VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, int pass)
 {
 	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
-	bo = req->busyobj;
+	bo = VBO_GetBusyObj(wrk, req);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	assert(bo->refcount == 2);
+	bo->refcount = 2;
+
+	oc->busyobj = bo;
+
+	assert(bo->refcount >= 1);
 	CHECK_OBJ_NOTNULL(bo->vcl, VCL_CONF_MAGIC);
 
+	bo->do_pass = pass;
+
+	bo->vary = req->vary_b;
+	req->vary_b = NULL;
+
 	AZ(bo->fetch_objcore);
-	bo->fetch_objcore = req->objcore;
-	req->objcore = NULL;
+	bo->fetch_objcore = oc;
 
 	AZ(bo->req);
 	bo->req = req;
@@ -536,4 +544,5 @@ VBF_Fetch(struct worker *wrk, struct req *req)
 		printf("XXX\n");
 		(void)usleep(100000);
 	}
+	return (bo);
 }
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 8010016..ee8e786 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -336,7 +336,7 @@ cnt_error(struct worker *wrk, struct req *req)
 }
 
 /*--------------------------------------------------------------------
- * Prepare to fetch body from backend
+ * Initiated a fetch (pass/miss) which we intend to deliver
  *
 DOT subgraph xcluster_body {
 DOT	fetch [
@@ -355,12 +355,8 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	AN(req->busyobj);
-	AN(req->objcore);
 	req->acct_req.fetch++;
-	VBF_Fetch(wrk, req);
 	AN(req->busyobj);
-	AZ(req->objcore);
 	assert(req->busyobj->refcount > 0);
 	(void)HTTP1_DiscardReqBody(req);
 	while (req->busyobj->state < BOS_FAILED) {
@@ -555,7 +551,6 @@ DOT
 static enum req_fsm_nxt
 cnt_miss(struct worker *wrk, struct req *req)
 {
-	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -564,32 +559,22 @@ cnt_miss(struct worker *wrk, struct req *req)
 	AZ(req->obj);
 	AZ(req->busyobj);
 
-	/*
-	 * We optimistically expect to need this most of the time
-	 * (This allows us to put the predictive Vary directly on the bo->ws)
-	 */
-	bo = VBO_GetBusyObj(wrk, req);
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	req->busyobj = bo;
-	bo->vary = req->vary_b;
-	req->vary_b = NULL;
-
 	VCL_miss_method(req->vcl, wrk, req, NULL, req->http->ws);
 	switch (wrk->handling) {
 	case VCL_RET_ERROR:
-		VBO_DerefBusyObj(wrk, &req->busyobj);
+		free(req->vary_b);
 		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
 		req->objcore = NULL;
 		req->req_step = R_STP_ERROR;
 		return (REQ_FSM_MORE);
 	case VCL_RET_RESTART:
-		VBO_DerefBusyObj(wrk, &req->busyobj);
+		free(req->vary_b);
 		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
 		req->objcore = NULL;
 		req->req_step = R_STP_RESTART;
 		return (REQ_FSM_MORE);
 	case VCL_RET_PASS:
-		VBO_DerefBusyObj(wrk, &req->busyobj);
+		free(req->vary_b);
 		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
 		req->objcore = NULL;
 		req->req_step = R_STP_PASS;
@@ -600,13 +585,11 @@ cnt_miss(struct worker *wrk, struct req *req)
 		WRONG("wrong return from vcl_miss{}");
 	}
 
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-
-	AN (req->objcore);
 	wrk->stats.cache_miss++;
 
-	req->objcore->busyobj = bo;
-	bo->refcount = 2;	/* One ref for req, one for vbf_fetch_body */
+	AN (req->objcore);
+	req->busyobj = VBF_Fetch(wrk, req, req->objcore, 0);
+	req->objcore = NULL;
 	req->req_step = R_STP_FETCH;
 	return (REQ_FSM_MORE);
 }
@@ -631,7 +614,6 @@ XDOT err_pass [label="ERROR",shape=plaintext]
 static enum req_fsm_nxt
 cnt_pass(struct worker *wrk, struct req *req)
 {
-	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -651,14 +633,10 @@ cnt_pass(struct worker *wrk, struct req *req)
 	assert (wrk->handling == VCL_RET_FETCH);
 	req->acct_req.pass++;
 
-	bo = VBO_GetBusyObj(wrk, req);
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	req->busyobj = bo;
-	bo->do_pass = 1;
 	req->objcore = HSH_NewObjCore(wrk);
-
-	req->objcore->busyobj = bo;
-	bo->refcount = 2;	/* One ref for req, one for vbf_fetch_body */
+	AN(req->objcore);
+	req->busyobj = VBF_Fetch(wrk, req, req->objcore, 1);
+	req->objcore = NULL;
 	req->req_step = R_STP_FETCH;
 	return (REQ_FSM_MORE);
 }
diff --git a/bin/varnishtest/tests/c00012.vtc b/bin/varnishtest/tests/c00012.vtc
index 79185e1..570fc93 100644
--- a/bin/varnishtest/tests/c00012.vtc
+++ b/bin/varnishtest/tests/c00012.vtc
@@ -25,7 +25,7 @@ client c1 {
 	rxresp
 	expect resp.status == 200
 	expect resp.bodylen == 7
-	expect resp.http.x-varnish == "1004"
+	expect resp.http.x-varnish == "1003"
 } 
 
 client c1 -run



From phk at varnish-cache.org  Thu Aug 15 13:08:17 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 15 Aug 2013 15:08:17 +0200
Subject: [master] 5841933 Welcome to the new grace-mode,
 now the first client does not get penalized by the backend fetch.
Message-ID: <E1V9xHl-0003g8-8P@project.varnish-software.com>

commit 58419339abd1ed8bed6e2c49d0feb55940deb579
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 15 13:07:18 2013 +0000

    Welcome to the new grace-mode, now the first client does not
    get penalized by the backend fetch.
    
    Rough edges are to be expected, including that I had to disable
    three Saint-mode test-cases, until I figure out how saint-mode should
    work in this scenario.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index ee8e786..f300f88 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -487,19 +487,13 @@ cnt_lookup(struct worker *wrk, struct req *req)
 
 	switch (wrk->handling) {
 	case VCL_RET_DELIVER:
-		if (boc != NULL && VDI_Healthy(req->director, req->digest)) {
-			// XXX: Start bg-fetch */
-			(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
-			req->objcore = boc;
-			req->req_step = R_STP_MISS;
-			return (REQ_FSM_MORE);
-		} else if (boc != NULL) {
-			(void)HSH_Deref(&wrk->stats, boc, NULL);
-			free(req->vary_b);
-			req->vary_b = NULL;
+		if (boc != NULL) {
+			req->busyobj = VBF_Fetch(wrk, req, boc, 0);
+			VBO_DerefBusyObj(wrk, &req->busyobj);
+		} else {
+			(void)HTTP1_DiscardReqBody(req);// XXX: handle err
 		}
 		wrk->stats.cache_hit++;
-		(void)HTTP1_DiscardReqBody(req);	// XXX: handle err
 		req->req_step = R_STP_PREPRESP;
 		return (REQ_FSM_MORE);
 	case VCL_RET_FETCH:
diff --git a/bin/varnishtest/tests/c00029.vtc b/bin/varnishtest/tests/c00029.vtc
index 985d595..de1a634 100644
--- a/bin/varnishtest/tests/c00029.vtc
+++ b/bin/varnishtest/tests/c00029.vtc
@@ -1,5 +1,7 @@
 varnishtest "Test that saintmode_threshold correctly marks a backend as sick"
 
+feature OldSaintMode
+
 server s1 {
 	rxreq
 	txresp
diff --git a/bin/varnishtest/tests/c00030.vtc b/bin/varnishtest/tests/c00030.vtc
index 51c5042..fccc4e5 100644
--- a/bin/varnishtest/tests/c00030.vtc
+++ b/bin/varnishtest/tests/c00030.vtc
@@ -1,5 +1,7 @@
 varnishtest "Test that saintmode_threshold in VCL"
 
+feature OldSaintMode
+
 server s1 {
 	rxreq
 	txresp
diff --git a/bin/varnishtest/tests/c00042.vtc b/bin/varnishtest/tests/c00042.vtc
index ae39d24..7de613d 100644
--- a/bin/varnishtest/tests/c00042.vtc
+++ b/bin/varnishtest/tests/c00042.vtc
@@ -1,4 +1,4 @@
-varnishtest "using req.ttl to force fetch"
+varnishtest "using req.hash_always_miss to force fetch"
 
 server s1 {
 	rxreq
@@ -13,8 +13,7 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.http.short) {
-			set req.ttl = 1s;
-			set req.grace = 0s;
+			set req.hash_always_miss = true;
 		}
 	}
 } -start
@@ -30,8 +29,6 @@ client c1 {
 	expect resp.status == 200
 	expect resp.bodylen == 1
 
-	delay 2
-
 	txreq -url "/1" -hdr "short: yes"
 	rxresp
 	expect resp.status == 200
diff --git a/bin/varnishtest/tests/s00003.vtc b/bin/varnishtest/tests/s00003.vtc
index 3ed6e98..02ceb63 100644
--- a/bin/varnishtest/tests/s00003.vtc
+++ b/bin/varnishtest/tests/s00003.vtc
@@ -1,5 +1,7 @@
 varnishtest "Check saint mode with sick pages"
 
+feature OldSaintMode
+
 server s1 {
 	timeout 10
 



From phk at varnish-cache.org  Sat Aug 17 20:00:01 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 17 Aug 2013 22:00:01 +0200
Subject: [master] 0e46b16 Forgot to commit the test-case for v4 grace
Message-ID: <E1VAmfJ-0001F9-IR@project.varnish-software.com>

commit 0e46b160ad7754e1f8c3ef636bce9b53b0aec831
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Aug 17 19:59:41 2013 +0000

    Forgot to commit the test-case for v4 grace

diff --git a/bin/varnishtest/tests/c00058.vtc b/bin/varnishtest/tests/c00058.vtc
new file mode 100644
index 0000000..f39e3e3
--- /dev/null
+++ b/bin/varnishtest/tests/c00058.vtc
@@ -0,0 +1,38 @@
+varnishtest "Test v4 grace"
+
+server s1 {
+	rxreq
+	txresp -bodylen 3
+	rxreq
+	txresp -bodylen 6
+	sema r2 sync 2
+} -start
+
+varnish v1 -vcl+backend { 
+	sub vcl_backend_response {
+		set beresp.ttl = 0.1s;
+		set beresp.grace = 10s;
+	}
+} -start
+
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.bodylen == 3
+
+	delay 1
+
+	# We still get old object
+	txreq
+	rxresp
+	expect resp.bodylen == 3
+
+	# But bg fetch was kicked off
+	sema r2 sync 2
+
+	# And now we get the new object
+	txreq
+	rxresp
+	expect resp.bodylen == 6
+} -run



From phk at varnish-cache.org  Mon Aug 19 10:08:47 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Aug 2013 12:08:47 +0200
Subject: [master] 2bbdf01 Make it more obvious which condtion fails.
Message-ID: <E1VBMOF-0005bJ-2I@project.varnish-software.com>

commit 2bbdf01bf593d78c0f1bf0036a96eccb6e0d5a9d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 19 09:29:41 2013 +0000

    Make it more obvious which condtion fails.

diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 37fa115..4af926b 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -128,13 +128,14 @@ Lck__Assert(const struct lock *lck, int held)
 {
 	struct ilck *ilck;
 
-	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
-	if (held)
-		assert(ilck->held &&
-		    pthread_equal(ilck->owner, pthread_self()));
-	else
-		assert(!ilck->held ||
-		    !pthread_equal(ilck->owner, pthread_self()));
+	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC); 
+	if (held) {
+		assert(ilck->held);
+		assert(pthread_equal(ilck->owner, pthread_self()));
+	} else {
+		assert(!ilck->held);
+		assert(!pthread_equal(ilck->owner, pthread_self()));
+	}
 }
 
 int __match_proto__()



From phk at varnish-cache.org  Mon Aug 19 10:08:47 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Aug 2013 12:08:47 +0200
Subject: [master] 7d22d7d Replace the cheesy sleep-loop with a proper cond-var.
Message-ID: <E1VBMOF-0005bO-3o@project.varnish-software.com>

commit 7d22d7d0b81551ee991e76e87bfcfedde5dd179c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 19 10:08:26 2013 +0000

    Replace the cheesy sleep-loop with a proper cond-var.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 5066281..b2399cc 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -500,6 +500,7 @@ struct busyobj {
 	unsigned		magic;
 #define BUSYOBJ_MAGIC		0x23b95567
 	struct lock		mtx;
+	pthread_cond_t		cond;
 	char			*end;
 	enum fetch_step		step;
 
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 962cc3c..ba49473 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -69,6 +69,7 @@ vbo_New(void)
 	bo->magic = BUSYOBJ_MAGIC;
 	bo->end = (char *)bo + sz;
 	Lck_New(&bo->mtx, lck_busyobj);
+	AZ(pthread_cond_init(&bo->cond, NULL));
 	return (bo);
 }
 
@@ -82,6 +83,7 @@ VBO_Free(struct busyobj **bop)
 	*bop = NULL;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AZ(bo->refcount);
+	AZ(pthread_cond_destroy(&bo->cond));
 	Lck_Delete(&bo->mtx);
 	MPL_Free(vbopool, bo);
 }
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 19ccc30..027fb5c 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -47,8 +47,12 @@
 static void
 vbf_release_req(struct busyobj *bo)
 {
-	if (bo->req != NULL)
-		bo->req = NULL;
+	if (bo->req == NULL)
+		return;
+	Lck_Lock(&bo->mtx);
+	bo->req = NULL;
+	AZ(pthread_cond_signal(&bo->cond));
+	Lck_Unlock(&bo->mtx);
 }
 
 /*--------------------------------------------------------------------
@@ -391,7 +395,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	 * Ready to fetch the body
 	 */
 
-	assert(bo->refcount >= 2);	/* one for each thread */
+	assert(bo->refcount >= 1);
 
 	if (obj->objcore->objhead != NULL) {
 		EXP_Insert(obj);
@@ -477,27 +481,31 @@ static void
 vbf_fetch_thread(struct worker *wrk, void *priv)
 {
 	struct busyobj *bo;
+	enum fetch_step stp;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
 
 	THR_SetBusyobj(bo);
-	bo->step = F_STP_MKBEREQ;
+	stp = F_STP_MKBEREQ;
 
-	while (bo->step != F_STP_DONE) {
-		switch(bo->step) {
+	while (stp != F_STP_DONE) {
+		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+		bo->step = stp;
+		switch(stp) {
 #define FETCH_STEP(l, U, arg)						\
 		case F_STP_##U:						\
-			bo->step = vbf_stp_##l arg;			\
-			VSLb(bo->vsl, SLT_Debug,			\
-			    "%s -> %s", #l, vbf_step_name(bo->step));	\
+			stp = vbf_stp_##l arg;				\
 			break;
 #include "tbl/steps.h"
 #undef FETCH_STEP
 		default:
 			WRONG("Illegal fetch_step");
 		}
+		if (stp != F_STP_DONE)				
+			VSLb(bo->vsl, SLT_Debug, "%s -> %s",
+			    vbf_step_name(bo->step), vbf_step_name(stp));
 	}
 	assert(WRW_IsReleased(wrk));
 	THR_SetBusyobj(NULL);
@@ -521,7 +529,6 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, int pass)
 
 	oc->busyobj = bo;
 
-	assert(bo->refcount >= 1);
 	CHECK_OBJ_NOTNULL(bo->vcl, VCL_CONF_MAGIC);
 
 	bo->do_pass = pass;
@@ -540,9 +547,12 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, int pass)
 
 	if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_QUEUE_FRONT))
 		vbf_fetch_thread(wrk, bo);
-	while (bo->req != NULL) {
-		printf("XXX\n");
-		(void)usleep(100000);
+	Lck_Lock(&bo->mtx);
+	while (1) {
+		if (bo->req == NULL) 
+			break;
+		(void)Lck_CondWait(&bo->cond, &bo->mtx, NULL);
 	}
+	Lck_Unlock(&bo->mtx);
 	return (bo);
 }



From phk at varnish-cache.org  Mon Aug 19 13:22:52 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Aug 2013 15:22:52 +0200
Subject: [master] 0e50763 Add better coverage test of vcc_backend.c
Message-ID: <E1VBPQ4-0005xW-L1@project.varnish-software.com>

commit 0e507635925252ff112e7ff0aa034fd595011fb0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 19 13:22:23 2013 +0000

    Add better coverage test of vcc_backend.c

diff --git a/bin/varnishtest/tests/v00038.vtc b/bin/varnishtest/tests/v00038.vtc
new file mode 100644
index 0000000..c155119
--- /dev/null
+++ b/bin/varnishtest/tests/v00038.vtc
@@ -0,0 +1,67 @@
+varnishtest "VCL compiler coverage test: vcc_backend.c
+
+varnish v1 -errvcl "IPv6 address [] wrong" {
+	backend b1 {
+		.host = "[0:0:0:0";
+	}
+}
+
+varnish v1 -errvcl "with exactly three digits" {
+	backend b1 {
+		.host = "127.0.0.1";
+		.probe = {
+			.expected_response = 1000;
+		}
+	}
+}
+
+varnish v1 -errvcl "Must specify .threshold with .window" {
+	backend b1 {
+		.host = "127.0.0.1";
+		.probe = {
+			.window = 32;
+		}
+	}
+}
+
+varnish v1 -errvcl "Threshold must be 64 or less" {
+	backend b1 {
+		.host = "127.0.0.1";
+		.probe = {
+			.threshold = 65;
+		}
+	}
+}
+
+varnish v1 -errvcl "Window must be 64 or less" {
+	backend b1 {
+		.host = "127.0.0.1";
+		.probe = {
+			.window = 65;
+			.threshold = 64;
+		}
+	}
+}
+
+varnish v1 -errvcl "Threshold can not be greater than window" {
+	backend b1 {
+		.host = "127.0.0.1";
+		.probe = {
+			.window = 63;
+			.threshold = 64;
+		}
+	}
+}
+
+varnish v1 -errvcl "NB: Backend Syntax has changed:" {
+	backend b1 {
+		set .host = "127.0.0.1";
+	}
+}
+
+varnish v1 -errvcl "Expected '{' or name of probe." {
+	backend b1 {
+		.host = "127.0.0.1";
+		.probe = "NONE";
+	}
+}



From phk at varnish-cache.org  Mon Aug 19 13:53:26 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Aug 2013 15:53:26 +0200
Subject: [master] 4580b7d Hmm, as always git cheats me when I add files.
Message-ID: <E1VBPte-000800-Vk@project.varnish-software.com>

commit 4580b7deeab503af3e5a064a2e5cc218971dea87
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 19 13:53:06 2013 +0000

    Hmm, as always git cheats me when I add files.

diff --git a/bin/varnishtest/tests/v00038.vtc b/bin/varnishtest/tests/v00038.vtc
index c155119..7de4517 100644
--- a/bin/varnishtest/tests/v00038.vtc
+++ b/bin/varnishtest/tests/v00038.vtc
@@ -1,4 +1,4 @@
-varnishtest "VCL compiler coverage test: vcc_backend.c
+varnishtest "VCL compiler coverage test: vcc_backend.c"
 
 varnish v1 -errvcl "IPv6 address [] wrong" {
 	backend b1 {
@@ -59,7 +59,7 @@ varnish v1 -errvcl "NB: Backend Syntax has changed:" {
 	}
 }
 
-varnish v1 -errvcl "Expected '{' or name of probe." {
+varnish v1 -errvcl "Expected '{' or name of probe, got" {
 	backend b1 {
 		.host = "127.0.0.1";
 		.probe = "NONE";
diff --git a/include/vss.h b/include/vss.h
index d98aa1d..af41057 100644
--- a/include/vss.h
+++ b/include/vss.h
@@ -29,7 +29,7 @@
 /* vss.c */
 struct vss_addr;
 
-int VSS_parse(const char *str, char **addr, char **port);
+const char *VSS_parse(const char *str, char **addr, char **port);
 int VSS_resolve(const char *addr, const char *port, struct vss_addr ***ta);
 int VSS_bind(const struct vss_addr *addr);
 int VSS_listen(const struct vss_addr *addr, int depth);
diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c
index bc0597a..2c1d8d6 100644
--- a/lib/libvarnish/vss.c
+++ b/lib/libvarnish/vss.c
@@ -71,7 +71,7 @@ struct vss_addr {
  * See also RFC5952
  */
 
-int
+const char *
 VSS_parse(const char *str, char **addr, char **port)
 {
 	const char *p;
@@ -83,7 +83,7 @@ VSS_parse(const char *str, char **addr, char **port)
 		if ((p = strchr(str, ']')) == NULL ||
 		    p == str + 1 ||
 		    (p[1] != '\0' && p[1] != ':'))
-			return (-1);
+			return ("IPv6 address [] wrong.");
 		*addr = strdup(str + 1);
 		XXXAN(*addr);
 		(*addr)[p - (str + 1)] = '\0';
@@ -109,7 +109,7 @@ VSS_parse(const char *str, char **addr, char **port)
 			XXXAN(*port);
 		}
 	}
-	return (0);
+	return (NULL);
 }
 
 /*
@@ -143,8 +143,7 @@ VSS_resolve(const char *addr, const char *port, struct vss_addr ***vap)
 	hints.ai_socktype = SOCK_STREAM;
 	hints.ai_flags = AI_PASSIVE;
 
-	ret = VSS_parse(addr, &hop, &adp);
-	if (ret)
+	if (VSS_parse(addr, &hop, &adp) != NULL)
 		return (0);
 
 	if (adp == NULL)
diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c
index a8773d0..2cb3469 100644
--- a/lib/libvcl/vcc_backend.c
+++ b/lib/libvcl/vcc_backend.c
@@ -75,14 +75,15 @@ static void
 Emit_Sockaddr(struct vcc *tl, const struct token *t_host, const char *port)
 {
 	const char *ipv4, *ipv4a, *ipv6, *ipv6a, *pa;
+	const char *err;
 	char *hop, *pop;
 
 	AN(t_host->dec);
 
-	if (VSS_parse(t_host->dec, &hop, &pop)) {
+	err = VSS_parse(t_host->dec, &hop, &pop);
+	if (err != NULL) {
 		VSB_printf(tl->sb,
-		    "Backend host '%.*s': wrong syntax (unbalanced [...] ?)\n",
-		    PF(t_host) );
+		    "Backend host '%.*s': %s\n", PF(t_host), err);
 		vcc_ErrWhere(tl, t_host);
 		return;
 	}
@@ -416,7 +417,7 @@ vcc_ParseHostDef(struct vcc *tl, int serial, const char *vgcname)
 			SkipToken(tl, ';');
 		} else if (vcc_IdIs(t_field, "probe")) {
 			VSB_printf(tl->sb,
-			    "Expected '{' or name of probe.");
+			    "Expected '{' or name of probe, got ");
 			vcc_ErrToken(tl, tl->t);
 			VSB_printf(tl->sb, " at\n");
 			vcc_ErrWhere(tl, tl->t);



From phk at varnish-cache.org  Tue Aug 20 07:23:42 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Aug 2013 09:23:42 +0200
Subject: [master] fb980db Polish
Message-ID: <E1VBgI2-0003Qa-Px@project.varnish-software.com>

commit fb980db0eb8f1c8992c2c1b548298394ed30658a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 20 07:23:35 2013 +0000

    Polish

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 027fb5c..e853c52 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -503,7 +503,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 		default:
 			WRONG("Illegal fetch_step");
 		}
-		if (stp != F_STP_DONE)				
+		if (stp != F_STP_DONE)
 			VSLb(bo->vsl, SLT_Debug, "%s -> %s",
 			    vbf_step_name(bo->step), vbf_step_name(stp));
 	}
@@ -549,7 +549,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, int pass)
 		vbf_fetch_thread(wrk, bo);
 	Lck_Lock(&bo->mtx);
 	while (1) {
-		if (bo->req == NULL) 
+		if (bo->req == NULL)
 			break;
 		(void)Lck_CondWait(&bo->cond, &bo->mtx, NULL);
 	}
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index 9f150ea..d45abf9 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -510,7 +510,7 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv)
 		    "Multiple attempts to access non-cached req.body");
 		return (i);
 	}
-	
+
 	do {
 		l = http1_iter_req_body(req, buf, sizeof buf);
 		if (l < 0) {
diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 4af926b..9b8444c 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -128,7 +128,7 @@ Lck__Assert(const struct lock *lck, int held)
 {
 	struct ilck *ilck;
 
-	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC); 
+	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
 	if (held) {
 		assert(ilck->held);
 		assert(pthread_equal(ilck->owner, pthread_self()));
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 81be78e..2d23723 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -304,7 +304,7 @@ pan_req(const struct req *req)
 	else
 		VSB_printf(pan_vsp, "  step = 0x%x,\n", req->req_step);
 
-	VSB_printf(pan_vsp, "  req_body = %s,\n", 
+	VSB_printf(pan_vsp, "  req_body = %s,\n",
 	    reqbody_status_2str(req->req_body_status));
 
 	if (req->err_code)
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index f300f88..9e47927 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -344,8 +344,8 @@ DOT		shape=record
 DOT		label="{cnt_fetch:|start fetch_thread}"
 DOT	]
 DOT }
-DOT fetch:out -> prepresp [style=bold,color=red]
-DOT fetch:out -> prepresp [style=bold,color=blue]
+DOT fetch -> prepresp [style=bold,color=red]
+DOT fetch -> prepresp [style=bold,color=blue]
  */
 
 static enum req_fsm_nxt
@@ -378,7 +378,6 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	return (REQ_FSM_MORE);
 }
 
-
 /*--------------------------------------------------------------------
  * LOOKUP
  * Hash things together and look object up in hash-table.
@@ -389,11 +388,11 @@ cnt_fetch(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_lookup {
 DOT	lookup [
 DOT		shape=record
-DOT		label="{<top>cnt_lookup:|hash lookup|{<busy>busy?|<e>exp?|<eb>expbusy?|<h>hit?|<miss>miss?|<hfp>hit-for-pass?}}"
+DOT		label="{<top>cnt_lookup:|hash lookup|{<busy>busy?|<e>exp?|<eb>exp+busy?|<h>hit?|<miss>miss?|<hfp>hit-for-pass?}}"
 DOT	]
 DOT	lookup2 [
 DOT		shape=record
-DOT		label="{<top>cnt_lookup:|{vcl_lookup\{\}|{req.*|obj.*|obj_stale.*}}}"
+DOT		label="{<top>cnt_lookup:|{vcl_lookup\{\}|req.*, obj.*}|{<deliver>deliver?|error?|restart?|<fetch>fetch?|<pass>pass?}}"
 DOT	]
 DOT }
 DOT lookup:busy:w -> lookup:top:w [label="(waitinglist)"]
@@ -402,6 +401,9 @@ DOT lookup:hfp:s -> pass [style=bold,color=red]
 DOT lookup:e:s -> lookup2 [style=bold,color=green]
 DOT lookup:eb:s -> lookup2 [style=bold,color=green]
 DOT lookup:h:s -> lookup2 [style=bold,color=green]
+DOT lookup2:pass:s -> pass [style=bold,color=red]
+DOT lookup2:fetch:s -> miss [style=bold,color=blue]
+DOT lookup2:deliver:s -> prepresp:nw [style=bold,color=green]
  */
 
 static enum req_fsm_nxt
@@ -534,11 +536,11 @@ cnt_lookup(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_miss {
 DOT	miss [
 DOT		shape=record
-DOT		label="{cnt_miss:|{vcl_miss\{\}|req.*}|{<err>error?|<rst>restart?}|{<fetch>fetch?|<pass>pass?}}"
+DOT		label="{cnt_miss:|{vcl_miss\{\}|req.*, bereq.*}|{<fetch>fetch?|<err>error?|<rst>restart?|<pass>pass?}}"
 DOT	]
 DOT }
-DOT miss:fetch -> fetch [label="fetch",style=bold,color=blue]
-DOT miss:pass -> pass [label="pass",style=bold,color=red]
+DOT miss:fetch:s -> fetch [label="fetch",style=bold,color=blue]
+DOT miss:pass:s -> pass [label="pass",style=bold,color=red]
 DOT
  */
 
@@ -595,10 +597,10 @@ cnt_miss(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_pass {
 DOT	pass [
 DOT		shape=record
-DOT		label="{cnt_pass:|{vcl_pass\{\}|req.*}|{<err>error?|<rst>restart?}|<pass>create anon obj}"
+DOT		label="{cnt_pass:|{vcl_pass\{\}|req.*, bereq.*}|{<fetch>fetch?|<err>error?|<rst>restart?}}"
 DOT	]
 DOT }
-DOT pass:pass -> fetch [style=bold, color=red]
+DOT pass:fetch:s -> fetch:n [style=bold, color=red]
 XDOT pass:rst -> rst_pass [label="restart",color=purple]
 XDOT rst_pass [label="RESTART",shape=plaintext]
 XDOT pass:err -> err_pass [label="error"]
@@ -608,6 +610,7 @@ XDOT err_pass [label="ERROR",shape=plaintext]
 static enum req_fsm_nxt
 cnt_pass(struct worker *wrk, struct req *req)
 {
+	struct objcore *oc;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -627,10 +630,9 @@ cnt_pass(struct worker *wrk, struct req *req)
 	assert (wrk->handling == VCL_RET_FETCH);
 	req->acct_req.pass++;
 
-	req->objcore = HSH_NewObjCore(wrk);
-	AN(req->objcore);
-	req->busyobj = VBF_Fetch(wrk, req, req->objcore, 1);
-	req->objcore = NULL;
+	oc = HSH_NewObjCore(wrk);
+	AN(oc);
+	req->busyobj = VBF_Fetch(wrk, req, oc, 1);
 	req->req_step = R_STP_FETCH;
 	return (REQ_FSM_MORE);
 }
@@ -642,7 +644,7 @@ cnt_pass(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_pipe {
 DOT	pipe [
 DOT		shape=record
-DOT		label="{cnt_pipe:|filter req.*-\>bereq.*|{vcl_pipe()|req.*\nbereq\.*}|{<pipe>pipe?|<error>error?}}"
+DOT		label="{cnt_pipe:|filter req.*-\>bereq.*|{vcl_pipe()|req.*, bereq\.*}|{<pipe>pipe?|<error>error?}}"
 DOT	]
 DOT	pipe_do [
 DOT		shape=ellipse
@@ -730,8 +732,8 @@ DOT	]
 DOT }
 DOT recv:pipe -> pipe [style=bold,color=orange]
 DOT recv:pass -> pass [style=bold,color=red]
-DOT recv:lookup -> lookup [style=bold,color=green]
-DOT recv:purge -> purge [style=bold,color=purple]
+DOT recv:lookup:s -> lookup [style=bold,color=green]
+DOT recv:purge:s -> purge [style=bold,color=purple]
 #DOT recv:error -> err_recv
 #DOT err_recv [label="ERROR",shape=plaintext]
  */



From phk at varnish-cache.org  Tue Aug 20 08:02:09 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Aug 2013 10:02:09 +0200
Subject: [master] ec31b72 polish
Message-ID: <E1VBgtF-0005Vy-2K@project.varnish-software.com>

commit ec31b72072855401b115045b62daac585a1b47c1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 20 08:01:57 2013 +0000

    polish

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 9e47927..bb52290 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -536,7 +536,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_miss {
 DOT	miss [
 DOT		shape=record
-DOT		label="{cnt_miss:|{vcl_miss\{\}|req.*, bereq.*}|{<fetch>fetch?|<err>error?|<rst>restart?|<pass>pass?}}"
+DOT		label="{cnt_miss:|{vcl_miss\{\}|req.*}|{<fetch>fetch?|<err>error?|<rst>restart?|<pass>pass?}}"
 DOT	]
 DOT }
 DOT miss:fetch:s -> fetch [label="fetch",style=bold,color=blue]
@@ -597,7 +597,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_pass {
 DOT	pass [
 DOT		shape=record
-DOT		label="{cnt_pass:|{vcl_pass\{\}|req.*, bereq.*}|{<fetch>fetch?|<err>error?|<rst>restart?}}"
+DOT		label="{cnt_pass:|{vcl_pass\{\}|req.*}|{<fetch>fetch?|<err>error?|<rst>restart?}}"
 DOT	]
 DOT }
 DOT pass:fetch:s -> fetch:n [style=bold, color=red]
diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc
index 2cf3f94..aac5776 100644
--- a/bin/varnishtest/tests/v00020.vtc
+++ b/bin/varnishtest/tests/v00020.vtc
@@ -242,7 +242,7 @@ varnish v1 -errvcl {'&&' must be followed by BOOL, found DURATION.} {
 
 varnish v1 -vcl {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_backend_response {
+	sub vcl_miss {
 		if (!req.url) {
 		}
 	}
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 225108a..8d57ba6 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -101,7 +101,7 @@ returns =(
 # 'backend' means all methods tagged "B"
 # 'both' means all methods tagged "B" or "C"
 
-sp_variables = (
+sp_variables = [
 	('client.ip',
 		'IP',
 		( 'both',),
@@ -224,58 +224,58 @@ sp_variables = (
 	),
 	('bereq.backend',
 		'BACKEND',
-		( 'backend', ),
-		( 'backend', ),
+		( 'pipe', 'backend', ),
+		( 'pipe', 'backend', ),
 	),
 	('bereq.backend.healthy',
 		'BOOL',
-		( 'backend', ),
+		( 'pipe', 'backend', ),
 		( ),
 	),
 	('bereq.method',
 		'STRING',
-		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
-		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
+		( 'pipe', 'backend', ),
+		( 'pipe', 'backend', ),
 	),
 	('bereq.request',
 		'STRING',
-		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
-		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
+		( 'pipe', 'backend', ),
+		( 'pipe', 'backend', ),
 	),
 	('bereq.url',
 		'STRING',
-		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
-		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
+		( 'pipe', 'backend', ),
+		( 'pipe', 'backend', ),
 	),
 	('bereq.proto',
 		'STRING',
-		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
-		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
+		( 'pipe', 'backend', ),
+		( 'pipe', 'backend', ),
 	),
 	('bereq.http.',
 		'HEADER',
-		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
-		( 'pipe', 'backend_fetch', 'miss', 'backend_response',),
+		( 'pipe', 'backend', ),
+		( 'pipe', 'backend', ),
 	),
 	('bereq.uncacheable',
 		'BOOL',
-		( 'backend_fetch', 'backend_response',),
-		( 'backend_fetch', 'backend_response',),
+		( 'backend', ),
+		( 'backend', ),
 	),
 	('bereq.connect_timeout',
 		'DURATION',
-		( 'pipe', 'backend_fetch', 'miss',),
-		( 'pipe', 'backend_fetch', 'miss',),
+		( 'pipe', 'backend', ),
+		( 'pipe', 'backend', ),
 	),
 	('bereq.first_byte_timeout',
 		'DURATION',
-		( 'backend_fetch', 'miss',),
-		( 'backend_fetch', 'miss',),
+		( 'pipe', 'backend', ),
+		( 'pipe', 'backend', ),
 	),
 	('bereq.between_bytes_timeout',
 		'DURATION',
-		( 'backend_fetch', 'miss',),
-		( 'backend_fetch', 'miss',),
+		( 'pipe', 'backend', ),
+		( 'pipe', 'backend', ),
 	),
 	('beresp.proto',
 		'STRING',
@@ -437,7 +437,7 @@ sp_variables = (
 		( 'all',),
 		( ),
 	),
-)
+]
 
 stv_variables = (
 	('free_space',	'BYTES',	"0."),
@@ -770,30 +770,41 @@ fo.close()
 #######################################################################
 
 def restrict(fo, spec):
-	if len(spec) == 0:
-		fo.write("\t\t0,\n")
-		return
-	if spec[0] == 'all':
-		spec = vcls
-	if spec[0] == 'client':
-		spec = vcls_client
-	if spec[0] == 'backend':
-		spec = vcls_backend
-	if spec[0] == 'both':
-		spec = vcls_client
-		spec += vcls_backend
+	d = dict()
+	for j in spec:
+		if j == 'all':
+			for i in vcls:
+				d[i] = True
+		elif j == 'backend':
+			for i in vcls_backend:
+				d[i] = True
+		elif j == 'client':
+			for i in vcls_client:
+				d[i] = True
+		elif j == 'both':
+			for i in vcls_client:
+				d[i] = True
+			for i in vcls_backend:
+				d[i] = True
+		else:
+			assert j in vcls
+			d[j] = True
 	p = ""
 	n = 0
-	for j in spec:
-		if n == 4:
-			fo.write("\n")
-			n = 0
-		if n == 0:
-			fo.write("\t\t")
-		n += 1
-		fo.write(p + "VCL_MET_" + j.upper())
+	l = list(d.keys())
+	l.sort()
+	w = 0
+	fo.write("\t\t")
+	for j in l:
+		x = p + "VCL_MET_" + j.upper()
+		if w + len(x) > 60:
+			fo.write("\n\t\t")
+			w = 0
+		fo.write(x)
+		w += len(x)
 		p = " | "
-
+	if len(d) == 0:
+		fo.write("0")
 	fo.write(",\n")
 
 #######################################################################
@@ -814,6 +825,7 @@ fo.write("""
 const struct var vcc_vars[] = {
 """)
 
+sp_variables.sort()
 for i in sp_variables:
 	typ = i[1]
 	cnam = i[0].replace(".", "_")



From phk at varnish-cache.org  Wed Aug 21 08:11:28 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 21 Aug 2013 10:11:28 +0200
Subject: [master] eba445d Restructure to put all the streaming code in one
 place.
Message-ID: <E1VC3Vo-0003K4-As@project.varnish-software.com>

commit eba445d4621c6e4819611f3f064da0a492b519f7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 21 08:11:16 2013 +0000

    Restructure to put all the streaming code in one place.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index bb52290..db27c51 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -79,43 +79,102 @@ DOT acceptor -> recv [style=bold,color=green]
  * We have a refcounted object on the session, and possibly the busyobj
  * which is fetching it, prepare a response.
  *
-DOT subgraph xcluster_prepresp {
-DOT	prepresp [
+DOT	stream [
 DOT		shape=record
-DOT		label="{cnt_prepresp:|Filter obj.-\>resp.|{vcl_deliver\{\}|{req.|resp.}}|{error?|restart?}|stream ?}"
+DOT		label="{cnt_stream:}"
 DOT	]
-DOT	prepresp -> deliver [style=bold,color=green,label=deliver]
-DOT	prepresp -> deliver [style=bold,color=red]
-DOT	prepresp -> deliver [style=bold,color=blue]
-DOT }
+DOT	stream:deliver:s -> DONE [style=bold,color=red]
+DOT	stream:deliver:s -> DONE [style=bold,color=blue]
  *
  */
 
 static enum req_fsm_nxt
-cnt_prepresp(struct worker *wrk, struct req *req)
+cnt_stream(struct worker *wrk, struct req *req)
 {
 	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	bo = req->busyobj;
-	CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 
 	req->res_mode = 0;
 
-	if (bo == NULL) {
-		if (!req->disable_esi && req->obj->esidata != NULL) {
-			/* In ESI mode, we can't know the aggregate length */
-			req->res_mode &= ~RES_LEN;
-			req->res_mode |= RES_ESI;
-		} else {
-			req->res_mode |= RES_LEN;
-		}
+	AZ(bo->do_esi);
+
+	if (wrk->handling == VCL_RET_RESTART) {
+		AN(bo->do_stream);
+		assert(req->obj == bo->fetch_obj);
+		req->obj = NULL;
+		VBO_DerefBusyObj(wrk, &req->busyobj);
+		AZ(req->obj);
+		http_Teardown(req->resp);
+		req->req_step = R_STP_RESTART;
+		return (REQ_FSM_MORE);
+	}
+	assert(wrk->handling == VCL_RET_DELIVER);
+
+	while (bo->state < BOS_FAILED)
+		(void)usleep(10000);
+	assert(bo->state >= BOS_FAILED);
+
+	if (bo->state == BOS_FAILED) {
+		(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
+		VBO_DerefBusyObj(wrk, &req->busyobj);
+		req->err_code = 503;
+		req->req_step = R_STP_ERROR;
+		return (REQ_FSM_MORE);
+	}
+	VBO_DerefBusyObj(wrk, &req->busyobj);
+
+	AZ(req->busyobj);
+
+	RES_WriteObj(req);
+
+	/* No point in saving the body if it is hit-for-pass */
+	if (req->obj->objcore->flags & OC_F_PASS)
+		STV_Freestore(req->obj);
+
+	assert(WRW_IsReleased(wrk));
+	(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
+	http_Teardown(req->resp);
+	return (REQ_FSM_DONE);
+}
+
+/*--------------------------------------------------------------------
+ * Deliver an already stored object
+ *
+DOT	deliver [
+DOT		shape=record
+DOT		label="{cnt_deliver:|Filter obj.-\>resp.|{vcl_deliver\{\}|{req.|resp.}}|{<stream>deliver(stream)?|restart?|<deliver>deliver?}}"
+DOT	]
+DOT deliver:deliver:s -> DONE [style=bold,color=green]
+DOT deliver:stream:s -> stream [style=bold,color=red]
+DOT deliver:stream:s -> stream [style=bold,color=blue]
+ *
+ */
+
+static enum req_fsm_nxt
+cnt_deliver(struct worker *wrk, struct req *req)
+{
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
+
+	req->res_mode = 0;
+
+	if (!req->disable_esi && req->obj->esidata != NULL) {
+		AZ(req->busyobj);
+		/* In ESI mode, we can't know the aggregate length */
+		req->res_mode &= ~RES_LEN;
+		req->res_mode |= RES_ESI;
 	} else {
-		AZ(bo->do_esi);
+		req->res_mode |= RES_LEN;
 	}
 
 	if (req->esi_level > 0) {
@@ -159,75 +218,24 @@ cnt_prepresp(struct worker *wrk, struct req *req)
 	RES_BuildHttp(req);
 
 	VCL_deliver_method(req->vcl, wrk, req, NULL, req->http->ws);
-	switch (wrk->handling) {
-	case VCL_RET_DELIVER:
-		break;
-	case VCL_RET_RESTART:
-		if (req->restarts >= cache_param->max_restarts)
-			break;
-		if (bo != NULL) {
-			AN(bo->do_stream);
-			assert(req->obj == bo->fetch_obj);
-			req->obj = NULL;
-			VBO_DerefBusyObj(wrk, &req->busyobj);
-		} else {
-			(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
-		}
+
+	/* Stop the insanity before it turns "Hotel California" on us */
+	if (req->restarts >= cache_param->max_restarts)
+		wrk->handling = VCL_RET_DELIVER;
+
+	if (req->busyobj != NULL) {
+		req->req_step = R_STP_STREAM;
+		return (REQ_FSM_MORE);
+	}
+
+	if (wrk->handling == VCL_RET_RESTART) {
+		(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
 		AZ(req->obj);
 		http_Teardown(req->resp);
 		req->req_step = R_STP_RESTART;
 		return (REQ_FSM_MORE);
-	default:
-		WRONG("Illegal action in vcl_deliver{}");
 	}
-	req->req_step = R_STP_DELIVER;
-	return (REQ_FSM_MORE);
-}
-
-/*--------------------------------------------------------------------
- * Deliver an already stored object
- *
-DOT subgraph xcluster_deliver {
-DOT	deliver [
-DOT		shape=record
-DOT		label="{cnt_deliver:|Send body}"
-DOT	]
-DOT }
-DOT deliver -> DONE [style=bold,color=green]
-DOT deliver -> DONE [style=bold,color=red]
-DOT deliver -> DONE [style=bold,color=blue]
- *
- */
-
-static enum req_fsm_nxt
-cnt_deliver(struct worker *wrk, struct req *req)
-{
-	struct busyobj *bo;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-	bo = req->busyobj;
-	CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC);
-
-	if (bo != NULL) {
-		while (bo->state < BOS_FAILED)
-			(void)usleep(10000);
-		assert(bo->state >= BOS_FAILED);
-
-		if (bo->state == BOS_FAILED) {
-			(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
-			VBO_DerefBusyObj(wrk, &req->busyobj);
-			req->err_code = 503;
-			req->req_step = R_STP_ERROR;
-			return (REQ_FSM_MORE);
-		}
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-	}
-
-	AZ(req->busyobj);
-	req->director = NULL;
-	req->restarts = 0;
+	assert(wrk->handling == VCL_RET_DELIVER);
 
 	RES_WriteObj(req);
 
@@ -240,6 +248,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	http_Teardown(req->resp);
 	return (REQ_FSM_DONE);
 }
+
 /*--------------------------------------------------------------------
  * Emit an error
  *
@@ -249,7 +258,7 @@ DOT		shape=record
 DOT		label="vcl_error()|resp."
 DOT	]
 DOT	ERROR -> vcl_error
-DOT	vcl_error-> prepresp [label=deliver]
+DOT	vcl_error-> deliver [label=deliver]
 DOT }
 DOT vcl_error-> rsterr [label="restart",color=purple]
 DOT rsterr [label="RESTART",shape=plaintext]
@@ -312,15 +321,16 @@ cnt_error(struct worker *wrk, struct req *req)
 		http_PutResponse(h, http_StatusMessage(req->err_code));
 	VCL_error_method(req->vcl, wrk, req, NULL, req->http->ws);
 
-	if (wrk->handling == VCL_RET_RESTART &&
-	    req->restarts <  cache_param->max_restarts) {
+	/* Stop the insanity before it turns "Hotel California" on us */
+	if (req->restarts >= cache_param->max_restarts)
+		wrk->handling = VCL_RET_DELIVER;
+
+	if (wrk->handling == VCL_RET_RESTART) {
 		HSH_Drop(wrk, &req->obj);
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 		req->req_step = R_STP_RESTART;
 		return (REQ_FSM_MORE);
-	} else if (wrk->handling == VCL_RET_RESTART)
-		wrk->handling = VCL_RET_DELIVER;
-
+	}
 
 	/* We always close when we take this path */
 	req->doclose = SC_TX_ERROR;
@@ -331,7 +341,7 @@ cnt_error(struct worker *wrk, struct req *req)
 	req->err_reason = NULL;
 	http_Teardown(bo->bereq);
 	VBO_DerefBusyObj(wrk, &req->busyobj);
-	req->req_step = R_STP_PREPRESP;
+	req->req_step = R_STP_DELIVER;
 	return (REQ_FSM_MORE);
 }
 
@@ -344,8 +354,8 @@ DOT		shape=record
 DOT		label="{cnt_fetch:|start fetch_thread}"
 DOT	]
 DOT }
-DOT fetch -> prepresp [style=bold,color=red]
-DOT fetch -> prepresp [style=bold,color=blue]
+DOT fetch -> deliver [style=bold,color=red]
+DOT fetch -> deliver [style=bold,color=blue]
  */
 
 static enum req_fsm_nxt
@@ -373,7 +383,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 		req->obj = req->busyobj->fetch_obj;
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 		assert(WRW_IsReleased(wrk));
-		req->req_step = R_STP_PREPRESP;
+		req->req_step = R_STP_DELIVER;
 	}
 	return (REQ_FSM_MORE);
 }
@@ -403,7 +413,7 @@ DOT lookup:eb:s -> lookup2 [style=bold,color=green]
 DOT lookup:h:s -> lookup2 [style=bold,color=green]
 DOT lookup2:pass:s -> pass [style=bold,color=red]
 DOT lookup2:fetch:s -> miss [style=bold,color=blue]
-DOT lookup2:deliver:s -> prepresp:nw [style=bold,color=green]
+DOT lookup2:deliver:s -> deliver:n [style=bold,color=green]
  */
 
 static enum req_fsm_nxt
@@ -496,7 +506,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 			(void)HTTP1_DiscardReqBody(req);// XXX: handle err
 		}
 		wrk->stats.cache_hit++;
-		req->req_step = R_STP_PREPRESP;
+		req->req_step = R_STP_DELIVER;
 		return (REQ_FSM_MORE);
 	case VCL_RET_FETCH:
 		(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index 6f219e4..30d331c 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -44,7 +44,7 @@ REQ_STEP(lookup,	LOOKUP,		(wrk, req))
 REQ_STEP(purge,		PURGE,		(wrk, req))
 REQ_STEP(miss,		MISS,		(wrk, req))
 REQ_STEP(fetch,		FETCH,		(wrk, req))
-REQ_STEP(prepresp,	PREPRESP,	(wrk, req))
+REQ_STEP(stream,	STREAM,		(wrk, req))
 REQ_STEP(deliver,	DELIVER,	(wrk, req))
 REQ_STEP(error,		ERROR,		(wrk, req))
 #endif



From phk at varnish-cache.org  Wed Aug 21 15:11:40 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 21 Aug 2013 17:11:40 +0200
Subject: [master] 39da3b4 Polish up bo->state before we unleash streaming
Message-ID: <E1VCA4S-0002fn-SP@project.varnish-software.com>

commit 39da3b442c8a4d9a754af4731aae7305ac364dbe
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 21 15:11:18 2013 +0000

    Polish up bo->state before we unleash streaming

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b2399cc..879b637 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -490,10 +490,10 @@ oc_getlru(const struct objcore *oc)
  */
 
 enum busyobj_state_e {
-	BOS_INVALID = 0,
-	BOS_FETCHING,
-	BOS_FAILED,
-	BOS_FINISHED
+	BOS_INVALID = 0,	/* don't touch (yet) */
+	BOS_FETCHING,		/* beresp.* can be examined */
+	BOS_FINISHED,		/* object is complete */
+	BOS_FAILED,		/* something went wrong */
 };
 
 struct busyobj {
@@ -797,6 +797,9 @@ struct busyobj *VBO_GetBusyObj(struct worker *, struct req *);
 void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj);
 void VBO_Free(struct busyobj **vbo);
 void VBO_extend(const struct busyobj *, ssize_t);
+void VBO_setstate(struct busyobj *bo, enum busyobj_state_e next);
+void VBO_waitstate(struct busyobj *bo, enum busyobj_state_e want);
+
 
 /* cache_http1_fetch.c [V1F] */
 int V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req);
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index ba49473..6b219d6 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -222,3 +222,26 @@ VBO_extend(const struct busyobj *bo, ssize_t l)
 	assert(l > 0);
 	bo->fetch_obj->len += l;
 }
+
+void
+VBO_setstate(struct busyobj *bo, enum busyobj_state_e next)
+{
+	Lck_Lock(&bo->mtx);
+	VSLb(bo->vsl, SLT_Debug, "XXX: %d -> %d", bo->state, next);
+	assert(next > bo->state);
+	bo->state = next;
+	AZ(pthread_cond_signal(&bo->cond));
+	Lck_Unlock(&bo->mtx);
+}
+
+void
+VBO_waitstate(struct busyobj *bo, enum busyobj_state_e want)
+{
+	Lck_Lock(&bo->mtx);
+	while (1) {
+		if (bo->state >= want)
+			break;
+		(void)Lck_CondWait(&bo->cond, &bo->mtx, NULL);
+	}
+	Lck_Unlock(&bo->mtx);
+}
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index e853c52..0032478 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -117,8 +117,10 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	http_PrintfHeader(bo->bereq,
 	    "X-Varnish: %u", bo->vsl->wid & VSL_IDENTMASK);
-	if (wrk->handling == VCL_RET_ABANDON)
+	if (wrk->handling == VCL_RET_ABANDON) {
+		VBO_setstate(bo, BOS_FAILED);
 		return (F_STP_ABANDON);
+	}
 	assert (wrk->handling == VCL_RET_FETCH);
 	return (F_STP_FETCHHDR);
 }
@@ -141,6 +143,8 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 	if (!bo->do_pass)
 		vbf_release_req(bo); /* XXX: retry ?? */
 
+	assert(bo->state == BOS_INVALID);
+
 	i = V1F_fetch_hdr(wrk, bo, bo->req);
 	/*
 	 * If we recycle a backend connection, there is a finite chance
@@ -204,6 +208,7 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 	if (wrk->handling == VCL_RET_DELIVER)
 		return (F_STP_FETCH);
 	if (wrk->handling == VCL_RET_RETRY) {
+		assert(bo->state == BOS_INVALID);
 		bo->retries++;
 		if (bo->retries <= cache_param->max_retries) {
 			VDI_CloseFd(&bo->vbc);
@@ -352,6 +357,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
 		bo->fetch_objcore = NULL;
 		VDI_CloseFd(&bo->vbc);
+		VBO_setstate(bo, BOS_FAILED);
 		return (F_STP_ABANDON);
 	}
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
@@ -407,6 +413,9 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->vfp == NULL)
 		bo->vfp = &VFP_nop;
 
+	assert(bo->state == BOS_INVALID);
+	VBO_setstate(bo, BOS_FETCHING);
+
 	V1F_fetch_body(wrk, bo);
 
 	assert(bo->refcount >= 1);
@@ -419,6 +428,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		(void)HSH_Deref(&wrk->stats, NULL, &obj);
 		return (F_STP_ABANDON);
 	}
+	VBO_setstate(bo, BOS_FINISHED);
 
 	VBO_DerefBusyObj(wrk, &bo);	// XXX ?
 	return (F_STP_DONE);
@@ -433,7 +443,7 @@ vbf_stp_abandon(struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
-	bo->state = BOS_FAILED;
+	assert(bo->state == BOS_FAILED);
 	vbf_release_req(bo);
 	VBO_DerefBusyObj(wrk, &bo);	// XXX ?
 	return (F_STP_DONE);
diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c
index 8fd37f9..fdc896c 100644
--- a/bin/varnishd/cache/cache_fetch_proc.c
+++ b/bin/varnishd/cache/cache_fetch_proc.c
@@ -57,7 +57,7 @@ VFP_Error2(struct busyobj *bo, const char *error, const char *more)
 {
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	if (bo->state == BOS_FETCHING) {
+	if (bo->state < BOS_FAILED) {
 		if (more == NULL)
 			VSLb(bo->vsl, SLT_FetchError, "%s", error);
 		else
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index a6ed005..65d2e93 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -324,7 +324,7 @@ V1F_fetch_body(struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
 
-	assert(bo->state == BOS_INVALID);
+	assert(bo->state == BOS_FETCHING);
 
 	/*
 	 * XXX: The busyobj needs a dstat, but it is not obvious which one
@@ -337,8 +337,6 @@ V1F_fetch_body(struct worker *wrk, struct busyobj *bo)
 	AZ(bo->vgz_rx);
 	AZ(VTAILQ_FIRST(&obj->store));
 
-	bo->state = BOS_FETCHING;
-
 	/* XXX: pick up estimate from objdr ? */
 	cl = 0;
 	cls = bo->should_close;
@@ -441,9 +439,6 @@ V1F_fetch_body(struct worker *wrk, struct busyobj *bo)
 			http_PrintfHeader(obj->http,
 			    "Content-Length: %zd", obj->len);
 		}
-
-		/* XXX: Atomic assignment, needs volatile/membar ? */
-		bo->state = BOS_FINISHED;
 	}
 	if (obj->objcore->objhead != NULL)
 		HSH_Complete(obj->objcore);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index db27c51..22282e3 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -117,9 +117,8 @@ cnt_stream(struct worker *wrk, struct req *req)
 	}
 	assert(wrk->handling == VCL_RET_DELIVER);
 
-	while (bo->state < BOS_FAILED)
-		(void)usleep(10000);
-	assert(bo->state >= BOS_FAILED);
+	VBO_waitstate(bo, BOS_FINISHED);
+	assert(bo->state >= BOS_FINISHED);
 
 	if (bo->state == BOS_FAILED) {
 		(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
@@ -369,7 +368,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	AN(req->busyobj);
 	assert(req->busyobj->refcount > 0);
 	(void)HTTP1_DiscardReqBody(req);
-	while (req->busyobj->state < BOS_FAILED) {
+	while (req->busyobj->state < BOS_FINISHED) {
 		printf("YYY\n");
 		(void)usleep(100000);
 	}



From phk at varnish-cache.org  Thu Aug 22 09:35:53 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 22 Aug 2013 11:35:53 +0200
Subject: [master] 3450da1 fiddle the VFP's a little bit.
Message-ID: <E1VCRJ3-0000hJ-Aa@project.varnish-software.com>

commit 3450da1fc4d11da6fb61e85921be6866e409444f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 22 08:07:15 2013 +0000

    fiddle the VFP's a little bit.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 879b637..fffe812 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -259,9 +259,9 @@ struct dstat {
 
 /* Fetch processors --------------------------------------------------*/
 
-typedef void vfp_begin_f(void *priv, size_t );
-typedef int vfp_bytes_f(void *priv, struct http_conn *, ssize_t);
-typedef int vfp_end_f(void *priv);
+typedef void vfp_begin_f(struct busyobj *bo, size_t );
+typedef int vfp_bytes_f(struct busyobj *bo, struct http_conn *, ssize_t);
+typedef int vfp_end_f(struct busyobj *bo);
 
 struct vfp {
 	vfp_begin_f	*begin;
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index 7da3608..e64d796 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -295,12 +295,11 @@ vfp_esi_bytes_gg(const struct busyobj *bo, struct vef_priv *vef,
 /*---------------------------------------------------------------------*/
 
 static void __match_proto__(vfp_begin_f)
-vfp_esi_begin(void *priv, size_t estimate)
+vfp_esi_begin(struct busyobj *bo, size_t estimate)
 {
-	struct busyobj *bo;
 	struct vef_priv *vef;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	(void)estimate;
 
 	ALLOC_OBJ(vef, VEF_MAGIC);
@@ -340,13 +339,12 @@ vfp_esi_begin(void *priv, size_t estimate)
 }
 
 static int __match_proto__(vfp_bytes_f)
-vfp_esi_bytes(void *priv, struct http_conn *htc, ssize_t bytes)
+vfp_esi_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
 {
-	struct busyobj *bo;
 	struct vef_priv *vef;
 	int i;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	vef = bo->vef_priv;
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 
@@ -365,15 +363,14 @@ vfp_esi_bytes(void *priv, struct http_conn *htc, ssize_t bytes)
 }
 
 static int __match_proto__(vfp_end_f)
-vfp_esi_end(void *priv)
+vfp_esi_end(struct busyobj *bo)
 {
-	struct busyobj *bo;
 	struct vsb *vsb;
 	struct vef_priv *vef;
 	ssize_t l;
 	int retval = 0;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AN(bo->vep);
 
 	if (bo->state == BOS_FAILED)
diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c
index fdc896c..cacfcc0 100644
--- a/bin/varnishd/cache/cache_fetch_proc.c
+++ b/bin/varnishd/cache/cache_fetch_proc.c
@@ -89,11 +89,10 @@ VFP_Error(struct busyobj *bo, const char *error)
  * as seen on the socket, or zero if unknown.
  */
 static void __match_proto__(vfp_begin_f)
-vfp_nop_begin(void *priv, size_t estimate)
+vfp_nop_begin(struct busyobj *bo, size_t estimate)
 {
-	struct busyobj *bo;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	if (estimate > 0)
 		(void)VFP_GetStorage(bo, estimate);
@@ -111,13 +110,12 @@ vfp_nop_begin(void *priv, size_t estimate)
  */
 
 static int __match_proto__(vfp_bytes_f)
-vfp_nop_bytes(void *priv, struct http_conn *htc, ssize_t bytes)
+vfp_nop_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
 {
 	ssize_t l, wl;
 	struct storage *st;
-	struct busyobj *bo;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	while (bytes > 0) {
 		st = VFP_GetStorage(bo, 0);
@@ -146,23 +144,10 @@ vfp_nop_bytes(void *priv, struct http_conn *htc, ssize_t bytes)
  */
 
 static int __match_proto__(vfp_end_f)
-vfp_nop_end(void *priv)
+vfp_nop_end(struct busyobj *bo)
 {
-	struct storage *st;
-	struct busyobj *bo;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
-	st = VTAILQ_LAST(&bo->fetch_obj->store, storagehead);
-	if (st == NULL)
-		return (0);
-
-	if (st->len == 0) {
-		VTAILQ_REMOVE(&bo->fetch_obj->store, st, list);
-		STV_free(st);
-		return (0);
-	}
-	if (st->len < st->space)
-		STV_trim(st, st->len, 1);
+	(void)bo;
 	return (0);
 }
 
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index f0374a7..be633d3 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -441,11 +441,10 @@ VGZ_Destroy(struct vgz **vgp)
  */
 
 static void __match_proto__(vfp_begin_f)
-vfp_gunzip_begin(void *priv, size_t estimate)
+vfp_gunzip_begin(struct busyobj *bo, size_t estimate)
 {
-	struct busyobj *bo;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	(void)estimate;
 	AZ(bo->vgz_rx);
 	bo->vgz_rx = VGZ_NewUngzip(bo->vsl, "U F -");
@@ -453,16 +452,15 @@ vfp_gunzip_begin(void *priv, size_t estimate)
 }
 
 static int __match_proto__(vfp_bytes_f)
-vfp_gunzip_bytes(void *priv, struct http_conn *htc, ssize_t bytes)
+vfp_gunzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
-	struct busyobj *bo;
 	ssize_t l, wl;
 	int i = -100;
 	size_t dl;
 	const void *dp;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	vg = bo->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -490,12 +488,11 @@ vfp_gunzip_bytes(void *priv, struct http_conn *htc, ssize_t bytes)
 }
 
 static int __match_proto__(vfp_end_f)
-vfp_gunzip_end(void *priv)
+vfp_gunzip_end(struct busyobj *bo)
 {
 	struct vgz *vg;
-	struct busyobj *bo;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	vg = bo->vgz_rx;
 	bo->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
@@ -521,28 +518,26 @@ struct vfp vfp_gunzip = {
  */
 
 static void __match_proto__(vfp_begin_f)
-vfp_gzip_begin(void *priv, size_t estimate)
+vfp_gzip_begin(struct busyobj *bo, size_t estimate)
 {
 	(void)estimate;
-	struct busyobj *bo;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AZ(bo->vgz_rx);
 	bo->vgz_rx = VGZ_NewGzip(bo->vsl, "G F -");
 	XXXAZ(vgz_getmbuf(bo->vgz_rx));
 }
 
 static int __match_proto__(vfp_bytes_f)
-vfp_gzip_bytes(void *priv, struct http_conn *htc, ssize_t bytes)
+vfp_gzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
-	struct busyobj *bo;
 	ssize_t l, wl;
 	int i = -100;
 	size_t dl;
 	const void *dp;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	vg = bo->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -567,15 +562,14 @@ vfp_gzip_bytes(void *priv, struct http_conn *htc, ssize_t bytes)
 }
 
 static int __match_proto__(vfp_end_f)
-vfp_gzip_end(void *priv)
+vfp_gzip_end(struct busyobj *bo)
 {
 	struct vgz *vg;
-	struct busyobj *bo;
 	size_t dl;
 	const void *dp;
 	int i;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	vg = bo->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	bo->vgz_rx = NULL;
@@ -610,11 +604,10 @@ struct vfp vfp_gzip = {
  */
 
 static void __match_proto__(vfp_begin_f)
-vfp_testgzip_begin(void *priv, size_t estimate)
+vfp_testgzip_begin(struct busyobj *bo, size_t estimate)
 {
-	struct busyobj *bo;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	(void)estimate;
 	bo->vgz_rx = VGZ_NewUngzip(bo->vsl, "u F -");
 	CHECK_OBJ_NOTNULL(bo->vgz_rx, VGZ_MAGIC);
@@ -622,17 +615,16 @@ vfp_testgzip_begin(void *priv, size_t estimate)
 }
 
 static int __match_proto__(vfp_bytes_f)
-vfp_testgzip_bytes(void *priv, struct http_conn *htc, ssize_t bytes)
+vfp_testgzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
-	struct busyobj *bo;
 	ssize_t l, wl;
 	int i = -100;
 	size_t dl;
 	const void *dp;
 	struct storage *st;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	vg = bo->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -666,12 +658,11 @@ vfp_testgzip_bytes(void *priv, struct http_conn *htc, ssize_t bytes)
 }
 
 static int __match_proto__(vfp_end_f)
-vfp_testgzip_end(void *priv)
+vfp_testgzip_end(struct busyobj *bo)
 {
 	struct vgz *vg;
-	struct busyobj *bo;
 
-	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	vg = bo->vgz_rx;
 	bo->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index 65d2e93..cd917a1 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -384,13 +384,18 @@ V1F_fetch_body(struct worker *wrk, struct busyobj *bo)
 	}
 	AZ(bo->vgz_rx);
 
-#if 0
 	/*
-	 * We always call vfp_nop_end() to ditch or trim the last storage
-	 * segment, to avoid having to replicate that code in all vfp's.
+	 * Trim or delete the last segment, if any
 	 */
-	AZ(vfp_nop_end(bo));
-#endif
+
+	st = VTAILQ_LAST(&bo->fetch_obj->store, storagehead);
+	if (st != NULL) {
+		if (st->len == 0) {
+			VTAILQ_REMOVE(&bo->fetch_obj->store, st, list);
+			STV_free(st);
+		} else if (st->len < st->space)
+			STV_trim(st, st->len, 1);
+	}
 
 	bo->vfp = NULL;
 



From phk at varnish-cache.org  Thu Aug 22 09:35:53 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 22 Aug 2013 11:35:53 +0200
Subject: [master] d114e14 Add VBO_waitlen()
Message-ID: <E1VCRJ3-0000hP-CH@project.varnish-software.com>

commit d114e1493de328cfcb3c94d08c84a5eeb3d763dc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 22 09:35:39 2013 +0000

    Add VBO_waitlen()

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index fffe812..0e3d7d8 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -796,7 +796,8 @@ void VBO_Init(void);
 struct busyobj *VBO_GetBusyObj(struct worker *, struct req *);
 void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj);
 void VBO_Free(struct busyobj **vbo);
-void VBO_extend(const struct busyobj *, ssize_t);
+void VBO_extend(struct busyobj *, ssize_t);
+ssize_t VBO_waitlen(struct busyobj *bo, ssize_t l);
 void VBO_setstate(struct busyobj *bo, enum busyobj_state_e next);
 void VBO_waitstate(struct busyobj *bo, enum busyobj_state_e want);
 
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 6b219d6..2a4d318 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -212,7 +212,7 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 }
 
 void
-VBO_extend(const struct busyobj *bo, ssize_t l)
+VBO_extend(struct busyobj *bo, ssize_t l)
 {
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -220,7 +220,25 @@ VBO_extend(const struct busyobj *bo, ssize_t l)
 	if (l == 0)
 		return;
 	assert(l > 0);
+	Lck_Lock(&bo->mtx);
 	bo->fetch_obj->len += l;
+	AZ(pthread_cond_signal(&bo->cond));
+	Lck_Unlock(&bo->mtx);
+}
+
+ssize_t
+VBO_waitlen(struct busyobj *bo, ssize_t l)
+{
+	Lck_Lock(&bo->mtx);
+	while (1) {
+		if (bo->fetch_obj->len > l || bo->state >= BOS_FINISHED) {
+			l = bo->fetch_obj->len;
+			break;
+		}
+		(void)Lck_CondWait(&bo->cond, &bo->mtx, NULL);
+	}
+	Lck_Unlock(&bo->mtx);
+	return (l);
 }
 
 void
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index cd917a1..c3e6b2b 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -393,8 +393,10 @@ V1F_fetch_body(struct worker *wrk, struct busyobj *bo)
 		if (st->len == 0) {
 			VTAILQ_REMOVE(&bo->fetch_obj->store, st, list);
 			STV_free(st);
-		} else if (st->len < st->space)
+		} else if (st->len < st->space) {
+			/* XXX: is this safe under streaming ? */
 			STV_trim(st, st->len, 1);
+		}
 	}
 
 	bo->vfp = NULL;



From perbu at varnish-cache.org  Sun Aug 25 08:15:19 2013
From: perbu at varnish-cache.org (Per Buer)
Date: Sun, 25 Aug 2013 10:15:19 +0200
Subject: [master] 5ce4bda Redone for clearity Also removed some redundant
 stuff.
Message-ID: <E1VDVTj-0005tJ-8t@project.varnish-software.com>

commit 5ce4bdaa0f08f35f976da9fdf50fee048689e2a0
Author: Per Buer <per.buer at gmail.com>
Date:   Sun Aug 25 10:15:32 2013 +0200

    Redone for clearity
    Also removed some redundant stuff.

diff --git a/doc/sphinx/tutorial/backend_servers.rst b/doc/sphinx/tutorial/backend_servers.rst
index 1b1aaf2..7ea3a20 100644
--- a/doc/sphinx/tutorial/backend_servers.rst
+++ b/doc/sphinx/tutorial/backend_servers.rst
@@ -12,28 +12,32 @@ file. If you installed from source this is
 /usr/local/etc/varnish/default.vcl, if you installed from a package it
 is probably /etc/varnish/default.vcl.
 
-Somewhere in the top there will be a section that looks a bit like this.::
+If you've been following the tutorial there is probably a section of
+the configuration that looks like this:::
 
-	  # backend default {
-	  #     .host = "127.0.0.1";
-	  #     .port = "8080";
-	  # }
+  backend default {
+      .host = "www.varnish-cache.org";
+      .port = "80";
+  }
 
-We comment in this bit of text and change the port setting from 8080
-to 80, making the text look like.::
+Since you probably don't want to be mirroring varnish-cache.org we
+need to get Varnish to fetch content from your own origin
+server. We've already bound Varnish to the public port 80 on the
+server so now we need to tie it to the origin.
+
+For this example, let's pretend the origin server is running on
+localhost, port 8080.
 
           backend default {
                 .host = "127.0.0.1";
-    		.port = "80";
+    		.port = "8080";
 	  }
 
-Now, this piece of configuration defines a backend in Varnish called
-*default*. When Varnish needs to get content from this backend it will
-connect to port 80 on localhost (127.0.0.1).
-
 Varnish can have several backends defined and can you can even join
 several backends together into clusters of backends for load balancing
-purposes. 
+purposes, having Varnish pick one backend based on different
+algorithms. 
 
-Now that we have the basic Varnish configuration done, let us start up
-Varnish on port 8080 so we can do some fundamental testing on it.
+A lot of the power of Varnish Cache comes from it's design, which
+might not be what you are expecting. Next, let's have a look at some of
+them.
diff --git a/doc/sphinx/tutorial/now_what.rst b/doc/sphinx/tutorial/now_what.rst
index 36b81b7..1e23e10 100644
--- a/doc/sphinx/tutorial/now_what.rst
+++ b/doc/sphinx/tutorial/now_what.rst
@@ -6,4 +6,6 @@ Now what?
 
 You've read through the tutorial. You should have Varnish up and
 running. You should know about the logs and you should have a rough
-idea of what VCL is.
\ No newline at end of file
+idea of what VCL is. Next, you might want to have a look at
+:ref:``users-guide-index``, where we go through the features of
+Varnish in more detail.
diff --git a/doc/sphinx/tutorial/peculiarities.rst b/doc/sphinx/tutorial/peculiarities.rst
index 9e83e99..c04bd81 100644
--- a/doc/sphinx/tutorial/peculiarities.rst
+++ b/doc/sphinx/tutorial/peculiarities.rst
@@ -6,6 +6,15 @@ There are a couple of things that are different with Varnish Cache, as
 opposed to other programs. One thing you've already seen - VCL. I'll
 just give you a very quick tour of the other pecularities.
 
+Configuration
+~~~~~~~~~~~~~
+
+The Varnish Configuration is written in VCL. When Varnish is ran this
+configuration is transformed into C code and then fed into a C
+compiler, loaded and run. So, as opposed to declaring various
+settings, you write polices on how the incomming traffic should be
+handled.
+
 
 varnishadm
 ~~~~~~~~~~



From phk at varnish-cache.org  Mon Aug 26 06:40:24 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 26 Aug 2013 08:40:24 +0200
Subject: [master] 1b5a1f6 Use VBO_setstate() so waiters wake up.
Message-ID: <E1VDqTQ-0007U8-5V@project.varnish-software.com>

commit 1b5a1f6d25b8a96618a469253479fc79fa98203d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 26 06:40:03 2013 +0000

    Use VBO_setstate() so waiters wake up.

diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c
index cacfcc0..37a2f4d 100644
--- a/bin/varnishd/cache/cache_fetch_proc.c
+++ b/bin/varnishd/cache/cache_fetch_proc.c
@@ -62,8 +62,8 @@ VFP_Error2(struct busyobj *bo, const char *error, const char *more)
 			VSLb(bo->vsl, SLT_FetchError, "%s", error);
 		else
 			VSLb(bo->vsl, SLT_FetchError, "%s: %s", error, more);
+		VBO_setstate(bo, BOS_FAILED);
 	}
-	bo->state = BOS_FAILED;
 	return (-1);
 }
 



From phk at varnish-cache.org  Mon Aug 26 07:21:53 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 26 Aug 2013 09:21:53 +0200
Subject: [master] 44f6acb Shuffle things a little bit closer to streaming
Message-ID: <E1VDr7Z-0006do-9W@project.varnish-software.com>

commit 44f6acb45bba91feb21b36dddc1b4c867583d61a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 26 07:21:33 2013 +0000

    Shuffle things a little bit closer to streaming

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 22282e3..bd97a59 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -97,6 +97,9 @@ cnt_stream(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	bo = req->busyobj;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	AN(bo->do_stream);
+
+	INCOMPL();
 
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
@@ -106,7 +109,6 @@ cnt_stream(struct worker *wrk, struct req *req)
 	AZ(bo->do_esi);
 
 	if (wrk->handling == VCL_RET_RESTART) {
-		AN(bo->do_stream);
 		assert(req->obj == bo->fetch_obj);
 		req->obj = NULL;
 		VBO_DerefBusyObj(wrk, &req->busyobj);
@@ -223,8 +225,13 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		wrk->handling = VCL_RET_DELIVER;
 
 	if (req->busyobj != NULL) {
-		req->req_step = R_STP_STREAM;
-		return (REQ_FSM_MORE);
+		AN(req->busyobj->do_stream);
+		/* Don't stream if already finished */
+		if (req->busyobj->state != BOS_FINISHED) {
+			req->req_step = R_STP_STREAM;
+			return (REQ_FSM_MORE);
+		}
+		VBO_DerefBusyObj(wrk, &req->busyobj);
 	}
 
 	if (wrk->handling == VCL_RET_RESTART) {
@@ -254,13 +261,11 @@ cnt_deliver(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_error {
 DOT	vcl_error [
 DOT		shape=record
-DOT		label="vcl_error()|resp."
+DOT		label="{{vcl_error()|resp.}|{<del>deliver?|<restart>restart?}}"
 DOT	]
 DOT	ERROR -> vcl_error
-DOT	vcl_error-> deliver [label=deliver]
+DOT	vcl_error:del:s -> deliver [label=deliver]
 DOT }
-DOT vcl_error-> rsterr [label="restart",color=purple]
-DOT rsterr [label="RESTART",shape=plaintext]
  */
 
 static enum req_fsm_nxt
@@ -350,40 +355,47 @@ cnt_error(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_body {
 DOT	fetch [
 DOT		shape=record
-DOT		label="{cnt_fetch:|start fetch_thread}"
+DOT		label="{cnt_fetch:|wait for fetch|{<ok>OK|<err>Failed}}"
 DOT	]
 DOT }
-DOT fetch -> deliver [style=bold,color=red]
-DOT fetch -> deliver [style=bold,color=blue]
+DOT fetch:ok:s -> deliver [style=bold,color=red]
+DOT fetch:ok:s -> deliver [style=bold,color=blue]
+DOT fetch:err:s -> vcl_error
  */
 
 static enum req_fsm_nxt
 cnt_fetch(struct worker *wrk, struct req *req)
 {
+	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	bo = req->busyobj;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	req->acct_req.fetch++;
-	AN(req->busyobj);
-	assert(req->busyobj->refcount > 0);
+	assert(bo->refcount > 0);
 	(void)HTTP1_DiscardReqBody(req);
-	while (req->busyobj->state < BOS_FINISHED) {
-		printf("YYY\n");
-		(void)usleep(100000);
-	}
-	if (req->busyobj->state == BOS_FAILED) {
+
+	/* If we are not allowed to stream, don't. */
+	if (1 || !bo->do_stream)  			// XXX
+		VBO_waitstate(bo, BOS_FINISHED);
+	else
+		VBO_waitstate(bo, BOS_FETCHING);
+
+	if (bo->state == BOS_FAILED) {
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 		req->err_code = 503;
 		req->req_step = R_STP_ERROR;
-	} else {
-		assert (req->busyobj->state == BOS_FINISHED);
-		req->err_code = req->busyobj->err_code;
-		req->obj = req->busyobj->fetch_obj;
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-		assert(WRW_IsReleased(wrk));
-		req->req_step = R_STP_DELIVER;
+		return (REQ_FSM_MORE);
 	}
+
+	assert (bo->state >= BOS_FETCHING);
+	req->err_code = bo->err_code;
+	req->obj = bo->fetch_obj;
+	VBO_DerefBusyObj(wrk, &req->busyobj);
+	assert(WRW_IsReleased(wrk));
+	req->req_step = R_STP_DELIVER;
 	return (REQ_FSM_MORE);
 }
 



From perbu at varnish-cache.org  Mon Aug 26 07:47:02 2013
From: perbu at varnish-cache.org (Per Buer)
Date: Mon, 26 Aug 2013 09:47:02 +0200
Subject: [master] fbfce35 typo spotted by federico
Message-ID: <E1VDrVu-0002wI-D0@project.varnish-software.com>

commit fbfce35d0b50e8350771af0119692ed6266d7f4d
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Aug 26 09:46:59 2013 +0200

    typo spotted by federico

diff --git a/doc/sphinx/tutorial/peculiarities.rst b/doc/sphinx/tutorial/peculiarities.rst
index c04bd81..af12eff 100644
--- a/doc/sphinx/tutorial/peculiarities.rst
+++ b/doc/sphinx/tutorial/peculiarities.rst
@@ -12,7 +12,7 @@ Configuration
 The Varnish Configuration is written in VCL. When Varnish is ran this
 configuration is transformed into C code and then fed into a C
 compiler, loaded and run. So, as opposed to declaring various
-settings, you write polices on how the incomming traffic should be
+settings, you write polices on how the incoming traffic should be
 handled.
 
 



From phk at varnish-cache.org  Mon Aug 26 08:40:07 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 26 Aug 2013 10:40:07 +0200
Subject: [master] e50f3e8 Move yet a little bit closer to streaming.
Message-ID: <E1VDsLH-0004zb-KM@project.varnish-software.com>

commit e50f3e801821a14a4d71f2c6d5264002d41653e0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 26 08:39:42 2013 +0000

    Move yet a little bit closer to streaming.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 0032478..bde06c8 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -205,6 +205,9 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 
 	VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws);
 
+	if (bo->do_esi)
+		bo->do_stream = 0;
+
 	if (wrk->handling == VCL_RET_DELIVER)
 		return (F_STP_FETCH);
 	if (wrk->handling == VCL_RET_RETRY) {
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index c3e6b2b..2d9928c 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -390,11 +390,11 @@ V1F_fetch_body(struct worker *wrk, struct busyobj *bo)
 
 	st = VTAILQ_LAST(&bo->fetch_obj->store, storagehead);
 	if (st != NULL) {
+		/* XXX: is any of this safe under streaming ? */
 		if (st->len == 0) {
 			VTAILQ_REMOVE(&bo->fetch_obj->store, st, list);
 			STV_free(st);
 		} else if (st->len < st->space) {
-			/* XXX: is this safe under streaming ? */
 			STV_trim(st, st->len, 1);
 		}
 	}
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index bd97a59..396ec18 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -174,7 +174,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		/* In ESI mode, we can't know the aggregate length */
 		req->res_mode &= ~RES_LEN;
 		req->res_mode |= RES_ESI;
-	} else {
+	} else if (req->busyobj == NULL) {
 		req->res_mode |= RES_LEN;
 	}
 
@@ -225,14 +225,16 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		wrk->handling = VCL_RET_DELIVER;
 
 	if (req->busyobj != NULL) {
-		AN(req->busyobj->do_stream);
+		VBO_waitstate(req->busyobj, BOS_FINISHED);
 		/* Don't stream if already finished */
 		if (req->busyobj->state != BOS_FINISHED) {
+			AN(req->busyobj->do_stream);
 			req->req_step = R_STP_STREAM;
 			return (REQ_FSM_MORE);
 		}
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 	}
+	AZ(req->busyobj);
 
 	if (wrk->handling == VCL_RET_RESTART) {
 		(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
@@ -377,11 +379,11 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	assert(bo->refcount > 0);
 	(void)HTTP1_DiscardReqBody(req);
 
-	/* If we are not allowed to stream, don't. */
-	if (1 || !bo->do_stream)  			// XXX
+	VBO_waitstate(bo, BOS_FETCHING);
+
+	/* bo->do_stream is not valid until after vcl_backend_response{} */
+	if (!bo->do_stream)
 		VBO_waitstate(bo, BOS_FINISHED);
-	else
-		VBO_waitstate(bo, BOS_FETCHING);
 
 	if (bo->state == BOS_FAILED) {
 		VBO_DerefBusyObj(wrk, &req->busyobj);
@@ -393,7 +395,8 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	assert (bo->state >= BOS_FETCHING);
 	req->err_code = bo->err_code;
 	req->obj = bo->fetch_obj;
-	VBO_DerefBusyObj(wrk, &req->busyobj);
+	if (bo->state == BOS_FINISHED)
+		VBO_DerefBusyObj(wrk, &req->busyobj);
 	assert(WRW_IsReleased(wrk));
 	req->req_step = R_STP_DELIVER;
 	return (REQ_FSM_MORE);
diff --git a/bin/varnishtest/tests/r00416.vtc b/bin/varnishtest/tests/r00416.vtc
index 3d0ed71..3ad2a86 100644
--- a/bin/varnishtest/tests/r00416.vtc
+++ b/bin/varnishtest/tests/r00416.vtc
@@ -76,7 +76,11 @@ server s1 {
 		-body "foo"
 } -start
 
-varnish v1 -vcl+backend {} -start
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
+} -start
 
 client c1 {
 	txreq \
diff --git a/bin/varnishtest/tests/r00730.vtc b/bin/varnishtest/tests/r00730.vtc
index 6ed2847..4e4fe24 100644
--- a/bin/varnishtest/tests/r00730.vtc
+++ b/bin/varnishtest/tests/r00730.vtc
@@ -8,6 +8,9 @@ server s1 {
 } -start
 
 varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
 } -start
 
 client c1 {
@@ -21,6 +24,9 @@ varnish v1 -vcl+backend {
 	sub vcl_recv {
 		return (pass);
 	}
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
 } 
 
 client c1 {
diff --git a/bin/varnishtest/tests/r01014.vtc b/bin/varnishtest/tests/r01014.vtc
index 53a4473..bb15012 100644
--- a/bin/varnishtest/tests/r01014.vtc
+++ b/bin/varnishtest/tests/r01014.vtc
@@ -5,9 +5,14 @@ server s1 {
 	txresp -nolen -hdr "Content-Encoding: gzip" -hdr "Content-Length:"
 } -start
 
-varnish v1 -vcl+backend { } -start
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
+} -start
 
 client c1 {
 	txreq
 	rxresp
+	expect resp.status == 503
 } -run
diff --git a/bin/varnishtest/tests/r01175.vtc b/bin/varnishtest/tests/r01175.vtc
index afb5642..e7c05bf 100644
--- a/bin/varnishtest/tests/r01175.vtc
+++ b/bin/varnishtest/tests/r01175.vtc
@@ -8,6 +8,7 @@ server s1 {
 varnish v1 -storage "-s test=malloc,1M" -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.storage = "test";
+		set beresp.do_stream = false;
 	}
 } -start
 



From martin at varnish-cache.org  Mon Aug 26 10:15:26 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Mon, 26 Aug 2013 12:15:26 +0200
Subject: [3.0] 0a7e6ca Fix duplicate Content-Length headers with pass and
 stream
Message-ID: <E1VDtpW-00028l-OW@project.varnish-software.com>

commit 0a7e6caa0c6bd93003d4733e96f5ea054ac84cbc
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Aug 26 12:14:05 2013 +0200

    Fix duplicate Content-Length headers with pass and stream
    
    Fixes: #1272
    
    Patch by: ehocdet

diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index f22d48a..160dda7 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -352,9 +352,11 @@ RES_StreamStart(struct sess *sp)
 		http_Unset(sp->wrk->resp, H_Content_Encoding);
 
 	if (!(sp->wrk->res_mode & RES_CHUNKED) &&
-	    sp->wrk->h_content_length != NULL)
+	    sp->wrk->h_content_length != NULL) {
+		http_Unset(sp->wrk->resp, H_Content_Length);
 		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
 		    "Content-Length: %s", sp->wrk->h_content_length);
+	}
 
 	sp->wrk->acct_tmp.hdrbytes +=
 	    http_Write(sp->wrk, sp->wrk->resp, 1);



From phk at varnish-cache.org  Mon Aug 26 10:42:02 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 26 Aug 2013 12:42:02 +0200
Subject: [master] 94a2445 Move back into cnt_stream{}
Message-ID: <E1VDuFG-0006CQ-UZ@project.varnish-software.com>

commit 94a24450e7ad97fa1c183a343165e78a00221d1a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 26 10:41:46 2013 +0000

    Move back into cnt_stream{}

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 396ec18..c50a53a 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -99,12 +99,10 @@ cnt_stream(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AN(bo->do_stream);
 
-	INCOMPL();
-
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 
-	req->res_mode = 0;
+	// req->res_mode = 0;
 
 	AZ(bo->do_esi);
 
@@ -225,7 +223,6 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		wrk->handling = VCL_RET_DELIVER;
 
 	if (req->busyobj != NULL) {
-		VBO_waitstate(req->busyobj, BOS_FINISHED);
 		/* Don't stream if already finished */
 		if (req->busyobj->state != BOS_FINISHED) {
 			AN(req->busyobj->do_stream);
@@ -394,7 +391,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 
 	assert (bo->state >= BOS_FETCHING);
 	req->err_code = bo->err_code;
-	req->obj = bo->fetch_obj;
+	req->obj = bo->fetch_obj;			// XXX: recnt ?
 	if (bo->state == BOS_FINISHED)
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 	assert(WRW_IsReleased(wrk));



From phk at varnish-cache.org  Mon Aug 26 11:38:27 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 26 Aug 2013 13:38:27 +0200
Subject: [master] 98ccaa9 Retire saint-mode.
Message-ID: <E1VDv7r-0006nB-3E@project.varnish-software.com>

commit 98ccaa9541b069359473c4126d004c9cea0c52c9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 26 11:38:08 2013 +0000

    Retire saint-mode.
    
    If there is a need, we'll bring it back as a VMOD.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 0e3d7d8..b4037c4 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -530,7 +530,6 @@ struct busyobj {
 	struct http		*beresp;
 	struct objcore		*fetch_objcore;
 	struct object		*fetch_obj;
-	uint8_t			digest[DIGEST_LEN];
 	struct exp		exp;
 	struct http_conn	htc;
 
@@ -759,7 +758,7 @@ void VBE_DiscardHealth(const struct director *vdi);
 
 
 struct vbc *VDI_GetFd(const struct director *, struct busyobj *);
-int VDI_Healthy(const struct director *, const uint8_t *digest);
+int VDI_Healthy(const struct director *);
 void VDI_CloseFd(struct vbc **vbp);
 void VDI_RecycleFd(struct vbc **vbp);
 void VDI_AddHostHeader(struct http *to, const struct vbc *vbc);
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index e802446..092f43f 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -42,7 +42,6 @@
 #include "cache_backend.h"
 #include "vrt.h"
 #include "vtcp.h"
-#include "vtim.h"
 
 static struct mempool	*vbcpool;
 
@@ -221,35 +220,6 @@ vbe_NewConn(void)
 }
 
 /*--------------------------------------------------------------------
- * Add backend trouble item
- */
-
-void
-VBE_AddTrouble(const struct busyobj *bo, double expires)
-{
-	struct trouble *tp;
-	struct vbc *vbc;
-	struct backend *be;
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	vbc = bo->vbc;
-	if (vbc == NULL)
-		return;
-	CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC);
-	be = vbc->backend;
-	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
-	ALLOC_OBJ(tp, TROUBLE_MAGIC);
-	if (tp == NULL)
-		return;
-	memcpy(tp->digest, bo->digest, sizeof tp->digest);
-	tp->timeout = expires;
-	Lck_Lock(&vbc->backend->mtx);
-	VTAILQ_INSERT_HEAD(&be->troublelist, tp, list);
-	be->n_trouble++;
-	Lck_Unlock(&vbc->backend->mtx);
-}
-
-/*--------------------------------------------------------------------
  * It evaluates if a backend is healthy _for_a_specific_object_.
  * That means that it relies on req->objcore->objhead. This is mainly for
  * saint-mode, but also takes backend->healthy into account. If
@@ -261,17 +231,10 @@ VBE_AddTrouble(const struct busyobj *bo, double expires)
  */
 
 static unsigned int
-vbe_Healthy(const struct vdi_simple *vs, const uint8_t *digest)
+vbe_Healthy(const struct vdi_simple *vs)
 {
-	struct trouble *tr;
-	struct trouble *tr2;
-	unsigned retval;
-	unsigned int threshold;
 	struct backend *backend;
-	VTAILQ_HEAD(, trouble)  troublelist;
-	double now;
 
-	AN(digest);
 	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
 	backend = vs->backend;
 	CHECK_OBJ_NOTNULL(backend, BACKEND_MAGIC);
@@ -282,48 +245,7 @@ vbe_Healthy(const struct vdi_simple *vs, const uint8_t *digest)
 	if (backend->admin_health == ah_sick)
 		return (0);
 
-	/* VRT/VCC sets threshold to UINT_MAX to mark that it's not
-	 * specified by VCL (thus use param).
-	 */
-	threshold = vs->vrt->saintmode_threshold;
-	if (threshold == UINT_MAX)
-		threshold = cache_param->saintmode_threshold;
-
-	if (backend->admin_health == ah_healthy)
-		threshold = UINT_MAX;
-
-	/* Saintmode is disabled, or list is empty */
-	if (threshold == 0 || backend->n_trouble == 0)
-		return (1);
-
-	now = VTIM_real();
-
-	retval = 1;
-	VTAILQ_INIT(&troublelist);
-	Lck_Lock(&backend->mtx);
-	VTAILQ_FOREACH_SAFE(tr, &backend->troublelist, list, tr2) {
-		CHECK_OBJ_NOTNULL(tr, TROUBLE_MAGIC);
-
-		if (tr->timeout < now) {
-			VTAILQ_REMOVE(&backend->troublelist, tr, list);
-			VTAILQ_INSERT_HEAD(&troublelist, tr, list);
-			backend->n_trouble--;
-			continue;
-		}
-
-		if (!memcmp(tr->digest, digest, sizeof tr->digest)) {
-			retval = 0;
-			break;
-		}
-	}
-	if (threshold <= backend->n_trouble)
-		retval = 0;
-	Lck_Unlock(&backend->mtx);
-
-	VTAILQ_FOREACH_SAFE(tr, &troublelist, list, tr2)
-		FREE_OBJ(tr);
-
-	return (retval);
+	return (1);
 }
 
 /*--------------------------------------------------------------------
@@ -379,7 +301,7 @@ vbe_GetVbe(struct busyobj *bo, struct vdi_simple *vs)
 		VBE_ReleaseConn(vc);
 	}
 
-	if (!vbe_Healthy(vs, bo->digest)) {
+	if (!vbe_Healthy(vs)) {
 		VSC_C_main->backend_unhealthy++;
 		return (NULL);
 	}
@@ -469,13 +391,13 @@ vdi_simple_getfd(const struct director *d, struct busyobj *bo)
 }
 
 static unsigned
-vdi_simple_healthy(const struct director *d, const uint8_t *digest)
+vdi_simple_healthy(const struct director *d)
 {
 	struct vdi_simple *vs;
 
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC);
-	return (vbe_Healthy(vs, digest));
+	return (vbe_Healthy(vs));
 }
 
 static void
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 8103af5..d186ecc 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -77,7 +77,7 @@ struct vrt_backend_probe;
 
 typedef struct vbc *vdi_getfd_f(const struct director *, struct busyobj *);
 typedef void vdi_fini_f(const struct director *);
-typedef unsigned vdi_healthy(const struct director *, const uint8_t *digest);
+typedef unsigned vdi_healthy(const struct director *);
 
 struct director {
 	unsigned		magic;
@@ -91,18 +91,6 @@ struct director {
 };
 
 /*--------------------------------------------------------------------
- * List of objectheads that have recently been rejected by VCL.
- */
-
-struct trouble {
-	unsigned		magic;
-#define TROUBLE_MAGIC		0x4211ab21
-	uint8_t			digest[DIGEST_LEN];
-	double			timeout;
-	VTAILQ_ENTRY(trouble)	list;
-};
-
-/*--------------------------------------------------------------------
  * An instance of a backend from a VCL program.
  */
 
@@ -137,9 +125,6 @@ struct backend {
 	unsigned		healthy;
 	enum admin_health	admin_health;
 
-	unsigned		n_trouble;
-	VTAILQ_HEAD(, trouble)	troublelist;
-
 	struct VSC_C_vbe	*vsc;
 };
 
@@ -166,7 +151,6 @@ struct vbc {
 
 /* cache_backend.c */
 void VBE_ReleaseConn(struct vbc *vc);
-void VBE_AddTrouble(const struct busyobj *, double expires);
 
 /* cache_backend_cfg.c */
 void VBE_DropRefConn(struct backend *);
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index c3e7c8c..24ed727 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -209,8 +209,6 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 
 	VTAILQ_INIT(&b->connlist);
 
-	VTAILQ_INIT(&b->troublelist);
-
 	/*
 	 * This backend may live longer than the VCL that instantiated it
 	 * so we cannot simply reference the VCL's copy of things.
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 2a4d318..63ad963 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -144,8 +144,6 @@ VBO_GetBusyObj(struct worker *wrk, struct req *req)
 
 	WS_Init(bo->ws, "bo", p, bo->end - p);
 
-	memcpy(bo->digest, req->digest, sizeof bo->digest);
-
 	bo->do_stream = 1;
 
 	bo->director = req->director;
diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c
index 366b864..5d7939a 100644
--- a/bin/varnishd/cache/cache_dir.c
+++ b/bin/varnishd/cache/cache_dir.c
@@ -123,9 +123,9 @@ VDI_GetFd(const struct director *d, struct busyobj *bo)
  */
 
 int
-VDI_Healthy(const struct director *d, const uint8_t *digest)
+VDI_Healthy(const struct director *d)
 {
 
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	return (d->healthy(d, digest));
+	return (d->healthy(d));
 }
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c50a53a..3dca4e4 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -102,8 +102,6 @@ cnt_stream(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 
-	// req->res_mode = 0;
-
 	AZ(bo->do_esi);
 
 	if (wrk->handling == VCL_RET_RESTART) {
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 3c86a6b..de2cf78 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -380,20 +380,6 @@ VRT_BOOL_string(unsigned val)
 	return (val ? "true" : "false");
 }
 
-/*--------------------------------------------------------------------
- * Add an objecthead to the saintmode list for the (hopefully) relevant
- * backend.
- */
-
-void
-VRT_l_beresp_saintmode(const struct vrt_ctx *ctx, double a)
-{
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	if (a > 0.)
-		VBE_AddTrouble(ctx->bo, a + VTIM_real());
-}
-
 /*--------------------------------------------------------------------*/
 
 void
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 763e6ff..1ed5df0 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -336,7 +336,7 @@ VRT_r_req_backend_healthy(const struct vrt_ctx *ctx)
 	if (ctx->req->director == NULL)
 		return (0);
 	CHECK_OBJ_NOTNULL(ctx->req->director, DIRECTOR_MAGIC);
-	return (VDI_Healthy(ctx->req->director, ctx->req->digest));
+	return (VDI_Healthy(ctx->req->director));
 }
 
 /*--------------------------------------------------------------------*/
@@ -368,7 +368,7 @@ VRT_r_bereq_backend_healthy(const struct vrt_ctx *ctx)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->bo->director, DIRECTOR_MAGIC);
-	return (VDI_Healthy(ctx->bo->director, ctx->bo->digest));
+	return (VDI_Healthy(ctx->bo->director));
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishtest/tests/c00029.vtc b/bin/varnishtest/tests/c00029.vtc
deleted file mode 100644
index de1a634..0000000
--- a/bin/varnishtest/tests/c00029.vtc
+++ /dev/null
@@ -1,114 +0,0 @@
-varnishtest "Test that saintmode_threshold correctly marks a backend as sick"
-
-feature OldSaintMode
-
-server s1 {
-	rxreq
-	txresp
-
-	rxreq
-	txresp
-
-	rxreq
-	txresp
-
-	rxreq
-	txresp
-
-	rxreq
-	txresp -hdr "X-Saint: yes"
-
-	accept
-
-	rxreq
-	txresp -hdr "X-Saint: yes"
-
-	accept
-
-	rxreq
-	txresp -hdr "X-Saint: yes"
-
-	accept
-
-	rxreq
-	txresp -hdr "X-Saint: yes"
-} -start
-
-varnish v1 -arg "-p saintmode_threshold=2" -vcl+backend {
-	sub vcl_recv {
-		set req.grace = 1h;
-	}
-
-	sub vcl_backend_response {
-		if (beresp.http.X-Saint == "yes") {
-			set beresp.saintmode = 20s;
-			set beresp.uncacheable = true;
-			set beresp.ttl = 0s;
-			return(deliver);
-		}
-		set beresp.grace = 1h;
-		set beresp.ttl = 1s;
-	}
-	sub vcl_deliver {
-		if (resp.http.X-saint == "yes") {
-			return (restart);
-		}
-		set resp.http.X-Restarts = req.restarts;
-	}
- } -start
-
-client c1 {
-        txreq -url "/one"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "0"
-
-        txreq -url "/two"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "0"
-
-        txreq -url "/three"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "0"
-
-        txreq -url "/four"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "0"
-
-	delay 2
-
-        txreq -url "/one"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "1"
-
-        txreq -url "/two"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "1"
-
-	# The saintmode_threshold should kick in now
-	# so expect no more restarts
-
-        txreq -url "/three"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "0"
-
-        txreq -url "/four"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "0"
-
-} -run
diff --git a/bin/varnishtest/tests/c00030.vtc b/bin/varnishtest/tests/c00030.vtc
deleted file mode 100644
index fccc4e5..0000000
--- a/bin/varnishtest/tests/c00030.vtc
+++ /dev/null
@@ -1,118 +0,0 @@
-varnishtest "Test that saintmode_threshold in VCL"
-
-feature OldSaintMode
-
-server s1 {
-	rxreq
-	txresp
-
-	rxreq
-	txresp
-
-	rxreq
-	txresp
-
-	rxreq
-	txresp
-
-	rxreq
-	txresp -hdr "X-Saint: yes"
-
-	accept
-
-	rxreq
-	txresp -hdr "X-Saint: yes"
-
-	accept
-
-	rxreq
-	txresp -hdr "X-Saint: yes"
-
-	accept
-
-	rxreq
-	txresp -hdr "X-Saint: yes"
-} -start
-
-varnish v1 -arg "-p saintmode_threshold=10" -vcl {
-	backend foo {
-		.host = "${s1_addr}";
-		.port = "${s1_port}";
-		.saintmode_threshold = 2;
-	}
-
-	sub vcl_recv {
-		set req.backend = foo;
-		set req.grace = 1h;
-	}
-
-	sub vcl_backend_response {
-		if (beresp.http.X-Saint == "yes") {
-			set beresp.saintmode = 20s;
-			set beresp.uncacheable = true;
-			set beresp.ttl = 0s;
-			return(deliver);
-		}
-		set beresp.grace = 1h;
-		set beresp.ttl = 1s;
-	}
-	sub vcl_deliver {
-		if (resp.http.X-saint == "yes") {
-			return (restart);
-		}
-		set resp.http.X-Restarts = req.restarts;
-	}
- } -start
-
-client c1 {
-        txreq -url "/one"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "0"
-
-        txreq -url "/two"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "0"
-
-        txreq -url "/three"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "0"
-
-        txreq -url "/four"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "0"
-
-	delay 2
-
-        txreq -url "/one"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "1"
-
-        txreq -url "/two"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "1"
-
-        txreq -url "/three"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "0"
-
-        txreq -url "/four"
-        rxresp
-        expect resp.status == 200
-        expect resp.http.X-Saint != "yes"
-	expect resp.http.X-Restarts == "0"
-
-} -run
diff --git a/bin/varnishtest/tests/r01091.vtc b/bin/varnishtest/tests/r01091.vtc
deleted file mode 100644
index f50274f..0000000
--- a/bin/varnishtest/tests/r01091.vtc
+++ /dev/null
@@ -1,44 +0,0 @@
-varnishtest "Test fallback director with saint mode"
-
-server s1 {
-        rxreq
-        txresp -hdr "Foo: 1"
-} -start
-
-server s2 {
-        rxreq
-        txresp -hdr "Foo: 2" -bodylen 1
-} -start
-
-varnish v1 -vcl+backend {
-	import directors from "${topbuild}/lib/libvmod_directors/.libs/libvmod_directors.so" ;
-
-	sub vcl_init {
-		new f1 = directors.fallback();
-		f1.add_backend(s1);
-		f1.add_backend(s2);
-	}
-
-        sub vcl_recv {
-                set req.backend = f1.backend();
-        }
-        sub vcl_backend_response {
-		if (beresp.http.foo == "1") {
-                        set beresp.uncacheable = true;
-                        set beresp.saintmode = 1h;
-                        set beresp.status = 606;
-                        return(deliver);
-                }
-        }
-	sub vcl_deliver {
-		if (resp.status == 606) {
-                        return(restart);
-		}
-	}
-} -start
-
-client c1 {
-        txreq
-        rxresp
-        expect resp.http.foo == "2"
-} -run
diff --git a/bin/varnishtest/tests/s00003.vtc b/bin/varnishtest/tests/s00003.vtc
deleted file mode 100644
index 02ceb63..0000000
--- a/bin/varnishtest/tests/s00003.vtc
+++ /dev/null
@@ -1,64 +0,0 @@
-varnishtest "Check saint mode with sick pages"
-
-feature OldSaintMode
-
-server s1 {
-	timeout 10
-
-	rxreq
-	expect req.url == "/"
-	txresp -status 200 -hdr "foo: 1"
-	rxreq
-	expect req.url == "/"
-	txresp -status 200 -hdr "foo: 2"
-	accept
-	rxreq
-	expect req.url == "/"
-	txresp -status 200 -hdr "foo: 3"
-} -start
-
-varnish v1 -vcl+backend {
-	sub vcl_backend_response {
-		set beresp.ttl = 1s;
-		set beresp.grace = 10m;
-		if (beresp.http.foo == "2")
-		{
-			set beresp.saintmode = 2s;
-			set beresp.status = 606;
-			set beresp.uncacheable = true;
-			set beresp.ttl = 0s;
-			return (deliver);
-		}
-		return(deliver);
-	}
-	sub vcl_deliver {
-		if (resp.status == 606) {
-			return (restart);
-		}
-	}
-} -start
-
-client c1 {
-	txreq -url "/"
-	rxresp
-	expect resp.status == 200
-	expect resp.http.foo == 1
-} -run
-
-delay 2
-
-client c2 {
-	txreq -url "/"
-	rxresp
-	expect resp.status == 200
-	expect resp.http.foo == 1
-} -run
-
-delay 2
-
-client c3 {
-	txreq -url "/"
-	rxresp
-	expect resp.status == 200
-	expect resp.http.foo == 3
-} -run
diff --git a/include/vrt.h b/include/vrt.h
index 701e131..1d079fc 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -133,7 +133,6 @@ struct vrt_backend {
 	double				first_byte_timeout;
 	double				between_bytes_timeout;
 	unsigned			max_connections;
-	unsigned			saintmode_threshold;
 	const struct vrt_backend_probe	*probe;
 };
 
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 8d57ba6..0125d73 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -282,11 +282,6 @@ sp_variables = [
 		( 'backend_response',),
 		( 'backend_response',),
 	),
-	('beresp.saintmode',
-		'DURATION',
-		( ),
-		( 'backend_response',),
-	),
 	('beresp.status',
 		'INT',
 		( 'backend_response',),
diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c
index 2cb3469..b013400 100644
--- a/lib/libvcl/vcc_backend.c
+++ b/lib/libvcl/vcc_backend.c
@@ -50,7 +50,6 @@
 
 #include "config.h"
 
-#include <limits.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -300,7 +299,6 @@ vcc_ParseHostDef(struct vcc *tl, int serial, const char *vgcname)
 	struct token *t_host = NULL;
 	struct token *t_port = NULL;
 	struct token *t_hosthdr = NULL;
-	unsigned saint = UINT_MAX;
 	struct fld_spec *fs;
 	struct vsb *vsb;
 	unsigned u;
@@ -317,7 +315,6 @@ vcc_ParseHostDef(struct vcc *tl, int serial, const char *vgcname)
 	    "?between_bytes_timeout",
 	    "?probe",
 	    "?max_connections",
-	    "?saintmode_threshold",
 	    NULL);
 
 	SkipToken(tl, '{');
@@ -391,21 +388,6 @@ vcc_ParseHostDef(struct vcc *tl, int serial, const char *vgcname)
 			ERRCHK(tl);
 			SkipToken(tl, ';');
 			Fb(tl, 0, "\t.max_connections = %u,\n", u);
-		} else if (vcc_IdIs(t_field, "saintmode_threshold")) {
-			u = vcc_UintVal(tl);
-			/* UINT_MAX == magic number to mark as unset, so
-			 * not allowed here.
-			 */
-			if (u == UINT_MAX) {
-				VSB_printf(tl->sb,
-				    "Value outside allowed range: ");
-				vcc_ErrToken(tl, tl->t);
-				VSB_printf(tl->sb, " at\n");
-				vcc_ErrWhere(tl, tl->t);
-			}
-			ERRCHK(tl);
-			saint = u;
-			SkipToken(tl, ';');
 		} else if (vcc_IdIs(t_field, "probe") && tl->t->tok == '{') {
 			Fb(tl, 0, "\t.probe = &vgc_probe__%d,\n", tl->nprobe);
 			vcc_ParseProbeSpec(tl);
@@ -452,8 +434,6 @@ vcc_ParseHostDef(struct vcc *tl, int serial, const char *vgcname)
 		EncToken(tl->fb, t_host);
 	Fb(tl, 0, ",\n");
 
-	Fb(tl, 0, "\t.saintmode_threshold = %d,\n",saint);
-
 	/* Close the struct */
 	Fb(tl, 0, "};\n");
 
diff --git a/lib/libvmod_directors/fall_back.c b/lib/libvmod_directors/fall_back.c
index 7c430ac..4c86b7e 100644
--- a/lib/libvmod_directors/fall_back.c
+++ b/lib/libvmod_directors/fall_back.c
@@ -45,12 +45,12 @@ struct vmod_directors_fallback {
 };
 
 static unsigned __match_proto__(vdi_healthy)
-vmod_rr_healthy(const struct director *dir, const uint8_t *digest)
+vmod_rr_healthy(const struct director *dir)
 {
 	struct vmod_directors_fallback *rr;
 
 	CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_FALLBACK_MAGIC);
-	return (vdir_any_healthy(rr->vd, digest));
+	return (vdir_any_healthy(rr->vd));
 }
 
 static struct vbc * __match_proto__(vdi_getfd_f)
@@ -65,7 +65,7 @@ vmod_rr_getfd(const struct director *dir, struct busyobj *bo)
 	for (u = 0; u < rr->vd->n_backend; u++) {
 		be = rr->vd->backend[u];
 		CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC);
-		if (be->healthy(be, bo->digest))
+		if (be->healthy(be))
 			break;
 	}
 	vdir_unlock(rr->vd);
diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c
index 290909f..35c8ed7 100644
--- a/lib/libvmod_directors/hash.c
+++ b/lib/libvmod_directors/hash.c
@@ -119,6 +119,6 @@ vmod_hash_backend(const struct vrt_ctx *ctx, struct vmod_directors_hash *rr,
 	r = vbe32dec(sha256);
 	r = scalbn(r, -32);
 	assert(r >= 0 && r <= 1.0);
-	be = vdir_pick_be(rr->vd, ctx->bo, r, rr->nloops);
+	be = vdir_pick_be(rr->vd, r, rr->nloops);
 	return (be);
 }
diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c
index 6d2be1e..a31ec1b 100644
--- a/lib/libvmod_directors/random.c
+++ b/lib/libvmod_directors/random.c
@@ -50,12 +50,12 @@ struct vmod_directors_random {
 };
 
 static unsigned __match_proto__(vdi_healthy)
-vmod_rr_healthy(const struct director *dir, const uint8_t *digest)
+vmod_rr_healthy(const struct director *dir)
 {
 	struct vmod_directors_random *rr;
 
 	CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_RANDOM_MAGIC);
-	return (vdir_any_healthy(rr->vd, digest));
+	return (vdir_any_healthy(rr->vd));
 }
 
 static struct vbc * __match_proto__(vdi_getfd_f)
@@ -68,7 +68,7 @@ vmod_rr_getfd(const struct director *dir, struct busyobj *bo)
 	CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_RANDOM_MAGIC);
 	r = scalbn(random(), -31);
 	assert(r >= 0 && r < 1.0);
-	be = vdir_pick_be(rr->vd, bo, r, rr->nloops);
+	be = vdir_pick_be(rr->vd, r, rr->nloops);
 	if (be == NULL)
 		return (NULL);
 	return (be->getfd(be, bo));
diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c
index fe0e528..f4b5eb6 100644
--- a/lib/libvmod_directors/round_robin.c
+++ b/lib/libvmod_directors/round_robin.c
@@ -46,12 +46,12 @@ struct vmod_directors_round_robin {
 };
 
 static unsigned __match_proto__(vdi_healthy)
-vmod_rr_healthy(const struct director *dir, const uint8_t *digest)
+vmod_rr_healthy(const struct director *dir)
 {
 	struct vmod_directors_round_robin *rr;
 
 	CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC);
-	return (vdir_any_healthy(rr->vd, digest));
+	return (vdir_any_healthy(rr->vd));
 }
 
 static struct vbc * __match_proto__(vdi_getfd_f)
@@ -68,7 +68,7 @@ vmod_rr_getfd(const struct director *dir, struct busyobj *bo)
 		be = rr->vd->backend[rr->nxt];
 		rr->nxt++;
 		CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC);
-		if (be->healthy(be, bo->digest))
+		if (be->healthy(be))
 			break;
 	}
 	vdir_unlock(rr->vd);
diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c
index 14d523d..8108a29 100644
--- a/lib/libvmod_directors/vdir.c
+++ b/lib/libvmod_directors/vdir.c
@@ -128,7 +128,7 @@ vdir_add_backend(struct vdir *vd, VCL_BACKEND be, double weight)
 }
 
 unsigned
-vdir_any_healthy(struct vdir *vd, const uint8_t *digest)
+vdir_any_healthy(struct vdir *vd)
 {
 	unsigned retval = 0;
 	VCL_BACKEND be;
@@ -139,7 +139,7 @@ vdir_any_healthy(struct vdir *vd, const uint8_t *digest)
 	for (u = 0; u < vd->n_backend; u++) {
 		be = vd->backend[u];
 		CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC);
-		if (be->healthy(be, digest)) {
+		if (be->healthy(be)) {
 			retval = 1;
 			break;
 		}
@@ -169,8 +169,7 @@ vdir_pick_by_weight(const struct vdir *vd, double w,
 }
 
 VCL_BACKEND
-vdir_pick_be(struct vdir *vd, const struct busyobj *bo, double w,
-    unsigned nloops)
+vdir_pick_be(struct vdir *vd, double w, unsigned nloops)
 {
 	struct vbitmap *vbm = NULL;
 	unsigned u, v, l;
@@ -186,7 +185,7 @@ vdir_pick_be(struct vdir *vd, const struct busyobj *bo, double w,
 		u = vdir_pick_by_weight(vd, w * tw, vbm);
 		be = vd->backend[u];
 		CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC);
-		if (be->healthy(be, bo->digest))
+		if (be->healthy(be))
 			break;
 		if (l == 0) {
 			vbm = vd->vbm;
diff --git a/lib/libvmod_directors/vdir.h b/lib/libvmod_directors/vdir.h
index fa7a458..39ad9a1 100644
--- a/lib/libvmod_directors/vdir.h
+++ b/lib/libvmod_directors/vdir.h
@@ -47,6 +47,5 @@ void vdir_delete(struct vdir **vdp);
 void vdir_lock(struct vdir *vd);
 void vdir_unlock(struct vdir *vd);
 unsigned vdir_add_backend(struct vdir *vd, VCL_BACKEND be, double weight);
-unsigned vdir_any_healthy(struct vdir *vd, const uint8_t *digest);
-VCL_BACKEND vdir_pick_be(struct vdir *vd, const struct busyobj *, double w,
-    unsigned nloops);
+unsigned vdir_any_healthy(struct vdir *vd);
+VCL_BACKEND vdir_pick_be(struct vdir *vd, double w, unsigned nloops);



From phk at varnish-cache.org  Tue Aug 27 07:48:49 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 27 Aug 2013 09:48:49 +0200
Subject: [master] 8489546 Introduce the object iterator functions,
 which will hide the nasty synchronization bits of streaming.
Message-ID: <E1VEE1B-0004Ix-Mu@project.varnish-software.com>

commit 8489546fceb56753ca44b95cca03a558088a62f5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 27 07:48:17 2013 +0000

    Introduce the object iterator functions, which will hide the
    nasty synchronization bits of streaming.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index b1f54ed..4759414 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -34,6 +34,7 @@ varnishd_SOURCES = \
 	cache/cache_lck.c \
 	cache/cache_main.c \
 	cache/cache_mempool.c \
+	cache/cache_obj.c \
 	cache/cache_panic.c \
 	cache/cache_pipe.c \
 	cache/cache_pool.c \
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b4037c4..1626351 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -129,6 +129,7 @@ struct vsb;
 struct waitinglist;
 struct worker;
 struct wrw;
+struct objiter;
 
 #define DIGEST_LEN		32
 
@@ -983,6 +984,11 @@ void MPL_Destroy(struct mempool **mpp);
 void *MPL_Get(struct mempool *mpl, unsigned *size);
 void MPL_Free(struct mempool *mpl, void *item);
 
+/* cache_obj.c */
+struct objiter *ObjIterBegin(struct object *);
+int ObjIter(struct objiter *, void **, ssize_t *);
+void ObjIterEnd(struct objiter **);
+
 /* cache_panic.c */
 void PAN_Init(void);
 const char *body_status_2str(enum body_status e);
diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c
new file mode 100644
index 0000000..65f443b
--- /dev/null
+++ b/bin/varnishd/cache/cache_obj.c
@@ -0,0 +1,80 @@
+/*-
+ * Copyright (c) 2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+
+#include "cache.h"
+
+struct objiter {
+	unsigned			magic;
+#define OBJITER_MAGIC			0x745fb151
+	struct object			*obj;
+	struct storage			*st;
+};
+
+struct objiter *
+ObjIterBegin(struct object *obj)
+{
+	struct objiter *oi;
+
+	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+	ALLOC_OBJ(oi, OBJITER_MAGIC);
+	if (oi != NULL)
+		oi->obj = obj;
+	return (oi);
+}
+
+int
+ObjIter(struct objiter *oi, void **p, ssize_t *l)
+{
+
+	CHECK_OBJ_NOTNULL(oi, OBJITER_MAGIC);
+	AN(p);
+	AN(l);
+
+	if (oi->st == NULL)
+		oi->st = VTAILQ_FIRST(&oi->obj->store);
+	else
+		oi->st = VTAILQ_NEXT(oi->st, list);
+	if (oi->st != NULL) {
+		*p = oi->st->ptr;
+		*l = oi->st->len;
+		return (1);
+	}
+	return (0);
+}
+
+void
+ObjIterEnd(struct objiter **oi)
+{
+
+	CHECK_OBJ_NOTNULL((*oi), OBJITER_MAGIC);
+	FREE_OBJ((*oi));
+}
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index b49f405..5e9f997 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -161,9 +161,11 @@ RES_BuildHttp(struct req *req)
 static void
 res_WriteGunzipObj(struct req *req)
 {
-	struct storage *st;
 	unsigned u = 0;
 	struct vgz *vg;
+	struct objiter *oi;
+	void *ptr;
+	ssize_t len;
 	int i;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -171,18 +173,21 @@ res_WriteGunzipObj(struct req *req)
 	vg = VGZ_NewUngzip(req->vsl, "U D -");
 	AZ(VGZ_WrwInit(vg));
 
-	VTAILQ_FOREACH(st, &req->obj->store, list) {
+	oi = ObjIterBegin(req->obj);
+	XXXAN(oi);
+
+	while (ObjIter(oi, &ptr, &len)) {
 		CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
-		u += st->len;
+		u += len;
 
-		i = VGZ_WrwGunzip(req, vg, st->ptr, st->len);
+		i = VGZ_WrwGunzip(req, vg, ptr, len);
 		/* XXX: error check */
 		(void)i;
 	}
 	VGZ_WrwFlush(req, vg);
 	(void)VGZ_Destroy(&vg);
 	assert(u == req->obj->len);
+	ObjIterEnd(&oi);
 }
 
 /*--------------------------------------------------------------------*/
@@ -191,39 +196,41 @@ static void
 res_WriteDirObj(struct req *req, ssize_t low, ssize_t high)
 {
 	ssize_t u = 0;
-	size_t ptr, off, len;
-	struct storage *st;
+	ssize_t idx, skip, len;
+	struct objiter *oi;
+	void *ptr;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	ptr = 0;
-	VTAILQ_FOREACH(st, &req->obj->store, list) {
-		CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
-		u += st->len;
-		len = st->len;
-		off = 0;
-		if (ptr + len <= low) {
+	oi = ObjIterBegin(req->obj);
+	XXXAN(oi);
+
+	idx = 0;
+	while (ObjIter(oi, &ptr, &len)) {
+		u += len;
+		skip = 0;
+		if (idx + len <= low) {
 			/* This segment is too early */
-			ptr += len;
+			idx += len;
 			continue;
 		}
-		if (ptr < low) {
+		if (idx < low) {
 			/* Chop front of segment off */
-			off += (low - ptr);
-			len -= (low - ptr);
-			ptr += (low - ptr);
+			skip += (low - idx);
+			len -= (low - idx);
+			idx += (low - idx);
 		}
-		if (ptr + len > high)
+		if (idx + len > high)
 			/* Chop tail of segment off */
-			len = 1 + high - ptr;
+			len = 1 + high - idx;
 
-		ptr += len;
+		idx += len;
 
 		req->acct_req.bodybytes += len;
-		(void)WRW_Write(req->wrk, st->ptr + off, len);
+		(void)WRW_Write(req->wrk, (char*)ptr + skip, len);
 	}
 	assert(u == req->obj->len);
+	ObjIterEnd(&oi);
 }
 
 /*--------------------------------------------------------------------



From phk at varnish-cache.org  Wed Aug 28 06:39:46 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 28 Aug 2013 08:39:46 +0200
Subject: [master] 72a19f2 Isolate req->busyobj in the state engine.
Message-ID: <E1VEZPu-00052Y-Sn@project.varnish-software.com>

commit 72a19f2fcfbc403c4d79b976d8f1e2eb5cf1a563
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 28 06:38:41 2013 +0000

    Isolate req->busyobj in the state engine.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 1626351..7f1b82f 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -996,7 +996,7 @@ const char *reqbody_status_2str(enum req_body_state_e e);
 const char *sess_close_2str(enum sess_close sc, int want_desc);
 
 /* cache_pipe.c */
-void PipeRequest(struct req *req);
+void PipeRequest(struct req *req, struct busyobj *bo);
 
 /* cache_pool.c */
 void Pool_Init(void);
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index e76151b..b14c8af 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -60,19 +60,17 @@ rdf(int fd0, int fd1)
 }
 
 void
-PipeRequest(struct req *req)
+PipeRequest(struct req *req, struct busyobj *bo)
 {
 	struct vbc *vc;
 	struct worker *wrk;
 	struct pollfd fds[2];
-	struct busyobj *bo;
 	int i;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC);
 	wrk = req->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	bo = req->busyobj;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	vc = VDI_GetFd(NULL, bo);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 3dca4e4..0d59693 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -685,23 +685,22 @@ cnt_pipe(struct worker *wrk, struct req *req)
 	AZ(req->busyobj);
 
 	req->acct_req.pipe++;
-	req->busyobj = VBO_GetBusyObj(wrk, req);
-	bo = req->busyobj;
+	bo = VBO_GetBusyObj(wrk, req);
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
 	http_FilterReq(bo->bereq, req->http, 0);	// XXX: 0 ?
 	http_PrintfHeader(bo->bereq,
 	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
 
-	VCL_pipe_method(req->vcl, wrk, req, NULL, req->http->ws);
+	VCL_pipe_method(req->vcl, wrk, req, bo, req->http->ws);
 
 	if (wrk->handling == VCL_RET_ERROR)
 		INCOMPL();
 	assert(wrk->handling == VCL_RET_PIPE);
 
-	PipeRequest(req);
+	PipeRequest(req, bo);
 	assert(WRW_IsReleased(wrk));
 	http_Teardown(bo->bereq);
-	VBO_DerefBusyObj(wrk, &req->busyobj);
+	VBO_DerefBusyObj(wrk, &bo);
 	return (REQ_FSM_DONE);
 }
 



From phk at varnish-cache.org  Wed Aug 28 11:54:40 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 28 Aug 2013 13:54:40 +0200
Subject: [master] bde1e57 Fix layout of SLT_TTL records from VCL
Message-ID: <E1VEeKe-00009a-T7@project.varnish-software.com>

commit bde1e57de1fbdf2ba41a77341c18b93a7a27359d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 28 11:54:20 2013 +0000

    Fix layout of SLT_TTL records from VCL

diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 1ed5df0..f72586e 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -464,13 +464,13 @@ VRT_r_##which##_##fld(const struct vrt_ctx *ctx)		\
 }
 
 static void
-vrt_wsp_exp(struct vsl_log *vsl, unsigned xid, double now, const struct exp *e)
+vrt_wsp_exp(struct vsl_log *vsl, double now, const struct exp *e)
 {
 	double dt;
 
 	dt = now - e->entered;
-	VSLb(vsl, SLT_TTL, "%u VCL %.0f %.0f %.0f %.0f %.0f",
-	    xid, e->ttl - dt, e->grace, e->keep, now, e->age + dt);
+	VSLb(vsl, SLT_TTL, "VCL %.0f %.0f %.0f %.0f %.0f",
+	    e->ttl - dt, e->grace, e->keep, now, e->age + dt);
 }
 
 VRT_DO_EXP(req, ctx->req->exp, ttl, 0, )
@@ -479,27 +479,21 @@ VRT_DO_EXP(req, ctx->req->exp, keep, 0, )
 
 VRT_DO_EXP(obj, ctx->req->obj->exp, grace, 0,
    EXP_Rearm(ctx->req->obj);
-   vrt_wsp_exp(ctx->vsl, ctx->req->obj->vxid,
-	ctx->req->t_req, &ctx->req->obj->exp);)
+   vrt_wsp_exp(ctx->vsl, ctx->req->t_req, &ctx->req->obj->exp);)
 VRT_DO_EXP(obj, ctx->req->obj->exp, ttl,
    (ctx->req->t_req - ctx->req->obj->exp.entered),
    EXP_Rearm(ctx->req->obj);
-   vrt_wsp_exp(ctx->vsl, ctx->req->obj->vxid,
-	ctx->req->t_req, &ctx->req->obj->exp);)
+   vrt_wsp_exp(ctx->vsl, ctx->req->t_req, &ctx->req->obj->exp);)
 VRT_DO_EXP(obj, ctx->req->obj->exp, keep, 0,
    EXP_Rearm(ctx->req->obj);
-   vrt_wsp_exp(ctx->vsl, ctx->req->obj->vxid,
-	ctx->req->t_req, &ctx->req->obj->exp);)
+   vrt_wsp_exp(ctx->vsl, ctx->req->t_req, &ctx->req->obj->exp);)
 
 VRT_DO_EXP(beresp, ctx->bo->exp, grace, 0,
-   vrt_wsp_exp(ctx->vsl, ctx->vsl->wid & VSL_IDENTMASK,
-	ctx->bo->exp.entered, &ctx->bo->exp);)
+   vrt_wsp_exp(ctx->vsl, ctx->bo->exp.entered, &ctx->bo->exp);)
 VRT_DO_EXP(beresp, ctx->bo->exp, ttl, 0,
-   vrt_wsp_exp(ctx->vsl, ctx->vsl->wid & VSL_IDENTMASK,
-	ctx->bo->exp.entered, &ctx->bo->exp);)
+   vrt_wsp_exp(ctx->vsl, ctx->bo->exp.entered, &ctx->bo->exp);)
 VRT_DO_EXP(beresp, ctx->bo->exp, keep, 0,
-   vrt_wsp_exp(ctx->vsl, ctx->vsl->wid & VSL_IDENTMASK,
-	ctx->bo->exp.entered, &ctx->bo->exp);)
+   vrt_wsp_exp(ctx->vsl, ctx->bo->exp.entered, &ctx->bo->exp);)
 
 /*--------------------------------------------------------------------
  * req.xid



From phk at varnish-cache.org  Wed Aug 28 12:29:27 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 28 Aug 2013 14:29:27 +0200
Subject: [master] 68a4240 Add a "private" oh which we can hang pass and error
 objectcores from,
 so that we do not need to special-case locking all over the place.
Message-ID: <E1VEesJ-0006QG-DB@project.varnish-software.com>

commit 68a424092d589932c782c9514d340f1752f094a5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 28 12:28:43 2013 +0000

    Add a "private" oh which we can hang pass and error objectcores
    from, so that we do not need to special-case locking all over the
    place.
    
    Give the busyobj a real reference to the objcore it's fetching.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7f1b82f..4598ad3 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -423,6 +423,7 @@ struct objcore {
 #define OC_F_LRUDONTMOVE	(1<<4)
 #define OC_F_PRIV		(1<<5)		/* Stevedore private flag */
 #define OC_F_LURK		(3<<6)		/* Ban-lurker-color */
+#define OC_F_PRIVATE		(1<<8)
 	unsigned		timer_idx;
 	VTAILQ_ENTRY(objcore)	list;
 	VTAILQ_ENTRY(objcore)	lru_list;
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 63ad963..cb22c54 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -243,7 +243,7 @@ void
 VBO_setstate(struct busyobj *bo, enum busyobj_state_e next)
 {
 	Lck_Lock(&bo->mtx);
-	VSLb(bo->vsl, SLT_Debug, "XXX: %d -> %d", bo->state, next);
+	VSLb(bo->vsl, SLT_Debug, "XXX BOS: %d -> %d", bo->state, next);
 	assert(next > bo->state);
 	bo->state = next;
 	AZ(pthread_cond_signal(&bo->cond));
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index bde06c8..90d07fd 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -207,6 +207,8 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 
 	if (bo->do_esi)
 		bo->do_stream = 0;
+	if (bo->do_pass)
+		bo->fetch_objcore->flags |= OC_F_PASS;
 
 	if (wrk->handling == VCL_RET_DELIVER)
 		return (F_STP_FETCH);
@@ -357,7 +359,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	}
 	bo->stats = NULL;
 	if (obj == NULL) {
-		AZ(HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL));
+		(void)HSH_Deref(&wrk->stats, bo->fetch_objcore, NULL);
 		bo->fetch_objcore = NULL;
 		VDI_CloseFd(&bo->vbc);
 		VBO_setstate(bo, BOS_FAILED);
@@ -406,13 +408,14 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	assert(bo->refcount >= 1);
 
-	if (obj->objcore->objhead != NULL) {
+	if (!(bo->fetch_obj->objcore->flags & OC_F_PRIVATE)) {
 		EXP_Insert(obj);
 		AN(obj->objcore->ban);
-		AZ(obj->ws_o->overflow);
-		HSH_Unbusy(&wrk->stats, obj->objcore);
 	}
 
+	AZ(obj->ws_o->overflow);
+	HSH_Unbusy(&wrk->stats, obj->objcore);
+
 	if (bo->vfp == NULL)
 		bo->vfp = &VFP_nop;
 
@@ -423,9 +426,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	assert(bo->refcount >= 1);
 
-	if (obj->objcore->objhead != NULL)
-		HSH_Ref(obj->objcore);
-
 	if (bo->state == BOS_FAILED) {
 		/* handle early failures */
 		(void)HSH_Deref(&wrk->stats, NULL, &obj);
@@ -433,6 +433,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	}
 	VBO_setstate(bo, BOS_FINISHED);
 
+VSLb(bo->vsl, SLT_Debug, "YYY REF %d %d", bo->refcount, bo->fetch_obj->objcore->refcnt);
 	VBO_DerefBusyObj(wrk, &bo);	// XXX ?
 	return (F_STP_DONE);
 }
@@ -550,6 +551,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, int pass)
 	req->vary_b = NULL;
 
 	AZ(bo->fetch_objcore);
+	HSH_Ref(oc);
 	bo->fetch_objcore = oc;
 
 	AZ(bo->req);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index ffd1087..9abe891 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -63,6 +63,7 @@
 #include "vsha256.h"
 
 static const struct hash_slinger *hash;
+static struct objhead *private_oh;
 
 /*---------------------------------------------------------------------*/
 
@@ -79,11 +80,25 @@ HSH_NewObjCore(struct worker *wrk)
 }
 
 /*---------------------------------------------------------------------*/
+
+static struct objhead *
+hsh_newobjhead(void)
+{
+	struct objhead *oh;
+
+	ALLOC_OBJ(oh, OBJHEAD_MAGIC);
+	XXXAN(oh);
+	oh->refcnt = 1;
+	VTAILQ_INIT(&oh->objcs);
+	Lck_New(&oh->mtx, lck_objhdr);
+	return (oh);
+}
+
+/*---------------------------------------------------------------------*/
 /* Precreate an objhead and object for later use */
 static void
 hsh_prealloc(struct worker *wrk)
 {
-	struct objhead *oh;
 	struct waitinglist *wl;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -93,12 +108,7 @@ hsh_prealloc(struct worker *wrk)
 	CHECK_OBJ_NOTNULL(wrk->nobjcore, OBJCORE_MAGIC);
 
 	if (wrk->nobjhead == NULL) {
-		ALLOC_OBJ(oh, OBJHEAD_MAGIC);
-		XXXAN(oh);
-		oh->refcnt = 1;
-		VTAILQ_INIT(&oh->objcs);
-		Lck_New(&oh->mtx, lck_objhdr);
-		wrk->nobjhead = oh;
+		wrk->nobjhead = hsh_newobjhead();
 		wrk->stats.n_objecthead++;
 	}
 	CHECK_OBJ_NOTNULL(wrk->nobjhead, OBJHEAD_MAGIC);
@@ -116,6 +126,28 @@ hsh_prealloc(struct worker *wrk)
 		hash->prep(wrk);
 }
 
+/*---------------------------------------------------------------------*/
+
+struct objcore *
+HSH_Private(struct worker *wrk)
+{
+	struct objcore *oc;
+
+	CHECK_OBJ_NOTNULL(private_oh, OBJHEAD_MAGIC);
+
+	oc = HSH_NewObjCore(wrk);
+	AN(oc);
+	oc->refcnt = 1;
+	oc->objhead = private_oh;
+	oc->flags |= OC_F_PRIVATE;
+	Lck_Lock(&private_oh->mtx);
+	VTAILQ_INSERT_TAIL(&private_oh->objcs, oc, list);
+	private_oh->refcnt++;
+	Lck_Unlock(&private_oh->mtx);
+	return (oc);
+}
+
+/*---------------------------------------------------------------------*/
 void
 HSH_Cleanup(struct worker *wrk)
 {
@@ -286,6 +318,7 @@ hsh_insert_busyobj(struct worker *wrk, struct objhead *oh)
 	struct objcore *oc;
 
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+	Lck_AssertHeld(&oh->mtx);
 
 	oc = wrk->nobjcore;
 	wrk->nobjcore = NULL;
@@ -687,44 +720,34 @@ HSH_Deref(struct dstat *ds, struct objcore *oc, struct object **oo)
 	}
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	assert(oc->refcnt > 0);
 
 	oh = oc->objhead;
-	if (oh != NULL) {
-		CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 
-		Lck_Lock(&oh->mtx);
-		assert(oh->refcnt > 0);
-		assert(oc->refcnt > 0);
-		r = --oc->refcnt;
-		if (!r)
-			VTAILQ_REMOVE(&oh->objcs, oc, list);
-		else {
-			/* Must have an object */
-			AN(oc->methods);
-		}
-		if (oh->waitinglist != NULL)
-			hsh_rush(ds, oh);
-		Lck_Unlock(&oh->mtx);
-		if (r != 0)
-			return (r);
+	Lck_Lock(&oh->mtx);
+	assert(oh->refcnt > 0);
+	r = --oc->refcnt;
+	if (!r)
+		VTAILQ_REMOVE(&oh->objcs, oc, list);
+	if (oh->waitinglist != NULL)
+		hsh_rush(ds, oh);
+	Lck_Unlock(&oh->mtx);
+	if (r != 0)
+		return (r);
 
-		BAN_DestroyObj(oc);
-		AZ(oc->ban);
-	} else
-		AZ(oc->refcnt);
+	BAN_DestroyObj(oc);
+	AZ(oc->ban);
 
-	if (oc->methods != NULL) {
+	if (oc->methods != NULL)
 		oc_freeobj(oc);
-		ds->n_object--;
-	}
+	ds->n_object--;
 	FREE_OBJ(oc);
 
 	ds->n_objectcore--;
-	if (oh != NULL) {
-		/* Drop our ref on the objhead */
-		assert(oh->refcnt > 0);
-		(void)HSH_DerefObjHead(ds, &oh);
-	}
+	/* Drop our ref on the objhead */
+	assert(oh->refcnt > 0);
+	(void)HSH_DerefObjHead(ds, &oh);
 	return (0);
 }
 
@@ -740,6 +763,14 @@ HSH_DerefObjHead(struct dstat *ds, struct objhead **poh)
 	*poh = NULL;
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 
+	if (oh == private_oh) {
+		Lck_Lock(&oh->mtx);
+		assert(oh->refcnt > 1);
+		oh->refcnt--;
+		Lck_Unlock(&oh->mtx);
+		return(1);
+	}
+
 	assert(oh->refcnt > 0);
 	r = hash->deref(oh);
 	if (!r)
@@ -755,4 +786,6 @@ HSH_Init(const struct hash_slinger *slinger)
 	hash = slinger;
 	if (hash->start != NULL)
 		hash->start();
+	private_oh = hsh_newobjhead();
+	private_oh->refcnt = 1;
 }
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 0d59693..3e4dabf 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -161,8 +161,11 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(req->obj->objcore, OBJCORE_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 
+	assert(req->obj->objcore->refcnt > 0);
+
 	req->res_mode = 0;
 
 	if (!req->disable_esi && req->obj->esidata != NULL) {
@@ -247,6 +250,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		STV_Freestore(req->obj);
 
 	assert(WRW_IsReleased(wrk));
+VSLb(req->vsl, SLT_Debug, "XXX REF %d", req->obj->objcore->refcnt);
 	(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
 	http_Teardown(req->resp);
 	return (REQ_FSM_DONE);
@@ -283,7 +287,7 @@ cnt_error(struct worker *wrk, struct req *req)
 	req->busyobj = bo;
 	AZ(bo->stats);
 	bo->stats = &wrk->stats;
-	bo->fetch_objcore = HSH_NewObjCore(wrk);
+	bo->fetch_objcore = HSH_Private(wrk);
 	req->obj = STV_NewObject(bo,
 	    TRANSIENT_STORAGE, cache_param->http_resp_size,
 	    (uint16_t)cache_param->http_max_hdr);
@@ -467,7 +471,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	AZ(req->objcore);
 	if (lr == HSH_MISS) {
 		/* Found nothing */
-		VSLb(req->vsl, SLT_Debug, "XXXX MISS\n");
+		VSLb(req->vsl, SLT_Debug, "XXXX MISS");
 		AZ(oc);
 		AN(boc);
 		AN(boc->flags & OC_F_BUSY);
@@ -483,7 +487,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 
 	if (oc->flags & OC_F_PASS) {
 		/* Found a hit-for-pass */
-		VSLb(req->vsl, SLT_Debug, "XXXX HIT-FOR-PASS\n");
+		VSLb(req->vsl, SLT_Debug, "XXXX HIT-FOR-PASS");
 		AZ(boc);
 		(void)HSH_Deref(&wrk->stats, oc, NULL);
 		req->objcore = NULL;
@@ -649,7 +653,7 @@ cnt_pass(struct worker *wrk, struct req *req)
 	assert (wrk->handling == VCL_RET_FETCH);
 	req->acct_req.pass++;
 
-	oc = HSH_NewObjCore(wrk);
+	oc = HSH_Private(wrk);
 	AN(oc);
 	req->busyobj = VBF_Fetch(wrk, req, oc, 1);
 	req->req_step = R_STP_FETCH;
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index e21a96e..79443c7 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -73,6 +73,7 @@ void HSH_AddString(const struct req *, const char *str);
 void HSH_Insert(struct worker *, const void *hash, struct objcore *);
 void HSH_Purge(struct worker *, struct objhead *, double ttl, double grace);
 void HSH_config(const char *h_arg);
+struct objcore *HSH_Private(struct worker *wrk);
 struct objcore *HSH_NewObjCore(struct worker *wrk);
 
 #ifdef VARNISH_CACHE_CHILD



From phk at varnish-cache.org  Thu Aug 29 09:15:20 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 29 Aug 2013 11:15:20 +0200
Subject: [master] 175d642 Turn "request done" into a proper busyobj state.
Message-ID: <E1VEyK0-0002kQ-CZ@project.varnish-software.com>

commit 175d642d2bdfc2b68619c45fb47aa985d5572820
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 29 09:15:03 2013 +0000

    Turn "request done" into a proper busyobj state.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 4598ad3..0784b62 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -493,6 +493,7 @@ oc_getlru(const struct objcore *oc)
 
 enum busyobj_state_e {
 	BOS_INVALID = 0,	/* don't touch (yet) */
+	BOS_REQ_DONE,		/* beresp.* can be examined */
 	BOS_FETCHING,		/* beresp.* can be examined */
 	BOS_FINISHED,		/* object is complete */
 	BOS_FAILED,		/* something went wrong */
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 90d07fd..c25be1c 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -47,12 +47,10 @@
 static void
 vbf_release_req(struct busyobj *bo)
 {
-	if (bo->req == NULL)
-		return;
-	Lck_Lock(&bo->mtx);
+	assert(bo->state == BOS_INVALID);
+	AN(bo->req);
 	bo->req = NULL;
-	AZ(pthread_cond_signal(&bo->cond));
-	Lck_Unlock(&bo->mtx);
+	VBO_setstate(bo, BOS_REQ_DONE);
 }
 
 /*--------------------------------------------------------------------
@@ -118,6 +116,8 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	http_PrintfHeader(bo->bereq,
 	    "X-Varnish: %u", bo->vsl->wid & VSL_IDENTMASK);
 	if (wrk->handling == VCL_RET_ABANDON) {
+		if (bo->req != NULL)
+			vbf_release_req(bo);
 		VBO_setstate(bo, BOS_FAILED);
 		return (F_STP_ABANDON);
 	}
@@ -140,10 +140,10 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
 
-	if (!bo->do_pass)
+	if (!bo->do_pass && bo->req != NULL)
 		vbf_release_req(bo); /* XXX: retry ?? */
 
-	assert(bo->state == BOS_INVALID);
+	assert(bo->state <= BOS_REQ_DONE);
 
 	i = V1F_fetch_hdr(wrk, bo, bo->req);
 	/*
@@ -156,7 +156,7 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 		i = V1F_fetch_hdr(wrk, bo, bo->req);
 	}
 
-	if (bo->do_pass)
+	if (bo->do_pass && bo->req != NULL)
 		vbf_release_req(bo); /* XXX : retry ?? */
 
 	AZ(bo->req);
@@ -213,7 +213,7 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 	if (wrk->handling == VCL_RET_DELIVER)
 		return (F_STP_FETCH);
 	if (wrk->handling == VCL_RET_RETRY) {
-		assert(bo->state == BOS_INVALID);
+		assert(bo->state == BOS_REQ_DONE);
 		bo->retries++;
 		if (bo->retries <= cache_param->max_retries) {
 			VDI_CloseFd(&bo->vbc);
@@ -419,7 +419,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->vfp == NULL)
 		bo->vfp = &VFP_nop;
 
-	assert(bo->state == BOS_INVALID);
+	assert(bo->state == BOS_REQ_DONE);
 	VBO_setstate(bo, BOS_FETCHING);
 
 	V1F_fetch_body(wrk, bo);
@@ -448,7 +448,6 @@ vbf_stp_abandon(struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	assert(bo->state == BOS_FAILED);
-	vbf_release_req(bo);
 	VBO_DerefBusyObj(wrk, &bo);	// XXX ?
 	return (F_STP_DONE);
 }
@@ -562,12 +561,6 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, int pass)
 
 	if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_QUEUE_FRONT))
 		vbf_fetch_thread(wrk, bo);
-	Lck_Lock(&bo->mtx);
-	while (1) {
-		if (bo->req == NULL)
-			break;
-		(void)Lck_CondWait(&bo->cond, &bo->mtx, NULL);
-	}
-	Lck_Unlock(&bo->mtx);
+	VBO_waitstate(bo, BOS_REQ_DONE);
 	return (bo);
 }



From phk at varnish-cache.org  Thu Aug 29 09:59:14 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 29 Aug 2013 11:59:14 +0200
Subject: [master] 69369f3 Tell VBF_Fetch() what we're trying to do,
 and have it linger until we have a usable state.
Message-ID: <E1VEz0U-00019B-RL@project.varnish-software.com>

commit 69369f3e1b0e689a494eea637d2394e5ecd1f919
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 29 09:58:44 2013 +0000

    Tell VBF_Fetch() what we're trying to do, and have it linger until
    we have a usable state.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 0784b62..9b68f79 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -845,7 +845,13 @@ int EXP_NukeOne(struct busyobj *, struct lru *lru);
 void EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru);
 
 /* cache_fetch.c */
-struct busyobj *VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, int pass);
+enum vbf_fetch_mode_e {
+	VBF_NORMAL = 0,
+	VBF_PASS = 1,
+	VBF_BACKGROUND = 2,
+};
+struct busyobj *VBF_Fetch(struct worker *wrk, struct req *req,
+    struct objcore *oc, enum vbf_fetch_mode_e);
 
 /* cache_fetch_proc.c */
 struct storage *VFP_GetStorage(struct busyobj *, ssize_t sz);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index c25be1c..d928603 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -528,7 +528,8 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
  */
 
 struct busyobj *
-VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, int pass)
+VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
+    enum vbf_fetch_mode_e mode)
 {
 	struct busyobj *bo;
 
@@ -544,7 +545,8 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, int pass)
 
 	CHECK_OBJ_NOTNULL(bo->vcl, VCL_CONF_MAGIC);
 
-	bo->do_pass = pass;
+	if (mode == VBF_PASS)
+		bo->do_pass = 1;
 
 	bo->vary = req->vary_b;
 	req->vary_b = NULL;
@@ -561,6 +563,11 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, int pass)
 
 	if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_QUEUE_FRONT))
 		vbf_fetch_thread(wrk, bo);
-	VBO_waitstate(bo, BOS_REQ_DONE);
+	if (mode == VBF_BACKGROUND) {
+		VBO_waitstate(bo, BOS_REQ_DONE);
+		VBO_DerefBusyObj(wrk, &bo);
+		return (NULL);
+	}
+	VBO_waitstate(bo, BOS_FETCHING);
 	return (bo);
 }
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 3e4dabf..29ae0e4 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -378,8 +378,6 @@ cnt_fetch(struct worker *wrk, struct req *req)
 	assert(bo->refcount > 0);
 	(void)HTTP1_DiscardReqBody(req);
 
-	VBO_waitstate(bo, BOS_FETCHING);
-
 	/* bo->do_stream is not valid until after vcl_backend_response{} */
 	if (!bo->do_stream)
 		VBO_waitstate(bo, BOS_FINISHED);
@@ -499,9 +497,6 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	oh = oc->objhead;
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 
-	/* We are not prepared to do streaming yet */
-	XXXAZ(req->busyobj);
-
 	o = oc_getobj(&wrk->stats, oc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	req->obj = o;
@@ -513,8 +508,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	switch (wrk->handling) {
 	case VCL_RET_DELIVER:
 		if (boc != NULL) {
-			req->busyobj = VBF_Fetch(wrk, req, boc, 0);
-			VBO_DerefBusyObj(wrk, &req->busyobj);
+			AZ(VBF_Fetch(wrk, req, boc, VBF_BACKGROUND));
 		} else {
 			(void)HTTP1_DiscardReqBody(req);// XXX: handle err
 		}
@@ -607,7 +601,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 	wrk->stats.cache_miss++;
 
 	AN (req->objcore);
-	req->busyobj = VBF_Fetch(wrk, req, req->objcore, 0);
+	req->busyobj = VBF_Fetch(wrk, req, req->objcore, VBF_NORMAL);
 	req->objcore = NULL;
 	req->req_step = R_STP_FETCH;
 	return (REQ_FSM_MORE);
@@ -655,7 +649,7 @@ cnt_pass(struct worker *wrk, struct req *req)
 
 	oc = HSH_Private(wrk);
 	AN(oc);
-	req->busyobj = VBF_Fetch(wrk, req, oc, 1);
+	req->busyobj = VBF_Fetch(wrk, req, oc, VBF_PASS);
 	req->req_step = R_STP_FETCH;
 	return (REQ_FSM_MORE);
 }



From perbu at varnish-cache.org  Fri Aug 30 06:49:59 2013
From: perbu at varnish-cache.org (Per Buer)
Date: Fri, 30 Aug 2013 08:49:59 +0200
Subject: [master] 7293eaf On virutal hosting in Varnis
Message-ID: <E1VFIWt-0002MF-3F@project.varnish-software.com>

commit 7293eaf9dc729ab2c2e5932ce18ca98b068431ef
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Aug 30 08:49:05 2013 +0200

    On virutal hosting in Varnis

diff --git a/doc/sphinx/tutorial/backend_servers.rst b/doc/sphinx/tutorial/backend_servers.rst
index 7ea3a20..ebc05ad 100644
--- a/doc/sphinx/tutorial/backend_servers.rst
+++ b/doc/sphinx/tutorial/backend_servers.rst
@@ -20,19 +20,23 @@ the configuration that looks like this:::
       .port = "80";
   }
 
+This means we set up a backend in Varnish that fetches content from
+the host www.varnish-cache.org on port 80. 
+
 Since you probably don't want to be mirroring varnish-cache.org we
 need to get Varnish to fetch content from your own origin
 server. We've already bound Varnish to the public port 80 on the
 server so now we need to tie it to the origin.
 
 For this example, let's pretend the origin server is running on
-localhost, port 8080.
+localhost, port 8080.::
 
           backend default {
                 .host = "127.0.0.1";
     		.port = "8080";
 	  }
 
+
 Varnish can have several backends defined and can you can even join
 several backends together into clusters of backends for load balancing
 purposes, having Varnish pick one backend based on different
diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index 586b221..7dc8c95 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -31,6 +31,7 @@ Varnish can have several backends defined and can you can even join
 several backends together into clusters of backends for load balancing
 purposes. 
 
+
 Multiple backends
 -----------------
 
@@ -73,8 +74,42 @@ really arbitrary data. You want to send mobile devices to a different
 backend? No problem. if (req.User-agent ~ /mobile/) .. should do the
 trick. 
 
+
+Backends and virtual hosts in Varnish
+-------------------------------------
+
+Varnish fully supports virtual hosts. They might work in a somewhat
+counter intuitive fashion since they are never declared
+explicitly. You set up the routing of incoming HTTP requests in
+vcl_recv. If you want this routing to be done on the basis of virtual
+hosts you just need to inspect req.http.host.
+
+You can have something like this:::
+
+  sub vcl_recv {
+     if (req.http.host ~ "foo.com") {
+       set req.backend = foo;
+     } elsif (req.http.host ~ "bar.com") {
+       set req.backend = bar;
+     }
+  }
+
+Note that the first regular expressions will match foo.com,
+www.foo.com, zoop.foo.com and any other host ending in foo.com. In
+this example this is intentional but you might want it to be a bit
+more tight, maybe relying on the == operator in stead, like this:::
+
+
+  sub vcl_recv {
+    if (req.http.host == "foo.com" or req.http.host == "www.foo.com") {
+      set req.backend = foo;
+    }
+  } 
+
+
 .. _users-guide-advanced_backend_servers-directors:
 
+
 Directors
 ---------
 



From perbu at varnish-cache.org  Fri Aug 30 06:49:59 2013
From: perbu at varnish-cache.org (Per Buer)
Date: Fri, 30 Aug 2013 08:49:59 +0200
Subject: [master] cdfdb28 removed unused file
Message-ID: <E1VFIWt-0002ML-4X@project.varnish-software.com>

commit cdfdb28db55f936c72c323aafa03ff18726e6926
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Aug 30 08:49:26 2013 +0200

    removed unused file

diff --git a/doc/sphinx/include/vcl-backends.rst b/doc/sphinx/include/vcl-backends.rst
deleted file mode 100644
index 0290bf7..0000000
--- a/doc/sphinx/include/vcl-backends.rst
+++ /dev/null
@@ -1,44 +0,0 @@
-Backend declarations
---------------------
-
-A backend declaration creates and initializes a named backend object:
-::
-
-  backend www {
-    .host = "www.example.com";
-    .port = "http";
-  }
-
-The backend object can later be used to select a backend at request time:
-::
-
-  if (req.http.host ~ "(?i)^(www.)?example.com$") {
-    set req.backend = www;
-  }
-
-To avoid overloading backend servers, .max_connections can be set to
-limit the maximum number of concurrent backend connections.
-
-The timeout parameters can be overridden in the backend declaration.
-The timeout parameters are .connect_timeout for the time to wait for a
-backend connection, .first_byte_timeout for the time to wait for the
-first byte from the backend and .between_bytes_timeout for time to
-wait between each received byte.
-
-These can be set in the declaration like this:
-::
-
-  backend www {
-    .host = "www.example.com";
-    .port = "http";
-    .connect_timeout = 1s;
-    .first_byte_timeout = 5s;
-    .between_bytes_timeout = 2s;
-  }
-
-To mark a backend as unhealthy after number of items have been added
-to its saintmode list ``.saintmode_threshold`` can be set to the maximum
-list size. Setting a value of 0 disables saint mode checking entirely
-for that backend.  The value in the backend declaration overrides the
-parameter.
-



From perbu at varnish-cache.org  Fri Aug 30 07:01:49 2013
From: perbu at varnish-cache.org (Per Buer)
Date: Fri, 30 Aug 2013 09:01:49 +0200
Subject: [master] 7bd56ff inline the backend stuff again
Message-ID: <E1VFIiL-00066k-H0@project.varnish-software.com>

commit 7bd56ffc2a9fbe22c10042b90954c6aedcc2918b
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Aug 30 09:01:46 2013 +0200

    inline the backend stuff again

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index ab63679..a99f2eb 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -27,9 +27,54 @@ When a new configuration is loaded, the varnishd management process
 translates the VCL code to C and compiles it to a shared object which
 is then dynamically linked into the server process.
 
-.. .. include:: ../include/vcl-syntax.rst
+For an overview of the VCL syntax please see the users guide.
+
+
+Backend declarations
+--------------------
+
+A backend declaration creates and initializes a named backend object:
+::
+
+  backend www {
+    .host = "www.example.com";
+    .port = "http";
+  }
+
+The backend object can later be used to select a backend at request time:
+::
+
+  if (req.http.host ~ "(?i)^(www.)?example.com$") {
+    set req.backend = www;
+  }
+
+To avoid overloading backend servers, .max_connections can be set to
+limit the maximum number of concurrent backend connections.
+
+The timeout parameters can be overridden in the backend declaration.
+The timeout parameters are .connect_timeout for the time to wait for a
+backend connection, .first_byte_timeout for the time to wait for the
+first byte from the backend and .between_bytes_timeout for time to
+wait between each received byte.
+
+These can be set in the declaration like this:
+::
+
+  backend www {
+    .host = "www.example.com";
+    .port = "http";
+    .connect_timeout = 1s;
+    .first_byte_timeout = 5s;
+    .between_bytes_timeout = 2s;
+  }
+
+To mark a backend as unhealthy after number of items have been added
+to its saintmode list ``.saintmode_threshold`` can be set to the maximum
+list size. Setting a value of 0 disables saint mode checking entirely
+for that backend.  The value in the backend declaration overrides the
+parameter.
+
 
-.. include:: ../include/vcl-backends.rst
 
 .. _ref-vcl-director: