From ingvar at projects.linpro.no  Sun Nov  2 21:39:28 2008
From: ingvar at projects.linpro.no (ingvar at projects.linpro.no)
Date: Sun,  2 Nov 2008 22:39:28 +0100 (CET)
Subject: r3358 - trunk/varnish-cache/redhat
Message-ID: <20081102213928.A35C61EC10F@projects.linpro.no>

Author: ingvar
Date: 2008-11-02 22:39:28 +0100 (Sun, 02 Nov 2008)
New Revision: 3358

Modified:
   trunk/varnish-cache/redhat/varnish.spec
Log:
* Sun Nov 02 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0.1-2
- Removed the requirement for kernel => 2.6.0. All supported
  platforms meets this, and it generates strange errors in EPEL



Modified: trunk/varnish-cache/redhat/varnish.spec
===================================================================
--- trunk/varnish-cache/redhat/varnish.spec	2008-10-30 07:00:00 UTC (rev 3357)
+++ trunk/varnish-cache/redhat/varnish.spec	2008-11-02 21:39:28 UTC (rev 3358)
@@ -1,7 +1,7 @@
 Summary: Varnish is a high-performance HTTP accelerator
 Name: varnish
 Version: 2.0.1
-Release: 1%{?dist}
+Release: 2%{?dist}
 License: BSD
 Group: System Environment/Daemons
 URL: http://www.varnish-cache.org/
@@ -12,7 +12,7 @@
 # configure script. Release tarballs would not need this
 #BuildRequires: automake autoconf libtool
 BuildRequires: ncurses-devel libxslt groff
-Requires: kernel >= 2.6.0 varnish-libs = %{version}-%{release}
+Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses
 Requires(pre): shadow-utils
@@ -44,7 +44,7 @@
 Summary: Development files for %{name}-libs
 Group: System Environment/Libraries
 BuildRequires: ncurses-devel
-Requires: kernel >= 2.6.0 varnish-libs = %{version}-%{release}
+Requires: varnish-libs = %{version}-%{release}
 
 %description libs-devel
 Development files for %{name}-libs
@@ -54,7 +54,7 @@
 #Summary: Files for static linking of %{name} library functions
 #Group: System Environment/Libraries
 #BuildRequires: ncurses-devel
-#Requires: kernel >= 2.6.0 varnish-libs-devel = %{version}-%{release}
+#Requires: varnish-libs-devel = %{version}-%{release}
 #
 #%description libs-static
 #Files for static linking of varnish library functions
@@ -230,6 +230,10 @@
 %postun libs -p /sbin/ldconfig
 
 %changelog
+* Sun Nov 02 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0.1-2
+- Removed the requirement for kernel => 2.6.0. All supported
+  platforms meets this, and it generates strange errors in EPEL
+
 * Fri Oct 17 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0.1-1
 - 2.0.1 released, a bugfix release. New upstream sources
 - Package now also available in EPEL



From phk at projects.linpro.no  Wed Nov  5 12:51:52 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Wed,  5 Nov 2008 13:51:52 +0100 (CET)
Subject: r3359 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081105125152.2BEEC1EC215@projects.linpro.no>

Author: phk
Date: 2008-11-05 13:51:51 +0100 (Wed, 05 Nov 2008)
New Revision: 3359

Modified:
   trunk/varnish-cache/bin/varnishd/cache_dir_random.c
Log:
Fix the broken logic in the random directors picking routine.

Drop the consistency check for health-changes, at the cost of a
slight bias[1] for hosts before, or after, then one that changed health,
according to how its health changed.

Count all failures to pick against the retry count.

Fixes #361


Modified: trunk/varnish-cache/bin/varnishd/cache_dir_random.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_dir_random.c	2008-11-02 21:39:28 UTC (rev 3358)
+++ trunk/varnish-cache/bin/varnishd/cache_dir_random.c	2008-11-05 12:51:51 UTC (rev 3359)
@@ -66,9 +66,9 @@
 static struct vbe_conn *
 vdi_random_getfd(struct sess *sp)
 {
-	int i, j, k;
+	int i, k;
 	struct vdi_random *vs;
-	double r, s1, s2;
+	double r, s1;
 	struct vbe_conn *vbe;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -78,43 +78,32 @@
 	k = 0;
 	for (k = 0; k < vs->retries; ) {
 
-		r = random() / 2147483648.0;	/* 2^31 */
-		assert(r >= 0.0 && r < 1.0);
-
+		/* Sum up the weights of healty backends */
 		s1 = 0.0;
-		j = 0;
-		for (i = 0; i < vs->nhosts; i++) {
-			if (!vs->hosts[i].backend->healthy)
-				continue;
-			s1 += vs->hosts[i].weight;
-			j++;
-		}
+		for (i = 0; i < vs->nhosts; i++)
+			if (vs->hosts[i].backend->healthy)
+				s1 += vs->hosts[i].weight;
 
-		if (j == 0)		/* No healthy hosts */
+		if (s1 == 0.0)
 			return (NULL);
 
+		/* Pick a random threshold in that interval */
+		r = random() / 2147483648.0;	/* 2^31 */
+		assert(r >= 0.0 && r < 1.0);
 		r *= s1;
 
-		s2 = 0;
+		s1 = 0.0;
 		for (i = 0; i < vs->nhosts; i++)  {
 			if (!vs->hosts[i].backend->healthy)
 				continue;
-			s2 += vs->hosts[i].weight;
-			if (r < s2)
-				break;
+			s1 += vs->hosts[i].weight;
+			if (r >= s1)
+				continue;
+			vbe = VBE_GetVbe(sp, vs->hosts[i].backend);
+			if (vbe != NULL)
+				return (vbe);
+			break;
 		}
-
-		if (s2 != s1) {
-			/*
-			 * Health bit changed in an unusable way while we
-			 * worked the problem.  Usable changes are any that
-			 * result in the same sum we prepared for.
-			 */
-			continue;
-		}
-		vbe = VBE_GetVbe(sp, vs->hosts[i].backend);
-		if (vbe != NULL)
-			return (vbe);
 		k++;
 	}
 	return (NULL);



From tfheen at projects.linpro.no  Thu Nov  6 11:45:45 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Thu,  6 Nov 2008 12:45:45 +0100 (CET)
Subject: r3360 - in trunk/varnish-cache: . lib/libvcl
Message-ID: <20081106114545.7A3C71EC535@projects.linpro.no>

Author: tfheen
Date: 2008-11-06 12:45:45 +0100 (Thu, 06 Nov 2008)
New Revision: 3360

Modified:
   trunk/varnish-cache/configure.ac
   trunk/varnish-cache/lib/libvcl/Makefile.am
Log:
Fix up tclsh invocation (again)

Move the || true bit of invoking tclsh to the Makefile, since
missing would otherwise not pass the file name to tclsh.

Thanks to des for spotting this.


Modified: trunk/varnish-cache/configure.ac
===================================================================
--- trunk/varnish-cache/configure.ac	2008-11-05 12:51:51 UTC (rev 3359)
+++ trunk/varnish-cache/configure.ac	2008-11-06 11:45:45 UTC (rev 3360)
@@ -232,7 +232,7 @@
 AM_MISSING_HAS_RUN
 AC_CHECK_PROGS(TCLSH, [tclsh tclsh8.4 tclsh8.5], :)
 if test "$TCLSH" = :; then
-  TCLSH="${am_missing_run}tclsh || true"
+  TCLSH="${am_missing_run}tclsh"
 fi
 
 # Solaris defines SO_{RCV,SND}TIMEO, but does not implement them.

Modified: trunk/varnish-cache/lib/libvcl/Makefile.am
===================================================================
--- trunk/varnish-cache/lib/libvcl/Makefile.am	2008-11-05 12:51:51 UTC (rev 3359)
+++ trunk/varnish-cache/lib/libvcl/Makefile.am	2008-11-06 11:45:45 UTC (rev 3360)
@@ -29,10 +29,10 @@
 	vcc_gen_fixed_token.tcl
 
 $(srcdir)/vcc_obj.c: $(srcdir)/vcc_gen_obj.tcl
-	cd $(srcdir) && @TCLSH@ vcc_gen_obj.tcl
+	cd $(srcdir) && @TCLSH@ vcc_gen_obj.tcl || true
 
 $(srcdir)/vcc_fixed_token.c: $(srcdir)/vcc_gen_fixed_token.tcl $(top_srcdir)/include/vcl.h $(top_srcdir)/include/vrt.h $(top_srcdir)/include/vrt_obj.h
-	cd $(srcdir) && @TCLSH@ vcc_gen_fixed_token.tcl
+	cd $(srcdir) && @TCLSH@ vcc_gen_fixed_token.tcl || true
 
 $(srcdir)/vcc_token_defs.h: $(srcdir)/vcc_gen_fixed_token.tcl $(top_srcdir)/include/vcl.h $(top_srcdir)/include/vrt.h $(top_srcdir)/include/vrt_obj.h
-	cd $(srcdir) && @TCLSH@ vcc_gen_fixed_token.tcl
+	cd $(srcdir) && @TCLSH@ vcc_gen_fixed_token.tcl || true



From tfheen at projects.linpro.no  Thu Nov  6 11:46:31 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Thu,  6 Nov 2008 12:46:31 +0100 (CET)
Subject: r3361 - trunk/varnish-cache/man
Message-ID: <20081106114631.7A6FD1EC874@projects.linpro.no>

Author: tfheen
Date: 2008-11-06 12:46:31 +0100 (Thu, 06 Nov 2008)
New Revision: 3361

Modified:
   trunk/varnish-cache/man/vcl.7so
Log:
Fix up $N vs \N in man page

The VCL man page documented the capturing parentheses as using $N
rather than \N which is actually used.

Fixes #359


Modified: trunk/varnish-cache/man/vcl.7so
===================================================================
--- trunk/varnish-cache/man/vcl.7so	2008-11-06 11:45:45 UTC (rev 3360)
+++ trunk/varnish-cache/man/vcl.7so	2008-11-06 11:46:31 UTC (rev 3361)
@@ -198,11 +198,11 @@
 .Fa sub .
 Within
 .Fa sub ,
-.Va $0
+.Va \\0
 (which can also be spelled
 .Va & )
 is replaced with the entire matched string, and
-.Va $n
+.Va \\n
 is replaced with the contents of subgroup
 .Ar n
 in the matched string.



From tfheen at projects.linpro.no  Thu Nov  6 11:57:06 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Thu,  6 Nov 2008 12:57:06 +0100 (CET)
Subject: r3362 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081106115706.2239E1EC535@projects.linpro.no>

Author: tfheen
Date: 2008-11-06 12:57:05 +0100 (Thu, 06 Nov 2008)
New Revision: 3362

Modified:
   trunk/varnish-cache/bin/varnishd/varnishd.1
Log:
Document the size parameter to -s malloc

Fixes #362


Modified: trunk/varnish-cache/bin/varnishd/varnishd.1
===================================================================
--- trunk/varnish-cache/bin/varnishd/varnishd.1	2008-11-06 11:46:31 UTC (rev 3361)
+++ trunk/varnish-cache/bin/varnishd/varnishd.1	2008-11-06 11:57:05 UTC (rev 3362)
@@ -236,9 +236,27 @@
 .Ss Storage Types
 The following storage types are available:
 .Bl -tag -width 4n
-.It Cm malloc
+.It Cm malloc Ns Op Ns , Ns Ar size Ns
 Storage for each object is allocated with
 .Xr malloc 3 .
+.Pp
+The
+.Ar size
+parameter specifies the maximum amount of memory varnishd will allocate.
+The size is assumed to be in bytes, unless followed by one of the
+following suffixes:
+.Bl -tag -width indent
+.It K, k
+The size is expressed in kibibytes.
+.It M, m
+The size is expressed in mebibytes.
+.It G, g
+The size is expressed in gibibytes.
+.It T, t
+The size is expressed in tebibytes.
+.El
+.Pp
+The default size is unlimited.
 .It Cm file Ns Op Ns , Ns Ar path Ns Op Ns , Ns Ar size Ns Op Ns , Ns Ar granularity
 Storage for each object is allocated from an arena backed by a file.
 This is the default.



From phk at projects.linpro.no  Sun Nov  9 13:46:57 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Sun,  9 Nov 2008 14:46:57 +0100 (CET)
Subject: r3363 - in trunk/varnish-cache: bin/varnishd lib/libvcl
Message-ID: <20081109134657.C6E7A1EC874@projects.linpro.no>

Author: phk
Date: 2008-11-09 14:46:57 +0100 (Sun, 09 Nov 2008)
New Revision: 3363

Modified:
   trunk/varnish-cache/bin/varnishd/cache_acceptor.c
   trunk/varnish-cache/bin/varnishd/cache_backend_poll.c
   trunk/varnish-cache/bin/varnishd/cache_hash.c
   trunk/varnish-cache/bin/varnishd/cache_http.c
   trunk/varnish-cache/bin/varnishd/mgt_vcc.c
   trunk/varnish-cache/bin/varnishd/stevedore.c
   trunk/varnish-cache/bin/varnishd/storage_file.c
   trunk/varnish-cache/lib/libvcl/vcc_token.c
Log:
Constifications requested by FlexeLint v9



Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_acceptor.c	2008-11-06 11:57:05 UTC (rev 3362)
+++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c	2008-11-09 13:46:57 UTC (rev 3363)
@@ -49,7 +49,7 @@
 #include "cache.h"
 #include "cache_acceptor.h"
 
-static struct acceptor *vca_acceptors[] = {
+static struct acceptor * const vca_acceptors[] = {
 #if defined(HAVE_KQUEUE)
 	&acceptor_kqueue,
 #endif
@@ -63,7 +63,7 @@
 	NULL,
 };
 
-static struct acceptor *vca_act;
+static struct acceptor const *vca_act;
 
 static pthread_t	vca_thread_acct;
 static struct timeval	tv_sndtimeo;

Modified: trunk/varnish-cache/bin/varnishd/cache_backend_poll.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_backend_poll.c	2008-11-06 11:57:05 UTC (rev 3362)
+++ trunk/varnish-cache/bin/varnishd/cache_backend_poll.c	2008-11-09 13:46:57 UTC (rev 3363)
@@ -86,7 +86,7 @@
 static VTAILQ_HEAD(, vbp_target)	vbp_list =
     VTAILQ_HEAD_INITIALIZER(vbp_list);
 
-static char default_request[] =
+static const char default_request[] =
     "GET / HTTP/1.1\r\n"
     "Connection: close\r\n"
     "\r\n";

Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c	2008-11-06 11:57:05 UTC (rev 3362)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c	2008-11-09 13:46:57 UTC (rev 3363)
@@ -66,7 +66,7 @@
 #include "cache.h"
 #include "stevedore.h"
 
-static struct hash_slinger      *hash;
+static const struct hash_slinger *hash;
 
 double
 HSH_Grace(double g)

Modified: trunk/varnish-cache/bin/varnishd/cache_http.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_http.c	2008-11-06 11:57:05 UTC (rev 3362)
+++ trunk/varnish-cache/bin/varnishd/cache_http.c	2008-11-09 13:46:57 UTC (rev 3363)
@@ -65,7 +65,7 @@
 	LOGMTX2(ax, HTTP_HDR_FIRST,	Header),	\
 	}
 
-static enum shmlogtag logmtx[][HTTP_HDR_FIRST + 1] = {
+static const enum shmlogtag logmtx[][HTTP_HDR_FIRST + 1] = {
 	[HTTP_Rx] = LOGMTX1(Rx),
 	[HTTP_Tx] = LOGMTX1(Tx),
 	[HTTP_Obj] = LOGMTX1(Obj)

Modified: trunk/varnish-cache/bin/varnishd/mgt_vcc.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_vcc.c	2008-11-06 11:57:05 UTC (rev 3362)
+++ trunk/varnish-cache/bin/varnishd/mgt_vcc.c	2008-11-09 13:46:57 UTC (rev 3363)
@@ -78,7 +78,7 @@
 /*
  * Keep this in synch with man/vcl.7 and etc/default.vcl!
  */
-static const char *default_vcl =
+static const char * const default_vcl =
 #include "default_vcl.h"
     ""	;
 

Modified: trunk/varnish-cache/bin/varnishd/stevedore.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/stevedore.c	2008-11-06 11:57:05 UTC (rev 3362)
+++ trunk/varnish-cache/bin/varnishd/stevedore.c	2008-11-09 13:46:57 UTC (rev 3363)
@@ -40,7 +40,7 @@
 static VTAILQ_HEAD(, stevedore)	stevedores =
     VTAILQ_HEAD_INITIALIZER(stevedores);
 
-static struct stevedore * volatile stv_next;
+static const struct stevedore * volatile stv_next;
 
 struct storage *
 STV_alloc(struct sess *sp, size_t size)

Modified: trunk/varnish-cache/bin/varnishd/storage_file.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/storage_file.c	2008-11-06 11:57:05 UTC (rev 3362)
+++ trunk/varnish-cache/bin/varnishd/storage_file.c	2008-11-09 13:46:57 UTC (rev 3363)
@@ -233,8 +233,8 @@
 	/* XXX: force block allocation here or in open ? */
 }
 
-static char default_size[] = "50%";
-static char default_filename[] = ".";
+static const char default_size[] = "50%";
+static const char default_filename[] = ".";
 
 static void
 smf_init(struct stevedore *parent, int ac, char * const *av)

Modified: trunk/varnish-cache/lib/libvcl/vcc_token.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_token.c	2008-11-06 11:57:05 UTC (rev 3362)
+++ trunk/varnish-cache/lib/libvcl/vcc_token.c	2008-11-09 13:46:57 UTC (rev 3363)
@@ -248,7 +248,7 @@
 static int8_t
 vcc_xdig(const char c)
 {
-	static const char *xdigit =
+	static const char * const xdigit =
 	    "0123456789abcdef"
 	    "0123456789ABCDEF";
 	const char *p;



From phk at projects.linpro.no  Sun Nov  9 14:25:22 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Sun,  9 Nov 2008 15:25:22 +0100 (CET)
Subject: r3364 - in trunk/varnish-cache: bin/varnishd lib/libvarnish lib/libvcl
Message-ID: <20081109142522.4D75B1EC535@projects.linpro.no>

Author: phk
Date: 2008-11-09 15:25:22 +0100 (Sun, 09 Nov 2008)
New Revision: 3364

Modified:
   trunk/varnish-cache/bin/varnishd/cache_expire.c
   trunk/varnish-cache/bin/varnishd/mgt_child.c
   trunk/varnish-cache/bin/varnishd/varnishd.c
   trunk/varnish-cache/lib/libvarnish/crc32.c
   trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c
   trunk/varnish-cache/lib/libvcl/vcc_priv.h
Log:
More FlexeLint v9 consts



Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_expire.c	2008-11-09 13:46:57 UTC (rev 3363)
+++ trunk/varnish-cache/bin/varnishd/cache_expire.c	2008-11-09 14:25:22 UTC (rev 3364)
@@ -69,8 +69,8 @@
  * housekeeping fields parts of an object.
  */
 
-static const char *tmr_prefetch	= "prefetch";
-static const char *tmr_ttl	= "ttl";
+static const char * const tmr_prefetch	= "prefetch";
+static const char * const tmr_ttl	= "ttl";
 
 struct objexp {
 	unsigned		magic;

Modified: trunk/varnish-cache/bin/varnishd/mgt_child.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_child.c	2008-11-09 13:46:57 UTC (rev 3363)
+++ trunk/varnish-cache/bin/varnishd/mgt_child.c	2008-11-09 14:25:22 UTC (rev 3364)
@@ -81,7 +81,7 @@
 	CH_DIED = 4
 }			child_state = CH_STOPPED;
 
-static const char *ch_state[] = {
+static const char * const ch_state[] = {
 	[CH_STOPPED] =	"stopped",
 	[CH_STARTING] =	"starting",
 	[CH_RUNNING] =	"running",

Modified: trunk/varnish-cache/bin/varnishd/varnishd.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/varnishd.c	2008-11-09 13:46:57 UTC (rev 3363)
+++ trunk/varnish-cache/bin/varnishd/varnishd.c	2008-11-09 14:25:22 UTC (rev 3364)
@@ -117,7 +117,7 @@
 extern struct stevedore smu_stevedore;
 #endif
 
-static struct choice stv_choice[] = {
+static const struct choice stv_choice[] = {
 	{ "file",	&smf_stevedore },
 	{ "malloc",	&sma_stevedore },
 #ifdef HAVE_LIBUMEM
@@ -158,7 +158,7 @@
 extern struct hash_slinger hsl_slinger;
 extern struct hash_slinger hcl_slinger;
 
-static struct choice hsh_choice[] = {
+static const struct choice hsh_choice[] = {
 	{ "classic",		&hcl_slinger },
 	{ "simple",		&hsl_slinger },
 	{ "simple_list",	&hsl_slinger },	/* backwards compat */

Modified: trunk/varnish-cache/lib/libvarnish/crc32.c
===================================================================
--- trunk/varnish-cache/lib/libvarnish/crc32.c	2008-11-09 13:46:57 UTC (rev 3363)
+++ trunk/varnish-cache/lib/libvarnish/crc32.c	2008-11-09 14:25:22 UTC (rev 3364)
@@ -37,7 +37,7 @@
 
 /*--------------------------------------------------------------------*/
 
-static uint32_t crc32bits[] = {
+static const uint32_t crc32bits[] = {
     0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f,
     0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
     0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2,

Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c	2008-11-09 13:46:57 UTC (rev 3363)
+++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c	2008-11-09 14:25:22 UTC (rev 3364)
@@ -104,7 +104,7 @@
 	}
 }
 
-const char *vcl_tnames[256] = {
+const char * const vcl_tnames[256] = {
 	['!'] = "'!'",
 	['%'] = "'%'",
 	['&'] = "'&'",

Modified: trunk/varnish-cache/lib/libvcl/vcc_priv.h
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_priv.h	2008-11-09 13:46:57 UTC (rev 3363)
+++ trunk/varnish-cache/lib/libvcl/vcc_priv.h	2008-11-09 14:25:22 UTC (rev 3364)
@@ -39,7 +39,7 @@
 #define isident(c) (isalpha(c) || isdigit(c) || (c) == '_' || (c) == '-')
 #define isvar(c) (isident(c) || (c) == '.')
 unsigned vcl_fixed_token(const char *p, const char **q);
-extern const char *vcl_tnames[256];
+extern const char * const vcl_tnames[256];
 void vcl_output_lang_h(struct vsb *sb);
 
 #define PF(t)	(int)((t)->e - (t)->b), (t)->b



From phk at projects.linpro.no  Sun Nov  9 14:26:25 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Sun,  9 Nov 2008 15:26:25 +0100 (CET)
Subject: r3365 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081109142625.0985B1EC874@projects.linpro.no>

Author: phk
Date: 2008-11-09 15:26:24 +0100 (Sun, 09 Nov 2008)
New Revision: 3365

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_acceptor_poll.c
   trunk/varnish-cache/bin/varnishd/flint.lnt
Log:
A couple of stylistisc FlexeLint v9 nits, and some additions to the .lnt
file.

Unfortunately the thread support in FlexeLint is not quite up to our
standards of creative locking.



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2008-11-09 14:25:22 UTC (rev 3364)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2008-11-09 14:26:24 UTC (rev 3365)
@@ -615,9 +615,25 @@
 #define MTX			pthread_mutex_t
 #define MTX_INIT(foo)		AZ(pthread_mutex_init(foo, NULL))
 #define MTX_DESTROY(foo)	AZ(pthread_mutex_destroy(foo))
+
+#ifdef __flexelint_v9__
 #define TRYLOCK(foo, r)						\
 do {								\
 	(r) = pthread_mutex_trylock(foo);			\
+} while (0)
+#define LOCK(foo)						\
+do {								\
+	AZ(pthread_mutex_lock(foo));				\
+} while (0)
+#define UNLOCK(foo)						\
+do {								\
+	AZ(pthread_mutex_unlock(foo));				\
+} while (0)
+
+#else
+#define TRYLOCK(foo, r)						\
+do {								\
+	(r) = pthread_mutex_trylock(foo);			\
 	assert(r == 0 || r == EBUSY);				\
 	if (params->diag_bitmap & 0x8) {			\
 		VSL(SLT_Debug, 0,				\
@@ -652,6 +668,7 @@
 		    "MTX_UNLOCK(%s,%s,%d," #foo ")",		\
 		    __func__, __FILE__, __LINE__);		\
 } while (0)
+#endif
 
 #if defined(HAVE_PTHREAD_MUTEX_ISOWNED_NP)
 #define ALOCKED(mutex)		AN(pthread_mutex_isowned_np((mutex)))
@@ -693,8 +710,7 @@
 {
 
 	Tcheck(t);
-	return
-	    ((unsigned)(t.e - t.b));
+	return ((unsigned)(t.e - t.b));
 }
 
 static inline void

Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor_poll.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_acceptor_poll.c	2008-11-09 14:25:22 UTC (rev 3364)
+++ trunk/varnish-cache/bin/varnishd/cache_acceptor_poll.c	2008-11-09 14:26:24 UTC (rev 3365)
@@ -55,7 +55,7 @@
 vca_pollspace(unsigned fd)
 {
 	struct pollfd *newpollfd = pollfd;
-	unsigned newnpoll = npoll;
+	unsigned newnpoll;
 
 	if (fd < npoll)
 		return;

Modified: trunk/varnish-cache/bin/varnishd/flint.lnt
===================================================================
--- trunk/varnish-cache/bin/varnishd/flint.lnt	2008-11-09 14:25:22 UTC (rev 3364)
+++ trunk/varnish-cache/bin/varnishd/flint.lnt	2008-11-09 14:26:24 UTC (rev 3365)
@@ -1,5 +1,47 @@
--passes=3
 
+-d__flexelint_v9__=1
+
+//-sem (pthread_mutex_lock, thread_lock)
+-sem (pthread_mutex_trylock, thread_lock)
+-sem (VBE_DropRefLocked, thread_unlock)
+-e459	// unlocked access from func-ptr
+-e454	// mutex not released (...ReleaseLocked)
+-e457	// unprotected access
+
+-esym(458, lbv_assert)	// unlocked access
+-esym(458, params)	// unlocked access
+
+-emacro(835, HTTPH)	// Info 835: A zero has been given as left argument to operator '&'
+-emacro(845, HTTPH)	// Info 845: The left argument to operator '&&' is certain to be 0
+//////////////
+-efunc(1791, pdiff)	// return last on line
+//////////////
+-efile(451, "sys/*.h")	// No include guard
+-efile(451, "machine/*.h")	// No include guard
+-efile(451, "vcl_returns.h")	// No include guard
+-efile(451, "cache_backend_poll.h")	// No include guard
+-efile(451, "steps.h")	// No include guard
+-efile(451, "http_headers.h")	// No include guard
+-efile(451, "stat_field.h")	// No include guard
+-efile(451, "acct_fields.h")	// No include guard
+-efile(451, "config.h")	// No include guard
+//////////////
+// -e458			// unprotected access
+// -e456			// merged locking paths
+-sem(vca_thread_acct, thread_mono)
+-sem(vca_epoll_thread, thread_mono)
+-sem(vca_kqueue_thread, thread_mono)
+-sem(vca_poll_thread, thread_mono)
+-sem(vca_ports_thread, thread_mono)
+-sem(exp_timer, thread_mono)
+-sem(wrk_herdtimer_thread, thread_mono)
+-sem(wrk_herder_thread, thread_mono)
+-esym(458, VSL_stats)
+-esym(458, heritage)
+-esym(458, name_key)
+//////////////
+-passes=1
+
 +libh mgt_event.h
 +libh ../../config.h
 



From phk at projects.linpro.no  Mon Nov 10 09:29:53 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 10 Nov 2008 10:29:53 +0100 (CET)
Subject: r3366 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081110092953.2C6061EC535@projects.linpro.no>

Author: phk
Date: 2008-11-10 10:29:52 +0100 (Mon, 10 Nov 2008)
New Revision: 3366

Modified:
   trunk/varnish-cache/bin/varnishd/cache_center.c
Log:
Add a debug CLI command to seed random(3).

This is a lot less useful than it could have been, as the Open Group
only mandates that:

	Like rand(), random() shall produce by default a sequence
	of numbers that can be duplicated by calling srandom() with
	1 as the seed.

But crucially leaves out *which* sequence of numbers.



Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2008-11-09 14:26:24 UTC (rev 3365)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2008-11-10 09:29:52 UTC (rev 3366)
@@ -1055,9 +1055,23 @@
 	cli_out(cli, "XID is %u", xids);
 }
 
+static void
+cli_debug_srandom(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)priv;
+	unsigned long seed;
+
+	if (av[2] != NULL)
+		seed = strtoul(av[2], NULL, 0);
+	srandom(seed);
+	cli_out(cli, "Random(3) seeded with %lu", seed);
+}
+
 static struct cli_proto debug_cmds[] = {
 	{ "debug.xid", "debug.xid",
 		"\tExamine or set XID\n", 0, 1, cli_debug_xid },
+	{ "debug.srandom", "debug.srandom",
+		"\tSeed the random(3) function\n", 0, 1, cli_debug_srandom },
 	{ NULL }
 };
 



From phk at projects.linpro.no  Mon Nov 10 09:37:21 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 10 Nov 2008 10:37:21 +0100 (CET)
Subject: r3367 - trunk/varnish-cache/bin/varnishtest
Message-ID: <20081110093721.DEA321EC6D7@projects.linpro.no>

Author: phk
Date: 2008-11-10 10:37:21 +0100 (Mon, 10 Nov 2008)
New Revision: 3367

Modified:
   trunk/varnish-cache/bin/varnishtest/vtc.c
Log:
Add a toplevel word which examines the sequence returned by
srandom(1) and stops the test if we do not get the same sequence
as we expect.

The Open Group does not define which deterministic sequence srandom(1)
should result in, on that it be deterministic, but I have high hopes
in the general sanity and expect that UNIX people across the board
have realized that for portability the same sequence should be
returned on all platforms.

At the very least FreeBSD and Linux/GLIBC, as seen on projects.linpro.no,
agree.



Modified: trunk/varnish-cache/bin/varnishtest/vtc.c
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc.c	2008-11-10 09:29:52 UTC (rev 3366)
+++ trunk/varnish-cache/bin/varnishtest/vtc.c	2008-11-10 09:37:21 UTC (rev 3367)
@@ -46,6 +46,8 @@
 const char	*vtc_file;
 char		*vtc_desc;
 
+static int	stop;
+
 /**********************************************************************
  * Read a file into memory
  */
@@ -188,6 +190,8 @@
 
 		assert(cp->cmd != NULL);
 		cp->cmd(token_s, priv, cmd, vl);
+		if (stop)
+			break;
 	}
 }
 
@@ -285,6 +289,52 @@
 }
 
 /**********************************************************************
+ * Check random generator
+ */
+
+#define NRNDEXPECT	12
+static const unsigned long random_expect[NRNDEXPECT] = {
+	1804289383,	846930886,	1681692777,	1714636915,
+	1957747793,	424238335,	719885386,	1649760492,
+	 596516649,	1189641421,	1025202362,	1350490027
+};
+
+#define RND_NEXT_1K	0x3bdcbe30
+
+static void
+cmd_random(CMD_ARGS)
+{
+	unsigned long l;
+	int i;
+
+	(void)cmd;
+	(void)priv;
+	if (av == NULL)
+		return;
+	srandom(1);
+	for (i = 0; i < NRNDEXPECT; i++) {
+		l = random();
+		if (l == random_expect[i])
+			continue;
+		vtc_log(vl, 4, "random[%d] = 0x%x (expect 0x%x)",
+		    i, l, random_expect[i]);
+		vtc_log(vl, 1, "SKIPPING test: unknown srandom(1) sequence.");
+		stop = 1;
+		break;
+	}
+	l = 0;
+	for (i = 0; i < 1000; i++) 
+		l += random();
+	if (l != RND_NEXT_1K) {
+		vtc_log(vl, 4, "sum(random[%d...%d]) = 0x%x (expect 0x%x)",
+		    NRNDEXPECT, NRNDEXPECT + 1000,
+		    l, RND_NEXT_1K);
+		vtc_log(vl, 1, "SKIPPING test: unknown srandom(1) sequence.");
+		stop = 1;
+	}
+}
+
+/**********************************************************************
  * Execute a file
  */
 
@@ -296,6 +346,7 @@
 	{ "test",	cmd_test },
 	{ "shell",	cmd_shell },
 	{ "sema",	cmd_sema },
+	{ "random",	cmd_random },
 	{ NULL,		NULL }
 };
 
@@ -304,6 +355,7 @@
 {
 	char *buf;
 
+	stop = 0;
 	vtc_file = fn;
 	vtc_desc = NULL;
 	vtc_log(vl, 1, "TEST %s starting", fn);



From phk at projects.linpro.no  Mon Nov 10 09:40:39 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 10 Nov 2008 10:40:39 +0100 (CET)
Subject: r3368 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <20081110094039.C54961ED1C0@projects.linpro.no>

Author: phk
Date: 2008-11-10 10:40:39 +0100 (Mon, 10 Nov 2008)
New Revision: 3368

Added:
   trunk/varnish-cache/bin/varnishtest/tests/v00022.vtc
Log:
Add a test of the random director that uses actual randomness.

This will be skipped on platforms where srandom(1) does not result
in the same deterministic sequence of random numbers as on FreeBSD
and Linux.



Added: trunk/varnish-cache/bin/varnishtest/tests/v00022.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/v00022.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/v00022.vtc	2008-11-10 09:40:39 UTC (rev 3368)
@@ -0,0 +1,86 @@
+# $Id$
+
+test "Deeper test of random director"
+
+random
+
+server s1 {
+	rxreq
+	txresp -body "1"
+} -start
+server s2 -listen 127.0.0.1:9180 {
+	rxreq
+	txresp -body "22"
+	rxreq
+	txresp -body "22"
+	rxreq
+	txresp -body "22"
+} -start
+server s3 -listen 127.0.0.1:9181 {
+	rxreq
+	txresp -body "333"
+} -start
+server s4 -listen 127.0.0.1:9182 {
+	rxreq
+	txresp -body "4444"
+	rxreq
+	txresp -body "4444"
+	rxreq
+	txresp -body "4444"
+	rxreq
+	txresp -body "4444"
+	rxreq
+	txresp -body "4444"
+} -start
+
+varnish v1 -vcl+backend {
+	director foo random {
+		{ .backend = s1; .weight = 1; }
+		{ .backend = s2; .weight = 1; }
+		{ .backend = s3; .weight = 1; }
+		{ .backend = s4; .weight = 1; }
+	}
+
+	sub vcl_recv {
+		set req.backend = foo;
+		pass;
+	}
+} -start
+
+# NB: Do not change the number 1
+# NB: Only srandom(1) is standardized as deterministic.
+
+varnish v1 -cliok "debug.srandom 1"
+
+client c1 {
+	txreq 
+	rxresp
+	expect resp.bodylen == 4
+	txreq 
+	rxresp
+	expect resp.bodylen == 2
+	txreq 
+	rxresp
+	expect resp.bodylen == 4
+	txreq 
+	rxresp
+	expect resp.bodylen == 4
+	txreq 
+	rxresp
+	expect resp.bodylen == 4
+	txreq 
+	rxresp
+	expect resp.bodylen == 1
+	txreq 
+	rxresp
+	expect resp.bodylen == 2
+	txreq 
+	rxresp
+	expect resp.bodylen == 4
+	txreq 
+	rxresp
+	expect resp.bodylen == 2
+	txreq 
+	rxresp
+	expect resp.bodylen == 3
+} -run



From tfheen at projects.linpro.no  Mon Nov 10 10:07:29 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Mon, 10 Nov 2008 11:07:29 +0100 (CET)
Subject: r3369 - branches/2.0/varnish-cache/bin/varnishd
Message-ID: <20081110100729.CDC321EC6D7@projects.linpro.no>

Author: tfheen
Date: 2008-11-10 11:07:29 +0100 (Mon, 10 Nov 2008)
New Revision: 3369

Modified:
   branches/2.0/varnish-cache/bin/varnishd/cache_dir_random.c
Log:
Merge r3359: Fix the broken logic in the random directors picking routine.

Drop the consistency check for health-changes, at the cost of a
slight bias[1] for hosts before, or after, then one that changed health,
according to how its health changed.

Count all failures to pick against the retry count.

Fixes #361 



Modified: branches/2.0/varnish-cache/bin/varnishd/cache_dir_random.c
===================================================================
--- branches/2.0/varnish-cache/bin/varnishd/cache_dir_random.c	2008-11-10 09:40:39 UTC (rev 3368)
+++ branches/2.0/varnish-cache/bin/varnishd/cache_dir_random.c	2008-11-10 10:07:29 UTC (rev 3369)
@@ -66,9 +66,9 @@
 static struct vbe_conn *
 vdi_random_getfd(struct sess *sp)
 {
-	int i, j, k;
+	int i, k;
 	struct vdi_random *vs;
-	double r, s1, s2;
+	double r, s1;
 	struct vbe_conn *vbe;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -78,45 +78,34 @@
 	k = 0;
 	for (k = 0; k < vs->retries; ) {
 
-		r = random() / 2147483648.0;	/* 2^31 */
-		assert(r >= 0.0 && r < 1.0);
-
+		/* Sum up the weights of healty backends */
 		s1 = 0.0;
-		j = 0;
-		for (i = 0; i < vs->nhosts; i++) {
-			if (!vs->hosts[i].backend->healthy)
-				continue;
-			s1 += vs->hosts[i].weight;
-			j++;
-		}	
+		for (i = 0; i < vs->nhosts; i++)
+			if (vs->hosts[i].backend->healthy)
+				s1 += vs->hosts[i].weight;
 
-		if (j == 0)		/* No healthy hosts */
+		if (s1 == 0.0)
 			return (NULL);
 
+		/* Pick a random threshold in that interval */
+		r = random() / 2147483648.0;	/* 2^31 */
+		assert(r >= 0.0 && r < 1.0);
 		r *= s1;
 
-		s2 = 0;
+		s1 = 0.0;
 		for (i = 0; i < vs->nhosts; i++)  {
 			if (!vs->hosts[i].backend->healthy)
 				continue;
-			s2 += vs->hosts[i].weight;
-			if (r < s2)
-				break;
+			s1 += vs->hosts[i].weight;
+			if (r >= s1)
+				continue;
+			vbe = VBE_GetVbe(sp, vs->hosts[i].backend);
+			if (vbe != NULL)
+				return (vbe);
+			break;
 		}
-
-		if (s2 != s1) {
-			/*
-			 * Health bit changed in an unusable way while we
-			 * worked the problem.  Usable changes are any that
-			 * result in the same sum we prepared for.
-			 */
-			continue;
-		}
-		vbe = VBE_GetVbe(sp, vs->hosts[i].backend);
-		if (vbe != NULL)
-			return (vbe);
 		k++;
-	} 
+	}
 	return (NULL);
 }
 



From tfheen at projects.linpro.no  Mon Nov 10 10:09:32 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Mon, 10 Nov 2008 11:09:32 +0100 (CET)
Subject: r3370 - in branches/2.0/varnish-cache: bin/varnishd bin/varnishtest
	include
Message-ID: <20081110100932.1F0AC1EC6D7@projects.linpro.no>

Author: tfheen
Date: 2008-11-10 11:09:31 +0100 (Mon, 10 Nov 2008)
New Revision: 3370

Added:
   branches/2.0/varnish-cache/bin/varnishtest/c00019.vtc
Modified:
   branches/2.0/varnish-cache/bin/varnishd/cache_ban.c
   branches/2.0/varnish-cache/bin/varnishd/heritage.h
   branches/2.0/varnish-cache/bin/varnishd/mgt_param.c
   branches/2.0/varnish-cache/include/stat_field.h
Log:
Merge r3329, r3330: Add a boolean parameter "purge_dups"

Add a boolean parameter "purge_dups", to make it possible to mark earlier
bans of the same regexp as "gone" to save duplicate regexp checking.

Prodded to by: jodok 

Add a test-case for dup purge elimination 



Modified: branches/2.0/varnish-cache/bin/varnishd/cache_ban.c
===================================================================
--- branches/2.0/varnish-cache/bin/varnishd/cache_ban.c	2008-11-10 10:07:29 UTC (rev 3369)
+++ branches/2.0/varnish-cache/bin/varnishd/cache_ban.c	2008-11-10 10:09:31 UTC (rev 3370)
@@ -49,6 +49,8 @@
 #define BAN_MAGIC		0x700b08ea
 	VTAILQ_ENTRY(ban)	list;
 	unsigned		refcount;
+	int			flags;
+#define BAN_F_GONE		(1 << 0)
 	regex_t			regexp;
 	char			*ban;
 	int			hash;
@@ -68,8 +70,9 @@
 int
 BAN_Add(struct cli *cli, const char *regexp, int hash)
 {
-	struct ban *b;
+	struct ban *b, *bi, *be;
 	char buf[512];
+	unsigned pcount;
 	int i;
 
 	ALLOC_OBJ(b, BAN_MAGIC);
@@ -97,8 +100,37 @@
 	ban_start = b;
 	VSL_stats->n_purge++;
 	VSL_stats->n_purge_add++;
+
+	if (params->purge_dups) {
+		be = VTAILQ_LAST(&ban_head, banhead);
+		be->refcount++;
+	} else
+		be = NULL;
 	UNLOCK(&ban_mtx);
 
+	if (be == NULL)
+		return (0);
+
+	/* Hunt down duplicates, and mark them as gone */
+	bi = b;
+	pcount = 0;
+	while(bi != be) {
+		bi = VTAILQ_NEXT(bi, list);
+		if (bi->flags & BAN_F_GONE)
+			continue;
+		if (b->hash != bi->hash)
+			continue;
+		if (strcmp(b->ban, bi->ban))
+			continue;
+		bi->flags |= BAN_F_GONE;
+		pcount++;
+	}
+	LOCK(&ban_mtx);
+	be->refcount--;
+	/* XXX: We should check if the tail can be removed */
+	VSL_stats->n_purge_dups += pcount;
+	UNLOCK(&ban_mtx);
+
 	return (0);
 }
 
@@ -168,7 +200,8 @@
 	tests = 0;
 	for (b = b0; b != o->ban; b = VTAILQ_NEXT(b, list)) {
 		tests++;
-		if (!regexec(&b->regexp, b->hash ? hash : url, 0, NULL, 0))
+		if (!(b->flags & BAN_F_GONE) &&
+		    !regexec(&b->regexp, b->hash ? hash : url, 0, NULL, 0))
 			break;
 	}
 
@@ -227,8 +260,8 @@
 	for (b0 = ban_start; b0 != NULL; b0 = VTAILQ_NEXT(b0, list)) {
 		if (b0->refcount == 0 && VTAILQ_NEXT(b0, list) == NULL)
 			break;
-		cli_out(cli, "%5u %s \"%s\"\n",
-		    b0->refcount,
+		cli_out(cli, "%5u %d %s \"%s\"\n",
+		    b0->refcount, b0->flags,
 		    b0->hash ? "hash" : "url ",
 		    b0->ban);
 	}

Modified: branches/2.0/varnish-cache/bin/varnishd/heritage.h
===================================================================
--- branches/2.0/varnish-cache/bin/varnishd/heritage.h	2008-11-10 10:07:29 UTC (rev 3369)
+++ branches/2.0/varnish-cache/bin/varnishd/heritage.h	2008-11-10 10:09:31 UTC (rev 3370)
@@ -183,6 +183,9 @@
 	/* Amount of time to sleep when running out of file
 	   descriptors.  In msecs */
 	unsigned		accept_fd_holdoff;
+
+	/* Get rid of duplicate purges */
+	unsigned		purge_dups;
 };
 
 extern volatile struct params *params;

Modified: branches/2.0/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- branches/2.0/varnish-cache/bin/varnishd/mgt_param.c	2008-11-10 10:07:29 UTC (rev 3369)
+++ branches/2.0/varnish-cache/bin/varnishd/mgt_param.c	2008-11-10 10:09:31 UTC (rev 3370)
@@ -817,6 +817,10 @@
 		"The TTL assigned to the synthesized error pages\n",
 		0,
 		"0", "seconds" },
+	{ "purge_dups", tweak_bool, &master.purge_dups, 0, 0,
+		"Detect and eliminate duplicate purges.\n",
+		0,
+		"off", "bool" },
 	{ NULL, NULL, NULL }
 };
 

Copied: branches/2.0/varnish-cache/bin/varnishtest/c00019.vtc (from rev 3330, trunk/varnish-cache/bin/varnishtest/c00019.vtc)
===================================================================
--- branches/2.0/varnish-cache/bin/varnishtest/c00019.vtc	                        (rev 0)
+++ branches/2.0/varnish-cache/bin/varnishtest/c00019.vtc	2008-11-10 10:09:31 UTC (rev 3370)
@@ -0,0 +1,83 @@
+# $Id$
+
+test "Check purge counters and duplicate purge elimination"
+
+server s1 {
+	rxreq
+	txresp -hdr "foo: 1" -body "foo1"
+	rxreq
+	txresp -hdr "foo: 2" -body "foo2"
+	rxreq
+	txresp -hdr "foo: 3" -body "foo3"
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+varnish v1 -cliok "purge.url FOO"
+
+# There is one "magic" purge from boot
+varnish v1 -expect n_purge_add == 2
+varnish v1 -cliok "purge.list"
+
+# Our fetch is not affected by the purge
+# as the FOO-purge was preexisting
+client c1 {
+	txreq -url /FOO
+	rxresp
+	expect resp.http.foo == 1
+} -run
+
+varnish v1 -cliok "purge.list"
+varnish v1 -expect n_purge_obj_test == 0
+varnish v1 -expect n_purge_re_test == 0
+
+# Add another purge
+varnish v1 -cliok "purge.url FOO"
+varnish v1 -expect n_purge_add == 3
+varnish v1 -cliok "purge.list"
+
+# The cached object will be purged, and a new
+# fetched from the backend
+client c1 {
+	txreq -url /FOO
+	rxresp
+	expect resp.http.foo == 2
+} -run
+
+varnish v1 -expect n_purge_obj_test == 1
+varnish v1 -expect n_purge_re_test == 1
+varnish v1 -cliok "purge.list"
+
+# Fetch the cached copy, just for grins
+client c1 {
+	txreq -url /FOO
+	rxresp
+	expect resp.http.foo == 2
+} -run
+
+
+# Now add another purge
+varnish v1 -cliok "purge.url FOO"
+varnish v1 -expect n_purge_add == 4
+
+# Enable dup removal of purges
+varnish v1 -cliok "param.set purge_dups on"
+
+# This should incapacitate the to previous FOO purges.
+varnish v1 -cliok "purge.url FOO"
+varnish v1 -expect n_purge_add == 5
+varnish v1 -expect n_purge_dups == 3
+varnish v1 -cliok "purge.list"
+
+# And we should get a fresh object from backend
+client c1 {
+	txreq -url /FOO
+	rxresp
+	expect resp.http.foo == 3
+} -run
+
+# With only two objects having ever been compared
+varnish v1 -expect n_purge_obj_test == 2
+varnish v1 -expect n_purge_re_test == 2
+varnish v1 -cliok "purge.list"
+

Modified: branches/2.0/varnish-cache/include/stat_field.h
===================================================================
--- branches/2.0/varnish-cache/include/stat_field.h	2008-11-10 10:07:29 UTC (rev 3369)
+++ branches/2.0/varnish-cache/include/stat_field.h	2008-11-10 10:09:31 UTC (rev 3370)
@@ -124,3 +124,4 @@
 MAC_STAT(n_purge_retire,	uint64_t, 'a', "N old purges deleted")
 MAC_STAT(n_purge_obj_test,	uint64_t, 'a', "N objects tested")
 MAC_STAT(n_purge_re_test,	uint64_t, 'a', "N regexps tested against")
+MAC_STAT(n_purge_dups,		uint64_t, 'a', "N duplicate purges removed")



From tfheen at projects.linpro.no  Mon Nov 10 10:10:09 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Mon, 10 Nov 2008 11:10:09 +0100 (CET)
Subject: r3371 - branches/2.0/varnish-cache/bin/varnishncsa
Message-ID: <20081110101009.609801EC535@projects.linpro.no>

Author: tfheen
Date: 2008-11-10 11:10:09 +0100 (Mon, 10 Nov 2008)
New Revision: 3371

Modified:
   branches/2.0/varnish-cache/bin/varnishncsa/varnishncsa.c
Log:
Merge r3344: Use a proper format string. 


Modified: branches/2.0/varnish-cache/bin/varnishncsa/varnishncsa.c
===================================================================
--- branches/2.0/varnish-cache/bin/varnishncsa/varnishncsa.c	2008-11-10 10:09:31 UTC (rev 3370)
+++ branches/2.0/varnish-cache/bin/varnishncsa/varnishncsa.c	2008-11-10 10:10:09 UTC (rev 3371)
@@ -441,7 +441,7 @@
 		if (lp->df_Host) {
 			if (strncmp(lp->df_Host, "http://", 7) != 0)
 				fprintf(fo, "http://");
-			fprintf(fo, lp->df_Host);
+			fprintf(fo, "%s", lp->df_Host);
 		}
 		fprintf(fo, "%s ", lp->df_Uq);
 		fprintf(fo, "%s\" ", lp->df_H);



From tfheen at projects.linpro.no  Mon Nov 10 10:10:49 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Mon, 10 Nov 2008 11:10:49 +0100 (CET)
Subject: r3372 - branches/2.0/varnish-cache/lib/libvcl
Message-ID: <20081110101049.CBC0C1ED2C2@projects.linpro.no>

Author: tfheen
Date: 2008-11-10 11:10:49 +0100 (Mon, 10 Nov 2008)
New Revision: 3372

Modified:
   branches/2.0/varnish-cache/lib/libvcl/vcc_compile.c
Log:
Merge r3346: Don't quote the entire inlined C source in the index table, just show C{

Spotted by: nkallen


Modified: branches/2.0/varnish-cache/lib/libvcl/vcc_compile.c
===================================================================
--- branches/2.0/varnish-cache/lib/libvcl/vcc_compile.c	2008-11-10 10:10:09 UTC (rev 3371)
+++ branches/2.0/varnish-cache/lib/libvcl/vcc_compile.c	2008-11-10 10:10:49 UTC (rev 3372)
@@ -286,8 +286,12 @@
 				pos++;
 
 		}
-		Fc(tl, 0, "  [%3u] = { %d, %8u, %4u, %3u, 0, \"%.*s\" },\n",
-		    t->cnt, sp->idx, t->b - sp->b, lin, pos + 1, PF(t));
+		Fc(tl, 0, "  [%3u] = { %d, %8u, %4u, %3u, 0, ",
+		    t->cnt, sp->idx, t->b - sp->b, lin, pos + 1);
+		if (t->tok == CSRC)
+			Fc(tl, 0, " \"C{\"},\n");
+		else
+			Fc(tl, 0, " \"%.*s\" },\n", PF(t));
 	}
 	Fc(tl, 0, "};\n");
 }



From tfheen at projects.linpro.no  Mon Nov 10 10:12:00 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Mon, 10 Nov 2008 11:12:00 +0100 (CET)
Subject: r3373 - branches/2.0/varnish-cache/bin/varnishd
Message-ID: <20081110101200.E98411ED1C0@projects.linpro.no>

Author: tfheen
Date: 2008-11-10 11:12:00 +0100 (Mon, 10 Nov 2008)
New Revision: 3373

Modified:
   branches/2.0/varnish-cache/bin/varnishd/cache_vrt.c
Log:
Merge r3353, r3354: resp.status was returning obj.status 



Modified: branches/2.0/varnish-cache/bin/varnishd/cache_vrt.c
===================================================================
--- branches/2.0/varnish-cache/bin/varnishd/cache_vrt.c	2008-11-10 10:10:49 UTC (rev 3372)
+++ branches/2.0/varnish-cache/bin/varnishd/cache_vrt.c	2008-11-10 10:12:00 UTC (rev 3373)
@@ -284,8 +284,8 @@
 VRT_r_resp_status(const struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-	return (atoi(sp->obj->http->hd[HTTP_HDR_STATUS].b));
+	CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC);
+	return (atoi(sp->http->hd[HTTP_HDR_STATUS].b));
 }
 
 /*--------------------------------------------------------------------*/



From tfheen at projects.linpro.no  Mon Nov 10 10:12:28 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Mon, 10 Nov 2008 11:12:28 +0100 (CET)
Subject: r3374 - branches/2.0/varnish-cache/bin/varnishtest
Message-ID: <20081110101228.6CFE01EC535@projects.linpro.no>

Author: tfheen
Date: 2008-11-10 11:12:28 +0100 (Mon, 10 Nov 2008)
New Revision: 3374

Added:
   branches/2.0/varnish-cache/bin/varnishtest/varnishtest.1
Modified:
   branches/2.0/varnish-cache/bin/varnishtest/Makefile.am
Log:
Merge r3356: Add a varnishtest man page 


Modified: branches/2.0/varnish-cache/bin/varnishtest/Makefile.am
===================================================================
--- branches/2.0/varnish-cache/bin/varnishtest/Makefile.am	2008-11-10 10:12:00 UTC (rev 3373)
+++ branches/2.0/varnish-cache/bin/varnishtest/Makefile.am	2008-11-10 10:12:28 UTC (rev 3374)
@@ -9,6 +9,8 @@
 
 bin_PROGRAMS =	varnishtest
 
+dist_man_MANS = varnishtest.1
+
 varnishtest_SOURCES = \
 		vtc.c \
 		vtc.h \

Copied: branches/2.0/varnish-cache/bin/varnishtest/varnishtest.1 (from rev 3356, trunk/varnish-cache/bin/varnishtest/varnishtest.1)
===================================================================
--- branches/2.0/varnish-cache/bin/varnishtest/varnishtest.1	                        (rev 0)
+++ branches/2.0/varnish-cache/bin/varnishtest/varnishtest.1	2008-11-10 10:12:28 UTC (rev 3374)
@@ -0,0 +1,220 @@
+.\"-
+.\" Copyright (c) 2006-2008 Linpro AS
+.\" All rights reserved.
+.\"
+.\" Author: Stig Sandbeck Mathisen <ssm at linpro.no>
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $Id$
+.\"
+.Dd October 29, 2008
+.Dt VARNISHTEST 1
+.Os
+.Sh NAME
+.Nm varnishtest
+.Nd Test program for Varnish
+.Sh SYNOPSIS
+.Nm
+.Op Fl n Ar iter
+.Op Fl q
+.Op Fl v
+.Ar file
+.Op Ar file ...
+.Sh DESCRIPTION
+The
+.Nm
+program is a script driven program used to test the varnish HTTP
+accelerator.
+.Pp
+The
+.Nm
+program, when started and given one or more script files, can create a
+number of threads representing backends, some threads representing
+clients, and a varnishd process.
+.Pp
+The following options are available:
+.Bl -tag -width Fl
+.It Fl n Ar iter
+Run 
+.Ar iter
+number of iterations.
+.It Fl q
+Be quiet.
+.It Fl v
+Be verbose.
+.It Ar file
+File to use as a script
+.El
+.Sh SCRIPTS
+.Ss Example script
+.Bd -literal -offset 4n
+# Start a varnish instance called "v1"
+varnish v1 -arg "-b localhost:9080" -start
+
+# Create a server thread called "s1"
+server s1 {
+    # Receive a request
+    rxreq 
+    # Send a standard response
+    txresp -hdr "Connection: close" -body "012345\\n"
+}
+
+# Start the server thread
+server s1 -start 
+
+# Create a client thread called "c1"
+client c1 {
+   # Send a request
+   txreq -url "/"
+   # Wait for a response
+   rxresp
+   # Insist that it be a success
+   expect resp.status == 200
+}
+
+# Run the client
+client c1 -run
+
+# Wait for the server to die
+server s1 -wait
+
+# (Forcefully) Stop the varnish instance.
+varnish v1 -stop
+.Ed
+
+.Ss Example script output
+The output, running this script looks as follows.
+.Pp
+The "bargraph" at the beginning of the line is an indication of the
+level of detail in the line.
+.Pp
+The second field where the message comes from
+.Pp
+The rest of the line is anyones guess :-)
+.Bd -literal -offset 4n
+#    TEST tests/b00000.vtc starting
+###  v1   CMD: cd ../varnishd && ./varnishd -d -d -n v1 -a :9081 -T :9001 -b localhost:9080
+###  v1   opening CLI connection
+#### v1   debug| NB: Storage size limited to 2GB on 32 bit architecture,\\n
+#### v1   debug| NB: otherwise we could run out of address space.\\n
+#### v1   debug| storage_file: filename: ./varnish.Shkoq5 (unlinked) size 2047 MB.\\n
+###  v1   CLI connection fd = 3
+#### v1   CLI TX| start
+#### v1   debug| Using old SHMFILE\\n
+#### v1   debug| Notice: locking SHMFILE in core failed: Operation not permitted\\n
+#### v1   debug| bind(): Address already in use\\n
+#### v1   debug| rolling(1)...
+#### v1   debug| \\n
+#### v1   debug| rolling(2)...\\n
+#### v1   debug| Debugging mode, enter "start" to start child\\n
+###  v1   CLI 200 <start>
+##   s1   Starting server
+###  s1   listen on :9080 (fd 6)
+##   c1   Starting client
+##   c1   Waiting for client
+##   s1   started on :9080
+##   c1   started
+###  c1   connect to :9081
+###  c1   connected to :9081 fd is 8
+#### c1   | GET / HTTP/1.1\\r\\n
+#### c1   | \\r\\n
+###  c1   rxresp
+#### s1   Accepted socket 7
+###  s1   rxreq
+#### s1   | GET / HTTP/1.1\\r\\n
+#### s1   | X-Varnish: 422080121\\r\\n
+#### s1   | X-Forwarded-For: 127.0.0.1\\r\\n
+#### s1   | Host: localhost\\r\\n
+#### s1   | \\r\\n
+#### s1   http[ 0] | GET
+#### s1   http[ 1] | /
+#### s1   http[ 2] | HTTP/1.1
+#### s1   http[ 3] | X-Varnish: 422080121
+#### s1   http[ 4] | X-Forwarded-For: 127.0.0.1
+#### s1   http[ 5] | Host: localhost
+#### s1   | HTTP/1.1 200 Ok\\r\\n
+#### s1   | Connection: close\\r\\n
+#### s1   | \\r\\n
+#### s1   | 012345\\n
+#### s1   | \\r\\n
+##   s1   ending
+#### c1   | HTTP/1.1 200 Ok\\r\\n
+#### c1   | Content-Length: 9\\r\\n
+#### c1   | Date: Mon, 16 Jun 2008 22:16:55 GMT\\r\\n
+#### c1   | X-Varnish: 422080121\\r\\n
+#### c1   | Age: 0\\r\\n
+#### c1   | Via: 1.1 varnish\\r\\n
+#### c1   | Connection: keep-alive\\r\\n
+#### c1   | \\r\\n
+#### c1   http[ 0] | HTTP/1.1
+#### c1   http[ 1] | 200
+#### c1   http[ 2] | Ok
+#### c1   http[ 3] | Content-Length: 9
+#### c1   http[ 4] | Date: Mon, 16 Jun 2008 22:16:55 GMT
+#### c1   http[ 5] | X-Varnish: 422080121
+#### c1   http[ 6] | Age: 0
+#### c1   http[ 7] | Via: 1.1 varnish
+#### c1   http[ 8] | Connection: keep-alive
+#### c1   EXPECT resp.status (200) == 200 (200) match
+##   c1   ending
+##   s1   Waiting for server
+#### v1   CLI TX| stop
+###  v1   CLI 200 <stop>
+#    TEST tests/b00000.vtc completed
+.Ed
+.Pp
+If instead of 200 we had expected 201 with the line:
+.Bd -literal -offset 4n
+expect resp.status == 201
+.Ed
+.Pp
+The output would have ended with:
+.Bd -literal -offset 4n
+#### c1   http[ 0] | HTTP/1.1
+#### c1   http[ 1] | 200
+#### c1   http[ 2] | Ok
+#### c1   http[ 3] | Content-Length: 9
+#### c1   http[ 4] | Date: Mon, 16 Jun 2008 22:26:35 GMT
+#### c1   http[ 5] | X-Varnish: 648043653 648043652
+#### c1   http[ 6] | Age: 6
+#### c1   http[ 7] | Via: 1.1 varnish
+#### c1   http[ 8] | Connection: keep-alive
+---- c1   EXPECT resp.status (200) == 201 (201) failed
+.Ed
+.Sh SEE ALSO
+.Xr varnishlog 1 ,
+.Xr varnishhist 1 ,
+.Xr varnishncsa 1 ,
+.Xr varnishstat 1 ,
+.Xr varnishtop 1 ,
+.Xr vcl 7
+.Sh HISTORY
+The
+.Nm
+program was developed by
+.An Poul-Henning Kamp Aq phk at phk.freebsd.dk
+in cooperation with Verdens Gang AS and Linpro AS.  This manual page
+was written by
+.An Stig Sandbeck Mathisen Aq ssm at linpro.no
+using examples by
+.An Poul-Henning Kamp Aq phk at phk.freebsd.dk



From tfheen at projects.linpro.no  Mon Nov 10 10:12:56 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Mon, 10 Nov 2008 11:12:56 +0100 (CET)
Subject: r3375 - branches/2.0/varnish-cache/bin/varnishtest
Message-ID: <20081110101256.89E3C1ED1C0@projects.linpro.no>

Author: tfheen
Date: 2008-11-10 11:12:56 +0100 (Mon, 10 Nov 2008)
New Revision: 3375

Modified:
   branches/2.0/varnish-cache/bin/varnishtest/varnishtest.1
Log:
Merge r3357: varnishtest man page: whitespace fix 


Modified: branches/2.0/varnish-cache/bin/varnishtest/varnishtest.1
===================================================================
--- branches/2.0/varnish-cache/bin/varnishtest/varnishtest.1	2008-11-10 10:12:28 UTC (rev 3374)
+++ branches/2.0/varnish-cache/bin/varnishtest/varnishtest.1	2008-11-10 10:12:56 UTC (rev 3375)
@@ -55,7 +55,7 @@
 The following options are available:
 .Bl -tag -width Fl
 .It Fl n Ar iter
-Run 
+Run
 .Ar iter
 number of iterations.
 .It Fl q
@@ -74,13 +74,13 @@
 # Create a server thread called "s1"
 server s1 {
     # Receive a request
-    rxreq 
+    rxreq
     # Send a standard response
     txresp -hdr "Connection: close" -body "012345\\n"
 }
 
 # Start the server thread
-server s1 -start 
+server s1 -start
 
 # Create a client thread called "c1"
 client c1 {
@@ -101,7 +101,6 @@
 # (Forcefully) Stop the varnish instance.
 varnish v1 -stop
 .Ed
-
 .Ss Example script output
 The output, running this script looks as follows.
 .Pp



From tfheen at projects.linpro.no  Mon Nov 10 10:14:19 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Mon, 10 Nov 2008 11:14:19 +0100 (CET)
Subject: r3376 - branches/2.0/varnish-cache/redhat
Message-ID: <20081110101419.AFA781EC535@projects.linpro.no>

Author: tfheen
Date: 2008-11-10 11:14:19 +0100 (Mon, 10 Nov 2008)
New Revision: 3376

Modified:
   branches/2.0/varnish-cache/redhat/varnish.spec
Log:
Merge r3358: Removed the requirement for kernel => 2.6.0

* Sun Nov 02 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0.1-2
- Removed the requirement for kernel => 2.6.0. All supported

    platforms meets this, and it generates strange errors in EPEL 


Modified: branches/2.0/varnish-cache/redhat/varnish.spec
===================================================================
--- branches/2.0/varnish-cache/redhat/varnish.spec	2008-11-10 10:12:56 UTC (rev 3375)
+++ branches/2.0/varnish-cache/redhat/varnish.spec	2008-11-10 10:14:19 UTC (rev 3376)
@@ -1,7 +1,7 @@
 Summary: Varnish is a high-performance HTTP accelerator
 Name: varnish
 Version: 2.0
-Release: 1%{?dist}
+Release: 2%{?dist}
 License: BSD
 Group: System Environment/Daemons
 URL: http://www.varnish-cache.org/
@@ -11,7 +11,7 @@
 # configure script. Release tarballs would not need this
 #BuildRequires: automake autoconf libtool
 BuildRequires: ncurses-devel libxslt groff
-Requires: kernel >= 2.6.0 varnish-libs = %{version}-%{release}
+Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses
 Requires(pre): shadow-utils
@@ -43,7 +43,7 @@
 Summary: Development files for %{name}-libs
 Group: System Environment/Libraries
 BuildRequires: ncurses-devel
-Requires: kernel >= 2.6.0 varnish-libs = %{version}-%{release}
+Requires: varnish-libs = %{version}-%{release}
 
 %description libs-devel
 Development files for %{name}-libs
@@ -53,7 +53,7 @@
 #Summary: Files for static linking of %{name} library functions
 #Group: System Environment/Libraries
 #BuildRequires: ncurses-devel
-#Requires: kernel >= 2.6.0 varnish-libs-devel = %{version}-%{release}
+#Requires: varnish-libs-devel = %{version}-%{release}
 #
 #%description libs-static
 #Files for static linking of varnish library functions
@@ -220,6 +220,18 @@
 %postun libs -p /sbin/ldconfig
 
 %changelog
+* Sun Nov 02 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0.1-2
+- Removed the requirement for kernel => 2.6.0. All supported
+  platforms meets this, and it generates strange errors in EPEL
+
+* Fri Oct 17 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0.1-1
+- 2.0.1 released, a bugfix release. New upstream sources
+- Package now also available in EPEL
+
+* Thu Oct 16 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0-2
+- Readded the debugflag patch. It's so practical
+- Added a strange workaround for make check on ppc64
+
 * Wed Oct 15 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0-1
 - 2.0 released. New upstream sources
 - Disabled jemalloc on ppc and ppc64. Added a note in README.redhat.



From tfheen at projects.linpro.no  Mon Nov 10 11:55:15 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Mon, 10 Nov 2008 12:55:15 +0100 (CET)
Subject: r3377 - trunk/varnish-cache/lib/libvcl
Message-ID: <20081110115515.F26A81EC6D7@projects.linpro.no>

Author: tfheen
Date: 2008-11-10 12:55:15 +0100 (Mon, 10 Nov 2008)
New Revision: 3377

Modified:
   trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl
Log:
Constify vcl_tnames in .tcl script too

Change the .tcl script corresponding to r3364, fixes build failure


Modified: trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl	2008-11-10 10:14:19 UTC (rev 3376)
+++ trunk/varnish-cache/lib/libvcl/vcc_gen_fixed_token.tcl	2008-11-10 11:55:15 UTC (rev 3377)
@@ -331,7 +331,7 @@
 puts $fo "}"
 
 puts $fo ""
-puts $fo "const char *vcl_tnames\[256\] = {"
+puts $fo "const char * const vcl_tnames\[256\] = {"
 foreach i $token2 {
 	puts $fo "\t\[[lindex $i 0]\] = \"[lindex $i 0]\","
 }



From tfheen at projects.linpro.no  Mon Nov 10 11:55:17 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Mon, 10 Nov 2008 12:55:17 +0100 (CET)
Subject: r3378 - trunk/varnish-cache/lib/libvcl
Message-ID: <20081110115517.ED74F1ED2CC@projects.linpro.no>

Author: tfheen
Date: 2008-11-10 12:55:17 +0100 (Mon, 10 Nov 2008)
New Revision: 3378

Modified:
   trunk/varnish-cache/lib/libvcl/vcc_compile.c
Log:
Make sure the VCL we try to load is a regular file

Fixes #368


Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_compile.c	2008-11-10 11:55:15 UTC (rev 3377)
+++ trunk/varnish-cache/lib/libvcl/vcc_compile.c	2008-11-10 11:55:17 UTC (rev 3378)
@@ -415,6 +415,11 @@
 		}
 	}
 	assert(0 == fstat(fd, &st));
+	if (! S_ISREG(st.st_mode)) {
+		vsb_printf(sb, "File '%s' is not a regular file\n", fn);
+		AZ(close(fd));
+		return (NULL);
+	}
 	f = malloc(st.st_size + 1);
 	assert(f != NULL);
 	i = read(fd, f, st.st_size);



From ingvar at projects.linpro.no  Mon Nov 10 12:34:49 2008
From: ingvar at projects.linpro.no (ingvar at projects.linpro.no)
Date: Mon, 10 Nov 2008 13:34:49 +0100 (CET)
Subject: r3379 - trunk/varnish-cache/redhat
Message-ID: <20081110123449.56DE71ED1C3@projects.linpro.no>

Author: ingvar
Date: 2008-11-10 13:34:49 +0100 (Mon, 10 Nov 2008)
New Revision: 3379

Modified:
   trunk/varnish-cache/redhat/varnish.spec
Log:
Specfile: 2.0.1 >  2.0.2

Modified: trunk/varnish-cache/redhat/varnish.spec
===================================================================
--- trunk/varnish-cache/redhat/varnish.spec	2008-11-10 11:55:17 UTC (rev 3378)
+++ trunk/varnish-cache/redhat/varnish.spec	2008-11-10 12:34:49 UTC (rev 3379)
@@ -1,6 +1,6 @@
 Summary: Varnish is a high-performance HTTP accelerator
 Name: varnish
-Version: 2.0.1
+Version: 2.0.2
 Release: 2%{?dist}
 License: BSD
 Group: System Environment/Daemons



From ingvar at projects.linpro.no  Mon Nov 10 12:43:59 2008
From: ingvar at projects.linpro.no (ingvar at projects.linpro.no)
Date: Mon, 10 Nov 2008 13:43:59 +0100 (CET)
Subject: r3380 - trunk/varnish-cache/redhat
Message-ID: <20081110124359.219741EC535@projects.linpro.no>

Author: ingvar
Date: 2008-11-10 13:43:58 +0100 (Mon, 10 Nov 2008)
New Revision: 3380

Modified:
   trunk/varnish-cache/redhat/varnish.spec
Log:
specfile: 2.0.1-2 -> 2.0.2-1 and a changelog item for 2.0.2

Modified: trunk/varnish-cache/redhat/varnish.spec
===================================================================
--- trunk/varnish-cache/redhat/varnish.spec	2008-11-10 12:34:49 UTC (rev 3379)
+++ trunk/varnish-cache/redhat/varnish.spec	2008-11-10 12:43:58 UTC (rev 3380)
@@ -1,7 +1,7 @@
 Summary: Varnish is a high-performance HTTP accelerator
 Name: varnish
 Version: 2.0.2
-Release: 2%{?dist}
+Release: 1%{?dist}
 License: BSD
 Group: System Environment/Daemons
 URL: http://www.varnish-cache.org/
@@ -230,6 +230,9 @@
 %postun libs -p /sbin/ldconfig
 
 %changelog
+* Mon Nov 10 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0.2-1
+  New upstream release 2.0.2. A bugfix release
+
 * Sun Nov 02 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0.1-2
 - Removed the requirement for kernel => 2.6.0. All supported
   platforms meets this, and it generates strange errors in EPEL



From phk at projects.linpro.no  Mon Nov 10 19:46:25 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 10 Nov 2008 20:46:25 +0100 (CET)
Subject: r3381 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081110194625.A27F51EC9A5@projects.linpro.no>

Author: phk
Date: 2008-11-10 20:46:25 +0100 (Mon, 10 Nov 2008)
New Revision: 3381

Modified:
   trunk/varnish-cache/bin/varnishd/Makefile.am
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_backend.c
   trunk/varnish-cache/bin/varnishd/cache_backend.h
   trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c
   trunk/varnish-cache/bin/varnishd/cache_ban.c
   trunk/varnish-cache/bin/varnishd/cache_cli.c
   trunk/varnish-cache/bin/varnishd/cache_expire.c
   trunk/varnish-cache/bin/varnishd/cache_hash.c
   trunk/varnish-cache/bin/varnishd/cache_main.c
   trunk/varnish-cache/bin/varnishd/cache_pool.c
   trunk/varnish-cache/bin/varnishd/cache_session.c
   trunk/varnish-cache/bin/varnishd/cache_vcl.c
   trunk/varnish-cache/bin/varnishd/hash_classic.c
   trunk/varnish-cache/bin/varnishd/hash_simple_list.c
   trunk/varnish-cache/bin/varnishd/shmlog.c
   trunk/varnish-cache/bin/varnishd/storage_file.c
   trunk/varnish-cache/bin/varnishd/storage_malloc.c
   trunk/varnish-cache/bin/varnishd/storage_synth.c
   trunk/varnish-cache/bin/varnishd/storage_umem.c
Log:
Take the full step and wrap all our mutex operations in proper C functions
instead of increasingly unwieldy macros.

Amongst other things, this will make it much easier to do lock
profiling, contest statistics, asserts etc.

This commit is largely mechanically generated and should not result in
any changed functionality.

Locks retain the "mtx" monicker, as a reminder that they are mutexes.

No performance impact expected.


Modified: trunk/varnish-cache/bin/varnishd/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishd/Makefile.am	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/Makefile.am	2008-11-10 19:46:25 UTC (rev 3381)
@@ -27,6 +27,7 @@
 	cache_http.c \
 	cache_httpconn.c \
 	cache_main.c \
+	cache_lck.c \
 	cache_panic.c \
 	cache_pipe.c \
 	cache_pool.c \

Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2008-11-10 19:46:25 UTC (rev 3381)
@@ -87,6 +87,7 @@
 struct vrt_backend;
 struct cli_proto;
 struct ban;
+struct lock { void *priv; };		// Opaque
 
 /*--------------------------------------------------------------------*/
 
@@ -295,7 +296,7 @@
 #define OBJHEAD_MAGIC		0x1b96615d
 	void			*hashpriv;
 
-	pthread_mutex_t		mtx;
+	struct lock		mtx;
 	VTAILQ_HEAD(,object)	objects;
 	char			*hash;
 	unsigned		hashlen;
@@ -512,6 +513,27 @@
 void THR_SetSession(const struct sess *sp);
 const struct sess * THR_GetSession(void);
 
+/* cache_lck.c */
+
+/* Internal functions, call only through macros below */
+void Lck__Lock(struct lock *lck, const char *p, const char *f, int l);
+void Lck__Unlock(struct lock *lck, const char *p, const char *f, int l);
+int Lck__Trylock(struct lock *lck, const char *p, const char *f, int l);
+void Lck__New(struct lock *lck, const char *w);
+void Lck__Assert(struct lock *lck, int held);
+
+/* public interface: */
+void LCK_Init(void);
+void Lck_Delete(struct lock *lck);
+void Lck_CondWait(pthread_cond_t *cond, struct lock *lck);
+
+#define Lck_New(a) Lck__New(a, #a);
+#define Lck_Lock(a) Lck__Lock(a, __func__, __FILE__, __LINE__)
+#define Lck_Unlock(a) Lck__Unlock(a, __func__, __FILE__, __LINE__)
+#define Lck_Trylock(a) Lck__Trylock(a, __func__, __FILE__, __LINE__)
+#define Lck_AssertHeld(a) Lck__Assert(a, 1)
+#define Lck_AssertNotHeld(a) Lck__Assert(a, 0)
+
 /* cache_panic.c */
 void PAN_Init(void);
 
@@ -612,72 +634,6 @@
 struct vsb *SMS_Makesynth(struct object *obj);
 void SMS_Finish(struct object *obj);
 
-#define MTX			pthread_mutex_t
-#define MTX_INIT(foo)		AZ(pthread_mutex_init(foo, NULL))
-#define MTX_DESTROY(foo)	AZ(pthread_mutex_destroy(foo))
-
-#ifdef __flexelint_v9__
-#define TRYLOCK(foo, r)						\
-do {								\
-	(r) = pthread_mutex_trylock(foo);			\
-} while (0)
-#define LOCK(foo)						\
-do {								\
-	AZ(pthread_mutex_lock(foo));				\
-} while (0)
-#define UNLOCK(foo)						\
-do {								\
-	AZ(pthread_mutex_unlock(foo));				\
-} while (0)
-
-#else
-#define TRYLOCK(foo, r)						\
-do {								\
-	(r) = pthread_mutex_trylock(foo);			\
-	assert(r == 0 || r == EBUSY);				\
-	if (params->diag_bitmap & 0x8) {			\
-		VSL(SLT_Debug, 0,				\
-		    "MTX_TRYLOCK(%s,%s,%d," #foo ") = %d",	\
-		    __func__, __FILE__, __LINE__, (r));		\
-	}							\
-} while (0)
-#define LOCK(foo)						\
-do {								\
-	if (!(params->diag_bitmap & 0x18)) {			\
-		AZ(pthread_mutex_lock(foo));			\
-	} else {						\
-		int ixjd = pthread_mutex_trylock(foo);		\
-		assert(ixjd == 0 || ixjd == EBUSY);		\
-		if (ixjd) {					\
-			VSL(SLT_Debug, 0,			\
-			    "MTX_CONTEST(%s,%s,%d," #foo ")",	\
-			    __func__, __FILE__, __LINE__);	\
-			AZ(pthread_mutex_lock(foo));		\
-		} else if (params->diag_bitmap & 0x8) {		\
-			VSL(SLT_Debug, 0,			\
-			    "MTX_LOCK(%s,%s,%d," #foo ")",	\
-			    __func__, __FILE__, __LINE__);	\
-		}						\
-	}							\
-} while (0)
-#define UNLOCK(foo)						\
-do {								\
-	AZ(pthread_mutex_unlock(foo));				\
-	if (params->diag_bitmap & 0x8)				\
-		VSL(SLT_Debug, 0,				\
-		    "MTX_UNLOCK(%s,%s,%d," #foo ")",		\
-		    __func__, __FILE__, __LINE__);		\
-} while (0)
-#endif
-
-#if defined(HAVE_PTHREAD_MUTEX_ISOWNED_NP)
-#define ALOCKED(mutex)		AN(pthread_mutex_isowned_np((mutex)))
-#elif defined(DIAGNOSTICS)
-#define ALOCKED(mutex)		AN(pthread_mutex_trylock((mutex)))
-#else
-#define ALOCKED(mutex)		(void)(mutex)
-#endif
-
 /*
  * A normal pointer difference is signed, but we never want a negative value
  * so this little tool will make sure we don't get that.

Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_backend.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/cache_backend.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -147,11 +147,11 @@
 	struct bereq *bereq;
 	volatile unsigned len;
 
-	LOCK(&VBE_mtx);
+	Lck_Lock(&VBE_mtx);
 	bereq = VTAILQ_FIRST(&bereq_head);
 	if (bereq != NULL)
 		VTAILQ_REMOVE(&bereq_head, bereq, list);
-	UNLOCK(&VBE_mtx);
+	Lck_Unlock(&VBE_mtx);
 	if (bereq != NULL) {
 		CHECK_OBJ(bereq, BEREQ_MAGIC);
 	} else {
@@ -177,9 +177,9 @@
 
 	CHECK_OBJ_NOTNULL(bereq, BEREQ_MAGIC);
 	WS_Reset(bereq->ws, NULL);
-	LOCK(&VBE_mtx);
+	Lck_Lock(&VBE_mtx);
 	VTAILQ_INSERT_HEAD(&bereq_head, bereq, list);
-	UNLOCK(&VBE_mtx);
+	Lck_Unlock(&VBE_mtx);
 }
 
 /*--------------------------------------------------------------------
@@ -195,13 +195,13 @@
 
 	vc = VTAILQ_FIRST(&vbe_conns);
 	if (vc != NULL) {
-		LOCK(&VBE_mtx);
+		Lck_Lock(&VBE_mtx);
 		vc = VTAILQ_FIRST(&vbe_conns);
 		if (vc != NULL) {
 			VSL_stats->backend_unused--;
 			VTAILQ_REMOVE(&vbe_conns, vc, list);
 		}
-		UNLOCK(&VBE_mtx);
+		Lck_Unlock(&VBE_mtx);
 	}
 	if (vc != NULL)
 		return (vc);
@@ -222,10 +222,10 @@
 	assert(vc->fd < 0);
 
 	if (params->cache_vbe_conns) {
-		LOCK(&VBE_mtx);
+		Lck_Lock(&VBE_mtx);
 		VTAILQ_INSERT_HEAD(&vbe_conns, vc, list);
 		VSL_stats->backend_unused++;
-		UNLOCK(&VBE_mtx);
+		Lck_Unlock(&VBE_mtx);
 	} else {
 		VSL_stats->n_vbe_conn--;
 		free(vc);
@@ -239,10 +239,10 @@
 {
 	int s;
 
-	LOCK(&bp->mtx);
+	Lck_Lock(&bp->mtx);
 	bp->refcount++;
 	bp->n_conn++;		/* It mostly works */
-	UNLOCK(&bp->mtx);
+	Lck_Unlock(&bp->mtx);
 
 	s = -1;
 	assert(bp->ipv6 != NULL || bp->ipv4 != NULL);
@@ -257,10 +257,10 @@
 		s = VBE_TryConnect(sp, PF_INET6, bp->ipv6, bp->ipv6len, bp);
 
 	if (s < 0) {
-		LOCK(&bp->mtx);
+		Lck_Lock(&bp->mtx);
 		bp->n_conn--;
 		bp->refcount--;		/* Only keep ref on success */
-		UNLOCK(&bp->mtx);
+		Lck_Unlock(&bp->mtx);
 	}
 	return (s);
 }
@@ -295,7 +295,7 @@
 
 	/* first look for vbe_conn's we can recycle */
 	while (1) {
-		LOCK(&bp->mtx);
+		Lck_Lock(&bp->mtx);
 		vc = VTAILQ_FIRST(&bp->connlist);
 		if (vc != NULL) {
 			bp->refcount++;
@@ -303,7 +303,7 @@
 			assert(vc->fd >= 0);
 			VTAILQ_REMOVE(&bp->connlist, vc, list);
 		}
-		UNLOCK(&bp->mtx);
+		Lck_Unlock(&bp->mtx);
 		if (vc == NULL)
 			break;
 		if (VBE_CheckFd(vc->fd)) {
@@ -379,7 +379,7 @@
 	bp = sp->vbe->backend;
 
 	WSL(sp->wrk, SLT_BackendReuse, sp->vbe->fd, "%s", bp->vcl_name);
-	LOCK(&bp->mtx);
+	Lck_Lock(&bp->mtx);
 	VSL_stats->backend_recycle++;
 	VTAILQ_INSERT_HEAD(&bp->connlist, sp->vbe, list);
 	sp->vbe = NULL;

Modified: trunk/varnish-cache/bin/varnishd/cache_backend.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_backend.h	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/cache_backend.h	2008-11-10 19:46:25 UTC (rev 3381)
@@ -109,7 +109,7 @@
 
 	VTAILQ_ENTRY(backend)	list;
 	int			refcount;
-	pthread_mutex_t		mtx;
+	struct lock		mtx;
 
 	struct sockaddr		*ipv4;
 	socklen_t		ipv4len;
@@ -129,7 +129,7 @@
 struct vbe_conn *VBE_GetVbe(struct sess *sp, struct backend *bp);
 
 /* cache_backend_cfg.c */
-extern MTX VBE_mtx;
+extern struct lock VBE_mtx;
 void VBE_DropRefConn(struct backend *);
 void VBE_DropRef(struct backend *);
 void VBE_DropRefLocked(struct backend *b);

Modified: trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -48,7 +48,7 @@
 #include "cache_backend.h"
 #include "cli_priv.h"
 
-MTX VBE_mtx;
+struct lock VBE_mtx;
 
 /*
  * The list of backends is not locked, it is only ever accessed from
@@ -105,7 +105,7 @@
 	assert(b->refcount > 0);
 
 	i = --b->refcount;
-	UNLOCK(&b->mtx);
+	Lck_Unlock(&b->mtx);
 	if (i > 0)
 		return;
 
@@ -128,7 +128,7 @@
 
 	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
 
-	LOCK(&b->mtx);
+	Lck_Lock(&b->mtx);
 	VBE_DropRefLocked(b);
 }
 
@@ -138,7 +138,7 @@
 
 	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
 
-	LOCK(&b->mtx);
+	Lck_Lock(&b->mtx);
 	assert(b->n_conn > 0);
 	b->n_conn--;
 	VBE_DropRefLocked(b);
@@ -207,7 +207,7 @@
 	/* Create new backend */
 	ALLOC_OBJ(b, BACKEND_MAGIC);
 	XXXAN(b);
-	MTX_INIT(&b->mtx);
+	Lck_New(&b->mtx);
 	b->refcount = 1;
 
 	VTAILQ_INIT(&b->connlist);
@@ -283,6 +283,6 @@
 VBE_Init(void)
 {
 
-	MTX_INIT(&VBE_mtx);
+	Lck_New(&VBE_mtx);
 	CLI_AddFuncs(DEBUG_CLI, debug_cmds);
 }

Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_ban.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/cache_ban.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -57,7 +57,7 @@
 };
 
 static VTAILQ_HEAD(banhead,ban) ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
-static MTX ban_mtx;
+static struct lock ban_mtx;
 
 /*
  * We maintain ban_start as a pointer to the first element of the list
@@ -95,7 +95,7 @@
 	b->hash = hash;
 	b->ban = strdup(regexp);
 	AN(b->ban);
-	LOCK(&ban_mtx);
+	Lck_Lock(&ban_mtx);
 	VTAILQ_INSERT_HEAD(&ban_head, b, list);
 	ban_start = b;
 	VSL_stats->n_purge++;
@@ -106,7 +106,7 @@
 		be->refcount++;
 	} else
 		be = NULL;
-	UNLOCK(&ban_mtx);
+	Lck_Unlock(&ban_mtx);
 
 	if (be == NULL)
 		return (0);
@@ -125,11 +125,11 @@
 		bi->flags |= BAN_F_GONE;
 		pcount++;
 	}
-	LOCK(&ban_mtx);
+	Lck_Lock(&ban_mtx);
 	be->refcount--;
 	/* XXX: We should check if the tail can be removed */
 	VSL_stats->n_purge_dups += pcount;
-	UNLOCK(&ban_mtx);
+	Lck_Unlock(&ban_mtx);
 
 	return (0);
 }
@@ -140,10 +140,10 @@
 
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	AZ(o->ban);
-	LOCK(&ban_mtx);
+	Lck_Lock(&ban_mtx);
 	o->ban = ban_start;
 	ban_start->refcount++;
-	UNLOCK(&ban_mtx);
+	Lck_Unlock(&ban_mtx);
 }
 
 void
@@ -155,7 +155,7 @@
 	if (o->ban == NULL)
 		return;
 	CHECK_OBJ_NOTNULL(o->ban, BAN_MAGIC);
-	LOCK(&ban_mtx);
+	Lck_Lock(&ban_mtx);
 	o->ban->refcount--;
 	o->ban = NULL;
 
@@ -168,7 +168,7 @@
 	} else {
 		b = NULL;
 	}
-	UNLOCK(&ban_mtx);
+	Lck_Unlock(&ban_mtx);
 	if (b != NULL) {
 		free(b->ban);
 		regfree(&b->regexp);
@@ -205,13 +205,13 @@
 			break;
 	}
 
-	LOCK(&ban_mtx);
+	Lck_Lock(&ban_mtx);
 	o->ban->refcount--;
 	if (b == o->ban)	/* not banned */
 		b0->refcount++;
 	VSL_stats->n_purge_obj_test++;
 	VSL_stats->n_purge_re_test += tests;
-	UNLOCK(&ban_mtx);
+	Lck_Unlock(&ban_mtx);
 
 	if (b == o->ban) {	/* not banned */
 		o->ban = b0;
@@ -285,7 +285,7 @@
 BAN_Init(void)
 {
 
-	MTX_INIT(&ban_mtx);
+	Lck_New(&ban_mtx);
 	CLI_AddFuncs(PUBLIC_CLI, ban_cmds);
 	/* Add an initial ban, since the list can never be empty */
 	(void)BAN_Add(NULL, ".", 0);

Modified: trunk/varnish-cache/bin/varnishd/cache_cli.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_cli.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/cache_cli.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -53,8 +53,8 @@
 #include "vlu.h"
 #include "vsb.h"
 
-pthread_t	cli_thread;
-static MTX	cli_mtx;
+pthread_t		cli_thread;
+static struct lock	cli_mtx;
 
 /*
  * The CLI commandlist is split in three:
@@ -81,12 +81,12 @@
 	case DEBUG_CLI:	 cp = &ccf_debug_cli;  break;
 	default: INCOMPL();
 	}
-	LOCK(&cli_mtx);
+	Lck_Lock(&cli_mtx);
 	c = cli_concat(*cp, p);
 	AN(c);
 	free(*cp);
 	*cp = c;
-	UNLOCK(&cli_mtx);
+	Lck_Unlock(&cli_mtx);
 }
 
 /*--------------------------------------------------------------------
@@ -105,7 +105,7 @@
 	VCL_Poll();
 	VBE_Poll();
 	vsb_clear(cli->sb);
-	LOCK(&cli_mtx);
+	Lck_Lock(&cli_mtx);
 	cli_dispatch(cli, ccf_master_cli, p);
 	if (cli->result == CLIS_UNKNOWN) {
 		vsb_clear(cli->sb);
@@ -117,7 +117,7 @@
 		cli->result = CLIS_OK;
 		cli_dispatch(cli, ccf_debug_cli, p);
 	}
-	UNLOCK(&cli_mtx);
+	Lck_Unlock(&cli_mtx);
 	vsb_finish(cli->sb);
 	AZ(vsb_overflowed(cli->sb));
 	i = cli_writeres(heritage.cli_out, cli);
@@ -242,7 +242,7 @@
 CLI_Init(void)
 {
 
-	MTX_INIT(&cli_mtx);
+	Lck_New(&cli_mtx);
 	cli_thread = pthread_self();
 
 	CLI_AddFuncs(MASTER_CLI, master_cmds);

Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_expire.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/cache_expire.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -86,7 +86,7 @@
 
 static pthread_t exp_thread;
 static struct binheap *exp_heap;
-static MTX exp_mtx;
+static struct lock exp_mtx;
 static VTAILQ_HEAD(,objexp) lru = VTAILQ_HEAD_INITIALIZER(lru);
 
 /*
@@ -176,12 +176,12 @@
 	assert(o->entered != 0 && !isnan(o->entered));
 	oe->lru_stamp = o->entered;
 	update_object_when(o);
-	LOCK(&exp_mtx);
+	Lck_Lock(&exp_mtx);
 	binheap_insert(exp_heap, oe);
 	assert(oe->timer_idx != BINHEAP_NOIDX);
 	VTAILQ_INSERT_TAIL(&lru, oe, list);
 	oe->on_lru = 1;
-	UNLOCK(&exp_mtx);
+	Lck_Unlock(&exp_mtx);
 }
 
 /*--------------------------------------------------------------------
@@ -196,7 +196,6 @@
 void
 EXP_Touch(const struct object *o, double now)
 {
-	int i;
 	struct objexp *oe;
 
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
@@ -206,8 +205,7 @@
 	CHECK_OBJ_NOTNULL(oe, OBJEXP_MAGIC);
 	if (oe->lru_stamp + params->lru_timeout > now)
 		return;
-	TRYLOCK(&exp_mtx, i);
-	if (i)
+	if (Lck_Trylock(&exp_mtx))
 		return;
 	if (oe->on_lru) {
 		VTAILQ_REMOVE(&lru, oe, list);
@@ -215,7 +213,7 @@
 		oe->lru_stamp = now;
 		VSL_stats->n_lru_moved++;
 	}
-	UNLOCK(&exp_mtx);
+	Lck_Unlock(&exp_mtx);
 }
 
 /*--------------------------------------------------------------------
@@ -238,13 +236,13 @@
 		return;
 	CHECK_OBJ_NOTNULL(oe, OBJEXP_MAGIC);
 	update_object_when(o);
-	LOCK(&exp_mtx);
+	Lck_Lock(&exp_mtx);
 	assert(oe->timer_idx != BINHEAP_NOIDX);
 	binheap_delete(exp_heap, oe->timer_idx); /* XXX: binheap_shuffle() ? */
 	assert(oe->timer_idx == BINHEAP_NOIDX);
 	binheap_insert(exp_heap, oe);
 	assert(oe->timer_idx != BINHEAP_NOIDX);
-	UNLOCK(&exp_mtx);
+	Lck_Unlock(&exp_mtx);
 }
 
 
@@ -278,11 +276,11 @@
 	VCL_Get(&sp->vcl);
 	t = TIM_real();
 	while (1) {
-		LOCK(&exp_mtx);
+		Lck_Lock(&exp_mtx);
 		oe = binheap_root(exp_heap);
 		CHECK_OBJ_ORNULL(oe, OBJEXP_MAGIC);
 		if (oe == NULL || oe->timer_when > t) { /* XXX: > or >= ? */
-			UNLOCK(&exp_mtx);
+			Lck_Unlock(&exp_mtx);
 			WSL_Flush(&ww, 0);
 			AZ(sleep(1));
 			VCL_Refresh(&sp->vcl);
@@ -305,7 +303,7 @@
 		}
 
 		assert(oe->on_lru);
-		UNLOCK(&exp_mtx);
+		Lck_Unlock(&exp_mtx);
 
 		WSL(&ww, SLT_ExpPick, 0, "%u %s", o->xid, oe->timer_what);
 
@@ -319,10 +317,10 @@
 				    o->xid);
 			}
 			update_object_when(o);
-			LOCK(&exp_mtx);
+			Lck_Lock(&exp_mtx);
 			binheap_insert(exp_heap, oe);
 			assert(oe->timer_idx != BINHEAP_NOIDX);
-			UNLOCK(&exp_mtx);
+			Lck_Unlock(&exp_mtx);
 		} else {
 			assert(oe->timer_what == tmr_ttl);
 			sp->obj = o;
@@ -332,11 +330,11 @@
 			assert(sp->handling == VCL_RET_DISCARD);
 			WSL(&ww, SLT_ExpKill, 0,
 			    "%u %d", o->xid, (int)(o->ttl - t));
-			LOCK(&exp_mtx);
+			Lck_Lock(&exp_mtx);
 			VTAILQ_REMOVE(&lru, o->objexp, list);
 			oe->on_lru = 0;
 			VSL_stats->n_expired++;
-			UNLOCK(&exp_mtx);
+			Lck_Unlock(&exp_mtx);
 			del_objexp(o);
 			HSH_Deref(o);
 		}
@@ -367,7 +365,7 @@
 	 * NB: Checking refcount here is no guarantee that it does not gain
 	 * another ref while we ponder its destiny without the lock held.
 	 */
-	LOCK(&exp_mtx);
+	Lck_Lock(&exp_mtx);
 	VTAILQ_FOREACH(oe, &lru, list) {
 		CHECK_OBJ_NOTNULL(oe, OBJEXP_MAGIC);
 		if (oe->timer_idx == BINHEAP_NOIDX)	/* exp_timer has it */
@@ -388,7 +386,7 @@
 		assert(oe->timer_idx == BINHEAP_NOIDX);
 		VSL_stats->n_lru_nuked++;
 	}
-	UNLOCK(&exp_mtx);
+	Lck_Unlock(&exp_mtx);
 
 	if (oe == NULL)
 		return (-1);
@@ -414,14 +412,14 @@
 	assert(sp->handling == VCL_RET_KEEP);
 
 	/* Insert in binheap and lru again */
-	LOCK(&exp_mtx);
+	Lck_Lock(&exp_mtx);
 	VSL_stats->n_lru_nuked--;		/* It was premature */
 	VSL_stats->n_lru_saved++;
 	binheap_insert(exp_heap, oe);
 	assert(oe->timer_idx != BINHEAP_NOIDX);
 	VTAILQ_INSERT_TAIL(&lru, oe, list);
 	oe->on_lru = 1;
-	UNLOCK(&exp_mtx);
+	Lck_Unlock(&exp_mtx);
 	return (0);
 }
 
@@ -456,7 +454,7 @@
 EXP_Init(void)
 {
 
-	MTX_INIT(&exp_mtx);
+	Lck_New(&exp_mtx);
 	exp_heap = binheap_new(NULL, object_cmp, object_update);
 	XXXAN(exp_heap);
 	AZ(pthread_create(&exp_thread, NULL, exp_timer, NULL));

Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -93,7 +93,7 @@
 		w->nobjhead->magic = OBJHEAD_MAGIC;
 		VTAILQ_INIT(&w->nobjhead->objects);
 		VTAILQ_INIT(&w->nobjhead->waitinglist);
-		MTX_INIT(&w->nobjhead->mtx);
+		Lck_New(&w->nobjhead->mtx);
 		VSL_stats->n_objecthead++;
 	} else
 		CHECK_OBJ_NOTNULL(w->nobjhead, OBJHEAD_MAGIC);
@@ -205,13 +205,13 @@
 		oh = sp->objhead;
 		sp->objhead = NULL;
 		CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-		LOCK(&oh->mtx);
+		Lck_Lock(&oh->mtx);
 	} else {
 		oh = hash->lookup(sp, w->nobjhead);
 		CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 		if (oh == w->nobjhead)
 			w->nobjhead = NULL;
-		LOCK(&oh->mtx);
+		Lck_Lock(&oh->mtx);
 	}
 
 	busy_o = NULL;
@@ -257,7 +257,7 @@
 		o->refcnt++;
 		if (o->hits < INT_MAX)
 			o->hits++;
-		UNLOCK(&oh->mtx);
+		Lck_Unlock(&oh->mtx);
 		if (params->log_hash)
 			WSP(sp, SLT_Hash, "%s", oh->hash);
 		(void)hash->deref(oh);
@@ -269,7 +269,7 @@
 		if (sp->esis == 0)
 			VTAILQ_INSERT_TAIL(&oh->waitinglist, sp, list);
 		sp->objhead = oh;
-		UNLOCK(&oh->mtx);
+		Lck_Unlock(&oh->mtx);
 		return (NULL);
 	}
 
@@ -285,7 +285,7 @@
 		o->parent = grace_o;
 		grace_o->refcnt++;
 	}
-	UNLOCK(&oh->mtx);
+	Lck_Unlock(&oh->mtx);
 	if (params->log_hash)
 		WSP(sp, SLT_Hash, "%s", oh->hash);
 	/*
@@ -333,7 +333,7 @@
 	oh = o->objhead;
 	if (oh != NULL) {
 		CHECK_OBJ(oh, OBJHEAD_MAGIC);
-		LOCK(&oh->mtx);
+		Lck_Lock(&oh->mtx);
 	}
 	o->busy = 0;
 	if (oh != NULL)
@@ -343,7 +343,7 @@
 	if (parent != NULL)
 		parent->child = NULL;
 	if (oh != NULL)
-		UNLOCK(&oh->mtx);
+		Lck_Unlock(&oh->mtx);
 	if (parent != NULL)
 		HSH_Deref(parent);
 }
@@ -357,12 +357,12 @@
 	oh = o->objhead;
 	if (oh != NULL) {
 		CHECK_OBJ(oh, OBJHEAD_MAGIC);
-		LOCK(&oh->mtx);
+		Lck_Lock(&oh->mtx);
 	}
 	assert(o->refcnt > 0);
 	o->refcnt++;
 	if (oh != NULL)
-		UNLOCK(&oh->mtx);
+		Lck_Unlock(&oh->mtx);
 }
 
 void
@@ -377,7 +377,7 @@
 		CHECK_OBJ(oh, OBJHEAD_MAGIC);
 
 		/* drop ref on object */
-		LOCK(&oh->mtx);
+		Lck_Lock(&oh->mtx);
 	}
 	assert(o->refcnt > 0);
 	r = --o->refcnt;
@@ -386,7 +386,7 @@
 	if (oh != NULL) {
 		if (!r)
 			VTAILQ_REMOVE(&oh->objects, o, list);
-		UNLOCK(&oh->mtx);
+		Lck_Unlock(&oh->mtx);
 	}
 
 	/* If still referenced, done */
@@ -411,7 +411,7 @@
 	if (hash->deref(oh))
 		return;
 	assert(VTAILQ_EMPTY(&oh->objects));
-	MTX_DESTROY(&oh->mtx);
+	Lck_Delete(&oh->mtx);
 	VSL_stats->n_objecthead--;
 	free(oh->hash);
 	FREE_OBJ(oh);

Modified: trunk/varnish-cache/bin/varnishd/cache_main.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_main.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/cache_main.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -101,6 +101,10 @@
 
 	THR_SetName("cache-main");
 
+	VSL_Init();	/* First, LCK needs it. */
+
+	LCK_Init();	/* Locking, must be first */
+
 	PAN_Init();
 	CLI_Init();
 	Fetch_Init();
@@ -113,7 +117,6 @@
 
 	VBE_Init();
 	VBP_Init();
-	VSL_Init();
 	WRK_Init();
 
 	EXP_Init();

Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -79,7 +79,7 @@
 struct wq {
 	unsigned		magic;
 #define WQ_MAGIC		0x606658fa
-	MTX			mtx;
+	struct lock		mtx;
 	struct workerhead	idle;
 	VTAILQ_HEAD(, workreq)	overflow;
 	unsigned		nthr;
@@ -95,7 +95,7 @@
 static unsigned			nthr_max;
 
 static pthread_cond_t		herder_cond;
-static MTX			herder_mtx;
+static struct lock		herder_mtx;
 
 /*--------------------------------------------------------------------
  * Write data to fd
@@ -249,7 +249,7 @@
 
 	VSL(SLT_WorkThread, 0, "%p start", w);
 
-	LOCK(&qp->mtx);
+	Lck_Lock(&qp->mtx);
 	qp->nthr++;
 	while (1) {
 		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
@@ -263,20 +263,20 @@
 			if (isnan(w->lastused))
 				w->lastused = TIM_real();
 			VTAILQ_INSERT_HEAD(&qp->idle, w, list);
-			AZ(pthread_cond_wait(&w->cond, &qp->mtx));
+			Lck_CondWait(&w->cond, &qp->mtx);
 		}
 		if (w->wrq == NULL)
 			break;
-		UNLOCK(&qp->mtx);
+		Lck_Unlock(&qp->mtx);
 		AN(w->wrq);
 		AN(w->wrq->func);
 		w->lastused = NAN;
 		w->wrq->func(w, w->wrq->priv);
 		w->wrq = NULL;
-		LOCK(&qp->mtx);
+		Lck_Lock(&qp->mtx);
 	}
 	qp->nthr--;
-	UNLOCK(&qp->mtx);
+	Lck_Unlock(&qp->mtx);
 
 	VSL(SLT_WorkThread, 0, "%p end", w);
 	if (w->vcl != NULL)
@@ -285,7 +285,7 @@
 	if (w->srcaddr != NULL)
 		free(w->srcaddr);
 	if (w->nobjhead != NULL) {
-		MTX_DESTROY(&w->nobjhead->mtx);
+		Lck_Delete(&w->nobjhead->mtx);
 		FREE_OBJ(w->nobjhead);
 	}
 	if (w->nobj!= NULL)
@@ -318,13 +318,13 @@
 	qp = wq[onq];
 	nq = onq;
 
-	LOCK(&qp->mtx);
+	Lck_Lock(&qp->mtx);
 
 	/* If there are idle threads, we tickle the first one into action */
 	w = VTAILQ_FIRST(&qp->idle);
 	if (w != NULL) {
 		VTAILQ_REMOVE(&qp->idle, w, list);
-		UNLOCK(&qp->mtx);
+		Lck_Unlock(&qp->mtx);
 		w->wrq = wrq;
 		AZ(pthread_cond_signal(&w->cond));
 		return (0);
@@ -333,14 +333,14 @@
 	/* If we have too much in the overflow already, refuse. */
 	if (qp->nqueue > ovfl_max) {
 		qp->ndrop++;
-		UNLOCK(&qp->mtx);
+		Lck_Unlock(&qp->mtx);
 		return (-1);
 	}
 
 	VTAILQ_INSERT_TAIL(&qp->overflow, wrq, list);
 	qp->noverflow++;
 	qp->nqueue++;
-	UNLOCK(&qp->mtx);
+	Lck_Unlock(&qp->mtx);
 	AZ(pthread_cond_signal(&herder_cond));
 	return (0);
 }
@@ -412,7 +412,7 @@
 		wq[u] = calloc(sizeof *wq[u], 1);
 		XXXAN(wq[u]);
 		wq[u]->magic = WQ_MAGIC;
-		MTX_INIT(&wq[u]->mtx);
+		Lck_New(&wq[u]->mtx);
 		VTAILQ_INIT(&wq[u]->overflow);
 		VTAILQ_INIT(&wq[u]->idle);
 	}
@@ -429,7 +429,7 @@
 {
 	struct worker *w = NULL;
 
-	LOCK(&qp->mtx);
+	Lck_Lock(&qp->mtx);
 	vs->n_wrk += qp->nthr;
 	vs->n_wrk_queue += qp->nqueue;
 	vs->n_wrk_drop += qp->ndrop;
@@ -442,7 +442,7 @@
 		else
 			w = NULL;
 	}
-	UNLOCK(&qp->mtx);
+	Lck_Unlock(&qp->mtx);
 
 	/* And give it a kiss on the cheek... */
 	if (w != NULL) {
@@ -572,9 +572,9 @@
 			 * We cannot avoid getting a mutex, so we have a
 			 * bogo mutex just for POSIX_STUPIDITY
 			 */
-			AZ(pthread_mutex_lock(&herder_mtx));
-			AZ(pthread_cond_wait(&herder_cond, &herder_mtx));
-			AZ(pthread_mutex_unlock(&herder_mtx));
+			Lck_Lock(&herder_mtx);
+			Lck_CondWait(&herder_cond, &herder_mtx);
+			Lck_Unlock(&herder_mtx);
 			wrk_breed_flock(wq[u]);
 		}
 	}
@@ -588,7 +588,7 @@
 	pthread_t tp;
 
 	AZ(pthread_cond_init(&herder_cond, NULL));
-	AZ(pthread_mutex_init(&herder_mtx, NULL));
+	Lck_New(&herder_mtx);
 
 	wrk_addpools(params->wthread_pools);
 	AZ(pthread_create(&tp, NULL, wrk_herdtimer_thread, NULL));

Modified: trunk/varnish-cache/bin/varnishd/cache_session.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_session.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/cache_session.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -78,7 +78,7 @@
 };
 
 static unsigned ses_qp;
-static MTX			ses_mem_mtx;
+static struct lock		ses_mem_mtx;
 
 /*--------------------------------------------------------------------*/
 
@@ -103,11 +103,11 @@
 	unsigned		magic;
 #define SRCADDRHEAD_MAGIC	0x38231a8b
 	VTAILQ_HEAD(,srcaddr)	head;
-	MTX			mtx;
+	struct lock		mtx;
 } *srchash;
 
 static unsigned			nsrchash;
-static MTX			stat_mtx;
+static struct lock 		stat_mtx;
 
 /*--------------------------------------------------------------------
  * Assign a srcaddr to this session.
@@ -140,7 +140,7 @@
 		XXXAN(sp->wrk->srcaddr);
 	}
 
-	LOCK(&ch->mtx);
+	Lck_Lock(&ch->mtx);
 	c3 = NULL;
 	VTAILQ_FOREACH_SAFE(c, &ch->head, list, c2) {
 		if (c->hash == u && !strcmp(c->addr, sp->addr)) {
@@ -155,7 +155,7 @@
 				VTAILQ_REMOVE(&ch->head, c3, list);
 				VSL_stats->n_srcaddr--;
 			}
-			UNLOCK(&ch->mtx);
+			Lck_Unlock(&ch->mtx);
 			if (c3 != NULL)
 				free(c3);
 			return;
@@ -183,7 +183,7 @@
 	VSL_stats->n_srcaddr_act++;
 	VTAILQ_INSERT_TAIL(&ch->head, c3, list);
 	sp->srcaddr = c3;
-	UNLOCK(&ch->mtx);
+	Lck_Unlock(&ch->mtx);
 }
 
 /*--------------------------------------------------------------------*/
@@ -198,13 +198,13 @@
 	CHECK_OBJ(sp->srcaddr, SRCADDR_MAGIC);
 	ch = sp->srcaddr->sah;
 	CHECK_OBJ(ch, SRCADDRHEAD_MAGIC);
-	LOCK(&ch->mtx);
+	Lck_Lock(&ch->mtx);
 	assert(sp->srcaddr->nref > 0);
 	sp->srcaddr->nref--;
 	if (sp->srcaddr->nref == 0)
 		VSL_stats->n_srcaddr_act--;
 	sp->srcaddr = NULL;
-	UNLOCK(&ch->mtx);
+	Lck_Unlock(&ch->mtx);
 }
 
 /*--------------------------------------------------------------------*/
@@ -228,21 +228,21 @@
 	if (sp->srcaddr != NULL) {
 		/* XXX: only report once per second ? */
 		CHECK_OBJ(sp->srcaddr, SRCADDR_MAGIC);
-		LOCK(&sp->srcaddr->sah->mtx);
+		Lck_Lock(&sp->srcaddr->sah->mtx);
 		ses_sum_acct(&sp->srcaddr->acct, a);
 		b = sp->srcaddr->acct;
-		UNLOCK(&sp->srcaddr->sah->mtx);
+		Lck_Unlock(&sp->srcaddr->sah->mtx);
 		WSL(sp->wrk, SLT_StatAddr, 0,
 		    "%s 0 %.0f %ju %ju %ju %ju %ju %ju %ju",
 		    sp->srcaddr->addr, sp->t_end - b.first,
 		    b.sess, b.req, b.pipe, b.pass,
 		    b.fetch, b.hdrbytes, b.bodybytes);
 	}
-	LOCK(&stat_mtx);
+	Lck_Lock(&stat_mtx);
 #define ACCT(foo)	VSL_stats->s_##foo += a->foo;
 #include "acct_fields.h"
 #undef ACCT
-	UNLOCK(&stat_mtx);
+	Lck_Unlock(&stat_mtx);
 	memset(a, 0, sizeof *a);
 }
 
@@ -266,9 +266,9 @@
 		 * If that queue is empty, flip queues holding the lock
 		 * and try the new unlocked queue.
 		 */
-		LOCK(&ses_mem_mtx);
+		Lck_Lock(&ses_mem_mtx);
 		ses_qp = 1 - ses_qp;
-		UNLOCK(&ses_mem_mtx);
+		Lck_Unlock(&ses_mem_mtx);
 		sm = VTAILQ_FIRST(&ses_free_mem[ses_qp]);
 	}
 	if (sm != NULL) {
@@ -343,9 +343,9 @@
 		VSL_stats->n_sess_mem--;
 		free(sm);
 	} else {
-		LOCK(&ses_mem_mtx);
+		Lck_Lock(&ses_mem_mtx);
 		VTAILQ_INSERT_HEAD(&ses_free_mem[1 - ses_qp], sm, list);
-		UNLOCK(&ses_mem_mtx);
+		Lck_Unlock(&ses_mem_mtx);
 	}
 }
 
@@ -362,8 +362,8 @@
 	for (i = 0; i < nsrchash; i++) {
 		srchash[i].magic = SRCADDRHEAD_MAGIC;
 		VTAILQ_INIT(&srchash[i].head);
-		MTX_INIT(&srchash[i].mtx);
+		Lck_New(&srchash[i].mtx);
 	}
-	MTX_INIT(&stat_mtx);
-	MTX_INIT(&ses_mem_mtx);
+	Lck_New(&stat_mtx);
+	Lck_New(&ses_mem_mtx);
 }

Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vcl.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/cache_vcl.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -64,7 +64,7 @@
     VTAILQ_HEAD_INITIALIZER(vcl_head);
 
 
-static MTX			vcl_mtx;
+static struct lock		vcl_mtx;
 static struct vcls		*vcl_active; /* protected by vcl_mtx */
 
 /*--------------------------------------------------------------------*/
@@ -83,13 +83,13 @@
 VCL_Get(struct VCL_conf **vcc)
 {
 
-	LOCK(&vcl_mtx);
+	Lck_Lock(&vcl_mtx);
 	AN(vcl_active);
 	*vcc = vcl_active->conf;
 	AN(*vcc);
 	AZ((*vcc)->discard);
 	(*vcc)->busy++;
-	UNLOCK(&vcl_mtx);
+	Lck_Unlock(&vcl_mtx);
 }
 
 void
@@ -100,14 +100,14 @@
 	vc = *vcc;
 	*vcc = NULL;
 
-	LOCK(&vcl_mtx);
+	Lck_Lock(&vcl_mtx);
 	assert(vc->busy > 0);
 	vc->busy--;
 	/*
 	 * We do not garbage collect discarded VCL's here, that happens
 	 * in VCL_Poll() which is called from the CLI thread.
 	 */
-	UNLOCK(&vcl_mtx);
+	Lck_Unlock(&vcl_mtx);
 }
 
 /*--------------------------------------------------------------------*/
@@ -167,10 +167,10 @@
 	}
 	REPLACE(vcl->name, name);
 	VTAILQ_INSERT_TAIL(&vcl_head, vcl, list);
-	LOCK(&vcl_mtx);
+	Lck_Lock(&vcl_mtx);
 	if (vcl_active == NULL)
 		vcl_active = vcl;
-	UNLOCK(&vcl_mtx);
+	Lck_Unlock(&vcl_mtx);
 	cli_out(cli, "Loaded \"%s\" as \"%s\"", fn , name);
 	vcl->conf->init_func(cli);
 	VSL_stats->n_vcl++;
@@ -264,9 +264,9 @@
 		cli_out(cli, "VCL '%s' unknown", av[2]);
 		return;
 	}
-	LOCK(&vcl_mtx);
+	Lck_Lock(&vcl_mtx);
 	if (vcl == vcl_active) {
-		UNLOCK(&vcl_mtx);
+		Lck_Unlock(&vcl_mtx);
 		cli_result(cli, CLIS_PARAM);
 		cli_out(cli, "VCL %s is the active VCL", av[2]);
 		return;
@@ -274,7 +274,7 @@
 	VSL_stats->n_vcl_discard++;
 	VSL_stats->n_vcl_avail--;
 	vcl->conf->discard = 1;
-	UNLOCK(&vcl_mtx);
+	Lck_Unlock(&vcl_mtx);
 	if (vcl->conf->busy == 0)
 		VCL_Nuke(vcl);
 }
@@ -292,9 +292,9 @@
 		cli_result(cli, CLIS_PARAM);
 		return;
 	}
-	LOCK(&vcl_mtx);
+	Lck_Lock(&vcl_mtx);
 	vcl_active = vcl;
-	UNLOCK(&vcl_mtx);
+	Lck_Unlock(&vcl_mtx);
 }
 
 /*--------------------------------------------------------------------*/
@@ -350,5 +350,5 @@
 {
 
 	CLI_AddFuncs(MASTER_CLI, vcl_cmds);
-	MTX_INIT(&vcl_mtx);
+	Lck_New(&vcl_mtx);
 }

Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_classic.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/hash_classic.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -58,7 +58,7 @@
 	unsigned		magic;
 #define HCL_HEAD_MAGIC		0x0f327016
 	VTAILQ_HEAD(, hcl_entry)	head;
-	MTX			mtx;
+	struct lock		mtx;
 };
 
 static unsigned			hcl_nhash = 16383;
@@ -110,7 +110,7 @@
 
 	for (u = 0; u < hcl_nhash; u++) {
 		VTAILQ_INIT(&hcl_head[u].head);
-		MTX_INIT(&hcl_head[u].mtx);
+		Lck_New(&hcl_head[u].mtx);
 		hcl_head[u].magic = HCL_HEAD_MAGIC;
 	}
 }
@@ -151,7 +151,7 @@
 	he2 = NULL;
 
 	for (r = 0; r < 2; r++ ) {
-		LOCK(&hp->mtx);
+		Lck_Lock(&hp->mtx);
 		VTAILQ_FOREACH(he, &hp->head, list) {
 			CHECK_OBJ_NOTNULL(he, HCL_ENTRY_MAGIC);
 			if (sp->lhashptr < he->oh->hashlen)
@@ -169,7 +169,7 @@
 				break;
 			he->refcnt++;
 			roh = he->oh;
-			UNLOCK(&hp->mtx);
+			Lck_Unlock(&hp->mtx);
 			/*
 			 * If we loose the race, we need to clean up
 			 * the work we did for our second attempt.
@@ -183,7 +183,7 @@
 			return (roh);
 		}
 		if (noh == NULL) {
-			UNLOCK(&hp->mtx);
+			Lck_Unlock(&hp->mtx);
 			return (NULL);
 		}
 		if (he2 != NULL) {
@@ -193,10 +193,10 @@
 				VTAILQ_INSERT_TAIL(&hp->head, he2, list);
 			he2->refcnt++;
 			noh = he2->oh;
-			UNLOCK(&hp->mtx);
+			Lck_Unlock(&hp->mtx);
 			return (noh);
 		}
-		UNLOCK(&hp->mtx);
+		Lck_Unlock(&hp->mtx);
 
 		he2 = calloc(sizeof *he2, 1);
 		XXXAN(he2);
@@ -234,12 +234,12 @@
 	assert(he->refcnt > 0);
 	assert(he->hash < hcl_nhash);
 	assert(hp == &hcl_head[he->hash]);
-	LOCK(&hp->mtx);
+	Lck_Lock(&hp->mtx);
 	if (--he->refcnt == 0)
 		VTAILQ_REMOVE(&hp->head, he, list);
 	else
 		he = NULL;
-	UNLOCK(&hp->mtx);
+	Lck_Unlock(&hp->mtx);
 	if (he == NULL)
 		return (1);
 	free(he);

Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_simple_list.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -51,7 +51,7 @@
 };
 
 static VTAILQ_HEAD(, hsl_entry)	hsl_head = VTAILQ_HEAD_INITIALIZER(hsl_head);
-static MTX hsl_mutex;
+static struct lock hsl_mtx;
 
 /*--------------------------------------------------------------------
  * The ->init method is called during process start and allows
@@ -62,7 +62,7 @@
 hsl_start(void)
 {
 
-	MTX_INIT(&hsl_mutex);
+	Lck_New(&hsl_mtx);
 }
 
 /*--------------------------------------------------------------------
@@ -78,7 +78,7 @@
 	struct hsl_entry *he, *he2;
 	int i;
 
-	LOCK(&hsl_mutex);
+	Lck_Lock(&hsl_mtx);
 	VTAILQ_FOREACH(he, &hsl_head, list) {
 		i = HSH_Compare(sp, he->obj);
 		if (i < 0)
@@ -87,7 +87,7 @@
 			break;
 		he->refcnt++;
 		nobj = he->obj;
-		UNLOCK(&hsl_mutex);
+		Lck_Unlock(&hsl_mtx);
 		return (nobj);
 	}
 	if (nobj != NULL) {
@@ -107,7 +107,7 @@
 		else
 			VTAILQ_INSERT_TAIL(&hsl_head, he2, list);
 	}
-	UNLOCK(&hsl_mutex);
+	Lck_Unlock(&hsl_mtx);
 	return (nobj);
 }
 
@@ -123,14 +123,14 @@
 
 	AN(obj->hashpriv);
 	he = obj->hashpriv;
-	LOCK(&hsl_mutex);
+	Lck_Lock(&hsl_mtx);
 	if (--he->refcnt == 0) {
 		VTAILQ_REMOVE(&hsl_head, he, list);
 		free(he);
 		ret = 0;
 	} else
 		ret = 1;
-	UNLOCK(&hsl_mutex);
+	Lck_Unlock(&hsl_mtx);
 	return (ret);
 }
 

Modified: trunk/varnish-cache/bin/varnishd/shmlog.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/shmlog.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/shmlog.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -64,7 +64,7 @@
 struct varnish_stats *VSL_stats;
 static struct shmloghead *loghead;
 static unsigned char *logstart;
-static MTX vsl_mtx;
+static pthread_mutex_t vsl_mtx;
 
 
 static void
@@ -287,7 +287,7 @@
 	assert(loghead->hdrsize == sizeof *loghead);
 	/* XXX more check sanity of loghead  ? */
 	logstart = (unsigned char *)loghead + loghead->start;
-	MTX_INIT(&vsl_mtx);
+	AZ(pthread_mutex_init(&vsl_mtx, NULL));
 	loghead->starttime = TIM_real();
 	loghead->panicstr[0] = '\0';
 	memset(VSL_stats, 0, sizeof *VSL_stats);

Modified: trunk/varnish-cache/bin/varnishd/storage_file.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/storage_file.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/storage_file.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -115,7 +115,7 @@
 	struct smfhead		order;
 	struct smfhead		free[NBUCKET];
 	struct smfhead		used;
-	MTX			mtx;
+	struct lock		mtx;
 };
 
 /*--------------------------------------------------------------------*/
@@ -609,7 +609,7 @@
 	/* XXX */
 	if (sum < MINPAGES * (off_t)getpagesize())
 		exit (2);
-	MTX_INIT(&sc->mtx);
+	Lck_New(&sc->mtx);
 
 	VSL_stats->sm_bfree += sc->filesize;
 }
@@ -625,18 +625,18 @@
 	assert(size > 0);
 	size += (sc->pagesize - 1);
 	size &= ~(sc->pagesize - 1);
-	LOCK(&sc->mtx);
+	Lck_Lock(&sc->mtx);
 	VSL_stats->sm_nreq++;
 	smf = alloc_smf(sc, size);
 	if (smf == NULL) {
-		UNLOCK(&sc->mtx);
+		Lck_Unlock(&sc->mtx);
 		return (NULL);
 	}
 	CHECK_OBJ_NOTNULL(smf, SMF_MAGIC);
 	VSL_stats->sm_nobj++;
 	VSL_stats->sm_balloc += smf->size;
 	VSL_stats->sm_bfree -= smf->size;
-	UNLOCK(&sc->mtx);
+	Lck_Unlock(&sc->mtx);
 	CHECK_OBJ_NOTNULL(&smf->s, STORAGE_MAGIC);	/*lint !e774 */
 	XXXAN(smf);
 	assert(smf->size == size);
@@ -668,12 +668,12 @@
 	size += (sc->pagesize - 1);
 	size &= ~(sc->pagesize - 1);
 	if (smf->size > size) {
-		LOCK(&sc->mtx);
+		Lck_Lock(&sc->mtx);
 		VSL_stats->sm_balloc -= (smf->size - size);
 		VSL_stats->sm_bfree += (smf->size - size);
 		trim_smf(smf, size);
 		assert(smf->size == size);
-		UNLOCK(&sc->mtx);
+		Lck_Unlock(&sc->mtx);
 		smf->s.space = size;
 	}
 }
@@ -690,12 +690,12 @@
 	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
 	CAST_OBJ_NOTNULL(smf, s->priv, SMF_MAGIC);
 	sc = smf->sc;
-	LOCK(&sc->mtx);
+	Lck_Lock(&sc->mtx);
 	VSL_stats->sm_nobj--;
 	VSL_stats->sm_balloc -= smf->size;
 	VSL_stats->sm_bfree += smf->size;
 	free_smf(smf);
-	UNLOCK(&sc->mtx);
+	Lck_Unlock(&sc->mtx);
 }
 
 /*--------------------------------------------------------------------*/

Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/storage_malloc.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/storage_malloc.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -44,7 +44,7 @@
 #include "stevedore.h"
 
 static size_t			sma_max = SIZE_MAX;
-static MTX			sma_mtx;
+static struct lock		sma_mtx;
 
 struct sma {
 	struct storage		s;
@@ -56,7 +56,7 @@
 {
 	struct sma *sma;
 
-	LOCK(&sma_mtx);
+	Lck_Lock(&sma_mtx);
 	VSL_stats->sma_nreq++;
 	if (VSL_stats->sma_nbytes + size > sma_max)
 		size = 0;
@@ -65,7 +65,7 @@
 		VSL_stats->sma_nbytes += size;
 		VSL_stats->sma_balloc += size;
 	}
-	UNLOCK(&sma_mtx);
+	Lck_Unlock(&sma_mtx);
 
 	if (size == 0)
 		return (NULL);
@@ -94,11 +94,11 @@
 	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
 	sma = s->priv;
 	assert(sma->sz == sma->s.space);
-	LOCK(&sma_mtx);
+	Lck_Lock(&sma_mtx);
 	VSL_stats->sma_nobj--;
 	VSL_stats->sma_nbytes -= sma->sz;
 	VSL_stats->sma_bfree += sma->sz;
-	UNLOCK(&sma_mtx);
+	Lck_Unlock(&sma_mtx);
 	free(sma->s.ptr);
 	free(sma);
 }
@@ -113,11 +113,11 @@
 	sma = s->priv;
 	assert(sma->sz == sma->s.space);
 	if ((p = realloc(sma->s.ptr, size)) != NULL) {
-		LOCK(&sma_mtx);
+		Lck_Lock(&sma_mtx);
 		VSL_stats->sma_nbytes -= (sma->sz - size);
 		VSL_stats->sma_bfree += sma->sz - size;
 		sma->sz = size;
-		UNLOCK(&sma_mtx);
+		Lck_Unlock(&sma_mtx);
 		sma->s.ptr = p;
 		sma->s.space = size;
 	}
@@ -150,7 +150,7 @@
 sma_open(const struct stevedore *st)
 {
 	(void)st;
-	AZ(pthread_mutex_init(&sma_mtx, NULL));
+	Lck_New(&sma_mtx);
 }
 
 struct stevedore sma_stevedore = {

Modified: trunk/varnish-cache/bin/varnishd/storage_synth.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/storage_synth.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/storage_synth.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -43,18 +43,18 @@
 #include "vsb.h"
 #include "stevedore.h"
 
-static MTX			sms_mtx;
+static struct lock		sms_mtx;
 
 static void
 sms_free(struct storage *sto)
 {
 
 	CHECK_OBJ_NOTNULL(sto, STORAGE_MAGIC);
-	LOCK(&sms_mtx);
+	Lck_Lock(&sms_mtx);
 	VSL_stats->sms_nobj--;
 	VSL_stats->sms_nbytes -= sto->len;
 	VSL_stats->sms_bfree += sto->len;
-	UNLOCK(&sms_mtx);
+	Lck_Unlock(&sms_mtx);
 	vsb_delete(sto->priv);
 	free(sto);
 }
@@ -63,7 +63,7 @@
 SMS_Init(void)
 {
 
-	AZ(pthread_mutex_init(&sms_mtx, NULL));
+	Lck_New(&sms_mtx);
 }
 
 static struct stevedore sms_stevedore = {
@@ -82,10 +82,10 @@
 	HSH_Freestore(obj);
 	obj->len = 0;
 
-	LOCK(&sms_mtx);
+	Lck_Lock(&sms_mtx);
 	VSL_stats->sms_nreq++;
 	VSL_stats->sms_nobj++;
-	UNLOCK(&sms_mtx);
+	Lck_Unlock(&sms_mtx);
 
 	sto = calloc(sizeof *sto, 1);
 	XXXAN(sto);

Modified: trunk/varnish-cache/bin/varnishd/storage_umem.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/storage_umem.c	2008-11-10 12:43:58 UTC (rev 3380)
+++ trunk/varnish-cache/bin/varnishd/storage_umem.c	2008-11-10 19:46:25 UTC (rev 3381)
@@ -61,7 +61,7 @@
 {
 	struct smu *smu;
 
-	LOCK(&smu_mtx);
+	Lck_Lock(&smu_mtx);
 	VSL_stats->sma_nreq++;
 	if (VSL_stats->sma_nbytes + size > smu_max)
 		size = 0;
@@ -70,7 +70,7 @@
 		VSL_stats->sma_nbytes += size;
 		VSL_stats->sma_balloc += size;
 	}
-	UNLOCK(&smu_mtx);
+	Lck_Unlock(&smu_mtx);
 
 	if (size == 0)
 		return (NULL);
@@ -99,11 +99,11 @@
 	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
 	smu = s->priv;
 	assert(smu->sz == smu->s.space);
-	LOCK(&smu_mtx);
+	Lck_Lock(&smu_mtx);
 	VSL_stats->sma_nobj--;
 	VSL_stats->sma_nbytes -= smu->sz;
 	VSL_stats->sma_bfree += smu->sz;
-	UNLOCK(&smu_mtx);
+	Lck_Unlock(&smu_mtx);
 	umem_free(smu->s.ptr, smu->s.space);
 	umem_free(smu, sizeof *smu);
 }
@@ -120,11 +120,11 @@
 	if ((p = umem_alloc(size, UMEM_DEFAULT)) != NULL) {
 		memcpy(p, smu->s.ptr, size);
 		umem_free(smu->s.ptr, smu->s.space);
-		LOCK(&smu_mtx);
+		Lck_Lock(&smu_mtx);
 		VSL_stats->sma_nbytes -= (smu->sz - size);
 		VSL_stats->sma_bfree += smu->sz - size;
 		smu->sz = size;
-		UNLOCK(&smu_mtx);
+		Lck_Unlock(&smu_mtx);
 		smu->s.ptr = p;
 		smu->s.space = size;
 	}



From phk at projects.linpro.no  Mon Nov 10 19:48:08 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 10 Nov 2008 20:48:08 +0100 (CET)
Subject: r3382 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081110194808.3FC3E1EC9A0@projects.linpro.no>

Author: phk
Date: 2008-11-10 20:48:08 +0100 (Mon, 10 Nov 2008)
New Revision: 3382

Added:
   trunk/varnish-cache/bin/varnishd/cache_lck.c
Log:
Remember to add this file in second attempt :-)



Added: trunk/varnish-cache/bin/varnishd/cache_lck.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_lck.c	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishd/cache_lck.c	2008-11-10 19:48:08 UTC (rev 3382)
@@ -0,0 +1,183 @@
+/*-
+ * Copyright (c) 2008 Linpro AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $Id$
+ *
+ * The geniuses who came up with pthreads did not think operations like
+ * pthread_assert_mutex_held() were important enough to include them in
+ * the API.
+ *
+ * Build our own locks on top of pthread mutexes and hope that the next
+ * civilization is better at such crucial details than this one.
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+
+#include <pthread.h>
+#ifdef HAVE_PTHREAD_NP_H
+#include <pthread_np.h>
+#endif
+#include <stdlib.h>
+
+#include "shmlog.h"
+#include "cache.h"
+
+struct ilck {
+	unsigned		magic;
+#define ILCK_MAGIC		0x7b86c8a5
+	pthread_mutex_t		mtx;
+	pthread_t		owner;
+	VTAILQ_ENTRY(ilck)	list;
+	const char		*w;
+};
+
+static VTAILQ_HEAD(, ilck)	ilck_head =
+    VTAILQ_HEAD_INITIALIZER(ilck_head);
+
+static pthread_mutex_t		lck_mtx;
+
+void
+Lck__Lock(struct lock *lck, const char *p, const char *f, int l)
+{
+	struct ilck *ilck;
+	int r;
+
+	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
+	if (!(params->diag_bitmap & 0x18)) {
+		AZ(pthread_mutex_lock(&ilck->mtx));
+		AZ(ilck->owner);
+		ilck->owner = pthread_self();
+		return;
+	}
+	r = pthread_mutex_trylock(&ilck->mtx);
+	assert(r == 0 || errno == EBUSY);
+	if (r) {
+		VSL(SLT_Debug, 0, "MTX_CONTEST(%s,%s,%d,%s)", p, f, l, ilck->w);
+		AZ(pthread_mutex_lock(&ilck->mtx));
+	} else if (params->diag_bitmap & 0x8) {
+		VSL(SLT_Debug, 0, "MTX_LOCK(%s,%s,%d,%s)", p, f, l, ilck->w);
+	}
+	AZ(ilck->owner);
+	ilck->owner = pthread_self();
+}
+
+void
+Lck__Unlock(struct lock *lck, const char *p, const char *f, int l)
+{
+	struct ilck *ilck;
+
+	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
+	assert(ilck->owner == pthread_self());
+	ilck->owner = NULL;
+	AZ(pthread_mutex_unlock(&ilck->mtx));
+	if (params->diag_bitmap & 0x8)
+		VSL(SLT_Debug, 0, "MTX_UNLOCK(%s,%s,%d,%s)", p, f, l, ilck->w);
+}
+
+int
+Lck__Trylock(struct lock *lck, const char *p, const char *f, int l)
+{
+	struct ilck *ilck;
+	int r;
+
+	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
+	r = pthread_mutex_lock(&ilck->mtx);
+	assert(r == 0 || errno == EBUSY);
+	if (params->diag_bitmap & 0x8)
+		VSL(SLT_Debug, 0,
+		    "MTX_TRYLOCK(%s,%s,%d,%s) = %d", p, f, l, ilck->w);
+	if (r == 0) {
+		AZ(ilck->owner);
+		ilck->owner = pthread_self();
+	}
+	return (r);
+}
+
+void
+Lck__Assert(struct lock *lck, int held)
+{
+	struct ilck *ilck;
+
+	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
+	if (held)
+		assert(ilck->owner == pthread_self());
+	else
+		assert(ilck->owner != pthread_self());
+}
+
+void
+Lck_CondWait(pthread_cond_t *cond, struct lock *lck)
+{
+	struct ilck *ilck;
+
+	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
+	assert(ilck->owner == pthread_self());
+	ilck->owner = NULL;
+	AZ(pthread_cond_wait(cond, &ilck->mtx));
+	AZ(ilck->owner);
+	ilck->owner = pthread_self();
+}
+
+void
+Lck__New(struct lock *lck, const char *w)
+{
+	struct ilck *ilck;
+
+	AZ(lck->priv);
+	ALLOC_OBJ(ilck, ILCK_MAGIC);
+	AN(ilck);
+	ilck->w = w;
+	AZ(pthread_mutex_init(&ilck->mtx, NULL));
+	AZ(pthread_mutex_lock(&lck_mtx));
+	VTAILQ_INSERT_TAIL(&ilck_head, ilck, list);
+	AZ(pthread_mutex_unlock(&lck_mtx));
+	lck->priv = ilck;
+}
+
+void
+Lck_Delete(struct lock *lck)
+{
+	struct ilck *ilck;
+
+	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
+	lck->priv = NULL;
+	AZ(pthread_mutex_lock(&lck_mtx));
+	VTAILQ_REMOVE(&ilck_head, ilck, list);
+	AZ(pthread_mutex_unlock(&lck_mtx));
+	AZ(pthread_mutex_destroy(&ilck->mtx));
+	FREE_OBJ(ilck);
+}
+
+
+void
+LCK_Init(void)
+{
+
+	AZ(pthread_mutex_init(&lck_mtx, NULL));
+}



From phk at projects.linpro.no  Tue Nov 11 13:15:01 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 11 Nov 2008 14:15:01 +0100 (CET)
Subject: r3383 - trunk/varnish-cache/lib/libvarnish
Message-ID: <20081111131501.62A441EC9A5@projects.linpro.no>

Author: phk
Date: 2008-11-11 14:15:01 +0100 (Tue, 11 Nov 2008)
New Revision: 3383

Modified:
   trunk/varnish-cache/lib/libvarnish/vlu.c
Log:
React to both NL and CR in VLU



Modified: trunk/varnish-cache/lib/libvarnish/vlu.c
===================================================================
--- trunk/varnish-cache/lib/libvarnish/vlu.c	2008-11-10 19:48:08 UTC (rev 3382)
+++ trunk/varnish-cache/lib/libvarnish/vlu.c	2008-11-11 13:15:01 UTC (rev 3383)
@@ -87,8 +87,11 @@
 
 	l->buf[l->bufp] = '\0';
 	for (p = l->buf; *p != '\0'; p = q) {
-		q = strchr(p, '\n');
-		if (q == NULL)
+		/* Find first CR or NL */
+		for (q = p; *q != '\0'; q++)
+			if (*q == '\n' || *q == '\r')
+				break;
+		if (*q == '\0')
 			break;
 		*q++ = '\0';
 		i = l->func(l->priv, p);



From phk at projects.linpro.no  Tue Nov 11 13:38:09 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 11 Nov 2008 14:38:09 +0100 (CET)
Subject: r3384 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081111133809.C2A4C1ED2C2@projects.linpro.no>

Author: phk
Date: 2008-11-11 14:38:09 +0100 (Tue, 11 Nov 2008)
New Revision: 3384

Modified:
   trunk/varnish-cache/bin/varnishd/cache_lck.c
Log:
Have I mentioned that I think POSIX is a bunch of amateurs ?

There is no NULL value for pthread_t and you have to call a function
to compare them.



Modified: trunk/varnish-cache/bin/varnishd/cache_lck.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_lck.c	2008-11-11 13:15:01 UTC (rev 3383)
+++ trunk/varnish-cache/bin/varnishd/cache_lck.c	2008-11-11 13:38:09 UTC (rev 3384)
@@ -52,6 +52,7 @@
 	unsigned		magic;
 #define ILCK_MAGIC		0x7b86c8a5
 	pthread_mutex_t		mtx;
+	int			held;
 	pthread_t		owner;
 	VTAILQ_ENTRY(ilck)	list;
 	const char		*w;
@@ -71,8 +72,9 @@
 	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
 	if (!(params->diag_bitmap & 0x18)) {
 		AZ(pthread_mutex_lock(&ilck->mtx));
-		AZ(ilck->owner);
+		AZ(ilck->held);
 		ilck->owner = pthread_self();
+		ilck->held = 1;
 		return;
 	}
 	r = pthread_mutex_trylock(&ilck->mtx);
@@ -83,8 +85,9 @@
 	} else if (params->diag_bitmap & 0x8) {
 		VSL(SLT_Debug, 0, "MTX_LOCK(%s,%s,%d,%s)", p, f, l, ilck->w);
 	}
-	AZ(ilck->owner);
+	AZ(ilck->held);
 	ilck->owner = pthread_self();
+	ilck->held = 1;
 }
 
 void
@@ -93,8 +96,9 @@
 	struct ilck *ilck;
 
 	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
-	assert(ilck->owner == pthread_self());
-	ilck->owner = NULL;
+	assert(pthread_equal(ilck->owner, pthread_self()));
+	AN(ilck->held);
+	ilck->held = 0;
 	AZ(pthread_mutex_unlock(&ilck->mtx));
 	if (params->diag_bitmap & 0x8)
 		VSL(SLT_Debug, 0, "MTX_UNLOCK(%s,%s,%d,%s)", p, f, l, ilck->w);
@@ -113,7 +117,8 @@
 		VSL(SLT_Debug, 0,
 		    "MTX_TRYLOCK(%s,%s,%d,%s) = %d", p, f, l, ilck->w);
 	if (r == 0) {
-		AZ(ilck->owner);
+		AZ(ilck->held);
+		ilck->held = 1;
 		ilck->owner = pthread_self();
 	}
 	return (r);
@@ -126,9 +131,11 @@
 
 	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
 	if (held)
-		assert(ilck->owner == pthread_self());
+		assert(ilck->held &&
+		    pthread_equal(ilck->owner, pthread_self()));
 	else
-		assert(ilck->owner != pthread_self());
+		assert(!ilck->held ||
+		    !pthread_equal(ilck->owner, pthread_self()));
 }
 
 void
@@ -137,10 +144,11 @@
 	struct ilck *ilck;
 
 	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
-	assert(ilck->owner == pthread_self());
-	ilck->owner = NULL;
+	AN(ilck->held);
+	assert(pthread_equal(ilck->owner, pthread_self()));
+	ilck->held = 0;
 	AZ(pthread_cond_wait(cond, &ilck->mtx));
-	AZ(ilck->owner);
+	AZ(ilck->held);
 	ilck->owner = pthread_self();
 }
 



From phk at projects.linpro.no  Tue Nov 11 13:43:01 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 11 Nov 2008 14:43:01 +0100 (CET)
Subject: r3385 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081111134301.E17FF1EC9A5@projects.linpro.no>

Author: phk
Date: 2008-11-11 14:43:01 +0100 (Tue, 11 Nov 2008)
New Revision: 3385

Modified:
   trunk/varnish-cache/bin/varnishd/cache_lck.c
Log:
Remember to set lock "held" in CondWait



Modified: trunk/varnish-cache/bin/varnishd/cache_lck.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_lck.c	2008-11-11 13:38:09 UTC (rev 3384)
+++ trunk/varnish-cache/bin/varnishd/cache_lck.c	2008-11-11 13:43:01 UTC (rev 3385)
@@ -149,6 +149,7 @@
 	ilck->held = 0;
 	AZ(pthread_cond_wait(cond, &ilck->mtx));
 	AZ(ilck->held);
+	ilck->held = 1;
 	ilck->owner = pthread_self();
 }
 



From phk at projects.linpro.no  Tue Nov 11 19:06:56 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 11 Nov 2008 20:06:56 +0100 (CET)
Subject: r3386 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081111190656.1FEE51ED2C2@projects.linpro.no>

Author: phk
Date: 2008-11-11 20:06:55 +0100 (Tue, 11 Nov 2008)
New Revision: 3386

Modified:
   trunk/varnish-cache/bin/varnishd/cache_center.c
Log:
Implement restart in vcl_hit.

Fixes #365



Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2008-11-11 13:43:01 UTC (rev 3385)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2008-11-11 19:06:55 UTC (rev 3386)
@@ -543,7 +543,9 @@
 		sp->step = STP_ERROR;
 		return (0);
 	case VCL_RET_RESTART:
-		INCOMPL();
+		sp->director = NULL;
+		sp->restarts++;
+		sp->step = STP_RECV;
 		return (0);
 	default:
 		WRONG("Illegal action in vcl_hit{}");



From phk at projects.linpro.no  Tue Nov 11 19:07:30 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 11 Nov 2008 20:07:30 +0100 (CET)
Subject: r3387 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <20081111190730.C237E1EC9A0@projects.linpro.no>

Author: phk
Date: 2008-11-11 20:07:30 +0100 (Tue, 11 Nov 2008)
New Revision: 3387

Added:
   trunk/varnish-cache/bin/varnishtest/tests/r00365.vtc
Log:
Regression test case for ticket 365: restart in hit.



Added: trunk/varnish-cache/bin/varnishtest/tests/r00365.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00365.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00365.vtc	2008-11-11 19:07:30 UTC (rev 3387)
@@ -0,0 +1,31 @@
+# $Id$
+
+test "Test restarts in vcl_hit"
+
+server s1 {
+	rxreq
+	expect req.url == "/foo"
+	txresp -status 200 -body "1"
+	rxreq
+	expect req.url == "/foo"
+	txresp -status 200 -body "22"
+} -start
+
+varnish v1 -vcl+backend { 
+	sub vcl_hit {
+		set obj.cacheable = false;
+		restart;
+	}
+} -start
+
+client c1 {
+	txreq -url "/foo"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 1
+
+	txreq -url "/foo"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 2
+} -run



From phk at projects.linpro.no  Tue Nov 11 20:22:05 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 11 Nov 2008 21:22:05 +0100 (CET)
Subject: r3388 - in trunk/varnish-cache/bin: varnishd varnishtest/tests
Message-ID: <20081111202205.851081EC9A0@projects.linpro.no>

Author: phk
Date: 2008-11-11 21:22:05 +0100 (Tue, 11 Nov 2008)
New Revision: 3388

Added:
   trunk/varnish-cache/bin/varnishtest/tests/v00023.vtc
Modified:
   trunk/varnish-cache/bin/varnishd/cache_vrt.c
Log:
Make sure that
	set obj.ttl = 0s
means that the object is not hit again by actually using "-1" instead.

This works around the rounding error which otherwise causes the object
to be inside TTL for up to one second - epsilon.



Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt.c	2008-11-11 19:07:30 UTC (rev 3387)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt.c	2008-11-11 20:22:05 UTC (rev 3388)
@@ -320,9 +320,16 @@
 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);	/* XXX */
 	WSP(sp, SLT_TTL, "%u VCL %.0f %.0f",
 	    sp->obj->xid, a, sp->t_req);
-	if (a < 0)
-		a = 0;
-	sp->obj->ttl = sp->t_req + a;
+	/*
+	 * If people set obj.ttl = 0s, they don't expect it to be cacheable
+	 * any longer, but it will still be for up to 1s - epsilon because
+	 * of the rounding to seconds.
+	 * We special case and make sure that rounding does not surprise.
+	 */
+	if (a <= 0)
+		sp->obj->ttl = sp->t_req - 1;
+	else
+		sp->obj->ttl = sp->t_req + a;
 	EXP_Rearm(sp->obj);
 }
 

Added: trunk/varnish-cache/bin/varnishtest/tests/v00023.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/v00023.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/v00023.vtc	2008-11-11 20:22:05 UTC (rev 3388)
@@ -0,0 +1,31 @@
+# $Id$
+
+test "Test that obj.ttl = 0s prevents subsequent hits"
+
+server s1 {
+	rxreq
+	expect req.url == "/foo"
+	txresp -status 200 -body "1"
+	rxreq
+	expect req.url == "/foo"
+	txresp -status 200 -body "22"
+} -start
+
+varnish v1 -vcl+backend { 
+	sub vcl_hit {
+		set obj.ttl = 0s;
+		restart;
+	}
+} -start
+
+client c1 {
+	txreq -url "/foo"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 1
+
+	txreq -url "/foo"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 2
+} -run



From phk at projects.linpro.no  Tue Nov 11 20:40:37 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 11 Nov 2008 21:40:37 +0100 (CET)
Subject: r3389 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <20081111204037.6256D1EC7FE@projects.linpro.no>

Author: phk
Date: 2008-11-11 21:40:37 +0100 (Tue, 11 Nov 2008)
New Revision: 3389

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/v00016.vtc
Log:
Fix this test-case to not rely on being able to compile /dev/null
now that this is no longer possible.



Modified: trunk/varnish-cache/bin/varnishtest/tests/v00016.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/v00016.vtc	2008-11-11 20:22:05 UTC (rev 3388)
+++ trunk/varnish-cache/bin/varnishtest/tests/v00016.vtc	2008-11-11 20:40:37 UTC (rev 3389)
@@ -2,16 +2,20 @@
 
 test "Various VCL compiler coverage tests"
 
+shell "true > /tmp/_varnishtest_empty_file"
+
 varnish v1 -vcl {
 	backend b { .host = "127.0.0.1"; }
-	include "/dev/null" ;
+	include "/tmp/_varnishtest_empty_file" ;
 }
 
 varnish v1 -badvcl {
 	backend b { .host = "127.0.0.1"; }
-	include "/dev/null" |
+	include "/tmp/_varnishtest_empty_file" |
 }
 
+shell "rm -f /tmp/_varnishtest_empty_file"
+
 varnish v1 -badvcl {
 	backend b { .host = "127.0.0.1"; }
 	include <<



From phk at projects.linpro.no  Fri Nov 14 00:19:34 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Fri, 14 Nov 2008 01:19:34 +0100 (CET)
Subject: r3390 - trunk/varnish-cache/lib/libvarnish
Message-ID: <20081114001934.3FA701EC9A0@projects.linpro.no>

Author: phk
Date: 2008-11-14 01:19:33 +0100 (Fri, 14 Nov 2008)
New Revision: 3390

Modified:
   trunk/varnish-cache/lib/libvarnish/binary_heap.c
Log:
Rework the binary heap, we use for expiry processing, to deal more
gracefully with large number of objects.

Previously we kept all objects in a single array which resultined
in increasingly infrequent but increasingly demanding calls to
calloc(3) with the consequent massive memory copies.  We also did
not release memory again if unused.

Now we stripe the array into rows of 64k objects each.

This number is a compromise between space wastage, max 1MB on a
64bit machine, and the desire to not add and delete rows all the
time.  With 64k objects in a row, even on a very busy server would
only add a new row every 5...10 seconds during ramp up.

Delete unused rows, but keep a hysteresis of an entire empty row
to avoid silly add-delete-add-delete-add-delete behaviour at row
boundaries.

Streamline some of the functions a bit.

Fixes #210




Modified: trunk/varnish-cache/lib/libvarnish/binary_heap.c
===================================================================
--- trunk/varnish-cache/lib/libvarnish/binary_heap.c	2008-11-11 20:40:37 UTC (rev 3389)
+++ trunk/varnish-cache/lib/libvarnish/binary_heap.c	2008-11-14 00:19:33 UTC (rev 3390)
@@ -32,8 +32,6 @@
  *
  * We use a malloc(3)/realloc(3) array to store the pointers using the
  * classical FORTRAN strategy.
- *
- * XXX: the array is not scaled back when items are deleted.
  */
 
 #include "config.h"
@@ -44,38 +42,71 @@
 #include "binary_heap.h"
 #include "libvarnish.h"
 
+/* Paramters ---------------------------------------------------------*/
+
+/*
+ * The number of elements in a row has to be a compromise between
+ * wasted space and number of memory allocations.
+ * With 64k objects per row, there will be at least 5...10 seconds
+ * between row additions on a very busy server.
+ * At the same time, the worst case amount of wasted memory is kept
+ * at a reasonable 1 MB -- two rows on 64bit system.
+ * Finally, but without practical significance: 16 bits should be
+ * easier for the compiler to optimize.
+ */
+#define ROW_SHIFT		16
+
 /* Private definitions -----------------------------------------------*/
 
-#define MIN_LENGTH		16
-
 #define ROOT_IDX		1
 
+#define ROW_WIDTH		(1 << ROW_SHIFT)
+
+/*lint -emacro(572, ROW) shift 0 >> by 16 */
+/*lint -emacro(835, ROW) 0 left of >> */
+/*lint -emacro(778, ROW) const >> evaluates to zero */
+#define ROW(b, n)		((b)->array[(n) >> ROW_SHIFT])
+
+/*lint -emacro(835, A) 0 left of & */
+#define A(b, n)			ROW(b, n)[(n) & (ROW_WIDTH - 1)]
+
 struct binheap {
 	unsigned		magic;
 #define BINHEAP_MAGIC		0xf581581aU	/* from /dev/random */
 	void			*priv;
 	binheap_cmp_t		*cmp;
 	binheap_update_t	*update;
-	void			**array;
+	void			***array;
+	unsigned		rows;
 	unsigned		length;
 	unsigned		next;
-	unsigned		granularity;
 };
 
 #define PARENT(u)	((u) / 2)
+/*lint -emacro(835, CHILD) 0 right of + */
 #define CHILD(u,n)	((u) * 2 + (n))
 
 /* Implementation ----------------------------------------------------*/
 
 static void
-binheap_update(const struct binheap *bh, unsigned u)
+binheap_addrow(struct binheap *bh)
 {
-	assert(bh->magic == BINHEAP_MAGIC);
-	assert(u < bh->next);
-	assert(bh->array[u] != NULL);
-	if (bh->update == NULL)
-		return;
-	bh->update(bh->priv, bh->array[u], u);
+	unsigned u;
+
+	/* First make sure we have space for another row */
+	if (&ROW(bh, bh->length) >= bh->array + bh->rows) {
+		u = bh->rows * 2;
+		bh->array = realloc(bh->array, sizeof(*bh->array) * u);
+		assert(bh->array != NULL);
+
+		/* NULL out new pointers */
+		while (bh->rows < u)
+			bh->array[bh->rows++] = NULL;
+	}
+	assert(ROW(bh, bh->length) == NULL);
+	ROW(bh, bh->length) = malloc(sizeof(**bh->array) * ROW_WIDTH);
+	assert(ROW(bh, bh->length));
+	bh->length += ROW_WIDTH;
 }
 
 struct binheap *
@@ -90,15 +121,27 @@
 	bh->cmp = cmp_f;
 	bh->update = update_f;
 	bh->next = ROOT_IDX;
-	bh->length = MIN_LENGTH;
-	bh->array = calloc(sizeof *bh->array, bh->length);
+	bh->rows = 16;		/* A tiny-ish number */
+	bh->array = calloc(sizeof *bh->array, bh->rows);
 	assert(bh->array != NULL);
-	bh->granularity = getpagesize() / sizeof *bh->array;
+	binheap_addrow(bh);
+	A(bh, ROOT_IDX) = NULL;
 	bh->magic = BINHEAP_MAGIC;
 	return (bh);
 }
 
 static void
+binheap_update(const struct binheap *bh, unsigned u)
+{
+	assert(bh->magic == BINHEAP_MAGIC);
+	assert(u < bh->next);
+	assert(A(bh, u) != NULL);
+	if (bh->update == NULL)
+		return;
+	bh->update(bh->priv, A(bh, u), u);
+}
+
+static void
 binhead_swap(const struct binheap *bh, unsigned u, unsigned v)
 {
 	void *p;
@@ -106,9 +149,9 @@
 	assert(bh->magic == BINHEAP_MAGIC);
 	assert(u < bh->next);
 	assert(v < bh->next);
-	p = bh->array[u];
-	bh->array[u] = bh->array[v];
-	bh->array[v] = p;
+	p = A(bh, u);
+	A(bh, u) = A(bh, v);
+	A(bh, v) = p;
 	binheap_update(bh, u);
 	binheap_update(bh, v);
 }
@@ -121,11 +164,10 @@
 	assert(bh->magic == BINHEAP_MAGIC);
 	while (u > ROOT_IDX) {
 		v = PARENT(u);
-		if (bh->cmp(bh->priv, bh->array[u], bh->array[v])) {
-			binhead_swap(bh, u, v);
-			u = v;
-		} else
+		if (!bh->cmp(bh->priv, A(bh, u), A(bh, v)))
 			break;
+		binhead_swap(bh, u, v);
+		u = v;
 	}
 	return (u);
 }
@@ -138,28 +180,15 @@
 	assert(bh->magic == BINHEAP_MAGIC);
 	while (1) {
 		v1 = CHILD(u, 0);
-		v2 = CHILD(u, 1);
 		if (v1 >= bh->next)
 			return;
-		if (v2 >= bh->next) {
-			if (!bh->cmp(bh->priv, bh->array[u], bh->array[v1]))
-				binhead_swap(bh, u, v1);
+		v2 = CHILD(u, 1);
+		if (v2 < bh->next && bh->cmp(bh->priv, A(bh, v2), A(bh, v1)))
+			v1 = v2;
+		if (bh->cmp(bh->priv, A(bh, u), A(bh, v1))) 
 			return;
-		}
-		if (bh->cmp(bh->priv, bh->array[v1], bh->array[v2])) {
-			if (!bh->cmp(bh->priv, bh->array[u], bh->array[v1])) {
-				binhead_swap(bh, u, v1);
-				u = v1;
-				continue;
-			}
-		} else {
-			if (!bh->cmp(bh->priv, bh->array[u], bh->array[v2])) {
-				binhead_swap(bh, u, v2);
-				u = v2;
-				continue;
-			}
-		}
-		return;
+		binhead_swap(bh, u, v1);
+		u = v1;
 	}
 }
 
@@ -171,18 +200,10 @@
 	assert(bh != NULL);
 	assert(bh->magic == BINHEAP_MAGIC);
 	assert(bh->length >= bh->next);
-	if (bh->length == bh->next) {
-		if (bh->length >= bh->granularity * 32)
-			bh->length += bh->granularity * 32;
-		else if (bh->length > bh->granularity)
-			bh->length += bh->granularity;
-		else
-			bh->length += bh->length;
-		bh->array = realloc(bh->array, bh->length * sizeof *bh->array);
-		assert(bh->array != NULL);
-	}
+	if (bh->length == bh->next)
+		binheap_addrow(bh);
 	u = bh->next++;
-	bh->array[u] = p;
+	A(bh, u) = p;
 	binheap_update(bh, u);
 	(void)binheap_trickleup(bh, u);
 }
@@ -193,11 +214,33 @@
 
 	assert(bh != NULL);
 	assert(bh->magic == BINHEAP_MAGIC);
-	if (bh->next == ROOT_IDX)
-		return (NULL);
-	return (bh->array[ROOT_IDX]);
+	return (A(bh, ROOT_IDX));
 }
 
+/*
+ * It may seem counter-intuitive that we delete by replacement with
+ * the tail object. "That's almost certain to not belong there, in
+ * particular when we delete the root ?" is the typical reaction.
+ *
+ * If we tried to trickle up into the empty position, we would,
+ * eventually, end up with a hole in the bottom row, at which point
+ * we would move the tail object there.
+ * But there is no guarantee that the tail object would not need to
+ * trickle up from that position, in fact, it might be the new root
+ * of this half of the subtree.
+ * The total number of operations is guaranteed to be at least
+ * N{height} downward selections, because we have to get the hole
+ * all the way down, but in addition to that, we may get up to
+ * N{height}-1 upward trickles.
+ *
+ * When we fill the hole with the tail object, the worst case is
+ * that it trickles all the way down to become the tail object
+ * again.
+ * In other words worst case is N{height} downward trickles.
+ * But there is a pretty decent chance that it does not make
+ * it all the way down.
+ */
+
 void
 binheap_delete(struct binheap *bh, unsigned idx)
 {
@@ -207,18 +250,28 @@
 	assert(bh->next > ROOT_IDX);
 	assert(idx < bh->next);
 	assert(idx > 0);
-	assert(bh->array[idx] != NULL);
-	bh->update(bh->priv, bh->array[idx], 0);
+	assert(A(bh, idx) != NULL);
+	bh->update(bh->priv, A(bh, idx), 0);
 	if (idx == --bh->next) {
-		bh->array[bh->next] = NULL;
+		A(bh, bh->next) = NULL;
 		return;
 	}
-	bh->array[idx] = bh->array[bh->next];
-	bh->array[bh->next] = NULL;
+	A(bh, idx) = A(bh, bh->next);
+	A(bh, bh->next) = NULL;
 	binheap_update(bh, idx);
 	idx = binheap_trickleup(bh, idx);
 	binheap_trickledown(bh, idx);
-	/* XXX: free part of array ? */
+
+	/*
+	 * We keep a hysteresis of one full row before we start to
+	 * return space to the OS to avoid silly behaviour around
+	 * row boundaries.
+	 */
+	if (bh->next + 2 * ROW_WIDTH <= bh->length) {
+		free(ROW(bh, bh->length - 1));
+		ROW(bh, bh->length - 1) = NULL;
+		bh->length -= ROW_WIDTH;
+	}
 }
 
 
@@ -226,7 +279,10 @@
 /* Test driver -------------------------------------------------------*/
 #include <stdio.h>
 
-#if 0
+#if 1
+
+#define N 23
+
 static int
 cmp(void *priv, void *a, void *b)
 {
@@ -247,14 +303,14 @@
 	unsigned u, *up;
 
 	printf("dump\n");
-	f = popen("dot -Tps >> /tmp/_.ps", "w");
+	f = popen("dot -Tps >> /tmp/_.ps 2>/dev/null", "w");
 	assert(f != NULL);
 	fprintf(f, "digraph binheap {\n");
 	fprintf(f, "size=\"7,10\"\n");
 	fprintf(f, "ptr [label=\"%s\"]\n", what);
 	fprintf(f, "ptr -> node_%u\n", ptr);
 	for (u = 1; u < bh->next; u++) {
-		up = bh->array[u];
+		up = A(bh, u);
 		fprintf(f, "node_%u [label=\"%u\"];\n", u, *up);
 		if (u > 0)
 			fprintf(f, "node_%u -> node_%u\n", PARENT(u), u);
@@ -263,7 +319,6 @@
 	pclose(f);
 }
 
-#define N 31
 int
 main(int argc, char **argv)
 {
@@ -286,7 +341,7 @@
 			break;
 		assert(*up >= lu);
 		lu = *up;
-		u = random() % bh->next;
+		u = (random() % (bh->next - 1)) + 1;
 		binheap_delete(bh, u);
 		if (1)
 			dump(bh, "Delete", u);
@@ -301,7 +356,7 @@
 };
 
 #define M 1311191
-#define N 131
+#define N 1311
 
 struct foo ff[N];
 
@@ -332,8 +387,8 @@
 
 	for (u = 2; u < bh->next; u++) {
 		v = PARENT(u);
-		fa = bh->array[u];
-		fb = bh->array[v];
+		fa = A(bh, u);
+		fb = A(bh, v);
 		assert(fa->key > fb->key);
 		continue;
 		printf("[%2u/%2u] %10u > [%2u/%2u] %10u %s\n",
@@ -363,11 +418,17 @@
 	for (u = 0; u < M; u++) {
 		v = random() % N;
 		if (ff[v].idx > 0) {
-			printf("Delete [%u] %'u\n", v, ff[v].key);
+			if (0)
+				printf("Delete [%u] %'u\n", v, ff[v].key);
+			else
+				printf("-%u", v);
 			binheap_delete(bh, ff[v].idx);
 		} else {
 			ff[v].key = random();
-			printf("Insert [%u] %'u\n", v, ff[v].key);
+			if (0) 
+				printf("Insert [%u] %'u\n", v, ff[v].key);
+			else
+				printf("+%u", v);
 			binheap_insert(bh, &ff[v]);
 		}
 		chk(bh);



From phk at projects.linpro.no  Fri Nov 14 09:18:23 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Fri, 14 Nov 2008 10:18:23 +0100 (CET)
Subject: r3391 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081114091823.D8BC61EC45F@projects.linpro.no>

Author: phk
Date: 2008-11-14 10:18:23 +0100 (Fri, 14 Nov 2008)
New Revision: 3391

Removed:
   trunk/varnish-cache/bin/varnishd/cache_vrt_acl.c
Modified:
   trunk/varnish-cache/bin/varnishd/Makefile.am
   trunk/varnish-cache/bin/varnishd/cache_vrt.c
Log:
Merge cache_vrt_acl.c into cache_vrt.c, one one-line function is silly
in a source-file.

(All the work is done by vcc generated code now)



Modified: trunk/varnish-cache/bin/varnishd/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishd/Makefile.am	2008-11-14 00:19:33 UTC (rev 3390)
+++ trunk/varnish-cache/bin/varnishd/Makefile.am	2008-11-14 09:18:23 UTC (rev 3391)
@@ -36,7 +36,6 @@
 	cache_vary.c \
 	cache_vcl.c \
 	cache_vrt.c \
-	cache_vrt_acl.c \
 	cache_vrt_esi.c \
 	cache_vrt_re.c \
 	cache_ws.c \

Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt.c	2008-11-14 00:19:33 UTC (rev 3390)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt.c	2008-11-14 09:18:23 UTC (rev 3391)
@@ -80,6 +80,14 @@
 
 /*--------------------------------------------------------------------*/
 
+void
+VRT_acl_log(const struct sess *sp, const char *msg)
+{
+	WSL(sp->wrk, SLT_VCL_acl, sp->fd, msg);
+}
+
+/*--------------------------------------------------------------------*/
+
 static struct http *
 vrt_selecthttp(const struct sess *sp, enum gethdr_e where)
 {

Deleted: trunk/varnish-cache/bin/varnishd/cache_vrt_acl.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt_acl.c	2008-11-14 00:19:33 UTC (rev 3390)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt_acl.c	2008-11-14 09:18:23 UTC (rev 3391)
@@ -1,45 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2008 Linpro AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $Id$
- *
- * Runtime support for compiled VCL programs, ACLs
- *
- */
-
-#include "config.h"
-
-#include "shmlog.h"
-#include "vrt.h"
-#include "cache.h"
-
-void
-VRT_acl_log(const struct sess *sp, const char *msg)
-{
-	WSL(sp->wrk, SLT_VCL_acl, sp->fd, msg);
-}



From phk at projects.linpro.no  Fri Nov 14 09:48:57 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Fri, 14 Nov 2008 10:48:57 +0100 (CET)
Subject: r3392 - trunk/varnish-cache/lib/libvcl
Message-ID: <20081114094857.56AF41EC0E8@projects.linpro.no>

Author: phk
Date: 2008-11-14 10:48:57 +0100 (Fri, 14 Nov 2008)
New Revision: 3392

Modified:
   trunk/varnish-cache/lib/libvcl/vcc_string.c
Log:
Inline VRT_re_test()



Modified: trunk/varnish-cache/lib/libvcl/vcc_string.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_string.c	2008-11-14 09:18:23 UTC (rev 3391)
+++ trunk/varnish-cache/lib/libvcl/vcc_string.c	2008-11-14 09:48:57 UTC (rev 3392)
@@ -33,6 +33,7 @@
 
 #include <stdio.h>
 #include <string.h>
+#include <regex.h>
 
 #include "vsb.h"
 
@@ -47,13 +48,22 @@
 char *
 vcc_regexp(struct tokenlist *tl, int sub)
 {
-	char buf[32], *p;
+	char buf[BUFSIZ], *p;
+	regex_t	t;
+	int i;
 
 	Expect(tl, CSTR);
-	if (VRT_re_test(tl->sb, tl->t->dec, sub)) {
+	memset(&t, 0, sizeof t);
+	i = regcomp(&t, tl->t->dec, REG_EXTENDED | (sub ? 0 : REG_NOSUB));
+	if (i != 0) {
+		(void)regerror(i, &t, buf, sizeof buf);
+		vsb_printf(tl->sb,
+		    "Regexp compilation error:\n\n%s\n\n", buf);
 		vcc_ErrWhere(tl, tl->t);
+		regfree(&t);
 		return (NULL);
 	}
+	regfree(&t);
 	sprintf(buf, "VGC_re_%u", tl->recnt++);
 	p = TlAlloc(tl, strlen(buf) + 1);
 	strcpy(p, buf);



From phk at projects.linpro.no  Fri Nov 14 09:49:28 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Fri, 14 Nov 2008 10:49:28 +0100 (CET)
Subject: r3393 - in trunk/varnish-cache: bin/varnishd include
Message-ID: <20081114094928.EF0031EC45F@projects.linpro.no>

Author: phk
Date: 2008-11-14 10:49:28 +0100 (Fri, 14 Nov 2008)
New Revision: 3393

Modified:
   trunk/varnish-cache/bin/varnishd/cache_vrt_re.c
   trunk/varnish-cache/include/vrt.h
Log:
Retire VRT_re_test()



Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_re.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt_re.c	2008-11-14 09:48:57 UTC (rev 3392)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt_re.c	2008-11-14 09:49:28 UTC (rev 3393)
@@ -84,25 +84,6 @@
 	return (0);
 }
 
-int
-VRT_re_test(struct vsb *sb, const char *re, int sub)
-{
-	int i;
-	regex_t	t;
-	char buf[BUFSIZ];
-
-	memset(&t, 0, sizeof t);
-	i = regcomp(&t, re, REG_EXTENDED | (sub ? 0 : REG_NOSUB));
-	if (i == 0) {
-		regfree(&t);
-		return (0);
-	}
-	(void)regerror(i, &t, buf, sizeof buf);
-	vsb_printf(sb, "Regexp compilation error:\n\n%s\n\n", buf);
-	regfree(&t);
-	return (1);
-}
-
 const char *
 VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
     const char *sub)

Modified: trunk/varnish-cache/include/vrt.h
===================================================================
--- trunk/varnish-cache/include/vrt.h	2008-11-14 09:48:57 UTC (rev 3392)
+++ trunk/varnish-cache/include/vrt.h	2008-11-14 09:49:28 UTC (rev 3393)
@@ -136,7 +136,6 @@
 void VRT_re_init(void **, const char *, int sub);
 void VRT_re_fini(void *);
 int VRT_re_match(const char *, void *re);
-int VRT_re_test(struct vsb *, const char *, int sub);
 const char *VRT_regsub(const struct sess *sp, int all, const char *,
     void *, const char *);
 



From phk at projects.linpro.no  Fri Nov 14 09:50:12 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Fri, 14 Nov 2008 10:50:12 +0100 (CET)
Subject: r3394 - trunk/varnish-cache/lib/libvcl
Message-ID: <20081114095012.CBD2E1EC0E8@projects.linpro.no>

Author: phk
Date: 2008-11-14 10:50:12 +0100 (Fri, 14 Nov 2008)
New Revision: 3394

Modified:
   trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c
Log:
Regenerated.



Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c	2008-11-14 09:49:28 UTC (rev 3393)
+++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c	2008-11-14 09:50:12 UTC (rev 3394)
@@ -167,10 +167,10 @@
 
 	/* ../../include/vcl.h */
 
-	vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 3339 2008-10-20 20");
-	vsb_cat(sb, ":05:25Z phk $\n *\n * NB:  This file is machine genera");
-	vsb_cat(sb, "ted, DO NOT EDIT!\n *\n * Edit vcc_gen_fixed_token.tcl");
-	vsb_cat(sb, " instead\n */\n\nstruct sess;\n");
+	vsb_cat(sb, "/*\n * $Id: vcc_gen_fixed_token.tcl 3377 2008-11-10 11");
+	vsb_cat(sb, ":55:15Z tfheen $\n *\n * NB:  This file is machine gen");
+	vsb_cat(sb, "erated, DO NOT EDIT!\n *\n * Edit vcc_gen_fixed_token.");
+	vsb_cat(sb, "tcl instead\n */\n\nstruct sess;\n");
 	vsb_cat(sb, "struct cli;\n\ntypedef void vcl_init_f(struct cli *);\n");
 	vsb_cat(sb, "typedef void vcl_fini_f(struct cli *);\n");
 	vsb_cat(sb, "typedef int vcl_func_f(struct sess *sp);\n");
@@ -223,8 +223,8 @@
 	vsb_cat(sb, " * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWI");
 	vsb_cat(sb, "SE) ARISING IN ANY WAY\n * OUT OF THE USE OF THIS SOFT");
 	vsb_cat(sb, "WARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n");
-	vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id: vrt.h 3324 2008-10-18 20:");
-	vsb_cat(sb, "50:10Z phk $\n *\n * Runtime support for compiled VCL ");
+	vsb_cat(sb, " * SUCH DAMAGE.\n *\n * $Id: vrt.h 3393 2008-11-14 09:");
+	vsb_cat(sb, "49:28Z phk $\n *\n * Runtime support for compiled VCL ");
 	vsb_cat(sb, "programs.\n *\n * XXX: When this file is changed, lib/");
 	vsb_cat(sb, "libvcl/vcc_gen_fixed_token.tcl\n");
 	vsb_cat(sb, " * XXX: *MUST* be rerun.\n */\n");
@@ -271,7 +271,6 @@
 	vsb_cat(sb, "g);\n\n/* Regexp related */\nvoid VRT_re_init(void **,");
 	vsb_cat(sb, " const char *, int sub);\nvoid VRT_re_fini(void *);\n");
 	vsb_cat(sb, "int VRT_re_match(const char *, void *re);\n");
-	vsb_cat(sb, "int VRT_re_test(struct vsb *, const char *, int sub);\n");
 	vsb_cat(sb, "const char *VRT_regsub(const struct sess *sp, int all,");
 	vsb_cat(sb, " const char *,\n    void *, const char *);\n");
 	vsb_cat(sb, "\nvoid VRT_panic(struct sess *sp,  const char *, ...);");



From phk at projects.linpro.no  Fri Nov 14 10:22:47 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Fri, 14 Nov 2008 11:22:47 +0100 (CET)
Subject: r3395 - trunk/varnish-cache/lib/libvarnish
Message-ID: <20081114102247.D05451EC9A0@projects.linpro.no>

Author: phk
Date: 2008-11-14 11:22:47 +0100 (Fri, 14 Nov 2008)
New Revision: 3395

Modified:
   trunk/varnish-cache/lib/libvarnish/flint.lnt
Log:
silence config.h noise



Modified: trunk/varnish-cache/lib/libvarnish/flint.lnt
===================================================================
--- trunk/varnish-cache/lib/libvarnish/flint.lnt	2008-11-14 09:50:12 UTC (rev 3394)
+++ trunk/varnish-cache/lib/libvarnish/flint.lnt	2008-11-14 10:22:47 UTC (rev 3395)
@@ -11,6 +11,8 @@
 
 -header(../../config.h)
 
+-efile(451, "../../config.h")
+
 // Fix strchr() semtics, it can only return NULL if arg2 != 0
 -sem(strchr, 1p, type(1), 2n == 0 ? (@p < 1p) : (@p < 1p || @p == 0 ))
 



From tfheen at projects.linpro.no  Fri Nov 14 12:58:33 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Fri, 14 Nov 2008 13:58:33 +0100 (CET)
Subject: r3396 - branches/2.0/varnish-cache/redhat
Message-ID: <20081114125833.A173E1EC45F@projects.linpro.no>

Author: tfheen
Date: 2008-11-14 13:58:33 +0100 (Fri, 14 Nov 2008)
New Revision: 3396

Modified:
   branches/2.0/varnish-cache/redhat/varnish.spec
Log:
Merge in 2.0.2 spec file change

Modified: branches/2.0/varnish-cache/redhat/varnish.spec
===================================================================
--- branches/2.0/varnish-cache/redhat/varnish.spec	2008-11-14 10:22:47 UTC (rev 3395)
+++ branches/2.0/varnish-cache/redhat/varnish.spec	2008-11-14 12:58:33 UTC (rev 3396)
@@ -1,7 +1,7 @@
 Summary: Varnish is a high-performance HTTP accelerator
 Name: varnish
-Version: 2.0
-Release: 2%{?dist}
+Version: 2.0.2
+Release: 1%{?dist}
 License: BSD
 Group: System Environment/Daemons
 URL: http://www.varnish-cache.org/
@@ -220,6 +220,9 @@
 %postun libs -p /sbin/ldconfig
 
 %changelog
+* Mon Nov 10 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0.2-1
+  New upstream release 2.0.2. A bugfix release
+
 * Sun Nov 02 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0.1-2
 - Removed the requirement for kernel => 2.6.0. All supported
   platforms meets this, and it generates strange errors in EPEL



From tfheen at projects.linpro.no  Fri Nov 14 12:59:02 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Fri, 14 Nov 2008 13:59:02 +0100 (CET)
Subject: r3397 - branches/2.0/varnish-cache
Message-ID: <20081114125902.D9F371EC0E8@projects.linpro.no>

Author: tfheen
Date: 2008-11-14 13:59:02 +0100 (Fri, 14 Nov 2008)
New Revision: 3397

Modified:
   branches/2.0/varnish-cache/configure.ac
Log:
Update version number in configure.ac

Modified: branches/2.0/varnish-cache/configure.ac
===================================================================
--- branches/2.0/varnish-cache/configure.ac	2008-11-14 12:58:33 UTC (rev 3396)
+++ branches/2.0/varnish-cache/configure.ac	2008-11-14 12:59:02 UTC (rev 3397)
@@ -3,7 +3,7 @@
 AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006-2008 Linpro AS / Verdens Gang AS])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [2.0.1], [varnish-dev at projects.linpro.no])
+AC_INIT([Varnish], [2.0.2], [varnish-dev at projects.linpro.no])
 AC_CONFIG_SRCDIR(include/varnishapi.h)
 AM_CONFIG_HEADER(config.h)
 



From tfheen at projects.linpro.no  Fri Nov 14 13:04:11 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Fri, 14 Nov 2008 14:04:11 +0100 (CET)
Subject: r3398 - tags
Message-ID: <20081114130411.1FDD81EC45F@projects.linpro.no>

Author: tfheen
Date: 2008-11-14 14:04:10 +0100 (Fri, 14 Nov 2008)
New Revision: 3398

Added:
   tags/varnish-2.0.2/
Log:
Release 2.0.2


Copied: tags/varnish-2.0.2 (from rev 3397, branches/2.0)



From petter at projects.linpro.no  Mon Nov 17 13:31:37 2008
From: petter at projects.linpro.no (petter at projects.linpro.no)
Date: Mon, 17 Nov 2008 14:31:37 +0100 (CET)
Subject: r3399 - trunk/varnish-cache/lib/libjemalloc
Message-ID: <20081117133137.E05631EC0E8@projects.linpro.no>

Author: petter
Date: 2008-11-17 14:31:37 +0100 (Mon, 17 Nov 2008)
New Revision: 3399

Modified:
   trunk/varnish-cache/lib/libjemalloc/jemalloc_linux.c
Log:
Fixed compiling on Linux with -Werror set. Added mising prototypes, removed an unused function and added the ATTRIBUTE_UNUSED to the attribute list for the rb_wrap() macro which generated unused functions.


Modified: trunk/varnish-cache/lib/libjemalloc/jemalloc_linux.c
===================================================================
--- trunk/varnish-cache/lib/libjemalloc/jemalloc_linux.c	2008-11-14 13:04:10 UTC (rev 3398)
+++ trunk/varnish-cache/lib/libjemalloc/jemalloc_linux.c	2008-11-17 13:31:37 UTC (rev 3399)
@@ -197,6 +197,7 @@
 #include <unistd.h>
 #include <fcntl.h>
 #include <pthread.h>
+#include <ansidecl.h>
 
 #include "rb.h"
 
@@ -1113,7 +1114,6 @@
 static bool	base_pages_alloc_mmap(size_t minsize);
 static bool	base_pages_alloc(size_t minsize);
 static void	*base_alloc(size_t size);
-static void	*base_calloc(size_t number, size_t size);
 static extent_node_t *base_node_alloc(void);
 static void	base_node_dealloc(extent_node_t *node);
 #ifdef MALLOC_STATS
@@ -1200,6 +1200,14 @@
  */
 /******************************************************************************/
 
+/*
+ * Functions missing prototypes which caused -Werror to fail. 
+ * Not sure if it has any side effects.
+ * */
+size_t malloc_usable_size(const void *ptr);
+void _malloc_thread_cleanup(void);
+
+
 static void
 wrtmessage(const char *p1, const char *p2, const char *p3, const char *p4)
 {
@@ -1625,17 +1633,6 @@
 	return (ret);
 }
 
-static void *
-base_calloc(size_t number, size_t size)
-{
-	void *ret;
-
-	ret = base_alloc(number * size);
-	memset(ret, 0, number * size);
-
-	return (ret);
-}
-
 static extent_node_t *
 base_node_alloc(void)
 {
@@ -1779,7 +1776,7 @@
 }
 
 /* Wrap red-black tree macros in functions. */
-rb_wrap(static, extent_tree_szad_, extent_tree_t, extent_node_t,
+rb_wrap(static ATTRIBUTE_UNUSED, extent_tree_szad_, extent_tree_t, extent_node_t,
     link_szad, extent_szad_comp)
 #endif
 
@@ -1793,7 +1790,7 @@
 }
 
 /* Wrap red-black tree macros in functions. */
-rb_wrap(static, extent_tree_ad_, extent_tree_t, extent_node_t, link_ad,
+rb_wrap(static ATTRIBUTE_UNUSED, extent_tree_ad_, extent_tree_t, extent_node_t, link_ad,
     extent_ad_comp)
 
 /*
@@ -2347,7 +2344,7 @@
 }
 
 /* Wrap red-black tree macros in functions. */
-rb_wrap(static, arena_chunk_tree_dirty_, arena_chunk_tree_t,
+rb_wrap(static ATTRIBUTE_UNUSED, arena_chunk_tree_dirty_, arena_chunk_tree_t,
     arena_chunk_t, link_dirty, arena_chunk_comp)
 
 static inline int
@@ -2363,7 +2360,7 @@
 }
 
 /* Wrap red-black tree macros in functions. */
-rb_wrap(static, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t,
+rb_wrap(static ATTRIBUTE_UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t,
     link, arena_run_comp)
 
 static inline int
@@ -2395,7 +2392,7 @@
 }
 
 /* Wrap red-black tree macros in functions. */
-rb_wrap(static, arena_avail_tree_, arena_avail_tree_t,
+rb_wrap(static ATTRIBUTE_UNUSED, arena_avail_tree_, arena_avail_tree_t,
     arena_chunk_map_t, link, arena_avail_comp)
 
 static inline void *



From phk at projects.linpro.no  Mon Nov 17 18:02:09 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 17 Nov 2008 19:02:09 +0100 (CET)
Subject: r3400 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081117180209.8B0E21EC5D4@projects.linpro.no>

Author: phk
Date: 2008-11-17 19:02:09 +0100 (Mon, 17 Nov 2008)
New Revision: 3400

Modified:
   trunk/varnish-cache/bin/varnishd/hash_simple_list.c
Log:
Rename variables to reflect that they are object heads, not objects.



Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_simple_list.c	2008-11-17 13:31:37 UTC (rev 3399)
+++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c	2008-11-17 18:02:09 UTC (rev 3400)
@@ -46,7 +46,7 @@
 
 struct hsl_entry {
 	VTAILQ_ENTRY(hsl_entry)	list;
-	struct objhead		*obj;
+	struct objhead		*oh;
 	unsigned		refcnt;
 };
 
@@ -73,34 +73,34 @@
  */
 
 static struct objhead *
-hsl_lookup(const struct sess *sp, struct objhead *nobj)
+hsl_lookup(const struct sess *sp, struct objhead *noh)
 {
 	struct hsl_entry *he, *he2;
 	int i;
 
 	Lck_Lock(&hsl_mtx);
 	VTAILQ_FOREACH(he, &hsl_head, list) {
-		i = HSH_Compare(sp, he->obj);
+		i = HSH_Compare(sp, he->oh);
 		if (i < 0)
 			continue;
 		if (i > 0)
 			break;
 		he->refcnt++;
-		nobj = he->obj;
+		noh = he->oh;
 		Lck_Unlock(&hsl_mtx);
-		return (nobj);
+		return (noh);
 	}
-	if (nobj != NULL) {
+	if (noh != NULL) {
 		he2 = calloc(sizeof *he2, 1);
 		XXXAN(he2);
-		he2->obj = nobj;
+		he2->oh = noh;
 		he2->refcnt = 1;
 
-		nobj->hashpriv = he2;
-		nobj->hash = malloc(sp->lhashptr);
-		XXXAN(nobj->hash);
-		nobj->hashlen = sp->lhashptr;
-		HSH_Copy(sp, nobj);
+		noh->hashpriv = he2;
+		noh->hash = malloc(sp->lhashptr);
+		XXXAN(noh->hash);
+		noh->hashlen = sp->lhashptr;
+		HSH_Copy(sp, noh);
 
 		if (he != NULL)
 			VTAILQ_INSERT_BEFORE(he, he2, list);
@@ -108,7 +108,7 @@
 			VTAILQ_INSERT_TAIL(&hsl_head, he2, list);
 	}
 	Lck_Unlock(&hsl_mtx);
-	return (nobj);
+	return (noh);
 }
 
 /*--------------------------------------------------------------------



From tfheen at projects.linpro.no  Tue Nov 18 13:29:34 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Tue, 18 Nov 2008 14:29:34 +0100 (CET)
Subject: r3401 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081118132934.981801EC11A@projects.linpro.no>

Author: tfheen
Date: 2008-11-18 14:29:34 +0100 (Tue, 18 Nov 2008)
New Revision: 3401

Modified:
   trunk/varnish-cache/bin/varnishd/storage_malloc.c
Log:
Make malloc print max storage size

storage_file prints the maximum storage size, make malloc do the same,
for consistency.


Modified: trunk/varnish-cache/bin/varnishd/storage_malloc.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/storage_malloc.c	2008-11-17 18:02:09 UTC (rev 3400)
+++ trunk/varnish-cache/bin/varnishd/storage_malloc.c	2008-11-18 13:29:34 UTC (rev 3401)
@@ -143,6 +143,9 @@
 		ARGV_ERR("(-smalloc) size \"%s\": %s\n", av[0], e);
 	if ((u != (uintmax_t)(size_t)u))
 		ARGV_ERR("(-smalloc) size \"%s\": too big\n", av[0]);
+
+	printf("storage_malloc: max size %ju MB.\n",
+	    u / (1024 * 1024));
 	sma_max = u;
 }
 



From tfheen at projects.linpro.no  Tue Nov 18 20:53:03 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Tue, 18 Nov 2008 21:53:03 +0100 (CET)
Subject: r3402 - trunk/varnish-cache/man
Message-ID: <20081118205303.439491EC0E8@projects.linpro.no>

Author: tfheen
Date: 2008-11-18 21:53:03 +0100 (Tue, 18 Nov 2008)
New Revision: 3402

Modified:
   trunk/varnish-cache/man/vcl.7so
Log:
Document grace

Thanks to perbu for suggested documentation
Fixes 355


Modified: trunk/varnish-cache/man/vcl.7so
===================================================================
--- trunk/varnish-cache/man/vcl.7so	2008-11-18 13:29:34 UTC (rev 3401)
+++ trunk/varnish-cache/man/vcl.7so	2008-11-18 20:53:03 UTC (rev 3402)
@@ -186,6 +186,24 @@
     pipe;
 }
 .Ed
+.Ss Grace 
+If the backend takes a long time to generate an object there is a risk
+of a thread pile up.
+In order to prevent this you can enable grace.
+This allows varnish to serve an expired version of the object while a
+fresh object is being generated by the backend.
+.Pp
+The following vcl code will make Varnish serve expired objects.
+All object will be kept up to two minutes past their expiration time
+or a fresh object is generated.
+.Bd -literal -offset 4n
+sub vcl_recv {
+    set req.grace = 2m;
+}
+sub vcl_fetch {
+    set obj.grace = 2m;
+}
+.Ed
 .Ss Functions
 The following built-in functions are available:
 .Bl -tag -width indent



From phk at projects.linpro.no  Wed Nov 19 10:10:17 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Wed, 19 Nov 2008 11:10:17 +0100 (CET)
Subject: r3403 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081119101017.082663BC007@projects.linpro.no>

Author: phk
Date: 2008-11-19 11:10:16 +0100 (Wed, 19 Nov 2008)
New Revision: 3403

Modified:
   trunk/varnish-cache/bin/varnishd/cache_hash.c
   trunk/varnish-cache/bin/varnishd/hash_simple_list.c
Log:
Rename some objhead pointers from obj to oh for consistency.



Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c	2008-11-18 20:53:03 UTC (rev 3402)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c	2008-11-19 10:10:16 UTC (rev 3403)
@@ -137,7 +137,7 @@
 }
 
 int
-HSH_Compare(const struct sess *sp, const struct objhead *obj)
+HSH_Compare(const struct sess *sp, const struct objhead *oh)
 {
 	int i;
 	unsigned u, v;
@@ -145,11 +145,11 @@
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(obj, OBJHEAD_MAGIC);
-	i = sp->lhashptr - obj->hashlen;
+	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+	i = sp->lhashptr - oh->hashlen;
 	if (i)
 		return (i);
-	b = obj->hash;
+	b = oh->hash;
 	for (u = 0; u < sp->ihashptr; u += 2) {
 		v = pdiff(sp->hashptr[u], sp->hashptr[u + 1]);
 		i = memcmp(sp->hashptr[u], b, v);
@@ -162,18 +162,21 @@
 	}
 	assert(*b == '\0');
 	b++;
-	assert(b == obj->hash + obj->hashlen);
+	assert(b == oh->hash + oh->hashlen);
 	return (0);
 }
 
 void
-HSH_Copy(const struct sess *sp, const struct objhead *obj)
+HSH_Copy(const struct sess *sp, const struct objhead *oh)
 {
 	unsigned u, v;
 	char *b;
 
-	assert(obj->hashlen >= sp->lhashptr);
-	b = obj->hash;
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+
+	assert(oh->hashlen >= sp->lhashptr);
+	b = oh->hash;
 	for (u = 0; u < sp->ihashptr; u += 2) {
 		v = pdiff(sp->hashptr[u], sp->hashptr[u + 1]);
 		memcpy(b, sp->hashptr[u], v);
@@ -181,7 +184,7 @@
 		*b++ = '#';
 	}
 	*b++ = '\0';
-	assert(b <= obj->hash + obj->hashlen);
+	assert(b <= oh->hash + oh->hashlen);
 }
 
 struct object *

Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_simple_list.c	2008-11-18 20:53:03 UTC (rev 3402)
+++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c	2008-11-19 10:10:16 UTC (rev 3403)
@@ -116,13 +116,13 @@
  */
 
 static int
-hsl_deref(const struct objhead *obj)
+hsl_deref(const struct objhead *oh)
 {
 	struct hsl_entry *he;
 	int ret;
 
-	AN(obj->hashpriv);
-	he = obj->hashpriv;
+	AN(oh->hashpriv);
+	he = oh->hashpriv;
 	Lck_Lock(&hsl_mtx);
 	if (--he->refcnt == 0) {
 		VTAILQ_REMOVE(&hsl_head, he, list);



From phk at projects.linpro.no  Wed Nov 19 10:21:30 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Wed, 19 Nov 2008 11:21:30 +0100 (CET)
Subject: r3404 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081119102130.BF39D1ED180@projects.linpro.no>

Author: phk
Date: 2008-11-19 11:21:30 +0100 (Wed, 19 Nov 2008)
New Revision: 3404

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/hash_classic.c
   trunk/varnish-cache/bin/varnishd/hash_simple_list.c
Log:
Move the hash'ers refcount up to objhead, it is generic.



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2008-11-19 10:10:16 UTC (rev 3403)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2008-11-19 10:21:30 UTC (rev 3404)
@@ -297,6 +297,7 @@
 	void			*hashpriv;
 
 	struct lock		mtx;
+	unsigned		refcnt;
 	VTAILQ_HEAD(,object)	objects;
 	char			*hash;
 	unsigned		hashlen;

Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_classic.c	2008-11-19 10:10:16 UTC (rev 3403)
+++ trunk/varnish-cache/bin/varnishd/hash_classic.c	2008-11-19 10:21:30 UTC (rev 3404)
@@ -49,7 +49,6 @@
 	VTAILQ_ENTRY(hcl_entry)	list;
 	struct hcl_hd		*head;
 	struct objhead		*oh;
-	unsigned		refcnt;
 	unsigned		digest;
 	unsigned		hash;
 };
@@ -167,7 +166,7 @@
 				continue;
 			if (i > 0)
 				break;
-			he->refcnt++;
+			he->oh->refcnt++;
 			roh = he->oh;
 			Lck_Unlock(&hp->mtx);
 			/*
@@ -191,7 +190,7 @@
 				VTAILQ_INSERT_BEFORE(he, he2, list);
 			else
 				VTAILQ_INSERT_TAIL(&hp->head, he2, list);
-			he2->refcnt++;
+			he2->oh->refcnt++;
 			noh = he2->oh;
 			Lck_Unlock(&hp->mtx);
 			return (noh);
@@ -231,11 +230,11 @@
 	CAST_OBJ_NOTNULL(he, oh->hashpriv, HCL_ENTRY_MAGIC);
 	hp = he->head;
 	CHECK_OBJ_NOTNULL(hp, HCL_HEAD_MAGIC);
-	assert(he->refcnt > 0);
+	assert(he->oh->refcnt > 0);
 	assert(he->hash < hcl_nhash);
 	assert(hp == &hcl_head[he->hash]);
 	Lck_Lock(&hp->mtx);
-	if (--he->refcnt == 0)
+	if (--he->oh->refcnt == 0)
 		VTAILQ_REMOVE(&hp->head, he, list);
 	else
 		he = NULL;

Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_simple_list.c	2008-11-19 10:10:16 UTC (rev 3403)
+++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c	2008-11-19 10:21:30 UTC (rev 3404)
@@ -47,7 +47,6 @@
 struct hsl_entry {
 	VTAILQ_ENTRY(hsl_entry)	list;
 	struct objhead		*oh;
-	unsigned		refcnt;
 };
 
 static VTAILQ_HEAD(, hsl_entry)	hsl_head = VTAILQ_HEAD_INITIALIZER(hsl_head);
@@ -85,7 +84,7 @@
 			continue;
 		if (i > 0)
 			break;
-		he->refcnt++;
+		he->oh->refcnt++;
 		noh = he->oh;
 		Lck_Unlock(&hsl_mtx);
 		return (noh);
@@ -94,7 +93,7 @@
 		he2 = calloc(sizeof *he2, 1);
 		XXXAN(he2);
 		he2->oh = noh;
-		he2->refcnt = 1;
+		he2->oh->refcnt = 1;
 
 		noh->hashpriv = he2;
 		noh->hash = malloc(sp->lhashptr);
@@ -124,7 +123,7 @@
 	AN(oh->hashpriv);
 	he = oh->hashpriv;
 	Lck_Lock(&hsl_mtx);
-	if (--he->refcnt == 0) {
+	if (--he->oh->refcnt == 0) {
 		VTAILQ_REMOVE(&hsl_head, he, list);
 		free(he);
 		ret = 0;



From phk at projects.linpro.no  Wed Nov 19 11:58:48 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Wed, 19 Nov 2008 12:58:48 +0100 (CET)
Subject: r3405 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081119115848.57AD11EC6D7@projects.linpro.no>

Author: phk
Date: 2008-11-19 12:58:48 +0100 (Wed, 19 Nov 2008)
New Revision: 3405

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_cli.c
   trunk/varnish-cache/bin/varnishd/cache_expire.c
   trunk/varnish-cache/bin/varnishd/cache_hash.c
   trunk/varnish-cache/bin/varnishd/cache_main.c
   trunk/varnish-cache/bin/varnishd/cache_pool.c
   trunk/varnish-cache/bin/varnishd/cache_vrt.c
   trunk/varnish-cache/bin/varnishd/hash_classic.c
   trunk/varnish-cache/bin/varnishd/hash_simple_list.c
   trunk/varnish-cache/bin/varnishd/hash_slinger.h
   trunk/varnish-cache/bin/varnishd/storage_synth.c
Log:
Kick the hash_slinger interface around a bit:

Isolate more stuff in hash_slinger.h.

Remove hash_slinger from cache.h, include in .c's as necessary.

Save a malloc per objhead by putting a few fields into the objhead
for the hash_slingers to use.

Preinitialize the refcount when we precreate the objhead.

Move the hash-string allocation into HSH_Copy(), no point in
duplication of mandatory step.




Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2008-11-19 10:21:30 UTC (rev 3404)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2008-11-19 11:58:48 UTC (rev 3405)
@@ -29,6 +29,12 @@
  * $Id$
  */
 
+/*
+ * This macro can be used in .h files to isolate bits that the manager
+ * should not (need to) see, such as pthread mutexes etc.
+ */
+#define VARNISH_CACHE_CHILD	1
+
 #include <sys/time.h>
 #include <sys/uio.h>
 #include <sys/socket.h>
@@ -213,8 +219,6 @@
 	void			*priv;
 };
 
-#include "hash_slinger.h"
-
 /* Backend Request ---------------------------------------------------*/
 
 struct bereq {
@@ -291,19 +295,6 @@
 	int			hits;
 };
 
-struct objhead {
-	unsigned		magic;
-#define OBJHEAD_MAGIC		0x1b96615d
-	void			*hashpriv;
-
-	struct lock		mtx;
-	unsigned		refcnt;
-	VTAILQ_HEAD(,object)	objects;
-	char			*hash;
-	unsigned		hashlen;
-	VTAILQ_HEAD(, sess)	waitinglist;
-};
-
 /* -------------------------------------------------------------------*/
 
 struct sess {
@@ -449,18 +440,6 @@
 int FetchReqBody(struct sess *sp);
 void Fetch_Init(void);
 
-/* cache_hash.c */
-void HSH_Prealloc(struct sess *sp);
-void HSH_Freestore(struct object *o);
-int HSH_Compare(const struct sess *sp, const struct objhead *o);
-void HSH_Copy(const struct sess *sp, const struct objhead *o);
-struct object *HSH_Lookup(struct sess *sp);
-void HSH_Unbusy(const struct sess *sp);
-void HSH_Ref(struct object *o);
-void HSH_Deref(struct object *o);
-double HSH_Grace(double g);
-void HSH_Init(void);
-
 /* cache_http.c */
 const char *http_StatusMessage(unsigned);
 void HTTP_Init(void);

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2008-11-19 10:21:30 UTC (rev 3404)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2008-11-19 11:58:48 UTC (rev 3405)
@@ -75,6 +75,7 @@
 #include "vcl.h"
 #include "cli_priv.h"
 #include "cache.h"
+#include "hash_slinger.h"
 
 static unsigned xids;
 

Modified: trunk/varnish-cache/bin/varnishd/cache_cli.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_cli.c	2008-11-19 10:21:30 UTC (rev 3404)
+++ trunk/varnish-cache/bin/varnishd/cache_cli.c	2008-11-19 11:58:48 UTC (rev 3405)
@@ -52,6 +52,7 @@
 #include "cache.h"
 #include "vlu.h"
 #include "vsb.h"
+#include "hash_slinger.h"
 
 pthread_t		cli_thread;
 static struct lock	cli_mtx;

Modified: trunk/varnish-cache/bin/varnishd/cache_expire.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_expire.c	2008-11-19 10:21:30 UTC (rev 3404)
+++ trunk/varnish-cache/bin/varnishd/cache_expire.c	2008-11-19 11:58:48 UTC (rev 3405)
@@ -60,6 +60,7 @@
 #include "shmlog.h"
 #include "binary_heap.h"
 #include "cache.h"
+#include "hash_slinger.h"
 
 /*
  * Objects have sideways references in the binary heap and the LRU list

Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c	2008-11-19 10:21:30 UTC (rev 3404)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c	2008-11-19 11:58:48 UTC (rev 3405)
@@ -65,6 +65,7 @@
 #include "shmlog.h"
 #include "cache.h"
 #include "stevedore.h"
+#include "hash_slinger.h"
 
 static const struct hash_slinger *hash;
 
@@ -91,12 +92,23 @@
 		w->nobjhead = calloc(sizeof *w->nobjhead, 1);
 		XXXAN(w->nobjhead);
 		w->nobjhead->magic = OBJHEAD_MAGIC;
+		w->nobjhead->refcnt = 1;
 		VTAILQ_INIT(&w->nobjhead->objects);
 		VTAILQ_INIT(&w->nobjhead->waitinglist);
 		Lck_New(&w->nobjhead->mtx);
 		VSL_stats->n_objecthead++;
 	} else
 		CHECK_OBJ_NOTNULL(w->nobjhead, OBJHEAD_MAGIC);
+
+#if 0
+	/* Make sure there is space enough for the hash-string */
+	if (w->nobjhead->hashlen < sp->lhashptr) {
+		w->objhead->hash = realloc(w->objhead->hash, sp->lhashptr);
+		w->objhead->hashlen = sp->lhashptr;
+		AN(w->objhead->hash);
+	}
+#endif
+
 	if (w->nobj == NULL) {
 		st = STV_alloc(sp, params->obj_workspace);
 		XXXAN(st);
@@ -167,7 +179,7 @@
 }
 
 void
-HSH_Copy(const struct sess *sp, const struct objhead *oh)
+HSH_Copy(const struct sess *sp, struct objhead *oh)
 {
 	unsigned u, v;
 	char *b;
@@ -175,7 +187,9 @@
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 
-	assert(oh->hashlen >= sp->lhashptr);
+	oh->hash = malloc(sp->lhashptr);
+	XXXAN(oh->hash);
+	oh->hashlen = sp->lhashptr;
 	b = oh->hash;
 	for (u = 0; u < sp->ihashptr; u += 2) {
 		v = pdiff(sp->hashptr[u], sp->hashptr[u + 1]);
@@ -210,6 +224,7 @@
 		CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 		Lck_Lock(&oh->mtx);
 	} else {
+		AN(w->nobjhead);
 		oh = hash->lookup(sp, w->nobjhead);
 		CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 		if (oh == w->nobjhead)

Modified: trunk/varnish-cache/bin/varnishd/cache_main.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_main.c	2008-11-19 10:21:30 UTC (rev 3404)
+++ trunk/varnish-cache/bin/varnishd/cache_main.c	2008-11-19 11:58:48 UTC (rev 3405)
@@ -39,6 +39,7 @@
 #include "shmlog.h"
 #include "cache.h"
 #include "stevedore.h"
+#include "hash_slinger.h"
 
 /*--------------------------------------------------------------------
  * Per thread storage for the session currently being processed by

Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-19 10:21:30 UTC (rev 3404)
+++ trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-19 11:58:48 UTC (rev 3405)
@@ -71,6 +71,7 @@
 #include "cli_priv.h"
 #include "cache.h"
 #include "stevedore.h"
+#include "hash_slinger.h"
 
 VTAILQ_HEAD(workerhead, worker);
 

Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt.c	2008-11-19 10:21:30 UTC (rev 3404)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt.c	2008-11-19 11:58:48 UTC (rev 3405)
@@ -50,6 +50,7 @@
 #include "vrt_obj.h"
 #include "vcl.h"
 #include "cache.h"
+#include "hash_slinger.h"
 #include "cache_backend.h"
 
 void *vrt_magic_string_end = &vrt_magic_string_end;

Modified: trunk/varnish-cache/bin/varnishd/hash_classic.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_classic.c	2008-11-19 10:21:30 UTC (rev 3404)
+++ trunk/varnish-cache/bin/varnishd/hash_classic.c	2008-11-19 11:58:48 UTC (rev 3405)
@@ -40,23 +40,14 @@
 
 #include "shmlog.h"
 #include "cache.h"
+#include "hash_slinger.h"
 
 /*--------------------------------------------------------------------*/
 
-struct hcl_entry {
-	unsigned		magic;
-#define HCL_ENTRY_MAGIC		0x0ba707bf
-	VTAILQ_ENTRY(hcl_entry)	list;
-	struct hcl_hd		*head;
-	struct objhead		*oh;
-	unsigned		digest;
-	unsigned		hash;
-};
-
 struct hcl_hd {
 	unsigned		magic;
 #define HCL_HEAD_MAGIC		0x0f327016
-	VTAILQ_HEAD(, hcl_entry)	head;
+	VTAILQ_HEAD(, objhead)	head;
 	struct lock		mtx;
 };
 
@@ -126,16 +117,13 @@
 static struct objhead *
 hcl_lookup(const struct sess *sp, struct objhead *noh)
 {
-	struct objhead *roh;
-	struct hcl_entry *he, *he2;
+	struct objhead *oh;
 	struct hcl_hd *hp;
-	unsigned u1, digest, r;
+	unsigned u1, digest;
 	unsigned u, v;
 	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC);
 	CHECK_OBJ_ORNULL(noh, OBJHEAD_MAGIC);
 
 	digest = ~0U;
@@ -147,73 +135,39 @@
 
 	u1 = digest % hcl_nhash;
 	hp = &hcl_head[u1];
-	he2 = NULL;
 
-	for (r = 0; r < 2; r++ ) {
-		Lck_Lock(&hp->mtx);
-		VTAILQ_FOREACH(he, &hp->head, list) {
-			CHECK_OBJ_NOTNULL(he, HCL_ENTRY_MAGIC);
-			if (sp->lhashptr < he->oh->hashlen)
-				continue;
-			if (sp->lhashptr > he->oh->hashlen)
-				break;
-			if (he->digest < digest)
-				continue;
-			if (he->digest > digest)
-				break;
-			i = HSH_Compare(sp, he->oh);
-			if (i < 0)
-				continue;
-			if (i > 0)
-				break;
-			he->oh->refcnt++;
-			roh = he->oh;
-			Lck_Unlock(&hp->mtx);
-			/*
-			 * If we loose the race, we need to clean up
-			 * the work we did for our second attempt.
-			 */
-			if (he2 != NULL)
-				free(he2);
-			if (noh != NULL && noh->hash != NULL) {
-				free(noh->hash);
-				noh->hash = NULL;
-			}
-			return (roh);
-		}
-		if (noh == NULL) {
-			Lck_Unlock(&hp->mtx);
-			return (NULL);
-		}
-		if (he2 != NULL) {
-			if (he != NULL)
-				VTAILQ_INSERT_BEFORE(he, he2, list);
-			else
-				VTAILQ_INSERT_TAIL(&hp->head, he2, list);
-			he2->oh->refcnt++;
-			noh = he2->oh;
-			Lck_Unlock(&hp->mtx);
-			return (noh);
-		}
+	Lck_Lock(&hp->mtx);
+	VTAILQ_FOREACH(oh, &hp->head, hoh_list) {
+		if (sp->lhashptr < oh->hashlen)
+			continue;
+		if (sp->lhashptr > oh->hashlen)
+			break;
+		if (oh->hoh_digest < digest)
+			continue;
+		if (oh->hoh_digest > digest)
+			break;
+		i = HSH_Compare(sp, oh);
+		if (i < 0)
+			continue;
+		if (i > 0)
+			break;
+		oh->refcnt++;
 		Lck_Unlock(&hp->mtx);
+		return (oh);
+	}
 
-		he2 = calloc(sizeof *he2, 1);
-		XXXAN(he2);
-		he2->magic = HCL_ENTRY_MAGIC;
-		he2->oh = noh;
-		he2->digest = digest;
-		he2->hash = u1;
-		he2->head = hp;
+	if (oh != NULL)
+		VTAILQ_INSERT_BEFORE(oh, noh, hoh_list);
+	else
+		VTAILQ_INSERT_TAIL(&hp->head, noh, hoh_list);
 
-		noh->hashpriv = he2;
-		AZ(noh->hash);
-		noh->hash = malloc(sp->lhashptr);
-		XXXAN(noh->hash);
-		noh->hashlen = sp->lhashptr;
-		HSH_Copy(sp, noh);
-	}
-	assert(he2 == NULL);		/* FlexeLint */
-	INCOMPL();
+	noh->hoh_digest = digest;
+	noh->hoh_head = hp;
+
+	HSH_Copy(sp, noh);
+
+	Lck_Unlock(&hp->mtx);
+	return (noh);
 }
 
 /*--------------------------------------------------------------------
@@ -221,28 +175,22 @@
  */
 
 static int
-hcl_deref(const struct objhead *oh)
+hcl_deref(struct objhead *oh)
 {
-	struct hcl_entry *he;
 	struct hcl_hd *hp;
+	int ret;
 
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	CAST_OBJ_NOTNULL(he, oh->hashpriv, HCL_ENTRY_MAGIC);
-	hp = he->head;
-	CHECK_OBJ_NOTNULL(hp, HCL_HEAD_MAGIC);
-	assert(he->oh->refcnt > 0);
-	assert(he->hash < hcl_nhash);
-	assert(hp == &hcl_head[he->hash]);
+	CAST_OBJ_NOTNULL(hp, oh->hoh_head, HCL_HEAD_MAGIC);
+	assert(oh->refcnt > 0);
 	Lck_Lock(&hp->mtx);
-	if (--he->oh->refcnt == 0)
-		VTAILQ_REMOVE(&hp->head, he, list);
-	else
-		he = NULL;
+	if (--oh->refcnt == 0) {
+		VTAILQ_REMOVE(&hp->head, oh, hoh_list);
+		ret = 0;
+	} else 
+		ret = 1;
 	Lck_Unlock(&hp->mtx);
-	if (he == NULL)
-		return (1);
-	free(he);
-	return (0);
+	return (ret);
 }
 
 /*--------------------------------------------------------------------*/

Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_simple_list.c	2008-11-19 10:21:30 UTC (rev 3404)
+++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c	2008-11-19 11:58:48 UTC (rev 3405)
@@ -33,23 +33,17 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
 #include "shmlog.h"
 #include "cache.h"
+#include "hash_slinger.h"
 
 /*--------------------------------------------------------------------*/
 
-struct hsl_entry {
-	VTAILQ_ENTRY(hsl_entry)	list;
-	struct objhead		*oh;
-};
-
-static VTAILQ_HEAD(, hsl_entry)	hsl_head = VTAILQ_HEAD_INITIALIZER(hsl_head);
+static VTAILQ_HEAD(, objhead)	hsl_head = VTAILQ_HEAD_INITIALIZER(hsl_head);
 static struct lock hsl_mtx;
 
 /*--------------------------------------------------------------------
@@ -74,38 +68,28 @@
 static struct objhead *
 hsl_lookup(const struct sess *sp, struct objhead *noh)
 {
-	struct hsl_entry *he, *he2;
+	struct objhead *oh;
 	int i;
 
 	Lck_Lock(&hsl_mtx);
-	VTAILQ_FOREACH(he, &hsl_head, list) {
-		i = HSH_Compare(sp, he->oh);
+	VTAILQ_FOREACH(oh, &hsl_head, hoh_list) {
+		i = HSH_Compare(sp, oh);
 		if (i < 0)
 			continue;
 		if (i > 0)
 			break;
-		he->oh->refcnt++;
-		noh = he->oh;
+		oh->refcnt++;
 		Lck_Unlock(&hsl_mtx);
-		return (noh);
+		return (oh);
 	}
-	if (noh != NULL) {
-		he2 = calloc(sizeof *he2, 1);
-		XXXAN(he2);
-		he2->oh = noh;
-		he2->oh->refcnt = 1;
 
-		noh->hashpriv = he2;
-		noh->hash = malloc(sp->lhashptr);
-		XXXAN(noh->hash);
-		noh->hashlen = sp->lhashptr;
-		HSH_Copy(sp, noh);
+	if (oh != NULL)
+		VTAILQ_INSERT_BEFORE(oh, noh, hoh_list);
+	else
+		VTAILQ_INSERT_TAIL(&hsl_head, noh, hoh_list);
 
-		if (he != NULL)
-			VTAILQ_INSERT_BEFORE(he, he2, list);
-		else
-			VTAILQ_INSERT_TAIL(&hsl_head, he2, list);
-	}
+	HSH_Copy(sp, noh);
+
 	Lck_Unlock(&hsl_mtx);
 	return (noh);
 }
@@ -115,17 +99,13 @@
  */
 
 static int
-hsl_deref(const struct objhead *oh)
+hsl_deref(struct objhead *oh)
 {
-	struct hsl_entry *he;
 	int ret;
 
-	AN(oh->hashpriv);
-	he = oh->hashpriv;
 	Lck_Lock(&hsl_mtx);
-	if (--he->oh->refcnt == 0) {
-		VTAILQ_REMOVE(&hsl_head, he, list);
-		free(he);
+	if (--oh->refcnt == 0) {
+		VTAILQ_REMOVE(&hsl_head, oh, hoh_list);
 		ret = 0;
 	} else
 		ret = 1;

Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_slinger.h	2008-11-19 10:21:30 UTC (rev 3404)
+++ trunk/varnish-cache/bin/varnishd/hash_slinger.h	2008-11-19 11:58:48 UTC (rev 3405)
@@ -30,12 +30,13 @@
  */
 
 struct sess;
+struct object;
 
 typedef void hash_init_f(int ac, char * const *av);
 typedef void hash_start_f(void);
 typedef struct objhead *
     hash_lookup_f(const struct sess *sp, struct objhead *nobj);
-typedef int hash_deref_f(const struct objhead *obj);
+typedef int hash_deref_f(struct objhead *obj);
 
 struct hash_slinger {
 	unsigned		magic;
@@ -46,3 +47,38 @@
 	hash_lookup_f		*lookup;
 	hash_deref_f		*deref;
 };
+
+/* cache_hash.c */
+void HSH_Prealloc(struct sess *sp);
+void HSH_Freestore(struct object *o);
+int HSH_Compare(const struct sess *sp, const struct objhead *o);
+void HSH_Copy(const struct sess *sp, struct objhead *o);
+struct object *HSH_Lookup(struct sess *sp);
+void HSH_Unbusy(const struct sess *sp);
+void HSH_Ref(struct object *o);
+void HSH_Deref(struct object *o);
+double HSH_Grace(double g);
+void HSH_Init(void);
+
+
+#ifdef VARNISH_CACHE_CHILD
+struct objhead {
+	unsigned		magic;
+#define OBJHEAD_MAGIC		0x1b96615d
+
+	struct lock		mtx;
+	unsigned		refcnt;
+	VTAILQ_HEAD(,object)	objects;
+	char			*hash;
+	unsigned		hashlen;
+	VTAILQ_HEAD(, sess)	waitinglist;
+
+	/*------------------------------------------------------------
+	 * The fields below are for the sole private use of the hash 
+	 * implementation.
+	 */
+	VTAILQ_ENTRY(objhead)	hoh_list;
+	void			*hoh_head;
+	unsigned		hoh_digest;
+};
+#endif /* VARNISH_CACHE_CHILD */

Modified: trunk/varnish-cache/bin/varnishd/storage_synth.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/storage_synth.c	2008-11-19 10:21:30 UTC (rev 3404)
+++ trunk/varnish-cache/bin/varnishd/storage_synth.c	2008-11-19 11:58:48 UTC (rev 3405)
@@ -42,6 +42,7 @@
 #include "cache.h"
 #include "vsb.h"
 #include "stevedore.h"
+#include "hash_slinger.h"
 
 static struct lock		sms_mtx;
 



From petter at projects.linpro.no  Wed Nov 19 14:13:58 2008
From: petter at projects.linpro.no (petter at projects.linpro.no)
Date: Wed, 19 Nov 2008 15:13:58 +0100 (CET)
Subject: r3406 - in trunk/varnish-cache: bin/varnishd bin/varnishtest/tests
	include lib/libvarnish lib/libvcl
Message-ID: <20081119141358.0CE5C1EC11A@projects.linpro.no>

Author: petter
Date: 2008-11-19 15:13:57 +0100 (Wed, 19 Nov 2008)
New Revision: 3406

Added:
   trunk/varnish-cache/bin/varnishtest/tests/b00020.vtc
   trunk/varnish-cache/bin/varnishtest/tests/b00021.vtc
   trunk/varnish-cache/bin/varnishtest/tests/b00022.vtc
   trunk/varnish-cache/bin/varnishtest/tests/b00023.vtc
   trunk/varnish-cache/bin/varnishtest/tests/b00024.vtc
   trunk/varnish-cache/bin/varnishtest/tests/b00025.vtc
   trunk/varnish-cache/bin/varnishtest/tests/b00026.vtc
Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_backend.c
   trunk/varnish-cache/bin/varnishd/cache_backend.h
   trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_fetch.c
   trunk/varnish-cache/bin/varnishd/cache_session.c
   trunk/varnish-cache/bin/varnishd/cache_vrt.c
   trunk/varnish-cache/bin/varnishd/heritage.h
   trunk/varnish-cache/bin/varnishd/mgt_param.c
   trunk/varnish-cache/include/libvarnish.h
   trunk/varnish-cache/include/vrt.h
   trunk/varnish-cache/include/vrt_obj.h
   trunk/varnish-cache/lib/libvarnish/tcp.c
   trunk/varnish-cache/lib/libvcl/vcc_backend.c
   trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c
   trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl
   trunk/varnish-cache/lib/libvcl/vcc_obj.c
Log:
Added support for setting read timeouts for backend requests (first_byte_timeout and between_bytes_timeout), in addition to make the connect_timeout available for the bereq object in vcl_miss and vcl_fetch.

first_byte_timeout is a read timeout from the connection to the backend is created to when the first byte arrives. It can be set as a parameter to varnish, as a field in the backend declaration or as bereq.first_byte_timeout in vcl_miss and vcl_pass.

between_bytes_timeout is a read timeout between each read from the backend. It can be set as a parameter to varnish, as a field in the backend declaration or as bereq.between_bytes_timeout in vcl_miss and vcl_pass.

The time unit for these timeout values are seconds. NOTE: The connect_timeout previously used milliseconds as time unit, so beware.


Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2008-11-19 14:13:57 UTC (rev 3406)
@@ -337,6 +337,11 @@
 	double			t_resp;
 	double			t_end;
 
+	/* Timeouts */
+	double connect_timeout;
+	double first_byte_timeout;
+	double between_bytes_timeout;
+
 	/* Acceptable grace period */
 	double			grace;
 
@@ -538,6 +543,8 @@
 void SES_Delete(struct sess *sp);
 void SES_RefSrcAddr(struct sess *sp);
 void SES_Charge(struct sess *sp);
+void SES_ResetBackendTimeouts(struct sess *sp);
+void SES_InheritBackendTimeouts(struct sess *sp);
 
 /* cache_shmlog.c */
 void VSL_Init(void);

Modified: trunk/varnish-cache/bin/varnishd/cache_backend.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_backend.c	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/bin/varnishd/cache_backend.c	2008-11-19 14:13:57 UTC (rev 3406)
@@ -95,7 +95,7 @@
 	if (s < 0)
 		return (s);
 
-	tmo = params->connect_timeout;
+	tmo = (int)(sp->connect_timeout * 1000);
 	if (bp->connect_timeout > 10e-3)
 		tmo = (int)(bp->connect_timeout * 1000);
 

Modified: trunk/varnish-cache/bin/varnishd/cache_backend.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_backend.h	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/bin/varnishd/cache_backend.h	2008-11-19 14:13:57 UTC (rev 3406)
@@ -104,6 +104,8 @@
 	char			*ident;
 	char			*vcl_name;
 	double			connect_timeout;
+	double			first_byte_timeout;
+	double			between_bytes_timeout;
 
 	uint32_t		hash;
 

Modified: trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/bin/varnishd/cache_backend_cfg.c	2008-11-19 14:13:57 UTC (rev 3406)
@@ -222,6 +222,8 @@
 	REPLACE(b->hosthdr, vb->hosthdr);
 
 	b->connect_timeout = vb->connect_timeout;
+	b->first_byte_timeout = vb->first_byte_timeout;
+	b->between_bytes_timeout = vb->between_bytes_timeout;
 	b->max_conn = vb->max_connections;
 
 	/*

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2008-11-19 14:13:57 UTC (rev 3406)
@@ -845,6 +845,8 @@
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 	AZ(sp->obj);
 
+	SES_ResetBackendTimeouts(sp);
+
 	/* By default we use the first backend */
 	AZ(sp->director);
 	sp->director = sp->vcl->director[0];

Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c	2008-11-19 14:13:57 UTC (rev 3406)
@@ -336,6 +336,8 @@
 	if (sp->vbe == NULL)
 		return (__LINE__);
 	vc = sp->vbe;
+	/* Inherit the backend timeouts from the selected backend */
+	SES_InheritBackendTimeouts(sp);
 
 	/*
 	 * Now that we know our backend, we can set a default Host:
@@ -369,8 +371,11 @@
 	VSL_stats->backend_req++;
 
 	HTC_Init(htc, bereq->ws, vc->fd);
-	do
+	TCP_set_read_timeout(vc->fd, sp->first_byte_timeout);
+	do {
 		i = HTC_Rx(htc);
+		TCP_set_read_timeout(vc->fd, sp->between_bytes_timeout);
+	}
 	while (i == 0);
 
 	if (i < 0) {

Modified: trunk/varnish-cache/bin/varnishd/cache_session.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_session.c	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/bin/varnishd/cache_session.c	2008-11-19 14:13:57 UTC (rev 3406)
@@ -58,6 +58,7 @@
 
 #include "shmlog.h"
 #include "cache.h"
+#include "cache_backend.h"
 
 /*--------------------------------------------------------------------*/
 
@@ -316,6 +317,8 @@
 	sp->http = &sm->http[0];
 	sp->http0 = &sm->http[1];
 
+	SES_ResetBackendTimeouts(sp);
+
 	return (sp);
 }
 
@@ -367,3 +370,38 @@
 	Lck_New(&stat_mtx);
 	Lck_New(&ses_mem_mtx);
 }
+
+void
+SES_ResetBackendTimeouts(struct sess *sp)
+{
+	sp->connect_timeout = params->connect_timeout;
+	sp->first_byte_timeout = params->first_byte_timeout;
+	sp->between_bytes_timeout = params->between_bytes_timeout;
+}
+
+void
+SES_InheritBackendTimeouts(struct sess *sp)
+{
+	struct backend *be = NULL;
+
+	AN(sp);
+	AN(sp->vbe);
+	AN(sp->vbe->backend);
+
+	be = sp->vbe->backend;
+	/* 
+	 * We only inherit the backend's timeout if the session timeout
+	 * has not already been set in the VCL, as the order of precedence
+	 * is parameter < backend definition < VCL.
+	 */
+	if (be->connect_timeout > 1e-3 && 
+			sp->connect_timeout == params->connect_timeout)
+		sp->connect_timeout = be->connect_timeout;
+	if (be->first_byte_timeout > 1e-3 && 
+			sp->first_byte_timeout == params->first_byte_timeout)
+		sp->first_byte_timeout = be->first_byte_timeout;
+	if (be->between_bytes_timeout > 1e-3 
+			&& sp->between_bytes_timeout == params->between_bytes_timeout)
+		sp->between_bytes_timeout = be->between_bytes_timeout;
+}
+

Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt.c	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt.c	2008-11-19 14:13:57 UTC (rev 3406)
@@ -299,6 +299,48 @@
 	return (atoi(sp->http->hd[HTTP_HDR_STATUS].b));
 }
 
+void
+VRT_l_bereq_connect_timeout(struct sess *sp, double num)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	sp->connect_timeout = (num > 0 ? num : 0);
+}
+
+double
+VRT_r_bereq_connect_timeout(struct sess *sp)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	return sp->connect_timeout;
+}
+
+void
+VRT_l_bereq_first_byte_timeout(struct sess *sp, double num)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	sp->first_byte_timeout = (num > 0 ? num : 0);
+}
+
+double
+VRT_r_bereq_first_byte_timeout(struct sess *sp)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	return sp->first_byte_timeout;
+}
+
+void
+VRT_l_bereq_between_bytes_timeout(struct sess *sp, double num)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	sp->between_bytes_timeout = (num > 0 ? num : 0);
+}
+
+double
+VRT_r_bereq_between_bytes_timeout(struct sess *sp)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	return sp->between_bytes_timeout;
+}
+
 /*--------------------------------------------------------------------*/
 
 void

Modified: trunk/varnish-cache/bin/varnishd/heritage.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/heritage.h	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/bin/varnishd/heritage.h	2008-11-19 14:13:57 UTC (rev 3406)
@@ -154,8 +154,12 @@
 	unsigned		cache_vbe_conns;
 
 	/* Default connection_timeout */
-	unsigned		connect_timeout;
+	double			connect_timeout;
 
+	/* Read timeouts for backend */
+	double			first_byte_timeout;
+	double	 		between_bytes_timeout;
+
 	/* How long to linger on sessions */
 	unsigned		session_linger;
 

Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-19 14:13:57 UTC (rev 3406)
@@ -97,6 +97,24 @@
 		cli_out(cli, "%u", *dst);
 }
 
+static void
+tweak_generic_timeout_double(struct cli *cli, volatile double *dst, const char *arg)
+{
+	double u;
+
+	if (arg != NULL) {
+		u = strtod(arg, NULL);
+		if (u < 0) {
+			cli_out(cli, "Timeout must be greater or equal to zero\n");
+			cli_result(cli, CLIS_PARAM);
+			return;
+		}
+		*dst = u;
+	} else
+		cli_out(cli, "%f", *dst);
+}
+
+
 /*--------------------------------------------------------------------*/
 
 static void
@@ -108,7 +126,14 @@
 	tweak_generic_timeout(cli, dest, arg);
 }
 
+static void
+tweak_timeout_double(struct cli *cli, const struct parspec *par, const char *arg)
+{
+	volatile double *dest;
 
+	dest = par->priv;
+	tweak_generic_timeout_double(cli, dest, arg);
+}
 /*--------------------------------------------------------------------*/
 
 static void
@@ -746,14 +771,31 @@
 		"Cache vbe_conn's or rely on malloc, that's the question.",
 		EXPERIMENTAL,
 		"off", "bool" },
-	{ "connect_timeout", tweak_uint,
+	{ "connect_timeout", tweak_timeout_double,
 		&master.connect_timeout,0, UINT_MAX,
 		"Default connection timeout for backend connections.  "
 		"We only try to connect to the backend for this many "
-		"milliseconds before giving up.  "
-		"VCL can override this default value for each backend.",
+		"seconds before giving up. "
+		"VCL can override this default value for each backend. "
+		"This does not apply to pipe. ",
 		0,
-		"400", "ms" },
+		"0.4", "s" },
+	{ "first_byte_timeout", tweak_timeout_double,
+		&master.first_byte_timeout,0, UINT_MAX,
+		"Default timeout for receiving first byte from backend. "
+		"We only wait for this many seconds for the first "
+		"byte before giving up. A value of 0 means it will never time out. "
+		"VCL can override this default value for each backend request.",
+		0,
+		"60", "s" },
+	{ "between_bytes_timeout", tweak_timeout_double,
+		&master.between_bytes_timeout,0, UINT_MAX,
+		"Default timeout between bytes when receiving data from backend. "
+		"We only wait for this many seconds between bytes "
+		"before giving up. A value of 0 means it will never time out. "
+		"VCL can override this default value for each backend request.",
+		0,
+		"60", "s" },
 	{ "accept_fd_holdoff", tweak_timeout,
 		&master.accept_fd_holdoff, 0,  3600*1000,
 		"If we run out of file descriptors, the accept thread will "

Added: trunk/varnish-cache/bin/varnishtest/tests/b00020.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/b00020.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/b00020.vtc	2008-11-19 14:13:57 UTC (rev 3406)
@@ -0,0 +1,34 @@
+# $Id: b00019.vtc 3300 2008-10-15 09:52:15Z tfheen $
+
+test "Check the between_bytes_timeout behaves from parameters"
+
+server s1 {
+	rxreq
+	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	delay 1.5
+	send "Baba\n"
+} -start
+
+varnish v1 -vcl+backend {} -start
+varnish v1 -cliok "param.set between_bytes_timeout 1"
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+} -run
+
+server s1 {
+	rxreq
+	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	delay 0.5
+	send "Baba\n"
+	delay 0.5
+	send "Baba\n"
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run

Added: trunk/varnish-cache/bin/varnishtest/tests/b00021.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/b00021.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/b00021.vtc	2008-11-19 14:13:57 UTC (rev 3406)
@@ -0,0 +1,37 @@
+# $Id: b00019.vtc 3300 2008-10-15 09:52:15Z tfheen $
+
+test "Check the between_bytes_timeout behaves from vcl"
+
+server s1 {
+	rxreq
+	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	delay 1.5
+	send "Baba\n"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_miss {
+		set bereq.between_bytes_timeout = 1s;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+} -run
+
+server s1 {
+	rxreq
+	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	delay 0.5
+	send "Baba\n"
+	delay 0.5
+	send "Baba\n"
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run

Added: trunk/varnish-cache/bin/varnishtest/tests/b00022.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/b00022.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/b00022.vtc	2008-11-19 14:13:57 UTC (rev 3406)
@@ -0,0 +1,39 @@
+# $Id: b00019.vtc 3300 2008-10-15 09:52:15Z tfheen $
+
+test "Check the between_bytes_timeout behaves from backend definition"
+
+server s1 {
+	rxreq
+	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	delay 1.5
+	send "Baba\n"
+} -start
+
+varnish v1 -vcl {
+	backend b1 {
+		.host = "127.0.0.1";
+		.port = "9080";
+		.between_bytes_timeout = 1s;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+} -run
+
+server s1 {
+	rxreq
+	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	delay 0.5
+	send "Baba\n"
+	delay 0.5
+	send "Baba\n"
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run

Added: trunk/varnish-cache/bin/varnishtest/tests/b00023.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/b00023.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/b00023.vtc	2008-11-19 14:13:57 UTC (rev 3406)
@@ -0,0 +1,31 @@
+# $Id: b00019.vtc 3300 2008-10-15 09:52:15Z tfheen $
+
+test "Check that the first_byte_timeout works from parameters"
+
+server s1 {
+	rxreq
+	delay 1.5
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+varnish v1 -cliok "param.set first_byte_timeout 1"
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+} -run
+
+
+server s1 {
+	rxreq
+	delay 0.5
+	txresp
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run

Added: trunk/varnish-cache/bin/varnishtest/tests/b00024.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/b00024.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/b00024.vtc	2008-11-19 14:13:57 UTC (rev 3406)
@@ -0,0 +1,34 @@
+# $Id: b00019.vtc 3300 2008-10-15 09:52:15Z tfheen $
+
+test "Check that the first_byte_timeout works from vcl"
+
+server s1 {
+	rxreq
+	delay 1.5
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_miss {
+		set bereq.first_byte_timeout = 1s;	
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+} -run
+
+
+server s1 {
+	rxreq
+	delay 0.5
+	txresp
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run

Added: trunk/varnish-cache/bin/varnishtest/tests/b00025.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/b00025.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/b00025.vtc	2008-11-19 14:13:57 UTC (rev 3406)
@@ -0,0 +1,36 @@
+# $Id: b00019.vtc 3300 2008-10-15 09:52:15Z tfheen $
+
+test "Check that the first_byte_timeout works from backend definition"
+
+server s1 {
+	rxreq
+	delay 1.5
+	txresp
+} -start
+
+varnish v1 -vcl {
+	backend b1 {
+		.host = "127.0.0.1";
+		.port = "9080";
+		.first_byte_timeout = 1s;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+} -run
+
+
+server s1 {
+	rxreq
+	delay 0.5
+	txresp
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run

Added: trunk/varnish-cache/bin/varnishtest/tests/b00026.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/b00026.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/b00026.vtc	2008-11-19 14:13:57 UTC (rev 3406)
@@ -0,0 +1,50 @@
+# $Id: b00019.vtc 3300 2008-10-15 09:52:15Z tfheen $
+
+test "Check the precedence for timeouts"
+
+server s1 -listen 127.0.0.1:9080 {
+	rxreq
+	expect req.url == "from_backend"
+	delay 1; 
+	txresp
+} -start
+server s2 -listen 127.0.0.1:9180 {
+	rxreq
+	expect req.url == "from_vcl"
+	delay 1.5;
+	txresp
+} -start
+
+varnish v1 -vcl {
+	backend b1 {
+		.host = "127.0.0.1";
+		.port = "9080";
+		.first_byte_timeout = 2s;
+	}
+	backend b2 {
+		.host = "127.0.0.1";
+		.port = "9180";
+		.first_byte_timeout = 1s;
+	}
+
+	sub vcl_recv {
+		if (req.url  == "from_backend") {
+			set req.backend = b1;
+			pass;
+		}
+		set req.backend = b2;
+	}
+	sub vcl_miss {
+		set bereq.first_byte_timeout = 2s;
+	}
+} -start
+varnish v1 -cliok "param.set first_byte_timeout 0.5"
+
+client c1 {
+	txreq -url "from_backend"
+	rxresp
+	expect resp.status == 200
+	txreq -url "from_vcl"
+	rxresp
+	expect resp.status == 200
+} -run

Modified: trunk/varnish-cache/include/libvarnish.h
===================================================================
--- trunk/varnish-cache/include/libvarnish.h	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/include/libvarnish.h	2008-11-19 14:13:57 UTC (rev 3406)
@@ -65,6 +65,7 @@
 int TCP_connect(int s, const struct sockaddr *name, socklen_t namelen,
     int msec);
 void TCP_close(int *s);
+void TCP_set_read_timeout(int socket, double seconds);
 #endif
 
 /* from libvarnish/time.c */

Modified: trunk/varnish-cache/include/vrt.h
===================================================================
--- trunk/varnish-cache/include/vrt.h	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/include/vrt.h	2008-11-19 14:13:57 UTC (rev 3406)
@@ -69,6 +69,8 @@
 	const unsigned char		*ipv6_sockaddr;
 
 	double				connect_timeout;
+	double				first_byte_timeout;
+	double				between_bytes_timeout;
 	unsigned			max_connections;
 	struct vrt_backend_probe	probe;
 };

Modified: trunk/varnish-cache/include/vrt_obj.h
===================================================================
--- trunk/varnish-cache/include/vrt_obj.h	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/include/vrt_obj.h	2008-11-19 14:13:57 UTC (rev 3406)
@@ -28,6 +28,12 @@
 void VRT_l_bereq_url(const struct sess *, const char *, ...);
 const char * VRT_r_bereq_proto(const struct sess *);
 void VRT_l_bereq_proto(const struct sess *, const char *, ...);
+double VRT_r_bereq_connect_timeout(struct sess *);
+void VRT_l_bereq_connect_timeout(struct sess *, double);
+double VRT_r_bereq_first_byte_timeout(struct sess *);
+void VRT_l_bereq_first_byte_timeout(struct sess *, double);
+double VRT_r_bereq_between_bytes_timeout(struct sess *);
+void VRT_l_bereq_between_bytes_timeout(struct sess *, double);
 const char * VRT_r_obj_proto(const struct sess *);
 void VRT_l_obj_proto(const struct sess *, const char *, ...);
 int VRT_r_obj_status(const struct sess *);

Modified: trunk/varnish-cache/lib/libvarnish/tcp.c
===================================================================
--- trunk/varnish-cache/lib/libvarnish/tcp.c	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/lib/libvarnish/tcp.c	2008-11-19 14:13:57 UTC (rev 3406)
@@ -47,6 +47,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include <math.h>
 
 #include "config.h"
 #ifndef HAVE_STRLCPY
@@ -210,3 +211,14 @@
 	    errno == ENOTCONN);
 	*s = -1;
 }
+
+void
+TCP_set_read_timeout(int s, double seconds)
+{
+	struct timeval timeout;
+	timeout.tv_sec = floor(seconds);
+	timeout.tv_usec = 1e6 * (seconds - timeout.tv_sec);
+#ifdef SO_RCVTIMEO_WORKS
+	AZ(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout));
+#endif
+}

Modified: trunk/varnish-cache/lib/libvcl/vcc_backend.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_backend.c	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/lib/libvcl/vcc_backend.c	2008-11-19 14:13:57 UTC (rev 3406)
@@ -473,6 +473,8 @@
 	    "?port",
 	    "?host_header",
 	    "?connect_timeout",
+	    "?first_byte_timeout",
+	    "?between_bytes_timeout",
 	    "?probe",
 	    "?max_connections",
 	    NULL);
@@ -537,6 +539,20 @@
 			Fb(tl, 0, ",\n");
 			ExpectErr(tl, ';');
 			vcc_NextToken(tl);
+		} else if (vcc_IdIs(t_field, "first_byte_timeout")) {
+			Fb(tl, 0, "\t.first_byte_timeout = ");
+			vcc_TimeVal(tl);
+			ERRCHK(tl);
+			Fb(tl, 0, ",\n");
+			ExpectErr(tl, ';');
+			vcc_NextToken(tl);
+		} else if (vcc_IdIs(t_field, "between_bytes_timeout")) {
+			Fb(tl, 0, "\t.between_bytes_timeout = ");
+			vcc_TimeVal(tl);
+			ERRCHK(tl);
+			Fb(tl, 0, ",\n");
+			ExpectErr(tl, ';');
+			vcc_NextToken(tl);
 		} else if (vcc_IdIs(t_field, "max_connections")) {
 			u = vcc_UintVal(tl);
 			vcc_NextToken(tl);

Modified: trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/lib/libvcl/vcc_fixed_token.c	2008-11-19 14:13:57 UTC (rev 3406)
@@ -243,6 +243,8 @@
 	vsb_cat(sb, "\tconst unsigned char\t\t*ipv4_sockaddr;\n");
 	vsb_cat(sb, "\tconst unsigned char\t\t*ipv6_sockaddr;\n");
 	vsb_cat(sb, "\n\tdouble\t\t\t\tconnect_timeout;\n");
+	vsb_cat(sb, "\tdouble\t\t\t\tfirst_byte_timeout;\n");
+	vsb_cat(sb, "\tdouble\t\t\t\tbetween_bytes_timeout;\n");
 	vsb_cat(sb, "\tunsigned\t\t\tmax_connections;\n");
 	vsb_cat(sb, "\tstruct vrt_backend_probe\tprobe;\n");
 	vsb_cat(sb, "};\n\n/*\n * A director with a predictable reply\n");
@@ -335,9 +337,16 @@
 	vsb_cat(sb, " const char *, ...);\nconst char * VRT_r_bereq_proto(c");
 	vsb_cat(sb, "onst struct sess *);\nvoid VRT_l_bereq_proto(const str");
 	vsb_cat(sb, "uct sess *, const char *, ...);\n");
-	vsb_cat(sb, "const char * VRT_r_obj_proto(const struct sess *);\n");
-	vsb_cat(sb, "void VRT_l_obj_proto(const struct sess *, const char *");
-	vsb_cat(sb, ", ...);\nint VRT_r_obj_status(const struct sess *);\n");
+	vsb_cat(sb, "double VRT_r_bereq_connect_timeout(struct sess *);\n");
+	vsb_cat(sb, "void VRT_l_bereq_connect_timeout(struct sess *, double");
+	vsb_cat(sb, ");\ndouble VRT_r_bereq_first_byte_timeout(struct sess ");
+	vsb_cat(sb, "*);\nvoid VRT_l_bereq_first_byte_timeout(struct sess *");
+	vsb_cat(sb, ", double);\ndouble VRT_r_bereq_between_bytes_timeout(s");
+	vsb_cat(sb, "truct sess *);\nvoid VRT_l_bereq_between_bytes_timeout");
+	vsb_cat(sb, "(struct sess *, double);\nconst char * VRT_r_obj_proto");
+	vsb_cat(sb, "(const struct sess *);\nvoid VRT_l_obj_proto(const str");
+	vsb_cat(sb, "uct sess *, const char *, ...);\n");
+	vsb_cat(sb, "int VRT_r_obj_status(const struct sess *);\n");
 	vsb_cat(sb, "void VRT_l_obj_status(const struct sess *, int);\n");
 	vsb_cat(sb, "const char * VRT_r_obj_response(const struct sess *);\n");
 	vsb_cat(sb, "void VRT_l_obj_response(const struct sess *, const cha");

Modified: trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/lib/libvcl/vcc_gen_obj.tcl	2008-11-19 14:13:57 UTC (rev 3406)
@@ -127,6 +127,21 @@
 	{     pipe pass      miss     fetch                        }
 	"const struct sess *"
     }
+    { bereq.connect_timeout
+	RW TIME
+	{     pass      miss     }
+	"struct sess *"
+    }
+    { bereq.first_byte_timeout
+	RW TIME
+	{     pass      miss     }
+	"struct sess *"
+    }
+    { bereq.between_bytes_timeout
+	RW TIME
+	{     pass      miss     }
+	"struct sess *"
+    }
 
     # The (possibly) cached object
     { obj.proto

Modified: trunk/varnish-cache/lib/libvcl/vcc_obj.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_obj.c	2008-11-19 11:58:48 UTC (rev 3405)
+++ trunk/varnish-cache/lib/libvcl/vcc_obj.c	2008-11-19 14:13:57 UTC (rev 3406)
@@ -108,6 +108,21 @@
 	    V_RW,	    "HDR_BEREQ",
 	    VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_MISS | VCL_MET_FETCH
 	},
+	{ "bereq.connect_timeout", TIME, 21,
+	    "VRT_r_bereq_connect_timeout(sp)",	    "VRT_l_bereq_connect_timeout(sp, ",
+	    V_RW,	    0,
+	    VCL_MET_PASS | VCL_MET_MISS
+	},
+	{ "bereq.first_byte_timeout", TIME, 24,
+	    "VRT_r_bereq_first_byte_timeout(sp)",	    "VRT_l_bereq_first_byte_timeout(sp, ",
+	    V_RW,	    0,
+	    VCL_MET_PASS | VCL_MET_MISS
+	},
+	{ "bereq.between_bytes_timeout", TIME, 27,
+	    "VRT_r_bereq_between_bytes_timeout(sp)",	    "VRT_l_bereq_between_bytes_timeout(sp, ",
+	    V_RW,	    0,
+	    VCL_MET_PASS | VCL_MET_MISS
+	},
 	{ "obj.proto", STRING, 9,
 	    "VRT_r_obj_proto(sp)",	    "VRT_l_obj_proto(sp, ",
 	    V_RW,	    0,



From tfheen at projects.linpro.no  Wed Nov 19 16:26:16 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Wed, 19 Nov 2008 17:26:16 +0100 (CET)
Subject: r3407 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081119162616.DA5971EC1FE@projects.linpro.no>

Author: tfheen
Date: 2008-11-19 17:26:16 +0100 (Wed, 19 Nov 2008)
New Revision: 3407

Modified:
   trunk/varnish-cache/bin/varnishd/varnishd.1
Log:
Correct defaults in varnishd.1

The defaults for thread_pool_min and thread_pools were wrong; fixed.


Modified: trunk/varnish-cache/bin/varnishd/varnishd.1
===================================================================
--- trunk/varnish-cache/bin/varnishd/varnishd.1	2008-11-19 14:13:57 UTC (rev 3406)
+++ trunk/varnish-cache/bin/varnishd/varnishd.1	2008-11-19 16:26:16 UTC (rev 3407)
@@ -520,7 +520,7 @@
 .Nm
 to respond faster to a sudden increase in traffic.
 .Pp
-The default is 1.
+The default is 5.
 .It Va thread_pools
 The number of worker thread pools.
 Higher values reduce lock contention but increase pressure on the
@@ -528,7 +528,7 @@
 Note that a decrease of this parameter will only take effect after a
 restart.
 .Pp
-The default is 1.
+The default is 2.
 .It Va thread_pool_timeout
 The amount of time a worker thread can be idle before it is killed,
 when the number of worker threads exceeds



From phk at projects.linpro.no  Thu Nov 20 08:50:57 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Thu, 20 Nov 2008 09:50:57 +0100 (CET)
Subject: r3408 - trunk/varnish-cache/bin/varnishtest
Message-ID: <20081120085057.0887D1EC6D7@projects.linpro.no>

Author: phk
Date: 2008-11-20 09:50:56 +0100 (Thu, 20 Nov 2008)
New Revision: 3408

Modified:
   trunk/varnish-cache/bin/varnishtest/vtc_server.c
Log:
Check ECONNRESET



Modified: trunk/varnish-cache/bin/varnishtest/vtc_server.c
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc_server.c	2008-11-19 16:26:16 UTC (rev 3407)
+++ trunk/varnish-cache/bin/varnishtest/vtc_server.c	2008-11-20 08:50:56 UTC (rev 3408)
@@ -98,7 +98,8 @@
 		vtc_log(vl, 3, "Accepted socket fd is %d", fd);
 		http_process(vl, s->spec, fd, 0);
 		vtc_log(vl, 3, "shutting fd %d", fd);
-		assert((shutdown(fd, SHUT_WR) == 0) || errno == ENOTCONN);
+		assert((shutdown(fd, SHUT_WR) == 0)
+		    || errno == ENOTCONN || errno == ECONNRESET);
 		TCP_close(&fd);
 	}
 	vtc_log(vl, 2, "Ending");



From phk at projects.linpro.no  Thu Nov 20 10:03:53 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Thu, 20 Nov 2008 11:03:53 +0100 (CET)
Subject: r3409 - trunk/varnish-cache/lib/libvcl
Message-ID: <20081120100353.5A7221EC6D7@projects.linpro.no>

Author: phk
Date: 2008-11-20 11:03:53 +0100 (Thu, 20 Nov 2008)
New Revision: 3409

Modified:
   trunk/varnish-cache/lib/libvcl/vcc_acl.c
Log:
Clean up the ACL generation code a bit.



Modified: trunk/varnish-cache/lib/libvcl/vcc_acl.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_acl.c	2008-11-20 08:50:56 UTC (rev 3408)
+++ trunk/varnish-cache/lib/libvcl/vcc_acl.c	2008-11-20 10:03:53 UTC (rev 3409)
@@ -60,59 +60,87 @@
 
 /* Compare two acl rules for ordering */
 
+#define CMP(a, b)							\
+	do {								\
+		if ((a) < (b))						\
+			return (-1);					\
+		else if ((b) < (a))					\
+			return (1);					\
+	} while (0)
+		
 static int
-vcl_acl_cmp(struct tokenlist *tl, struct acl_e *ae1, struct acl_e *ae2)
+vcl_acl_cmp(struct acl_e *ae1, struct acl_e *ae2)
 {
 	unsigned char *p1, *p2;
 	unsigned m;
 
-	(void)tl;
 	p1 = ae1->data;
 	p2 = ae2->data;
 	m = ae1->mask;
 	if (ae2->mask < m)
 		m = ae2->mask;
 	for (; m >= 8; m -= 8) {
-		if (*p1 < *p2)
-			return (-1);
-		if (*p1 > *p2)
-			return (1);
+		CMP(*p1, *p2);
 		p1++;
 		p2++;
 	}
 	if (m) {
 		m = 0xff00 >> m;
 		m &= 0xff;
-		if ((*p1 & m) < (*p2 & m))
-			return (-1);
-		if ((*p1 & m) > (*p2 & m))
-			return (1);
+		CMP(*p1 & m, *p2 & m);
 	}
-	if (ae1->mask > ae2->mask)
-		return (-1);
-	if (ae1->mask < ae2->mask)
-		return (1);
+	/* Long mask is less than short mask */
+	CMP(ae2->mask, ae1->mask);
 
 	return (0);
 }
 
 
 static void
-vcl_acl_add_entry(struct tokenlist *tl, struct acl_e *ae)
+vcc_acl_add_entry(struct tokenlist *tl, const struct acl_e *ae, int l,
+    const unsigned char *u, int fam)
 {
-	struct acl_e *ae2;
+	struct acl_e *ae2, *aen;
 	int i;
 
+	if (fam == PF_INET && ae->mask > 32) {
+		vsb_printf(tl->sb,
+		    "Too wide mask (%u) for IPv4 address", ae->mask);
+		vcc_ErrWhere(tl, ae->t_mask);
+		return;
+	}
+	if (fam == PF_INET6 && ae->mask > 128) {
+		vsb_printf(tl->sb,
+		    "Too wide mask (%u) for IPv6 address", ae->mask);
+		vcc_ErrWhere(tl, ae->t_mask);
+		return;
+	}
+
+	/* Make a copy from the template */
+	aen = TlAlloc(tl, sizeof *ae2);
+	AN(aen);
+	*aen = *ae;
+
+	/* We treat family as part of address, it saves code */
+	assert(fam <= 0xff);
+	aen->data[0] = fam & 0xff;
+	aen->mask += 8;
+
+	memcpy(aen->data + 1, u, l);
+
 	VTAILQ_FOREACH(ae2, &tl->acl, list) {
-		i = vcl_acl_cmp(tl, ae, ae2);
+		i = vcl_acl_cmp(aen, ae2);
 		if (i == 0) {
-			/* If the two rules agree, silently ignore it */
-			if (ae->not == ae2->not)
+			/*
+			 * If the two rules agree, silently ignore it
+			 * XXX: is that counter intuitive ?
+			 */
+			if (aen->not == ae2->not)
 				return;
 			vsb_printf(tl->sb, "Conflicting ACL entries:\n");
 			vcc_ErrWhere(tl, ae2->t_addr);
 			vsb_printf(tl->sb, "vs:\n");
-			vcc_ErrWhere(tl, ae->t_addr);
+			vcc_ErrWhere(tl, aen->t_addr);
 			return;
 		}
 		/*
@@ -126,46 +154,14 @@
 		 * be used to gather statistics.
 		 */
 		if (i < 0) {
-			VTAILQ_INSERT_BEFORE(ae2, ae, list);
+			VTAILQ_INSERT_BEFORE(ae2, aen, list);
 			return;
 		}
 	}
-	VTAILQ_INSERT_TAIL(&tl->acl, ae, list);
+	VTAILQ_INSERT_TAIL(&tl->acl, aen, list);
 }
 
 static void
-vcc_acl_emit_entry(struct tokenlist *tl, const struct acl_e *ae, int l,
-    const unsigned char *u, int fam)
-{
-	struct acl_e *ae2;
-
-	if (fam == PF_INET && ae->mask > 32) {
-		vsb_printf(tl->sb,
-		    "Too wide mask (%u) for IPv4 address", ae->mask);
-		vcc_ErrWhere(tl, ae->t_mask);
-		return;
-	}
-	if (fam == PF_INET6 && ae->mask > 128) {
-		vsb_printf(tl->sb,
-		    "Too wide mask (%u) for IPv6 address", ae->mask);
-		vcc_ErrWhere(tl, ae->t_mask);
-		return;
-	}
-
-	ae2 = TlAlloc(tl, sizeof *ae2);
-	AN(ae2);
-	*ae2 = *ae;
-
-	ae2->data[0] = fam & 0xff;
-	ae2->mask += 8;	/* family matching */
-
-	memcpy(ae2->data + 1, u, l);
-
-	vcl_acl_add_entry(tl, ae2);
-
-}
-
-static void
 vcc_acl_try_getaddrinfo(struct tokenlist *tl, struct acl_e *ae)
 {
 	struct addrinfo *res0, *res, hint;
@@ -211,7 +207,7 @@
 			if (ae->t_mask == NULL)
 				ae->mask = 32;
 			i4++;
-			vcc_acl_emit_entry(tl, ae, 4, u, res->ai_family);
+			vcc_acl_add_entry(tl, ae, 4, u, res->ai_family);
 			break;
 		case PF_INET6:
 			assert(PF_INET6 < 256);
@@ -221,7 +217,7 @@
 			if (ae->t_mask == NULL)
 				ae->mask = 128;
 			i6++;
-			vcc_acl_emit_entry(tl, ae, 16, u, res->ai_family);
+			vcc_acl_add_entry(tl, ae, 16, u, res->ai_family);
 			break;
 		default:
 			vsb_printf(tl->sb,
@@ -270,7 +266,7 @@
 	}
 	if (ae->t_mask == NULL)
 		ae->mask = 8 + 8 * i;
-	vcc_acl_emit_entry(tl, ae, 4, b, AF_INET);
+	vcc_acl_add_entry(tl, ae, 4, b, AF_INET);
 	return (1);
 }
 
@@ -321,9 +317,12 @@
 	ERRCHK(tl);
 }
 
+/*********************************************************************
+ * Emit a function to match the ACL we have collected
+ */
+
 static void
-vcc_acl_bot(const struct tokenlist *tl, const char *acln, int silent,
-    const char *pfx)
+vcc_acl_emit(const struct tokenlist *tl, const char *acln, int anon)
 {
 	struct acl_e *ae;
 	int depth, l, m, i;
@@ -333,7 +332,7 @@
 
 	Fh(tl, 0, "\nstatic int\n");
 	Fh(tl, 0, "match_acl_%s_%s(const struct sess *sp, const void *p)\n",
-	    pfx, acln);
+	    anon ? "anon" : "named", acln);
 	Fh(tl, 0, "{\n");
 	Fh(tl, 0, "\tconst unsigned char *a;\n");
 	assert(sizeof (unsigned char) == 1);
@@ -375,30 +374,31 @@
 		/* Back down, if necessary */
 		oc = "";
 		while (l <= depth) {
-			Fh(tl, 0, "\t%*s}\n",
-			    -depth, "");
+			Fh(tl, 0, "\t%*s}\n", -depth, "");
 			depth--;
 			oc = "else ";
 		}
+
 		m = ae->mask;
 		m -= l * 8;
+
+		/* Do whole byte compares */
 		for (i = l; m >= 8; m -= 8, i++) {
-			if (i == 0) {
+			if (i == 0)
 				Fh(tl, 0, "\t%*s%sif (fam == %d) {\n",
 				    -i, "", oc, ae->data[i]);
-			} else {
+			else
 				Fh(tl, 0, "\t%*s%sif (a[%d] == %d) {\n",
 				    -i, "", oc, i - 1, ae->data[i]);
-			}
 			at[i] = ae->data[i];
 			depth = i;
 			oc = "";
 		}
+
 		if (m > 0) {
+			/* Do fractional byte compares */
 			Fh(tl, 0, "\t%*s%sif ((a[%d] & 0x%x) == %d) {\n",
-			    -i, "",
-			    oc,
-			    i - 1, (0xff00 >> m) & 0xff,
+			    -i, "", oc, i - 1, (0xff00 >> m) & 0xff,
 			    ae->data[i] & ((0xff00 >> m) & 0xff));
 			at[i] = 256;
 			depth = i;
@@ -407,11 +407,9 @@
 
 		i = (ae->mask + 7) / 8;
 
-		if (!silent) {
+		if (!anon) {
 			Fh(tl, 0, "\t%*sVRT_acl_log(sp, \"%sMATCH %s \" ",
-			    -i, "",
-			    ae->not ? "NEG_" : "",
-			    acln,
+			    -i, "", ae->not ? "NEG_" : "", acln,
 			    PF(ae->t_addr));
 			EncToken(tl->fh, ae->t_addr);
 			if (ae->t_mask != NULL)
@@ -422,9 +420,12 @@
 		Fh(tl, 0, "\t%*sreturn (%d);\n", -i, "", ae->not ? 0 : 1);
 	}
 
+	/* Unwind */
 	for (; 0 <= depth; depth--)
 		Fh(tl, 0, "\t%*.*s}\n", depth, depth, "");
-	if (!silent)
+
+	/* Deny by default */
+	if (!anon)
 		Fh(tl, 0, "\tVRT_acl_log(sp, \"NO_MATCH %s\");\n", acln);
 	Fh(tl, 0, "\treturn (0);\n}\n");
 }
@@ -453,7 +454,7 @@
 		asprintf(&acln, "%u", tl->cnt);
 		assert(acln != NULL);
 		vcc_acl_entry(tl);
-		vcc_acl_bot(tl, acln, 1, "anon");
+		vcc_acl_emit(tl, acln, 1);
 		Fb(tl, 1, "%smatch_acl_anon_%s(sp, %s)\n",
 		    (tcond == T_NEQ ? "!" : ""), acln, vp->rname);
 		free(acln);
@@ -497,7 +498,7 @@
 	ExpectErr(tl, '}');
 	vcc_NextToken(tl);
 
-	vcc_acl_bot(tl, acln, 0, "named");
+	vcc_acl_emit(tl, acln, 0);
 
 	free(acln);
 }



From phk at projects.linpro.no  Thu Nov 20 10:19:56 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Thu, 20 Nov 2008 11:19:56 +0100 (CET)
Subject: r3410 - in trunk/varnish-cache/lib: libvarnish libvcl
Message-ID: <20081120101956.97B4E1EC0E8@projects.linpro.no>

Author: phk
Date: 2008-11-20 11:19:56 +0100 (Thu, 20 Nov 2008)
New Revision: 3410

Modified:
   trunk/varnish-cache/lib/libvarnish/vss.c
   trunk/varnish-cache/lib/libvcl/vcc_parse.c
   trunk/varnish-cache/lib/libvcl/vcc_xref.c
Log:
Various nits.



Modified: trunk/varnish-cache/lib/libvarnish/vss.c
===================================================================
--- trunk/varnish-cache/lib/libvarnish/vss.c	2008-11-20 10:03:53 UTC (rev 3409)
+++ trunk/varnish-cache/lib/libvarnish/vss.c	2008-11-20 10:19:56 UTC (rev 3410)
@@ -155,7 +155,7 @@
 	XXXAN(va);
 	*vap = va;
 	for (res = res0, i = 0; res != NULL; res = res->ai_next, ++i) {
-		va[i] = calloc(1, sizeof *va[i]);
+		va[i] = calloc(1, sizeof(*va[i]));
 		XXXAN(va[i]);
 		va[i]->va_family = res->ai_family;
 		va[i]->va_socktype = res->ai_socktype;

Modified: trunk/varnish-cache/lib/libvcl/vcc_parse.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_parse.c	2008-11-20 10:03:53 UTC (rev 3409)
+++ trunk/varnish-cache/lib/libvcl/vcc_parse.c	2008-11-20 10:19:56 UTC (rev 3410)
@@ -127,7 +127,7 @@
 static double
 RateUnit(struct tokenlist *tl)
 {
-	double sc = 1.0;
+	double sc;
 
 	assert(tl->t->tok == ID);
 	sc = SizeUnit(tl);

Modified: trunk/varnish-cache/lib/libvcl/vcc_xref.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_xref.c	2008-11-20 10:03:53 UTC (rev 3409)
+++ trunk/varnish-cache/lib/libvcl/vcc_xref.c	2008-11-20 10:19:56 UTC (rev 3410)
@@ -28,7 +28,7 @@
  *
  * $Id$
  *
- * This fine contains code for two cross-reference or consistency checks.
+ * This file contains code for two cross-reference or consistency checks.
  *
  * The first check is simply that all functions, acls and backends are
  * both defined and referenced.  Complaints about referenced but undefined
@@ -90,7 +90,7 @@
 		vcc_ErrToken(tl, r->name);
 		vsb_printf(tl->sb, " has unknown type %d\n",
 		    r->type);
-		return "???";
+		return "?";
 	}
 }
 



From petter at projects.linpro.no  Thu Nov 20 11:01:27 2008
From: petter at projects.linpro.no (petter at projects.linpro.no)
Date: Thu, 20 Nov 2008 12:01:27 +0100 (CET)
Subject: r3411 - in trunk/varnish-cache: bin/varnishd man
Message-ID: <20081120110127.0B57C1EC5D4@projects.linpro.no>

Author: petter
Date: 2008-11-20 12:01:26 +0100 (Thu, 20 Nov 2008)
New Revision: 3411

Modified:
   trunk/varnish-cache/bin/varnishd/mgt_param.c
   trunk/varnish-cache/bin/varnishd/varnishd.1
   trunk/varnish-cache/man/vcl.7so
Log:
Added documentaiton on the timeouts

Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-20 10:19:56 UTC (rev 3410)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-20 11:01:26 UTC (rev 3411)
@@ -773,11 +773,11 @@
 		"off", "bool" },
 	{ "connect_timeout", tweak_timeout_double,
 		&master.connect_timeout,0, UINT_MAX,
-		"Default connection timeout for backend connections.  "
+		"Default connection timeout for backend connections. "
 		"We only try to connect to the backend for this many "
 		"seconds before giving up. "
-		"VCL can override this default value for each backend. "
-		"This does not apply to pipe. ",
+		"VCL can override this default value for each backend and "
+		"backend request."
 		0,
 		"0.4", "s" },
 	{ "first_byte_timeout", tweak_timeout_double,
@@ -785,7 +785,8 @@
 		"Default timeout for receiving first byte from backend. "
 		"We only wait for this many seconds for the first "
 		"byte before giving up. A value of 0 means it will never time out. "
-		"VCL can override this default value for each backend request.",
+		"VCL can override this default value for each backend and "
+		"backend request. This parameter does not apply to pipe.",
 		0,
 		"60", "s" },
 	{ "between_bytes_timeout", tweak_timeout_double,
@@ -793,7 +794,8 @@
 		"Default timeout between bytes when receiving data from backend. "
 		"We only wait for this many seconds between bytes "
 		"before giving up. A value of 0 means it will never time out. "
-		"VCL can override this default value for each backend request.",
+		"VCL can override this default value for each backend request and "
+		"backend request. This parameter does not apply to pipe.",
 		0,
 		"60", "s" },
 	{ "accept_fd_holdoff", tweak_timeout,

Modified: trunk/varnish-cache/bin/varnishd/varnishd.1
===================================================================
--- trunk/varnish-cache/bin/varnishd/varnishd.1	2008-11-20 10:19:56 UTC (rev 3410)
+++ trunk/varnish-cache/bin/varnishd/varnishd.1	2008-11-20 11:01:26 UTC (rev 3411)
@@ -405,6 +405,15 @@
 .Pp
 The default is
 .Dv off .
+.It Va between_bytes_timeout
+Default timeout between bytes when receiving data from backend.
+We only wait for this many seconds between bytes before giving up.
+A value of 0 means it will never time out.
+VCL can override this default value for each backend and backend request.
+This parameter does not apply to pipe.
+.Pp
+The default is
+.Dv 60 seconds
 .It Va client_http11
 Whether to force the use of HTTP/1.1 when responding to client
 requests, or just use the same protocol version as that used by the
@@ -412,6 +421,13 @@
 .Pp
 The default is
 .Dv off .
+.It Va connect_timeout
+Default connection timeout for backend connections.
+We only try to connect to the backend for this many seconds before giving up.
+VCL can override this default value for each backend and backend request.
+.Pp
+The default is
+.Dv 0.4 seconds
 .It Va default_ttl
 The default time-to-live assigned to objects if neither the backend
 nor the configuration assign one.
@@ -427,6 +443,15 @@
 backend server does not specify a content length.
 .Pp
 The default is 128 kilobytes.
+.It Va first_byte_timeout
+Default timeout for receiving first byte from backend.
+We only wait for this many seconds for the first byte before giving up.
+A value of 0 means it will never time out.
+VCL can override this default value for each backend and backend request.
+This parameter does not apply to pipe.
+.Pp
+The default is
+.Dv 60 seconds
 .It Va group
 The name of an unprivileged group to which the child process should
 switch before it starts accepting connections.

Modified: trunk/varnish-cache/man/vcl.7so
===================================================================
--- trunk/varnish-cache/man/vcl.7so	2008-11-20 10:19:56 UTC (rev 3410)
+++ trunk/varnish-cache/man/vcl.7so	2008-11-20 11:01:26 UTC (rev 3411)
@@ -92,6 +92,26 @@
     set req.backend = www;
 }
 .Ed
+.Pp
+The timeout parameters can be overridden in the backend declaration.
+The timeout parameters are
+.Fa .connect_timeout
+for the time to wait for a backend connection,
+.Fa .first_byte_timeout
+for the time to wait for the first byte from the backend and
+.Fa .between_bytes_timeout
+for time to wait between each received byte.
+.Pp
+These can be set in the declaration like this:
+.Bd -literal -offset 4n
+backend www {
+    .host = "www.example.com";
+    .port = "http";
+    .connect_timeout = 1s;
+    .first_byte_timeout = 5s;
+    .between_bytes_timeout = 2s;
+}
+.Ed
 .Ss Directors
 Directors choose from different backends based on health status and a
 per-director algorithm.
@@ -516,6 +536,14 @@
 .It Va bereq.http. Ns Ar header
 The corresponding HTTP
 .Ar header .
+.It Va bereq.connect_timeout
+The time in seconds to wait for a backend connection.
+.It Va bereq.first_byte_timeout
+The time in seconds to wait for the first byte from the backend.
+Not available in pipe mode.
+.It Va bereq.between_bytes_timeout
+The time in seconds to wait between each received byte from the backend.
+Not available in pipe mode.
 .El
 .Pp
 The following variables are available after the requested object has



From petter at projects.linpro.no  Fri Nov 21 07:21:13 2008
From: petter at projects.linpro.no (petter at projects.linpro.no)
Date: Fri, 21 Nov 2008 08:21:13 +0100 (CET)
Subject: r3412 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081121072113.CA60D1EC200@projects.linpro.no>

Author: petter
Date: 2008-11-21 08:21:13 +0100 (Fri, 21 Nov 2008)
New Revision: 3412

Modified:
   trunk/varnish-cache/bin/varnishd/mgt_param.c
Log:
Soooorry. 


Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-20 11:01:26 UTC (rev 3411)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-21 07:21:13 UTC (rev 3412)
@@ -777,7 +777,7 @@
 		"We only try to connect to the backend for this many "
 		"seconds before giving up. "
 		"VCL can override this default value for each backend and "
-		"backend request."
+		"backend request.",
 		0,
 		"0.4", "s" },
 	{ "first_byte_timeout", tweak_timeout_double,



From phk at projects.linpro.no  Fri Nov 21 09:50:23 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Fri, 21 Nov 2008 10:50:23 +0100 (CET)
Subject: r3413 - in trunk/varnish-cache: bin/varnishd include lib/libvarnish
Message-ID: <20081121095023.036D41EC55D@projects.linpro.no>

Author: phk
Date: 2008-11-21 10:50:22 +0100 (Fri, 21 Nov 2008)
New Revision: 3413

Added:
   trunk/varnish-cache/lib/libvarnish/subproc.c
Modified:
   trunk/varnish-cache/bin/varnishd/mgt_vcc.c
   trunk/varnish-cache/include/libvarnish.h
   trunk/varnish-cache/lib/libvarnish/Makefile.am
Log:
Move the code for running stuff in a sub-process out to a library
function, and give it the ability to limit how many lines of
output we get back from the subprocess, in order to muzzle the
C-compiler somewhat.



Modified: trunk/varnish-cache/bin/varnishd/mgt_vcc.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_vcc.c	2008-11-21 07:21:13 UTC (rev 3412)
+++ trunk/varnish-cache/bin/varnishd/mgt_vcc.c	2008-11-21 09:50:22 UTC (rev 3413)
@@ -47,7 +47,6 @@
 #include "compat/asprintf.h"
 #endif
 #include "vsb.h"
-#include "vlu.h"
 
 #include "vqueue.h"
 
@@ -124,16 +123,13 @@
  * Errors goes in sb;
  */
 
-static int
-mgt_cc_vlu(void *priv, const char *str)
+static void
+run_cc(void *priv)
 {
-	struct vsb *vsb;
-
-	vsb = priv;
-	vsb_printf(vsb, "C-compiler said: %s\n", str);
-	return (0);
+	(void)execl("/bin/sh", "/bin/sh", "-c", priv, NULL);
 }
 
+
 static char *
 mgt_run_cc(const char *source, struct vsb *sb)
 {
@@ -142,10 +138,8 @@
 	char sf[] = "./vcl.########.c";
 	char of[sizeof sf + 1];
 	char *retval;
-	int rv, p[2], sfd, srclen, status;
-	pid_t pid;
+	int sfd, srclen;
 	void *dlh;
-	struct vlu *vlu;
 
 	/* Create temporary C source file */
 	sfd = vtmpfile(sf);
@@ -178,57 +172,8 @@
 	AZ(vsb_overflowed(&cmdsb));
 	/* XXX check vsb state */
 
-	if (pipe(p) < 0) {
-		vsb_printf(sb, "%s(): pipe() failed: %s",
-		    __func__, strerror(errno));
+	if (SUB_run(sb, run_cc, cmdline, "C-compiler", 10)) {
 		(void)unlink(sf);
-		return (NULL);
-	}
-	assert(p[0] > STDERR_FILENO);
-	assert(p[1] > STDERR_FILENO);
-	if ((pid = fork()) < 0) {
-		vsb_printf(sb, "%s(): fork() failed: %s",
-		    __func__, strerror(errno));
-		AZ(close(p[0]));
-		AZ(close(p[1]));
-		(void)unlink(sf);
-		return (NULL);
-	}
-	if (pid == 0) {
-		AZ(close(STDIN_FILENO));
-		assert(open("/dev/null", O_RDONLY) == STDIN_FILENO);
-		assert(dup2(p[1], STDOUT_FILENO) == STDOUT_FILENO);
-		assert(dup2(p[1], STDERR_FILENO) == STDERR_FILENO);
-		/* Close all other fds */
-		for (sfd = STDERR_FILENO + 1; sfd < 100; sfd++)
-			(void)close(sfd);
-		(void)execl("/bin/sh", "/bin/sh", "-c", cmdline, NULL);
-		_exit(1);
-	}
-	AZ(close(p[1]));
-	vlu = VLU_New(sb, mgt_cc_vlu, 0);
-	while (!VLU_Fd(p[0], vlu))
-		continue;
-	AZ(close(p[0]));
-	VLU_Destroy(vlu);
-	(void)unlink(sf);
-	do {
-		rv = waitpid(pid, &status, 0);
-		if (rv < 0 && errno != EINTR) {
-			vsb_printf(sb, "%s(): waitpid() failed: %s",
-			    __func__, strerror(errno));
-			(void)unlink(of);
-			return (NULL);
-		}
-	} while (rv < 0);
-	if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
-		vsb_printf(sb, "%s(): Compiler failed", __func__);
-		if (WIFEXITED(status))
-			vsb_printf(sb, ", exit %d", WEXITSTATUS(status));
-		if (WIFSIGNALED(status))
-			vsb_printf(sb, ", signal %d", WTERMSIG(status));
-		if (WCOREDUMP(status))
-			vsb_printf(sb, ", core dumped");
 		(void)unlink(of);
 		return (NULL);
 	}

Modified: trunk/varnish-cache/include/libvarnish.h
===================================================================
--- trunk/varnish-cache/include/libvarnish.h	2008-11-21 07:21:13 UTC (rev 3412)
+++ trunk/varnish-cache/include/libvarnish.h	2008-11-21 09:50:22 UTC (rev 3413)
@@ -37,6 +37,8 @@
 #define NULL ((void*)0)
 #endif
 
+struct vsb;
+
 /* from libvarnish/argv.c */
 void FreeArgv(char **argv);
 char **ParseArgv(const char *s, int flag);
@@ -50,6 +52,10 @@
 /* from libvarnish/num.c */
 const char *str2bytes(const char *p, uintmax_t *r, uintmax_t rel);
 
+/* from libvarnish/subproc.c */
+typedef void sub_func_f(void*);
+int SUB_run(struct vsb *sb, sub_func_f *func, void *priv, const char *name, int maxlines);
+
 /* from libvarnish/tcp.c */
 /* NI_MAXHOST and NI_MAXSERV are ridiculously long for numeric format */
 #define TCP_ADDRBUFSIZE		64

Modified: trunk/varnish-cache/lib/libvarnish/Makefile.am
===================================================================
--- trunk/varnish-cache/lib/libvarnish/Makefile.am	2008-11-21 07:21:13 UTC (rev 3412)
+++ trunk/varnish-cache/lib/libvarnish/Makefile.am	2008-11-21 09:50:22 UTC (rev 3413)
@@ -10,6 +10,7 @@
 	argv.c \
 	assert.c \
 	binary_heap.c \
+	subproc.c \
 	cli.c \
 	cli_common.c \
 	crc32.c \

Added: trunk/varnish-cache/lib/libvarnish/subproc.c
===================================================================
--- trunk/varnish-cache/lib/libvarnish/subproc.c	                        (rev 0)
+++ trunk/varnish-cache/lib/libvarnish/subproc.c	2008-11-21 09:50:22 UTC (rev 3413)
@@ -0,0 +1,133 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2008 Linpro AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $Id$
+ *
+ * Run stuff in a child process
+ */
+
+#include "config.h"
+
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+#include <stdio.h>
+
+#include <sys/wait.h>
+
+#include "vsb.h"
+#include "vlu.h"
+#include "libvarnish.h"
+
+struct sub_priv {
+	const char	*name;
+	struct vsb	*sb;
+	int		lines;
+	int		maxlines;
+};
+
+static int
+sub_vlu(void *priv, const char *str)
+{
+	struct sub_priv *sp;
+
+	sp = priv;
+	if (!sp->lines++)
+		vsb_printf(sp->sb, "Message from %s:\n", sp->name);
+	if (sp->maxlines > 0 && sp->lines <= sp->maxlines)
+		vsb_printf(sp->sb, "%s\n", str);
+	return (0);
+}
+
+int
+SUB_run(struct vsb *sb, sub_func_f *func, void *priv, const char *name, int maxlines)
+{
+	int rv, p[2], sfd, status;
+	pid_t pid;
+	struct vlu *vlu;
+	struct sub_priv sp;
+
+	sp.sb = sb;
+	sp.name = name;
+	sp.lines = 0;
+	sp.maxlines = maxlines;
+
+	if (pipe(p) < 0) {
+		vsb_printf(sb, "Starting %s: pipe() failed: %s",
+		    name, strerror(errno));
+		return (-1);
+	}
+	assert(p[0] > STDERR_FILENO);
+	assert(p[1] > STDERR_FILENO);
+	if ((pid = fork()) < 0) {
+		vsb_printf(sb, "Starting %s: fork() failed: %s",
+		    name, strerror(errno));
+		AZ(close(p[0]));
+		AZ(close(p[1]));
+		return (-1);
+	}
+	if (pid == 0) {
+		AZ(close(STDIN_FILENO));
+		assert(open("/dev/null", O_RDONLY) == STDIN_FILENO);
+		assert(dup2(p[1], STDOUT_FILENO) == STDOUT_FILENO);
+		assert(dup2(p[1], STDERR_FILENO) == STDERR_FILENO);
+		/* Close all other fds */
+		for (sfd = STDERR_FILENO + 1; sfd < 100; sfd++)
+			(void)close(sfd);
+		func(priv);
+		_exit(1);
+	}
+	AZ(close(p[1]));
+	vlu = VLU_New(&sp, sub_vlu, 0);
+	while (!VLU_Fd(p[0], vlu))
+		continue;
+	AZ(close(p[0]));
+	VLU_Destroy(vlu);
+	if (sp.lines > sp.maxlines)
+		vsb_printf(sb, "[%d lines truncated]\n",
+		    sp.lines - sp.maxlines);
+	do {
+		rv = waitpid(pid, &status, 0);
+		if (rv < 0 && errno != EINTR) {
+			vsb_printf(sb, "Running %s: waitpid() failed: %s",
+			    name, strerror(errno));
+			return (-1);
+		}
+	} while (rv < 0);
+	if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
+		vsb_printf(sb, "Running %s failed", name);
+		if (WIFEXITED(status))
+			vsb_printf(sb, ", exit %d", WEXITSTATUS(status));
+		if (WIFSIGNALED(status))
+			vsb_printf(sb, ", signal %d", WTERMSIG(status));
+		if (WCOREDUMP(status))
+			vsb_printf(sb, ", core dumped");
+		return (-1);
+	}
+	return (0);
+}



From phk at projects.linpro.no  Fri Nov 21 11:32:57 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Fri, 21 Nov 2008 12:32:57 +0100 (CET)
Subject: r3414 - in trunk/varnish-cache: include lib/libvarnish lib/libvcl
Message-ID: <20081121113257.16ACB1EC200@projects.linpro.no>

Author: phk
Date: 2008-11-21 12:32:56 +0100 (Fri, 21 Nov 2008)
New Revision: 3414

Modified:
   trunk/varnish-cache/include/libvarnish.h
   trunk/varnish-cache/lib/libvarnish/vtmpfile.c
   trunk/varnish-cache/lib/libvcl/vcc_compile.c
Log:
Add a vreadfile() utility function, which reads a file into malloc'ed
memory



Modified: trunk/varnish-cache/include/libvarnish.h
===================================================================
--- trunk/varnish-cache/include/libvarnish.h	2008-11-21 09:50:22 UTC (rev 3413)
+++ trunk/varnish-cache/include/libvarnish.h	2008-11-21 11:32:56 UTC (rev 3414)
@@ -86,6 +86,7 @@
 
 /* from libvarnish/vtmpfile.c */
 int vtmpfile(char *);
+char *vreadfile(int fd);
 
 /*
  * assert(), AN() and AZ() are static checks that should not happen.

Modified: trunk/varnish-cache/lib/libvarnish/vtmpfile.c
===================================================================
--- trunk/varnish-cache/lib/libvarnish/vtmpfile.c	2008-11-21 09:50:22 UTC (rev 3413)
+++ trunk/varnish-cache/lib/libvarnish/vtmpfile.c	2008-11-21 11:32:56 UTC (rev 3414)
@@ -35,7 +35,10 @@
 #include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <unistd.h>
 
+#include <sys/stat.h>
+
 #include "libvarnish.h"
 
 int
@@ -74,3 +77,21 @@
 	}
 	/* not reached */
 }
+
+char *
+vreadfile(int fd)
+{
+	struct stat st;
+	char *f;
+	int i;
+
+	assert(0 == fstat(fd, &st));
+	if (!S_ISREG(st.st_mode))
+		return (NULL);
+	f = malloc(st.st_size + 1);
+	assert(f != NULL);
+	i = read(fd, f, st.st_size);
+	assert(i == st.st_size);
+	f[i] = '\0';
+	return (f);
+}

Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_compile.c	2008-11-21 09:50:22 UTC (rev 3413)
+++ trunk/varnish-cache/lib/libvcl/vcc_compile.c	2008-11-21 11:32:56 UTC (rev 3414)
@@ -402,8 +402,6 @@
 vcc_file_source(struct vsb *sb, const char *fn, int fd)
 {
 	char *f;
-	int i;
-	struct stat st;
 	struct source *sp;
 
 	if (fd < 0) {
@@ -414,19 +412,10 @@
 			return (NULL);
 		}
 	}
-	assert(0 == fstat(fd, &st));
-	if (! S_ISREG(st.st_mode)) {
-		vsb_printf(sb, "File '%s' is not a regular file\n", fn);
-		AZ(close(fd));
-		return (NULL);
-	}
-	f = malloc(st.st_size + 1);
-	assert(f != NULL);
-	i = read(fd, f, st.st_size);
-	assert(i == st.st_size);
+	f = vreadfile(fd);
+	AN(f);
 	AZ(close(fd));
-	f[i] = '\0';
-	sp = vcc_new_source(f, f + i, fn);
+	sp = vcc_new_source(f, NULL, fn);
 	sp->freeit = f;
 	return (sp);
 }



From phk at projects.linpro.no  Fri Nov 21 12:09:47 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Fri, 21 Nov 2008 13:09:47 +0100 (CET)
Subject: r3415 - in trunk/varnish-cache: bin/varnishd include lib/libvarnish
	lib/libvcl
Message-ID: <20081121120947.0A3101EC200@projects.linpro.no>

Author: phk
Date: 2008-11-21 13:09:46 +0100 (Fri, 21 Nov 2008)
New Revision: 3415

Modified:
   trunk/varnish-cache/bin/varnishd/mgt.h
   trunk/varnish-cache/bin/varnishd/mgt_vcc.c
   trunk/varnish-cache/bin/varnishd/varnishd.c
   trunk/varnish-cache/include/libvarnish.h
   trunk/varnish-cache/include/libvcl.h
   trunk/varnish-cache/lib/libvarnish/vtmpfile.c
   trunk/varnish-cache/lib/libvcl/vcc_compile.c
Log:
Simplify how we manage the -f argument:

The VCL file specified to -f must be read relative to the directory
from which varnishd is started, before we chdir to the workdir.

We used to deal with this by opening the file and passing the file
handle down.  It's simpler to just read the file and pass the actual
VCL code down.



Modified: trunk/varnish-cache/bin/varnishd/mgt.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt.h	2008-11-21 11:32:56 UTC (rev 3414)
+++ trunk/varnish-cache/bin/varnishd/mgt.h	2008-11-21 12:09:46 UTC (rev 3415)
@@ -58,7 +58,7 @@
 
 /* mgt_vcc.c */
 void mgt_vcc_init(void);
-int mgt_vcc_default(const char *bflag, const char *fflag, int f_fd, int Cflag);
+int mgt_vcc_default(const char *bflag, char *vcl, int Cflag);
 int mgt_push_vcls_and_start(unsigned *status, char **p);
 int mgt_has_vcl(void);
 extern char *mgt_cc_cmd;

Modified: trunk/varnish-cache/bin/varnishd/mgt_vcc.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_vcc.c	2008-11-21 11:32:56 UTC (rev 3414)
+++ trunk/varnish-cache/bin/varnishd/mgt_vcc.c	2008-11-21 12:09:46 UTC (rev 3415)
@@ -201,40 +201,27 @@
 /*--------------------------------------------------------------------*/
 
 static char *
-mgt_VccCompile(struct vsb *sb, const char *b, const char *e, int C_flag)
+mgt_VccCompile(struct vsb **sb, const char *b, int C_flag)
 {
 	char *csrc, *vf = NULL;
 
-	csrc = VCC_Compile(sb, b, e);
-	if (csrc != NULL) {
-		if (C_flag)
-			(void)fputs(csrc, stdout);
-		vf = mgt_run_cc(csrc, sb);
-		if (C_flag && vf != NULL)
-			AZ(unlink(vf));
-		free(csrc);
-	}
-	return (vf);
-}
+	*sb = vsb_newauto();
+	XXXAN(*sb);
+	csrc = VCC_Compile(*sb, b, NULL);
 
-static char *
-mgt_VccCompileFile(struct vsb *sb, const char *fn, int C_flag, int fd)
-{
-	char *csrc, *vf = NULL;
-
-	csrc = VCC_CompileFile(sb, fn, fd);
 	if (csrc != NULL) {
 		if (C_flag)
 			(void)fputs(csrc, stdout);
-		vf = mgt_run_cc(csrc, sb);
+		vf = mgt_run_cc(csrc, *sb);
 		if (C_flag && vf != NULL)
 			AZ(unlink(vf));
 		free(csrc);
 	}
+	vsb_finish(*sb);
+	AZ(vsb_overflowed(*sb));
 	return (vf);
 }
 
-
 /*--------------------------------------------------------------------*/
 
 static struct vclprog *
@@ -290,16 +277,15 @@
 /*--------------------------------------------------------------------*/
 
 int
-mgt_vcc_default(const char *b_arg, const char *f_arg, int f_fd, int C_flag)
+mgt_vcc_default(const char *b_arg, char *vcl, int C_flag)
 {
 	char *addr, *port;
-	char *buf, *vf;
+	char *vf;
 	struct vsb *sb;
 	struct vclprog *vp;
 
-	sb = vsb_newauto();
-	XXXAN(sb);
 	if (b_arg != NULL) {
+		AZ(vcl);
 		/*
 		 * XXX: should do a "HEAD /" on the -b argument to see that
 		 * XXX: it even works.  On the other hand, we should do that
@@ -318,26 +304,21 @@
 			 */
 			free(port);
 			fprintf(stderr, "invalid backend address\n");
-			vsb_delete(sb);
 			return (1);
 		}
 
-		buf = NULL;
-		asprintf(&buf,
+		asprintf(&vcl,
 		    "backend default {\n"
 		    "    .host = \"%s\";\n"
 		    "    .port = \"%s\";\n"
 		    "}\n", addr, port ? port : "http");
 		free(addr);
 		free(port);
-		AN(buf);
-		vf = mgt_VccCompile(sb, buf, NULL, C_flag);
-		free(buf);
-	} else {
-		vf = mgt_VccCompileFile(sb, f_arg, C_flag, f_fd);
+		AN(vcl);
 	}
-	vsb_finish(sb);
-	AZ(vsb_overflowed(sb));
+
+	vf = mgt_VccCompile(&sb, vcl, C_flag);
+	free(vcl);
 	if (vsb_len(sb) > 0)
 		fprintf(stderr, "%s", vsb_data(sb));
 	vsb_delete(sb);
@@ -432,11 +413,7 @@
 		return;
 	}
 
-	sb = vsb_newauto();
-	XXXAN(sb);
-	vf = mgt_VccCompile(sb, av[3], NULL, 0);
-	vsb_finish(sb);
-	AZ(vsb_overflowed(sb));
+	vf = mgt_VccCompile(&sb, av[3], 0);
 	if (vsb_len(sb) > 0)
 		cli_out(cli, "%s", vsb_data(sb));
 	vsb_delete(sb);
@@ -459,7 +436,7 @@
 void
 mcf_config_load(struct cli *cli, const char * const *av, void *priv)
 {
-	char *vf;
+	char *vf, *vcl;
 	struct vsb *sb;
 	unsigned status;
 	char *p = NULL;
@@ -473,11 +450,16 @@
 		return;
 	}
 
-	sb = vsb_newauto();
-	XXXAN(sb);
-	vf = mgt_VccCompileFile(sb, av[3], 0, -1);
-	vsb_finish(sb);
-	AZ(vsb_overflowed(sb));
+	vcl = vreadfile(av[3]);
+	if (vcl == NULL) {
+		cli_out(cli, "Cannot open '%s'", av[3]);
+		cli_result(cli, CLIS_PARAM);
+		return;
+	}
+
+	vf = mgt_VccCompile(&sb, vcl, 0);
+	free(vcl);
+
 	if (vsb_len(sb) > 0)
 		cli_out(cli, "%s", vsb_data(sb));
 	vsb_delete(sb);

Modified: trunk/varnish-cache/bin/varnishd/varnishd.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/varnishd.c	2008-11-21 11:32:56 UTC (rev 3414)
+++ trunk/varnish-cache/bin/varnishd/varnishd.c	2008-11-21 12:09:46 UTC (rev 3415)
@@ -424,14 +424,13 @@
 	const char *l_arg = "80m";
 	uintmax_t l_size;
 	const char *q;
-	int f_fd = -1;
 	const char *h_arg = "classic";
 	const char *n_arg = NULL;
 	const char *P_arg = NULL;
 	const char *s_arg = "file";
 	int s_arg_given = 0;
 	const char *T_arg = NULL;
-	char *p;
+	char *p, *vcl = NULL;
 	struct cli cli[1];
 	struct pidfh *pfh = NULL;
 	char dirname[1024];
@@ -567,9 +566,9 @@
 	}
 
 	if (f_arg != NULL) {
-		f_fd = open(f_arg, O_RDONLY);
-		if (f_fd < 0) {
-			fprintf(stderr, "Cannot open '%s': %s\n",
+		vcl = vreadfile(f_arg);
+		if (vcl == NULL) {
+			fprintf(stderr, "Cannot read '%s': %s\n",
 			    f_arg, strerror(errno));
 			exit(1);
 		}
@@ -606,7 +605,7 @@
 	}
 
 	if (b_arg != NULL || f_arg != NULL)
-		if (mgt_vcc_default(b_arg, f_arg, f_fd, C_flag))
+		if (mgt_vcc_default(b_arg, vcl, C_flag))
 			exit (2);
 
 	if (C_flag)

Modified: trunk/varnish-cache/include/libvarnish.h
===================================================================
--- trunk/varnish-cache/include/libvarnish.h	2008-11-21 11:32:56 UTC (rev 3414)
+++ trunk/varnish-cache/include/libvarnish.h	2008-11-21 12:09:46 UTC (rev 3415)
@@ -86,7 +86,7 @@
 
 /* from libvarnish/vtmpfile.c */
 int vtmpfile(char *);
-char *vreadfile(int fd);
+char *vreadfile(const char *fn);
 
 /*
  * assert(), AN() and AZ() are static checks that should not happen.

Modified: trunk/varnish-cache/include/libvcl.h
===================================================================
--- trunk/varnish-cache/include/libvcl.h	2008-11-21 11:32:56 UTC (rev 3414)
+++ trunk/varnish-cache/include/libvcl.h	2008-11-21 12:09:46 UTC (rev 3415)
@@ -30,7 +30,6 @@
  */
 
 char *VCC_Compile(struct vsb *sb, const char *b, const char *e);
-char *VCC_CompileFile(struct vsb *sb, const char *fn, int fd);
 void VCC_InitCompile(const char *default_vcl);
 
 

Modified: trunk/varnish-cache/lib/libvarnish/vtmpfile.c
===================================================================
--- trunk/varnish-cache/lib/libvarnish/vtmpfile.c	2008-11-21 11:32:56 UTC (rev 3414)
+++ trunk/varnish-cache/lib/libvarnish/vtmpfile.c	2008-11-21 12:09:46 UTC (rev 3415)
@@ -78,8 +78,8 @@
 	/* not reached */
 }
 
-char *
-vreadfile(int fd)
+static char *
+vreadfd(int fd)
 {
 	struct stat st;
 	char *f;
@@ -95,3 +95,19 @@
 	f[i] = '\0';
 	return (f);
 }
+
+char *
+vreadfile(const char *fn)
+{
+	int fd, err;
+	char *r;
+
+	fd = open(fn, O_RDONLY);
+	if (fd < 0)
+		return (NULL);
+	r = vreadfd(fd);
+	err = errno;
+	AZ(close(fd));
+	errno = err;
+	return (r);
+}

Modified: trunk/varnish-cache/lib/libvcl/vcc_compile.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_compile.c	2008-11-21 11:32:56 UTC (rev 3414)
+++ trunk/varnish-cache/lib/libvcl/vcc_compile.c	2008-11-21 12:09:46 UTC (rev 3415)
@@ -399,22 +399,17 @@
 /*--------------------------------------------------------------------*/
 
 static struct source *
-vcc_file_source(struct vsb *sb, const char *fn, int fd)
+vcc_file_source(struct vsb *sb, const char *fn)
 {
 	char *f;
 	struct source *sp;
 
-	if (fd < 0) {
-		fd = open(fn, O_RDONLY);
-		if (fd < 0) {
-			vsb_printf(sb, "Cannot open file '%s': %s\n",
-			    fn, strerror(errno));
-			return (NULL);
-		}
+	f = vreadfile(fn);
+	if (f == NULL) {
+		vsb_printf(sb, "Cannot read file '%s': %s\n",
+		    fn, strerror(errno));
+		return (NULL);
 	}
-	f = vreadfile(fd);
-	AN(f);
-	AZ(close(fd));
 	sp = vcc_new_source(f, NULL, fn);
 	sp->freeit = f;
 	return (sp);
@@ -450,7 +445,7 @@
 		}
 		assert(t2 != NULL);
 
-		sp = vcc_file_source(tl->sb, t1->dec, -1);
+		sp = vcc_file_source(tl->sb, t1->dec);
 		if (sp == NULL) {
 			vcc_ErrWhere(tl, t1);
 			return;
@@ -668,24 +663,6 @@
 }
 
 /*--------------------------------------------------------------------
- * Compile the VCL code from the file named.  Error messages, if any
- * are formatted into the vsb.
- */
-
-char *
-VCC_CompileFile(struct vsb *sb, const char *fn, int fd)
-{
-	struct source *sp;
-	char *r;
-
-	sp = vcc_file_source(sb, fn, fd);
-	if (sp == NULL)
-		return (NULL);
-	r = vcc_CompileSource(sb, sp);
-	return (r);
-}
-
-/*--------------------------------------------------------------------
  * Initialize the compiler and register the default VCL code for later
  * compilation runs.
  */



From phk at projects.linpro.no  Fri Nov 21 12:53:34 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Fri, 21 Nov 2008 13:53:34 +0100 (CET)
Subject: r3416 - trunk/varnish-cache/lib/libvarnish
Message-ID: <20081121125334.817A51EC11A@projects.linpro.no>

Author: phk
Date: 2008-11-21 13:53:34 +0100 (Fri, 21 Nov 2008)
New Revision: 3416

Modified:
   trunk/varnish-cache/lib/libvarnish/subproc.c
Log:
Make it possible to supress all or no lines of output



Modified: trunk/varnish-cache/lib/libvarnish/subproc.c
===================================================================
--- trunk/varnish-cache/lib/libvarnish/subproc.c	2008-11-21 12:09:46 UTC (rev 3415)
+++ trunk/varnish-cache/lib/libvarnish/subproc.c	2008-11-21 12:53:34 UTC (rev 3416)
@@ -59,7 +59,7 @@
 	sp = priv;
 	if (!sp->lines++)
 		vsb_printf(sp->sb, "Message from %s:\n", sp->name);
-	if (sp->maxlines > 0 && sp->lines <= sp->maxlines)
+	if (sp->maxlines < 0 || sp->lines <= sp->maxlines)
 		vsb_printf(sp->sb, "%s\n", str);
 	return (0);
 }
@@ -108,7 +108,7 @@
 		continue;
 	AZ(close(p[0]));
 	VLU_Destroy(vlu);
-	if (sp.lines > sp.maxlines)
+	if (sp.maxlines >= 0 && sp.lines > sp.maxlines)
 		vsb_printf(sb, "[%d lines truncated]\n",
 		    sp.lines - sp.maxlines);
 	do {



From phk at projects.linpro.no  Fri Nov 21 13:00:35 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Fri, 21 Nov 2008 14:00:35 +0100 (CET)
Subject: r3417 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081121130035.CEAB71EC200@projects.linpro.no>

Author: phk
Date: 2008-11-21 14:00:35 +0100 (Fri, 21 Nov 2008)
New Revision: 3417

Modified:
   trunk/varnish-cache/bin/varnishd/mgt_vcc.c
Log:
Also run the VCL->C compiler stage in a sub-process.

This isolates the mangement process from the compilers bugs
and memory usage.


Modified: trunk/varnish-cache/bin/varnishd/mgt_vcc.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_vcc.c	2008-11-21 12:53:34 UTC (rev 3416)
+++ trunk/varnish-cache/bin/varnishd/mgt_vcc.c	2008-11-21 13:00:35 UTC (rev 3417)
@@ -84,12 +84,15 @@
 /*
  * Prepare the compiler command line
  */
-static void
-mgt_make_cc_cmd(struct vsb *sb, const char *sf, const char *of)
+static struct vsb *
+mgt_make_cc_cmd(const char *sf, const char *of)
 {
+	struct vsb *sb;
 	int pct;
 	char *p;
 
+	sb = vsb_newauto();
+	XXXAN(sb);
 	for (p = mgt_cc_cmd, pct = 0; *p; ++p) {
 		if (pct) {
 			switch (*p) {
@@ -116,11 +119,13 @@
 	}
 	if (pct)
 		vsb_putc(sb, '%');
+	vsb_finish(sb);
+	AZ(vsb_overflowed(sb));
+	return (sb);
 }
 
 /*--------------------------------------------------------------------
- * Invoke system C compiler on source and return resulting dlfile.
- * Errors goes in sb;
+ * Invoke system C compiler in a sub-process
  */
 
 static void
@@ -129,60 +134,114 @@
 	(void)execl("/bin/sh", "/bin/sh", "-c", priv, NULL);
 }
 
+/*--------------------------------------------------------------------
+ * Invoke system VCC compiler in a sub-process
+ */
 
+struct vcc_priv {
+	char		*sf;
+	const char	*vcl;
+};
+
+static void
+run_vcc(void *priv)
+{
+	char *csrc;
+	struct vsb *sb;
+	struct vcc_priv *vp;
+	int fd, i, l;
+
+	vp = priv;
+	sb = vsb_newauto();
+	XXXAN(sb);
+	csrc = VCC_Compile(sb, vp->vcl, NULL);
+	vsb_finish(sb);
+	AZ(vsb_overflowed(sb));
+	if (vsb_len(sb))
+		printf("%s", vsb_data(sb));
+	vsb_delete(sb);
+	if (csrc == NULL)
+		exit (1);
+
+	fd = open(vp->sf, O_WRONLY);
+	if (fd < 0) {
+		fprintf(stderr, "Cannot open %s", vp->sf);
+		exit (1);
+	}
+	l = strlen(csrc);
+	i = write(fd, csrc, l);
+	if (i != l) {
+		fprintf(stderr, "Cannot write %s", vp->sf);
+		exit (1);
+	}
+	close(fd);
+	free(csrc);
+	exit (0);
+}
+
+/*--------------------------------------------------------------------
+ * Compile a VCL program, return shared object, errors in sb.
+ */
+
 static char *
-mgt_run_cc(const char *source, struct vsb *sb)
+mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag)
 {
-	char cmdline[1024];
-	struct vsb cmdsb;
+	char *csrc;
+	struct vsb *cmdsb;
 	char sf[] = "./vcl.########.c";
 	char of[sizeof sf + 1];
 	char *retval;
-	int sfd, srclen;
+	int sfd, i;
 	void *dlh;
+	struct vcc_priv vp;
 
 	/* Create temporary C source file */
 	sfd = vtmpfile(sf);
 	if (sfd < 0) {
-		vsb_printf(sb,
-		    "%s(): failed to create %s: %s",
-		    __func__, sf, strerror(errno));
+		vsb_printf(sb, "Failed to create %s: %s", sf, strerror(errno));
 		return (NULL);
 	}
-	srclen = strlen(source);
-	if (write(sfd, source, srclen) != srclen) {
-		vsb_printf(sb,
-		    "Failed to write C source to file: %s",
-		    strerror(errno));
-		AZ(unlink(sf));
-		AZ(close(sfd));
+	AZ(close(sfd));
+
+	/* Run the VCC compiler in a sub-process */
+	vp.sf = sf;
+	vp.vcl = vcl;
+	if (SUB_run(sb, run_vcc, &vp, "VCC-compiler", -1)) {
+		(void)unlink(sf);
 		return (NULL);
 	}
-	AZ(close(sfd));
 
-	/* Name the output shared library by overwriting the final 'c' */
+	if (C_flag) {
+		csrc = vreadfile(sf);
+		(void)fputs(csrc, stdout);
+		free(csrc);
+	}
+
+	/* Name the output shared library by "s/[.]c$/[.]so/" */
 	memcpy(of, sf, sizeof sf);
 	assert(sf[sizeof sf - 2] == 'c');
 	of[sizeof sf - 2] = 's';
 	of[sizeof sf - 1] = 'o';
 	of[sizeof sf] = '\0';
-	AN(vsb_new(&cmdsb, cmdline, sizeof cmdline, 0));
-	mgt_make_cc_cmd(&cmdsb, sf, of);
-	vsb_finish(&cmdsb);
-	AZ(vsb_overflowed(&cmdsb));
-	/* XXX check vsb state */
 
-	if (SUB_run(sb, run_cc, cmdline, "C-compiler", 10)) {
-		(void)unlink(sf);
+	/* Build the C-compiler command line */
+	cmdsb = mgt_make_cc_cmd(sf, of);
+
+	/* Run the C-compiler in a sub-shell */
+	i = SUB_run(sb, run_cc, vsb_data(cmdsb), "C-compiler", 10);
+
+	(void)unlink(sf);
+	vsb_delete(cmdsb);
+
+	if (i) {
 		(void)unlink(of);
 		return (NULL);
 	}
 
-	/* Next, try to load the object into the management process */
+	/* Try to load the object into the management process */
 	if ((dlh = dlopen(of, RTLD_NOW | RTLD_LOCAL)) == NULL) {
 		vsb_printf(sb,
-		    "%s(): failed to load compiled VCL program:\n  %s",
-		    __func__, dlerror());
+		    "Compiled VCL program failed to load:\n  %s", dlerror());
 		(void)unlink(of);
 		return (NULL);
 	}
@@ -203,20 +262,11 @@
 static char *
 mgt_VccCompile(struct vsb **sb, const char *b, int C_flag)
 {
-	char *csrc, *vf = NULL;
+	char *vf = NULL;
 
 	*sb = vsb_newauto();
 	XXXAN(*sb);
-	csrc = VCC_Compile(*sb, b, NULL);
-
-	if (csrc != NULL) {
-		if (C_flag)
-			(void)fputs(csrc, stdout);
-		vf = mgt_run_cc(csrc, *sb);
-		if (C_flag && vf != NULL)
-			AZ(unlink(vf));
-		free(csrc);
-	}
+	vf = mgt_run_cc(b, *sb, C_flag);
 	vsb_finish(*sb);
 	AZ(vsb_overflowed(*sb));
 	return (vf);
@@ -322,8 +372,11 @@
 	if (vsb_len(sb) > 0)
 		fprintf(stderr, "%s", vsb_data(sb));
 	vsb_delete(sb);
-	if (C_flag)
+	if (C_flag) {
+		if (vf != NULL) 
+			AZ(unlink(vf));
 		return (0);
+	}
 	if (vf == NULL) {
 		fprintf(stderr, "\nVCL compilation failed\n");
 		return (1);



From tfheen at projects.linpro.no  Sat Nov 22 01:35:16 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Sat, 22 Nov 2008 02:35:16 +0100 (CET)
Subject: r3418 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081122013516.669F31EC7CD@projects.linpro.no>

Author: tfheen
Date: 2008-11-22 02:35:16 +0100 (Sat, 22 Nov 2008)
New Revision: 3418

Modified:
   trunk/varnish-cache/bin/varnishd/cache_backend_poll.h
Log:
Fix typo


Modified: trunk/varnish-cache/bin/varnishd/cache_backend_poll.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_backend_poll.h	2008-11-21 13:00:35 UTC (rev 3417)
+++ trunk/varnish-cache/bin/varnishd/cache_backend_poll.h	2008-11-22 01:35:16 UTC (rev 3418)
@@ -30,7 +30,7 @@
  */
 
 BITMAP(good_ipv4, '4', "Good IPv4", 0)
-BITMAP(good_ipv6, '6', "Good IPv4", 0)
+BITMAP(good_ipv6, '6', "Good IPv6", 0)
 BITMAP( err_xmit, 'x', "Error Xmit", 0)
 BITMAP(good_xmit, 'X', "Good Xmit", 0)
 BITMAP( err_shut, 's', "Error Shut", 0)



From sky at projects.linpro.no  Mon Nov 24 09:25:56 2008
From: sky at projects.linpro.no (sky at projects.linpro.no)
Date: Mon, 24 Nov 2008 10:25:56 +0100 (CET)
Subject: r3419 - in branches/nuke/varnish-cache: . bin/varnishd
	bin/varnishncsa bin/varnishreplay bin/varnishtest doc include
	lib/libvcl redhat
Message-ID: <20081124092556.B07541EC7C5@projects.linpro.no>

Author: sky
Date: 2008-11-24 10:25:56 +0100 (Mon, 24 Nov 2008)
New Revision: 3419

Modified:
   branches/nuke/varnish-cache/bin/varnishd/cache_ban.c
   branches/nuke/varnish-cache/bin/varnishd/cache_dir_random.c
   branches/nuke/varnish-cache/bin/varnishd/cache_hash.c
   branches/nuke/varnish-cache/bin/varnishd/cache_vrt.c
   branches/nuke/varnish-cache/bin/varnishd/cache_vrt_esi.c
   branches/nuke/varnish-cache/bin/varnishd/heritage.h
   branches/nuke/varnish-cache/bin/varnishd/mgt_param.c
   branches/nuke/varnish-cache/bin/varnishncsa/varnishncsa.c
   branches/nuke/varnish-cache/bin/varnishreplay/varnishreplay.c
   branches/nuke/varnish-cache/bin/varnishtest/Makefile.am
   branches/nuke/varnish-cache/configure.ac
   branches/nuke/varnish-cache/doc/changes-2.0-2.0.1.xml
   branches/nuke/varnish-cache/include/stat_field.h
   branches/nuke/varnish-cache/lib/libvcl/vcc_compile.c
   branches/nuke/varnish-cache/redhat/varnish.spec
Log:
rebase on release 2.0.2 

Modified: branches/nuke/varnish-cache/bin/varnishd/cache_ban.c
===================================================================
--- branches/nuke/varnish-cache/bin/varnishd/cache_ban.c	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/bin/varnishd/cache_ban.c	2008-11-24 09:25:56 UTC (rev 3419)
@@ -49,6 +49,8 @@
 #define BAN_MAGIC		0x700b08ea
 	VTAILQ_ENTRY(ban)	list;
 	unsigned		refcount;
+	int			flags;
+#define BAN_F_GONE		(1 << 0)
 	regex_t			regexp;
 	char			*ban;
 	int			hash;
@@ -68,8 +70,9 @@
 int
 BAN_Add(struct cli *cli, const char *regexp, int hash)
 {
-	struct ban *b;
+	struct ban *b, *bi, *be;
 	char buf[512];
+	unsigned pcount;
 	int i;
 
 	ALLOC_OBJ(b, BAN_MAGIC);
@@ -97,8 +100,37 @@
 	ban_start = b;
 	VSL_stats->n_purge++;
 	VSL_stats->n_purge_add++;
+
+	if (params->purge_dups) {
+		be = VTAILQ_LAST(&ban_head, banhead);
+		be->refcount++;
+	} else
+		be = NULL;
 	UNLOCK(&ban_mtx);
 
+	if (be == NULL)
+		return (0);
+
+	/* Hunt down duplicates, and mark them as gone */
+	bi = b;
+	pcount = 0;
+	while(bi != be) {
+		bi = VTAILQ_NEXT(bi, list);
+		if (bi->flags & BAN_F_GONE)
+			continue;
+		if (b->hash != bi->hash)
+			continue;
+		if (strcmp(b->ban, bi->ban))
+			continue;
+		bi->flags |= BAN_F_GONE;
+		pcount++;
+	}
+	LOCK(&ban_mtx);
+	be->refcount--;
+	/* XXX: We should check if the tail can be removed */
+	VSL_stats->n_purge_dups += pcount;
+	UNLOCK(&ban_mtx);
+
 	return (0);
 }
 
@@ -168,7 +200,8 @@
 	tests = 0;
 	for (b = b0; b != o->ban; b = VTAILQ_NEXT(b, list)) {
 		tests++;
-		if (!regexec(&b->regexp, b->hash ? hash : url, 0, NULL, 0))
+		if (!(b->flags & BAN_F_GONE) &&
+		    !regexec(&b->regexp, b->hash ? hash : url, 0, NULL, 0))
 			break;
 	}
 
@@ -227,8 +260,8 @@
 	for (b0 = ban_start; b0 != NULL; b0 = VTAILQ_NEXT(b0, list)) {
 		if (b0->refcount == 0 && VTAILQ_NEXT(b0, list) == NULL)
 			break;
-		cli_out(cli, "%5u %s \"%s\"\n",
-		    b0->refcount,
+		cli_out(cli, "%5u %d %s \"%s\"\n",
+		    b0->refcount, b0->flags,
 		    b0->hash ? "hash" : "url ",
 		    b0->ban);
 	}

Modified: branches/nuke/varnish-cache/bin/varnishd/cache_dir_random.c
===================================================================
--- branches/nuke/varnish-cache/bin/varnishd/cache_dir_random.c	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/bin/varnishd/cache_dir_random.c	2008-11-24 09:25:56 UTC (rev 3419)
@@ -66,9 +66,9 @@
 static struct vbe_conn *
 vdi_random_getfd(struct sess *sp)
 {
-	int i, j, k;
+	int i, k;
 	struct vdi_random *vs;
-	double r, s1, s2;
+	double r, s1;
 	struct vbe_conn *vbe;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -78,45 +78,34 @@
 	k = 0;
 	for (k = 0; k < vs->retries; ) {
 
-		r = random() / 2147483648.0;	/* 2^31 */
-		assert(r >= 0.0 && r < 1.0);
-
+		/* Sum up the weights of healty backends */
 		s1 = 0.0;
-		j = 0;
-		for (i = 0; i < vs->nhosts; i++) {
-			if (!vs->hosts[i].backend->healthy)
-				continue;
-			s1 += vs->hosts[i].weight;
-			j++;
-		}	
+		for (i = 0; i < vs->nhosts; i++)
+			if (vs->hosts[i].backend->healthy)
+				s1 += vs->hosts[i].weight;
 
-		if (j == 0)		/* No healthy hosts */
+		if (s1 == 0.0)
 			return (NULL);
 
+		/* Pick a random threshold in that interval */
+		r = random() / 2147483648.0;	/* 2^31 */
+		assert(r >= 0.0 && r < 1.0);
 		r *= s1;
 
-		s2 = 0;
+		s1 = 0.0;
 		for (i = 0; i < vs->nhosts; i++)  {
 			if (!vs->hosts[i].backend->healthy)
 				continue;
-			s2 += vs->hosts[i].weight;
-			if (r < s2)
-				break;
+			s1 += vs->hosts[i].weight;
+			if (r >= s1)
+				continue;
+			vbe = VBE_GetVbe(sp, vs->hosts[i].backend);
+			if (vbe != NULL)
+				return (vbe);
+			break;
 		}
-
-		if (s2 != s1) {
-			/*
-			 * Health bit changed in an unusable way while we
-			 * worked the problem.  Usable changes are any that
-			 * result in the same sum we prepared for.
-			 */
-			continue;
-		}
-		vbe = VBE_GetVbe(sp, vs->hosts[i].backend);
-		if (vbe != NULL)
-			return (vbe);
 		k++;
-	} 
+	}
 	return (NULL);
 }
 

Modified: branches/nuke/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- branches/nuke/varnish-cache/bin/varnishd/cache_hash.c	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/bin/varnishd/cache_hash.c	2008-11-24 09:25:56 UTC (rev 3419)
@@ -281,7 +281,8 @@
 
 	if (busy_o != NULL) {
 		/* There are one or more busy objects, wait for them */
-		VTAILQ_INSERT_TAIL(&oh->waitinglist, sp, list);
+		if (sp->esis == 0)
+			VTAILQ_INSERT_TAIL(&oh->waitinglist, sp, list);
 		sp->objhead = oh;
 		UNLOCK(&oh->mtx);
 		return (NULL);

Modified: branches/nuke/varnish-cache/bin/varnishd/cache_vrt.c
===================================================================
--- branches/nuke/varnish-cache/bin/varnishd/cache_vrt.c	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/bin/varnishd/cache_vrt.c	2008-11-24 09:25:56 UTC (rev 3419)
@@ -284,8 +284,8 @@
 VRT_r_resp_status(const struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-	return (atoi(sp->obj->http->hd[HTTP_HDR_STATUS].b));
+	CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC);
+	return (atoi(sp->http->hd[HTTP_HDR_STATUS].b));
 }
 
 /*--------------------------------------------------------------------*/

Modified: branches/nuke/varnish-cache/bin/varnishd/cache_vrt_esi.c
===================================================================
--- branches/nuke/varnish-cache/bin/varnishd/cache_vrt_esi.c	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/bin/varnishd/cache_vrt_esi.c	2008-11-24 09:25:56 UTC (rev 3419)
@@ -796,7 +796,6 @@
 void
 ESI_Deliver(struct sess *sp)
 {
-
 	struct esi_bit *eb;
 	struct object *obj;
 
@@ -839,7 +838,16 @@
 		sp->step = STP_RECV;
 		http_ForceGet(sp->http);
 		http_Unset(sp->http, H_Content_Length);
-		CNT_Session(sp);
+		while (1) {
+			CNT_Session(sp);
+			if (sp->step == STP_DONE)
+				break;
+			AN(sp->wrk);
+			WSL_Flush(sp->wrk, 0);
+			DSL(0x20, SLT_Debug, sp->id, "loop waiting for ESI");
+			usleep(10000);
+		}
+		assert(sp->step == STP_DONE);
 		sp->esis--;
 		sp->obj = obj;
 

Modified: branches/nuke/varnish-cache/bin/varnishd/heritage.h
===================================================================
--- branches/nuke/varnish-cache/bin/varnishd/heritage.h	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/bin/varnishd/heritage.h	2008-11-24 09:25:56 UTC (rev 3419)
@@ -183,6 +183,9 @@
 	/* Amount of time to sleep when running out of file
 	   descriptors.  In msecs */
 	unsigned		accept_fd_holdoff;
+
+	/* Get rid of duplicate purges */
+	unsigned		purge_dups;
 };
 
 extern volatile struct params *params;

Modified: branches/nuke/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- branches/nuke/varnish-cache/bin/varnishd/mgt_param.c	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/bin/varnishd/mgt_param.c	2008-11-24 09:25:56 UTC (rev 3419)
@@ -817,6 +817,10 @@
 		"The TTL assigned to the synthesized error pages\n",
 		0,
 		"0", "seconds" },
+	{ "purge_dups", tweak_bool, &master.purge_dups, 0, 0,
+		"Detect and eliminate duplicate purges.\n",
+		0,
+		"off", "bool" },
 	{ NULL, NULL, NULL }
 };
 

Modified: branches/nuke/varnish-cache/bin/varnishncsa/varnishncsa.c
===================================================================
--- branches/nuke/varnish-cache/bin/varnishncsa/varnishncsa.c	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/bin/varnishncsa/varnishncsa.c	2008-11-24 09:25:56 UTC (rev 3419)
@@ -441,7 +441,7 @@
 		if (lp->df_Host) {
 			if (strncmp(lp->df_Host, "http://", 7) != 0)
 				fprintf(fo, "http://");
-			fprintf(fo, lp->df_Host);
+			fprintf(fo, "%s", lp->df_Host);
 		}
 		fprintf(fo, "%s ", lp->df_Uq);
 		fprintf(fo, "%s\" ", lp->df_H);

Modified: branches/nuke/varnish-cache/bin/varnishreplay/varnishreplay.c
===================================================================
--- branches/nuke/varnish-cache/bin/varnishreplay/varnishreplay.c	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/bin/varnishreplay/varnishreplay.c	2008-11-24 09:25:56 UTC (rev 3419)
@@ -743,7 +743,8 @@
 	signal(SIGPIPE, SIG_IGN);
 
 	pthread_attr_init(&thread_attr);
-	pthread_attr_setstacksize(&thread_attr, 16384);
+	/* XXX: seting the stack size manually reduces the memory usasage and increases speed */
+	pthread_attr_setstacksize(&thread_attr, 32768);
 
 	while (VSL_Dispatch(vd, gen_traffic, NULL) == 0)
 		/* nothing */ ;

Modified: branches/nuke/varnish-cache/bin/varnishtest/Makefile.am
===================================================================
--- branches/nuke/varnish-cache/bin/varnishtest/Makefile.am	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/bin/varnishtest/Makefile.am	2008-11-24 09:25:56 UTC (rev 3419)
@@ -9,6 +9,8 @@
 
 bin_PROGRAMS =	varnishtest
 
+dist_man_MANS = varnishtest.1
+
 varnishtest_SOURCES = \
 		vtc.c \
 		vtc.h \

Modified: branches/nuke/varnish-cache/configure.ac
===================================================================
--- branches/nuke/varnish-cache/configure.ac	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/configure.ac	2008-11-24 09:25:56 UTC (rev 3419)
@@ -3,7 +3,7 @@
 AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006-2008 Linpro AS / Verdens Gang AS])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [2.0.1], [varnish-dev at projects.linpro.no])
+AC_INIT([Varnish], [2.0.2], [varnish-dev at projects.linpro.no])
 AC_CONFIG_SRCDIR(include/varnishapi.h)
 AM_CONFIG_HEADER(config.h)
 

Modified: branches/nuke/varnish-cache/doc/changes-2.0-2.0.1.xml
===================================================================
--- branches/nuke/varnish-cache/doc/changes-2.0-2.0.1.xml	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/doc/changes-2.0-2.0.1.xml	2008-11-24 09:25:56 UTC (rev 3419)
@@ -17,6 +17,7 @@
       <para>There was an off-by-one error in the ACL compilation.
       Now fixed.</para>
     </change>
+  </subsystem>
 
   <subsystem>
     <name>Red Hat spec file</name>

Modified: branches/nuke/varnish-cache/include/stat_field.h
===================================================================
--- branches/nuke/varnish-cache/include/stat_field.h	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/include/stat_field.h	2008-11-24 09:25:56 UTC (rev 3419)
@@ -124,6 +124,7 @@
 MAC_STAT(n_purge_retire,	uint64_t, 'a', "N old purges deleted")
 MAC_STAT(n_purge_obj_test,	uint64_t, 'a', "N objects tested")
 MAC_STAT(n_purge_re_test,	uint64_t, 'a', "N regexps tested against")
+MAC_STAT(n_purge_dups,		uint64_t, 'a', "N duplicate purges removed")
 
 MAC_STAT(n_nuke_hit,            uint64_t, 'a', "N object headers marked as nuked")
 MAC_STAT(n_nuke_miss,           uint64_t, 'a', "N object headers not in cache")

Modified: branches/nuke/varnish-cache/lib/libvcl/vcc_compile.c
===================================================================
--- branches/nuke/varnish-cache/lib/libvcl/vcc_compile.c	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/lib/libvcl/vcc_compile.c	2008-11-24 09:25:56 UTC (rev 3419)
@@ -286,8 +286,12 @@
 				pos++;
 
 		}
-		Fc(tl, 0, "  [%3u] = { %d, %8u, %4u, %3u, 0, \"%.*s\" },\n",
-		    t->cnt, sp->idx, t->b - sp->b, lin, pos + 1, PF(t));
+		Fc(tl, 0, "  [%3u] = { %d, %8u, %4u, %3u, 0, ",
+		    t->cnt, sp->idx, t->b - sp->b, lin, pos + 1);
+		if (t->tok == CSRC)
+			Fc(tl, 0, " \"C{\"},\n");
+		else
+			Fc(tl, 0, " \"%.*s\" },\n", PF(t));
 	}
 	Fc(tl, 0, "};\n");
 }

Modified: branches/nuke/varnish-cache/redhat/varnish.spec
===================================================================
--- branches/nuke/varnish-cache/redhat/varnish.spec	2008-11-22 01:35:16 UTC (rev 3418)
+++ branches/nuke/varnish-cache/redhat/varnish.spec	2008-11-24 09:25:56 UTC (rev 3419)
@@ -1,6 +1,6 @@
 Summary: Varnish is a high-performance HTTP accelerator
 Name: varnish
-Version: 2.0
+Version: 2.0.2
 Release: 1%{?dist}
 License: BSD
 Group: System Environment/Daemons
@@ -11,7 +11,7 @@
 # configure script. Release tarballs would not need this
 #BuildRequires: automake autoconf libtool
 BuildRequires: ncurses-devel libxslt groff
-Requires: kernel >= 2.6.0 varnish-libs = %{version}-%{release}
+Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses
 Requires(pre): shadow-utils
@@ -43,7 +43,7 @@
 Summary: Development files for %{name}-libs
 Group: System Environment/Libraries
 BuildRequires: ncurses-devel
-Requires: kernel >= 2.6.0 varnish-libs = %{version}-%{release}
+Requires: varnish-libs = %{version}-%{release}
 
 %description libs-devel
 Development files for %{name}-libs
@@ -53,7 +53,7 @@
 #Summary: Files for static linking of %{name} library functions
 #Group: System Environment/Libraries
 #BuildRequires: ncurses-devel
-#Requires: kernel >= 2.6.0 varnish-libs-devel = %{version}-%{release}
+#Requires: varnish-libs-devel = %{version}-%{release}
 #
 #%description libs-static
 #Files for static linking of varnish library functions
@@ -220,6 +220,21 @@
 %postun libs -p /sbin/ldconfig
 
 %changelog
+* Mon Nov 10 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0.2-1
+  New upstream release 2.0.2. A bugfix release
+
+* Sun Nov 02 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0.1-2
+- Removed the requirement for kernel => 2.6.0. All supported
+  platforms meets this, and it generates strange errors in EPEL
+
+* Fri Oct 17 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0.1-1
+- 2.0.1 released, a bugfix release. New upstream sources
+- Package now also available in EPEL
+
+* Thu Oct 16 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0-2
+- Readded the debugflag patch. It's so practical
+- Added a strange workaround for make check on ppc64
+
 * Wed Oct 15 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0-1
 - 2.0 released. New upstream sources
 - Disabled jemalloc on ppc and ppc64. Added a note in README.redhat.



From phk at projects.linpro.no  Mon Nov 24 10:05:55 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 24 Nov 2008 11:05:55 +0100 (CET)
Subject: r3420 - in trunk/varnish-cache: include lib/libvarnish
Message-ID: <20081124100555.36A0C1EC200@projects.linpro.no>

Author: phk
Date: 2008-11-24 11:05:55 +0100 (Mon, 24 Nov 2008)
New Revision: 3420

Modified:
   trunk/varnish-cache/include/vsb.h
   trunk/varnish-cache/lib/libvarnish/vsb.c
Log:
Update license to remove the advertising clause, reflecting similar change
in the FreeBSD original.

Approved by:	des


Modified: trunk/varnish-cache/include/vsb.h
===================================================================
--- trunk/varnish-cache/include/vsb.h	2008-11-24 09:25:56 UTC (rev 3419)
+++ trunk/varnish-cache/include/vsb.h	2008-11-24 10:05:55 UTC (rev 3420)
@@ -11,8 +11,6 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote products
- *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES

Modified: trunk/varnish-cache/lib/libvarnish/vsb.c
===================================================================
--- trunk/varnish-cache/lib/libvarnish/vsb.c	2008-11-24 09:25:56 UTC (rev 3419)
+++ trunk/varnish-cache/lib/libvarnish/vsb.c	2008-11-24 10:05:55 UTC (rev 3420)
@@ -11,8 +11,6 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote products
- *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES



From phk at projects.linpro.no  Mon Nov 24 10:17:24 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 24 Nov 2008 11:17:24 +0100 (CET)
Subject: r3421 - in trunk/varnish-cache: . bin/varnishlog include
	include/compat lib/libvarnishcompat
Message-ID: <20081124101724.B637D1EC7C5@projects.linpro.no>

Author: phk
Date: 2008-11-24 11:17:24 +0100 (Mon, 24 Nov 2008)
New Revision: 3421

Removed:
   trunk/varnish-cache/include/compat/vis.h
   trunk/varnish-cache/lib/libvarnishcompat/vis.c
Modified:
   trunk/varnish-cache/bin/varnishlog/varnishlog.c
   trunk/varnish-cache/configure.ac
   trunk/varnish-cache/include/Makefile.am
   trunk/varnish-cache/lib/libvarnishcompat/Makefile.am
Log:
Eliminate <vis.h> and it's compat version, it is unused.



Modified: trunk/varnish-cache/bin/varnishlog/varnishlog.c
===================================================================
--- trunk/varnish-cache/bin/varnishlog/varnishlog.c	2008-11-24 10:05:55 UTC (rev 3420)
+++ trunk/varnish-cache/bin/varnishlog/varnishlog.c	2008-11-24 10:17:24 UTC (rev 3421)
@@ -47,12 +47,6 @@
 #include "compat/daemon.h"
 #endif
 
-#ifdef HAVE_VIS_H
-#include <vis.h>
-#else
-#include "compat/vis.h"
-#endif
-
 #include "vsb.h"
 #include "vpf.h"
 

Modified: trunk/varnish-cache/configure.ac
===================================================================
--- trunk/varnish-cache/configure.ac	2008-11-24 10:05:55 UTC (rev 3420)
+++ trunk/varnish-cache/configure.ac	2008-11-24 10:17:24 UTC (rev 3421)
@@ -80,7 +80,6 @@
 AC_CHECK_HEADERS([stddef.h])
 AC_CHECK_HEADERS([stdlib.h])
 AC_CHECK_HEADERS([unistd.h])
-AC_CHECK_HEADERS([vis.h])
 
 # Checks for typedefs, structures, and compiler characteristics.
 AC_C_CONST
@@ -166,7 +165,6 @@
 AC_CHECK_FUNCS([srandomdev])
 AC_CHECK_FUNCS([strlcat strlcpy])
 AC_CHECK_FUNCS([strndup])
-AC_CHECK_FUNCS([vis strvis strvisx])
 AC_CHECK_FUNCS([daemon])
 AC_SYS_LARGEFILE
 

Modified: trunk/varnish-cache/include/Makefile.am
===================================================================
--- trunk/varnish-cache/include/Makefile.am	2008-11-24 10:05:55 UTC (rev 3420)
+++ trunk/varnish-cache/include/Makefile.am	2008-11-24 10:17:24 UTC (rev 3421)
@@ -20,7 +20,6 @@
 	compat/strlcpy.h \
 	compat/strndup.h \
 	compat/vasprintf.h \
-	compat/vis.h \
 	flopen.h \
 	http_headers.h \
 	libvarnish.h \

Deleted: trunk/varnish-cache/include/compat/vis.h
===================================================================
--- trunk/varnish-cache/include/compat/vis.h	2008-11-24 10:05:55 UTC (rev 3420)
+++ trunk/varnish-cache/include/compat/vis.h	2008-11-24 10:17:24 UTC (rev 3421)
@@ -1,87 +0,0 @@
-/*-
- * Copyright (c) 1990, 1993
- *	The Regents of the University of California.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)vis.h	8.1 (Berkeley) 6/2/93
- * $FreeBSD: src/include/vis.h,v 1.11 2003/10/30 10:40:49 phk Exp $
- * $Id$
- */
-
-#ifndef _VIS_H_
-#define	_VIS_H_
-
-/*
- * to select alternate encoding format
- */
-#define	VIS_OCTAL	0x01	/* use octal \ddd format */
-#define	VIS_CSTYLE	0x02	/* use \[nrft0..] where appropriate */
-
-/*
- * to alter set of characters encoded (default is to encode all
- * non-graphic except space, tab, and newline).
- */
-#define	VIS_SP		0x04	/* also encode space */
-#define	VIS_TAB		0x08	/* also encode tab */
-#define	VIS_NL		0x10	/* also encode newline */
-#define	VIS_WHITE	(VIS_SP | VIS_TAB | VIS_NL)
-#define	VIS_SAFE	0x20	/* only encode "unsafe" characters */
-
-/*
- * other
- */
-#define	VIS_NOSLASH	0x40	/* inhibit printing '\' */
-#define	VIS_HTTPSTYLE	0x80	/* http-style escape % HEX HEX */
-#define	VIS_GLOB	0x100	/* encode glob(3) magics */
-
-/*
- * unvis return codes
- */
-#define	UNVIS_VALID	 1	/* character valid */
-#define	UNVIS_VALIDPUSH	 2	/* character valid, push back passed char */
-#define	UNVIS_NOCHAR	 3	/* valid sequence, no character produced */
-#define	UNVIS_SYNBAD	-1	/* unrecognized escape sequence */
-#define	UNVIS_ERROR	-2	/* decoder in unknown state (unrecoverable) */
-
-/*
- * unvis flags
- */
-#define	UNVIS_END	1	/* no more characters */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-char	*vis(char *, int, int, int);
-int	strvis(char *, const char *, int);
-int	strvisx(char *, const char *, size_t, int);
-int	strunvis(char *, const char *);
-int	strunvisx(char *, const char *, int);
-int	unvis(char *, int, int *, int);
-#ifdef __cplusplus
-};
-#endif
-
-#endif /* !_VIS_H_ */

Modified: trunk/varnish-cache/lib/libvarnishcompat/Makefile.am
===================================================================
--- trunk/varnish-cache/lib/libvarnishcompat/Makefile.am	2008-11-24 10:05:55 UTC (rev 3420)
+++ trunk/varnish-cache/lib/libvarnishcompat/Makefile.am	2008-11-24 10:17:24 UTC (rev 3421)
@@ -14,5 +14,4 @@
 	srandomdev.c \
 	strlcat.c \
 	strlcpy.c \
-	strndup.c \
-	vis.c
+	strndup.c

Deleted: trunk/varnish-cache/lib/libvarnishcompat/vis.c
===================================================================
--- trunk/varnish-cache/lib/libvarnishcompat/vis.c	2008-11-24 10:05:55 UTC (rev 3420)
+++ trunk/varnish-cache/lib/libvarnishcompat/vis.c	2008-11-24 10:17:24 UTC (rev 3421)
@@ -1,212 +0,0 @@
-/*-
- * Copyright (c) 1989, 1993
- *	The Regents of the University of California.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * @(#)vis.c	8.1 (Berkeley) 7/19/93
- * $FreeBSD: src/lib/libc/gen/vis.c,v 1.13 2003/10/30 12:41:50 phk Exp $
- * $Id$
- */
-
-#include "config.h"
-
-#if !defined(HAVE_VIS) || !defined(HAVE_STRVIS) || !defined(HAVE_STRVISX)
-
-#include <sys/types.h>
-#include <limits.h>
-#include <ctype.h>
-#include <stdio.h>
-
-#include "compat/vis.h"
-
-#define	isoctal(c)	(((u_char)(c)) >= '0' && ((u_char)(c)) <= '7')
-
-/*
- * vis - visually encode characters
- */
-#ifndef HAVE_VIS
-char *
-vis(dst, c, flag, nextc)
-	char *dst;
-	int c, nextc;
-	int flag;
-{
-	c = (unsigned char)c;
-
-	if (flag & VIS_HTTPSTYLE) {
-		/* Described in RFC 1808 */
-		if (!(isalnum(c) /* alpha-numeric */
-		    /* safe */
-		    || c == '$' || c == '-' || c == '_' || c == '.' || c == '+'
-		    /* extra */
-		    || c == '!' || c == '*' || c == '\'' || c == '('
-		    || c == ')' || c == ',')) {
-			*dst++ = '%';
-			snprintf(dst, 4, (c < 16 ? "0%X" : "%X"), c);
-			dst += 2;
-			goto done;
-		}
-	}
-
-	if ((flag & VIS_GLOB) &&
-	    (c == '*' || c == '?' || c == '[' || c == '#'))
-		;
-	else if (isgraph(c) ||
-	   ((flag & VIS_SP) == 0 && c == ' ') ||
-	   ((flag & VIS_TAB) == 0 && c == '\t') ||
-	   ((flag & VIS_NL) == 0 && c == '\n') ||
-	   ((flag & VIS_SAFE) && (c == '\b' || c == '\007' || c == '\r'))) {
-		*dst++ = c;
-		if (c == '\\' && (flag & VIS_NOSLASH) == 0)
-			*dst++ = '\\';
-		*dst = '\0';
-		return (dst);
-	}
-
-	if (flag & VIS_CSTYLE) {
-		switch(c) {
-		case '\n':
-			*dst++ = '\\';
-			*dst++ = 'n';
-			goto done;
-		case '\r':
-			*dst++ = '\\';
-			*dst++ = 'r';
-			goto done;
-		case '\b':
-			*dst++ = '\\';
-			*dst++ = 'b';
-			goto done;
-		case '\a':
-			*dst++ = '\\';
-			*dst++ = 'a';
-			goto done;
-		case '\v':
-			*dst++ = '\\';
-			*dst++ = 'v';
-			goto done;
-		case '\t':
-			*dst++ = '\\';
-			*dst++ = 't';
-			goto done;
-		case '\f':
-			*dst++ = '\\';
-			*dst++ = 'f';
-			goto done;
-		case ' ':
-			*dst++ = '\\';
-			*dst++ = 's';
-			goto done;
-		case '\0':
-			*dst++ = '\\';
-			*dst++ = '0';
-			if (isoctal(nextc)) {
-				*dst++ = '0';
-				*dst++ = '0';
-			}
-			goto done;
-		}
-	}
-	if (((c & 0177) == ' ') || isgraph(c) || (flag & VIS_OCTAL)) {
-		*dst++ = '\\';
-		*dst++ = ((u_char)c >> 6 & 07) + '0';
-		*dst++ = ((u_char)c >> 3 & 07) + '0';
-		*dst++ = ((u_char)c & 07) + '0';
-		goto done;
-	}
-	if ((flag & VIS_NOSLASH) == 0)
-		*dst++ = '\\';
-	if (c & 0200) {
-		c &= 0177;
-		*dst++ = 'M';
-	}
-	if (iscntrl(c)) {
-		*dst++ = '^';
-		if (c == 0177)
-			*dst++ = '?';
-		else
-			*dst++ = c + '@';
-	} else {
-		*dst++ = '-';
-		*dst++ = c;
-	}
-done:
-	*dst = '\0';
-	return (dst);
-}
-#endif
-
-/*
- * strvis, strvisx - visually encode characters from src into dst
- *
- *	Dst must be 4 times the size of src to account for possible
- *	expansion.  The length of dst, not including the trailing NUL,
- *	is returned.
- *
- *	Strvisx encodes exactly len bytes from src into dst.
- *	This is useful for encoding a block of data.
- */
-#ifndef HAVE_STRVIS
-int
-strvis(dst, src, flag)
-	char *dst;
-	const char *src;
-	int flag;
-{
-	char c;
-	char *start;
-
-	for (start = dst; (c = *src); )
-		dst = vis(dst, c, flag, *++src);
-	*dst = '\0';
-	return (dst - start);
-}
-#endif
-
-#ifndef HAVE_STRVISX
-int
-strvisx(dst, src, len, flag)
-	char *dst;
-	const char *src;
-	size_t len;
-	int flag;
-{
-	int c;
-	char *start;
-
-	for (start = dst; len > 1; len--) {
-		c = *src;
-		dst = vis(dst, c, flag, *++src);
-	}
-	if (len)
-		dst = vis(dst, *src, flag, '\0');
-	*dst = '\0';
-
-	return (dst - start);
-}
-#endif
-
-#endif



From phk at projects.linpro.no  Mon Nov 24 12:24:55 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 24 Nov 2008 13:24:55 +0100 (CET)
Subject: r3422 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081124122455.DAB4C1EC200@projects.linpro.no>

Author: phk
Date: 2008-11-24 13:24:55 +0100 (Mon, 24 Nov 2008)
New Revision: 3422

Modified:
   trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Do not count chunked encoding headers in stat.hdrbytes.



Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c	2008-11-24 10:17:24 UTC (rev 3421)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c	2008-11-24 12:24:55 UTC (rev 3422)
@@ -146,8 +146,7 @@
 	} else if (sp->wantbody) {
 		if (sp->esis > 0 && sp->http->protover >= 1.1) {
 			sprintf(lenbuf, "%x\r\n", sp->obj->len);
-			sp->wrk->acct.hdrbytes +=
-			    WRK_Write(sp->wrk, lenbuf, -1);
+			(void)WRK_Write(sp->wrk, lenbuf, -1);
 		}
 
 		VTAILQ_FOREACH(st, &sp->obj->store, list) {
@@ -171,11 +170,11 @@
 			}
 #endif /* SENDFILE_WORKS */
 			VSL_stats->n_objwrite++;
-			WRK_Write(sp->wrk, st->ptr, st->len);
+			(void)WRK_Write(sp->wrk, st->ptr, st->len);
 		}
 		assert(u == sp->obj->len);
 		if (sp->esis > 0 && sp->http->protover >= 1.1)
-			WRK_Write(sp->wrk, "\r\n", -1);
+			(void)WRK_Write(sp->wrk, "\r\n", -1);
 	}
 	if (WRK_Flush(sp->wrk))
 		vca_close_session(sp, "remote closed");



From phk at projects.linpro.no  Mon Nov 24 12:27:25 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 24 Nov 2008 13:27:25 +0100 (CET)
Subject: r3423 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081124122725.3A30A1EC1FE@projects.linpro.no>

Author: phk
Date: 2008-11-24 13:27:25 +0100 (Mon, 24 Nov 2008)
New Revision: 3423

Modified:
   trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c
Log:
Count ESI processed objects in acct.bodybytes



Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c	2008-11-24 12:24:55 UTC (rev 3422)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c	2008-11-24 12:27:25 UTC (rev 3423)
@@ -800,12 +800,14 @@
 	struct object *obj;
 
 	VTAILQ_FOREACH(eb, &sp->obj->esibits, list) {
+		assert(sp->wrk->wfd = &sp->fd);
 		if (Tlen(eb->verbatim)) {
 			if (sp->http->protover >= 1.1)
-				WRK_Write(sp->wrk, eb->chunk_length, -1);
-			WRK_Write(sp->wrk, eb->verbatim.b, Tlen(eb->verbatim));
+				(void)WRK_Write(sp->wrk, eb->chunk_length, -1);
+			sp->wrk->acct.bodybytes += WRK_Write(sp->wrk,
+			    eb->verbatim.b, Tlen(eb->verbatim));
 			if (sp->http->protover >= 1.1)
-				WRK_Write(sp->wrk, "\r\n", -1);
+				(void)WRK_Write(sp->wrk, "\r\n", -1);
 		}
 		if (eb->include.b == NULL ||
 		    sp->esis >= params->max_esi_includes)
@@ -814,9 +816,10 @@
 		/*
 		 * We flush here, because the next transaction is
 		 * quite likely to take some time, so we should get
-		 * as many bits to the client as we can already
+		 * as many bits to the client as we can already.
 		 */
-		WRK_Flush(sp->wrk);
+		if (WRK_Flush(sp->wrk))
+			break;
 
 		sp->esis++;
 		obj = sp->obj;
@@ -852,8 +855,9 @@
 		sp->obj = obj;
 
 	}
+	assert(sp->wrk->wfd = &sp->fd);
 	if (sp->esis == 0 && sp->http->protover >= 1.1)
-		WRK_Write(sp->wrk, "0\r\n\r\n", -1);
+		(void)WRK_Write(sp->wrk, "0\r\n\r\n", -1);
 }
 
 /*--------------------------------------------------------------------*/



From phk at projects.linpro.no  Mon Nov 24 12:30:29 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 24 Nov 2008 13:30:29 +0100 (CET)
Subject: r3424 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081124123029.70B051EC46B@projects.linpro.no>

Author: phk
Date: 2008-11-24 13:30:29 +0100 (Mon, 24 Nov 2008)
New Revision: 3424

Modified:
   trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c
Log:
Use == for comparison.



Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c	2008-11-24 12:27:25 UTC (rev 3423)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c	2008-11-24 12:30:29 UTC (rev 3424)
@@ -800,7 +800,7 @@
 	struct object *obj;
 
 	VTAILQ_FOREACH(eb, &sp->obj->esibits, list) {
-		assert(sp->wrk->wfd = &sp->fd);
+		assert(sp->wrk->wfd == &sp->fd);
 		if (Tlen(eb->verbatim)) {
 			if (sp->http->protover >= 1.1)
 				(void)WRK_Write(sp->wrk, eb->chunk_length, -1);
@@ -855,7 +855,7 @@
 		sp->obj = obj;
 
 	}
-	assert(sp->wrk->wfd = &sp->fd);
+	assert(sp->wrk->wfd == &sp->fd);
 	if (sp->esis == 0 && sp->http->protover >= 1.1)
 		(void)WRK_Write(sp->wrk, "0\r\n\r\n", -1);
 }



From phk at projects.linpro.no  Mon Nov 24 13:03:41 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 24 Nov 2008 14:03:41 +0100 (CET)
Subject: r3425 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081124130341.6F4251EC200@projects.linpro.no>

Author: phk
Date: 2008-11-24 14:03:41 +0100 (Mon, 24 Nov 2008)
New Revision: 3425

Modified:
   trunk/varnish-cache/bin/varnishd/cache_pool.c
Log:
Log write errors under SLT_Debug.



Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-24 12:30:29 UTC (rev 3424)
+++ trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-24 13:03:41 UTC (rev 3425)
@@ -126,6 +126,9 @@
 		i = writev(*w->wfd, w->iov, w->niov);
 		if (i != w->liov)
 			w->werr++;
+		WSL(w, SLT_Debug, *w->wfd,
+		    "Write error, len = %d/%d, errno = %s",
+		    i, w->liov, strerror(errno));
 	}
 	w->liov = 0;
 	w->niov = 0;



From tfheen at projects.linpro.no  Mon Nov 24 14:04:42 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Mon, 24 Nov 2008 15:04:42 +0100 (CET)
Subject: r3426 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081124140442.772911EC1FE@projects.linpro.no>

Author: tfheen
Date: 2008-11-24 15:04:42 +0100 (Mon, 24 Nov 2008)
New Revision: 3426

Modified:
   trunk/varnish-cache/bin/varnishd/cache_backend_poll.c
Log:
Fix typo (s/timeout/interval/) in default parameters for backend health

Thanks to Jonny @ globo for noticing.


Modified: trunk/varnish-cache/bin/varnishd/cache_backend_poll.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_backend_poll.c	2008-11-24 13:03:41 UTC (rev 3425)
+++ trunk/varnish-cache/bin/varnishd/cache_backend_poll.c	2008-11-24 14:04:42 UTC (rev 3426)
@@ -261,7 +261,7 @@
 	if (vt->probe.timeout == 0.0)
 		vt->probe.timeout = 2.0;
 	if (vt->probe.interval == 0.0)
-		vt->probe.timeout = 5.0;
+		vt->probe.interval = 5.0;
 	if (vt->probe.window == 0)
 		vt->probe.window = 8;
 	if (vt->probe.threshold == 0)



From phk at projects.linpro.no  Mon Nov 24 14:41:47 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 24 Nov 2008 15:41:47 +0100 (CET)
Subject: r3427 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081124144147.2B9791EC200@projects.linpro.no>

Author: phk
Date: 2008-11-24 15:41:36 +0100 (Mon, 24 Nov 2008)
New Revision: 3427

Modified:
   trunk/varnish-cache/bin/varnishd/cache_pool.c
Log:

Only emit debug message for writes that fail


Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-24 14:04:42 UTC (rev 3426)
+++ trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-24 14:41:36 UTC (rev 3427)
@@ -124,11 +124,12 @@
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 	if (*w->wfd >= 0 && w->niov > 0 && w->werr == 0) {
 		i = writev(*w->wfd, w->iov, w->niov);
-		if (i != w->liov)
+		if (i != w->liov) {
 			w->werr++;
-		WSL(w, SLT_Debug, *w->wfd,
-		    "Write error, len = %d/%d, errno = %s",
-		    i, w->liov, strerror(errno));
+			WSL(w, SLT_Debug, *w->wfd,
+			    "Write error, len = %d/%d, errno = %s",
+			    i, w->liov, strerror(errno));
+		}
 	}
 	w->liov = 0;
 	w->niov = 0;



From phk at projects.linpro.no  Mon Nov 24 15:53:26 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 24 Nov 2008 16:53:26 +0100 (CET)
Subject: r3428 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081124155326.D9D961EC1FE@projects.linpro.no>

Author: phk
Date: 2008-11-24 16:53:26 +0100 (Mon, 24 Nov 2008)
New Revision: 3428

Modified:
   trunk/varnish-cache/bin/varnishd/heritage.h
   trunk/varnish-cache/bin/varnishd/mgt_param.c
   trunk/varnish-cache/bin/varnishd/shmlog.c
Log:
Make the maximum record length in the shm log a paramter "shm_reclen".



Modified: trunk/varnish-cache/bin/varnishd/heritage.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/heritage.h	2008-11-24 14:41:36 UTC (rev 3427)
+++ trunk/varnish-cache/bin/varnishd/heritage.h	2008-11-24 15:53:26 UTC (rev 3428)
@@ -98,6 +98,8 @@
 	unsigned		obj_workspace;
 	unsigned		shm_workspace;
 
+	unsigned		shm_reclen;
+
 	/* Acceptor hints */
 	unsigned		sess_timeout;
 	unsigned		pipe_timeout;

Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-24 14:41:36 UTC (rev 3427)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-24 15:53:26 UTC (rev 3428)
@@ -636,6 +636,11 @@
 		"Minimum is 4096 bytes.",
 		DELAYED_EFFECT,
 		"8192", "bytes" },
+	{ "shm_reclen", tweak_uint, &master.shm_reclen, 16, 65535,
+		"Maximum number of bytes in SHM log record.\n"
+		"Maximum is 65535 bytes.",
+		0,
+		"255", "bytes" },
 	{ "default_grace", tweak_uint, &master.default_grace, 0, UINT_MAX,
 		"Default grace period.  We will deliver an object "
 		"this long after it has expired, provided another thread "

Modified: trunk/varnish-cache/bin/varnishd/shmlog.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/shmlog.c	2008-11-24 14:41:36 UTC (rev 3427)
+++ trunk/varnish-cache/bin/varnishd/shmlog.c	2008-11-24 15:53:26 UTC (rev 3428)
@@ -81,6 +81,8 @@
 vsl_hdr(enum shmlogtag tag, unsigned char *p, unsigned len, unsigned id)
 {
 
+	assert(len < 0x10000);
+	assert(id < 0x10000);
 	p[__SHMLOG_LEN_HIGH] = (len >> 8) & 0xff;
 	p[__SHMLOG_LEN_LOW] = len & 0xff;
 	p[__SHMLOG_ID_HIGH] = (id >> 8) & 0xff;
@@ -100,14 +102,15 @@
 VSLR(enum shmlogtag tag, int id, txt t)
 {
 	unsigned char *p;
-	unsigned l;
+	unsigned l, mlen;
 
 	Tcheck(t);
+	mlen = params->shm_reclen;
 
 	/* Truncate */
 	l = Tlen(t);
-	if (l > 255) {
-		l = 255;
+	if (l > mlen) {
+		l = mlen;
 		t.e = t.b + l;
 	}
 
@@ -136,11 +139,12 @@
 {
 	va_list ap;
 	unsigned char *p;
-	unsigned n;
+	unsigned n, mlen;
 	txt t;
 
 	AN(fmt);
 	va_start(ap, fmt);
+	mlen = params->shm_reclen;
 
 	if (strchr(fmt, '%') == NULL) {
 		t.b = TRUST_ME(fmt);
@@ -153,13 +157,14 @@
 		assert(loghead->ptr < loghead->size);
 
 		/* Wrap if we cannot fit a full size record */
-		if (loghead->ptr + SHMLOG_NEXTTAG + 255 + 1 >= loghead->size)
+		if (loghead->ptr + SHMLOG_NEXTTAG + mlen + 1 >= loghead->size)
 			vsl_wrap();
 
 		p = logstart + loghead->ptr;
-		n = vsnprintf((char *)(p + SHMLOG_DATA), 256, fmt, ap);
-		if (n > 255)
-			n = 255;	/* we truncate long fields */
+		/* +1 for the NUL */
+		n = vsnprintf((char *)(p + SHMLOG_DATA), mlen + 1, fmt, ap);
+		if (n > mlen)
+			n = mlen;		/* we truncate long fields */
 		vsl_hdr(tag, p, n, id);
 		loghead->ptr += SHMLOG_NEXTTAG + n;
 		assert(loghead->ptr < loghead->size);
@@ -203,14 +208,15 @@
 WSLR(struct worker *w, enum shmlogtag tag, int id, txt t)
 {
 	unsigned char *p;
-	unsigned l;
+	unsigned l, mlen;
 
 	Tcheck(t);
+	mlen = params->shm_reclen;
 
 	/* Truncate */
 	l = Tlen(t);
-	if (l > 255) {
-		l = 255;
+	if (l > mlen) {
+		l = mlen;
 		t.e = t.b + l;
 	}
 
@@ -234,11 +240,12 @@
 {
 	va_list ap;
 	unsigned char *p;
-	unsigned n;
+	unsigned n, mlen;
 	txt t;
 
 	AN(fmt);
 	va_start(ap, fmt);
+	mlen = params->shm_reclen;
 
 	if (strchr(fmt, '%') == NULL) {
 		t.b = TRUST_ME(fmt);
@@ -248,13 +255,14 @@
 		assert(w->wlp < w->wle);
 
 		/* Wrap if we cannot fit a full size record */
-		if (w->wlp + SHMLOG_NEXTTAG + 255 + 1 >= w->wle)
+		if (w->wlp + SHMLOG_NEXTTAG + mlen + 1 >= w->wle)
 			WSL_Flush(w, 1);
 
 		p = w->wlp;
-		n = vsnprintf((char *)(p + SHMLOG_DATA), 256, fmt, ap);
-		if (n > 255)
-			n = 255;	/* we truncate long fields */
+		/* +1 for the NUL */
+		n = vsnprintf((char *)(p + SHMLOG_DATA), mlen + 1, fmt, ap);
+		if (n > mlen)
+			n = mlen;	/* we truncate long fields */
 		vsl_hdr(tag, p, n, id);
 		w->wlp += SHMLOG_NEXTTAG + n;
 		assert(w->wlp < w->wle);



From phk at projects.linpro.no  Mon Nov 24 17:47:21 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 24 Nov 2008 18:47:21 +0100 (CET)
Subject: r3429 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081124174721.411841EC200@projects.linpro.no>

Author: phk
Date: 2008-11-24 18:47:21 +0100 (Mon, 24 Nov 2008)
New Revision: 3429

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_fetch.c
   trunk/varnish-cache/bin/varnishd/cache_http.c
   trunk/varnish-cache/bin/varnishd/cache_pipe.c
   trunk/varnish-cache/bin/varnishd/cache_pool.c
   trunk/varnish-cache/bin/varnishd/cache_response.c
   trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c
Log:
Rename the write-buffering functions to WRW_*().

Make reservation and release explicit.

Add asserts that this it happens.


Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2008-11-24 15:53:26 UTC (rev 3428)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2008-11-24 17:47:21 UTC (rev 3429)
@@ -529,12 +529,15 @@
 void WRK_Init(void);
 int WRK_Queue(struct workreq *wrq);
 void WRK_QueueSession(struct sess *sp);
-void WRK_Reset(struct worker *w, int *fd);
-unsigned WRK_Flush(struct worker *w);
-unsigned WRK_Write(struct worker *w, const void *ptr, int len);
-unsigned WRK_WriteH(struct worker *w, const txt *hh, const char *suf);
+
+void WRW_Reserve(struct worker *w, int *fd);
+void WRW_Release(struct worker *w);
+unsigned WRW_Flush(struct worker *w);
+unsigned WRW_FlushRelease(struct worker *w);
+unsigned WRW_Write(struct worker *w, const void *ptr, int len);
+unsigned WRW_WriteH(struct worker *w, const txt *hh, const char *suf);
 #ifdef SENDFILE_WORKS
-void WRK_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
+void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 #endif  /* SENDFILE_WORKS */
 
 /* cache_session.c [SES] */

Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c	2008-11-24 15:53:26 UTC (rev 3428)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c	2008-11-24 17:47:21 UTC (rev 3429)
@@ -288,8 +288,8 @@
 			content_length -= rdcnt;
 			if (!sp->sendbody)
 				continue;
-			WRK_Write(sp->wrk, buf, rdcnt);	/* XXX: stats ? */
-			if (WRK_Flush(sp->wrk))
+			(void)WRW_Write(sp->wrk, buf, rdcnt); /* XXX: stats ? */
+			if (WRW_Flush(sp->wrk))
 				return (2);
 		}
 	}
@@ -348,7 +348,7 @@
 		VBE_AddHostHeader(sp);
 
 	TCP_blocking(vc->fd);	/* XXX: we should timeout instead */
-	WRK_Reset(w, &vc->fd);
+	WRW_Reserve(w, &vc->fd);
 	http_Write(w, hp, 0);	/* XXX: stats ? */
 
 	/* Deal with any message-body the request might have */
@@ -358,7 +358,7 @@
 		return (__LINE__);
 	}
 
-	if (WRK_Flush(w)) {
+	if (WRW_FlushRelease(w)) {
 		VBE_ClosedFd(sp);
 		/* XXX: other cleanup ? */
 		return (__LINE__);

Modified: trunk/varnish-cache/bin/varnishd/cache_http.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_http.c	2008-11-24 15:53:26 UTC (rev 3428)
+++ trunk/varnish-cache/bin/varnishd/cache_http.c	2008-11-24 17:47:21 UTC (rev 3429)
@@ -809,28 +809,28 @@
 
 	if (resp) {
 		AN(hp->hd[HTTP_HDR_STATUS].b);
-		l = WRK_WriteH(w, &hp->hd[HTTP_HDR_PROTO], " ");
+		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], " ");
 		WSLH(w, *w->wfd, hp, HTTP_HDR_PROTO);
-		l += WRK_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " ");
+		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " ");
 		WSLH(w, *w->wfd, hp, HTTP_HDR_STATUS);
-		l += WRK_WriteH(w, &hp->hd[HTTP_HDR_RESPONSE], "\r\n");
+		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_RESPONSE], "\r\n");
 		WSLH(w, *w->wfd, hp, HTTP_HDR_RESPONSE);
 	} else {
 		AN(hp->hd[HTTP_HDR_URL].b);
-		l = WRK_WriteH(w, &hp->hd[HTTP_HDR_REQ], " ");
+		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_REQ], " ");
 		WSLH(w, *w->wfd, hp, HTTP_HDR_REQ);
-		l += WRK_WriteH(w, &hp->hd[HTTP_HDR_URL], " ");
+		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_URL], " ");
 		WSLH(w, *w->wfd, hp, HTTP_HDR_URL);
-		l += WRK_WriteH(w, &hp->hd[HTTP_HDR_PROTO], "\r\n");
+		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], "\r\n");
 		WSLH(w, *w->wfd, hp, HTTP_HDR_PROTO);
 	}
 	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
 		AN(hp->hd[u].b);
 		AN(hp->hd[u].e);
-		l += WRK_WriteH(w, &hp->hd[u], "\r\n");
+		l += WRW_WriteH(w, &hp->hd[u], "\r\n");
 		WSLH(w, *w->wfd, hp, u);
 	}
-	l += WRK_Write(w, "\r\n", -1);
+	l += WRW_Write(w, "\r\n", -1);
 	return (l);
 }
 

Modified: trunk/varnish-cache/bin/varnishd/cache_pipe.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pipe.c	2008-11-24 15:53:26 UTC (rev 3428)
+++ trunk/varnish-cache/bin/varnishd/cache_pipe.c	2008-11-24 17:47:21 UTC (rev 3429)
@@ -83,14 +83,14 @@
 	vc = sp->vbe;
 	TCP_blocking(vc->fd);
 
-	WRK_Reset(w, &vc->fd);
+	WRW_Reserve(w, &vc->fd);
 	w->acct.hdrbytes += http_Write(w, bereq->http, 0);
 
 	if (sp->htc->pipeline.b != NULL)
 		w->acct.bodybytes +=
-		    WRK_Write(w, sp->htc->pipeline.b, Tlen(sp->htc->pipeline));
+		    WRW_Write(w, sp->htc->pipeline.b, Tlen(sp->htc->pipeline));
 
-	if (WRK_Flush(w)) {
+	if (WRW_FlushRelease(w)) {
 		vca_close_session(sp, "pipe");
 		VBE_ClosedFd(sp);
 		return;

Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-24 15:53:26 UTC (rev 3428)
+++ trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-24 17:47:21 UTC (rev 3429)
@@ -106,22 +106,35 @@
  */
 
 void
-WRK_Reset(struct worker *w, int *fd)
+WRW_Reserve(struct worker *w, int *fd)
 {
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AZ(w->wfd);
 	w->werr = 0;
 	w->liov = 0;
 	w->niov = 0;
 	w->wfd = fd;
 }
 
+void
+WRW_Release(struct worker *w)
+{
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	w->werr = 0;
+	w->liov = 0;
+	w->niov = 0;
+	w->wfd = NULL;
+}
+
 unsigned
-WRK_Flush(struct worker *w)
+WRW_Flush(struct worker *w)
 {
 	ssize_t i;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AN(w->wfd);
 	if (*w->wfd >= 0 && w->niov > 0 && w->werr == 0) {
 		i = writev(*w->wfd, w->iov, w->niov);
 		if (i != w->liov) {
@@ -137,32 +150,46 @@
 }
 
 unsigned
-WRK_WriteH(struct worker *w, const txt *hh, const char *suf)
+WRW_FlushRelease(struct worker *w)
 {
 	unsigned u;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AN(w->wfd);
+	u = WRW_Flush(w);
+	WRW_Release(w);
+	return (u);
+}
+
+unsigned
+WRW_WriteH(struct worker *w, const txt *hh, const char *suf)
+{
+	unsigned u;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AN(w->wfd);
 	AN(w);
 	AN(hh);
 	AN(hh->b);
 	AN(hh->e);
-	u = WRK_Write(w, hh->b, hh->e - hh->b);
+	u = WRW_Write(w, hh->b, hh->e - hh->b);
 	if (suf != NULL)
-		u += WRK_Write(w, suf, -1);
+		u += WRW_Write(w, suf, -1);
 	return (u);
 }
 
 unsigned
-WRK_Write(struct worker *w, const void *ptr, int len)
+WRW_Write(struct worker *w, const void *ptr, int len)
 {
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AN(w->wfd);
 	if (len == 0 || *w->wfd < 0)
 		return (0);
 	if (len == -1)
 		len = strlen(ptr);
 	if (w->niov == MAX_IOVS)
-		(void)WRK_Flush(w);
+		(void)WRW_Flush(w);
 	w->iov[w->niov].iov_base = TRUST_ME(ptr);
 	w->iov[w->niov].iov_len = len;
 	w->liov += len;
@@ -172,10 +199,11 @@
 
 #ifdef SENDFILE_WORKS
 void
-WRK_Sendfile(struct worker *w, int fd, off_t off, unsigned len)
+WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len)
 {
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AN(w->wfd);
 	assert(fd >= 0);
 	assert(len > 0);
 

Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c	2008-11-24 15:53:26 UTC (rev 3428)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c	2008-11-24 17:47:21 UTC (rev 3429)
@@ -137,16 +137,24 @@
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	WRK_Reset(sp->wrk, &sp->fd);
+	WRW_Reserve(sp->wrk, &sp->fd);
+
 	if (sp->esis == 0)
 		sp->wrk->acct.hdrbytes += http_Write(sp->wrk, sp->http, 1);
 
 	if (sp->wantbody && !VTAILQ_EMPTY(&sp->obj->esibits)) {
+		if (WRW_FlushRelease(sp->wrk)) {
+			vca_close_session(sp, "remote closed");
+			return;
+		}
 		ESI_Deliver(sp);
-	} else if (sp->wantbody) {
+		return;
+	}
+
+	if (sp->wantbody) {
 		if (sp->esis > 0 && sp->http->protover >= 1.1) {
 			sprintf(lenbuf, "%x\r\n", sp->obj->len);
-			(void)WRK_Write(sp->wrk, lenbuf, -1);
+			(void)WRW_Write(sp->wrk, lenbuf, -1);
 		}
 
 		VTAILQ_FOREACH(st, &sp->obj->store, list) {
@@ -164,18 +172,18 @@
 			if (st->fd >= 0 &&
 			    st->len >= params->sendfile_threshold) {
 				VSL_stats->n_objsendfile++;
-				WRK_Sendfile(sp->wrk, st->fd,
+				WRW_Sendfile(sp->wrk, st->fd,
 				    st->where, st->len);
 				continue;
 			}
 #endif /* SENDFILE_WORKS */
 			VSL_stats->n_objwrite++;
-			(void)WRK_Write(sp->wrk, st->ptr, st->len);
+			(void)WRW_Write(sp->wrk, st->ptr, st->len);
 		}
 		assert(u == sp->obj->len);
 		if (sp->esis > 0 && sp->http->protover >= 1.1)
-			(void)WRK_Write(sp->wrk, "\r\n", -1);
+			(void)WRW_Write(sp->wrk, "\r\n", -1);
 	}
-	if (WRK_Flush(sp->wrk))
+	if (WRW_FlushRelease(sp->wrk))
 		vca_close_session(sp, "remote closed");
 }

Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c	2008-11-24 15:53:26 UTC (rev 3428)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c	2008-11-24 17:47:21 UTC (rev 3429)
@@ -799,27 +799,24 @@
 	struct esi_bit *eb;
 	struct object *obj;
 
+	WRW_Reserve(sp->wrk, &sp->fd);
 	VTAILQ_FOREACH(eb, &sp->obj->esibits, list) {
-		assert(sp->wrk->wfd == &sp->fd);
 		if (Tlen(eb->verbatim)) {
 			if (sp->http->protover >= 1.1)
-				(void)WRK_Write(sp->wrk, eb->chunk_length, -1);
-			sp->wrk->acct.bodybytes += WRK_Write(sp->wrk,
+				(void)WRW_Write(sp->wrk, eb->chunk_length, -1);
+			sp->wrk->acct.bodybytes += WRW_Write(sp->wrk,
 			    eb->verbatim.b, Tlen(eb->verbatim));
 			if (sp->http->protover >= 1.1)
-				(void)WRK_Write(sp->wrk, "\r\n", -1);
+				(void)WRW_Write(sp->wrk, "\r\n", -1);
 		}
 		if (eb->include.b == NULL ||
 		    sp->esis >= params->max_esi_includes)
 			continue;
 
-		/*
-		 * We flush here, because the next transaction is
-		 * quite likely to take some time, so we should get
-		 * as many bits to the client as we can already.
-		 */
-		if (WRK_Flush(sp->wrk))
-			break;
+		if (WRW_FlushRelease(sp->wrk)) {
+			vca_close_session(sp, "remote closed");
+			return;
+		}
 
 		sp->esis++;
 		obj = sp->obj;
@@ -853,11 +850,12 @@
 		assert(sp->step == STP_DONE);
 		sp->esis--;
 		sp->obj = obj;
-
+		WRW_Reserve(sp->wrk, &sp->fd);
 	}
-	assert(sp->wrk->wfd == &sp->fd);
 	if (sp->esis == 0 && sp->http->protover >= 1.1)
-		(void)WRK_Write(sp->wrk, "0\r\n\r\n", -1);
+		(void)WRW_Write(sp->wrk, "0\r\n\r\n", -1);
+	if (WRW_FlushRelease(sp->wrk))
+		vca_close_session(sp, "remote closed");
 }
 
 /*--------------------------------------------------------------------*/



From tfheen at projects.linpro.no  Mon Nov 24 19:08:36 2008
From: tfheen at projects.linpro.no (tfheen at projects.linpro.no)
Date: Mon, 24 Nov 2008 20:08:36 +0100 (CET)
Subject: r3430 - trunk/varnish-cache/lib/libjemalloc
Message-ID: <20081124190836.B3DDC1EC7C2@projects.linpro.no>

Author: tfheen
Date: 2008-11-24 20:08:36 +0100 (Mon, 24 Nov 2008)
New Revision: 3430

Modified:
   trunk/varnish-cache/lib/libjemalloc/jemalloc_linux.c
Log:
Include relevant bit of ansidecl.h directly

Avoids build-dependency on binutils-dev


Modified: trunk/varnish-cache/lib/libjemalloc/jemalloc_linux.c
===================================================================
--- trunk/varnish-cache/lib/libjemalloc/jemalloc_linux.c	2008-11-24 17:47:21 UTC (rev 3429)
+++ trunk/varnish-cache/lib/libjemalloc/jemalloc_linux.c	2008-11-24 19:08:36 UTC (rev 3430)
@@ -197,10 +197,20 @@
 #include <unistd.h>
 #include <fcntl.h>
 #include <pthread.h>
-#include <ansidecl.h>
 
 #include "rb.h"
 
+/* Prevent -Werror to complain about unused parameters when compiling
+   with non-ancient GCC.  Added directly here instead of grabbed from
+   ansidecl.h to save a build dependency on binutils-dev */
+#if __GNUC__ >= 3
+#ifndef ATTRIBUTE_UNUSED
+#define ATTRIBUTE_UNUSED __attribute__ ((__unused__))
+#endif /* ATTRIBUTE_UNUSED */
+#else
+#define ATTRIBUTE_UNUSED
+#endif
+
 #ifdef MALLOC_DEBUG
    /* Disable inlining to make debugging easier. */
 #  define inline



From phk at projects.linpro.no  Mon Nov 24 19:44:36 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 24 Nov 2008 20:44:36 +0100 (CET)
Subject: r3431 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081124194436.696551EC1FE@projects.linpro.no>

Author: phk
Date: 2008-11-24 20:44:36 +0100 (Mon, 24 Nov 2008)
New Revision: 3431

Modified:
   trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c
   trunk/varnish-cache/bin/varnishd/mgt_param.c
Log:
Control the ESI parsing debug records with the esi_syntax bitmap parameter.



Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c	2008-11-24 19:08:36 UTC (rev 3430)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c	2008-11-24 19:44:36 UTC (rev 3431)
@@ -158,8 +158,9 @@
 	VTAILQ_INSERT_TAIL(&ew->sp->obj->esibits, ew->eb, list);
 	ew->eb->verbatim = ew->dst;
 	sprintf(ew->eb->chunk_length, "%x\r\n", Tlen(ew->dst));
-	VSL(SLT_Debug, ew->sp->fd, "AddBit: %d <%.*s>",
-	    Tlen(ew->dst), Tlen(ew->dst), ew->dst.b);
+	if (params->esi_syntax & 0x4)
+		VSL(SLT_Debug, ew->sp->fd, "AddBit: %d <%.*s>",
+		    Tlen(ew->dst), Tlen(ew->dst), ew->dst.b);
 	return(ew->eb);
 }
 
@@ -172,8 +173,9 @@
 esi_addverbatim(struct esi_work *ew)
 {
 
-	VSL(SLT_Debug, ew->sp->fd, "AddVer: %d <%.*s>",
-	    Tlen(ew->o), Tlen(ew->o), ew->o.b);
+	if (params->esi_syntax & 0x4)
+		VSL(SLT_Debug, ew->sp->fd, "AddVer: %d <%.*s>",
+		    Tlen(ew->o), Tlen(ew->o), ew->o.b);
 	if (ew->o.b != ew->dst.e)
 		memmove(ew->dst.e, ew->o.b, Tlen(ew->o));
 	ew->dst.e += Tlen(ew->o);
@@ -283,9 +285,9 @@
 	VSL(SLT_Debug, 0, "Incl \"%.*s\"", t.e - t.b, t.b);
 	eb = esi_addbit(ew);
 	while (esi_attrib(ew, &t, &tag, &val) == 1) {
-		VSL(SLT_Debug, 0, "<%.*s> -> <%.*s>",
-		    tag.e - tag.b, tag.b,
-		    val.e - val.b, val.b);
+		if (params->esi_syntax & 0x4)
+			VSL(SLT_Debug, 0, "<%.*s> -> <%.*s>",
+			    tag.e - tag.b, tag.b, val.e - val.b, val.b);
 		if (Tlen(tag) != 3 || memcmp(tag.b, "src", 3))
 			continue;
 		if (Tlen(val) == 0) {
@@ -496,8 +498,9 @@
 			r = p + 1;
 		}
 
-		VSL(SLT_Debug, ew->sp->fd, "Element: clos=%d [%.*s]",
-		    celem, q - r, r);
+		if (params->esi_syntax & 0x4)
+			VSL(SLT_Debug, ew->sp->fd, "Element: clos=%d [%.*s]",
+			    celem, q - r, r);
 
 		if (r + 9 < q && !memcmp(r, "esi:remove", 10)) {
 
@@ -612,8 +615,9 @@
 {
 	char *p;
 
-	VSL(SLT_Debug, ew->sp->fd, "Parse: %d <%.*s>",
-	    Tlen(ew->t), Tlen(ew->t), ew->t.b);
+	if (params->esi_syntax & 0x4)
+		VSL(SLT_Debug, ew->sp->fd, "Parse: %d <%.*s>",
+		    Tlen(ew->t), Tlen(ew->t), ew->t.b);
 	p = esi_parse2(ew);
 	assert(ew->o.b >= ew->t.b);
 	assert(ew->o.e <= ew->t.e);

Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-24 19:08:36 UTC (rev 3430)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-24 19:44:36 UTC (rev 3431)
@@ -764,6 +764,7 @@
 		"Bitmap controlling ESI parsing code:\n"
 		"  0x00000001 - Don't check if it looks like XML\n"
 		"  0x00000002 - Ignore non-esi elements\n"
+		"  0x00000004 - Emit parsing debug records\n"
 		"Use 0x notation and do the bitor in your head :-)\n",
 		0,
 		"0", "bitmap" },



From phk at projects.linpro.no  Mon Nov 24 20:21:51 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Mon, 24 Nov 2008 21:21:51 +0100 (CET)
Subject: r3432 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081124202151.C00FF1EC7C2@projects.linpro.no>

Author: phk
Date: 2008-11-24 21:21:51 +0100 (Mon, 24 Nov 2008)
New Revision: 3432

Modified:
   trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c
Log:
Log debugs with correct id



Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c	2008-11-24 19:44:36 UTC (rev 3431)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c	2008-11-24 20:21:51 UTC (rev 3432)
@@ -282,11 +282,11 @@
 	unsigned u, v;
 	struct ws *ws;
 
-	VSL(SLT_Debug, 0, "Incl \"%.*s\"", t.e - t.b, t.b);
+	VSL(SLT_Debug, ew->sp->fd, "Incl \"%.*s\"", t.e - t.b, t.b);
 	eb = esi_addbit(ew);
 	while (esi_attrib(ew, &t, &tag, &val) == 1) {
 		if (params->esi_syntax & 0x4)
-			VSL(SLT_Debug, 0, "<%.*s> -> <%.*s>",
+			VSL(SLT_Debug, ew->sp->fd, "<%.*s> -> <%.*s>",
 			    tag.e - tag.b, tag.b, val.e - val.b, val.b);
 		if (Tlen(tag) != 3 || memcmp(tag.b, "src", 3))
 			continue;



From phk at projects.linpro.no  Tue Nov 25 08:37:34 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 25 Nov 2008 09:37:34 +0100 (CET)
Subject: r3433 - in trunk/varnish-cache/bin: varnishd varnishtest/tests
Message-ID: <20081125083734.C5BBE1EC220@projects.linpro.no>

Author: phk
Date: 2008-11-25 09:37:34 +0100 (Tue, 25 Nov 2008)
New Revision: 3433

Added:
   trunk/varnish-cache/bin/varnishtest/tests/r00386.vtc
Modified:
   trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c
Log:
When we receive an If-Modified-Since on an ESI object, do not process the conditional
for the child object and pretend to send a 304 reply for them, if we have decided to
deliver the main object.

Fixes #386


Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c	2008-11-24 20:21:51 UTC (rev 3432)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt_esi.c	2008-11-25 08:37:34 UTC (rev 3433)
@@ -841,7 +841,14 @@
 		sp->director = NULL;
 		sp->step = STP_RECV;
 		http_ForceGet(sp->http);
+
+		/* Don't do conditionals */
+		sp->http->conds = 0;
+		http_Unset(sp->http, H_If_Modified_Since);
+
+		/* Client content already taken care of */
 		http_Unset(sp->http, H_Content_Length);
+
 		while (1) {
 			CNT_Session(sp);
 			if (sp->step == STP_DONE)

Added: trunk/varnish-cache/bin/varnishtest/tests/r00386.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00386.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00386.vtc	2008-11-25 08:37:34 UTC (rev 3433)
@@ -0,0 +1,29 @@
+# $Id$
+
+test "#386, failure to insert include"
+
+server s1 {
+	rxreq
+	expect req.url == "/body"
+	txresp -hdr "Last-Modified: Tue, 25 Nov 2008 00:00:00 GMT" -body "BODY"
+	rxreq 
+	expect req.url == "/"
+	txresp -body {<X><esi:include src="/body"/></X>}
+} -start
+
+varnish v1 -arg "-p diag_bitmap=0x20" -vcl+backend {
+	sub vcl_fetch {
+		if (req.url == "/") {
+			esi;
+		}
+	}
+} -start
+
+client c1 {
+	txreq -url /body
+	rxresp
+	expect resp.bodylen == 4
+	txreq -url / -hdr "If-Modified-Since:  Tue, 25 Nov 2008 00:00:00 GMT"
+	rxresp
+	expect resp.bodylen == 11
+} -start



From phk at projects.linpro.no  Tue Nov 25 10:20:16 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 25 Nov 2008 11:20:16 +0100 (CET)
Subject: r3434 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081125102016.444751ED248@projects.linpro.no>

Author: phk
Date: 2008-11-25 11:20:16 +0100 (Tue, 25 Nov 2008)
New Revision: 3434

Modified:
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_hash.c
   trunk/varnish-cache/bin/varnishd/cache_vrt.c
   trunk/varnish-cache/bin/varnishd/hash_slinger.h
Log:
Isolate some hash-string building nastyness in cache_hash.c



Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2008-11-25 08:37:34 UTC (rev 3433)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2008-11-25 10:20:16 UTC (rev 3434)
@@ -586,28 +586,12 @@
 cnt_lookup(struct sess *sp)
 {
 	struct object *o;
-	char *p;
-	uintptr_t u;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
 	if (sp->obj == NULL) {
-
-		/* Allocate the pointers we need, align properly. */
-		sp->lhashptr = 1;	/* space for NUL */
-		sp->ihashptr = 0;
-		sp->nhashptr = sp->vcl->nhashcount * 2;
-		p = WS_Alloc(sp->http->ws,
-		    sizeof(const char *) * (sp->nhashptr + 1));
-		XXXAN(p);
-		/* Align pointer properly (?) */
-		u = (uintptr_t)p;
-		u &= sizeof(const char *) - 1;
-		if (u)
-			p += sizeof(const char *) - u;
-		sp->hashptr = (void*)p;
-
+		HSH_Prepare(sp, sp->vcl->nhashcount);
 		VCL_hash_method(sp);
 		assert(sp->handling == VCL_RET_HASH);
 	}

Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c	2008-11-25 08:37:34 UTC (rev 3433)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c	2008-11-25 10:20:16 UTC (rev 3434)
@@ -201,6 +201,48 @@
 	assert(b <= oh->hash + oh->hashlen);
 }
 
+void
+HSH_Prepare(struct sess *sp, unsigned nhashcount)
+{
+	char *p;
+	unsigned u;
+
+	/* Allocate the pointers we need, align properly. */
+	sp->lhashptr = 1;       /* space for NUL */
+	sp->ihashptr = 0;
+	sp->nhashptr = nhashcount * 2;
+	p = WS_Alloc(sp->http->ws, sizeof(const char *) * (sp->nhashptr + 1));
+	XXXAN(p);
+	/* Align pointer properly (?) */
+	u = (uintptr_t)p;
+	u &= sizeof(const char *) - 1;
+	if (u)
+		p += sizeof(const char *) - u;
+	sp->hashptr = (void*)p;
+}
+
+void
+HSH_AddString(struct sess *sp, const char *str)
+{
+	int l;
+
+	if (str == NULL)
+		str = "";
+	l = strlen(str);
+
+	/*
+	* XXX: handle this by bouncing sp->vcl->nhashcount when it fails
+	* XXX: and dispose of this request either by reallocating the
+	* XXX: hashptr (if possible) or restarting/error the request
+	*/
+	xxxassert(sp->ihashptr < sp->nhashptr);
+
+	sp->hashptr[sp->ihashptr] = str;
+	sp->hashptr[sp->ihashptr + 1] = str + l;
+	sp->ihashptr += 2;
+	sp->lhashptr += l + 1;
+}
+
 struct object *
 HSH_Lookup(struct sess *sp)
 {

Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt.c	2008-11-25 08:37:34 UTC (rev 3433)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt.c	2008-11-25 10:20:16 UTC (rev 3434)
@@ -585,23 +585,8 @@
 void
 VRT_l_req_hash(struct sess *sp, const char *str)
 {
-	int l;
 
-	if (str == NULL)
-		str = "";
-	l = strlen(str);
-
-	/*
-	 * XXX: handle this by bouncing sp->vcl->nhashcount when it fails
-	 * XXX: and dispose of this request either by reallocating the
-	 * XXX: hashptr (if possible) or restarting/error the request
-	 */
-	xxxassert(sp->ihashptr < sp->nhashptr);
-
-	sp->hashptr[sp->ihashptr] = str;
-	sp->hashptr[sp->ihashptr + 1] = str + l;
-	sp->ihashptr += 2;
-	sp->lhashptr += l + 1;
+	HSH_AddString(sp, str);
 }
 
 /*--------------------------------------------------------------------*/

Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_slinger.h	2008-11-25 08:37:34 UTC (rev 3433)
+++ trunk/varnish-cache/bin/varnishd/hash_slinger.h	2008-11-25 10:20:16 UTC (rev 3434)
@@ -59,6 +59,8 @@
 void HSH_Deref(struct object *o);
 double HSH_Grace(double g);
 void HSH_Init(void);
+void HSH_AddString(struct sess *sp, const char *str);
+void HSH_Prepare(struct sess *sp, unsigned hashcount);
 
 
 #ifdef VARNISH_CACHE_CHILD



From phk at projects.linpro.no  Tue Nov 25 11:07:20 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 25 Nov 2008 12:07:20 +0100 (CET)
Subject: r3435 - in trunk/varnish-cache: include lib/libvarnish
Message-ID: <20081125110720.2A4081EC7C2@projects.linpro.no>

Author: phk
Date: 2008-11-25 12:07:19 +0100 (Tue, 25 Nov 2008)
New Revision: 3435

Added:
   trunk/varnish-cache/include/vsha256.h
   trunk/varnish-cache/lib/libvarnish/vsha256.c
Log:
Add SHA256 hashing code.

This code was written by Colin Percival for the FreeBSD project.



Added: trunk/varnish-cache/include/vsha256.h
===================================================================
--- trunk/varnish-cache/include/vsha256.h	                        (rev 0)
+++ trunk/varnish-cache/include/vsha256.h	2008-11-25 11:07:19 UTC (rev 3435)
@@ -0,0 +1,50 @@
+/*-
+ * Copyright 2005 Colin Percival
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD: head/lib/libmd/sha256.h 154479 2006-01-17 15:35:57Z phk $
+ */
+
+#ifndef _SHA256_H_
+#define _SHA256_H_
+
+#include <sys/types.h>
+
+typedef struct SHA256Context {
+	uint32_t state[8];
+	uint32_t count[2];
+	unsigned char buf[64];
+} SHA256_CTX;
+
+__BEGIN_DECLS
+void	SHA256_Init(SHA256_CTX *);
+void	SHA256_Update(SHA256_CTX *, const void *, size_t);
+void	SHA256_Final(unsigned char [32], SHA256_CTX *);
+char   *SHA256_End(SHA256_CTX *, char *);
+char   *SHA256_File(const char *, char *);
+char   *SHA256_FileChunk(const char *, char *, off_t, off_t);
+char   *SHA256_Data(const void *, unsigned int, char *);
+__END_DECLS
+
+#endif /* !_SHA256_H_ */

Added: trunk/varnish-cache/lib/libvarnish/vsha256.c
===================================================================
--- trunk/varnish-cache/lib/libvarnish/vsha256.c	                        (rev 0)
+++ trunk/varnish-cache/lib/libvarnish/vsha256.c	2008-11-25 11:07:19 UTC (rev 3435)
@@ -0,0 +1,300 @@
+/*-
+ * Copyright 2005 Colin Percival
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD: head/lib/libmd/sha256c.c 154479 2006-01-17 15:35:57Z phk $");
+
+#include <sys/endian.h>
+#include <sys/types.h>
+
+#include <string.h>
+
+#include "vsha256.h"
+
+#if BYTE_ORDER == BIG_ENDIAN
+
+/* Copy a vector of big-endian uint32_t into a vector of bytes */
+#define be32enc_vect(dst, src, len)	\
+	memcpy((void *)dst, (const void *)src, (size_t)len)
+
+/* Copy a vector of bytes into a vector of big-endian uint32_t */
+#define be32dec_vect(dst, src, len)	\
+	memcpy((void *)dst, (const void *)src, (size_t)len)
+
+#else /* BYTE_ORDER != BIG_ENDIAN */
+
+/*
+ * Encode a length len/4 vector of (uint32_t) into a length len vector of
+ * (unsigned char) in big-endian form.  Assumes len is a multiple of 4.
+ */
+static void
+be32enc_vect(unsigned char *dst, const uint32_t *src, size_t len)
+{
+	size_t i;
+
+	for (i = 0; i < len / 4; i++)
+		be32enc(dst + i * 4, src[i]);
+}
+
+/*
+ * Decode a big-endian length len vector of (unsigned char) into a length
+ * len/4 vector of (uint32_t).  Assumes len is a multiple of 4.
+ */
+static void
+be32dec_vect(uint32_t *dst, const unsigned char *src, size_t len)
+{
+	size_t i;
+
+	for (i = 0; i < len / 4; i++)
+		dst[i] = be32dec(src + i * 4);
+}
+
+#endif /* BYTE_ORDER != BIG_ENDIAN */
+
+/* Elementary functions used by SHA256 */
+#define Ch(x, y, z)	((x & (y ^ z)) ^ z)
+#define Maj(x, y, z)	((x & (y | z)) | (y & z))
+#define SHR(x, n)	(x >> n)
+#define ROTR(x, n)	((x >> n) | (x << (32 - n)))
+#define S0(x)		(ROTR(x, 2) ^ ROTR(x, 13) ^ ROTR(x, 22))
+#define S1(x)		(ROTR(x, 6) ^ ROTR(x, 11) ^ ROTR(x, 25))
+#define s0(x)		(ROTR(x, 7) ^ ROTR(x, 18) ^ SHR(x, 3))
+#define s1(x)		(ROTR(x, 17) ^ ROTR(x, 19) ^ SHR(x, 10))
+
+/* SHA256 round function */
+#define RND(a, b, c, d, e, f, g, h, k)			\
+	t0 = h + S1(e) + Ch(e, f, g) + k;		\
+	t1 = S0(a) + Maj(a, b, c);			\
+	d += t0;					\
+	h  = t0 + t1;
+
+/* Adjusted round function for rotating state */
+#define RNDr(S, W, i, k)			\
+	RND(S[(64 - i) % 8], S[(65 - i) % 8],	\
+	    S[(66 - i) % 8], S[(67 - i) % 8],	\
+	    S[(68 - i) % 8], S[(69 - i) % 8],	\
+	    S[(70 - i) % 8], S[(71 - i) % 8],	\
+	    W[i] + k)
+
+/*
+ * SHA256 block compression function.  The 256-bit state is transformed via
+ * the 512-bit input block to produce a new state.
+ */
+static void
+SHA256_Transform(uint32_t * state, const unsigned char block[64])
+{
+	uint32_t W[64];
+	uint32_t S[8];
+	uint32_t t0, t1;
+	int i;
+
+	/* 1. Prepare message schedule W. */
+	be32dec_vect(W, block, 64);
+	for (i = 16; i < 64; i++)
+		W[i] = s1(W[i - 2]) + W[i - 7] + s0(W[i - 15]) + W[i - 16];
+
+	/* 2. Initialize working variables. */
+	memcpy(S, state, 32);
+
+	/* 3. Mix. */
+	RNDr(S, W, 0, 0x428a2f98);
+	RNDr(S, W, 1, 0x71374491);
+	RNDr(S, W, 2, 0xb5c0fbcf);
+	RNDr(S, W, 3, 0xe9b5dba5);
+	RNDr(S, W, 4, 0x3956c25b);
+	RNDr(S, W, 5, 0x59f111f1);
+	RNDr(S, W, 6, 0x923f82a4);
+	RNDr(S, W, 7, 0xab1c5ed5);
+	RNDr(S, W, 8, 0xd807aa98);
+	RNDr(S, W, 9, 0x12835b01);
+	RNDr(S, W, 10, 0x243185be);
+	RNDr(S, W, 11, 0x550c7dc3);
+	RNDr(S, W, 12, 0x72be5d74);
+	RNDr(S, W, 13, 0x80deb1fe);
+	RNDr(S, W, 14, 0x9bdc06a7);
+	RNDr(S, W, 15, 0xc19bf174);
+	RNDr(S, W, 16, 0xe49b69c1);
+	RNDr(S, W, 17, 0xefbe4786);
+	RNDr(S, W, 18, 0x0fc19dc6);
+	RNDr(S, W, 19, 0x240ca1cc);
+	RNDr(S, W, 20, 0x2de92c6f);
+	RNDr(S, W, 21, 0x4a7484aa);
+	RNDr(S, W, 22, 0x5cb0a9dc);
+	RNDr(S, W, 23, 0x76f988da);
+	RNDr(S, W, 24, 0x983e5152);
+	RNDr(S, W, 25, 0xa831c66d);
+	RNDr(S, W, 26, 0xb00327c8);
+	RNDr(S, W, 27, 0xbf597fc7);
+	RNDr(S, W, 28, 0xc6e00bf3);
+	RNDr(S, W, 29, 0xd5a79147);
+	RNDr(S, W, 30, 0x06ca6351);
+	RNDr(S, W, 31, 0x14292967);
+	RNDr(S, W, 32, 0x27b70a85);
+	RNDr(S, W, 33, 0x2e1b2138);
+	RNDr(S, W, 34, 0x4d2c6dfc);
+	RNDr(S, W, 35, 0x53380d13);
+	RNDr(S, W, 36, 0x650a7354);
+	RNDr(S, W, 37, 0x766a0abb);
+	RNDr(S, W, 38, 0x81c2c92e);
+	RNDr(S, W, 39, 0x92722c85);
+	RNDr(S, W, 40, 0xa2bfe8a1);
+	RNDr(S, W, 41, 0xa81a664b);
+	RNDr(S, W, 42, 0xc24b8b70);
+	RNDr(S, W, 43, 0xc76c51a3);
+	RNDr(S, W, 44, 0xd192e819);
+	RNDr(S, W, 45, 0xd6990624);
+	RNDr(S, W, 46, 0xf40e3585);
+	RNDr(S, W, 47, 0x106aa070);
+	RNDr(S, W, 48, 0x19a4c116);
+	RNDr(S, W, 49, 0x1e376c08);
+	RNDr(S, W, 50, 0x2748774c);
+	RNDr(S, W, 51, 0x34b0bcb5);
+	RNDr(S, W, 52, 0x391c0cb3);
+	RNDr(S, W, 53, 0x4ed8aa4a);
+	RNDr(S, W, 54, 0x5b9cca4f);
+	RNDr(S, W, 55, 0x682e6ff3);
+	RNDr(S, W, 56, 0x748f82ee);
+	RNDr(S, W, 57, 0x78a5636f);
+	RNDr(S, W, 58, 0x84c87814);
+	RNDr(S, W, 59, 0x8cc70208);
+	RNDr(S, W, 60, 0x90befffa);
+	RNDr(S, W, 61, 0xa4506ceb);
+	RNDr(S, W, 62, 0xbef9a3f7);
+	RNDr(S, W, 63, 0xc67178f2);
+
+	/* 4. Mix local working variables into global state */
+	for (i = 0; i < 8; i++)
+		state[i] += S[i];
+}
+
+static unsigned char PAD[64] = {
+	0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+};
+
+/* Add padding and terminating bit-count. */
+static void
+SHA256_Pad(SHA256_CTX * ctx)
+{
+	unsigned char len[8];
+	uint32_t r, plen;
+
+	/*
+	 * Convert length to a vector of bytes -- we do this now rather
+	 * than later because the length will change after we pad.
+	 */
+	be32enc_vect(len, ctx->count, 8);
+
+	/* Add 1--64 bytes so that the resulting length is 56 mod 64 */
+	r = (ctx->count[1] >> 3) & 0x3f;
+	plen = (r < 56) ? (56 - r) : (120 - r);
+	SHA256_Update(ctx, PAD, (size_t)plen);
+
+	/* Add the terminating bit-count */
+	SHA256_Update(ctx, len, 8);
+}
+
+/* SHA-256 initialization.  Begins a SHA-256 operation. */
+void
+SHA256_Init(SHA256_CTX * ctx)
+{
+
+	/* Zero bits processed so far */
+	ctx->count[0] = ctx->count[1] = 0;
+
+	/* Magic initialization constants */
+	ctx->state[0] = 0x6A09E667;
+	ctx->state[1] = 0xBB67AE85;
+	ctx->state[2] = 0x3C6EF372;
+	ctx->state[3] = 0xA54FF53A;
+	ctx->state[4] = 0x510E527F;
+	ctx->state[5] = 0x9B05688C;
+	ctx->state[6] = 0x1F83D9AB;
+	ctx->state[7] = 0x5BE0CD19;
+}
+
+/* Add bytes into the hash */
+void
+SHA256_Update(SHA256_CTX * ctx, const void *in, size_t len)
+{
+	uint32_t bitlen[2];
+	uint32_t r;
+	const unsigned char *src = in;
+
+	/* Number of bytes left in the buffer from previous updates */
+	r = (ctx->count[1] >> 3) & 0x3f;
+
+	/* Convert the length into a number of bits */
+	bitlen[1] = ((uint32_t)len) << 3;
+	bitlen[0] = (uint32_t)(len >> 29);
+
+	/* Update number of bits */
+	if ((ctx->count[1] += bitlen[1]) < bitlen[1])
+		ctx->count[0]++;
+	ctx->count[0] += bitlen[0];
+
+	/* Handle the case where we don't need to perform any transforms */
+	if (len < 64 - r) {
+		memcpy(&ctx->buf[r], src, len);
+		return;
+	}
+
+	/* Finish the current block */
+	memcpy(&ctx->buf[r], src, 64 - r);
+	SHA256_Transform(ctx->state, ctx->buf);
+	src += 64 - r;
+	len -= 64 - r;
+
+	/* Perform complete blocks */
+	while (len >= 64) {
+		SHA256_Transform(ctx->state, src);
+		src += 64;
+		len -= 64;
+	}
+
+	/* Copy left over data into buffer */
+	memcpy(ctx->buf, src, len);
+}
+
+/*
+ * SHA-256 finalization.  Pads the input data, exports the hash value,
+ * and clears the context state.
+ */
+void
+SHA256_Final(unsigned char digest[32], SHA256_CTX * ctx)
+{
+
+	/* Add padding */
+	SHA256_Pad(ctx);
+
+	/* Write the hash */
+	be32enc_vect(digest, ctx->state, 32);
+
+	/* Clear the context state */
+	memset((void *)ctx, 0, sizeof(*ctx));
+}



From phk at projects.linpro.no  Tue Nov 25 11:07:39 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 25 Nov 2008 12:07:39 +0100 (CET)
Subject: r3436 - trunk/varnish-cache/lib/libvarnish
Message-ID: <20081125110739.B95EA1EC46B@projects.linpro.no>

Author: phk
Date: 2008-11-25 12:07:39 +0100 (Tue, 25 Nov 2008)
New Revision: 3436

Modified:
   trunk/varnish-cache/lib/libvarnish/Makefile.am
Log:
Hook SHA256 into the build



Modified: trunk/varnish-cache/lib/libvarnish/Makefile.am
===================================================================
--- trunk/varnish-cache/lib/libvarnish/Makefile.am	2008-11-25 11:07:19 UTC (rev 3435)
+++ trunk/varnish-cache/lib/libvarnish/Makefile.am	2008-11-25 11:07:39 UTC (rev 3436)
@@ -24,6 +24,7 @@
 	vlu.c \
 	vpf.c \
 	vsb.c \
+	vsha256.c \
 	vss.c \
 	vtmpfile.c
 



From phk at projects.linpro.no  Tue Nov 25 11:09:39 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 25 Nov 2008 12:09:39 +0100 (CET)
Subject: r3437 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081125110939.29AFC1EC46B@projects.linpro.no>

Author: phk
Date: 2008-11-25 12:09:38 +0100 (Tue, 25 Nov 2008)
New Revision: 3437

Modified:
   trunk/varnish-cache/bin/varnishd/heritage.h
   trunk/varnish-cache/bin/varnishd/mgt_param.c
Log:
Add a parameter to enable SHA256 hashing.

This does not do anything yet.



Modified: trunk/varnish-cache/bin/varnishd/heritage.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/heritage.h	2008-11-25 11:07:39 UTC (rev 3436)
+++ trunk/varnish-cache/bin/varnishd/heritage.h	2008-11-25 11:09:38 UTC (rev 3437)
@@ -174,6 +174,9 @@
 	/* Default grace period */
 	unsigned		default_grace;
 
+	/* Use sha256 hasing */
+	unsigned		hash_sha256;
+
 	/* Log hash string to shm */
 	unsigned		log_hash;
 

Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-25 11:07:39 UTC (rev 3436)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-25 11:09:38 UTC (rev 3437)
@@ -839,6 +839,10 @@
 		"NB: Must be specified with -p to have effect.\n",
 		0,
 		"8192", "bytes" },
+	{ "hash_sha256", tweak_bool, &master.hash_sha256, 0, 0,
+		"Use SHA256 compression of hash-strings",
+		0,
+		"off", "bool" },
 	{ "log_hashstring", tweak_bool, &master.log_hash, 0, 0,
 		"Log the hash string to shared memory log.\n",
 		0,



From phk at projects.linpro.no  Tue Nov 25 11:45:59 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 25 Nov 2008 12:45:59 +0100 (CET)
Subject: r3438 - in trunk/varnish-cache: include lib/libvarnish
Message-ID: <20081125114559.703F41EC200@projects.linpro.no>

Author: phk
Date: 2008-11-25 12:45:59 +0100 (Tue, 25 Nov 2008)
New Revision: 3438

Modified:
   trunk/varnish-cache/include/vsha256.h
   trunk/varnish-cache/lib/libvarnish/vsha256.c
Log:
Neuther the FreeBSD specifics of SHA256 implementation.

In the end, it comes down to lack of POSIX definition of a way to find
out byte-endianess, sigh...



Modified: trunk/varnish-cache/include/vsha256.h
===================================================================
--- trunk/varnish-cache/include/vsha256.h	2008-11-25 11:09:38 UTC (rev 3437)
+++ trunk/varnish-cache/include/vsha256.h	2008-11-25 11:45:59 UTC (rev 3438)
@@ -29,7 +29,7 @@
 #ifndef _SHA256_H_
 #define _SHA256_H_
 
-#include <sys/types.h>
+#include <stdint.h>
 
 typedef struct SHA256Context {
 	uint32_t state[8];
@@ -41,10 +41,6 @@
 void	SHA256_Init(SHA256_CTX *);
 void	SHA256_Update(SHA256_CTX *, const void *, size_t);
 void	SHA256_Final(unsigned char [32], SHA256_CTX *);
-char   *SHA256_End(SHA256_CTX *, char *);
-char   *SHA256_File(const char *, char *);
-char   *SHA256_FileChunk(const char *, char *, off_t, off_t);
-char   *SHA256_Data(const void *, unsigned int, char *);
 __END_DECLS
 
 #endif /* !_SHA256_H_ */

Modified: trunk/varnish-cache/lib/libvarnish/vsha256.c
===================================================================
--- trunk/varnish-cache/lib/libvarnish/vsha256.c	2008-11-25 11:09:38 UTC (rev 3437)
+++ trunk/varnish-cache/lib/libvarnish/vsha256.c	2008-11-25 11:45:59 UTC (rev 3438)
@@ -22,19 +22,17 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
+ * From: $FreeBSD: head/lib/libmd/sha256c.c 154479 2006-01-17 15:35:57Z phk $
  */
 
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD: head/lib/libmd/sha256c.c 154479 2006-01-17 15:35:57Z phk $");
+#include <stdint.h>
 
-#include <sys/endian.h>
-#include <sys/types.h>
-
 #include <string.h>
 
 #include "vsha256.h"
 
-#if BYTE_ORDER == BIG_ENDIAN
+#if defined(BYTE_ORDER) && BYTE_ORDER == BIG_ENDIAN
 
 /* Copy a vector of big-endian uint32_t into a vector of bytes */
 #define be32enc_vect(dst, src, len)	\
@@ -44,8 +42,27 @@
 #define be32dec_vect(dst, src, len)	\
 	memcpy((void *)dst, (const void *)src, (size_t)len)
 
-#else /* BYTE_ORDER != BIG_ENDIAN */
+#else /* BYTE_ORDER != BIG_ENDIAN or in doubt... */
 
+static __inline uint32_t
+mybe32dec(const void *pp)
+{
+        unsigned char const *p = (unsigned char const *)pp;
+
+        return ((p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3]);
+}
+
+static __inline void
+mybe32enc(void *pp, uint32_t u)
+{
+        unsigned char *p = (unsigned char *)pp;
+
+        p[0] = (u >> 24) & 0xff;
+        p[1] = (u >> 16) & 0xff;
+        p[2] = (u >> 8) & 0xff;
+        p[3] = u & 0xff;
+}
+
 /*
  * Encode a length len/4 vector of (uint32_t) into a length len vector of
  * (unsigned char) in big-endian form.  Assumes len is a multiple of 4.
@@ -56,7 +73,7 @@
 	size_t i;
 
 	for (i = 0; i < len / 4; i++)
-		be32enc(dst + i * 4, src[i]);
+		mybe32enc(dst + i * 4, src[i]);
 }
 
 /*
@@ -69,10 +86,10 @@
 	size_t i;
 
 	for (i = 0; i < len / 4; i++)
-		dst[i] = be32dec(src + i * 4);
+		dst[i] = mybe32dec(src + i * 4);
 }
 
-#endif /* BYTE_ORDER != BIG_ENDIAN */
+#endif 
 
 /* Elementary functions used by SHA256 */
 #define Ch(x, y, z)	((x & (y ^ z)) ^ z)



From phk at projects.linpro.no  Tue Nov 25 12:02:10 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 25 Nov 2008 13:02:10 +0100 (CET)
Subject: r3439 - in trunk/varnish-cache: . lib/libvarnish
Message-ID: <20081125120210.D67531ECEFC@projects.linpro.no>

Author: phk
Date: 2008-11-25 13:02:10 +0100 (Tue, 25 Nov 2008)
New Revision: 3439

Modified:
   trunk/varnish-cache/configure.ac
   trunk/varnish-cache/lib/libvarnish/vsha256.c
Log:
Try to get the endianess optimization working, by including an assortment
of possibly relevant headers and only go with the fast path if we have
credible information that this is a big-endian platform.



Modified: trunk/varnish-cache/configure.ac
===================================================================
--- trunk/varnish-cache/configure.ac	2008-11-25 11:45:59 UTC (rev 3438)
+++ trunk/varnish-cache/configure.ac	2008-11-25 12:02:10 UTC (rev 3439)
@@ -70,11 +70,13 @@
 AC_HEADER_STDC
 AC_HEADER_SYS_WAIT
 AC_HEADER_TIME
+AC_CHECK_HEADERS([sys/endian.h])
 AC_CHECK_HEADERS([sys/filio.h])
 AC_CHECK_HEADERS([sys/mount.h])
 AC_CHECK_HEADERS([sys/socket.h])
 AC_CHECK_HEADERS([sys/statvfs.h])
 AC_CHECK_HEADERS([sys/vfs.h])
+AC_CHECK_HEADERS([endian.h])
 AC_CHECK_HEADERS([netinet/in.h])
 AC_CHECK_HEADERS([pthread_np.h])
 AC_CHECK_HEADERS([stddef.h])

Modified: trunk/varnish-cache/lib/libvarnish/vsha256.c
===================================================================
--- trunk/varnish-cache/lib/libvarnish/vsha256.c	2008-11-25 11:45:59 UTC (rev 3438)
+++ trunk/varnish-cache/lib/libvarnish/vsha256.c	2008-11-25 12:02:10 UTC (rev 3439)
@@ -26,10 +26,22 @@
  * From: $FreeBSD: head/lib/libmd/sha256c.c 154479 2006-01-17 15:35:57Z phk $
  */
 
+#include "config.h"
+
 #include <stdint.h>
-
 #include <string.h>
 
+#ifdef HAVE_ENDIAN_H
+#include <endian.h>
+#define BYTE_ORDER	__BYTE_ORDER
+#define BIG_ENDIAN	__BIG_ENDIAN
+#endif
+#ifdef HAVE_SYS_ENDIAN_H
+#include <sys/endian.h>
+#define BYTE_ORDER	_BYTE_ORDER
+#define BIG_ENDIAN	_BIG_ENDIAN
+#endif
+
 #include "vsha256.h"
 
 #if defined(BYTE_ORDER) && BYTE_ORDER == BIG_ENDIAN



From phk at projects.linpro.no  Tue Nov 25 13:39:15 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 25 Nov 2008 14:39:15 +0100 (CET)
Subject: r3440 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081125133915.D98751ECEFC@projects.linpro.no>

Author: phk
Date: 2008-11-25 14:39:15 +0100 (Tue, 25 Nov 2008)
New Revision: 3440

Modified:
   trunk/varnish-cache/bin/varnishd/mgt_param.c
Log:
Sort the parameters alphabetically, it's too hard to find anything right
now.



Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-25 12:02:10 UTC (rev 3439)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-25 13:39:15 UTC (rev 3440)
@@ -77,6 +77,8 @@
 };
 
 static struct params master;
+static int nparspec;
+static struct parspec const ** parspec;
 
 /*--------------------------------------------------------------------*/
 
@@ -402,7 +404,7 @@
 		free(ta);
 	}
 	FreeArgv(av);
-	if (cli->result != CLIS_OK) {
+	if (cli != NULL && cli->result != CLIS_OK) {
 		clean_listen_sock_head(&lsh);
 		return;
 	}
@@ -492,7 +494,7 @@
  * change its default value.
  * XXX: we should generate the relevant section of varnishd.1 from here.
  */
-static const struct parspec parspec[] = {
+static const struct parspec input_parspec[] = {
 	{ "user", tweak_user, NULL, 0, 0,
 		"The unprivileged user to run as.  Setting this will "
 		"also set \"group\" to the specified user's primary group.",
@@ -915,6 +917,7 @@
 void
 mcf_param_show(struct cli *cli, const char * const *av, void *priv)
 {
+	int i;
 	const struct parspec *pp;
 	int lfmt;
 
@@ -923,7 +926,8 @@
 		lfmt = 0;
 	else
 		lfmt = 1;
-	for (pp = parspec; pp->name != NULL; pp++) {
+	for (i = 0; i < nparspec; i++) {
+		pp = parspec[i];
 		if (av[2] != NULL && !lfmt && strcmp(pp->name, av[2]))
 			continue;
 		cli_out(cli, "%-*s ", margin, pp->name);
@@ -977,9 +981,11 @@
 void
 MCF_ParamSet(struct cli *cli, const char *param, const char *val)
 {
+	int i;
 	const struct parspec *pp;
 
-	for (pp = parspec; pp->name != NULL; pp++) {
+	for (i = 0; i < nparspec; i++) {
+		pp = parspec[i];
 		if (!strcmp(pp->name, param)) {
 			pp->func(cli, pp, val);
 			if (cli->result != CLIS_OK) {
@@ -1009,20 +1015,69 @@
 	MCF_ParamSet(cli, av[2], av[3]);
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Add a group of parameters to the global set and sort by name.
+ */
 
-void
-MCF_ParamInit(struct cli *cli)
+static int
+parspec_cmp(const void *a, const void *b)
 {
+	struct parspec * const * pa = a;
+	struct parspec * const * pb = b;
+	return (strcmp((*pa)->name, (*pb)->name));
+}
+
+static void
+MCF_AddParams(const struct parspec *ps)
+{
 	const struct parspec *pp;
+	int n;
 
-	for (pp = parspec; pp->name != NULL; pp++) {
-		cli_out(cli, "Set Default for %s = %s\n", pp->name, pp->def);
+	n = 0;
+	for (pp = ps; pp->name != NULL; pp++) {
 		if (strlen(pp->name) + 1 > margin)
 			margin = strlen(pp->name) + 1;
+		n++;
+	}
+	parspec = realloc(parspec, (nparspec + n + 1) * sizeof *parspec);
+	for (pp = ps; pp->name != NULL; pp++)
+		parspec[nparspec++] = pp;
+	parspec[nparspec] = NULL;
+	qsort (parspec, nparspec, sizeof parspec[0], parspec_cmp);
+}
+
+/*--------------------------------------------------------------------
+ * Set defaults for all parameters
+ */
+
+static void
+MCF_SetDefaults(struct cli *cli)
+{
+	const struct parspec *pp;
+	int i;
+
+	for (i = 0; i < nparspec; i++) {
+		pp = parspec[i];
+		if (cli != NULL)
+			cli_out(cli,
+			    "Set Default for %s = %s\n", pp->name, pp->def);
 		pp->func(cli, pp, pp->def);
-		if (cli->result != CLIS_OK)
+		if (cli != NULL && cli->result != CLIS_OK)
 			return;
 	}
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+MCF_ParamInit(struct cli *cli)
+{
+
+	MCF_AddParams(input_parspec);
+
+	/* XXX: We do this twice, to get past any interdependencies */
+	MCF_SetDefaults(NULL);
+	MCF_SetDefaults(cli);
+
 	params = &master;
 }



From phk at projects.linpro.no  Tue Nov 25 13:42:40 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 25 Nov 2008 14:42:40 +0100 (CET)
Subject: r3441 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081125134240.A15F61EC200@projects.linpro.no>

Author: phk
Date: 2008-11-25 14:42:40 +0100 (Tue, 25 Nov 2008)
New Revision: 3441

Added:
   trunk/varnish-cache/bin/varnishd/param.h
Modified:
   trunk/varnish-cache/bin/varnishd/mgt_param.c
Log:
Move parameter declaration into a .h file of its own.



Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-25 13:39:15 UTC (rev 3440)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-25 13:42:40 UTC (rev 3441)
@@ -50,35 +50,15 @@
 #include "mgt_cli.h"
 
 #include "heritage.h"
+#include "param.h"
 
 #include "vss.h"
 
 #define MAGIC_INIT_STRING	"\001"
-
-struct parspec;
-static int margin;
-
-typedef void tweak_t(struct cli *, const struct parspec *, const char *arg);
-
-struct parspec {
-	const char	*name;
-	tweak_t		*func;
-	volatile void	*priv;
-	unsigned	umin;
-	unsigned	umax;
-	const char	*descr;
-	int		 flags;
-#define DELAYED_EFFECT 1
-#define EXPERIMENTAL   2
-#define MUST_RESTART   4
-#define MUST_RELOAD    8
-	const char	*def;
-	const char	*units;
-};
-
 static struct params master;
 static int nparspec;
 static struct parspec const ** parspec;
+static int margin;
 
 /*--------------------------------------------------------------------*/
 

Added: trunk/varnish-cache/bin/varnishd/param.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/param.h	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishd/param.h	2008-11-25 13:42:40 UTC (rev 3441)
@@ -0,0 +1,50 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2008 Linpro AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $Id: mgt_param.c 3440 2008-11-25 13:39:15Z phk $
+ */
+
+struct parspec;
+
+typedef void tweak_t(struct cli *, const struct parspec *, const char *arg);
+
+struct parspec {
+	const char	*name;
+	tweak_t		*func;
+	volatile void	*priv;
+	unsigned	umin;
+	unsigned	umax;
+	const char	*descr;
+	int		 flags;
+#define DELAYED_EFFECT 1
+#define EXPERIMENTAL   2
+#define MUST_RESTART   4
+#define MUST_RELOAD    8
+	const char	*def;
+	const char	*units;
+};



From phk at projects.linpro.no  Tue Nov 25 13:44:53 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 25 Nov 2008 14:44:53 +0100 (CET)
Subject: r3442 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081125134453.364071ECEFC@projects.linpro.no>

Author: phk
Date: 2008-11-25 14:44:52 +0100 (Tue, 25 Nov 2008)
New Revision: 3442

Added:
   trunk/varnish-cache/bin/varnishd/vparam.h
Removed:
   trunk/varnish-cache/bin/varnishd/param.h
Modified:
   trunk/varnish-cache/bin/varnishd/mgt_param.c
Log:
Name that file "vparam.h" instead of "param.h" which is just too
generic for comfort.



Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-25 13:42:40 UTC (rev 3441)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-25 13:44:52 UTC (rev 3442)
@@ -50,7 +50,7 @@
 #include "mgt_cli.h"
 
 #include "heritage.h"
-#include "param.h"
+#include "vparam.h"
 
 #include "vss.h"
 

Deleted: trunk/varnish-cache/bin/varnishd/param.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/param.h	2008-11-25 13:42:40 UTC (rev 3441)
+++ trunk/varnish-cache/bin/varnishd/param.h	2008-11-25 13:44:52 UTC (rev 3442)
@@ -1,50 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2008 Linpro AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $Id: mgt_param.c 3440 2008-11-25 13:39:15Z phk $
- */
-
-struct parspec;
-
-typedef void tweak_t(struct cli *, const struct parspec *, const char *arg);
-
-struct parspec {
-	const char	*name;
-	tweak_t		*func;
-	volatile void	*priv;
-	unsigned	umin;
-	unsigned	umax;
-	const char	*descr;
-	int		 flags;
-#define DELAYED_EFFECT 1
-#define EXPERIMENTAL   2
-#define MUST_RESTART   4
-#define MUST_RELOAD    8
-	const char	*def;
-	const char	*units;
-};

Copied: trunk/varnish-cache/bin/varnishd/vparam.h (from rev 3441, trunk/varnish-cache/bin/varnishd/param.h)
===================================================================
--- trunk/varnish-cache/bin/varnishd/vparam.h	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishd/vparam.h	2008-11-25 13:44:52 UTC (rev 3442)
@@ -0,0 +1,50 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2008 Linpro AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $Id: mgt_param.c 3440 2008-11-25 13:39:15Z phk $
+ */
+
+struct parspec;
+
+typedef void tweak_t(struct cli *, const struct parspec *, const char *arg);
+
+struct parspec {
+	const char	*name;
+	tweak_t		*func;
+	volatile void	*priv;
+	unsigned	umin;
+	unsigned	umax;
+	const char	*descr;
+	int		 flags;
+#define DELAYED_EFFECT 1
+#define EXPERIMENTAL   2
+#define MUST_RESTART   4
+#define MUST_RELOAD    8
+	const char	*def;
+	const char	*units;
+};


Property changes on: trunk/varnish-cache/bin/varnishd/vparam.h
___________________________________________________________________
Name: svn:mergeinfo
   + 



From phk at projects.linpro.no  Tue Nov 25 14:09:39 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 25 Nov 2008 15:09:39 +0100 (CET)
Subject: r3443 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081125140939.A8C191EC200@projects.linpro.no>

Author: phk
Date: 2008-11-25 15:09:39 +0100 (Tue, 25 Nov 2008)
New Revision: 3443

Added:
   trunk/varnish-cache/bin/varnishd/mgt_pool.c
Modified:
   trunk/varnish-cache/bin/varnishd/Makefile.am
   trunk/varnish-cache/bin/varnishd/cache_pool.c
   trunk/varnish-cache/bin/varnishd/mgt_param.c
   trunk/varnish-cache/bin/varnishd/vparam.h
Log:
Make it possible to declare paramters in other source files, and move
the thread-pool related params into a new file mgt_pool.c as proof.

The paramters happen in management process context, and should therefore
not end up in cache_* files for namespace and sanity reasons.



Modified: trunk/varnish-cache/bin/varnishd/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishd/Makefile.am	2008-11-25 13:44:52 UTC (rev 3442)
+++ trunk/varnish-cache/bin/varnishd/Makefile.am	2008-11-25 14:09:39 UTC (rev 3443)
@@ -45,6 +45,7 @@
 	mgt_child.c \
 	mgt_cli.c \
 	mgt_param.c \
+	mgt_pool.c \
 	mgt_vcc.c \
 	rfc2616.c \
 	shmlog.c \

Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-25 13:44:52 UTC (rev 3442)
+++ trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-25 14:09:39 UTC (rev 3443)
@@ -629,3 +629,5 @@
 	AZ(pthread_create(&tp, NULL, wrk_herder_thread, NULL));
 	AZ(pthread_detach(tp));
 }
+
+/*--------------------------------------------------------------------*/

Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-25 13:44:52 UTC (rev 3442)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2008-11-25 14:09:39 UTC (rev 3443)
@@ -55,13 +55,26 @@
 #include "vss.h"
 
 #define MAGIC_INIT_STRING	"\001"
-static struct params master;
+struct params master;
 static int nparspec;
 static struct parspec const ** parspec;
 static int margin;
 
 /*--------------------------------------------------------------------*/
 
+static const struct parspec *
+mcf_findpar(const char *name)
+{
+	int i;
+
+	for (i = 0; i < nparspec; i++)
+		if (!strcmp(parspec[i]->name, name)) 
+			return (parspec[i]);
+	return (NULL);
+}
+
+/*--------------------------------------------------------------------*/
+
 static void
 tweak_generic_timeout(struct cli *cli, volatile unsigned *dst, const char *arg)
 {
@@ -99,7 +112,7 @@
 
 /*--------------------------------------------------------------------*/
 
-static void
+void
 tweak_timeout(struct cli *cli, const struct parspec *par, const char *arg)
 {
 	volatile unsigned *dest;
@@ -160,7 +173,7 @@
 
 /*--------------------------------------------------------------------*/
 
-static void
+void
 tweak_generic_uint(struct cli *cli, volatile unsigned *dest, const char *arg,
     unsigned min, unsigned max)
 {
@@ -191,7 +204,7 @@
 
 /*--------------------------------------------------------------------*/
 
-static void
+void
 tweak_uint(struct cli *cli, const struct parspec *par, const char *arg)
 {
 	volatile unsigned *dest;
@@ -283,29 +296,6 @@
 /*--------------------------------------------------------------------*/
 
 static void
-tweak_thread_pool_min(struct cli *cli, const struct parspec *par,
-    const char *arg)
-{
-
-	tweak_generic_uint(cli, &master.wthread_min, arg,
-	    par->umin, master.wthread_max);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-tweak_thread_pool_max(struct cli *cli, const struct parspec *par,
-    const char *arg)
-{
-
-	(void)par;
-	tweak_generic_uint(cli, &master.wthread_max, arg,
-	    master.wthread_min, UINT_MAX);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
 clean_listen_sock_head(struct listen_sock_head *lsh)
 {
 	struct listen_sock *ls, *ls2;
@@ -493,107 +483,6 @@
 		"flush of the cache use \"url.purge .\"",
 		0,
 		"120", "seconds" },
-	{ "thread_pools", tweak_uint, &master.wthread_pools, 1, UINT_MAX,
-		"Number of worker thread pools.\n"
-		"\n"
-		"Increasing number of worker pools decreases lock "
-		"contention.\n"
-		"\n"
-		"Too many pools waste CPU and RAM resources, and more than "
-		"one pool for each CPU is probably detrimal to performance.\n"
-		"\n"
-		"Can be increased on the fly, but decreases require a "
-		"restart to take effect.",
-		EXPERIMENTAL | DELAYED_EFFECT,
-		"2", "pools" },
-	{ "thread_pool_max", tweak_thread_pool_max, NULL, 1, 0,
-		"The maximum number of worker threads in all pools combined.\n"
-		"\n"
-		"Do not set this higher than you have to, since excess "
-		"worker threads soak up RAM and CPU and generally just get "
-		"in the way of getting work done.\n",
-		EXPERIMENTAL | DELAYED_EFFECT,
-		"500", "threads" },
-	{ "thread_pool_min", tweak_thread_pool_min, NULL, 2, 0,
-		"The minimum number of threads in each worker pool.\n"
-		"\n"
-		"Increasing this may help ramp up faster from low load "
-		"situations where threads have expired.\n"
-		"\n"
-		"Minimum is 2 threads.",
-		EXPERIMENTAL | DELAYED_EFFECT,
-		"5", "threads" },
-	{ "thread_pool_timeout", tweak_timeout, &master.wthread_timeout, 1, 0,
-		"Thread idle threshold.\n"
-		"\n"
-		"Threads in excess of thread_pool_min, which have been idle "
-		"for at least this long are candidates for purging.\n"
-		"\n"
-		"Minimum is 1 second.",
-		EXPERIMENTAL | DELAYED_EFFECT,
-		"300", "seconds" },
-	{ "thread_pool_purge_delay",
-		tweak_timeout, &master.wthread_purge_delay, 100, 0,
-		"Wait this long between purging threads.\n"
-		"\n"
-		"This controls the decay of thread pools when idle(-ish).\n"
-		"\n"
-		"Minimum is 100 milliseconds.",
-		EXPERIMENTAL | DELAYED_EFFECT,
-		"1000", "milliseconds" },
-	{ "thread_pool_add_threshold",
-		tweak_uint, &master.wthread_add_threshold, 0, UINT_MAX,
-		"Overflow threshold for worker thread creation.\n"
-		"\n"
-		"Setting this too low, will result in excess worker threads, "
-		"which is generally a bad idea.\n"
-		"\n"
-		"Setting it too high results in insuffient worker threads.\n",
-		EXPERIMENTAL,
-		"2", "requests" },
-	{ "thread_pool_add_delay",
-		tweak_timeout, &master.wthread_add_delay, 0, UINT_MAX,
-		"Wait at least this long between creating threads.\n"
-		"\n"
-		"Setting this too long results in insuffient worker threads.\n"
-		"\n"
-		"Setting this too short increases the risk of worker "
-		"thread pile-up.\n",
-		EXPERIMENTAL,
-		"20", "milliseconds" },
-	{ "thread_pool_fail_delay",
-		tweak_timeout, &master.wthread_fail_delay, 100, UINT_MAX,
-		"Wait at least this long after a failed thread creation "
-		"before trying to create another thread.\n"
-		"\n"
-		"Failure to create a worker thread is often a sign that "
-		" the end is near, because the process is running out of "
-		"RAM resources for thread stacks.\n"
-		"This delay tries to not rush it on needlessly.\n"
-		"\n"
-		"If thread creation failures are a problem, check that "
-		"thread_pool_max is not too high.\n"
-		"\n"
-		"It may also help to increase thread_pool_timeout and "
-		"thread_pool_min, to reduce the rate at which treads are "
-		"destroyed and later recreated.\n",
-		EXPERIMENTAL,
-		"200", "milliseconds" },
-	{ "overflow_max", tweak_uint, &master.overflow_max, 0, UINT_MAX,
-		"Percentage permitted overflow queue length.\n"
-		"\n"
-		"This sets the ratio of queued requests to worker threads, "
-		"above which sessions will be dropped instead of queued.\n",
-		EXPERIMENTAL,
-		"100", "%" },
-	{ "rush_exponent", tweak_uint, &master.rush_exponent, 2, UINT_MAX,
-		"How many parked request we start for each completed "
-		"request on the object.\n"
-		"NB: Even with the implict delay of delivery, "
-		"this parameter controls an exponential increase in "
-		"number of worker threads.  ",
-		EXPERIMENTAL,
-		"3", "requests per request" },
 	{ "sess_workspace", tweak_uint, &master.sess_workspace, 1024, UINT_MAX,
 		"Bytes of HTTP protocol workspace allocated for sessions. "
 		"This space must be big enough for the entire HTTP protocol "
@@ -961,24 +850,21 @@
 void
 MCF_ParamSet(struct cli *cli, const char *param, const char *val)
 {
-	int i;
 	const struct parspec *pp;
 
-	for (i = 0; i < nparspec; i++) {
-		pp = parspec[i];
-		if (!strcmp(pp->name, param)) {
-			pp->func(cli, pp, val);
-			if (cli->result != CLIS_OK) {
-			} else if (child_pid >= 0 && pp->flags & MUST_RESTART) {
-				cli_out(cli, "Change will take effect"
-				    " when child is restarted");
-			} else if (pp->flags & MUST_RELOAD) {
-				cli_out(cli, "Change will take effect"
-				    " when VCL script is reloaded");
-			}
-			MCF_ParamSync();
-			return;
+	pp = mcf_findpar(param);
+	if (pp != NULL) {
+		pp->func(cli, pp, val);
+		if (cli->result != CLIS_OK) {
+		} else if (child_pid >= 0 && pp->flags & MUST_RESTART) {
+			cli_out(cli, "Change will take effect"
+			    " when child is restarted");
+		} else if (pp->flags & MUST_RELOAD) {
+			cli_out(cli, "Change will take effect"
+			    " when VCL script is reloaded");
 		}
+		MCF_ParamSync();
+		return;
 	}
 	cli_result(cli, CLIS_PARAM);
 	cli_out(cli, "Unknown parameter \"%s\".", param);
@@ -1015,6 +901,8 @@
 
 	n = 0;
 	for (pp = ps; pp->name != NULL; pp++) {
+		if (mcf_findpar(pp->name) != NULL)
+			fprintf(stderr, "Duplicate param: %s\n", pp->name);
 		if (strlen(pp->name) + 1 > margin)
 			margin = strlen(pp->name) + 1;
 		n++;
@@ -1054,6 +942,7 @@
 {
 
 	MCF_AddParams(input_parspec);
+	MCF_AddParams(WRK_parspec);
 
 	/* XXX: We do this twice, to get past any interdependencies */
 	MCF_SetDefaults(NULL);

Added: trunk/varnish-cache/bin/varnishd/mgt_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_pool.c	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishd/mgt_pool.c	2008-11-25 14:09:39 UTC (rev 3443)
@@ -0,0 +1,184 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2008 Linpro AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $Id: cache_pool.c 3429 2008-11-24 17:47:21Z phk $
+ *
+ * We maintain a number of worker thread pools, to spread lock contention.
+ *
+ * Pools can be added on the fly, as a means to mitigate lock contention,
+ * but can only be removed again by a restart. (XXX: we could fix that)
+ *
+ * Two threads herd the pools, one eliminates idle threads and aggregates
+ * statistics for all the pools, the other thread creates new threads
+ * on demand, subject to various numerical constraints.
+ *
+ * The algorithm for when to create threads needs to be reactive enough
+ * to handle startup spikes, but sufficiently attenuated to not cause
+ * thread pileups.  This remains subject for improvement.
+ */
+
+#include "config.h"
+#include <limits.h>
+#include <sys/types.h>
+
+#include "cli_priv.h"
+#include "mgt.h"
+
+#include "vparam.h"
+#include "heritage.h"
+
+/*--------------------------------------------------------------------*/
+
+static void
+tweak_thread_pool_min(struct cli *cli, const struct parspec *par,
+    const char *arg)
+{
+
+	tweak_generic_uint(cli, &master.wthread_min, arg,
+	    par->umin, master.wthread_max);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+tweak_thread_pool_max(struct cli *cli, const struct parspec *par,
+    const char *arg)
+{
+
+	(void)par;
+	tweak_generic_uint(cli, &master.wthread_max, arg,
+	    master.wthread_min, UINT_MAX);
+}
+
+/*--------------------------------------------------------------------*/
+
+const struct parspec WRK_parspec[] = {
+	{ "thread_pools", tweak_uint, &master.wthread_pools, 1, UINT_MAX,
+		"Number of worker thread pools.\n"
+		"\n"
+		"Increasing number of worker pools decreases lock "
+		"contention.\n"
+		"\n"
+		"Too many pools waste CPU and RAM resources, and more than "
+		"one pool for each CPU is probably detrimal to performance.\n"
+		"\n"
+		"Can be increased on the fly, but decreases require a "
+		"restart to take effect.",
+		EXPERIMENTAL | DELAYED_EFFECT,
+		"2", "pools" },
+	{ "thread_pool_max", tweak_thread_pool_max, NULL, 1, 0,
+		"The maximum number of worker threads in all pools combined.\n"
+		"\n"
+		"Do not set this higher than you have to, since excess "
+		"worker threads soak up RAM and CPU and generally just get "
+		"in the way of getting work done.\n",
+		EXPERIMENTAL | DELAYED_EFFECT,
+		"500", "threads" },
+	{ "thread_pool_min", tweak_thread_pool_min, NULL, 2, 0,
+		"The minimum number of threads in each worker pool.\n"
+		"\n"
+		"Increasing this may help ramp up faster from low load "
+		"situations where threads have expired.\n"
+		"\n"
+		"Minimum is 2 threads.",
+		EXPERIMENTAL | DELAYED_EFFECT,
+		"5", "threads" },
+	{ "thread_pool_timeout", tweak_timeout, &master.wthread_timeout, 1, 0,
+		"Thread idle threshold.\n"
+		"\n"
+		"Threads in excess of thread_pool_min, which have been idle "
+		"for at least this long are candidates for purging.\n"
+		"\n"
+		"Minimum is 1 second.",
+		EXPERIMENTAL | DELAYED_EFFECT,
+		"300", "seconds" },
+	{ "thread_pool_purge_delay",
+		tweak_timeout, &master.wthread_purge_delay, 100, 0,
+		"Wait this long between purging threads.\n"
+		"\n"
+		"This controls the decay of thread pools when idle(-ish).\n"
+		"\n"
+		"Minimum is 100 milliseconds.",
+		EXPERIMENTAL | DELAYED_EFFECT,
+		"1000", "milliseconds" },
+	{ "thread_pool_add_threshold",
+		tweak_uint, &master.wthread_add_threshold, 0, UINT_MAX,
+		"Overflow threshold for worker thread creation.\n"
+		"\n"
+		"Setting this too low, will result in excess worker threads, "
+		"which is generally a bad idea.\n"
+		"\n"
+		"Setting it too high results in insuffient worker threads.\n",
+		EXPERIMENTAL,
+		"2", "requests" },
+	{ "thread_pool_add_delay",
+		tweak_timeout, &master.wthread_add_delay, 0, UINT_MAX,
+		"Wait at least this long between creating threads.\n"
+		"\n"
+		"Setting this too long results in insuffient worker threads.\n"
+		"\n"
+		"Setting this too short increases the risk of worker "
+		"thread pile-up.\n",
+		EXPERIMENTAL,
+		"20", "milliseconds" },
+	{ "thread_pool_fail_delay",
+		tweak_timeout, &master.wthread_fail_delay, 100, UINT_MAX,
+		"Wait at least this long after a failed thread creation "
+		"before trying to create another thread.\n"
+		"\n"
+		"Failure to create a worker thread is often a sign that "
+		" the end is near, because the process is running out of "
+		"RAM resources for thread stacks.\n"
+		"This delay tries to not rush it on needlessly.\n"
+		"\n"
+		"If thread creation failures are a problem, check that "
+		"thread_pool_max is not too high.\n"
+		"\n"
+		"It may also help to increase thread_pool_timeout and "
+		"thread_pool_min, to reduce the rate at which treads are "
+		"destroyed and later recreated.\n",
+		EXPERIMENTAL,
+		"200", "milliseconds" },
+	{ "overflow_max", tweak_uint, &master.overflow_max, 0, UINT_MAX,
+		"Percentage permitted overflow queue length.\n"
+		"\n"
+		"This sets the ratio of queued requests to worker threads, "
+		"above which sessions will be dropped instead of queued.\n",
+		EXPERIMENTAL,
+		"100", "%" },
+	{ "rush_exponent", tweak_uint, &master.rush_exponent, 2, UINT_MAX,
+		"How many parked request we start for each completed "
+		"request on the object.\n"
+		"NB: Even with the implict delay of delivery, "
+		"this parameter controls an exponential increase in "
+		"number of worker threads.  ",
+		EXPERIMENTAL,
+		"3", "requests per request" },
+	{ NULL, NULL, NULL }
+};
+

Modified: trunk/varnish-cache/bin/varnishd/vparam.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/vparam.h	2008-11-25 13:44:52 UTC (rev 3442)
+++ trunk/varnish-cache/bin/varnishd/vparam.h	2008-11-25 14:09:39 UTC (rev 3443)
@@ -48,3 +48,14 @@
 	const char	*def;
 	const char	*units;
 };
+
+void tweak_generic_uint(struct cli *cli,
+    volatile unsigned *dest, const char *arg, unsigned min, unsigned max);
+void tweak_uint(struct cli *cli, const struct parspec *par, const char *arg);
+void tweak_timeout(struct cli *cli,
+    const struct parspec *par, const char *arg);
+
+extern struct params master;
+
+/* mgt_pool.c */
+extern const struct parspec WRK_parspec[];



From phk at projects.linpro.no  Tue Nov 25 16:04:47 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Tue, 25 Nov 2008 17:04:47 +0100 (CET)
Subject: r3444 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081125160447.63F731ED71D@projects.linpro.no>

Author: phk
Date: 2008-11-25 17:04:47 +0100 (Tue, 25 Nov 2008)
New Revision: 3444

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_hash.c
   trunk/varnish-cache/bin/varnishd/cache_pool.c
Log:
Add code to calculate a SHA256 over the hash string if param hash_sha256
is set.



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2008-11-25 14:09:39 UTC (rev 3443)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2008-11-25 16:04:47 UTC (rev 3444)
@@ -93,6 +93,8 @@
 struct vrt_backend;
 struct cli_proto;
 struct ban;
+struct SHA256Context;
+
 struct lock { void *priv; };		// Opaque
 
 /*--------------------------------------------------------------------*/
@@ -202,6 +204,8 @@
 
 	unsigned char		*wlb, *wlp, *wle;
 	unsigned		wlr;
+
+	struct SHA256Context	*sha256ctx;
 };
 
 /* Work Request for worker thread ------------------------------------*/

Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c	2008-11-25 14:09:39 UTC (rev 3443)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c	2008-11-25 16:04:47 UTC (rev 3444)
@@ -66,6 +66,7 @@
 #include "cache.h"
 #include "stevedore.h"
 #include "hash_slinger.h"
+#include "vsha256.h"
 
 static const struct hash_slinger *hash;
 
@@ -219,6 +220,8 @@
 	if (u)
 		p += sizeof(const char *) - u;
 	sp->hashptr = (void*)p;
+	if (params->hash_sha256)
+		SHA256_Init(sp->wrk->sha256ctx);
 }
 
 void
@@ -241,6 +244,10 @@
 	sp->hashptr[sp->ihashptr + 1] = str + l;
 	sp->ihashptr += 2;
 	sp->lhashptr += l + 1;
+	if (params->hash_sha256) {
+		SHA256_Update(sp->wrk->sha256ctx, str, l);
+		SHA256_Update(sp->wrk->sha256ctx, "#", 1);
+	}
 }
 
 struct object *
@@ -250,6 +257,7 @@
 	struct http *h;
 	struct objhead *oh;
 	struct object *o, *busy_o, *grace_o;
+	unsigned char sha256[32];
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
@@ -257,6 +265,10 @@
 	AN(hash);
 	w = sp->wrk;
 	h = sp->http;
+	if (params->hash_sha256) {
+		SHA256_Final(sha256, sp->wrk->sha256ctx);
+		/* WSP(sp, SLT_Debug, "SHA256: <%.32s>", sha256); */
+	}
 
 	HSH_Prealloc(sp);
 	if (sp->objhead != NULL) {

Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-25 14:09:39 UTC (rev 3443)
+++ trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-25 16:04:47 UTC (rev 3444)
@@ -72,6 +72,7 @@
 #include "cache.h"
 #include "stevedore.h"
 #include "hash_slinger.h"
+#include "vsha256.h"
 
 VTAILQ_HEAD(workerhead, worker);
 
@@ -269,6 +270,7 @@
 	struct worker *w, ww;
 	struct wq *qp;
 	unsigned char wlog[params->shm_workspace];
+	struct SHA256Context sha256;
 
 	THR_SetName("cache-worker");
 	w = &ww;
@@ -278,6 +280,7 @@
 	w->lastused = NAN;
 	w->wlb = w->wlp = wlog;
 	w->wle = wlog + sizeof wlog;
+	w->sha256ctx = &sha256;
 	AZ(pthread_cond_init(&w->cond, NULL));
 
 	VSL(SLT_WorkThread, 0, "%p start", w);



From phk at projects.linpro.no  Wed Nov 26 12:05:44 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Wed, 26 Nov 2008 13:05:44 +0100 (CET)
Subject: r3445 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081126120544.AA6F51ED1C0@projects.linpro.no>

Author: phk
Date: 2008-11-26 13:05:44 +0100 (Wed, 26 Nov 2008)
New Revision: 3445

Modified:
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_pool.c
Log:
Add asserts to find where WRW is leaking in ticket 390



Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2008-11-25 16:04:47 UTC (rev 3444)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2008-11-26 12:05:44 UTC (rev 3445)
@@ -176,6 +176,7 @@
 	sp->restarts = 0;
 
 	RES_WriteObj(sp);
+	AZ(sp->wrk->wfd);
 	HSH_Deref(sp->obj);
 	sp->obj = NULL;
 	sp->step = STP_DONE;
@@ -383,6 +384,7 @@
 	AN(sp->director);
 	AZ(sp->vbe);
 	i = Fetch(sp);
+	AZ(sp->wrk->wfd);
 	AZ(sp->vbe);
 	AN(sp->director);
 
@@ -799,6 +801,7 @@
 	assert(sp->handling == VCL_RET_PIPE);
 
 	PipeSession(sp);
+	AZ(sp->wrk->wfd);
 	sp->step = STP_DONE;
 	return (0);
 }
@@ -1025,6 +1028,7 @@
 		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
 	}
 	WSL_Flush(w, 0);
+	AZ(w->wfd);
 }
 
 /*

Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-25 16:04:47 UTC (rev 3444)
+++ trunk/varnish-cache/bin/varnishd/cache_pool.c	2008-11-26 12:05:44 UTC (rev 3445)
@@ -308,6 +308,8 @@
 		AN(w->wrq->func);
 		w->lastused = NAN;
 		w->wrq->func(w, w->wrq->priv);
+		AZ(w->wfd);
+		assert(w->wlp == w->wlb);
 		w->wrq = NULL;
 		Lck_Lock(&qp->mtx);
 	}



From phk at projects.linpro.no  Wed Nov 26 16:10:19 2008
From: phk at projects.linpro.no (phk at projects.linpro.no)
Date: Wed, 26 Nov 2008 17:10:19 +0100 (CET)
Subject: r3446 - trunk/varnish-cache/bin/varnishd
Message-ID: <20081126161019.AD6D31EC7D3@projects.linpro.no>

Author: phk
Date: 2008-11-26 17:10:19 +0100 (Wed, 26 Nov 2008)
New Revision: 3446

Modified:
   trunk/varnish-cache/bin/varnishd/cache_fetch.c
Log:
Always release WRW, also on error.



Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c	2008-11-26 12:05:44 UTC (rev 3445)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c	2008-11-26 16:10:19 UTC (rev 3446)
@@ -353,13 +353,8 @@
 
 	/* Deal with any message-body the request might have */
 	i = FetchReqBody(sp);
-	if (i > 0) {
+	if (WRW_FlushRelease(w) || i > 0) {
 		VBE_ClosedFd(sp);
-		return (__LINE__);
-	}
-
-	if (WRW_FlushRelease(w)) {
-		VBE_ClosedFd(sp);
 		/* XXX: other cleanup ? */
 		return (__LINE__);
 	}