From phk at varnish-cache.org  Tue Oct  1 08:49:26 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 10:49:26 +0200
Subject: [master] 7c67df5 Polish the RST we generate for params.
Message-ID: <E1VQve2-0004FV-25@project.varnish-software.com>

commit 7c67df5547936ca3f690268f9e46048681493f4f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 24 05:51:34 2013 +0000

    Polish the RST we generate for params.
    
    Add a link-tag for each param for easy cross-referencing using
    	(:ref:`ref_param_$param`)

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index e2af69b..e75191e 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -916,13 +916,17 @@ MCF_DumpRstParam(void)
 {
 	const struct parspec *pp;
 	const char *p, *q;
-	int i;
+	int i, j;
 
 	printf("\n.. The following is the autogenerated "
 	    "output from varnishd -x dumprstparam\n\n");
 	for (i = 0; i < nparspec; i++) {
 		pp = parspecs[i];
+		printf(".. _ref_param_%s:\n\n", pp->name);
 		printf("%s\n", pp->name);
+		for (j = 0; j < strlen(pp->name); j++)
+			printf("~");
+		printf("\n");
 		if (pp->units != NULL && *pp->units != '\0')
 			printf("\t- Units: %s\n", pp->units);
 		printf("\t- Default: %s\n", pp->def);
diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c
index faeff24..1816e1b 100644
--- a/bin/varnishd/mgt/mgt_param_bits.c
+++ b/bin/varnishd/mgt/mgt_param_bits.c
@@ -239,13 +239,13 @@ tweak_feature(struct cli *cli, const struct parspec *par, const char *arg)
 const struct parspec VSL_parspec[] = {
 	{ "vsl_mask", tweak_vsl_mask, NULL, 0, 0,
 		"Mask individual VSL messages from being logged.\n"
-		"\tdefault\tSet default value\n"
+		"\tdefault\tSet default value\n\n"
 		"Use +/- prefixe in front of VSL tag name, to mask/unmask "
 		"individual VSL messages.",
 		0, "default", "" },
 	{ "debug", tweak_debug, NULL, 0, 0,
 		"Enable/Disable various kinds of debugging.\n"
-		"\tnone\t\tDisable all debugging\n"
+		"\tnone\t\tDisable all debugging\n\n"
 		"Use +/- prefix to set/reset individual bits:\n"
 #define DEBUG_BIT(U, l, p, d) "\t" #l "\t" p d "\n"
 #include "tbl/debug_bits.h"
@@ -253,7 +253,7 @@ const struct parspec VSL_parspec[] = {
 		, 0, "none", "" },
 	{ "feature", tweak_feature, NULL, 0, 0,
 		"Enable/Disable various minor features.\n"
-		"\tnone\t\tDisable all features.\n"
+		"\tnone\t\tDisable all features.\n\n"
 		"Use +/- prefix to enable/disable individual feature:\n"
 #define FEATURE_BIT(U, l, p, d, ld) "\t" #l "\t" p d "\n"
 #include "tbl/feature_bits.h"



From phk at varnish-cache.org  Tue Oct  1 08:49:26 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 10:49:26 +0200
Subject: [master] 5986cac Update the param descriptions
Message-ID: <E1VQve2-0004Fa-3v@project.varnish-software.com>

commit 5986cac5e5910320c5deaff7f29ea9a79ed9bc6d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 24 05:52:32 2013 +0000

    Update the param descriptions

diff --git a/doc/sphinx/reference/params.rst b/doc/sphinx/reference/params.rst
index 1ec8f58..d11a63c 100644
--- a/doc/sphinx/reference/params.rst
+++ b/doc/sphinx/reference/params.rst
@@ -1,21 +1,30 @@
 
 .. The following is the autogenerated output from varnishd -x dumprstparam
 
+.. _ref_param_accept_filter:
+
 accept_filter
+~~~~~~~~~~~~~
 	- Units: bool
 	- Default: on
 	- Flags: must_restart
 
 	Enable kernel accept-filters, if supported by the kernel.
 
+.. _ref_param_acceptor_sleep_decay:
+
 acceptor_sleep_decay
+~~~~~~~~~~~~~~~~~~~~
 	- Default: 0.900
 	- Flags: experimental
 
 	If we run out of resources, such as file descriptors or worker threads, the acceptor will sleep between accepts.
 	This parameter (multiplicatively) reduce the sleep duration for each succesfull accept. (ie: 0.9 = reduce by 10%)
 
+.. _ref_param_acceptor_sleep_incr:
+
 acceptor_sleep_incr
+~~~~~~~~~~~~~~~~~~~
 	- Units: s
 	- Default: 0.001
 	- Flags: experimental
@@ -23,7 +32,10 @@ acceptor_sleep_incr
 	If we run out of resources, such as file descriptors or worker threads, the acceptor will sleep between accepts.
 	This parameter control how much longer we sleep, each time we fail to accept a new connection.
 
+.. _ref_param_acceptor_sleep_max:
+
 acceptor_sleep_max
+~~~~~~~~~~~~~~~~~~
 	- Units: s
 	- Default: 0.050
 	- Flags: experimental
@@ -31,44 +43,65 @@ acceptor_sleep_max
 	If we run out of resources, such as file descriptors or worker threads, the acceptor will sleep between accepts.
 	This parameter limits how long it can sleep between attempts to accept new connections.
 
+.. _ref_param_auto_restart:
+
 auto_restart
+~~~~~~~~~~~~
 	- Units: bool
 	- Default: on
 
 	Restart child process automatically if it dies.
 
+.. _ref_param_ban_dups:
+
 ban_dups
+~~~~~~~~
 	- Units: bool
 	- Default: on
 
 	Detect and eliminate duplicate bans.
 
+.. _ref_param_ban_lurker_sleep:
+
 ban_lurker_sleep
+~~~~~~~~~~~~~~~~
 	- Units: s
 	- Default: 0.01
 
 	How long time does the ban lurker thread sleeps between successful attempts to push the last item up the ban  list.  It always sleeps a second when nothing can be done.
 	A value of zero disables the ban lurker.
 
+.. _ref_param_between_bytes_timeout:
+
 between_bytes_timeout
+~~~~~~~~~~~~~~~~~~~~~
 	- Units: s
 	- Default: 60
 
 	Default timeout between bytes when receiving data from backend. We only wait for this many seconds between bytes before giving up. A value of 0 means it will never time out. VCL can override this default value for each backend request and backend request. This parameter does not apply to pipe.
 
+.. _ref_param_busyobj_worker_cache:
+
 busyobj_worker_cache
+~~~~~~~~~~~~~~~~~~~~
 	- Units: bool
 	- Default: off
 
-	Cache free busyobj per worker thread.Disable this if you have very high hitrates and wantto save the memory of one busyobj per worker thread.
+	Cache free busyobj per worker thread. Disable this if you have very high hitrates and want to save the memory of one busyobj per worker thread. 
+
+.. _ref_param_cc_command:
 
 cc_command
-	- Default: exec gcc -std=gnu99 -g -O2 -pthread -fpic -shared -Wl,-x -o %o %s
+~~~~~~~~~~
+	- Default: exec clang -std=gnu99  -Qunused-arguments -D_THREAD_SAFE -pthread -fpic -shared -Wl,-x -o %o %s
 	- Flags: must_reload
 
 	Command used for compiling the C source code to a dlopen(3) loadable object.  Any occurrence of %s in the string will be replaced with the source file name, and %o will be replaced with the output file name.
 
+.. _ref_param_cli_buffer:
+
 cli_buffer
+~~~~~~~~~~
 	- Units: bytes
 	- Default: 8k
 
@@ -76,42 +109,61 @@ cli_buffer
 	You may need to increase this if you have big VCL files and use the vcl.inline CLI command.
 	NB: Must be specified with -p to have effect.
 
+.. _ref_param_cli_limit:
+
 cli_limit
+~~~~~~~~~
 	- Units: bytes
 	- Default: 48k
 
 	Maximum size of CLI response.  If the response exceeds this limit, the reponse code will be 201 instead of 200 and the last line will indicate the truncation.
 
+.. _ref_param_cli_timeout:
+
 cli_timeout
+~~~~~~~~~~~
 	- Units: seconds
 	- Default: 10
 
 	Timeout for the childs replies to CLI requests from the mgt_param.
 
+.. _ref_param_clock_skew:
+
 clock_skew
+~~~~~~~~~~
 	- Units: s
 	- Default: 10
 
 	How much clockskew we are willing to accept between the backend and our own clock.
 
+.. _ref_param_connect_timeout:
+
 connect_timeout
+~~~~~~~~~~~~~~~
 	- Units: s
-	- Default: 0.7
+	- Default: 3.5
 
 	Default connection timeout for backend connections. We only try to connect to the backend for this many seconds before giving up. VCL can override this default value for each backend and backend request.
 
+.. _ref_param_critbit_cooloff:
+
 critbit_cooloff
+~~~~~~~~~~~~~~~
 	- Units: s
 	- Default: 180.0
 	- Flags: wizard
 
 	How long time the critbit hasher keeps deleted objheads on the cooloff list.
 
+.. _ref_param_debug:
+
 debug
+~~~~~
 	- Default: none
 
 	Enable/Disable various kinds of debugging.
 		none		Disable all debugging
+
 	Use +/- prefix to set/reset individual bits::
 
 		req_state	VSL Request state engine
@@ -122,8 +174,12 @@ debug
 		hashedge	Edge cases in Hash
 		vclrel		Rapid VCL release
 		lurker		VSL Ban lurker
+		esi_chop	Chop ESI fetch to bits
+
+.. _ref_param_default_grace:
 
 default_grace
+~~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 10
 	- Flags: delayed
@@ -131,14 +187,20 @@ default_grace
 	Default grace period.  We will deliver an object this long after it has expired, provided another thread is attempting to get a new copy.
 	Objects already cached will not be affected by changes made until they are fetched from the backend again.
 
+.. _ref_param_default_keep:
+
 default_keep
+~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 0
 	- Flags: delayed
 
 	Default keep period.  We will keep a useless object around this long, making it available for conditional backend fetches.  That means that the object will be removed from the cache at the end of ttl+grace+keep.
 
+.. _ref_param_default_ttl:
+
 default_ttl
+~~~~~~~~~~~
 	- Units: seconds
 	- Default: 120
 
@@ -146,37 +208,37 @@ default_ttl
 	Objects already cached will not be affected by changes made until they are fetched from the backend again.
 	To force an immediate effect at the expense of a total flush of the cache use "ban obj.http.date ~ ."
 
-esi_syntax
-	- Units: bitmap
-	- Default: 0
-
-	Bitmap controlling ESI parsing code::
-
-	  0x00000001 - Don't check if it looks like XML
-	  0x00000002 - Ignore non-esi elements
-	  0x00000004 - Emit parsing debug records
-	  0x00000008 - Force-split parser input (debugging)
-
-	Use 0x notation and do the bitor in your head :-)
+.. _ref_param_expiry_sleep:
 
 expiry_sleep
+~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 1
 
 	How long the expiry thread sleeps when there is nothing for it to do.
 
+.. _ref_param_feature:
+
 feature
+~~~~~~~
 	- Default: none
 
 	Enable/Disable various minor features.
 		none		Disable all features.
+
 	Use +/- prefix to enable/disable individual feature::
 
 		short_panic	Short panic message.
 		wait_silo	Wait for persistent silo.
 		no_coredump	No coredumps.
+		esi_ignore_https	Treat HTTPS as HTTP in ESI:includes
+		esi_disable_xml_check	Don't check of body looks like XML
+		esi_ignore_other_elements	Ignore non-esi XML-elements
+
+.. _ref_param_fetch_chunksize:
 
 fetch_chunksize
+~~~~~~~~~~~~~~~
 	- Units: bytes
 	- Default: 128k
 	- Flags: experimental
@@ -184,26 +246,38 @@ fetch_chunksize
 	The default chunksize used by fetcher. This should be bigger than the majority of objects with short TTLs.
 	Internal limits in the storage_file module makes increases above 128kb a dubious idea.
 
+.. _ref_param_fetch_maxchunksize:
+
 fetch_maxchunksize
+~~~~~~~~~~~~~~~~~~
 	- Units: bytes
 	- Default: 256m
 	- Flags: experimental
 
 	The maximum chunksize we attempt to allocate from storage. Making this too large may cause delays and storage fragmentation.
 
+.. _ref_param_first_byte_timeout:
+
 first_byte_timeout
+~~~~~~~~~~~~~~~~~~
 	- Units: s
 	- Default: 60
 
 	Default timeout for receiving first byte from backend. We only wait for this many seconds for the first byte before giving up. A value of 0 means it will never time out. VCL can override this default value for each backend and backend request. This parameter does not apply to pipe.
 
+.. _ref_param_group:
+
 group
+~~~~~
 	- Default: nogroup
 	- Flags: must_restart
 
 	The unprivileged group to run as.
 
+.. _ref_param_gzip_buffer:
+
 gzip_buffer
+~~~~~~~~~~~
 	- Units: bytes
 	- Default: 32k
 	- Flags: experimental
@@ -211,29 +285,40 @@ gzip_buffer
 	Size of malloc buffer used for gzip processing.
 	These buffers are used for in-transit data, for instance gunzip'ed data being sent to a client.Making this space to small results in more overhead, writes to sockets etc, making it too big is probably just a waste of memory.
 
+.. _ref_param_gzip_level:
+
 gzip_level
+~~~~~~~~~~
 	- Default: 6
 
 	Gzip compression level: 0=debug, 1=fast, 9=best
 
+.. _ref_param_gzip_memlevel:
+
 gzip_memlevel
+~~~~~~~~~~~~~
 	- Default: 8
 
 	Gzip memory level 1=slow/least, 9=fast/most compression.
 	Memory impact is 1=1k, 2=2k, ... 9=256k.
 
+.. _ref_param_http_gzip_support:
+
 http_gzip_support
+~~~~~~~~~~~~~~~~~
 	- Units: bool
 	- Default: on
-	- Flags: experimental
 
-	Enable gzip support. When enabled Varnish will compress uncompressed objects before they are stored in the cache. If a client does not support gzip encoding Varnish will uncompress compressed objects on demand. Varnish will also rewrite the Accept-Encoding header of clients indicating support for gzip to::
+	Enable gzip support. When enabled Varnish request compressed objects from the backend and store them compressed. If a client does not support gzip encoding Varnish will uncompress compressed objects on demand. Varnish will also rewrite the Accept-Encoding header of clients indicating support for gzip to::
 
 	  Accept-Encoding: gzip
 
 	Clients that do not support gzip will have their Accept-Encoding header removed. For more information on how gzip is implemented please see the chapter on gzip in the Varnish reference.
 
+.. _ref_param_http_max_hdr:
+
 http_max_hdr
+~~~~~~~~~~~~
 	- Units: header lines
 	- Default: 64
 
@@ -241,39 +326,57 @@ http_max_hdr
 	Cheap, ~20 bytes, in terms of workspace memory.
 	Note that the first line occupies five header lines.
 
+.. _ref_param_http_range_support:
+
 http_range_support
+~~~~~~~~~~~~~~~~~~
 	- Units: bool
 	- Default: on
 
 	Enable support for HTTP Range headers.
 
+.. _ref_param_http_req_hdr_len:
+
 http_req_hdr_len
+~~~~~~~~~~~~~~~~
 	- Units: bytes
 	- Default: 8k
 
 	Maximum length of any HTTP client request header we will allow.  The limit is inclusive its continuation lines.
 
+.. _ref_param_http_req_size:
+
 http_req_size
+~~~~~~~~~~~~~
 	- Units: bytes
 	- Default: 32k
 
 	Maximum number of bytes of HTTP client request we will deal with.  This is a limit on all bytes up to the double blank line which ends the HTTP request.
 	The memory for the request is allocated from the client workspace (param: workspace_client) and this parameter limits how much of that the request is allowed to take up.
 
+.. _ref_param_http_resp_hdr_len:
+
 http_resp_hdr_len
+~~~~~~~~~~~~~~~~~
 	- Units: bytes
 	- Default: 8k
 
 	Maximum length of any HTTP backend response header we will allow.  The limit is inclusive its continuation lines.
 
+.. _ref_param_http_resp_size:
+
 http_resp_size
+~~~~~~~~~~~~~~
 	- Units: bytes
 	- Default: 32k
 
 	Maximum number of bytes of HTTP backend resonse we will deal with.  This is a limit on all bytes up to the double blank line which ends the HTTP request.
 	The memory for the request is allocated from the worker workspace (param: thread_pool_workspace) and this parameter limits how much of that the request is allowed to take up.
 
+.. _ref_param_idle_send_timeout:
+
 idle_send_timeout
+~~~~~~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 60
 	- Flags: delayed
@@ -282,28 +385,40 @@ idle_send_timeout
 	seconds the session is closed.
 	See setsockopt(2) under SO_SNDTIMEO for more information.
 
+.. _ref_param_listen_address:
+
 listen_address
+~~~~~~~~~~~~~~
 	- Default: :80
 	- Flags: must_restart
 
 	Whitespace separated list of network endpoints where Varnish will accept requests.
 	Possible formats: host, host:port, :port
 
+.. _ref_param_listen_depth:
+
 listen_depth
+~~~~~~~~~~~~
 	- Units: connections
 	- Default: 1024
 	- Flags: must_restart
 
 	Listen queue depth.
 
+.. _ref_param_log_local_address:
+
 log_local_address
+~~~~~~~~~~~~~~~~~
 	- Units: bool
 	- Default: on
 
 	Log the local address on the TCP connection in the SessionOpen VSL record.
 	Disabling this saves a getsockname(2) system call per TCP connection.
 
+.. _ref_param_lru_interval:
+
 lru_interval
+~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 2
 	- Flags: experimental
@@ -311,43 +426,64 @@ lru_interval
 	Grace period before object moves on LRU list.
 	Objects are only moved to the front of the LRU list if they have not been moved there already inside this timeout period.  This reduces the amount of lock operations necessary for LRU list access.
 
+.. _ref_param_max_esi_depth:
+
 max_esi_depth
+~~~~~~~~~~~~~
 	- Units: levels
 	- Default: 5
 
 	Maximum depth of esi:include processing.
 
+.. _ref_param_max_restarts:
+
 max_restarts
+~~~~~~~~~~~~
 	- Units: restarts
 	- Default: 4
 
 	Upper limit on how many times a request can restart.
 	Be aware that restarts are likely to cause a hit against the backend, so don't increase thoughtlessly.
 
+.. _ref_param_max_retries:
+
+max_retries
+~~~~~~~~~~~
+	- Units: retries
+	- Default: 4
+
+	Upper limit on how many times a backend fetch can retry.
+
+.. _ref_param_nuke_limit:
+
 nuke_limit
+~~~~~~~~~~
 	- Units: allocations
 	- Default: 50
 	- Flags: experimental
 
 	Maximum number of objects we attempt to nuke in orderto make space for a object body.
 
-obj_readonly
-	- Units: bool
-	- Default: false
-
-	If set, we do not update obj.hits and obj.lastuse to avoid dirtying VM pages associated with cached objects.
+.. _ref_param_pcre_match_limit:
 
 pcre_match_limit
+~~~~~~~~~~~~~~~~
 	- Default: 10000
 
 	The limit for the  number of internal matching function calls in a pcre_exec() execution.
 
+.. _ref_param_pcre_match_limit_recursion:
+
 pcre_match_limit_recursion
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 	- Default: 10000
 
 	The limit for the  number of internal matching function recursions in a pcre_exec() execution.
 
+.. _ref_param_ping_interval:
+
 ping_interval
+~~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 3
 	- Flags: must_restart
@@ -355,13 +491,19 @@ ping_interval
 	Interval between pings from parent to child.
 	Zero will disable pinging entirely, which makes it possible to attach a debugger to the child.
 
+.. _ref_param_pipe_timeout:
+
 pipe_timeout
+~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 60
 
 	Idle timeout for PIPE sessions. If nothing have been received in either direction for this many seconds, the session is closed.
 
+.. _ref_param_pool_req:
+
 pool_req
+~~~~~~~~
 	- Default: 10,100,10
 
 	Parameters for per worker pool request memory pool.
@@ -371,7 +513,10 @@ pool_req
 	   max_pool -- maximum size of free pool.
 	   max_age -- max age of free element.
 
+.. _ref_param_pool_sess:
+
 pool_sess
+~~~~~~~~~
 	- Default: 10,100,10
 
 	Parameters for per worker pool session memory pool.
@@ -381,7 +526,10 @@ pool_sess
 	   max_pool -- maximum size of free pool.
 	   max_age -- max age of free element.
 
+.. _ref_param_pool_vbc:
+
 pool_vbc
+~~~~~~~~
 	- Default: 10,100,10
 
 	Parameters for backend connection memory pool.
@@ -391,7 +539,10 @@ pool_vbc
 	   max_pool -- maximum size of free pool.
 	   max_age -- max age of free element.
 
+.. _ref_param_pool_vbo:
+
 pool_vbo
+~~~~~~~~
 	- Default: 10,100,10
 
 	Parameters for backend object fetch memory pool.
@@ -401,13 +552,19 @@ pool_vbo
 	   max_pool -- maximum size of free pool.
 	   max_age -- max age of free element.
 
+.. _ref_param_prefer_ipv6:
+
 prefer_ipv6
+~~~~~~~~~~~
 	- Units: bool
 	- Default: off
 
 	Prefer IPv6 address when connecting to backends which have both IPv4 and IPv6 addresses.
 
+.. _ref_param_rush_exponent:
+
 rush_exponent
+~~~~~~~~~~~~~
 	- Units: requests per request
 	- Default: 3
 	- Flags: experimental
@@ -415,14 +572,10 @@ rush_exponent
 	How many parked request we start for each completed request on the object.
 	NB: Even with the implict delay of delivery, this parameter controls an exponential increase in number of worker threads.
 
-saintmode_threshold
-	- Units: objects
-	- Default: 10
-	- Flags: experimental
-
-	The maximum number of objects held off by saint mode before no further will be made to the backend until one times out.  A value of 0 disables saintmode.
+.. _ref_param_send_timeout:
 
 send_timeout
+~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 600
 	- Flags: delayed
@@ -431,54 +584,88 @@ send_timeout
 	seconds the session is closed.
 	See setsockopt(2) under SO_SNDTIMEO for more information.
 
+.. _ref_param_session_max:
+
 session_max
+~~~~~~~~~~~
 	- Units: sessions
 	- Default: 100000
 
 	Maximum number of sessions we will allocate from one pool before just dropping connections.
 	This is mostly an anti-DoS measure, and setting it plenty high should not hurt, as long as you have the memory for it.
 
+.. _ref_param_shm_reclen:
+
 shm_reclen
+~~~~~~~~~~
 	- Units: bytes
 	- Default: 255
 
 	Maximum number of bytes in SHM log record.
 	Maximum is 65535 bytes.
 
+.. _ref_param_shortlived:
+
 shortlived
+~~~~~~~~~~
 	- Units: s
 	- Default: 10.0
 
 	Objects created with TTL shorter than this are always put in transient storage.
 
+.. _ref_param_sigsegv_handler:
+
+sigsegv_handler
+~~~~~~~~~~~~~~~
+	- Units: bool
+	- Default: off
+	- Flags: must_restart
+
+	Install a signal handler which tries to dump debug information on segmentation faults.
+
+.. _ref_param_syslog_cli_traffic:
+
 syslog_cli_traffic
+~~~~~~~~~~~~~~~~~~
 	- Units: bool
 	- Default: on
 
 	Log all CLI traffic to syslog(LOG_INFO).
 
+.. _ref_param_tcp_keepalive_intvl:
+
 tcp_keepalive_intvl
+~~~~~~~~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 5
 	- Flags: experimental
 
 	The number of seconds between TCP keep-alive probes. Note that this setting will only take effect when it is less thanthe system default.
 
+.. _ref_param_tcp_keepalive_probes:
+
 tcp_keepalive_probes
+~~~~~~~~~~~~~~~~~~~~
 	- Units: probes
 	- Default: 5
 	- Flags: experimental
 
 	The maximum number of TCP keep-alive probes to send before giving up and killing the connection if no response is obtained from the other end. Note that this setting will only take effect when it is less than the system default.
 
+.. _ref_param_tcp_keepalive_time:
+
 tcp_keepalive_time
+~~~~~~~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 600
 	- Flags: experimental
 
 	The number of seconds a connection needs to be idle before TCP begins sending out keep-alive probes. Note that this setting will only take effect when it is less than the system default.
 
+.. _ref_param_thread_pool_add_delay:
+
 thread_pool_add_delay
+~~~~~~~~~~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 0
 	- Flags: experimental
@@ -490,7 +677,10 @@ thread_pool_add_delay
 
 	Setting this too high results in insuffient worker threads.
 
+.. _ref_param_thread_pool_destroy_delay:
+
 thread_pool_destroy_delay
+~~~~~~~~~~~~~~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 1
 	- Flags: delayed, experimental
@@ -501,7 +691,10 @@ thread_pool_destroy_delay
 
 	Minimum is 0.01 second.
 
+.. _ref_param_thread_pool_fail_delay:
+
 thread_pool_fail_delay
+~~~~~~~~~~~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 0.2
 	- Flags: experimental
@@ -514,7 +707,10 @@ thread_pool_fail_delay
 
 	It may also help to increase thread_pool_timeout and thread_pool_min, to reduce the rate at which treads are destroyed and later recreated.
 
+.. _ref_param_thread_pool_max:
+
 thread_pool_max
+~~~~~~~~~~~~~~~
 	- Units: threads
 	- Default: 5000
 	- Flags: delayed
@@ -525,7 +721,10 @@ thread_pool_max
 
 	Minimum is 10 threads.
 
+.. _ref_param_thread_pool_min:
+
 thread_pool_min
+~~~~~~~~~~~~~~~
 	- Units: threads
 	- Default: 100
 	- Flags: delayed
@@ -536,7 +735,10 @@ thread_pool_min
 
 	Minimum is 10 threads.
 
+.. _ref_param_thread_pool_stack:
+
 thread_pool_stack
+~~~~~~~~~~~~~~~~~
 	- Units: bytes
 	- Default: 48k
 	- Flags: experimental
@@ -545,7 +747,10 @@ thread_pool_stack
 	This is likely rounded up to a multiple of 4k by the kernel.
 	The kernel/OS has a lower limit which will be enforced.
 
+.. _ref_param_thread_pool_timeout:
+
 thread_pool_timeout
+~~~~~~~~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 300
 	- Flags: delayed, experimental
@@ -556,7 +761,10 @@ thread_pool_timeout
 
 	Minimum is 10 seconds.
 
+.. _ref_param_thread_pools:
+
 thread_pools
+~~~~~~~~~~~~
 	- Units: pools
 	- Default: 2
 	- Flags: delayed, experimental
@@ -569,7 +777,10 @@ thread_pools
 
 	Can be increased on the fly, but decreases require a restart to take effect.
 
+.. _ref_param_thread_queue_limit:
+
 thread_queue_limit
+~~~~~~~~~~~~~~~~~~
 	- Default: 20
 	- Flags: experimental
 
@@ -577,7 +788,10 @@ thread_queue_limit
 
 	This sets the number of requests we will queue, waiting for an available thread.  Above this limit sessions will be dropped instead of queued.
 
+.. _ref_param_thread_stats_rate:
+
 thread_stats_rate
+~~~~~~~~~~~~~~~~~
 	- Units: requests
 	- Default: 10
 	- Flags: experimental
@@ -585,14 +799,20 @@ thread_stats_rate
 	Worker threads accumulate statistics, and dump these into the global stats counters if the lock is free when they finish a request.
 	This parameters defines the maximum number of requests a worker thread may handle, before it is forced to dump its accumulated stats into the global counters.
 
+.. _ref_param_timeout_idle:
+
 timeout_idle
+~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 5
 
 	Idle timeout for client connections.
 	A connection is considered idle, until we receive a non-white-space character on it.
 
+.. _ref_param_timeout_linger:
+
 timeout_linger
+~~~~~~~~~~~~~~
 	- Units: seconds
 	- Default: 0.050
 	- Flags: experimental
@@ -601,48 +821,72 @@ timeout_linger
 	When sessions are reused, as much as half of all reuses happen within the first 100 msec of the previous request completing.
 	Setting this too high results in worker threads not doing anything for their keep, setting it too low just means that more sessions take a detour around the waiter.
 
+.. _ref_param_timeout_req:
+
 timeout_req
+~~~~~~~~~~~
 	- Units: seconds
 	- Default: 2
 
 	Max time to receive clients request header, measured from first non-white-space character to double CRNL.
 
+.. _ref_param_user:
+
 user
+~~~~
 	- Default: nobody
 	- Flags: must_restart
 
 	The unprivileged user to run as.
 
+.. _ref_param_vcc_allow_inline_c:
+
 vcc_allow_inline_c
+~~~~~~~~~~~~~~~~~~
 	- Units: bool
-	- Default: on
+	- Default: off
 
 	Allow inline C code in VCL.
 
+.. _ref_param_vcc_err_unref:
+
 vcc_err_unref
+~~~~~~~~~~~~~
 	- Units: bool
 	- Default: on
 
 	Unreferenced VCL objects result in error.
 
+.. _ref_param_vcc_unsafe_path:
+
 vcc_unsafe_path
+~~~~~~~~~~~~~~~
 	- Units: bool
 	- Default: on
 
 	Allow '/' in vmod & include paths.
 	Allow 'import ... from ...'.
 
+.. _ref_param_vcl_dir:
+
 vcl_dir
-	- Default: /tmp/z/v/etc/varnish
+~~~~~~~
+	- Default: /opt/varnish/etc/varnish
 
 	Directory from which relative VCL filenames (vcl.load and include) are opened.
 
+.. _ref_param_vmod_dir:
+
 vmod_dir
-	- Default: /tmp/z/v/lib/varnish/vmods
+~~~~~~~~
+	- Default: /opt/varnish/lib/varnish/vmods
 
 	Directory where VCL modules are to be found.
 
+.. _ref_param_vsl_buffer:
+
 vsl_buffer
+~~~~~~~~~~
 	- Units: bytes
 	- Default: 4k
 
@@ -651,48 +895,70 @@ vsl_buffer
 	Setting this too high costs memory, setting it too low will cause more VSL flushes and likely increase lock-contention on the VSL mutex.
 	Minimum is 1k bytes.
 
+.. _ref_param_vsl_mask:
+
 vsl_mask
+~~~~~~~~
 	- Default: default
 
 	Mask individual VSL messages from being logged.
 		default	Set default value
+
 	Use +/- prefixe in front of VSL tag name, to mask/unmask individual VSL messages.
 
+.. _ref_param_vsl_space:
+
 vsl_space
+~~~~~~~~~
 	- Units: bytes
 	- Default: 80M
 	- Flags: must_restart
 
 	The amount of space to allocate for the VSL fifo buffer in the VSM memory segment.  If you make this too small, varnish{ncsa|log} etc will not be able to keep up.  Making it too large just costs memory resources.
 
+.. _ref_param_vsm_space:
+
 vsm_space
+~~~~~~~~~
 	- Units: bytes
 	- Default: 1M
 	- Flags: must_restart
 
 	The amount of space to allocate for stats counters in the VSM memory segment.  If you make this too small, some counters will be invisible.  Making it too large just costs memory resources.
 
+.. _ref_param_waiter:
+
 waiter
+~~~~~~
 	- Default: platform dependent
 	- Flags: must_restart, wizard
 
 	Select the waiter kernel interface.
 
+.. _ref_param_workspace_backend:
+
 workspace_backend
+~~~~~~~~~~~~~~~~~
 	- Units: bytes
 	- Default: 64k
 	- Flags: delayed
 
 	Bytes of HTTP protocol workspace for backend HTTP req/resp.  If larger than 4k, use a multiple of 4k for VM efficiency.
 
+.. _ref_param_workspace_client:
+
 workspace_client
+~~~~~~~~~~~~~~~~
 	- Units: bytes
 	- Default: 64k
 	- Flags: delayed
 
 	Bytes of HTTP protocol workspace for clients HTTP req/resp.  If larger than 4k, use a multiple of 4k for VM efficiency.
 
+.. _ref_param_workspace_thread:
+
 workspace_thread
+~~~~~~~~~~~~~~~~
 	- Units: bytes
 	- Default: 2048
 	- Flags: delayed



From phk at varnish-cache.org  Tue Oct  1 08:49:26 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 10:49:26 +0200
Subject: [master] 772a8ce Reference parameters in the reference manual directly
Message-ID: <E1VQve2-0004Fj-5Q@project.varnish-software.com>

commit 772a8ce1b8f299f1d2c6dd6fafe1db7ef3ed4b6e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 24 05:53:03 2013 +0000

    Reference parameters in the reference manual directly

diff --git a/doc/sphinx/users-guide/run_security.rst b/doc/sphinx/users-guide/run_security.rst
index 09d6314..4267a93 100644
--- a/doc/sphinx/users-guide/run_security.rst
+++ b/doc/sphinx/users-guide/run_security.rst
@@ -96,19 +96,24 @@ Parameters can be set from the command line, and made "read-only"
 interface.
 
 Pretty much any parameter can be used to totally mess up your
-HTTP service, but a few can do more damage than that::
+HTTP service, but a few can do more damage than that:
 
-	user			-- access to local system via VCL
-	group			-- access to local system via VCL
-	listen_address		-- trojan other service ports (ssh!)
-	cc_command		-- execute arbitrary programs
+:ref:`ref_param_user` and :ref:`ref_param_group`
+	Access to local system via VCL
 
-Furthermore you may want to look at::
+:ref:`ref_param_listen_address`
+	Trojan other TCP sockets, like ssh
 
-	syslog_cli_traffic	-- know what is going on
-	vcc_unsafe_path		-- retrict VCL/VMODS to vcl_dir+vmod_dir
-	vcl_dir			-- VCL include dir
-	vmod_dir		-- VMOD import dir
+:ref:`ref_param_cc_command`
+	Execute arbitrary programs
+
+Furthermore you may want to look at and lock down:
+
+:ref:`ref_param_syslog_cli_traffic`
+	Log all CLI commands to syslog(8), so you know what goes on.
+
+:ref:`ref_param_vcc_unsafe_path`
+	Retrict VCL/VMODS to :ref:`ref_param_vcl_dir` and :ref:`ref_param_vmod_dir`
 
 The CLI interface
 -----------------
@@ -150,7 +155,7 @@ The params mentioned above can restrict VMOD so they can only
 be imported from a designated directory, restricting VCL wranglers
 to a pre-approved subset of VMODs.
 
-If you do that, we belive that your local system cannot be compromised
+If you do that, we believe that your local system cannot be compromised
 from VCL code.
 
 HTTP requests



From phk at varnish-cache.org  Tue Oct  1 08:49:26 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 10:49:26 +0200
Subject: [master] 0cebf05 Limit depth of toc
Message-ID: <E1VQve2-0004Fr-72@project.varnish-software.com>

commit 0cebf05725d800f059fe02d029d57a6dc14393d8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 24 05:56:12 2013 +0000

    Limit depth of toc

diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index 3af59c9..ab27df2 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -5,6 +5,7 @@ The Varnish Reference Manual
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 .. toctree::
+	:maxdepth: 4
 
 	vcl.rst
 	varnish-cli.rst



From phk at varnish-cache.org  Tue Oct  1 08:49:26 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 10:49:26 +0200
Subject: [master] be9f0b6 add a missing va_end()
Message-ID: <E1VQve2-0004Fz-8a@project.varnish-software.com>

commit be9f0b62eede0efc5e0114724a485713f09e6bd0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 08:43:57 2013 +0000

    add a missing va_end()
    
    Spotted by: Coverity

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index d137c19..850e7a3 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -277,6 +277,7 @@ VRT_hashdata(const struct vrt_ctx *ctx, const char *str, ...)
 		HSH_AddString(ctx->req, p);
 		VSLb(ctx->vsl, SLT_Hash, "%s", str);
 	}
+	va_end(ap);
 	/*
 	 * Add a 'field-separator' to make it more difficult to
 	 * manipulate the hash.



From martin at varnish-cache.org  Tue Oct  1 12:48:17 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:17 +0200
Subject: [master] fafd6f2 Add a XXX for a feature to be added
Message-ID: <E1VQzNB-0005eb-LH@project.varnish-software.com>

commit fafd6f2155c0ddb54150d1be688a7edb3fe4080f
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Sep 9 14:29:50 2013 +0200

    Add a XXX for a feature to be added

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index b5d10f8..27d3b87 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -68,6 +68,7 @@ enum VSL_transaction_e {
 struct VSL_transaction {
 	unsigned		level;
 	int32_t			vxid;
+	/* int32_t		vxid_parent; /\* XXX: Implement this *\/ */
 	enum VSL_transaction_e	type;
 	struct VSL_cursor	*c;
 };



From martin at varnish-cache.org  Tue Oct  1 12:48:17 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:17 +0200
Subject: [master] 773e4c1 Require -v argument to logexp
Message-ID: <E1VQzNB-0005eg-ML@project.varnish-software.com>

commit 773e4c1cc55e1b2be4800c5ca12d8cef8f7ddcd3
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Sep 17 14:55:52 2013 +0200

    Require -v argument to logexp

diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c
index a5cbf86..c0a21c4 100644
--- a/bin/varnishtest/vtc_logexp.c
+++ b/bin/varnishtest/vtc_logexp.c
@@ -97,6 +97,7 @@ struct logexp {
 	char				*query;
 
 	struct VSM_data			*vsm;
+	struct vsb			*n_arg;
 	struct VSL_data			*vsl;
 	struct VSLQ			*vslq;
 	pthread_t			tp;
@@ -132,6 +133,8 @@ logexp_delete(struct logexp *le)
 	free(le->name);
 	free(le->query);
 	VSM_Delete(le->vsm);
+	if (le->n_arg)
+		VSB_delete(le->n_arg);
 	FREE_OBJ(le);
 }
 
@@ -308,6 +311,15 @@ logexp_start(struct logexp *le)
 	AZ(le->vsl);
 	AZ(le->vslq);
 
+	if (le->n_arg == NULL) {
+		vtc_log(le->vl, 0, "-v argument not given");
+		return;
+	}
+	if (VSM_n_Arg(le->vsm, VSB_data(le->n_arg)) <= 0) {
+		vtc_log(le->vl, 0, "-v argument error: %s",
+		    VSM_Error(le->vsm));
+		return;
+	}
 	if (VSM_Open(le->vsm)) {
 		vtc_log(le->vl, 0, "VSM_Open: %s", VSM_Error(le->vsm));
 		return;
@@ -453,7 +465,7 @@ cmd_logexp(CMD_ARGS)
 {
 	struct logexp *le, *le2;
 	const char tmpdir[] = "${tmpdir}";
-	struct vsb *vsb, *vsb2;
+	struct vsb *vsb;
 
 	(void)priv;
 	(void)cmd;
@@ -510,19 +522,18 @@ cmd_logexp(CMD_ARGS)
 				vtc_log(le->vl, 0, "Missing -v argument");
 				return;
 			}
+			if (le->n_arg != NULL) {
+				VSB_delete(le->n_arg);
+				le->n_arg = NULL;
+			}
 			vsb = VSB_new_auto();
+			AN(vsb);
 			AZ(VSB_printf(vsb, "%s/%s", tmpdir, av[1]));
 			AZ(VSB_finish(vsb));
-			vsb2 = macro_expand(le->vl, VSB_data(vsb));
+			le->n_arg = macro_expand(le->vl, VSB_data(vsb));
 			VSB_delete(vsb);
-			if (vsb2 == NULL)
-				return;
-			if (VSM_n_Arg(le->vsm, VSB_data(vsb2)) <= 0) {
-				vtc_log(le->vl, 0, "-v argument error: %s",
-				    VSM_Error(le->vsm));
+			if (le->n_arg == NULL)
 				return;
-			}
-			VSB_delete(vsb2);
 			av++;
 			continue;
 		}



From martin at varnish-cache.org  Tue Oct  1 12:48:17 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:17 +0200
Subject: [master] 8539170 Return 1 to VSL_Dispatch on completing the test
 script.
Message-ID: <E1VQzNB-0005eo-NT@project.varnish-software.com>

commit 853917027d2c4197c362325d9960e6f2b0a4b076
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 18 11:06:54 2013 +0200

    Return 1 to VSL_Dispatch on completing the test script.
    
    Return 1 on completing the test script. This will prevent the API from
    reporting any more transactions matching the query, which would
    trigger an assert.

diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c
index c0a21c4..f2170b2 100644
--- a/bin/varnishtest/vtc_logexp.c
+++ b/bin/varnishtest/vtc_logexp.c
@@ -247,7 +247,7 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				logexp_next(le);
 				if (le->test == NULL)
 					/* End of test script */
-					return (0);
+					return (1);
 			}
 			if (skip)
 				le->skip_cnt++;



From martin at varnish-cache.org  Tue Oct  1 12:48:17 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:17 +0200
Subject: [master] 00346d0 Add VSL expression parser and executor
Message-ID: <E1VQzNB-0005ez-Oz@project.varnish-software.com>

commit 00346d0e7a8c2a20a55f092ddf07189445e2528e
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jul 30 15:29:23 2013 +0200

    Add VSL expression parser and executor
    
    This is based on the libvcl parser

diff --git a/bin/varnishtest/tests/l00000.vtc b/bin/varnishtest/tests/l00000.vtc
index 90d9a7a..adf7c8f 100644
--- a/bin/varnishtest/tests/l00000.vtc
+++ b/bin/varnishtest/tests/l00000.vtc
@@ -29,7 +29,7 @@ logexpect l1 -v v1 -g session {
 } -start
 
 # Check with a query (this selects only the backend request)
-logexpect l2 -v v1 -g vxid -q "bereq 1001" {
+logexpect l2 -v v1 -g vxid -q "Begin eq 'bereq 1001'" {
 	expect 0 1002	Begin
 	expect * =	End
 } -start
diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index ac4fabc..375ab29 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -11,6 +11,8 @@ libvarnishapi_la_LDFLAGS = $(AM_LDFLAGS) -version-info 1:0:0
 libvarnishapi_la_SOURCES = \
 	vsm_api.h \
 	vsl_api.h \
+	vxp.h \
+	vxp_tokens.h \
 	\
 	../libvarnish/vas.c \
 	../libvarnish/vav.c \
@@ -31,6 +33,10 @@ libvarnishapi_la_SOURCES = \
 	vsl_query.c \
 	vsl.c \
 	vsc.c \
+	vxp.c \
+	vxp_parse.c \
+	vxp_lexer.c \
+	vxp_fixed_token.c \
 	libvarnishapi.map
 
 libvarnishapi_la_CFLAGS = \
@@ -43,3 +49,31 @@ libvarnishapi_la_LDFLAGS += -Wl,--version-script=$(srcdir)/libvarnishapi.map
 else
 libvarnishapi_la_LDFLAGS += -export-symbols-regex '^V'
 endif
+
+EXTRA_DIST = \
+	generate.py
+
+BUILT_SOURCES = \
+	vxp_fixed_token.c \
+	vxp_tokens.h
+
+CLEANFILES = \
+	$(builddir)/vxp_fixed_token.c \
+	$(builddir)/vxp_tokens.h
+
+vxp_fixed_token.c vxp_tokens.h: \
+	$(srcdir)/generate.py
+	@PYTHON@ $(srcdir)/generate.py $(srcdir) $(top_builddir)
+
+EXTRA_PROGRAMS = vxp_test
+
+vxp_test_LDADD = @PCRE_LIBS@ \
+	${RT_LIBS} ${LIBM} ${PTHREAD_LIBS}
+
+vxp_test_CFLAGS = \
+	-DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' \
+	-DVXP_DEBUG
+
+vxp_test_SOURCES = \
+	$(libvarnishapi_la_SOURCES) \
+	vxp_test.c
diff --git a/lib/libvarnishapi/generate.py b/lib/libvarnishapi/generate.py
new file mode 100755
index 0000000..1377f71
--- /dev/null
+++ b/lib/libvarnishapi/generate.py
@@ -0,0 +1,196 @@
+#!/usr/bin/env python
+#-
+# Copyright (c) 2006 Verdens Gang AS
+# Copyright (c) 2006-2013 Varnish Software AS
+# All rights reserved.
+#
+# Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+# Author: Martin Blix Grydeland <martin at varnish-software.com>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# Generate various .c and .h files for the VSL query expression parser
+# and the interfaces for it.
+
+import sys
+import copy
+
+srcroot = "../.."
+buildroot = "../.."
+if len(sys.argv) == 3:
+        srcroot = sys.argv[1]
+        buildroot = sys.argv[2]
+
+#######################################################################
+# These are our tokens
+
+tokens = {
+        # Numerical comparisons
+        "T_EQ":         "==",
+        "T_NEQ":        "!=",
+        "T_LEQ":        "<=",
+        "T_GEQ":        ">=",
+
+        # String comparisons
+        "T_SEQ":        "eq",
+        "T_SNEQ":       "ne",
+
+        # Regular expression matching
+        "T_NOMATCH":    "!~",
+
+        # Boolean operators
+        "T_NOT":        "not",
+        "T_AND":        "and",
+        "T_OR":         "or",
+
+        # Miscellaneous
+        None:           "<>~![]{}()",
+
+        # These have handwritten recognizers
+        "VAL":          None,
+        "EOI":          None,
+}
+
+#######################################################################
+# Emit a function to recognize tokens in a string
+
+def emit_vxp_fixed_token(fo, tokens):
+        recog = list()
+        emit = dict()
+        for i in tokens:
+                j = tokens[i]
+                if (j != None):
+                        recog.append(j)
+                        emit[j] = i
+
+        recog.sort()
+        rrecog = copy.copy(recog)
+        rrecog.sort(key = lambda x: -len(x))
+
+        fo.write("""
+unsigned
+vxp_fixed_token(const char *p, const char **q)
+{
+
+\tswitch (p[0]) {
+""")
+        last_initial = None
+        for i in recog:
+                if (i[0] == last_initial):
+                        continue
+                last_initial = i[0]
+                fo.write("\tcase '%s':\n" % last_initial)
+                for j in rrecog:
+                        if (j[0] != last_initial):
+                                continue
+
+			fo.write("\t\tif (")
+			k = 1
+			l = len(j)
+			while (k < l):
+				fo.write("p[%d] == '%s'" % (k, j[k]))
+				fo.write(" &&\n\t\t    ")
+				k += 1
+			fo.write("(isword(p[%d]) ? !isword(p[%d]) : 1)) {\n" %
+				 (l - 1, l))
+			fo.write("\t\t\t*q = p + %d;\n" % l)
+			fo.write("\t\t\treturn (%s);\n" % emit[j])
+			fo.write("\t\t}\n");
+		fo.write("\t\treturn (0);\n")
+
+        fo.write("\tdefault:\n\t\treturn (0);\n\t}\n}\n")
+
+#######################################################################
+# Emit the vxp_tnames (token->string) conversion array
+
+def emit_vxp_tnames(fo, tokens):
+        fo.write("\nconst char * const vxp_tnames[256] = {\n")
+        l = list(tokens.keys())
+        l.sort()
+        for i in l:
+                j = tokens[i]
+                if j == None:
+                        j = i
+                if i[0] == "'":
+                        j = i
+                fo.write("\t[%s] = \"%s\",\n" % (i, j))
+        fo.write("};\n")
+
+#######################################################################
+
+def polish_tokens(tokens):
+        # Expand single char tokens
+        st = tokens[None]
+        del tokens[None]
+
+        for i in st:
+                tokens["'" + i + "'"] = i
+
+#######################################################################
+
+def file_header(fo):
+        fo.write("""/*
+ * NB:  This file is machine generated, DO NOT EDIT!
+ *
+ * Edit and run generate.py instead
+ */
+""")
+
+#######################################################################
+
+polish_tokens(tokens)
+
+fo = open(buildroot + "/lib/libvarnishapi/vxp_tokens.h", "w")
+
+file_header(fo)
+
+j = 128
+l = list(tokens.keys())
+l.sort()
+for i in l:
+        if i[0] == "'":
+                continue
+        fo.write("#define\t%s %d\n" % (i, j))
+        j += 1
+        assert j < 256
+
+fo.close()
+
+#######################################################################
+
+fo = open(buildroot + "/lib/libvarnishapi/vxp_fixed_token.c", "w")
+
+file_header(fo)
+fo.write("""
+
+#include "config.h"
+
+#include <ctype.h>
+#include <stdio.h>
+
+#include "vxp.h"
+""")
+
+emit_vxp_fixed_token(fo, tokens)
+emit_vxp_tnames(fo, tokens)
+
+fo.close()
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 436d7c9..09554cb 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -32,42 +32,121 @@
 #include <unistd.h>
 #include <stdlib.h>
 #include <stdint.h>
+#include <string.h>
 
 #include "vas.h"
 #include "miniobj.h"
 #include "vre.h"
+#include "vsb.h"
 
 #include "vapi/vsl.h"
 #include "vsl_api.h"
+#include "vxp.h"
+
+#define NEEDLESS_RETURN(foo) return(foo)
 
 struct vslq_query {
 	unsigned		magic;
 #define VSLQ_QUERY_MAGIC	0x122322A5
 
-	vre_t			*regex;
+	struct vex		*vex;
 };
 
+static int
+vslq_test(const struct vex *vex, struct VSL_transaction * const ptrans[])
+{
+	struct VSL_transaction *t;
+	int i, reclen, vallen;
+	const char *recdata;
+
+	CHECK_OBJ_NOTNULL(vex, VEX_MAGIC);
+	CHECK_OBJ_NOTNULL(vex->tag, VEX_TAG_MAGIC);
+	CHECK_OBJ_NOTNULL(vex->val, VEX_VAL_MAGIC);
+	AN(vex->val->val_string);
+
+	vallen = strlen(vex->val->val_string);
+	for (t = ptrans[0]; t != NULL; t = *++ptrans) {
+		AZ(VSL_ResetCursor(t->c));
+		while (1) {
+			i = VSL_Next(t->c);
+			if (i < 0)
+				return (i);
+			if (i == 0)
+				break;
+			assert(i == 1);
+			AN(t->c->rec.ptr);
+
+			if (vex->tag->tag != VSL_TAG(t->c->rec.ptr))
+				continue;
+
+			reclen = VSL_LEN(t->c->rec.ptr);
+			recdata = VSL_CDATA(t->c->rec.ptr);
+			if (reclen == vallen &&
+			    !strncmp(vex->val->val_string, recdata, reclen))
+				return (1);
+		}
+	}
+
+	return (0);
+}
+
+static int
+vslq_exec(const struct vex *vex, struct VSL_transaction * const ptrans[])
+{
+	int r;
+
+	CHECK_OBJ_NOTNULL(vex, VEX_MAGIC);
+
+	switch (vex->tok) {
+	case T_OR:
+		AN(vex->a);
+		AN(vex->b);
+		r = vslq_exec(vex->a, ptrans);
+		if (r != 0)
+			return (r);
+		return (vslq_exec(vex->b, ptrans));
+	case T_AND:
+		AN(vex->a);
+		AN(vex->b);
+		r = vslq_exec(vex->a, ptrans);
+		if (r <= 0)
+			return (r);
+		return (vslq_exec(vex->b, ptrans));
+	case T_NOT:
+		AN(vex->a);
+		AZ(vex->b);
+		r = vslq_exec(vex->a, ptrans);
+		if (r < 0)
+			return (r);
+		return (!r);
+	default:
+		return (vslq_test(vex, ptrans));
+	}
+	NEEDLESS_RETURN(0);
+}
+
 struct vslq_query *
 vslq_newquery(struct VSL_data *vsl, enum VSL_grouping_e grouping,
     const char *querystring)
 {
-	struct vslq_query *query;
-	const char *error;
-	int pos;
-	vre_t *regex;
+	struct vsb *vsb;
+	struct vex *vex;
+	struct vslq_query *query = NULL;
 
 	(void)grouping;
 	AN(querystring);
-	regex = VRE_compile(querystring, 0, &error, &pos);
-	if (regex == NULL) {
-		vsl_diag(vsl, "failed to compile regex at pos %d: %s",
-		    pos, error);
-		return (NULL);
-	}
 
-	ALLOC_OBJ(query, VSLQ_QUERY_MAGIC);
-	if (query != NULL)
-		query->regex = regex;
+	vsb = VSB_new_auto();
+	AN(vsb);
+	vex = vex_New(querystring, vsb);
+	VSB_finish(vsb);
+	if (vex == NULL)
+		vsl_diag(vsl, "Query expression error:\n%s", VSB_data(vsb));
+	else {
+		ALLOC_OBJ(query, VSLQ_QUERY_MAGIC);
+		query->vex = vex;
+	}
+	VSB_delete(vsb);
 	return (query);
 }
 
@@ -81,45 +160,24 @@ vslq_deletequery(struct vslq_query **pquery)
 	*pquery = NULL;
 	CHECK_OBJ_NOTNULL(query, VSLQ_QUERY_MAGIC);
 
-	AN(query->regex);
-	VRE_free(&query->regex);
-	AZ(query->regex);
+	AN(query->vex);
+	vex_Free(&query->vex);
+	AZ(query->vex);
 
 	FREE_OBJ(query);
 }
 
 int
-vslq_runquery(const struct vslq_query *query, struct VSL_transaction * const ptrans[])
+vslq_runquery(const struct vslq_query *query,
+    struct VSL_transaction * const ptrans[])
 {
 	struct VSL_transaction *t;
-	struct VSL_cursor *c;
-	int i, len;
-	const char *data;
+	int r;
 
 	CHECK_OBJ_NOTNULL(query, VSLQ_QUERY_MAGIC);
-	AN(query->regex);
 
-	t = ptrans[0];
-	while (t) {
-		c = t->c;
-		while (1) {
-			i = VSL_Next(c);
-			if (i == 0)
-				break;
-			assert(i == 1);
-			AN(c->rec.ptr);
-			len = VSL_LEN(c->rec.ptr);
-			data = VSL_CDATA(c->rec.ptr);
-			i = VRE_exec(query->regex, data, len, 0, 0, NULL, 0,
-			    NULL);
-			if (i != VRE_ERROR_NOMATCH) {
-				AZ(VSL_ResetCursor(c));
-				return (1);
-			}
-		}
-		AZ(VSL_ResetCursor(c));
-		t = *++ptrans;
-	}
-
-	return (0);
+	r = vslq_exec(query->vex, ptrans);
+	for (t = ptrans[0]; t != NULL; t = *++ptrans)
+		AZ(VSL_ResetCursor(t->c));
+	return (r);
 }
diff --git a/lib/libvarnishapi/vxp.c b/lib/libvarnishapi/vxp.c
new file mode 100644
index 0000000..fffa550
--- /dev/null
+++ b/lib/libvarnishapi/vxp.c
@@ -0,0 +1,232 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#include "vsb.h"
+#include "vas.h"
+#include "miniobj.h"
+
+#include "vxp.h"
+
+static void
+vxp_ErrToken(const struct vxp *vxp, const struct token *t)
+{
+
+	if (t->tok == EOI)
+		VSB_printf(vxp->sb, "end of input");
+	else
+		VSB_printf(vxp->sb, "'%.*s'", PF(t));
+}
+
+static void
+vxp_Pos(struct vxp *vxp, struct vsb *vsb, const struct token *t, int tokoff)
+{
+	unsigned pos;
+
+	AN(vxp);
+	AN(vsb);
+	AN(t);
+	assert(t->b >= vxp->b);
+	pos = (unsigned)(t->b - vxp->b);
+	if (tokoff > 0)
+		pos += tokoff;
+	VSB_printf(vsb, "(Pos %u)", pos + 1);
+}
+
+static void
+vxp_quote(struct vxp *vxp, const char *b, const char *e, int tokoff)
+{
+	const char *p;
+	char c;
+
+	assert(b <= e);
+	assert(b >= vxp->b);
+	assert(e <= vxp->e);
+	for (p = vxp->b; p < vxp->e; p++) {
+		if (isspace(*p))
+			VSB_bcat(vxp->sb, " ", 1);
+		else
+			VSB_bcat(vxp->sb, p, 1);
+	}
+	VSB_putc(vxp->sb, '\n');
+	for (p = vxp->b; p < vxp->e; p++) {
+		if (p >= b && p < e) {
+			if (p - b == tokoff)
+				c = '^';
+			else
+				c = '#';
+		} else
+			c = '-';
+		VSB_putc(vxp->sb, c);
+	}
+	VSB_putc(vxp->sb, '\n');
+}
+
+void
+vxp_ErrWhere(struct vxp *vxp, const struct token *t, int tokoff)
+{
+
+	AN(vxp);
+	AN(t);
+	vxp_Pos(vxp, vxp->sb, t, tokoff);
+	VSB_putc(vxp->sb, '\n');
+	vxp_quote(vxp, t->b, t->e, tokoff);
+	VSB_putc(vxp->sb, '\n');
+	vxp->err = 1;
+}
+
+void
+vxp_NextToken(struct vxp *vxp)
+{
+
+	AN(vxp->t);
+	vxp->t = VTAILQ_NEXT(vxp->t, list);
+	if (vxp->t == NULL) {
+		VSB_printf(vxp->sb,
+		    "Ran out of input, something is missing or"
+		    " maybe unbalanced parenthesis\n");
+		vxp->err = 1;
+	}
+}
+
+void
+vxp__Expect(struct vxp *vxp, unsigned tok)
+{
+
+	if (vxp->t->tok == tok)
+		return;
+	VSB_printf(vxp->sb, "Expected %s got ", vxp_tnames[tok]);
+	vxp_ErrToken(vxp, vxp->t);
+	VSB_putc(vxp->sb, ' ');
+	vxp_ErrWhere(vxp, vxp->t, -1);
+}
+
+static void
+vxp_DoFree(struct vxp *vxp, void *p)
+{
+	struct membit *mb;
+
+	mb = calloc(sizeof *mb, 1);
+	AN(mb);
+	mb->ptr = p;
+	VTAILQ_INSERT_TAIL(&vxp->membits, mb, list);
+}
+
+void *
+vxp_Alloc(struct vxp *vxp, unsigned len)
+{
+	void *p;
+
+	p = calloc(len, 1);
+	AN(p);
+	vxp_DoFree(vxp, p);
+	return (p);
+}
+
+static struct vxp *
+vxp_New(struct vsb *sb)
+{
+	struct vxp *vxp;
+
+	AN(sb);
+
+	ALLOC_OBJ(vxp, VXP_MAGIC);
+	AN(vxp);
+	VTAILQ_INIT(&vxp->membits);
+	VTAILQ_INIT(&vxp->tokens);
+	vxp->sb = sb;
+
+	return (vxp);
+}
+
+static void
+vxp_Delete(struct vxp **pvxp)
+{
+	struct vxp *vxp;
+	struct membit *mb;
+
+	AN(pvxp);
+	vxp = *pvxp;
+	*pvxp = NULL;
+	CHECK_OBJ_NOTNULL(vxp, VXP_MAGIC);
+
+	while (!VTAILQ_EMPTY(&vxp->membits)) {
+		mb = VTAILQ_FIRST(&vxp->membits);
+		VTAILQ_REMOVE(&vxp->membits, mb, list);
+		free(mb->ptr);
+		free(mb);
+	}
+
+	FREE_OBJ(vxp);
+}
+
+struct vex *
+vex_New(const char *query, struct vsb *sb)
+{
+	struct vxp *vxp;
+	struct vex *vex;
+
+	AN(query);
+	AN(sb);
+	vxp = vxp_New(sb);
+	vxp->b = query;
+	vxp->e = query + strlen(query);
+
+	vxp_Lexer(vxp);
+
+#ifdef VXP_DEBUG
+	vxp_PrintTokens(vxp);
+#endif
+
+	if (vxp->err) {
+		vxp_Delete(&vxp);
+		AZ(vxp);
+		return (NULL);
+	}
+
+	vex = vxp_Parse(vxp);
+
+#ifdef VXP_DEBUG
+	if (vex != NULL)
+		vex_PrintTree(vex);
+#endif
+
+	vxp_Delete(&vxp);
+	AZ(vxp);
+
+	return (vex);
+}
diff --git a/lib/libvarnishapi/vxp.h b/lib/libvarnishapi/vxp.h
new file mode 100644
index 0000000..9243e0b
--- /dev/null
+++ b/lib/libvarnishapi/vxp.h
@@ -0,0 +1,140 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/types.h>
+
+#include "vqueue.h"
+#include "vre.h"
+
+#include "vxp_tokens.h"
+
+#define isword(c) \
+	(isalpha(c) || isdigit(c) || (c) == '_' || (c) == '-' || (c) == '.')
+
+#define PF(t)	(int)((t)->e - (t)->b), (t)->b
+
+/* From vex_fixed_token.c */
+unsigned vxp_fixed_token(const char *p, const char **q);
+extern const char * const vxp_tnames[256];
+
+struct membit {
+	VTAILQ_ENTRY(membit)	list;
+	void			*ptr;
+};
+
+struct token {
+	unsigned		tok;
+	const char		*b;
+	const char		*e;
+	VTAILQ_ENTRY(token)	list;
+	unsigned		cnt;
+	char			*dec;
+};
+
+struct vxp {
+	unsigned		magic;
+#define VXP_MAGIC		0x59C7F6AC
+
+	const char		*src;
+	const char		*b;
+	const char		*e;
+
+	VTAILQ_HEAD(, token)	tokens;
+	VTAILQ_HEAD(, membit)	membits;
+	struct token		*t;
+
+	struct vsb		*sb;
+	int			err;
+};
+
+struct vex;
+
+struct vex_tag {
+	unsigned		magic;
+#define VEX_TAG_MAGIC		0x1AD3D78D
+	int			tag;
+	int			field;
+	int			level_min;
+	int			level_max;
+};
+
+enum vex_val_e {
+	VEX__UNSET,
+	VEX_INT,
+	VEX_FLOAT,
+	VEX_STRING,
+	VEX_REGEX,
+};
+
+struct vex_val {
+	unsigned		magic;
+#define VEX_VAL_MAGIC		0x3F109965
+	enum vex_val_e		type;
+	long long		val_int;
+	double			val_float;
+	char			*val_string;
+	vre_t			*val_regex;
+};
+
+struct vex {
+	unsigned		magic;
+#define VEX_MAGIC		0xC7DB792D
+	unsigned		tok;
+	struct vex		*a, *b;
+	struct vex_tag		*tag;
+	struct vex_val		*val;
+};
+
+/* VXP internals */
+
+#define ERRCHK(tl)	do { if ((tl)->err) return; } while (0)
+#define Expect(a, b)	vxp__Expect(a, b)
+#define ExpectErr(a, b)	\
+    do { vxp__Expect(a, b); ERRCHK(a); } while (0)
+#define SkipToken(a, b) \
+    do { vxp__Expect(a, b); ERRCHK(a); vxp_NextToken(a); } while (0)
+
+void vxp__Expect(struct vxp *vxp, unsigned tok);
+void vxp_ErrWhere(struct vxp *vxp, const struct token *t, int tokoff);
+void vxp_NextToken(struct vxp *vxp);
+void * vxp_Alloc(struct vxp *vxp, unsigned len);
+void vxp_Lexer(struct vxp *vxp);
+struct vex * vxp_Parse(struct vxp *vxp);
+
+/* API internal interface */
+
+struct vex * vex_New(const char *query, struct vsb *sb);
+void vex_Free(struct vex **pvex);
+
+/* Debug routines */
+#ifdef VXP_DEBUG
+void vxp_PrintTokens(const struct vxp *vxp);
+void vex_PrintTree(const struct vex *vex);
+#endif /* VXP_DEBUG */
diff --git a/lib/libvarnishapi/vxp_lexer.c b/lib/libvarnishapi/vxp_lexer.c
new file mode 100644
index 0000000..076edaa
--- /dev/null
+++ b/lib/libvarnishapi/vxp_lexer.c
@@ -0,0 +1,191 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#include "vsb.h"
+#include "vas.h"
+
+#include "vxp.h"
+
+static void
+vxp_add_token(struct vxp *vxp, unsigned tok, const char *b, const char *e)
+{
+	struct token *t;
+
+	t = vxp_Alloc(vxp, sizeof *t);
+	AN(t);
+	t->tok = tok;
+	t->b = b;
+	t->e = e;
+	if (vxp->t != NULL)
+		VTAILQ_INSERT_AFTER(&vxp->tokens, vxp->t, t, list);
+	else
+		VTAILQ_INSERT_TAIL(&vxp->tokens, t, list);
+	vxp->t = t;
+}
+
+static int
+vxp_decstr(struct vxp *vxp, int quoted)
+{
+	const char *b, *e, *p;
+	char *s;
+	unsigned l;
+	int esc = 0;
+
+	assert(vxp->t->tok == VAL);
+
+	b = vxp->t->b;
+	e = vxp->t->e;
+	if (quoted) {
+		assert(e - b >= 2);
+		b++;
+		e--;
+	}
+	l = e - b;
+	s = vxp->t->dec = vxp_Alloc(vxp, l + 1);
+	AN(vxp->t->dec);
+	for (p = b; p < e; p++) {
+		if (!esc && *p == '\\') {
+			esc = 1;
+			continue;
+		}
+		esc = 0;
+		*s++ = *p;
+	}
+	*s = '\0';
+	if (esc || p != e) {
+		VSB_printf(vxp->sb, "Syntax error ");
+		vxp_ErrWhere(vxp, vxp->t, -1);
+		return (1);
+	}
+	return (0);
+}
+
+/*
+ * Build a token list
+ */
+
+void
+vxp_Lexer(struct vxp *vxp)
+{
+	const char *p, *q;
+	unsigned u;
+	char quote;
+
+	for (p = vxp->b; p < vxp->e; ) {
+
+		/* Skip any whitespace */
+		if (isspace(*p)) {
+			p++;
+			continue;
+		}
+
+		/* Match for the fixed tokens */
+		u = vxp_fixed_token(p, &q);
+		if (u != 0) {
+			AN(q);
+			vxp_add_token(vxp, u, p, q);
+			p = q;
+			continue;
+		}
+
+		/* Match quoted strings */
+		if (*p == '"' || *p == '\'') {
+			quote = *p;
+			for (q = p + 1; q < vxp->e; q++) {
+				if (q[-1] == '\\')
+					continue;
+				if (*q == quote) {
+					q++;
+					quote = '\0';
+					break;
+				}
+			}
+			vxp_add_token(vxp, VAL, p, q);
+			if (quote != '\0') {
+				VSB_printf(vxp->sb, "Unterminated string ");
+				vxp_ErrWhere(vxp, vxp->t, q - p - 1);
+				return;
+			}
+			if (vxp_decstr(vxp, 1))
+				return;
+			p = q;
+			continue;
+		}
+
+		/* Match bareword */
+		if (isword(*p)) {
+			for (q = p; q < vxp->e; q++)
+				if (!isword(*q))
+					break;
+			vxp_add_token(vxp, VAL, p, q);
+			if (vxp_decstr(vxp, 0))
+				return;
+			p = q;
+			continue;
+		}
+
+		/* Error */
+		vxp_add_token(vxp, EOI, p, p + 1);
+		VSB_printf(vxp->sb, "Syntax error ");
+		vxp_ErrWhere(vxp, vxp->t, q - p);
+		return;
+	}
+
+	/* Finished */
+	vxp_add_token(vxp, EOI, vxp->e, vxp->e);
+}
+
+#ifdef VXP_DEBUG
+void
+vxp_PrintTokens(const struct vxp *vxp)
+{
+	struct token *t;
+
+	fprintf(stderr, "Token list:\n");
+	fprintf(stderr, "  %-5s %-20s %s\n", "TOK", "SUBSTR", "DECODED");
+	VTAILQ_FOREACH(t, &vxp->tokens, list) {
+		fprintf(stderr, "  ");
+		fprintf(stderr, "%-5s", vxp_tnames[t->tok]);
+		fprintf(stderr, " %-20.*s", (unsigned)(t->e - t->b), t->b);
+		if (t->dec)
+			fprintf(stderr, " '%s'", t->dec);
+		fprintf(stderr, "\n");
+	}
+	fprintf(stderr, "\n");
+}
+#endif /* VXP_DEBUG */
diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
new file mode 100644
index 0000000..390be8d
--- /dev/null
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -0,0 +1,477 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <stdint.h>
+#include <string.h>
+#include <ctype.h>
+#include <stdint.h>
+
+#include "vas.h"
+#include "vsb.h"
+#include "miniobj.h"
+#include "vapi/vsl.h"
+
+#include "vxp.h"
+
+static void vxp_expr_or(struct vxp *vxp, struct vex **pvex);
+
+static void
+vxp_expr_tag(struct vxp *vxp, struct vex_tag **ptag)
+{
+
+	/* XXX: Tag wildcards */
+	AN(ptag);
+	AZ(*ptag);
+	if (vxp->t->tok != VAL) {
+		VSB_printf(vxp->sb, "Expected VSL tag got '%.*s' ", PF(vxp->t));
+		vxp_ErrWhere(vxp, vxp->t, -1);
+		return;
+	}
+	ALLOC_OBJ(*ptag, VEX_TAG_MAGIC);
+	AN(*ptag);
+	(*ptag)->tag = VSL_Name2Tag(vxp->t->dec, -1);
+	if ((*ptag)->tag == -1) {
+		VSB_printf(vxp->sb, "Could not match '%.*s' to any tag ",
+		    PF(vxp->t));
+		vxp_ErrWhere(vxp, vxp->t, -1);
+		return;
+	} else if ((*ptag)->tag == -2) {
+		VSB_printf(vxp->sb, "'%.*s' matches multiple tags ",
+		    PF(vxp->t));
+		vxp_ErrWhere(vxp, vxp->t, -1);
+		return;
+	}
+	vxp_NextToken(vxp);
+
+	/* XXX: Tag limiting operators ([], {}) */
+}
+
+static void
+vxp_expr_num(struct vxp *vxp, struct vex_val **pval)
+{
+	char *endptr;
+
+	AN(pval);
+	AZ(*pval);
+	if (vxp->t->tok != VAL) {
+		VSB_printf(vxp->sb, "Expected number got '%.*s' ", PF(vxp->t));
+		vxp_ErrWhere(vxp, vxp->t, -1);
+		return;
+	}
+	AN(vxp->t->dec);
+	ALLOC_OBJ(*pval, VEX_VAL_MAGIC);
+	AN(*pval);
+	if (strchr(vxp->t->dec, '.')) {
+		(*pval)->type = VEX_FLOAT;
+		(*pval)->val_float = strtod(vxp->t->dec, &endptr);
+		while (isspace(*endptr))
+			endptr++;
+		if (*endptr != '\0') {
+			VSB_printf(vxp->sb, "Floating point parse error ");
+			vxp_ErrWhere(vxp, vxp->t, -1);
+			return;
+		}
+	} else {
+		(*pval)->type = VEX_INT;
+		(*pval)->val_int = strtoll(vxp->t->dec, &endptr, 0);
+		while (isspace(*endptr))
+			endptr++;
+		if (*endptr != '\0') {
+			VSB_printf(vxp->sb, "Integer parse error ");
+			vxp_ErrWhere(vxp, vxp->t, -1);
+			return;
+		}
+	}
+	vxp_NextToken(vxp);
+}
+
+static void
+vxp_expr_str(struct vxp *vxp, struct vex_val **pval)
+{
+
+	AN(pval);
+	AZ(*pval);
+	if (vxp->t->tok != VAL) {
+		VSB_printf(vxp->sb, "Expected string got '%.*s' ", PF(vxp->t));
+		vxp_ErrWhere(vxp, vxp->t, -1);
+		return;
+	}
+	AN(vxp->t->dec);
+	ALLOC_OBJ(*pval, VEX_VAL_MAGIC);
+	AN(*pval);
+	(*pval)->type = VEX_STRING;
+	(*pval)->val_string = strdup(vxp->t->dec);
+	AN((*pval)->val_string);
+	vxp_NextToken(vxp);
+}
+
+static void
+vxp_expr_regex(struct vxp *vxp, struct vex_val **pval)
+{
+	const char *errptr;
+	int erroff;
+
+	/* XXX: Caseless option */
+
+	AN(pval);
+	AZ(*pval);
+	if (vxp->t->tok != VAL) {
+		VSB_printf(vxp->sb, "Expected regular expression got '%.*s' ",
+		    PF(vxp->t));
+		vxp_ErrWhere(vxp, vxp->t, -1);
+		return;
+	}
+	AN(vxp->t->dec);
+	ALLOC_OBJ(*pval, VEX_VAL_MAGIC);
+	AN(*pval);
+	(*pval)->type = VEX_REGEX;
+	(*pval)->val_string = strdup(vxp->t->dec);
+	(*pval)->val_regex = VRE_compile(vxp->t->dec, 0, &errptr, &erroff);
+	if ((*pval)->val_regex == NULL) {
+		AN(errptr);
+		VSB_printf(vxp->sb, "Regular expression error: %s ", errptr);
+		vxp_ErrWhere(vxp, vxp->t, erroff);
+		return;
+	}
+	vxp_NextToken(vxp);
+}
+
+/*
+ * SYNTAX:
+ *   expr_cmp:
+ *     tag
+ *     tag <operator> num|str|regex
+ */
+
+static void
+vxp_expr_cmp(struct vxp *vxp, struct vex **pvex)
+{
+
+	AN(pvex);
+	AZ(*pvex);
+	ALLOC_OBJ(*pvex, VEX_MAGIC);
+	AN(*pvex);
+	vxp_expr_tag(vxp, &(*pvex)->tag);
+	ERRCHK(vxp);
+
+	/* Test operator */
+	switch (vxp->t->tok) {
+
+	/* Single tag expressions don't take any more tokens */
+	case EOI:
+	case T_AND:
+	case T_OR:
+	case ')':
+		return;
+
+	/* Valid operators */
+	case T_EQ:		/* == */
+	case T_NEQ:		/* != */
+	case T_SEQ:		/* eq */
+	case T_SNEQ:		/* ne */
+	case '~':		/* ~ */
+	case T_NOMATCH:		/* !~ */
+		(*pvex)->tok = vxp->t->tok;
+		break;
+
+	/* Error */
+	default:
+		VSB_printf(vxp->sb, "Expected operator got '%.*s' ",
+		    PF(vxp->t));
+		vxp_ErrWhere(vxp, vxp->t, -1);
+		return;
+	}
+	vxp_NextToken(vxp);
+	ERRCHK(vxp);
+
+	/* Value */
+	switch((*pvex)->tok) {
+	case '\0':
+		WRONG("Missing token");
+	case T_EQ:		/* == */
+	case T_GEQ:		/* >= */
+	case T_LEQ:		/* <= */
+	case T_NEQ:		/* != */
+		vxp_expr_num(vxp, &(*pvex)->val);
+		break;
+	case T_SEQ:		/* eq */
+	case T_SNEQ:		/* ne */
+		vxp_expr_str(vxp, &(*pvex)->val);
+		break;
+	case '~':		/* ~ */
+	case T_NOMATCH:		/* !~ */
+		vxp_expr_regex(vxp, &(*pvex)->val);
+		break;
+	default:
+		INCOMPL();
+	}
+}
+
+/*
+ * SYNTAX:
+ *   expr_group:
+ *     '(' expr_or ')'
+ *     expr_not
+ */
+
+static void
+vxp_expr_group(struct vxp *vxp, struct vex **pvex)
+{
+
+	AN(pvex);
+	AZ(*pvex);
+
+	if (vxp->t->tok == '(') {
+		SkipToken(vxp, '(');
+		vxp_expr_or(vxp, pvex);
+		ERRCHK(vxp);
+		SkipToken(vxp, ')');
+		return;
+	}
+
+	vxp_expr_cmp(vxp, pvex);
+}
+
+/*
+ * SYNTAX:
+ *   expr_not:
+ *     '!' expr_group
+ *     expr_group
+ */
+
+static void
+vxp_expr_not(struct vxp *vxp, struct vex **pvex)
+{
+
+	AN(pvex);
+	AZ(*pvex);
+
+	if (vxp->t->tok == '!') {
+		ALLOC_OBJ(*pvex, VEX_MAGIC);
+		AN(*pvex);
+		(*pvex)->tok = vxp->t->tok;
+		vxp_NextToken(vxp);
+		vxp_expr_group(vxp, &(*pvex)->a);
+		return;
+	}
+
+	vxp_expr_group(vxp, pvex);
+	return;
+}
+
+/*
+ * SYNTAX:
+ *   expr_and:
+ *     expr_not { 'and' expr_not }*
+ */
+
+static void
+vxp_expr_and(struct vxp *vxp, struct vex **pvex)
+{
+	struct vex *a;
+
+	AN(pvex);
+	AZ(*pvex);
+	vxp_expr_not(vxp, pvex);
+	ERRCHK(vxp);
+	while (vxp->t->tok == T_AND) {
+		a = *pvex;
+		ALLOC_OBJ(*pvex, VEX_MAGIC);
+		AN(*pvex);
+		(*pvex)->tok = vxp->t->tok;
+		(*pvex)->a = a;
+		vxp_NextToken(vxp);
+		ERRCHK(vxp);
+		vxp_expr_not(vxp, &(*pvex)->b);
+		ERRCHK(vxp);
+	}
+}
+
+/*
+ * SYNTAX:
+ *   expr_or:
+ *     expr_and { 'or' expr_and }*
+ */
+
+static void
+vxp_expr_or(struct vxp *vxp, struct vex **pvex)
+{
+	struct vex *a;
+
+	AN(pvex);
+	AZ(*pvex);
+	vxp_expr_and(vxp, pvex);
+	ERRCHK(vxp);
+	while (vxp->t->tok == T_OR) {
+		a = *pvex;
+		ALLOC_OBJ(*pvex, VEX_MAGIC);
+		AN(*pvex);
+		(*pvex)->tok = vxp->t->tok;
+		(*pvex)->a = a;
+		vxp_NextToken(vxp);
+		ERRCHK(vxp);
+		vxp_expr_and(vxp, &(*pvex)->b);
+		ERRCHK(vxp);
+	}
+}
+
+/*
+ * SYNTAX:
+ *   expr:
+ *     expr_or EOI
+ */
+
+static void
+vxp_expr(struct vxp *vxp, struct vex **pvex)
+{
+	vxp_expr_or(vxp, pvex);
+	ERRCHK(vxp);
+	ExpectErr(vxp, EOI);
+}
+
+/*
+ * Build a struct vex tree from the token list in vxp
+ */
+
+struct vex *
+vxp_Parse(struct vxp *vxp)
+{
+	struct vex *vex = NULL;
+
+	vxp->t = VTAILQ_FIRST(&vxp->tokens);
+	if (vxp->t == NULL)
+		return (NULL);
+
+	vxp_expr(vxp, &vex);
+
+	if (vxp->err) {
+		if (vex)
+			vex_Free(&vex);
+		AZ(vex);
+		return (NULL);
+	}
+
+	return (vex);
+}
+
+/*
+ * Free a struct vex tree
+ */
+
+void
+vex_Free(struct vex **pvex)
+{
+
+	if ((*pvex)->tag != NULL)
+		FREE_OBJ((*pvex)->tag);
+	if ((*pvex)->val != NULL) {
+		if ((*pvex)->val->val_string)
+			free((*pvex)->val->val_string);
+		if ((*pvex)->val->val_regex)
+			VRE_free(&(*pvex)->val->val_regex);
+		FREE_OBJ((*pvex)->val);
+	}
+	if ((*pvex)->a != NULL) {
+		vex_Free(&(*pvex)->a);
+		AZ((*pvex)->a);
+	}
+	if ((*pvex)->b != NULL) {
+		vex_Free(&(*pvex)->b);
+		AZ((*pvex)->b);
+	}
+	FREE_OBJ(*pvex);
+	*pvex = NULL;
+}
+
+#ifdef VXP_DEBUG
+
+static void
+vex_print_val(const struct vex_val *val)
+{
+
+	CHECK_OBJ_NOTNULL(val, VEX_VAL_MAGIC);
+	switch (val->type) {
+	case VEX_INT:
+		fprintf(stderr, "INT=%jd", (intmax_t)val->val_int);
+		break;
+	case VEX_FLOAT:
+		fprintf(stderr, "FLOAT=%f", val->val_float);
+		break;
+	case VEX_STRING:
+		AN(val->val_string);
+		fprintf(stderr, "STRING='%s'", val->val_string);
+		break;
+	case VEX_REGEX:
+		AN(val->val_string);
+		AN(val->val_regex);
+		fprintf(stderr, "REGEX='%s'", val->val_string);
+		break;
+	default:
+		WRONG("value type");
+		break;
+	}
+}
+
+static void
+vex_print(const struct vex *vex, int indent)
+{
+	CHECK_OBJ_NOTNULL(vex, VEX_MAGIC);
+
+	fprintf(stderr, "%*s%s", indent, "", vxp_tnames[vex->tok]);
+	if (vex->tag != NULL) {
+		CHECK_OBJ_NOTNULL(vex->tag, VEX_TAG_MAGIC);
+		fprintf(stderr, " tag=%s", VSL_tags[vex->tag->tag]);
+	}
+	if (vex->val != NULL) {
+		fprintf(stderr, " ");
+		vex_print_val(vex->val);
+	}
+	fprintf(stderr, "\n");
+	if (vex->a != NULL)
+		vex_print(vex->a, indent + 2);
+	if (vex->b != NULL)
+		vex_print(vex->b, indent + 2);
+}
+
+void
+vex_PrintTree(const struct vex *vex)
+{
+
+	CHECK_OBJ_NOTNULL(vex, VEX_MAGIC);
+	fprintf(stderr, "VEX tree:\n");
+	vex_print(vex, 2);
+}
+
+#endif /* VXP_DEBUG */
diff --git a/lib/libvarnishapi/vxp_test.c b/lib/libvarnishapi/vxp_test.c
new file mode 100644
index 0000000..19be714
--- /dev/null
+++ b/lib/libvarnishapi/vxp_test.c
@@ -0,0 +1,41 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+
+#include "vxp.h"
+#include "vas.h"
+#include "vsb.h"
+
+int
+main(int argc, char **argv)
+{
+	int i;
+	unsigned l;
+	char *s;
+	struct vsb *vsb;
+	struct vex *vex;
+
+	l = 0;
+	for (i = 1; i < argc; i++)
+		l += strlen(argv[i]) + 1;
+	s = calloc(l + 1, sizeof (char));
+	for (i = 1; i < argc; strcat(s, " "), i++)
+		strcat(s, argv[i]);
+
+	vsb = VSB_new_auto();
+	AN(vsb);
+	vex = vex_New(s, vsb);
+
+	if (vex == NULL) {
+		VSB_finish(vsb);
+		fprintf(stderr, "Error:\n%s", VSB_data(vsb));
+		exit(1);
+	}
+	VSB_delete(vsb);
+
+	vex_Free(&vex);
+	AZ(vex);
+
+	return (0);
+}



From martin at varnish-cache.org  Tue Oct  1 12:48:17 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:17 +0200
Subject: [master] 7abf829 Add testcase for the query operators
Message-ID: <E1VQzNB-0005f5-QT@project.varnish-software.com>

commit 7abf8292a063427ab7fbd585cca5f9d24d03cd8e
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 18 12:17:31 2013 +0200

    Add testcase for the query operators

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
new file mode 100644
index 0000000..a7c7f32
--- /dev/null
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -0,0 +1,22 @@
+varnishtest "Test VSL query operators"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+} -start
+
+logexpect l1 -v v1
+
+client c1 {
+	txreq -hdr "Foo: bar"
+	rxresp
+	expect resp.status == 200
+} -run
+
+logexpect l1 -d 1 -g request -q "Begin eq 'req 1000'" {
+	expect * *	Begin
+	expect * =	ReqEnd
+} -run



From martin at varnish-cache.org  Tue Oct  1 12:48:17 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:17 +0200
Subject: [master] eecbd7c Ne-operator - merge with query lang
Message-ID: <E1VQzNB-0005fD-Rg@project.varnish-software.com>

commit eecbd7cb0b98992a7035b950dc37d781c001f22a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 18 12:58:47 2013 +0200

    Ne-operator - merge with query lang

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index a7c7f32..c4beb0d 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -16,7 +16,14 @@ client c1 {
 	expect resp.status == 200
 } -run
 
+# Test 'eq' operator
 logexpect l1 -d 1 -g request -q "Begin eq 'req 1000'" {
 	expect * *	Begin
 	expect * =	ReqEnd
 } -run
+
+# Test 'ne' operator
+logexpect l1 -d 1 -g request -q "Begin ne 'req 1001'" {
+	expect * *	Begin
+	expect * =	ReqEnd
+} -run
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 09554cb..7932f3c 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -53,18 +53,48 @@ struct vslq_query {
 };
 
 static int
+vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
+{
+	int reclen;
+	const char *recdata;
+
+	AN(vex);
+	AN(rec);
+
+	reclen = VSL_LEN(rec->ptr);
+	recdata = VSL_CDATA(rec->ptr);
+
+	switch (vex->tok) {
+	case T_SEQ:		/* eq */
+		assert(vex->val->type == VEX_STRING);
+		if (reclen == strlen(vex->val->val_string) &&
+		    !strncmp(vex->val->val_string, recdata, reclen))
+			return (1);
+		return (0);
+	case T_SNEQ:		/* ne */
+		assert(vex->val->type == VEX_STRING);
+		if (reclen != strlen(vex->val->val_string) ||
+		    strncmp(vex->val->val_string, recdata, reclen))
+			return (1);
+		return (0);
+	default:
+		INCOMPL();
+	}
+
+	return (0);
+}
+
+static int
 vslq_test(const struct vex *vex, struct VSL_transaction * const ptrans[])
 {
 	struct VSL_transaction *t;
-	int i, reclen, vallen;
-	const char *recdata;
+	int i;
 
 	CHECK_OBJ_NOTNULL(vex, VEX_MAGIC);
 	CHECK_OBJ_NOTNULL(vex->tag, VEX_TAG_MAGIC);
 	CHECK_OBJ_NOTNULL(vex->val, VEX_VAL_MAGIC);
 	AN(vex->val->val_string);
 
-	vallen = strlen(vex->val->val_string);
 	for (t = ptrans[0]; t != NULL; t = *++ptrans) {
 		AZ(VSL_ResetCursor(t->c));
 		while (1) {
@@ -79,11 +109,11 @@ vslq_test(const struct vex *vex, struct VSL_transaction * const ptrans[])
 			if (vex->tag->tag != VSL_TAG(t->c->rec.ptr))
 				continue;
 
-			reclen = VSL_LEN(t->c->rec.ptr);
-			recdata = VSL_CDATA(t->c->rec.ptr);
-			if (reclen == vallen &&
-			    !strncmp(vex->val->val_string, recdata, reclen))
-				return (1);
+			i = vslq_test_rec(vex, &t->c->rec);
+			if (i)
+				return (i);
+
+
 		}
 	}
 



From martin at varnish-cache.org  Tue Oct  1 12:48:17 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:17 +0200
Subject: [master] facaef0 Fix test case for 'ne' operator
Message-ID: <E1VQzNB-0005fL-Sn@project.varnish-software.com>

commit facaef06cfc10fef7d470ebcea5e0d3d62ebbd8a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 18 16:10:56 2013 +0200

    Fix test case for 'ne' operator

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index c4beb0d..4709acb 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -23,7 +23,8 @@ logexpect l1 -d 1 -g request -q "Begin eq 'req 1000'" {
 } -run
 
 # Test 'ne' operator
-logexpect l1 -d 1 -g request -q "Begin ne 'req 1001'" {
+logexpect l1 -d 1 -g request -q "ReqProtocol ne 'HTTP/1.0'" {
 	expect * *	Begin
 	expect * =	ReqEnd
 } -run
+



From martin at varnish-cache.org  Tue Oct  1 12:48:17 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:17 +0200
Subject: [master] ae2f4b0 Keep value string len in vex expression tree.
Message-ID: <E1VQzNB-0005fT-Tz@project.varnish-software.com>

commit ae2f4b052288a5deed639ffc8ed9fcb12bbb4087
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 18 16:15:44 2013 +0200

    Keep value string len in vex expression tree.
    
    This saves a strlen for every string comparison.

diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 7932f3c..de6d43b 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -55,11 +55,14 @@ struct vslq_query {
 static int
 vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 {
+	const struct vex_val *val;
 	int reclen;
 	const char *recdata;
 
 	AN(vex);
 	AN(rec);
+	val = vex->val;
+	AN(val);
 
 	reclen = VSL_LEN(rec->ptr);
 	recdata = VSL_CDATA(rec->ptr);
@@ -67,13 +70,13 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	switch (vex->tok) {
 	case T_SEQ:		/* eq */
 		assert(vex->val->type == VEX_STRING);
-		if (reclen == strlen(vex->val->val_string) &&
+		if (reclen == val->val_stringlen &&
 		    !strncmp(vex->val->val_string, recdata, reclen))
 			return (1);
 		return (0);
 	case T_SNEQ:		/* ne */
 		assert(vex->val->type == VEX_STRING);
-		if (reclen != strlen(vex->val->val_string) ||
+		if (reclen != val->val_stringlen ||
 		    strncmp(vex->val->val_string, recdata, reclen))
 			return (1);
 		return (0);
diff --git a/lib/libvarnishapi/vxp.h b/lib/libvarnishapi/vxp.h
index 9243e0b..35d1d33 100644
--- a/lib/libvarnishapi/vxp.h
+++ b/lib/libvarnishapi/vxp.h
@@ -100,6 +100,7 @@ struct vex_val {
 	long long		val_int;
 	double			val_float;
 	char			*val_string;
+	size_t			val_stringlen;
 	vre_t			*val_regex;
 };
 
diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index 390be8d..7ce5093 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -134,6 +134,7 @@ vxp_expr_str(struct vxp *vxp, struct vex_val **pval)
 	(*pval)->type = VEX_STRING;
 	(*pval)->val_string = strdup(vxp->t->dec);
 	AN((*pval)->val_string);
+	(*pval)->val_stringlen = strlen((*pval)->val_string);
 	vxp_NextToken(vxp);
 }
 



From martin at varnish-cache.org  Tue Oct  1 12:48:17 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:17 +0200
Subject: [master] 62b13a5 Log begin/end and query from logexpect
Message-ID: <E1VQzNB-0005fb-V4@project.varnish-software.com>

commit 62b13a57799ecfdbae71a0544f8943c1986c9ece
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 18 17:01:20 2013 +0200

    Log begin/end and query from logexpect
    
    This to make it easier to see which logexpect of the test script is
    actually running.

diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c
index f2170b2..8a0f384 100644
--- a/bin/varnishtest/vtc_logexp.c
+++ b/bin/varnishtest/vtc_logexp.c
@@ -273,6 +273,9 @@ logexp_thread(void *priv)
 
 	i = 0;
 	AZ(le->test);
+	vtc_log(le->vl, 4, "beg|");
+	if (le->query != NULL)
+		vtc_log(le->vl, 4, "qry| %s", le->query);
 	logexp_next(le);
 	while (le->test) {
 		i = VSLQ_Dispatch(le->vslq, logexp_dispatch, le);
@@ -281,7 +284,7 @@ logexp_thread(void *priv)
 		if (i == 0 && le->test)
 			VTIM_sleep(0.01);
 	}
-	vtc_log(le->vl, 4, "end of test script");
+	vtc_log(le->vl, 4, "end|");
 
 	return (NULL);
 }



From martin at varnish-cache.org  Tue Oct  1 12:48:17 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:17 +0200
Subject: [master] 5fb1da6 Make all VSL records NUL-terminated
Message-ID: <E1VQzNB-0005fj-WB@project.varnish-software.com>

commit 5fb1da6b569e75b232f2e1b555014231a4d7d064
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Sep 19 14:38:48 2013 +0200

    Make all VSL records NUL-terminated
    
    Make all VSL records NUL-terminated so that it is safe to use string
    functions on the SHMLOG directly.

diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 0093c74..4aa08ce 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -218,13 +218,14 @@ VSL(enum VSL_tag_e tag, uint32_t vxid, const char *fmt, ...)
 
 
 	if (strchr(fmt, '%') == NULL) {
-		vslr(tag, vxid, fmt, strlen(fmt));
+		vslr(tag, vxid, fmt, strlen(fmt) + 1);
 	} else {
 		va_start(ap, fmt);
 		n = vsnprintf(buf, mlen, fmt, ap);
 		va_end(ap);
-		if (n > mlen)
-			n = mlen;
+		if (n > mlen - 1)
+			n = mlen - 1;
+		buf[n++] = '\0'; /* NUL-terminated */
 		vslr(tag, vxid, buf, n);
 	}
 }
@@ -261,6 +262,7 @@ void
 VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t)
 {
 	unsigned l, mlen;
+	char *p;
 
 	Tcheck(t);
 	if (vsl_tag_is_masked(tag))
@@ -269,16 +271,18 @@ VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t)
 
 	/* Truncate */
 	l = Tlen(t);
-	if (l > mlen)
-		l = mlen;
+	if (l > mlen - 1)
+		l = mlen - 1;
 
 	assert(vsl->wlp < vsl->wle);
 
 	/* Flush if necessary */
-	if (VSL_END(vsl->wlp, l) >= vsl->wle)
+	if (VSL_END(vsl->wlp, l + 1) >= vsl->wle)
 		VSL_Flush(vsl, 1);
-	assert(VSL_END(vsl->wlp, l) < vsl->wle);
-	memcpy(VSL_DATA(vsl->wlp), t.b, l);
+	assert(VSL_END(vsl->wlp, l + 1) < vsl->wle);
+	p = VSL_DATA(vsl->wlp);
+	memcpy(p, t.b, l);
+	p[l++] = '\0';		/* NUL-terminated */
 	vsl->wlp = vsl_hdr(tag, vsl->wlp, l, vsl->wid);
 	assert(vsl->wlp < vsl->wle);
 	vsl->wlr++;
@@ -321,15 +325,16 @@ VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...)
 	mlen = cache_param->shm_reclen;
 
 	/* Flush if we cannot fit a full size record */
-	if (VSL_END(vsl->wlp, mlen) >= vsl->wle)
+	if (VSL_END(vsl->wlp, mlen + 1) >= vsl->wle)
 		VSL_Flush(vsl, 1);
 
 	p = VSL_DATA(vsl->wlp);
 	va_start(ap, fmt);
 	n = vsnprintf(p, mlen, fmt, ap);
 	va_end(ap);
-	if (n > mlen)
-		n = mlen;	/* we truncate long fields */
+	if (n > mlen - 1)
+		n = mlen - 1;	/* we truncate long fields */
+	p[n++] = '\0';		/* NUL-terminated */
 	vsl->wlp = vsl_hdr(tag, vsl->wlp, n, vsl->wid);
 	assert(vsl->wlp < vsl->wle);
 	vsl->wlr++;
diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h
index 2d484f6..196bf98 100644
--- a/include/vapi/vsl_int.h
+++ b/include/vapi/vsl_int.h
@@ -60,7 +60,10 @@
  * Each logrecord consist of:
  *	[n]		= ((type & 0xff) << 24) | (length & 0xffff)
  *	[n + 1]		= ((marker & 0x03) << 30) | (identifier & 0x3fffffff)
- *	[n + 2] ... [m]	= content
+ *	[n + 2] ... [m]	= content (NUL-terminated)
+ *
+ * Logrecords are NUL-terminated so that string functions can be run
+ * directly on the shmlog data.
  *
  * Notice that the constants in these macros cannot be changed without
  * changing corresponding magic numbers in varnishd/cache/cache_shmlog.c



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 18f5b8b Add '==' operator to query language
Message-ID: <E1VQzNC-0005fr-0z@project.varnish-software.com>

commit 18f5b8b6cea1e8d82dce84df1650676727cd3bb1
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Sep 19 15:01:06 2013 +0200

    Add '==' operator to query language

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index 4709acb..427435a 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -17,14 +17,22 @@ client c1 {
 } -run
 
 # Test 'eq' operator
-logexpect l1 -d 1 -g request -q "Begin eq 'req 1000'" {
-	expect * *	Begin
+logexpect l1 -d 1 -g vxid -q "Begin eq 'req 1000'" {
+	expect * *	Begin	req
 	expect * =	ReqEnd
+	expect * =	End
 } -run
 
 # Test 'ne' operator
-logexpect l1 -d 1 -g request -q "ReqProtocol ne 'HTTP/1.0'" {
-	expect * *	Begin
+logexpect l1 -d 1 -g vxid -q "ReqProtocol ne 'HTTP/1.0'" {
+	expect * *	Begin	req
 	expect * =	ReqEnd
+	expect * =	End
 } -run
 
+# Test '==' operator
+logexpect l1 -d 1 -g vxid -q "RespStatus == 200" {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index de6d43b..937d21f 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -33,6 +33,7 @@
 #include <stdlib.h>
 #include <stdint.h>
 #include <string.h>
+#include <ctype.h>
 
 #include "vas.h"
 #include "miniobj.h"
@@ -58,6 +59,8 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	const struct vex_val *val;
 	int reclen;
 	const char *recdata;
+	long long recint;
+	char *endptr;
 
 	AN(vex);
 	AN(rec);
@@ -67,17 +70,49 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	reclen = VSL_LEN(rec->ptr);
 	recdata = VSL_CDATA(rec->ptr);
 
+	/* Prepare */
 	switch (vex->tok) {
+	case T_EQ:		/* == */
+	case T_NEQ:		/* != */
+	case '<':
+	case '>':
+	case T_LEQ:		/* <= */
+	case T_GEQ:		/* >= */
+		/* Numerical comparison */
+		switch (val->type) {
+		case VEX_INT:
+			recint = strtoll(recdata, &endptr, 0);
+			if (*endptr == '\0' || isspace(*endptr))
+				break;
+			/* Can't parse - no match */
+			return (0);
+		default:
+			INCOMPL();
+		}
+		break;
+	}
+
+	/* Compare */
+	switch (vex->tok) {
+	case T_EQ:		/* == */
+		switch (val->type) {
+		case VEX_INT:
+			if (val->val_int == recint)
+				return (1);
+			return (0);
+		default:
+			INCOMPL();
+		}
 	case T_SEQ:		/* eq */
-		assert(vex->val->type == VEX_STRING);
-		if (reclen == val->val_stringlen &&
-		    !strncmp(vex->val->val_string, recdata, reclen))
+		assert(val->type == VEX_STRING);
+		if (reclen == val->val_stringlen + 1 &&
+		    !strncmp(val->val_string, recdata, reclen))
 			return (1);
 		return (0);
 	case T_SNEQ:		/* ne */
-		assert(vex->val->type == VEX_STRING);
-		if (reclen != val->val_stringlen ||
-		    strncmp(vex->val->val_string, recdata, reclen))
+		assert(val->type == VEX_STRING);
+		if (reclen != val->val_stringlen + 1 ||
+		    strncmp(val->val_string, recdata, reclen))
 			return (1);
 		return (0);
 	default:
@@ -96,7 +131,6 @@ vslq_test(const struct vex *vex, struct VSL_transaction * const ptrans[])
 	CHECK_OBJ_NOTNULL(vex, VEX_MAGIC);
 	CHECK_OBJ_NOTNULL(vex->tag, VEX_TAG_MAGIC);
 	CHECK_OBJ_NOTNULL(vex->val, VEX_VAL_MAGIC);
-	AN(vex->val->val_string);
 
 	for (t = ptrans[0]; t != NULL; t = *++ptrans) {
 		AZ(VSL_ResetCursor(t->c));



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] efc4154 Add '!=' operator to query language
Message-ID: <E1VQzNC-0005fz-2h@project.varnish-software.com>

commit efc4154b6091cf5a72e15c297a08e0b6d1c48ed1
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Sep 19 15:03:08 2013 +0200

    Add '!=' operator to query language

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index 427435a..e9f7503 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -36,3 +36,10 @@ logexpect l1 -d 1 -g vxid -q "RespStatus == 200" {
 	expect * =	ReqEnd
 	expect * =	End
 } -run
+
+# Test '!=' operator
+logexpect l1 -d 1 -g vxid -q "RespStatus != 503" {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 937d21f..2d4abf1 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -103,6 +103,15 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 		default:
 			INCOMPL();
 		}
+	case T_NEQ:		/* != */
+		switch (val->type) {
+		case VEX_INT:
+			if (val->val_int != recint)
+				return (1);
+			return (0);
+		default:
+			INCOMPL();
+		}
 	case T_SEQ:		/* eq */
 		assert(val->type == VEX_STRING);
 		if (reclen == val->val_stringlen + 1 &&



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 9bcf907 Add support for floats
Message-ID: <E1VQzNC-0005g7-3i@project.varnish-software.com>

commit 9bcf907cf28db63693ed872403192b66a269462e
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Sep 19 16:31:43 2013 +0200

    Add support for floats

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index e9f7503..5b54b94 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -30,16 +30,30 @@ logexpect l1 -d 1 -g vxid -q "ReqProtocol ne 'HTTP/1.0'" {
 	expect * =	End
 } -run
 
-# Test '==' operator
+# Test '==' operator on integers
 logexpect l1 -d 1 -g vxid -q "RespStatus == 200" {
 	expect * *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
-# Test '!=' operator
+# Test '==' operator on floats
+logexpect l1 -d 1 -g vxid -q "RespStatus == 200." {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
+
+# Test '!=' operator on integers
 logexpect l1 -d 1 -g vxid -q "RespStatus != 503" {
 	expect * *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
+
+# Test '!=' operator on floats
+logexpect l1 -d 1 -g vxid -q "RespStatus != 503." {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 2d4abf1..74ff646 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -60,6 +60,7 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	int reclen;
 	const char *recdata;
 	long long recint;
+	double recfloat;
 	char *endptr;
 
 	AN(vex);
@@ -86,6 +87,12 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 				break;
 			/* Can't parse - no match */
 			return (0);
+		case VEX_FLOAT:
+			recfloat = strtod(recdata, &endptr);
+			if (*endptr == '\0' || isspace(*endptr))
+				break;
+			/* Can't parse - no match */
+			return (0);
 		default:
 			INCOMPL();
 		}
@@ -100,8 +107,12 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 			if (val->val_int == recint)
 				return (1);
 			return (0);
+		case VEX_FLOAT:
+			if (val->val_float == recfloat)
+				return (1);
+			return (0);
 		default:
-			INCOMPL();
+			WRONG("Wrong value type");
 		}
 	case T_NEQ:		/* != */
 		switch (val->type) {
@@ -109,8 +120,12 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 			if (val->val_int != recint)
 				return (1);
 			return (0);
+		case VEX_FLOAT:
+			if (val->val_float != recfloat)
+				return (1);
+			return (0);
 		default:
-			INCOMPL();
+			WRONG("Wrong value type");
 		}
 	case T_SEQ:		/* eq */
 		assert(val->type == VEX_STRING);



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 2968fdf Add '<' and '>' operators
Message-ID: <E1VQzNC-0005gF-53@project.varnish-software.com>

commit 2968fdf2f118ed2dfcacf3d0f3c98ff7fd7569f1
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Sep 19 17:00:48 2013 +0200

    Add '<' and '>' operators

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index 5b54b94..2d9e83f 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -57,3 +57,31 @@ logexpect l1 -d 1 -g vxid -q "RespStatus != 503." {
 	expect * =	ReqEnd
 	expect * =	End
 } -run
+
+# Test '<' operator on integers
+logexpect l1 -d 1 -g vxid -q "RespStatus < 201" {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
+
+# Test '<' operator on floats
+logexpect l1 -d 1 -g vxid -q "RespStatus < 201." {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
+
+# Test '>' operator on integers
+logexpect l1 -d 1 -g vxid -q "RespStatus > 199" {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
+
+# Test '>' operator on floats
+logexpect l1 -d 1 -g vxid -q "RespStatus > 199." {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 74ff646..062be7e 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -104,11 +104,11 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	case T_EQ:		/* == */
 		switch (val->type) {
 		case VEX_INT:
-			if (val->val_int == recint)
+			if (recint == val->val_int)
 				return (1);
 			return (0);
 		case VEX_FLOAT:
-			if (val->val_float == recfloat)
+			if (recfloat == val->val_float)
 				return (1);
 			return (0);
 		default:
@@ -117,11 +117,37 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	case T_NEQ:		/* != */
 		switch (val->type) {
 		case VEX_INT:
-			if (val->val_int != recint)
+			if (recint != val->val_int)
 				return (1);
 			return (0);
 		case VEX_FLOAT:
-			if (val->val_float != recfloat)
+			if (recfloat != val->val_float)
+				return (1);
+			return (0);
+		default:
+			WRONG("Wrong value type");
+		}
+	case '<':		/* < */
+		switch (val->type) {
+		case VEX_INT:
+			if (recint < val->val_int)
+				return (1);
+			return (0);
+		case VEX_FLOAT:
+			if (recfloat < val->val_float)
+				return (1);
+			return (0);
+		default:
+			WRONG("Wrong value type");
+		}
+	case '>':
+		switch (val->type) {
+		case VEX_INT:
+			if (recint > val->val_int)
+				return (1);
+			return (0);
+		case VEX_FLOAT:
+			if (recfloat > val->val_float)
 				return (1);
 			return (0);
 		default:
@@ -130,13 +156,13 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	case T_SEQ:		/* eq */
 		assert(val->type == VEX_STRING);
 		if (reclen == val->val_stringlen + 1 &&
-		    !strncmp(val->val_string, recdata, reclen))
+		    !strncmp(recdata, val->val_string, reclen))
 			return (1);
 		return (0);
 	case T_SNEQ:		/* ne */
 		assert(val->type == VEX_STRING);
 		if (reclen != val->val_stringlen + 1 ||
-		    strncmp(val->val_string, recdata, reclen))
+		    strncmp(recdata, val->val_string, reclen))
 			return (1);
 		return (0);
 	default:
diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index 7ce5093..7e3ed5a 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -199,6 +199,10 @@ vxp_expr_cmp(struct vxp *vxp, struct vex **pvex)
 
 	/* Valid operators */
 	case T_EQ:		/* == */
+	case '<':		/* < */
+	case '>':		/* > */
+	case T_GEQ:		/* >= */
+	case T_LEQ:		/* <= */
 	case T_NEQ:		/* != */
 	case T_SEQ:		/* eq */
 	case T_SNEQ:		/* ne */
@@ -222,6 +226,8 @@ vxp_expr_cmp(struct vxp *vxp, struct vex **pvex)
 	case '\0':
 		WRONG("Missing token");
 	case T_EQ:		/* == */
+	case '<':		/* < */
+	case '>':		/* > */
 	case T_GEQ:		/* >= */
 	case T_LEQ:		/* <= */
 	case T_NEQ:		/* != */



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] b6f7816 Add '<=' and '>=' operators
Message-ID: <E1VQzNC-0005gN-6L@project.varnish-software.com>

commit b6f7816acc4363ac1874c194ef9b23507619c40a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Sep 19 17:10:30 2013 +0200

    Add '<=' and '>=' operators

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index 2d9e83f..36609ad 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -85,3 +85,31 @@ logexpect l1 -d 1 -g vxid -q "RespStatus > 199." {
 	expect * =	ReqEnd
 	expect * =	End
 } -run
+
+# Test '<=' operator on integers
+logexpect l1 -d 1 -g vxid -q "RespStatus <= 200" {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
+
+# Test '<=' operator on floats
+logexpect l1 -d 1 -g vxid -q "RespStatus <= 200." {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
+
+# Test '>=' operator on integers
+logexpect l1 -d 1 -g vxid -q "RespStatus >= 200" {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
+
+# Test '>=' operator on floats
+logexpect l1 -d 1 -g vxid -q "RespStatus >= 200." {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 062be7e..12fcd28 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -153,6 +153,32 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 		default:
 			WRONG("Wrong value type");
 		}
+	case T_LEQ:		/* <= */
+		switch (val->type) {
+		case VEX_INT:
+			if (recint <= val->val_int)
+				return (1);
+			return (0);
+		case VEX_FLOAT:
+			if (recfloat <= val->val_float)
+				return (1);
+			return (0);
+		default:
+			WRONG("Wrong value type");
+		}
+	case T_GEQ:		/* >= */
+		switch (val->type) {
+		case VEX_INT:
+			if (recint >= val->val_int)
+				return (1);
+			return (0);
+		case VEX_FLOAT:
+			if (recfloat >= val->val_float)
+				return (1);
+			return (0);
+		default:
+			WRONG("Wrong value type");
+		}
 	case T_SEQ:		/* eq */
 		assert(val->type == VEX_STRING);
 		if (reclen == val->val_stringlen + 1 &&



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 530f0b2 Rename tag -> lhs and val -> rhs in the parser.
Message-ID: <E1VQzNC-0005gV-7N@project.varnish-software.com>

commit 530f0b20612886ee17903d30837c8719438176d7
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Sep 20 13:58:46 2013 +0200

    Rename tag -> lhs and val -> rhs in the parser.

diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 12fcd28..3659a72 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -56,7 +56,7 @@ struct vslq_query {
 static int
 vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 {
-	const struct vex_val *val;
+	const struct vex_rhs *rhs;
 	int reclen;
 	const char *recdata;
 	long long recint;
@@ -65,8 +65,8 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 
 	AN(vex);
 	AN(rec);
-	val = vex->val;
-	AN(val);
+	rhs = vex->rhs;
+	AN(rhs);
 
 	reclen = VSL_LEN(rec->ptr);
 	recdata = VSL_CDATA(rec->ptr);
@@ -80,7 +80,7 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	case T_LEQ:		/* <= */
 	case T_GEQ:		/* >= */
 		/* Numerical comparison */
-		switch (val->type) {
+		switch (rhs->type) {
 		case VEX_INT:
 			recint = strtoll(recdata, &endptr, 0);
 			if (*endptr == '\0' || isspace(*endptr))
@@ -102,93 +102,93 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	/* Compare */
 	switch (vex->tok) {
 	case T_EQ:		/* == */
-		switch (val->type) {
+		switch (rhs->type) {
 		case VEX_INT:
-			if (recint == val->val_int)
+			if (recint == rhs->val_int)
 				return (1);
 			return (0);
 		case VEX_FLOAT:
-			if (recfloat == val->val_float)
+			if (recfloat == rhs->val_float)
 				return (1);
 			return (0);
 		default:
-			WRONG("Wrong value type");
+			WRONG("Wrong RHS type");
 		}
 	case T_NEQ:		/* != */
-		switch (val->type) {
+		switch (rhs->type) {
 		case VEX_INT:
-			if (recint != val->val_int)
+			if (recint != rhs->val_int)
 				return (1);
 			return (0);
 		case VEX_FLOAT:
-			if (recfloat != val->val_float)
+			if (recfloat != rhs->val_float)
 				return (1);
 			return (0);
 		default:
-			WRONG("Wrong value type");
+			WRONG("Wrong RHS type");
 		}
 	case '<':		/* < */
-		switch (val->type) {
+		switch (rhs->type) {
 		case VEX_INT:
-			if (recint < val->val_int)
+			if (recint < rhs->val_int)
 				return (1);
 			return (0);
 		case VEX_FLOAT:
-			if (recfloat < val->val_float)
+			if (recfloat < rhs->val_float)
 				return (1);
 			return (0);
 		default:
-			WRONG("Wrong value type");
+			WRONG("Wrong RHS type");
 		}
 	case '>':
-		switch (val->type) {
+		switch (rhs->type) {
 		case VEX_INT:
-			if (recint > val->val_int)
+			if (recint > rhs->val_int)
 				return (1);
 			return (0);
 		case VEX_FLOAT:
-			if (recfloat > val->val_float)
+			if (recfloat > rhs->val_float)
 				return (1);
 			return (0);
 		default:
-			WRONG("Wrong value type");
+			WRONG("Wrong RHS type");
 		}
 	case T_LEQ:		/* <= */
-		switch (val->type) {
+		switch (rhs->type) {
 		case VEX_INT:
-			if (recint <= val->val_int)
+			if (recint <= rhs->val_int)
 				return (1);
 			return (0);
 		case VEX_FLOAT:
-			if (recfloat <= val->val_float)
+			if (recfloat <= rhs->val_float)
 				return (1);
 			return (0);
 		default:
-			WRONG("Wrong value type");
+			WRONG("Wrong RHS type");
 		}
 	case T_GEQ:		/* >= */
-		switch (val->type) {
+		switch (rhs->type) {
 		case VEX_INT:
-			if (recint >= val->val_int)
+			if (recint >= rhs->val_int)
 				return (1);
 			return (0);
 		case VEX_FLOAT:
-			if (recfloat >= val->val_float)
+			if (recfloat >= rhs->val_float)
 				return (1);
 			return (0);
 		default:
-			WRONG("Wrong value type");
+			WRONG("Wrong RHS type");
 		}
 	case T_SEQ:		/* eq */
-		assert(val->type == VEX_STRING);
-		if (reclen == val->val_stringlen + 1 &&
-		    !strncmp(recdata, val->val_string, reclen))
+		assert(rhs->type == VEX_STRING);
+		if (reclen == rhs->val_stringlen + 1 &&
+		    !strncmp(recdata, rhs->val_string, reclen))
 			return (1);
 		return (0);
 	case T_SNEQ:		/* ne */
-		assert(val->type == VEX_STRING);
-		if (reclen != val->val_stringlen + 1 ||
-		    strncmp(recdata, val->val_string, reclen))
+		assert(rhs->type == VEX_STRING);
+		if (reclen != rhs->val_stringlen + 1 ||
+		    strncmp(recdata, rhs->val_string, reclen))
 			return (1);
 		return (0);
 	default:
@@ -205,8 +205,8 @@ vslq_test(const struct vex *vex, struct VSL_transaction * const ptrans[])
 	int i;
 
 	CHECK_OBJ_NOTNULL(vex, VEX_MAGIC);
-	CHECK_OBJ_NOTNULL(vex->tag, VEX_TAG_MAGIC);
-	CHECK_OBJ_NOTNULL(vex->val, VEX_VAL_MAGIC);
+	CHECK_OBJ_NOTNULL(vex->lhs, VEX_LHS_MAGIC);
+	CHECK_OBJ_NOTNULL(vex->rhs, VEX_RHS_MAGIC);
 
 	for (t = ptrans[0]; t != NULL; t = *++ptrans) {
 		AZ(VSL_ResetCursor(t->c));
@@ -219,7 +219,7 @@ vslq_test(const struct vex *vex, struct VSL_transaction * const ptrans[])
 			assert(i == 1);
 			AN(t->c->rec.ptr);
 
-			if (vex->tag->tag != VSL_TAG(t->c->rec.ptr))
+			if (vex->lhs->tag != VSL_TAG(t->c->rec.ptr))
 				continue;
 
 			i = vslq_test_rec(vex, &t->c->rec);
diff --git a/lib/libvarnishapi/vxp.h b/lib/libvarnishapi/vxp.h
index 35d1d33..73b3bc0 100644
--- a/lib/libvarnishapi/vxp.h
+++ b/lib/libvarnishapi/vxp.h
@@ -76,16 +76,19 @@ struct vxp {
 
 struct vex;
 
-struct vex_tag {
+struct vex_lhs {
+	/* Left-hand-side of a vex expression. Stores the information
+	   about which records and what parts of those records the
+	   expression should be applied to */
 	unsigned		magic;
-#define VEX_TAG_MAGIC		0x1AD3D78D
+#define VEX_LHS_MAGIC		0x1AD3D78D
 	int			tag;
 	int			field;
 	int			level_min;
 	int			level_max;
 };
 
-enum vex_val_e {
+enum vex_rhs_e {
 	VEX__UNSET,
 	VEX_INT,
 	VEX_FLOAT,
@@ -93,10 +96,12 @@ enum vex_val_e {
 	VEX_REGEX,
 };
 
-struct vex_val {
+struct vex_rhs {
+	/* Right-hand-side of a vex expression. Stores the value that the
+	   records from LHS should be matched against */
 	unsigned		magic;
-#define VEX_VAL_MAGIC		0x3F109965
-	enum vex_val_e		type;
+#define VEX_RHS_MAGIC		0x3F109965
+	enum vex_rhs_e		type;
 	long long		val_int;
 	double			val_float;
 	char			*val_string;
@@ -109,8 +114,8 @@ struct vex {
 #define VEX_MAGIC		0xC7DB792D
 	unsigned		tok;
 	struct vex		*a, *b;
-	struct vex_tag		*tag;
-	struct vex_val		*val;
+	struct vex_lhs		*lhs;
+	struct vex_rhs		*rhs;
 };
 
 /* VXP internals */
diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index 7e3ed5a..2cc1288 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -48,26 +48,26 @@
 static void vxp_expr_or(struct vxp *vxp, struct vex **pvex);
 
 static void
-vxp_expr_tag(struct vxp *vxp, struct vex_tag **ptag)
+vxp_expr_lhs(struct vxp *vxp, struct vex_lhs **plhs)
 {
 
 	/* XXX: Tag wildcards */
-	AN(ptag);
-	AZ(*ptag);
+	AN(plhs);
+	AZ(*plhs);
 	if (vxp->t->tok != VAL) {
 		VSB_printf(vxp->sb, "Expected VSL tag got '%.*s' ", PF(vxp->t));
 		vxp_ErrWhere(vxp, vxp->t, -1);
 		return;
 	}
-	ALLOC_OBJ(*ptag, VEX_TAG_MAGIC);
-	AN(*ptag);
-	(*ptag)->tag = VSL_Name2Tag(vxp->t->dec, -1);
-	if ((*ptag)->tag == -1) {
+	ALLOC_OBJ(*plhs, VEX_LHS_MAGIC);
+	AN(*plhs);
+	(*plhs)->tag = VSL_Name2Tag(vxp->t->dec, -1);
+	if ((*plhs)->tag == -1) {
 		VSB_printf(vxp->sb, "Could not match '%.*s' to any tag ",
 		    PF(vxp->t));
 		vxp_ErrWhere(vxp, vxp->t, -1);
 		return;
-	} else if ((*ptag)->tag == -2) {
+	} else if ((*plhs)->tag == -2) {
 		VSB_printf(vxp->sb, "'%.*s' matches multiple tags ",
 		    PF(vxp->t));
 		vxp_ErrWhere(vxp, vxp->t, -1);
@@ -75,27 +75,27 @@ vxp_expr_tag(struct vxp *vxp, struct vex_tag **ptag)
 	}
 	vxp_NextToken(vxp);
 
-	/* XXX: Tag limiting operators ([], {}) */
+	/* XXX: Lhs limiting operators ([], {}) */
 }
 
 static void
-vxp_expr_num(struct vxp *vxp, struct vex_val **pval)
+vxp_expr_num(struct vxp *vxp, struct vex_rhs **prhs)
 {
 	char *endptr;
 
-	AN(pval);
-	AZ(*pval);
+	AN(prhs);
+	AZ(*prhs);
 	if (vxp->t->tok != VAL) {
 		VSB_printf(vxp->sb, "Expected number got '%.*s' ", PF(vxp->t));
 		vxp_ErrWhere(vxp, vxp->t, -1);
 		return;
 	}
 	AN(vxp->t->dec);
-	ALLOC_OBJ(*pval, VEX_VAL_MAGIC);
-	AN(*pval);
+	ALLOC_OBJ(*prhs, VEX_RHS_MAGIC);
+	AN(*prhs);
 	if (strchr(vxp->t->dec, '.')) {
-		(*pval)->type = VEX_FLOAT;
-		(*pval)->val_float = strtod(vxp->t->dec, &endptr);
+		(*prhs)->type = VEX_FLOAT;
+		(*prhs)->val_float = strtod(vxp->t->dec, &endptr);
 		while (isspace(*endptr))
 			endptr++;
 		if (*endptr != '\0') {
@@ -104,8 +104,8 @@ vxp_expr_num(struct vxp *vxp, struct vex_val **pval)
 			return;
 		}
 	} else {
-		(*pval)->type = VEX_INT;
-		(*pval)->val_int = strtoll(vxp->t->dec, &endptr, 0);
+		(*prhs)->type = VEX_INT;
+		(*prhs)->val_int = strtoll(vxp->t->dec, &endptr, 0);
 		while (isspace(*endptr))
 			endptr++;
 		if (*endptr != '\0') {
@@ -118,36 +118,36 @@ vxp_expr_num(struct vxp *vxp, struct vex_val **pval)
 }
 
 static void
-vxp_expr_str(struct vxp *vxp, struct vex_val **pval)
+vxp_expr_str(struct vxp *vxp, struct vex_rhs **prhs)
 {
 
-	AN(pval);
-	AZ(*pval);
+	AN(prhs);
+	AZ(*prhs);
 	if (vxp->t->tok != VAL) {
 		VSB_printf(vxp->sb, "Expected string got '%.*s' ", PF(vxp->t));
 		vxp_ErrWhere(vxp, vxp->t, -1);
 		return;
 	}
 	AN(vxp->t->dec);
-	ALLOC_OBJ(*pval, VEX_VAL_MAGIC);
-	AN(*pval);
-	(*pval)->type = VEX_STRING;
-	(*pval)->val_string = strdup(vxp->t->dec);
-	AN((*pval)->val_string);
-	(*pval)->val_stringlen = strlen((*pval)->val_string);
+	ALLOC_OBJ(*prhs, VEX_RHS_MAGIC);
+	AN(*prhs);
+	(*prhs)->type = VEX_STRING;
+	(*prhs)->val_string = strdup(vxp->t->dec);
+	AN((*prhs)->val_string);
+	(*prhs)->val_stringlen = strlen((*prhs)->val_string);
 	vxp_NextToken(vxp);
 }
 
 static void
-vxp_expr_regex(struct vxp *vxp, struct vex_val **pval)
+vxp_expr_regex(struct vxp *vxp, struct vex_rhs **prhs)
 {
 	const char *errptr;
 	int erroff;
 
 	/* XXX: Caseless option */
 
-	AN(pval);
-	AZ(*pval);
+	AN(prhs);
+	AZ(*prhs);
 	if (vxp->t->tok != VAL) {
 		VSB_printf(vxp->sb, "Expected regular expression got '%.*s' ",
 		    PF(vxp->t));
@@ -155,12 +155,12 @@ vxp_expr_regex(struct vxp *vxp, struct vex_val **pval)
 		return;
 	}
 	AN(vxp->t->dec);
-	ALLOC_OBJ(*pval, VEX_VAL_MAGIC);
-	AN(*pval);
-	(*pval)->type = VEX_REGEX;
-	(*pval)->val_string = strdup(vxp->t->dec);
-	(*pval)->val_regex = VRE_compile(vxp->t->dec, 0, &errptr, &erroff);
-	if ((*pval)->val_regex == NULL) {
+	ALLOC_OBJ(*prhs, VEX_RHS_MAGIC);
+	AN(*prhs);
+	(*prhs)->type = VEX_REGEX;
+	(*prhs)->val_string = strdup(vxp->t->dec);
+	(*prhs)->val_regex = VRE_compile(vxp->t->dec, 0, &errptr, &erroff);
+	if ((*prhs)->val_regex == NULL) {
 		AN(errptr);
 		VSB_printf(vxp->sb, "Regular expression error: %s ", errptr);
 		vxp_ErrWhere(vxp, vxp->t, erroff);
@@ -172,8 +172,8 @@ vxp_expr_regex(struct vxp *vxp, struct vex_val **pval)
 /*
  * SYNTAX:
  *   expr_cmp:
- *     tag
- *     tag <operator> num|str|regex
+ *     lhs
+ *     lhs <operator> num|str|regex
  */
 
 static void
@@ -184,13 +184,13 @@ vxp_expr_cmp(struct vxp *vxp, struct vex **pvex)
 	AZ(*pvex);
 	ALLOC_OBJ(*pvex, VEX_MAGIC);
 	AN(*pvex);
-	vxp_expr_tag(vxp, &(*pvex)->tag);
+	vxp_expr_lhs(vxp, &(*pvex)->lhs);
 	ERRCHK(vxp);
 
 	/* Test operator */
 	switch (vxp->t->tok) {
 
-	/* Single tag expressions don't take any more tokens */
+	/* Single lhs expressions don't take any more tokens */
 	case EOI:
 	case T_AND:
 	case T_OR:
@@ -231,15 +231,15 @@ vxp_expr_cmp(struct vxp *vxp, struct vex **pvex)
 	case T_GEQ:		/* >= */
 	case T_LEQ:		/* <= */
 	case T_NEQ:		/* != */
-		vxp_expr_num(vxp, &(*pvex)->val);
+		vxp_expr_num(vxp, &(*pvex)->rhs);
 		break;
 	case T_SEQ:		/* eq */
 	case T_SNEQ:		/* ne */
-		vxp_expr_str(vxp, &(*pvex)->val);
+		vxp_expr_str(vxp, &(*pvex)->rhs);
 		break;
 	case '~':		/* ~ */
 	case T_NOMATCH:		/* !~ */
-		vxp_expr_regex(vxp, &(*pvex)->val);
+		vxp_expr_regex(vxp, &(*pvex)->rhs);
 		break;
 	default:
 		INCOMPL();
@@ -401,14 +401,14 @@ void
 vex_Free(struct vex **pvex)
 {
 
-	if ((*pvex)->tag != NULL)
-		FREE_OBJ((*pvex)->tag);
-	if ((*pvex)->val != NULL) {
-		if ((*pvex)->val->val_string)
-			free((*pvex)->val->val_string);
-		if ((*pvex)->val->val_regex)
-			VRE_free(&(*pvex)->val->val_regex);
-		FREE_OBJ((*pvex)->val);
+	if ((*pvex)->lhs != NULL)
+		FREE_OBJ((*pvex)->lhs);
+	if ((*pvex)->rhs != NULL) {
+		if ((*pvex)->rhs->val_string)
+			free((*pvex)->rhs->val_string);
+		if ((*pvex)->rhs->val_regex)
+			VRE_free(&(*pvex)->rhs->val_regex);
+		FREE_OBJ((*pvex)->rhs);
 	}
 	if ((*pvex)->a != NULL) {
 		vex_Free(&(*pvex)->a);
@@ -425,28 +425,28 @@ vex_Free(struct vex **pvex)
 #ifdef VXP_DEBUG
 
 static void
-vex_print_val(const struct vex_val *val)
+vex_print_rhs(const struct vex_rhs *rhs)
 {
 
-	CHECK_OBJ_NOTNULL(val, VEX_VAL_MAGIC);
-	switch (val->type) {
+	CHECK_OBJ_NOTNULL(rhs, VEX_RHS_MAGIC);
+	switch (rhs->type) {
 	case VEX_INT:
-		fprintf(stderr, "INT=%jd", (intmax_t)val->val_int);
+		fprintf(stderr, "INT=%jd", (intmax_t)rhs->val_int);
 		break;
 	case VEX_FLOAT:
-		fprintf(stderr, "FLOAT=%f", val->val_float);
+		fprintf(stderr, "FLOAT=%f", rhs->val_float);
 		break;
 	case VEX_STRING:
-		AN(val->val_string);
-		fprintf(stderr, "STRING='%s'", val->val_string);
+		AN(rhs->val_string);
+		fprintf(stderr, "STRING='%s'", rhs->val_string);
 		break;
 	case VEX_REGEX:
-		AN(val->val_string);
-		AN(val->val_regex);
-		fprintf(stderr, "REGEX='%s'", val->val_string);
+		AN(rhs->val_string);
+		AN(rhs->val_regex);
+		fprintf(stderr, "REGEX='%s'", rhs->val_string);
 		break;
 	default:
-		WRONG("value type");
+		WRONG("rhs type");
 		break;
 	}
 }
@@ -457,13 +457,13 @@ vex_print(const struct vex *vex, int indent)
 	CHECK_OBJ_NOTNULL(vex, VEX_MAGIC);
 
 	fprintf(stderr, "%*s%s", indent, "", vxp_tnames[vex->tok]);
-	if (vex->tag != NULL) {
-		CHECK_OBJ_NOTNULL(vex->tag, VEX_TAG_MAGIC);
-		fprintf(stderr, " tag=%s", VSL_tags[vex->tag->tag]);
+	if (vex->lhs != NULL) {
+		CHECK_OBJ_NOTNULL(vex->lhs, VEX_LHS_MAGIC);
+		fprintf(stderr, " tag=%s", VSL_tags[vex->lhs->tag]);
 	}
-	if (vex->val != NULL) {
+	if (vex->rhs != NULL) {
 		fprintf(stderr, " ");
-		vex_print_val(vex->val);
+		vex_print_rhs(vex->rhs);
 	}
 	fprintf(stderr, "\n");
 	if (vex->a != NULL)



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] d0fba00 Variable renames for consistency
Message-ID: <E1VQzNC-0005ge-8R@project.varnish-software.com>

commit d0fba0072ce2a561f1b11b7db9bc75951961023f
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Sep 20 14:10:18 2013 +0200

    Variable renames for consistency

diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 3659a72..d025237 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -57,19 +57,17 @@ static int
 vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 {
 	const struct vex_rhs *rhs;
-	int reclen;
-	const char *recdata;
-	long long recint;
-	double recfloat;
-	char *endptr;
+	long long lhs_int;
+	double lhs_float;
+	const char *lhs_string;
+	char *p;
 
 	AN(vex);
 	AN(rec);
 	rhs = vex->rhs;
 	AN(rhs);
 
-	reclen = VSL_LEN(rec->ptr);
-	recdata = VSL_CDATA(rec->ptr);
+	lhs_string = VSL_CDATA(rec->ptr);
 
 	/* Prepare */
 	switch (vex->tok) {
@@ -82,19 +80,19 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 		/* Numerical comparison */
 		switch (rhs->type) {
 		case VEX_INT:
-			recint = strtoll(recdata, &endptr, 0);
-			if (*endptr == '\0' || isspace(*endptr))
+			lhs_int = strtoll(lhs_string, &p, 0);
+			if (*p == '\0' || isspace(*p))
 				break;
 			/* Can't parse - no match */
 			return (0);
 		case VEX_FLOAT:
-			recfloat = strtod(recdata, &endptr);
-			if (*endptr == '\0' || isspace(*endptr))
+			lhs_float = strtod(lhs_string, &p);
+			if (*p == '\0' || isspace(*p))
 				break;
 			/* Can't parse - no match */
 			return (0);
 		default:
-			INCOMPL();
+			WRONG("Wrong RHS type");
 		}
 		break;
 	}
@@ -104,11 +102,11 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	case T_EQ:		/* == */
 		switch (rhs->type) {
 		case VEX_INT:
-			if (recint == rhs->val_int)
+			if (lhs_int == rhs->val_int)
 				return (1);
 			return (0);
 		case VEX_FLOAT:
-			if (recfloat == rhs->val_float)
+			if (lhs_float == rhs->val_float)
 				return (1);
 			return (0);
 		default:
@@ -117,11 +115,11 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	case T_NEQ:		/* != */
 		switch (rhs->type) {
 		case VEX_INT:
-			if (recint != rhs->val_int)
+			if (lhs_int != rhs->val_int)
 				return (1);
 			return (0);
 		case VEX_FLOAT:
-			if (recfloat != rhs->val_float)
+			if (lhs_float != rhs->val_float)
 				return (1);
 			return (0);
 		default:
@@ -130,11 +128,11 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	case '<':		/* < */
 		switch (rhs->type) {
 		case VEX_INT:
-			if (recint < rhs->val_int)
+			if (lhs_int < rhs->val_int)
 				return (1);
 			return (0);
 		case VEX_FLOAT:
-			if (recfloat < rhs->val_float)
+			if (lhs_float < rhs->val_float)
 				return (1);
 			return (0);
 		default:
@@ -143,11 +141,11 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	case '>':
 		switch (rhs->type) {
 		case VEX_INT:
-			if (recint > rhs->val_int)
+			if (lhs_int > rhs->val_int)
 				return (1);
 			return (0);
 		case VEX_FLOAT:
-			if (recfloat > rhs->val_float)
+			if (lhs_float > rhs->val_float)
 				return (1);
 			return (0);
 		default:
@@ -156,11 +154,11 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	case T_LEQ:		/* <= */
 		switch (rhs->type) {
 		case VEX_INT:
-			if (recint <= rhs->val_int)
+			if (lhs_int <= rhs->val_int)
 				return (1);
 			return (0);
 		case VEX_FLOAT:
-			if (recfloat <= rhs->val_float)
+			if (lhs_float <= rhs->val_float)
 				return (1);
 			return (0);
 		default:
@@ -169,11 +167,11 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	case T_GEQ:		/* >= */
 		switch (rhs->type) {
 		case VEX_INT:
-			if (recint >= rhs->val_int)
+			if (lhs_int >= rhs->val_int)
 				return (1);
 			return (0);
 		case VEX_FLOAT:
-			if (recfloat >= rhs->val_float)
+			if (lhs_float >= rhs->val_float)
 				return (1);
 			return (0);
 		default:
@@ -181,14 +179,12 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 		}
 	case T_SEQ:		/* eq */
 		assert(rhs->type == VEX_STRING);
-		if (reclen == rhs->val_stringlen + 1 &&
-		    !strncmp(recdata, rhs->val_string, reclen))
+		if (!strcmp(lhs_string, rhs->val_string))
 			return (1);
 		return (0);
 	case T_SNEQ:		/* ne */
 		assert(rhs->type == VEX_STRING);
-		if (reclen != rhs->val_stringlen + 1 ||
-		    strncmp(recdata, rhs->val_string, reclen))
+		if (strcmp(lhs_string, rhs->val_string))
 			return (1);
 		return (0);
 	default:



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 71a5a71 Implement numerical tests by macros to eliminate
 code duplication.
Message-ID: <E1VQzNC-0005gn-9U@project.varnish-software.com>

commit 71a5a7183a938aa41217339c209b5c6c65592e86
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Sep 20 14:30:38 2013 +0200

    Implement numerical tests by macros to eliminate code duplication.

diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index d025237..43eb7ba 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -53,6 +53,20 @@ struct vslq_query {
 	struct vex		*vex;
 };
 
+#define VSLQ_TEST_NUMOP(TYPE, PRE_LHS, OP, PRE_RHS)	\
+	switch (TYPE) {					\
+	case VEX_INT:					\
+		if (PRE_LHS##_int OP PRE_RHS##_int)	\
+			return (1);			\
+		return (0);				\
+	case VEX_FLOAT:					\
+		if (PRE_LHS##_float OP PRE_RHS##_float)	\
+			return (1);			\
+		return (0);				\
+	default:					\
+		WRONG("Wrong RHS type");		\
+	}
+
 static int
 vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 {
@@ -100,83 +114,17 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	/* Compare */
 	switch (vex->tok) {
 	case T_EQ:		/* == */
-		switch (rhs->type) {
-		case VEX_INT:
-			if (lhs_int == rhs->val_int)
-				return (1);
-			return (0);
-		case VEX_FLOAT:
-			if (lhs_float == rhs->val_float)
-				return (1);
-			return (0);
-		default:
-			WRONG("Wrong RHS type");
-		}
+		VSLQ_TEST_NUMOP(rhs->type, lhs, ==, rhs->val);
 	case T_NEQ:		/* != */
-		switch (rhs->type) {
-		case VEX_INT:
-			if (lhs_int != rhs->val_int)
-				return (1);
-			return (0);
-		case VEX_FLOAT:
-			if (lhs_float != rhs->val_float)
-				return (1);
-			return (0);
-		default:
-			WRONG("Wrong RHS type");
-		}
+		VSLQ_TEST_NUMOP(rhs->type, lhs, !=, rhs->val);
 	case '<':		/* < */
-		switch (rhs->type) {
-		case VEX_INT:
-			if (lhs_int < rhs->val_int)
-				return (1);
-			return (0);
-		case VEX_FLOAT:
-			if (lhs_float < rhs->val_float)
-				return (1);
-			return (0);
-		default:
-			WRONG("Wrong RHS type");
-		}
+		VSLQ_TEST_NUMOP(rhs->type, lhs, <, rhs->val);
 	case '>':
-		switch (rhs->type) {
-		case VEX_INT:
-			if (lhs_int > rhs->val_int)
-				return (1);
-			return (0);
-		case VEX_FLOAT:
-			if (lhs_float > rhs->val_float)
-				return (1);
-			return (0);
-		default:
-			WRONG("Wrong RHS type");
-		}
+		VSLQ_TEST_NUMOP(rhs->type, lhs, >, rhs->val);
 	case T_LEQ:		/* <= */
-		switch (rhs->type) {
-		case VEX_INT:
-			if (lhs_int <= rhs->val_int)
-				return (1);
-			return (0);
-		case VEX_FLOAT:
-			if (lhs_float <= rhs->val_float)
-				return (1);
-			return (0);
-		default:
-			WRONG("Wrong RHS type");
-		}
+		VSLQ_TEST_NUMOP(rhs->type, lhs, <=, rhs->val);
 	case T_GEQ:		/* >= */
-		switch (rhs->type) {
-		case VEX_INT:
-			if (lhs_int >= rhs->val_int)
-				return (1);
-			return (0);
-		case VEX_FLOAT:
-			if (lhs_float >= rhs->val_float)
-				return (1);
-			return (0);
-		default:
-			WRONG("Wrong RHS type");
-		}
+		VSLQ_TEST_NUMOP(rhs->type, lhs, >=, rhs->val);
 	case T_SEQ:		/* eq */
 		assert(rhs->type == VEX_STRING);
 		if (!strcmp(lhs_string, rhs->val_string))



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 4e1bc12 Regular expression operators
Message-ID: <E1VQzNC-0005gv-Az@project.varnish-software.com>

commit 4e1bc12c64f0d8fca08eb4ff510f6435f3964e01
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Sep 20 14:56:12 2013 +0200

    Regular expression operators

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index 36609ad..839accf 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -113,3 +113,17 @@ logexpect l1 -d 1 -g vxid -q "RespStatus >= 200." {
 	expect * =	ReqEnd
 	expect * =	End
 } -run
+
+# Test '~' operator
+logexpect l1 -d 1 -g vxid -q "RespStatus ~ '^200$'" {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
+
+# Test '!~' operator
+logexpect l1 -d 1 -g vxid -q "RespStatus !~ '^404$'" {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 43eb7ba..dbf539b 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -74,7 +74,9 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	long long lhs_int;
 	double lhs_float;
 	const char *lhs_string;
+	size_t lhs_stringlen;
 	char *p;
+	int i;
 
 	AN(vex);
 	AN(rec);
@@ -82,6 +84,7 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	AN(rhs);
 
 	lhs_string = VSL_CDATA(rec->ptr);
+	lhs_stringlen = VSL_LEN(rec->ptr) - 1;
 
 	/* Prepare */
 	switch (vex->tok) {
@@ -127,16 +130,32 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 		VSLQ_TEST_NUMOP(rhs->type, lhs, >=, rhs->val);
 	case T_SEQ:		/* eq */
 		assert(rhs->type == VEX_STRING);
-		if (!strcmp(lhs_string, rhs->val_string))
+		if (lhs_stringlen == rhs->val_stringlen &&
+		    !strncmp(lhs_string, rhs->val_string, lhs_stringlen))
 			return (1);
 		return (0);
 	case T_SNEQ:		/* ne */
 		assert(rhs->type == VEX_STRING);
-		if (strcmp(lhs_string, rhs->val_string))
+		if (lhs_stringlen != rhs->val_stringlen ||
+		    strncmp(lhs_string, rhs->val_string, lhs_stringlen))
+			return (1);
+		return (0);
+	case '~':		/* ~ */
+		assert(rhs->type == VEX_REGEX && rhs->val_regex != NULL);
+		i = VRE_exec(rhs->val_regex, lhs_string, lhs_stringlen, 0, 0,
+		    NULL, 0, NULL);
+		if (i != VRE_ERROR_NOMATCH)
+			return (1);
+		return (0);
+	case T_NOMATCH:		/* !~ */
+		assert(rhs->type == VEX_REGEX && rhs->val_regex != NULL);
+		i = VRE_exec(rhs->val_regex, lhs_string, lhs_stringlen, 0, 0,
+		    NULL, 0, NULL);
+		if (i == VRE_ERROR_NOMATCH)
 			return (1);
 		return (0);
 	default:
-		INCOMPL();
+		WRONG("Bad expression token");
 	}
 
 	return (0);



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 22c4b13 Add tests for the boolean operators and fix not
 operator
Message-ID: <E1VQzNC-0005h3-C3@project.varnish-software.com>

commit 22c4b1304d91c2554d77d72e520b43797e493b37
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Sep 20 15:14:26 2013 +0200

    Add tests for the boolean operators and fix not operator

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index 839accf..9b3197d 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -127,3 +127,24 @@ logexpect l1 -d 1 -g vxid -q "RespStatus !~ '^404$'" {
 	expect * =	ReqEnd
 	expect * =	End
 } -run
+
+# Test boolean and
+logexpect l1 -d 1 -g vxid -q "RespStatus == 200 and RespStatus ~ '^200$'" {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
+
+# Test boolean or
+logexpect l1 -d 1 -g vxid -q "RespStatus == 404 or RespStatus ~ '^200$'" {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
+
+# Test boolean !
+logexpect l1 -d 1 -g vxid -q "RespStatus == 404 or !RespStatus ~ '^404$'" {
+	expect * *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
diff --git a/lib/libvarnishapi/generate.py b/lib/libvarnishapi/generate.py
index 1377f71..6245ef8 100755
--- a/lib/libvarnishapi/generate.py
+++ b/lib/libvarnishapi/generate.py
@@ -58,7 +58,6 @@ tokens = {
         "T_NOMATCH":    "!~",
 
         # Boolean operators
-        "T_NOT":        "not",
         "T_AND":        "and",
         "T_OR":         "or",
 
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index dbf539b..ba3d23b 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -218,7 +218,7 @@ vslq_exec(const struct vex *vex, struct VSL_transaction * const ptrans[])
 		if (r <= 0)
 			return (r);
 		return (vslq_exec(vex->b, ptrans));
-	case T_NOT:
+	case '!':
 		AN(vex->a);
 		AZ(vex->b);
 		r = vslq_exec(vex->a, ptrans);



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 7b4e6cb Improve test cases by anchoring the expected lines
 to the first
Message-ID: <E1VQzNC-0005hB-DQ@project.varnish-software.com>

commit 7b4e6cbf8f9fd1efb6a7cce667b2e7ed46870989
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Sep 20 16:29:06 2013 +0200

    Improve test cases by anchoring the expected lines to the first

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index 9b3197d..0de5614 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -18,133 +18,133 @@ client c1 {
 
 # Test 'eq' operator
 logexpect l1 -d 1 -g vxid -q "Begin eq 'req 1000'" {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test 'ne' operator
 logexpect l1 -d 1 -g vxid -q "ReqProtocol ne 'HTTP/1.0'" {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '==' operator on integers
 logexpect l1 -d 1 -g vxid -q "RespStatus == 200" {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '==' operator on floats
 logexpect l1 -d 1 -g vxid -q "RespStatus == 200." {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '!=' operator on integers
 logexpect l1 -d 1 -g vxid -q "RespStatus != 503" {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '!=' operator on floats
 logexpect l1 -d 1 -g vxid -q "RespStatus != 503." {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '<' operator on integers
 logexpect l1 -d 1 -g vxid -q "RespStatus < 201" {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '<' operator on floats
 logexpect l1 -d 1 -g vxid -q "RespStatus < 201." {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '>' operator on integers
 logexpect l1 -d 1 -g vxid -q "RespStatus > 199" {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '>' operator on floats
 logexpect l1 -d 1 -g vxid -q "RespStatus > 199." {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '<=' operator on integers
 logexpect l1 -d 1 -g vxid -q "RespStatus <= 200" {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '<=' operator on floats
 logexpect l1 -d 1 -g vxid -q "RespStatus <= 200." {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '>=' operator on integers
 logexpect l1 -d 1 -g vxid -q "RespStatus >= 200" {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '>=' operator on floats
 logexpect l1 -d 1 -g vxid -q "RespStatus >= 200." {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '~' operator
 logexpect l1 -d 1 -g vxid -q "RespStatus ~ '^200$'" {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test '!~' operator
 logexpect l1 -d 1 -g vxid -q "RespStatus !~ '^404$'" {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test boolean and
 logexpect l1 -d 1 -g vxid -q "RespStatus == 200 and RespStatus ~ '^200$'" {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test boolean or
 logexpect l1 -d 1 -g vxid -q "RespStatus == 404 or RespStatus ~ '^200$'" {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run
 
 # Test boolean !
 logexpect l1 -d 1 -g vxid -q "RespStatus == 404 or !RespStatus ~ '^404$'" {
-	expect * *	Begin	req
+	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
 } -run



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 66c12d8 Add grouping and and/or precedence tests
Message-ID: <E1VQzNC-0005hJ-ET@project.varnish-software.com>

commit 66c12d8feedce91eb67f81b4062fbc46df6a6844
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Sep 20 16:29:35 2013 +0200

    Add grouping and and/or precedence tests

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index 0de5614..4d8c12d 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -148,3 +148,17 @@ logexpect l1 -d 1 -g vxid -q "RespStatus == 404 or !RespStatus ~ '^404$'" {
 	expect * =	ReqEnd
 	expect * =	End
 } -run
+
+# Test grouping
+logexpect l1 -d 1 -g vxid -q "(RespStatus == 200 or RespStatus == 404) and RespStatus == 200" {
+	expect 0 *	Begin	req
+	expect * =	ReqEnd
+	expect * =	End
+} -run
+
+# Test and/or precedence
+logexpect l1 -d 1 -g vxid -q "RespStatus == 200 or RespStatus == 503 and RespStatus == 404" {
+	  expect 0 *	Begin	req
+	  expect * =	ReqEnd
+	  expect * =	End
+} -run



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 78c82e5 Add field operator
Message-ID: <E1VQzNC-0005hR-Fo@project.varnish-software.com>

commit 78c82e5efe0db40614f749932591fa45bb4ea679
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Sep 20 18:53:39 2013 +0200

    Add field operator

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index 4d8c12d..3030d3e 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -6,6 +6,9 @@ server s1 {
 } -start
 
 varnish v1 -vcl+backend {
+	sub vcl_deliver {
+		set resp.http.x-test = "123 321";
+	}
 } -start
 
 logexpect l1 -v v1
@@ -162,3 +165,10 @@ logexpect l1 -d 1 -g vxid -q "RespStatus == 200 or RespStatus == 503 and RespSta
 	  expect * =	ReqEnd
 	  expect * =	End
 } -run
+
+# Test field
+logexpect l1 -d 1 -g vxid -q "RespHeader[2] == 123" {
+	  expect 0 *	Begin	req
+	  expect * =	ReqEnd
+	  expect * =	End
+} -run
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index ba3d23b..e42705a 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -73,8 +73,7 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	const struct vex_rhs *rhs;
 	long long lhs_int;
 	double lhs_float;
-	const char *lhs_string;
-	size_t lhs_stringlen;
+	const char *b, *e;
 	char *p;
 	int i;
 
@@ -83,8 +82,26 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	rhs = vex->rhs;
 	AN(rhs);
 
-	lhs_string = VSL_CDATA(rec->ptr);
-	lhs_stringlen = VSL_LEN(rec->ptr) - 1;
+	b = VSL_CDATA(rec->ptr);
+	e = b + VSL_LEN(rec->ptr) - 1;
+
+	/* Field */
+	if (vex->lhs->field > 0) {
+		for (e = b, i = 0; *e && i < vex->lhs->field; i++) {
+			b = e;
+			/* Skip ws */
+			while (*b && isspace(*b))
+				b++;
+			e = b;
+			/* Skip non-ws */
+			while (*e && !isspace(*e))
+				e++;
+		}
+		assert(b <= e);
+		if (*b == '\0' || i < vex->lhs->field)
+			/* Missing field - no match */
+			return (0);
+	}
 
 	/* Prepare */
 	switch (vex->tok) {
@@ -95,15 +112,18 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	case T_LEQ:		/* <= */
 	case T_GEQ:		/* >= */
 		/* Numerical comparison */
+		if (*b == '\0')
+			/* Empty string doesn't match */
+			return (0);
 		switch (rhs->type) {
 		case VEX_INT:
-			lhs_int = strtoll(lhs_string, &p, 0);
+			lhs_int = strtoll(b, &p, 0);
 			if (*p == '\0' || isspace(*p))
 				break;
 			/* Can't parse - no match */
 			return (0);
 		case VEX_FLOAT:
-			lhs_float = strtod(lhs_string, &p);
+			lhs_float = strtod(b, &p);
 			if (*p == '\0' || isspace(*p))
 				break;
 			/* Can't parse - no match */
@@ -130,27 +150,25 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 		VSLQ_TEST_NUMOP(rhs->type, lhs, >=, rhs->val);
 	case T_SEQ:		/* eq */
 		assert(rhs->type == VEX_STRING);
-		if (lhs_stringlen == rhs->val_stringlen &&
-		    !strncmp(lhs_string, rhs->val_string, lhs_stringlen))
+		if (e - b == rhs->val_stringlen &&
+		    !strncmp(b, rhs->val_string, e - b))
 			return (1);
 		return (0);
 	case T_SNEQ:		/* ne */
 		assert(rhs->type == VEX_STRING);
-		if (lhs_stringlen != rhs->val_stringlen ||
-		    strncmp(lhs_string, rhs->val_string, lhs_stringlen))
+		if (e - b != rhs->val_stringlen ||
+		    strncmp(b, rhs->val_string, e - b))
 			return (1);
 		return (0);
 	case '~':		/* ~ */
 		assert(rhs->type == VEX_REGEX && rhs->val_regex != NULL);
-		i = VRE_exec(rhs->val_regex, lhs_string, lhs_stringlen, 0, 0,
-		    NULL, 0, NULL);
+		i = VRE_exec(rhs->val_regex, b, e - b, 0, 0, NULL, 0, NULL);
 		if (i != VRE_ERROR_NOMATCH)
 			return (1);
 		return (0);
 	case T_NOMATCH:		/* !~ */
 		assert(rhs->type == VEX_REGEX && rhs->val_regex != NULL);
-		i = VRE_exec(rhs->val_regex, lhs_string, lhs_stringlen, 0, 0,
-		    NULL, 0, NULL);
+		i = VRE_exec(rhs->val_regex, b, e - b, 0, 0, NULL, 0, NULL);
 		if (i == VRE_ERROR_NOMATCH)
 			return (1);
 		return (0);
diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index 2cc1288..aad8bde 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -50,6 +50,7 @@ static void vxp_expr_or(struct vxp *vxp, struct vex **pvex);
 static void
 vxp_expr_lhs(struct vxp *vxp, struct vex_lhs **plhs)
 {
+	char *p;
 
 	/* XXX: Tag wildcards */
 	AN(plhs);
@@ -75,7 +76,27 @@ vxp_expr_lhs(struct vxp *vxp, struct vex_lhs **plhs)
 	}
 	vxp_NextToken(vxp);
 
-	/* XXX: Lhs limiting operators ([], {}) */
+	if (vxp->t->tok == '[') {
+		/* LHS field [] */
+		vxp_NextToken(vxp);
+		if (vxp->t->tok != VAL) {
+			VSB_printf(vxp->sb, "Expected integer got '%.*s' ",
+			    PF(vxp->t));
+			vxp_ErrWhere(vxp, vxp->t, -1);
+			return;
+		}
+		(*plhs)->field = (int)strtol(vxp->t->dec, &p, 0);
+		if (*p || (*plhs)->field <= 0) {
+			VSB_printf(vxp->sb, "Expected positive integer");
+			vxp_ErrWhere(vxp, vxp->t, -1);
+			return;
+		}
+		vxp_NextToken(vxp);
+		ExpectErr(vxp, ']');
+		vxp_NextToken(vxp);
+	}
+
+	/* XXX: LHS Level {} */
 }
 
 static void
@@ -460,6 +481,8 @@ vex_print(const struct vex *vex, int indent)
 	if (vex->lhs != NULL) {
 		CHECK_OBJ_NOTNULL(vex->lhs, VEX_LHS_MAGIC);
 		fprintf(stderr, " tag=%s", VSL_tags[vex->lhs->tag]);
+		if (vex->lhs->field >= 0)
+			fprintf(stderr, "[%d]", vex->lhs->field);
 	}
 	if (vex->rhs != NULL) {
 		fprintf(stderr, " ");



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] dbfcff3 Add SLT__MAX define (256) for max tag of a VSL record
Message-ID: <E1VQzNC-0005ha-H8@project.varnish-software.com>

commit dbfcff3000138638774dee211ae384147c577855
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Sep 23 10:31:16 2013 +0200

    Add SLT__MAX define (256) for max tag of a VSL record

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 27d3b87..feb2faa 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -95,7 +95,7 @@ typedef int VSLQ_dispatch_f(struct VSL_data *vsl,
 	 *   !=0: Makes VSLQ_Dispatch return with this return value immediatly
 	 */
 
-extern const char *VSL_tags[256];
+extern const char *VSL_tags[SLT__MAX];
 	/*
 	 * Tag to string array.  Contains NULL for invalid tags.
 	 */
diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h
index 196bf98..02a9cb0 100644
--- a/include/vapi/vsl_int.h
+++ b/include/vapi/vsl_int.h
@@ -104,6 +104,7 @@ struct VSL_head {
  * The identifiers in shmlogtag are "SLT_" + XML tag.  A script may be run
  * on this file to extract the table rather than handcode it
  */
+#define SLT__MAX 256
 enum VSL_tag_e {
 	SLT__Bogus = 0,
 #define SLTM(foo,sdesc,ldesc)	SLT_##foo,
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index d7a5fa8..a5c9108 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -57,7 +57,7 @@
 
 /*--------------------------------------------------------------------*/
 
-const char *VSL_tags[256] = {
+const char *VSL_tags[SLT__MAX] = {
 #  define SLTM(foo,sdesc,ldesc)       [SLT_##foo] = #foo,
 #  include "tbl/vsl_tags.h"
 #  undef SLTM
@@ -91,8 +91,8 @@ VSL_New(void)
 	if (vsl == NULL)
 		return (NULL);
 
-	vsl->vbm_select = vbit_init(256);
-	vsl->vbm_supress = vbit_init(256);
+	vsl->vbm_select = vbit_init(SLT__MAX);
+	vsl->vbm_supress = vbit_init(SLT__MAX);
 	VTAILQ_INIT(&vsl->vslf_select);
 	VTAILQ_INIT(&vsl->vslf_suppress);
 
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 00039ad..8a2c48d 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -67,7 +67,7 @@ VSL_Name2Tag(const char *name, int l)
 	if (l == -1)
 		l = strlen(name);
 	n = -1;
-	for (i = 0; i < 256; i++) {
+	for (i = 0; i < SLT__MAX; i++) {
 		if (VSL_tags[i] != NULL &&
 		    !strncasecmp(name, VSL_tags[i], l)) {
 			if (strlen(VSL_tags[i]) == l) {
@@ -220,7 +220,7 @@ VSL_Arg(struct VSL_data *vsl, int opt, const char *arg)
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 	/* If first option is 'i', set all bits for supression */
 	if ((opt == 'i' || opt == 'I') && !(vsl->flags & F_SEEN_ixIX))
-		for (i = 0; i < 256; i++)
+		for (i = 0; i < SLT__MAX; i++)
 			vbit_set(vsl->vbm_supress, i);
 
 	switch (opt) {



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 8bd29f9 Don't hard code array size
Message-ID: <E1VQzNC-0005hi-IK@project.varnish-software.com>

commit 8bd29f9cb8b40f000af59f2b9ebbb7baaffcd2c8
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Sep 23 13:28:37 2013 +0200

    Don't hard code array size

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index feb2faa..8aa9152 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -63,6 +63,7 @@ enum VSL_transaction_e {
 	VSL_t_esireq,
 	VSL_t_bereq,
 	VSL_t_raw,
+	VSL_t__MAX,
 };
 
 struct VSL_transaction {
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index a5c9108..0889f54 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -201,7 +201,7 @@ VSL_Match(struct VSL_data *vsl, const struct VSL_cursor *c)
 	return (1);
 }
 
-static const char * const VSL_transactions[256] = {
+static const char * const VSL_transactions[VSL_t__MAX] = {
 	/*                 12345678901234 */
 	[VSL_t_unknown] = "<< Unknown  >>",
 	[VSL_t_sess]	= "<< Session  >>",



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] e2fe0d8 Fix linebreak
Message-ID: <E1VQzNC-0005hq-Ji@project.varnish-software.com>

commit e2fe0d8cebe0cd4aec296e1fc59806e4482f7612
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Sep 23 13:55:58 2013 +0200

    Fix linebreak

diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index 46f5004..5b7307c 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -94,4 +94,5 @@ struct vslq_query;
 struct vslq_query *vslq_newquery(struct VSL_data *vsl,
     enum VSL_grouping_e grouping, const char *query);
 void vslq_deletequery(struct vslq_query **pquery);
-int vslq_runquery(const struct vslq_query *query, struct VSL_transaction * const ptrans[]);
+int vslq_runquery(const struct vslq_query *query,
+    struct VSL_transaction * const ptrans[]);



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 787a4e9 Update documentation of VSL_Name2Tag
Message-ID: <E1VQzNC-0005hy-L9@project.varnish-software.com>

commit 787a4e9d83f7c2f9b8ce7999d4c57cf137884cca
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Sep 24 12:35:51 2013 +0200

    Update documentation of VSL_Name2Tag

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 8aa9152..7f6f98d 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -103,7 +103,13 @@ extern const char *VSL_tags[SLT__MAX];
 
 int VSL_Name2Tag(const char *name, int l);
 	/*
-	 * Convert string to tag number (= enum VSL_tag_e)
+	 * Convert string to tag number (= enum VSL_tag_e). Name can be a
+	 * substring from the beginning of a tag when that substring is
+	 * unique. Matching is case insensitive.
+	 *
+	 * Arguments:
+	 *    name:	A tag name (or substring) to match against
+	 *       l:	The length of name, or -1 to use strlen.
 	 *
 	 * Return values:
 	 *	>=0:	Tag number



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] a2519bc Add a VSL_Glob2Tags function to the API.
Message-ID: <E1VQzNC-0005i6-MJ@project.varnish-software.com>

commit a2519bc9477791baa4ac60792443bdb1b2f297be
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Sep 24 12:31:46 2013 +0200

    Add a VSL_Glob2Tags function to the API.
    
    VSL_Glob2Tags allows matching VSL tags against a glob expression.

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 7f6f98d..7c998de 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -96,6 +96,15 @@ typedef int VSLQ_dispatch_f(struct VSL_data *vsl,
 	 *   !=0: Makes VSLQ_Dispatch return with this return value immediatly
 	 */
 
+typedef void VSL_glob2tags_f(int tag, void *priv);
+	/*
+	 * The callback function type for use with VSL_Glob2Tags.
+	 *
+	 * Arguments:
+	 *    tag: Tag number (= enum VSL_tag_e)
+	 *   priv: The priv argument from VSL_Glob2Tag
+	 */
+
 extern const char *VSL_tags[SLT__MAX];
 	/*
 	 * Tag to string array.  Contains NULL for invalid tags.
@@ -117,6 +126,25 @@ int VSL_Name2Tag(const char *name, int l);
 	 *	-2:	Multiple tags match substring
 	 */
 
+int VSL_Glob2Tags(const char *glob, int l, VSL_glob2tags_f *func, void *priv);
+	/*
+	 * Convert a string to multiple tag matches. The string can have
+	 * either a prefix or postfix wildcard (*) character. For each
+	 * matching tag func is called. Matching is done case insensitive.
+	 *
+	 * Arguments:
+	 *   glob: The string to match
+	 *      l: The length of glob. -1 to use strlen.
+	 *   func: The function to call (can be NULL)
+	 *   priv: An argument that will be passed to func.
+	 *
+	 * Return values:
+	 *     >0: Number of times func was called for matching tags.
+	 *     -1: No tag matches
+	 *     -2: Multiple tags match non-glob input
+	 *     -3: Syntax error
+	 */
+
 int VSLQ_Name2Grouping(const char *name, int l);
 	/*
 	 * Convert string to grouping (= enum VSL_grouping_e)
diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 384c0e1..42315a0 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -115,5 +115,6 @@ LIBVARNISHAPI_1.3 {
 	VSLQ_Dispatch;
 	VSLQ_Flush;
 	VSLQ_Name2Grouping;
+	VSL_Glob2Tags;
 	# Variables:
 } LIBVARNISHAPI_1.0;
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 8a2c48d..da54295 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -83,6 +83,75 @@ VSL_Name2Tag(const char *name, int l)
 	return (n);
 }
 
+int
+VSL_Glob2Tags(const char *glob, int l, VSL_glob2tags_f *func, void *priv)
+{
+	int i, r, l2;
+	int pre = 0;
+	int post = 0;
+	char buf[64];
+
+	AN(glob);
+	if (l < 0)
+		l = strlen(glob);
+	if (l == 0 || l > sizeof buf - 1)
+		return (-1);
+	if (strchr(glob, '*') != NULL) {
+		if (glob[0] == '*') {
+			/* Prefix wildcard */
+			pre = 1;
+			glob++;
+			l--;
+		}
+		if (l > 0 && glob[l - 1] == '*') {
+			/* Postfix wildcard */
+			post = 1;
+			l--;
+		}
+	}
+	if (pre && post)
+		/* Support only post or prefix wildcards */
+		return (-3);
+	memcpy(buf, glob, l);
+	buf[l] = '\0';
+	if (strchr(buf, '*') != NULL)
+		/* No multiple wildcards */
+		return (-3);
+	if (pre == 0 && post == 0) {
+		/* No wildcards, use VSL_Name2Tag */
+		i = VSL_Name2Tag(buf, l);
+		if (i < 0)
+			return (i);
+		if (func != NULL)
+			(func)(i, priv);
+		return (1);
+	}
+
+	r = 0;
+	for (i = 0; i < SLT__MAX; i++) {
+		if (VSL_tags[i] == NULL)
+			continue;
+		l2 = strlen(VSL_tags[i]);
+		if (l2 < l)
+			continue;
+		if (pre) {
+			/* Prefix wildcard match */
+			if (strcasecmp(buf, VSL_tags[i] + l2 - l))
+				continue;
+		} else {
+			/* Postfix wildcard match */
+			if (strncasecmp(buf, VSL_tags[i], l))
+				continue;
+		}
+		if (func != NULL)
+			(func)(i, priv);
+		r++;
+	}
+	if (r == 0)
+		return (-1);
+	return (r);
+}
+
 static const char * const vsl_grouping[] = {
 	[VSL_g_raw]	= "raw",
 	[VSL_g_vxid]	= "vxid",



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 570a04f Add a VSL_List2Tags function for finding tags.
Message-ID: <E1VQzNC-0005iE-Ng@project.varnish-software.com>

commit 570a04f5cd88f8fdd61211236b15f839e4d3a1ca
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Sep 24 15:03:41 2013 +0200

    Add a VSL_List2Tags function for finding tags.
    
    This function takes a commaseparated list and feeds each part to
    VSL_Glob2Tags.

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 7c998de..4371521 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -96,13 +96,14 @@ typedef int VSLQ_dispatch_f(struct VSL_data *vsl,
 	 *   !=0: Makes VSLQ_Dispatch return with this return value immediatly
 	 */
 
-typedef void VSL_glob2tags_f(int tag, void *priv);
+typedef void VSL_tagfind_f(int tag, void *priv);
 	/*
-	 * The callback function type for use with VSL_Glob2Tags.
+	 * The callback function type for use with VSL_Glob2Tags and
+	 * VSL_List2Tags..
 	 *
 	 * Arguments:
 	 *    tag: Tag number (= enum VSL_tag_e)
-	 *   priv: The priv argument from VSL_Glob2Tag
+	 *   priv: The priv argument
 	 */
 
 extern const char *VSL_tags[SLT__MAX];
@@ -126,7 +127,7 @@ int VSL_Name2Tag(const char *name, int l);
 	 *	-2:	Multiple tags match substring
 	 */
 
-int VSL_Glob2Tags(const char *glob, int l, VSL_glob2tags_f *func, void *priv);
+int VSL_Glob2Tags(const char *glob, int l, VSL_tagfind_f *func, void *priv);
 	/*
 	 * Convert a string to multiple tag matches. The string can have
 	 * either a prefix or postfix wildcard (*) character. For each
@@ -145,6 +146,25 @@ int VSL_Glob2Tags(const char *glob, int l, VSL_glob2tags_f *func, void *priv);
 	 *     -3: Syntax error
 	 */
 
+int VSL_List2Tags(const char *list, int l, VSL_tagfind_f *func, void *priv);
+	/*
+	 * Convert a comma-separated list of tag globs to tag
+	 * matches. Calls VSL_Glob2Tags for each comma-separated part of
+	 * list.
+	 *
+	 * Arguments:
+	 *   list: The list of globs
+	 *      l: The length of list. -1 to use strlen
+	 *   func: The function to call (can be NULL)
+	 *   priv: An argument that will be passed to func.
+	 *
+	 * Return valus:
+	 *     >0: Number of times func was called for matching tags.
+	 *     -1: No tag matches for list element
+	 *     -2: Multiple tags match non-glob list element
+	 *     -3: Syntax error
+	 */
+
 int VSLQ_Name2Grouping(const char *name, int l);
 	/*
 	 * Convert string to grouping (= enum VSL_grouping_e)
diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 42315a0..8862129 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -116,5 +116,6 @@ LIBVARNISHAPI_1.3 {
 	VSLQ_Flush;
 	VSLQ_Name2Grouping;
 	VSL_Glob2Tags;
+	VSL_List2Tags;
 	# Variables:
 } LIBVARNISHAPI_1.0;
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index da54295..efc2eb8 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -84,7 +84,7 @@ VSL_Name2Tag(const char *name, int l)
 }
 
 int
-VSL_Glob2Tags(const char *glob, int l, VSL_glob2tags_f *func, void *priv)
+VSL_Glob2Tags(const char *glob, int l, VSL_tagfind_f *func, void *priv)
 {
 	int i, r, l2;
 	int pre = 0;
@@ -152,6 +152,36 @@ VSL_Glob2Tags(const char *glob, int l, VSL_glob2tags_f *func, void *priv)
 	return (r);
 }
 
+int
+VSL_List2Tags(const char *list, int l, VSL_tagfind_f *func, void *priv)
+{
+	const char *p, *q, *e;
+	int r, t;
+
+	if (l < 0)
+		l = strlen(list);
+	p = list;
+	e = p + l;
+	t = 0;
+	while (p < e) {
+		while (p < e && *p == ',')
+			p++;
+		if (p == e)
+			break;
+		q = p;
+		while (q < e && *q != ',')
+			q++;
+		r = VSL_Glob2Tags(p, q - p, func, priv);
+		if (r < 0)
+			return (r);
+		t += r;
+		p = q;
+	}
+	if (t == 0)
+		return (-1);
+	return (t);
+}
+
 static const char * const vsl_grouping[] = {
 	[VSL_g_raw]	= "raw",
 	[VSL_g_vxid]	= "vxid",



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 8b66fc6 Add a VSL tag globbing functions test program.
Message-ID: <E1VQzNC-0005iM-PS@project.varnish-software.com>

commit 8b66fc69ed998a0b343e62cc117f05f3be2bc7c8
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Sep 24 12:36:41 2013 +0200

    Add a VSL tag globbing functions test program.

diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index 375ab29..6f36ef6 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -65,7 +65,7 @@ vxp_fixed_token.c vxp_tokens.h: \
 	$(srcdir)/generate.py
 	@PYTHON@ $(srcdir)/generate.py $(srcdir) $(top_builddir)
 
-EXTRA_PROGRAMS = vxp_test
+EXTRA_PROGRAMS = vxp_test vsl_glob_test
 
 vxp_test_LDADD = @PCRE_LIBS@ \
 	${RT_LIBS} ${LIBM} ${PTHREAD_LIBS}
@@ -77,3 +77,10 @@ vxp_test_CFLAGS = \
 vxp_test_SOURCES = \
 	$(libvarnishapi_la_SOURCES) \
 	vxp_test.c
+
+vsl_glob_test_SOURCES = \
+	vsl_glob_test.c
+
+vsl_glob_test_LDADD = @PCRE_LIBS@ ${RT_LIBS} ${LIBM} libvarnishapi.la
+
+vsl_glob_test_CFLAGS = -I$(top_srcdir)/include
diff --git a/lib/libvarnishapi/vsl_glob_test.c b/lib/libvarnishapi/vsl_glob_test.c
new file mode 100644
index 0000000..87bb6fa
--- /dev/null
+++ b/lib/libvarnishapi/vsl_glob_test.c
@@ -0,0 +1,73 @@
+/*-
+ * Copyright (c) 2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Test what VSL_Name2Tag and VSL_Glob2Tags produces
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+
+#include "vas.h"
+#include "vapi/vsl.h"
+
+static void
+cb(int tag, void *priv)
+{
+	(void)priv;
+
+	printf("\t%d (%s)\n", tag, VSL_tags[tag]);
+}
+
+int
+main(int argc, char *argv[])
+{
+	int i;
+
+	if (argc != 2) {
+		fprintf(stderr, "vsl_glob_test <tagname/glob>\n");
+		exit(1);
+	}
+
+	i = VSL_Name2Tag(argv[1], -1);
+	printf("VSL_Name2Tag returns %d", i);
+	if (i >= 0)
+		printf(" (%s)", VSL_tags[i]);
+	printf("\n");
+
+	printf("VSL_Glob2Tags:\n");
+	i = VSL_Glob2Tags(argv[1], -1, cb, NULL);
+	printf("VSL_Glob2Tags returns %d\n", i);
+
+	printf("VSL_List2Tags:\n");
+	i = VSL_List2Tags(argv[1], -1, cb, NULL);
+	printf("VSL_List2Tags returns %d\n", i);
+
+	return (0);
+}



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 79b6333 Use VSL_List2Tags in -ix option processing
Message-ID: <E1VQzNC-0005iU-Qm@project.varnish-software.com>

commit 79b633308d440b60484de6a14227ba3f71f147c9
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Sep 24 13:03:58 2013 +0200

    Use VSL_List2Tags in -ix option processing

diff --git a/include/vapi/vapi_options.h b/include/vapi/vapi_options.h
index df3dc83..6eceaf6 100644
--- a/include/vapi/vapi_options.h
+++ b/include/vapi/vapi_options.h
@@ -66,8 +66,9 @@
 	)
 
 #define VSL_OPT_i							\
-	VOPT("i:", "[-i tag]", "Include tag",				\
-	    "Include log records of this tag in output. Multiple -i"	\
+	VOPT("i:", "[-i taglist]", "Include tags",			\
+	    "Include log records of these tags in output. Taglist is"   \
+	    " a comma-separated list of tag globs. Multiple -i"		\
 	    " options may be given.\n"					\
 	    "\n"							\
 	    VSL_iI_PS							\
@@ -109,9 +110,10 @@
 	)
 
 #define VSL_OPT_x							\
-	VOPT("x:", "[-x tag]", "Exclude tag",				\
-	    "Exclude log records of this tag in output. Multiple -x"	\
-	    " options may be given."					\
+	VOPT("x:", "[-x taglist]", "Exclude tags",			\
+	    "Exclude log records of these tags in output. Taglist is"   \
+	    " a comma-separated list of tag globs. Multiple -x"		\
+	    " options may be given.\n"					\
 	)
 
 #define VSL_OPT_X							\
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index efc2eb8..b32a470 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -212,42 +212,40 @@ VSLQ_Name2Grouping(const char *name, int l)
 	return (n);
 }
 
+static void
+vsl_vbm_bitset(int bit, void *priv)
+{
+
+	vbit_set((struct vbitmap *)priv, bit);
+}
+
+static void
+vsl_vbm_bitclr(int bit, void *priv)
+{
+
+	vbit_clr((struct vbitmap *)priv, bit);
+}
+
 static int
 vsl_ix_arg(struct VSL_data *vsl, int opt, const char *arg)
 {
-	int i, l;
-	const char *b, *e;
+	int i;
 
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 	vsl->flags |= F_SEEN_ixIX;
 
-	for (b = arg; *b; b = e) {
-		while (isspace(*b))
-			b++;
-		e = strchr(b, ',');
-		if (e == NULL)
-			e = strchr(b, '\0');
-		l = e - b;
-		if (*e == ',')
-			e++;
-		while (isspace(b[l - 1]))
-			l--;
-		i = VSL_Name2Tag(b, l);
-		if (i >= 0) {
-			if (opt == 'x')
-				vbit_set(vsl->vbm_supress, i);
-			else
-				vbit_clr(vsl->vbm_supress, i);
-		} else if (i == -2) {
-			return (vsl_diag(vsl,
-			    "-%c: \"%*.*s\" matches multiple tags\n",
-			    (char)opt, l, l, b));
-		} else {
-			return (vsl_diag(vsl,
-			    "-%c: Could not match \"%*.*s\" to any tag\n",
-			    (char)opt, l, l, b));
-		}
-	}
+	i = VSL_List2Tags(arg, -1, opt == 'x' ? vsl_vbm_bitset : vsl_vbm_bitclr,
+	    vsl->vbm_supress);
+	if (i == -1)
+		return (vsl_diag(vsl, "-%c: \"%s\" matches zero tags",
+			(char)opt, arg));
+	else if (i == -2)
+		return (vsl_diag(vsl, "-%c: \"%s\" is ambiguous",
+			(char)opt, arg));
+	else if (i == -3)
+		return (vsl_diag(vsl, "-%c: Syntax error in \"%s\"",
+			(char)opt, arg));
+
 	return (1);
 }
 



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] a1726ad Use VSL_List2Tags in -IX option processing.
Message-ID: <E1VQzNC-0005ic-S7@project.varnish-software.com>

commit a1726adcdaf8439a25b42e89bf585df1e12f8476
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Sep 24 17:07:14 2013 +0200

    Use VSL_List2Tags in -IX option processing.
    
    Use VSL_List2Tags in -IX option processing so that the filters can be
    applied to a list of tags.

diff --git a/include/vapi/vapi_options.h b/include/vapi/vapi_options.h
index 6eceaf6..c9e6e02 100644
--- a/include/vapi/vapi_options.h
+++ b/include/vapi/vapi_options.h
@@ -75,10 +75,10 @@
 	)
 
 #define VSL_OPT_I							\
-	VOPT("I:", "[-I <[tag:]regex>]", "Include by regex",		\
+	VOPT("I:", "[-I <[taglist:]regex>]", "Include by regex",	\
 	    "Include by regex matching. Output only records matching"	\
-	    " tag and regular expression. Applies to any tag if tag"	\
-	    " is * or empty.\n"						\
+	    " taglist and regular expression. Applies to any tag if"	\
+	    " taglist is absent.\n"					\
 	    "\n"							\
 	    VSL_iI_PS							\
 	)
@@ -117,8 +117,8 @@
 	)
 
 #define VSL_OPT_X							\
-	VOPT("X:", "[-X <[tag:]regex>]", "Exclude by regex",		\
+	VOPT("X:", "[-X <[taglist:]regex>]", "Exclude by regex",	\
 	    "Exclude by regex matching. Do not output records matching"	\
-	    " tag and regular expression. Applies to any tag if tag"	\
-	    " is * or empty."						\
+	    " taglist and regular expression. Applies to any tag if"	\
+	    " taglist is absent."					\
 	)
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 0889f54..cfb424a 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -108,6 +108,8 @@ vsl_IX_free(vslf_list *list)
 		vslf = VTAILQ_FIRST(list);
 		CHECK_OBJ_NOTNULL(vslf, VSLF_MAGIC);
 		VTAILQ_REMOVE(list, vslf, list);
+		if (vslf->tags)
+			vbit_destroy(vslf->tags);
 		AN(vslf->vre);
 		VRE_free(&vslf->vre);
 		AZ(vslf->vre);
@@ -167,7 +169,7 @@ vsl_match_IX(struct VSL_data *vsl, const vslf_list *list, const struct VSL_curso
 
 	VTAILQ_FOREACH(vslf, list, list) {
 		CHECK_OBJ_NOTNULL(vslf, VSLF_MAGIC);
-		if (vslf->tag >= 0 && vslf->tag != tag)
+		if (vslf->tags != NULL && !vbit_test(vslf->tags, tag))
 			continue;
 		if (VRE_exec(vslf->vre, cdata, len, 0, 0, NULL, 0, NULL) >= 0)
 			return (1);
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index 5b7307c..7094fec 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -63,7 +63,7 @@ struct vslf {
 #define VSLF_MAGIC			0x08650B39
 	VTAILQ_ENTRY(vslf)		list;
 
-	int				tag;
+	struct vbitmap			*tags;
 	vre_t				*vre;
 };
 
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index b32a470..dc58a66 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -256,47 +256,46 @@ vsl_IX_arg(struct VSL_data *vsl, int opt, const char *arg)
 	const char *b, *e, *err;
 	vre_t *vre;
 	struct vslf *vslf;
+	struct vbitmap *tags = NULL;
 
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 	vsl->flags |= F_SEEN_ixIX;
 
-	l = 0;
 	b = arg;
 	e = strchr(b, ':');
 	if (e) {
-		while (isspace(*b))
-			b++;
+		tags = vbit_init(SLT__MAX);
+		AN(tags);
 		l = e - b;
-		while (l > 0 && isspace(b[l - 1]))
-			l--;
-	}
-	if (l > 0 && strncmp(b, "*", l))
-		i = VSL_Name2Tag(b, l);
-	else
-		i = -3;
-	if (i == -2)
-		return (vsl_diag(vsl,
-			"-%c: \"%*.*s\" matches multiple tags\n",
-			(char)opt, l, l, b));
-	else if (i == -1)
-		return (vsl_diag(vsl,
-			"-%c: Could not match \"%*.*s\" to any tag\n",
-			(char)opt, l, l, b));
-	assert(i >= -3);
-
-	if (e)
+		i = VSL_List2Tags(b, l, vsl_vbm_bitset, tags);
+		if (i < 0)
+			vbit_destroy(tags);
+		if (i == -1)
+			return (vsl_diag(vsl,
+				"-%c: \"%*.*s\" matches zero tags",
+				(char)opt, l, l, b));
+		else if (i == -2)
+			return (vsl_diag(vsl,
+				"-%c: \"%*.*s\" is ambiguous",
+				(char)opt, l, l, b));
+		else if (i == 3)
+			return (vsl_diag(vsl,
+				"-%c: Syntax error in \"%*.*s\"",
+				(char)opt, l, l, b));
 		b = e + 1;
+	}
+
 	vre = VRE_compile(b, 0, &err, &off);
-	if (vre == NULL)
+	if (vre == NULL) {
+		if (tags)
+			vbit_destroy(tags);
 		return (vsl_diag(vsl, "-%c: Regex error at position %d (%s)\n",
 			(char)opt, off, err));
+	}
 
 	ALLOC_OBJ(vslf, VSLF_MAGIC);
-	if (vslf == NULL) {
-		VRE_free(&vre);
-		return (vsl_diag(vsl, "Out of memory"));
-	}
-	vslf->tag = i;
+	AN(vslf);
+	vslf->tags = tags;
 	vslf->vre = vre;
 
 	if (opt == 'I')



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 24ad822 Make synthetic SLT_VSL records NUL-terminated
Message-ID: <E1VQzNC-0005il-TX@project.varnish-software.com>

commit 24ad8224301a4896b8c09b79d8f8fa30a7db3913
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 25 11:26:22 2013 +0200

    Make synthetic SLT_VSL records NUL-terminated

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index bb2da3f..15cb044 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -758,20 +758,23 @@ static int
 vtx_diag(struct VSLQ *vslq, struct vtx *vtx, const char *fmt, ...)
 {
 	va_list ap;
-	uint32_t buf[256];
-	int i, len;
+	uint32_t chunk[256];
+	char *buf;
+	int l, buflen;
 	struct VSLC_ptr rec;
 
-	len = sizeof buf - 2 * sizeof (uint32_t);
+	buf = (char *)&chunk[2];
+	buflen = sizeof chunk - 2 * sizeof (uint32_t);
 	va_start(ap, fmt);
-	i = vsnprintf((char *)&buf[2], len, fmt, ap);
-	assert(i >= 0);
+	l = vsnprintf(buf, buflen, fmt, ap);
+	assert(l >= 0);
 	va_end(ap);
-	if (i < len)
-		len = i;
-	buf[1] = vtx->key.vxid;
-	buf[0] = ((((unsigned)SLT_VSL & 0xff) << 24) | len);
-	rec.ptr = buf;
+	if (l > buflen - 1)
+		l = buflen - 1;
+	buf[l++] = '\0';	/* NUL-terminated */
+	chunk[1] = vtx->key.vxid;
+	chunk[0] = ((((unsigned)SLT_VSL & 0xff) << 24) | l);
+	rec.ptr = chunk;
 	rec.priv = 0;
 	vtx_append(vslq, vtx, &rec, VSL_NEXT(rec.ptr) - rec.ptr, 1);
 



From martin at varnish-cache.org  Tue Oct  1 12:48:18 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:18 +0200
Subject: [master] 618908d Enable taglists in VSL query expressions
Message-ID: <E1VQzNC-0005it-V7@project.varnish-software.com>

commit 618908db7c66f1adad78e6e1b958213c472e0a44
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 25 13:21:02 2013 +0200

    Enable taglists in VSL query expressions

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index 3030d3e..b585fa2 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -172,3 +172,10 @@ logexpect l1 -d 1 -g vxid -q "RespHeader[2] == 123" {
 	  expect * =	ReqEnd
 	  expect * =	End
 } -run
+
+# Test taglists
+logexpect l1 -d 1 -g vxid -q "Debug,Resp* == 200" {
+	  expect 0 *	Begin	req
+	  expect * =	ReqEnd
+	  expect * =	End
+} -run
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index 7094fec..65007fd 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -40,6 +40,8 @@
 int vsl_diag(struct VSL_data *vsl, const char *fmt, ...)
     __printflike(2, 3);
 int vsl_skip(struct VSL_cursor *c, ssize_t words);
+void vsl_vbm_bitset(int bit, void *priv);
+void vsl_vbm_bitclr(int bit, void *priv);
 
 typedef void vslc_delete_f(struct VSL_cursor *);
 typedef int vslc_next_f(struct VSL_cursor *);
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index dc58a66..7795165 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -212,14 +212,14 @@ VSLQ_Name2Grouping(const char *name, int l)
 	return (n);
 }
 
-static void
+void __match_proto__(VSL_tagfind_f)
 vsl_vbm_bitset(int bit, void *priv)
 {
 
 	vbit_set((struct vbitmap *)priv, bit);
 }
 
-static void
+void __match_proto__(VSL_tagfind_f)
 vsl_vbm_bitclr(int bit, void *priv)
 {
 
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index e42705a..1ef92e9 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -39,6 +39,7 @@
 #include "miniobj.h"
 #include "vre.h"
 #include "vsb.h"
+#include "vbm.h"
 
 #include "vapi/vsl.h"
 #include "vsl_api.h"
@@ -187,6 +188,7 @@ vslq_test(const struct vex *vex, struct VSL_transaction * const ptrans[])
 
 	CHECK_OBJ_NOTNULL(vex, VEX_MAGIC);
 	CHECK_OBJ_NOTNULL(vex->lhs, VEX_LHS_MAGIC);
+	AN(vex->lhs->tags);
 	CHECK_OBJ_NOTNULL(vex->rhs, VEX_RHS_MAGIC);
 
 	for (t = ptrans[0]; t != NULL; t = *++ptrans) {
@@ -200,14 +202,12 @@ vslq_test(const struct vex *vex, struct VSL_transaction * const ptrans[])
 			assert(i == 1);
 			AN(t->c->rec.ptr);
 
-			if (vex->lhs->tag != VSL_TAG(t->c->rec.ptr))
+			if (!vbit_test(vex->lhs->tags, VSL_TAG(t->c->rec.ptr)))
 				continue;
 
 			i = vslq_test_rec(vex, &t->c->rec);
 			if (i)
 				return (i);
-
-
 		}
 	}
 
diff --git a/lib/libvarnishapi/vxp.h b/lib/libvarnishapi/vxp.h
index 73b3bc0..5d653bc 100644
--- a/lib/libvarnishapi/vxp.h
+++ b/lib/libvarnishapi/vxp.h
@@ -35,8 +35,8 @@
 
 #include "vxp_tokens.h"
 
-#define isword(c) \
-	(isalpha(c) || isdigit(c) || (c) == '_' || (c) == '-' || (c) == '.')
+#define isword(c)  (isalpha(c) || isdigit(c) || (c) == '_' || (c) == '-' || \
+	    (c) == '.' || (c) == '*' || (c) == ',')
 
 #define PF(t)	(int)((t)->e - (t)->b), (t)->b
 
@@ -82,7 +82,7 @@ struct vex_lhs {
 	   expression should be applied to */
 	unsigned		magic;
 #define VEX_LHS_MAGIC		0x1AD3D78D
-	int			tag;
+	struct vbitmap		*tags;
 	int			field;
 	int			level_min;
 	int			level_max;
diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index aad8bde..83e6594 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -40,8 +40,10 @@
 
 #include "vas.h"
 #include "vsb.h"
+#include "vbm.h"
 #include "miniobj.h"
 #include "vapi/vsl.h"
+#include "vsl_api.h"
 
 #include "vxp.h"
 
@@ -51,29 +53,36 @@ static void
 vxp_expr_lhs(struct vxp *vxp, struct vex_lhs **plhs)
 {
 	char *p;
+	int i;
 
-	/* XXX: Tag wildcards */
 	AN(plhs);
 	AZ(*plhs);
 	if (vxp->t->tok != VAL) {
-		VSB_printf(vxp->sb, "Expected VSL tag got '%.*s' ", PF(vxp->t));
+		VSB_printf(vxp->sb, "Expected VSL taglist got '%.*s' ",
+		    PF(vxp->t));
 		vxp_ErrWhere(vxp, vxp->t, -1);
 		return;
 	}
 	ALLOC_OBJ(*plhs, VEX_LHS_MAGIC);
 	AN(*plhs);
-	(*plhs)->tag = VSL_Name2Tag(vxp->t->dec, -1);
-	if ((*plhs)->tag == -1) {
-		VSB_printf(vxp->sb, "Could not match '%.*s' to any tag ",
-		    PF(vxp->t));
+	(*plhs)->tags = vbit_init(SLT__MAX);
+	i = VSL_List2Tags(vxp->t->dec, -1, vsl_vbm_bitset, (*plhs)->tags);
+	if (i == -1) {
+		VSB_printf(vxp->sb, "Taglist matches zero tags");
 		vxp_ErrWhere(vxp, vxp->t, -1);
 		return;
-	} else if ((*plhs)->tag == -2) {
-		VSB_printf(vxp->sb, "'%.*s' matches multiple tags ",
-		    PF(vxp->t));
+	}
+	if (i == -2) {
+		VSB_printf(vxp->sb, "Taglist is ambiguous");
+		vxp_ErrWhere(vxp, vxp->t, -1);
+		return;
+	}
+	if (i == -3) {
+		VSB_printf(vxp->sb, "Syntax error in taglist");
 		vxp_ErrWhere(vxp, vxp->t, -1);
 		return;
 	}
+	assert(i > 0);
 	vxp_NextToken(vxp);
 
 	if (vxp->t->tok == '[') {
@@ -422,8 +431,11 @@ void
 vex_Free(struct vex **pvex)
 {
 
-	if ((*pvex)->lhs != NULL)
+	if ((*pvex)->lhs != NULL) {
+		if ((*pvex)->lhs->tags != NULL)
+			vbit_destroy((*pvex)->lhs->tags);
 		FREE_OBJ((*pvex)->lhs);
+	}
 	if ((*pvex)->rhs != NULL) {
 		if ((*pvex)->rhs->val_string)
 			free((*pvex)->rhs->val_string);
@@ -473,6 +485,25 @@ vex_print_rhs(const struct vex_rhs *rhs)
 }
 
 static void
+vex_print_tags(const struct vbitmap *vbm)
+{
+	int i;
+	int first = 1;
+
+	for (i = 0; i < SLT__MAX; i++) {
+		if (VSL_tags[i] == NULL)
+			continue;
+		if (!vbit_test(vbm, i))
+			continue;
+		if (first)
+			first = 0;
+		else
+			fprintf(stderr, ",");
+		fprintf(stderr, "%s", VSL_tags[i]);
+	}
+}
+
+static void
 vex_print(const struct vex *vex, int indent)
 {
 	CHECK_OBJ_NOTNULL(vex, VEX_MAGIC);
@@ -480,7 +511,10 @@ vex_print(const struct vex *vex, int indent)
 	fprintf(stderr, "%*s%s", indent, "", vxp_tnames[vex->tok]);
 	if (vex->lhs != NULL) {
 		CHECK_OBJ_NOTNULL(vex->lhs, VEX_LHS_MAGIC);
-		fprintf(stderr, " tag=%s", VSL_tags[vex->lhs->tag]);
+		AN(vex->lhs->tags);
+		fprintf(stderr, " lhs=(");
+		vex_print_tags(vex->lhs->tags);
+		fprintf(stderr, ")");
 		if (vex->lhs->field >= 0)
 			fprintf(stderr, "[%d]", vex->lhs->field);
 	}



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] 17b7c07 Use 'not' instead of '!' for boolean not operation
Message-ID: <E1VQzND-0005j2-0I@project.varnish-software.com>

commit 17b7c076302f16933d196a1b988e1bde6c83a87e
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 25 13:31:02 2013 +0200

    Use 'not' instead of '!' for boolean not operation
    
    This makes it consistent with the 'and' and 'or' operators

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index b585fa2..f98ebc9 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -145,8 +145,8 @@ logexpect l1 -d 1 -g vxid -q "RespStatus == 404 or RespStatus ~ '^200$'" {
 	expect * =	End
 } -run
 
-# Test boolean !
-logexpect l1 -d 1 -g vxid -q "RespStatus == 404 or !RespStatus ~ '^404$'" {
+# Test boolean not
+logexpect l1 -d 1 -g vxid -q "RespStatus == 404 or not RespStatus ~ '^404$'" {
 	expect 0 *	Begin	req
 	expect * =	ReqEnd
 	expect * =	End
diff --git a/lib/libvarnishapi/generate.py b/lib/libvarnishapi/generate.py
index 6245ef8..9f3d668 100755
--- a/lib/libvarnishapi/generate.py
+++ b/lib/libvarnishapi/generate.py
@@ -60,9 +60,10 @@ tokens = {
         # Boolean operators
         "T_AND":        "and",
         "T_OR":         "or",
+	"T_NOT":	"not",
 
         # Miscellaneous
-        None:           "<>~![]{}()",
+        None:           "<>~[]{}()",
 
         # These have handwritten recognizers
         "VAL":          None,
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 1ef92e9..e36d4ba 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -236,7 +236,7 @@ vslq_exec(const struct vex *vex, struct VSL_transaction * const ptrans[])
 		if (r <= 0)
 			return (r);
 		return (vslq_exec(vex->b, ptrans));
-	case '!':
+	case T_NOT:
 		AN(vex->a);
 		AZ(vex->b);
 		r = vslq_exec(vex->a, ptrans);
diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index 83e6594..6847ab2 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -304,7 +304,7 @@ vxp_expr_group(struct vxp *vxp, struct vex **pvex)
 /*
  * SYNTAX:
  *   expr_not:
- *     '!' expr_group
+ *     'not' expr_group
  *     expr_group
  */
 
@@ -315,7 +315,7 @@ vxp_expr_not(struct vxp *vxp, struct vex **pvex)
 	AN(pvex);
 	AZ(*pvex);
 
-	if (vxp->t->tok == '!') {
+	if (vxp->t->tok == T_NOT) {
 		ALLOC_OBJ(*pvex, VEX_MAGIC);
 		AN(*pvex);
 		(*pvex)->tok = vxp->t->tok;



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] ad7153b Don't copy VSL record to tmp buf
Message-ID: <E1VQzND-0005jA-1i@project.varnish-software.com>

commit ad7153beef3d80b645d055e5792474d560e7f50a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 25 13:40:37 2013 +0200

    Don't copy VSL record to tmp buf
    
    Now that VSL records are null-terminated, it is safe to run sscanf
    without copying to a temporary null-terminated buffer.

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 15cb044..4bd7de6 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -488,28 +488,25 @@ vtx_check_ready(struct VSLQ *vslq, struct vtx *vtx)
 }
 
 static int
-vtx_parsetag_bl(const char *str, unsigned len, enum VSL_transaction_e *ptype,
+vtx_parsetag_bl(const char *str, enum VSL_transaction_e *ptype,
     unsigned *pvxid)
 {
-	char ibuf[len + 1];
-	char tbuf[7];
+	char buf[7];
 	unsigned vxid;
 	int i;
 	enum VSL_transaction_e type = VSL_t_unknown;
 
 	AN(str);
-	memcpy(ibuf, str, len);
-	ibuf[len] = '\0';
-	i = sscanf(ibuf, "%6s %u", tbuf, &vxid);
+	i = sscanf(str, "%6s %u", buf, &vxid);
 	if (i < 1)
 		return (-1);
-	if (!strcmp(tbuf, "sess"))
+	if (!strcmp(buf, "sess"))
 		type = VSL_t_sess;
-	else if (!strcmp(tbuf, "req"))
+	else if (!strcmp(buf, "req"))
 		type = VSL_t_req;
-	else if (!strcmp(tbuf, "esireq"))
+	else if (!strcmp(buf, "esireq"))
 		type = VSL_t_esireq;
-	else if (!strcmp(tbuf, "bereq"))
+	else if (!strcmp(buf, "bereq"))
 		type = VSL_t_bereq;
 	else
 		return (-1);
@@ -550,7 +547,7 @@ vtx_scan_begintag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 	if (vtx->flags & VTX_F_READY)
 		return (vtx_diag_tag(vslq, vtx, ptr, "link too late"));
 
-	i = vtx_parsetag_bl(VSL_CDATA(ptr), VSL_LEN(ptr), &type, &p_vxid);
+	i = vtx_parsetag_bl(VSL_CDATA(ptr), &type, &p_vxid);
 	if (i < 1)
 		return (vtx_diag_tag(vslq, vtx, ptr, "parse error"));
 
@@ -598,7 +595,7 @@ vtx_scan_linktag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 	if (vtx->flags & VTX_F_READY)
 		return (vtx_diag_tag(vslq, vtx, ptr, "link too late"));
 
-	i = vtx_parsetag_bl(VSL_CDATA(ptr), VSL_LEN(ptr), &c_type, &c_vxid);
+	i = vtx_parsetag_bl(VSL_CDATA(ptr), &c_type, &c_vxid);
 	if (i < 2)
 		return (vtx_diag_tag(vslq, vtx, ptr, "parse error"));
 	assert(i == 2);



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] b0080ea Enable record prefix filtering for operations
Message-ID: <E1VQzND-0005jI-3B@project.varnish-software.com>

commit b0080ea4ad92aa54ebfd67b2f2c8846710d67587
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 25 15:17:58 2013 +0200

    Enable record prefix filtering for operations

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index f98ebc9..7d22734 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -179,3 +179,10 @@ logexpect l1 -d 1 -g vxid -q "Debug,Resp* == 200" {
 	  expect * =	ReqEnd
 	  expect * =	End
 } -run
+
+# Test record prefix
+logexpect l1 -d 1 -g vxid -q "Resp*:x-test eq '123 321'" {
+	  expect 0 *	Begin	req
+	  expect * =	ReqEnd
+	  expect * =	End
+} -run
diff --git a/lib/libvarnishapi/generate.py b/lib/libvarnishapi/generate.py
index 9f3d668..b08e769 100755
--- a/lib/libvarnishapi/generate.py
+++ b/lib/libvarnishapi/generate.py
@@ -63,7 +63,7 @@ tokens = {
 	"T_NOT":	"not",
 
         # Miscellaneous
-        None:           "<>~[]{}()",
+        None:           "<>~[]{}():",
 
         # These have handwritten recognizers
         "VAL":          None,
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index e36d4ba..8a3a441 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -86,6 +86,18 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 	b = VSL_CDATA(rec->ptr);
 	e = b + VSL_LEN(rec->ptr) - 1;
 
+	/* Prefix */
+	if (vex->lhs->prefix != NULL) {
+		if (strncasecmp(b, vex->lhs->prefix, vex->lhs->prefixlen))
+			return (0);
+		if (b[vex->lhs->prefixlen] != ':')
+			return (0);
+		b += vex->lhs->prefixlen + 1;
+		/* Skip ws */
+		while (*b && isspace(*b))
+			b++;
+	}
+
 	/* Field */
 	if (vex->lhs->field > 0) {
 		for (e = b, i = 0; *e && i < vex->lhs->field; i++) {
diff --git a/lib/libvarnishapi/vxp.h b/lib/libvarnishapi/vxp.h
index 5d653bc..9a285b5 100644
--- a/lib/libvarnishapi/vxp.h
+++ b/lib/libvarnishapi/vxp.h
@@ -83,6 +83,8 @@ struct vex_lhs {
 	unsigned		magic;
 #define VEX_LHS_MAGIC		0x1AD3D78D
 	struct vbitmap		*tags;
+	char			*prefix;
+	int			prefixlen;
 	int			field;
 	int			level_min;
 	int			level_max;
diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index 6847ab2..988fde4 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -85,6 +85,22 @@ vxp_expr_lhs(struct vxp *vxp, struct vex_lhs **plhs)
 	assert(i > 0);
 	vxp_NextToken(vxp);
 
+	if (vxp->t->tok == ':') {
+		/* Record prefix */
+		vxp_NextToken(vxp);
+		if (vxp->t->tok != VAL) {
+			VSB_printf(vxp->sb, "Expected string got '%.*s' ",
+			    PF(vxp->t));
+			vxp_ErrWhere(vxp, vxp->t, -1);
+			return;
+		}
+		AN(vxp->t->dec);
+		(*plhs)->prefix = strdup(vxp->t->dec);
+		AN((*plhs)->prefix);
+		(*plhs)->prefixlen = strlen((*plhs)->prefix);
+		vxp_NextToken(vxp);
+	}
+
 	if (vxp->t->tok == '[') {
 		/* LHS field [] */
 		vxp_NextToken(vxp);
@@ -434,6 +450,8 @@ vex_Free(struct vex **pvex)
 	if ((*pvex)->lhs != NULL) {
 		if ((*pvex)->lhs->tags != NULL)
 			vbit_destroy((*pvex)->lhs->tags);
+		if ((*pvex)->lhs->prefix != NULL)
+			free((*pvex)->lhs->prefix);
 		FREE_OBJ((*pvex)->lhs);
 	}
 	if ((*pvex)->rhs != NULL) {
@@ -462,21 +480,22 @@ vex_print_rhs(const struct vex_rhs *rhs)
 {
 
 	CHECK_OBJ_NOTNULL(rhs, VEX_RHS_MAGIC);
+	fprintf(stderr, "rhs=");
 	switch (rhs->type) {
 	case VEX_INT:
-		fprintf(stderr, "INT=%jd", (intmax_t)rhs->val_int);
+		fprintf(stderr, "INT(%jd)", (intmax_t)rhs->val_int);
 		break;
 	case VEX_FLOAT:
-		fprintf(stderr, "FLOAT=%f", rhs->val_float);
+		fprintf(stderr, "FLOAT(%f)", rhs->val_float);
 		break;
 	case VEX_STRING:
 		AN(rhs->val_string);
-		fprintf(stderr, "STRING='%s'", rhs->val_string);
+		fprintf(stderr, "STRING(%s)", rhs->val_string);
 		break;
 	case VEX_REGEX:
 		AN(rhs->val_string);
 		AN(rhs->val_regex);
-		fprintf(stderr, "REGEX='%s'", rhs->val_string);
+		fprintf(stderr, "REGEX(%s)", rhs->val_string);
 		break;
 	default:
 		WRONG("rhs type");
@@ -515,6 +534,10 @@ vex_print(const struct vex *vex, int indent)
 		fprintf(stderr, " lhs=(");
 		vex_print_tags(vex->lhs->tags);
 		fprintf(stderr, ")");
+		if (vex->lhs->prefix) {
+			assert(vex->lhs->prefixlen == strlen(vex->lhs->prefix));
+			fprintf(stderr, ":%s", vex->lhs->prefix);
+		}
 		if (vex->lhs->field >= 0)
 			fprintf(stderr, "[%d]", vex->lhs->field);
 	}



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] dec8a96 Add tag presence "operator"
Message-ID: <E1VQzND-0005jR-5D@project.varnish-software.com>

commit dec8a963b21a2374a601dd776d184a0f6a9580fb
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 25 15:43:47 2013 +0200

    Add tag presence "operator"

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index 7d22734..6dc2b7d 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -186,3 +186,10 @@ logexpect l1 -d 1 -g vxid -q "Resp*:x-test eq '123 321'" {
 	  expect * =	ReqEnd
 	  expect * =	End
 } -run
+
+# Test tag presence (no operator)
+logexpect l1 -d 1 -g vxid -q "RespStatus" {
+	  expect 0 *	Begin	req
+	  expect * =	ReqEnd
+	  expect * =	End
+} -run
diff --git a/lib/libvarnishapi/generate.py b/lib/libvarnishapi/generate.py
index b08e769..1ef6d93 100755
--- a/lib/libvarnishapi/generate.py
+++ b/lib/libvarnishapi/generate.py
@@ -68,6 +68,9 @@ tokens = {
         # These have handwritten recognizers
         "VAL":          None,
         "EOI":          None,
+
+	# Special
+	"T_TRUE":	None,
 }
 
 #######################################################################
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 8a3a441..7098c4d 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -80,8 +80,6 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 
 	AN(vex);
 	AN(rec);
-	rhs = vex->rhs;
-	AN(rhs);
 
 	b = VSL_CDATA(rec->ptr);
 	e = b + VSL_LEN(rec->ptr) - 1;
@@ -116,6 +114,13 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 			return (0);
 	}
 
+	if (vex->tok == T_TRUE)
+		/* Always true */
+		return (1);
+
+	rhs = vex->rhs;
+	CHECK_OBJ_NOTNULL(rhs, VEX_RHS_MAGIC);
+
 	/* Prepare */
 	switch (vex->tok) {
 	case T_EQ:		/* == */
@@ -201,7 +206,6 @@ vslq_test(const struct vex *vex, struct VSL_transaction * const ptrans[])
 	CHECK_OBJ_NOTNULL(vex, VEX_MAGIC);
 	CHECK_OBJ_NOTNULL(vex->lhs, VEX_LHS_MAGIC);
 	AN(vex->lhs->tags);
-	CHECK_OBJ_NOTNULL(vex->rhs, VEX_RHS_MAGIC);
 
 	for (t = ptrans[0]; t != NULL; t = *++ptrans) {
 		AZ(VSL_ResetCursor(t->c));
diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index 988fde4..c105bc7 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -241,6 +241,7 @@ vxp_expr_cmp(struct vxp *vxp, struct vex **pvex)
 	case T_AND:
 	case T_OR:
 	case ')':
+		(*pvex)->tok = T_TRUE;
 		return;
 
 	/* Valid operators */



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] 054d8e9 Add some missing trailing spaces in error strings
Message-ID: <E1VQzND-0005jZ-6k@project.varnish-software.com>

commit 054d8e955182d5c8a4820baf7939f56819e9a5ec
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 25 15:55:43 2013 +0200

    Add some missing trailing spaces in error strings

diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index c105bc7..ec82b96 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -68,17 +68,17 @@ vxp_expr_lhs(struct vxp *vxp, struct vex_lhs **plhs)
 	(*plhs)->tags = vbit_init(SLT__MAX);
 	i = VSL_List2Tags(vxp->t->dec, -1, vsl_vbm_bitset, (*plhs)->tags);
 	if (i == -1) {
-		VSB_printf(vxp->sb, "Taglist matches zero tags");
+		VSB_printf(vxp->sb, "Taglist matches zero tags ");
 		vxp_ErrWhere(vxp, vxp->t, -1);
 		return;
 	}
 	if (i == -2) {
-		VSB_printf(vxp->sb, "Taglist is ambiguous");
+		VSB_printf(vxp->sb, "Taglist is ambiguous ");
 		vxp_ErrWhere(vxp, vxp->t, -1);
 		return;
 	}
 	if (i == -3) {
-		VSB_printf(vxp->sb, "Syntax error in taglist");
+		VSB_printf(vxp->sb, "Syntax error in taglist ");
 		vxp_ErrWhere(vxp, vxp->t, -1);
 		return;
 	}
@@ -112,7 +112,7 @@ vxp_expr_lhs(struct vxp *vxp, struct vex_lhs **plhs)
 		}
 		(*plhs)->field = (int)strtol(vxp->t->dec, &p, 0);
 		if (*p || (*plhs)->field <= 0) {
-			VSB_printf(vxp->sb, "Expected positive integer");
+			VSB_printf(vxp->sb, "Expected positive integer ");
 			vxp_ErrWhere(vxp, vxp->t, -1);
 			return;
 		}



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] 2f4a74f Free memory before exit in test program
Message-ID: <E1VQzND-0005jh-83@project.varnish-software.com>

commit 2f4a74f678720df04e828a5a9bed6c670b3aa3fa
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Sep 25 16:35:44 2013 +0200

    Free memory before exit in test program
    
    This to silence valgrind errors

diff --git a/lib/libvarnishapi/vxp_test.c b/lib/libvarnishapi/vxp_test.c
index 19be714..d4a8b4c 100644
--- a/lib/libvarnishapi/vxp_test.c
+++ b/lib/libvarnishapi/vxp_test.c
@@ -30,12 +30,15 @@ main(int argc, char **argv)
 	if (vex == NULL) {
 		VSB_finish(vsb);
 		fprintf(stderr, "Error:\n%s", VSB_data(vsb));
+		VSB_delete(vsb);
+		free(s);
 		exit(1);
 	}
 	VSB_delete(vsb);
 
 	vex_Free(&vex);
 	AZ(vex);
+	free(s);
 
 	return (0);
 }



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] 782251a Redo SLT_VSL diagnostic log records from the API
Message-ID: <E1VQzND-0005jp-A6@project.varnish-software.com>

commit 782251a341fab3eb3a8dc6f6db91cf94ed5e633a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Sep 27 14:29:16 2013 +0200

    Redo SLT_VSL diagnostic log records from the API
    
    A vtx_diag log record could realloc the memory buffers while
    processing a vtx, causing ptrs to previous memory to be still used.
    
    Fix this by having generated log records be malloced and kept on a
    separate list, which is always reported first on a reset cursor.
    
    Fixes: #1351

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 4bd7de6..1544604 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -67,6 +67,15 @@ struct vtx_chunk {
 	ssize_t			offset;
 };
 
+struct vtx_diag {
+	unsigned		magic;
+#define VTX_DIAG_MAGIC		0xC654479F
+
+	VTAILQ_ENTRY(vtx_diag)	list;
+	uint32_t		chunk[2 + 256 / sizeof (uint32_t)];
+};
+
+
 struct vslc_vtx {
 	unsigned		magic;
 #define VSLC_VTX_MAGIC		0x74C6523F
@@ -75,6 +84,8 @@ struct vslc_vtx {
 
 	struct vtx		*vtx;
 
+	struct vtx_diag		*diag;	/* Current diag message pointer */
+
 	unsigned		chunk;	/* Current chunk */
 	ssize_t			offset; /* Offset of next record */
 };
@@ -107,13 +118,16 @@ struct vtx {
 	unsigned		n_descend;
 
 	struct vslc_vtx		c;
-	ssize_t			len;
+
+	VTAILQ_HEAD(,vtx_diag)	diag;
 
 	struct vtx_chunk	chunk[VTX_CHUNKS];
 	unsigned		n_chunk;
 
 	uint32_t		*buf;
 	ssize_t			bufsize;
+
+	ssize_t			len;
 };
 
 struct VSLQ {
@@ -138,9 +152,9 @@ struct VSLQ {
 };
 
 /*lint -esym(534, vtx_diag) */
-static int vtx_diag(struct VSLQ *vslq, struct vtx *vtx, const char *fmt, ...);
+static int vtx_diag(struct vtx *vtx, const char *fmt, ...);
 /*lint -esym(534, vtx_diag_tag) */
-static int vtx_diag_tag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr,
+static int vtx_diag_tag(struct vtx *vtx, const uint32_t *ptr,
     const char *reason);
 
 static inline int
@@ -209,6 +223,18 @@ vslc_vtx_next(struct VSL_cursor *cursor)
 	assert(&c->cursor == cursor);
 	CHECK_OBJ_NOTNULL(c->vtx, VTX_MAGIC);
 
+	if (c->diag == NULL && VTAILQ_FIRST(&c->vtx->diag) != NULL) {
+		/* Send first diag msg */
+		c->diag = VTAILQ_FIRST(&c->vtx->diag);
+		c->cursor.rec.ptr = c->diag->chunk;
+		return (1);
+	} else if (c->diag != NULL && VTAILQ_NEXT(c->diag, list) != NULL) {
+		/* Send next diag msg */
+		c->diag = VTAILQ_NEXT(c->diag, list);
+		c->cursor.rec.ptr = c->diag->chunk;
+		return (1);
+	}
+
 	assert (c->offset <= c->vtx->len);
 	if (c->offset == c->vtx->len)
 		return (0);
@@ -245,6 +271,7 @@ vslc_vtx_reset(struct VSL_cursor *cursor)
 
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VTX_MAGIC);
 	assert(&c->cursor == cursor);
+	c->diag = NULL;
 	c->chunk = 0;
 	c->offset = 0;
 	c->cursor.rec.ptr = NULL;
@@ -290,10 +317,11 @@ vtx_new(struct VSLQ *vslq)
 	vtx->n_child = 0;
 	vtx->n_childready = 0;
 	vtx->n_descend = 0;
-	(void)vslc_vtx_reset(&vtx->c.cursor);
-	vtx->len = 0;
+	VTAILQ_INIT(&vtx->diag);
 	memset(vtx->chunk, 0, sizeof vtx->chunk);
 	vtx->n_chunk = 0;
+	vtx->len = 0;
+	(void)vslc_vtx_reset(&vtx->c.cursor);
 
 	VTAILQ_INSERT_TAIL(&vslq->incomplete, vtx, list_incomplete);
 	vslq->n_incomplete++;
@@ -319,6 +347,7 @@ vtx_retire(struct VSLQ *vslq, struct vtx **pvtx)
 {
 	struct vtx *vtx;
 	struct vtx *child;
+	struct vtx_diag *diag;
 
 	AN(vslq);
 	AN(pvtx);
@@ -347,6 +376,12 @@ vtx_retire(struct VSLQ *vslq, struct vtx **pvtx)
 	AZ(vtx->n_descend);
 	AN(VRB_REMOVE(vtx_tree, &vslq->tree, &vtx->key));
 
+	while (!VTAILQ_EMPTY(&vtx->diag)) {
+		diag = VTAILQ_FIRST(&vtx->diag);
+		VTAILQ_REMOVE(&vtx->diag, diag, list);
+		FREE_OBJ(diag);
+	}
+
 	if (vtx->n_chunk)
 		VTAILQ_REMOVE(&vslq->shmlist, vtx, list_shm);
 
@@ -464,7 +499,7 @@ vtx_check_ready(struct VSLQ *vslq, struct vtx *vtx)
 	AZ(vtx->flags & VTX_F_READY);
 
 	if (vtx->type == VSL_t_unknown)
-		vtx_diag(vslq, vtx, "vtx of unknown type marked complete");
+		vtx_diag(vtx, "vtx of unknown type marked complete");
 
 	ready = vtx;
 	while (1) {
@@ -545,16 +580,16 @@ vtx_scan_begintag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 	assert(VSL_TAG(ptr) == SLT_Begin);
 
 	if (vtx->flags & VTX_F_READY)
-		return (vtx_diag_tag(vslq, vtx, ptr, "link too late"));
+		return (vtx_diag_tag(vtx, ptr, "link too late"));
 
 	i = vtx_parsetag_bl(VSL_CDATA(ptr), &type, &p_vxid);
 	if (i < 1)
-		return (vtx_diag_tag(vslq, vtx, ptr, "parse error"));
+		return (vtx_diag_tag(vtx, ptr, "parse error"));
 
 	/* Check/set vtx type */
 	assert(type != VSL_t_unknown);
 	if (vtx->type != VSL_t_unknown && vtx->type != type)
-		return (vtx_diag_tag(vslq, vtx, ptr, "type mismatch"));
+		return (vtx_diag_tag(vtx, ptr, "type mismatch"));
 	vtx->type = type;
 
 	if (i == 1 || p_vxid == 0)
@@ -573,9 +608,9 @@ vtx_scan_begintag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 		return (0);
 
 	if (vtx->parent != NULL)
-		return (vtx_diag_tag(vslq, vtx, ptr, "duplicate link"));
+		return (vtx_diag_tag(vtx, ptr, "duplicate link"));
 	if (p_vtx->flags & VTX_F_READY)
-		return (vtx_diag_tag(vslq, vtx, ptr, "link too late"));
+		return (vtx_diag_tag(vtx, ptr, "link too late"));
 
 	vtx_set_parent(p_vtx, vtx);
 
@@ -593,11 +628,11 @@ vtx_scan_linktag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 	assert(VSL_TAG(ptr) == SLT_Link);
 
 	if (vtx->flags & VTX_F_READY)
-		return (vtx_diag_tag(vslq, vtx, ptr, "link too late"));
+		return (vtx_diag_tag(vtx, ptr, "link too late"));
 
 	i = vtx_parsetag_bl(VSL_CDATA(ptr), &c_type, &c_vxid);
 	if (i < 2)
-		return (vtx_diag_tag(vslq, vtx, ptr, "parse error"));
+		return (vtx_diag_tag(vtx, ptr, "parse error"));
 	assert(i == 2);
 
 	if (vslq->grouping == VSL_g_vxid)
@@ -612,11 +647,11 @@ vtx_scan_linktag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 		/* Link already exists */
 		return (0);
 	if (c_vtx->parent != NULL)
-		return (vtx_diag_tag(vslq, vtx, ptr, "duplicate link"));
+		return (vtx_diag_tag(vtx, ptr, "duplicate link"));
 	if (c_vtx->flags & VTX_F_READY)
-		return (vtx_diag_tag(vslq, vtx, ptr, "link too late"));
+		return (vtx_diag_tag(vtx, ptr, "link too late"));
 	if (c_vtx->type != VSL_t_unknown && c_vtx->type != c_type)
-		return (vtx_diag_tag(vslq, vtx, ptr, "type mismatch"));
+		return (vtx_diag_tag(vtx, ptr, "type mismatch"));
 	c_vtx->type = c_type;
 
 	vtx_set_parent(vtx, c_vtx);
@@ -635,20 +670,16 @@ vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
 		ptr = vtx->c.cursor.rec.ptr;
 		tag = VSL_TAG(ptr);
 
-		if (tag == SLT__Batch)
-			continue;
-
-		if (tag == SLT_VSL)
-			/* Don't process these to avoid looping */
+		if (tag == SLT__Batch || tag == SLT_VSL)
 			continue;
 
 		if (vtx->flags & VTX_F_COMPLETE) {
-			vtx_diag_tag(vslq, vtx, ptr, "late log rec");
+			vtx_diag_tag(vtx, ptr, "late log rec");
 			continue;
 		}
 
 		if (vtx->type == VSL_t_unknown && tag != SLT_Begin)
-			vtx_diag_tag(vslq, vtx, ptr, "early log rec");
+			vtx_diag_tag(vtx, ptr, "early log rec");
 
 		switch (tag) {
 		case SLT_Begin:
@@ -682,7 +713,7 @@ vtx_force(struct VSLQ *vslq, struct vtx *vtx, const char *reason)
 {
 	AZ(vtx->flags & VTX_F_COMPLETE);
 	AZ(vtx->flags & VTX_F_READY);
-	vtx_diag(vslq, vtx, reason);
+	vtx_diag(vtx, reason);
 
 	VTAILQ_REMOVE(&vslq->incomplete, vtx, list_incomplete);
 	vtx->flags |= VTX_F_COMPLETE;
@@ -752,16 +783,18 @@ vslq_callback(const struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
 }
 
 static int
-vtx_diag(struct VSLQ *vslq, struct vtx *vtx, const char *fmt, ...)
+vtx_diag(struct vtx *vtx, const char *fmt, ...)
 {
+	struct vtx_diag *diag;
 	va_list ap;
-	uint32_t chunk[256];
 	char *buf;
 	int l, buflen;
-	struct VSLC_ptr rec;
 
-	buf = (char *)&chunk[2];
-	buflen = sizeof chunk - 2 * sizeof (uint32_t);
+	ALLOC_OBJ(diag, VTX_DIAG_MAGIC);
+	AN(diag);
+
+	buf = (char *)&diag->chunk[2];
+	buflen = sizeof (diag->chunk) - 2 * sizeof (uint32_t);
 	va_start(ap, fmt);
 	l = vsnprintf(buf, buflen, fmt, ap);
 	assert(l >= 0);
@@ -769,20 +802,17 @@ vtx_diag(struct VSLQ *vslq, struct vtx *vtx, const char *fmt, ...)
 	if (l > buflen - 1)
 		l = buflen - 1;
 	buf[l++] = '\0';	/* NUL-terminated */
-	chunk[1] = vtx->key.vxid;
-	chunk[0] = ((((unsigned)SLT_VSL & 0xff) << 24) | l);
-	rec.ptr = chunk;
-	rec.priv = 0;
-	vtx_append(vslq, vtx, &rec, VSL_NEXT(rec.ptr) - rec.ptr, 1);
+	diag->chunk[1] = vtx->key.vxid;
+	diag->chunk[0] = ((((unsigned)SLT_VSL & 0xff) << 24) | l);
+	VTAILQ_INSERT_TAIL(&vtx->diag, diag, list);
 
 	return (-1);
 }
 
 static int
-vtx_diag_tag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr,
-    const char *reason)
+vtx_diag_tag(struct vtx *vtx, const uint32_t *ptr, const char *reason)
 {
-	return (vtx_diag(vslq, vtx, "%s (%s: %.*s)", reason,
+	return (vtx_diag(vtx, "%s (%s: %.*s)", reason,
 		VSL_tags[VSL_TAG(ptr)],	(int)VSL_LEN(ptr), VSL_CDATA(ptr)));
 }
 



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] 80240d7 Move query input to be an option instead of cmd line
 args
Message-ID: <E1VQzND-0005jy-Bn@project.varnish-software.com>

commit 80240d7746a5912d18ef5134e02f6c1f3ddfaa62
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Sep 30 12:36:47 2013 +0200

    Move query input to be an option instead of cmd line args
    
    Having the query input to the utils be cmd line args leads users to
    believe that the args are interpreted as tokens themselves. The parser
    doesn't see each argument as a token, but instead makes one string
    from all the arguments. This would cause quoting issues.
    
    By having the query expression be an option instead, it becomes
    clearer that the input is one string only, and any quoting when
    constructing an expression from some other input is left to the user.

diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index d3d9e29..a10c595 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -54,7 +54,7 @@ static void
 usage(void)
 {
 	const char **opt;
-	fprintf(stderr, "Usage: varnishlog <options> [query expression]\n\n");
+	fprintf(stderr, "Usage: varnishlog <options>\n\n");
 	fprintf(stderr, "Options:\n");
 	for (opt = vopt_usage; *opt != NULL; opt += 2)
 		fprintf(stderr, "  %-25s %s\n", *opt, *(opt + 1));
@@ -76,8 +76,8 @@ main(int argc, char * const *argv)
 		}
 	}
 
-	if (optind < argc)
-		VUT.query = argv[optind];
+	if (optind != argc)
+		usage();
 
 	VUT_Setup();
 	VUT_Main(NULL, NULL);
diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h
index 5c09c58..53519b7 100644
--- a/bin/varnishlog/varnishlog_options.h
+++ b/bin/varnishlog/varnishlog_options.h
@@ -39,6 +39,7 @@ VSL_OPT_I
 VSM_OPT_n
 VSM_OPT_N
 VUT_OPT_P
+VUT_OPT_q
 VSL_OPT_r
 VSL_OPT_u
 VSL_OPT_v
diff --git a/include/vut.h b/include/vut.h
index 0fdf8de..2ba148c 100644
--- a/include/vut.h
+++ b/include/vut.h
@@ -38,10 +38,10 @@ struct VUT {
 	int		D_opt;
 	int		g_arg;
 	char		*P_arg;
+	char		*q_arg;
 	char		*r_arg;
 	int		u_opt;
 	char		*w_arg;
-	const char	*query;
 
 	/* State */
 	struct VSL_data	*vsl;
diff --git a/include/vut_options.h b/include/vut_options.h
index 3034b41..7328b91 100644
--- a/include/vut_options.h
+++ b/include/vut_options.h
@@ -38,3 +38,8 @@
 	VOPT("P:", "[-P file]", "PID file",				\
 		"Write the process' PID to the specified file."		\
 	)
+
+#define VUT_OPT_q							\
+	VOPT("q:", "[-q query]", "VSL query",				\
+		"Specifies the VSL query to use."			\
+	)
diff --git a/lib/libvarnishapi/vxp_test.c b/lib/libvarnishapi/vxp_test.c
index d4a8b4c..1104039 100644
--- a/lib/libvarnishapi/vxp_test.c
+++ b/lib/libvarnishapi/vxp_test.c
@@ -2,43 +2,56 @@
 #include <stdio.h>
 #include <errno.h>
 #include <string.h>
+#include <unistd.h>
 
 #include "vxp.h"
 #include "vas.h"
 #include "vsb.h"
+#include "miniobj.h"
+
+static void
+usage(void)
+{
+	fprintf(stderr, "Usage: vxp_test -q <query-expression>\n");
+	exit(1);
+}
 
 int
 main(int argc, char **argv)
 {
-	int i;
-	unsigned l;
-	char *s;
 	struct vsb *vsb;
 	struct vex *vex;
+	char *q_arg = NULL;
+	char opt;
 
-	l = 0;
-	for (i = 1; i < argc; i++)
-		l += strlen(argv[i]) + 1;
-	s = calloc(l + 1, sizeof (char));
-	for (i = 1; i < argc; strcat(s, " "), i++)
-		strcat(s, argv[i]);
+	while ((opt = getopt(argc, argv, "q:")) != -1) {
+		switch (opt) {
+		case 'q':
+			REPLACE(q_arg, optarg);
+			break;
+		default:
+			usage();
+		}
+	}
+	if (q_arg == NULL || optind != argc)
+		usage();
 
 	vsb = VSB_new_auto();
 	AN(vsb);
-	vex = vex_New(s, vsb);
+	vex = vex_New(q_arg, vsb);
 
 	if (vex == NULL) {
 		VSB_finish(vsb);
 		fprintf(stderr, "Error:\n%s", VSB_data(vsb));
 		VSB_delete(vsb);
-		free(s);
+		free(q_arg);
 		exit(1);
 	}
 	VSB_delete(vsb);
 
 	vex_Free(&vex);
 	AZ(vex);
-	free(s);
+	free(q_arg);
 
 	return (0);
 }
diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index d3e5abd..4cf5c71 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -136,6 +136,10 @@ VUT_Arg(int opt, const char *arg)
 		/* PID file */
 		REPLACE(VUT.P_arg, arg);
 		return (1);
+	case 'q':
+		/* Query to use */
+		REPLACE(VUT.q_arg, arg);
+		return (1);
 	case 'r':
 		/* Binary file input */
 		REPLACE(VUT.r_arg, arg);
@@ -201,7 +205,7 @@ VUT_Setup(void)
 	}
 
 	/* Create query */
-	VUT.vslq = VSLQ_New(VUT.vsl, &c, VUT.g_arg, VUT.query);
+	VUT.vslq = VSLQ_New(VUT.vsl, &c, VUT.g_arg, VUT.q_arg);
 	if (VUT.vslq == NULL)
 		VUT_Error(1, "Query parse error (%s)", VSL_Error(VUT.vsl));
 	AZ(c);
@@ -289,7 +293,7 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 				VSL_ResetError(VUT.vsl);
 				continue;
 			}
-			VUT.vslq = VSLQ_New(VUT.vsl, &c, VUT.g_arg, VUT.query);
+			VUT.vslq = VSLQ_New(VUT.vsl, &c, VUT.g_arg, VUT.q_arg);
 			AN(VUT.vslq);
 			AZ(c);
 		}
@@ -339,7 +343,7 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 				VSL_ResetError(VUT.vsl);
 				continue;
 			}
-			VUT.vslq = VSLQ_New(VUT.vsl, &c, VUT.g_arg, VUT.query);
+			VUT.vslq = VSLQ_New(VUT.vsl, &c, VUT.g_arg, VUT.q_arg);
 			AN(VUT.vslq);
 			AZ(c);
 		}



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] 307d94f Make ',' a token when lexing the query expressions
Message-ID: <E1VQzND-0005k7-DN@project.varnish-software.com>

commit 307d94faec14b92c9c8ef27a688899bb3feaaeed
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Sep 30 12:58:57 2013 +0200

    Make ',' a token when lexing the query expressions
    
    Teach the parser how to read a comma separated list of tag globs,
    rather than having the ',' character be recognised as valid for
    barewords.

diff --git a/lib/libvarnishapi/generate.py b/lib/libvarnishapi/generate.py
index 1ef6d93..87bd044 100755
--- a/lib/libvarnishapi/generate.py
+++ b/lib/libvarnishapi/generate.py
@@ -63,7 +63,7 @@ tokens = {
 	"T_NOT":	"not",
 
         # Miscellaneous
-        None:           "<>~[]{}():",
+        None:           "<>~[]{}():,",
 
         # These have handwritten recognizers
         "VAL":          None,
diff --git a/lib/libvarnishapi/vxp.h b/lib/libvarnishapi/vxp.h
index 9a285b5..c0b979f 100644
--- a/lib/libvarnishapi/vxp.h
+++ b/lib/libvarnishapi/vxp.h
@@ -36,7 +36,7 @@
 #include "vxp_tokens.h"
 
 #define isword(c)  (isalpha(c) || isdigit(c) || (c) == '_' || (c) == '-' || \
-	    (c) == '.' || (c) == '*' || (c) == ',')
+	    (c) == '.' || (c) == '*')
 
 #define PF(t)	(int)((t)->e - (t)->b), (t)->b
 
diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index ec82b96..bdf64c5 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -57,33 +57,40 @@ vxp_expr_lhs(struct vxp *vxp, struct vex_lhs **plhs)
 
 	AN(plhs);
 	AZ(*plhs);
-	if (vxp->t->tok != VAL) {
-		VSB_printf(vxp->sb, "Expected VSL taglist got '%.*s' ",
-		    PF(vxp->t));
-		vxp_ErrWhere(vxp, vxp->t, -1);
-		return;
-	}
 	ALLOC_OBJ(*plhs, VEX_LHS_MAGIC);
 	AN(*plhs);
 	(*plhs)->tags = vbit_init(SLT__MAX);
-	i = VSL_List2Tags(vxp->t->dec, -1, vsl_vbm_bitset, (*plhs)->tags);
-	if (i == -1) {
-		VSB_printf(vxp->sb, "Taglist matches zero tags ");
-		vxp_ErrWhere(vxp, vxp->t, -1);
-		return;
-	}
-	if (i == -2) {
-		VSB_printf(vxp->sb, "Taglist is ambiguous ");
-		vxp_ErrWhere(vxp, vxp->t, -1);
-		return;
-	}
-	if (i == -3) {
-		VSB_printf(vxp->sb, "Syntax error in taglist ");
-		vxp_ErrWhere(vxp, vxp->t, -1);
-		return;
+	while (1) {
+		/* The tags this expression applies to */
+		if (vxp->t->tok != VAL) {
+			VSB_printf(vxp->sb, "Expected VSL tag name got '%.*s' ",
+			    PF(vxp->t));
+			vxp_ErrWhere(vxp, vxp->t, -1);
+			return;
+		}
+		i = VSL_Glob2Tags(vxp->t->dec, -1, vsl_vbm_bitset,
+		    (*plhs)->tags);
+		if (i == -1) {
+			VSB_printf(vxp->sb, "Tag name matches zero tags ");
+			vxp_ErrWhere(vxp, vxp->t, -1);
+			return;
+		}
+		if (i == -2) {
+			VSB_printf(vxp->sb, "Tag name is ambiguous ");
+			vxp_ErrWhere(vxp, vxp->t, -1);
+			return;
+		}
+		if (i == -3) {
+			VSB_printf(vxp->sb, "Syntax error in tag name ");
+			vxp_ErrWhere(vxp, vxp->t, -1);
+			return;
+		}
+		assert(i > 0);
+		vxp_NextToken(vxp);
+		if (vxp->t->tok != ',')
+			break;
+		vxp_NextToken(vxp);
 	}
-	assert(i > 0);
-	vxp_NextToken(vxp);
 
 	if (vxp->t->tok == ':') {
 		/* Record prefix */



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] e984abc Add level limits to query language
Message-ID: <E1VQzND-0005kF-F8@project.varnish-software.com>

commit e984abcfc4e924efef002d2922a6eaadfc8504f6
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Sep 30 14:33:53 2013 +0200

    Add level limits to query language

diff --git a/bin/varnishtest/tests/l00001.vtc b/bin/varnishtest/tests/l00001.vtc
index 6dc2b7d..1da61df 100644
--- a/bin/varnishtest/tests/l00001.vtc
+++ b/bin/varnishtest/tests/l00001.vtc
@@ -193,3 +193,21 @@ logexpect l1 -d 1 -g vxid -q "RespStatus" {
 	  expect * =	ReqEnd
 	  expect * =	End
 } -run
+
+# Test level limits equal
+logexpect l1 -d 1 -g vxid -q "{1}Begin ~ req" {
+	  expect 0 *	Begin	req
+	  expect * =	End
+} -run
+
+# Test level limits less than or equal
+logexpect l1 -d 1 -g vxid -q "{2-}Begin ~ req" {
+	  expect 0 *	Begin	req
+	  expect * =	End
+} -run
+
+# Test level limits greater than or equal
+logexpect l1 -d 1 -g vxid -q "{0+}Begin ~ req" {
+	  expect 0 *	Begin	req
+	  expect * =	End
+} -run
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 7098c4d..7f1b66b 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -208,6 +208,22 @@ vslq_test(const struct vex *vex, struct VSL_transaction * const ptrans[])
 	AN(vex->lhs->tags);
 
 	for (t = ptrans[0]; t != NULL; t = *++ptrans) {
+		if (vex->lhs->level >= 0) {
+			if (vex->lhs->level_pm < 0) {
+				/* OK if less than or equal */
+				if (t->level > vex->lhs->level)
+					continue;
+			} else if (vex->lhs->level_pm > 0) {
+				/* OK if greater than or equal */
+				if (t->level < vex->lhs->level)
+					continue;
+			} else {
+				/* OK if equal */
+				if (t->level != vex->lhs->level)
+					continue;
+			}
+		}
+
 		AZ(VSL_ResetCursor(t->c));
 		while (1) {
 			i = VSL_Next(t->c);
diff --git a/lib/libvarnishapi/vxp.h b/lib/libvarnishapi/vxp.h
index c0b979f..20df7ed 100644
--- a/lib/libvarnishapi/vxp.h
+++ b/lib/libvarnishapi/vxp.h
@@ -36,7 +36,7 @@
 #include "vxp_tokens.h"
 
 #define isword(c)  (isalpha(c) || isdigit(c) || (c) == '_' || (c) == '-' || \
-	    (c) == '.' || (c) == '*')
+	    (c) == '+' || (c) == '.' || (c) == '*')
 
 #define PF(t)	(int)((t)->e - (t)->b), (t)->b
 
@@ -86,8 +86,8 @@ struct vex_lhs {
 	char			*prefix;
 	int			prefixlen;
 	int			field;
-	int			level_min;
-	int			level_max;
+	int			level;
+	int			level_pm;
 };
 
 enum vex_rhs_e {
diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index bdf64c5..83dbd52 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -60,6 +60,40 @@ vxp_expr_lhs(struct vxp *vxp, struct vex_lhs **plhs)
 	ALLOC_OBJ(*plhs, VEX_LHS_MAGIC);
 	AN(*plhs);
 	(*plhs)->tags = vbit_init(SLT__MAX);
+	(*plhs)->level = -1;
+
+	if (vxp->t->tok == '{') {
+		/* Transaction level limits */
+		vxp_NextToken(vxp);
+		if (vxp->t->tok != VAL) {
+			VSB_printf(vxp->sb, "Expected integer got '%.*s' ",
+			    PF(vxp->t));
+			vxp_ErrWhere(vxp, vxp->t, -1);
+			return;
+		}
+		(*plhs)->level = (int)strtol(vxp->t->dec, &p, 0);
+		if ((*plhs)->level < 0) {
+			VSB_printf(vxp->sb, "Expected positive integer ");
+			vxp_ErrWhere(vxp, vxp->t, -1);
+			return;
+		}
+		if (*p == '-') {
+			(*plhs)->level_pm = -1;
+			p++;
+		} else if (*p == '+') {
+			(*plhs)->level_pm = 1;
+			p++;
+		}
+		if (*p) {
+			VSB_printf(vxp->sb, "Syntax error in level limit ");
+			vxp_ErrWhere(vxp, vxp->t, -1);
+			return;
+		}
+		vxp_NextToken(vxp);
+		ExpectErr(vxp, '}');
+		vxp_NextToken(vxp);
+	}
+
 	while (1) {
 		/* The tags this expression applies to */
 		if (vxp->t->tok != VAL) {
@@ -127,8 +161,6 @@ vxp_expr_lhs(struct vxp *vxp, struct vex_lhs **plhs)
 		ExpectErr(vxp, ']');
 		vxp_NextToken(vxp);
 	}
-
-	/* XXX: LHS Level {} */
 }
 
 static void
@@ -539,14 +571,19 @@ vex_print(const struct vex *vex, int indent)
 	if (vex->lhs != NULL) {
 		CHECK_OBJ_NOTNULL(vex->lhs, VEX_LHS_MAGIC);
 		AN(vex->lhs->tags);
-		fprintf(stderr, " lhs=(");
+		fprintf(stderr, " lhs=");
+		if (vex->lhs->level >= 0)
+			fprintf(stderr, "{%d%s}", vex->lhs->level,
+			    vex->lhs->level_pm < 0 ? "-" :
+			    vex->lhs->level_pm > 0 ? "+" : "");
+		fprintf(stderr, "(");
 		vex_print_tags(vex->lhs->tags);
 		fprintf(stderr, ")");
 		if (vex->lhs->prefix) {
 			assert(vex->lhs->prefixlen == strlen(vex->lhs->prefix));
 			fprintf(stderr, ":%s", vex->lhs->prefix);
 		}
-		if (vex->lhs->field >= 0)
+		if (vex->lhs->field > 0)
 			fprintf(stderr, "[%d]", vex->lhs->field);
 	}
 	if (vex->rhs != NULL) {



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] eab1696 Add query expression documentation
Message-ID: <E1VQzND-0005kP-Gm@project.varnish-software.com>

commit eab169644fd483c207afc84d2a7354c5a8fbf99a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Sep 26 17:01:30 2013 +0200

    Add query expression documentation

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 1ba6eb1..3152943 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -128,6 +128,7 @@ EXTRA_DIST = \
 	reference/vmod.rst \
 	reference/vmod_std.rst \
 	reference/vsm.rst \
+	reference/vsl-query.rst \
 	tutorial/index.rst \
 	tutorial/introduction.rst \
 	tutorial/starting_varnish.rst \
diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index ab27df2..13f7458 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -23,6 +23,7 @@ The Varnish Reference Manual
 	vmod.rst
 	vmod_std.rst
 	vsl.rst
+	vsl-query.rst
 
 .. todo::
         The programs:
diff --git a/doc/sphinx/reference/vsl-query.rst b/doc/sphinx/reference/vsl-query.rst
new file mode 100644
index 0000000..73f69c3
--- /dev/null
+++ b/doc/sphinx/reference/vsl-query.rst
@@ -0,0 +1,235 @@
+.. _ref-vsl-query:
+
+=============================
+Varnish VSL Query Expressions
+=============================
+
+:Author: Martin Blix Grydeland
+:Date: 2013-09-26
+:Version: 0.1
+:Manual section: 7
+
+OVERVIEW
+========
+
+The Varnish VSL Query Expressions extracts transactions from the
+Varnish shared memory log, and perform queries on the transactions
+before reporting matches.
+
+A transaction is a set of log lines that belongs together, e.g. a
+client request. The API monitors the log, and collects all log records
+that make up a transaction before reporting on that
+transaction. Transactions can also be grouped, meaning backend
+transactions are reported together with the client transaction that
+initiated it.
+
+A query is run on a group of transactions. A query expression is true
+if there is a log record within the group that satisfies the
+condition. It is false only if none of the log records satisfied the
+condition. Query expressions can be combined using boolean functions.
+
+GROUPING
+========
+
+When grouping transactions, there is a hierarchy structure showing
+which transaction initiated what. The level increases by one by a
+'initiated by' relation, so for example a backend transaction will
+have one higher level than the client transaction that initiated it on
+a cache miss. Levels start counting at 1, except when using raw where
+it will always be 0.
+
+The grouping modes are:
+
+* Session
+
+  All transactions initiated by a client connection is reported
+  together. All log data is buffered until the client connection is
+  closed, which can cause session grouping mode to potentially consume
+  a lot of memory.
+
+* Request
+
+  Transactions are grouped by request, where the set will include the
+  request itself, and any backend requests or ESI-subrequests. Session
+  data is not reported. This is the default.
+
+* VXID
+
+  Transactions are not grouped, so each VXID is reported in it's
+  entirety. Sessions, requests, ESI-requests and backend requests are
+  all reported individually. Non-transactional data is not reported
+  (VXID == 0).
+
+* Raw
+
+  Every log record will make up a transaction of it's own. All data,
+  including non-transactional data will be reported.
+
+Example transaction hierarchy ::
+
+  Lvl 1: Client request (cache miss)
+    Lvl 2: Backend request
+    Lvl 2: ESI subrequest (cache miss)
+      Lvl 3: Backend request
+      Lvl 3: Backend request (VCL restart)
+      Lvl 3: ESI subrequest (cache miss)
+        Lvl 4: Backend request
+    Lvl 2: ESI subrequest (cache hit)
+
+QUERY LANGUAGE
+==============
+
+A query expression consists of a record selection criteria, and
+optionally an operator and a value to match against ::
+
+  <record selection criteria> <operator> <operand>
+
+Record selection criteria
+-------------------------
+
+The record selection criteria determines what kind records from the
+transaction group the expression applies to. Syntax: ::
+
+  {level}taglist:record-prefix[field]
+
+Taglist is mandatory, the other components are optional.
+
+The level limits the expression to a transaction at that level. If
+left unspecified the expression is applied to transactions at all
+levels. Level is a positive integer or zero. If level is followed by a
+'+' character, it expresses greater than or equal. If level is
+followed by a '-', it expresses less than or equal.
+
+The taglist is a comma-separated list of VSL record tags that this
+expression should be applied against. Each list element can be a tag
+name or a tag glob. Globs allow a '*' either in the beginning of
+the name or at the end, and will select all tags that match either the
+prefix or subscript. A single '*' will select all tags.
+
+The record prefix will further limit the record set matched against to
+those records that has this prefix as it's first part of the record
+content followed by a colon. The part of the log record matched
+against will then be limited to what follows the prefix and
+colon. This is useful when matching against specific HTTP headers. The
+record prefix matching is done case insensitive.
+
+The field will, if present, treat the log record as a white space
+separated list of fields, and only the nth part of the record will be
+matched against. Fields start counting at 1.
+
+An expression using only a record selection criteria will be true if
+there is any record in the transaction group that is selected by the
+criteria.
+
+Operators
+---------
+
+The following matching operators are available:
+
+* == != < <= > >=
+
+  Numerical comparison. The record contents will be converted to
+  either an integer or a float before comparison, depending on the
+  type of the operand.
+
+* eq ne
+
+  String comparison. 'eq' tests string equality, 'ne' tests for not
+  equality.
+
+* ~ !~
+
+  Regular expression matching. '~' is a positive match, '!~' is a
+  non-match.
+
+Operand
+-------
+
+The operand is the value the selected records will be matched
+against.
+
+An operand can be quoted or unquoted. Quotes can be either single or
+double quotes, and for quoted operands a backslash can be used to
+escape the quotes.
+
+Unquoted operands can only consist of the following characters: ::
+
+  a-z A-Z 0-9 + - _ . *
+
+The following types of operands are available:
+
+* Integer
+
+  A number without any fractional part, valid for the numerical
+  comparison operators. The integer type is used when the operand does
+  not contain any period (.) characters.
+
+* Float
+
+  A number with a fractional part, valid for the numerical comparison
+  operators. The float type is used when the operand does contain a
+  period (.) character.
+
+* String
+
+  A sequence of characters, valid for the string equality operators.
+
+* Regular expression
+
+  A PCRE regular expression. Valid for the regular expression
+  operators.
+
+Boolean functions
+-----------------
+
+Query expressions can be linked together using boolean functions. The
+following are available, in decreasing precedence:
+
+* not <expr>
+
+  Inverts the result of <expr>
+
+* <expr1> and <expr2>
+
+  True only if both expr1 and expr2 are true
+
+* <expr1> or <expr2>
+
+  True if either of expr1 or expr2 is true
+
+Expressions can be grouped using parenthesis.
+
+QUERY EXPRESSION EXAMPLES
+=========================
+
+* Transaction group contains a request URL that equals to "/foo" ::
+
+    ReqURL eq "/foo"
+
+* Transaction group contains a request cookie header ::
+
+    ReqHeader:cookie
+
+* Transaction group doesn't contain a request cookie header ::
+
+    not ReqHeader:cookie
+
+* Transaction group contains a request user-agent header that contains
+  "iPod" and the request delivery time exceeds 1 second ::
+
+    ReqHeader:user-agent ~ "iPod" and ReqEnd[5] > 1.
+
+* Transaction group contains a backend response status larger than or
+  equal to 500 ::
+
+    BerespStatus >= 500
+
+* Transaction group contains a request response status of 304, but
+  where the request did not contain an if-modified-since header ::
+
+    ReqStatus == 304 and not ReqHeader:if-modified-since
+
+* Transactions that has backend failures or long delivery time on
+  their ESI subrequests. (Assumes request grouping mode). ::
+
+    BerespStatus >= 500 or {1+}ReqEnd[5] > 1.



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] bcc5630 Remove query doc from varnishlog
Message-ID: <E1VQzND-0005kY-IY@project.varnish-software.com>

commit bcc56307fbbd69597a5e6919ea110d5b948987c6
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Sep 30 14:51:38 2013 +0200

    Remove query doc from varnishlog
    
    The query language has it's own doc page in varnish-query

diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index 6c1fd9d..3a1f1f2 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -69,170 +69,6 @@ The following options are available:
 
 	XXX: Not yet implemented
 
-
-DESCRIPTION
-===========
-
-Varnishlog is a utility to extract and query the Varnish shared memory
-log.
-
-Varnishlog operates on transactions. A transaction is a set of log
-lines that belongs together, e.g. a client request. Varnishlog will
-monitor the log, and collect all log records that make up a
-transaction before reporting on that transaction. Transactions can
-also be grouped, meaning backend transactions are reported together
-with the client transaction that initiated it.
-
-The grouping levels are:
-
-* Session
-
-  All transactions initiated by a client connection is reported
-  together. All log data is buffered until the client connection is
-  closed.
-
-* Request
-
-  Transactions are grouped by request, where the set will include the
-  request itself, and any backend requests or ESI-subrequests. Session
-  data is not reported. This is the default.
-
-* VXID
-
-  Transactions are not grouped, so each VXID is reported in it's
-  entirity. Sessions, requests, ESI-requests and backend requests are
-  all reported individually. Non-transactional data is not reported
-  (VXID == 0).
-
-* Raw
-
-  Every log record will make up a transaction of it's own. All data,
-  including non-transactional data will be reported.
-
-
-Grouping
-========
-
-When grouping transactions, there is a hirarchy structure showing
-which transaction initiated what. The level increases by one by a
-'initiated by' relation, so for example a backend transaction will
-have one higher level than the client transaction that initiated it on
-a cache miss.
-
-Example transaction hirarchy ::
-
-  Lvl 1: Client request (cache miss)
-    Lvl 2: Backend request
-    Lvl 2: ESI subrequest (cache miss)
-      Lvl 3: Backend request
-      Lvl 3: Backend request (VCL restart)
-      Lvl 3: ESI subrequest (cache miss)
-        Lvl 4: Backend request
-    Lvl 2: ESI subrequest (cache hit)
-
-Query operators will unless limited see a grouped set of transactions
-together, and matching will be done on any log line from the complete
-set. See QUERY LANGUAGE for how to limit a match to a specific part of
-the set.
-
-Running queries in session grouping mode can potentially consume a lot
-of memory.
-
-
-QUERY LANGUAGE
-==============
-
-XXX: As a POC only a single string is accepted as a query expression,
-and this will be used as a regular expression that will be matched
-against any log line of the set. The rest of the query language is yet
-to be implemented.
-
-The query expression is given as a single command line
-argument. Additional arguments will give an error.
-
-An expression consists of a single tag, or a comparison between a tag
-and a constant.
-
-A single tag expression is considered true if there is one or more
-records with that tag in the transaction.
-
-For all comparisons, the LHS must be a tag, and the RHS must be a
-constant.
-
-Constants must be quoted if they contain whitespace. You can use
-either single or double quotes.
-
-A comparison expression is true if the comparison is true for one or
-more records with that tag in the transaction.
-
-(be)?re(q|sp).(url|request|status|response) expands to their specific
-tags.
-
-(be)?re(q|sp).http.<header> expands to their corresponding
-(Ber|R)(eq|esp)Header tag, and for this comparison the value will be
-s/^(?i)<header>: //
-
-<tag>{n} will only match on a transaction at the nth level (see
-grouping). Levels starts counting at 0. If n is followed by a '+',
-it will only match at level n or higher. If n is followed by a '-', it
-will only match at level n or lower.
-
-<tag>[n] will consider the value of the tag to be a white-space
-separated field list, and extract the nth field for the comparison.
-
-<tag>#n adds a repetition counter to this match, and is true only if
-the match is true n times. n+ means n or more, n- means n or less.
-
-'==', '!=', '<', '<=', '>' and '>=' are numerical comparisons. Integer
-by default, or floating point if the RHS contains a dot. LHS will be
-transformed (atoi/atof) for comparison.
-
-'eq' and 'ne' are for string comparison.
-
-'~' and '!~' are PCRE regular expression comparisons.
-
-'not' is for negation
-
-'and' is concatenation
-
-'or' is alteration
-
-'not' has highest precedence, alternation and concatenation have equal
-precedence and associate left to right. Paranthesis can be used to
-group statements.
-
-QUERY EXAMPLES
-==============
-
-The following commands will list the entire client transaction of
-requests where the url is "/foo" ::
-
-	$ varnishlog -c 'req.url eq "/foo"'
-	$ varnishlog -c 'ReqURL eq "/foo"'
-
-The following command will list the URL of all requests that has a
-cookie-header ::
-
-	$ varnishlog -c -i ReqURL req.http.cookie
-	$ varnishlog -c -i ReqURL 'ReqHeader ~ "^Cookie: "'
-
-Report the User-Agent of logged in clients where the request delivery
-time exceeds exceeds 0.5 seconds ::
-
-	$ varnishlog -c -I RxHeader:User-Agent 'req.http.cookie ~
-	"logged_in" and ReqEnd[5] > 0.5'
-
-Report delivery status code of client requests that had one or more
-503 errors from backend requests ::
-
-	$ varnishlog -i TxStatus 'beresp.status == 503'
-
-Report transaction set on requests that has backend failures
-or long delivery time on their ESI subrequests ::
-
-	$ varnishlog 'beresp.status{2+} >= 500 or ReqEnd{1+}[5] > 0.5'
-
-
 TAGS
 ====
 The following log entry tags are currently defined:
@@ -296,6 +132,7 @@ SEE ALSO
 * varnishncsa(1)
 * varnishstat(1)
 * varnishtop(1)
+* vsl-query(7)
 
 HISTORY
 =======



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] 441c29d Fix query expression error printing
Message-ID: <E1VQzND-0005kh-KG@project.varnish-software.com>

commit 441c29d27087eb4c1ae965f8bcbf9f25e0ae90fc
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Sep 30 15:19:45 2013 +0200

    Fix query expression error printing

diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 7f1b66b..40780bb 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -297,7 +297,7 @@ vslq_newquery(struct VSL_data *vsl, enum VSL_grouping_e grouping,
 	vex = vex_New(querystring, vsb);
 	VSB_finish(vsb);
 	if (vex == NULL)
-		vsl_diag(vsl, "Query expression error:\n%s", VSB_data(vsb));
+		vsl_diag(vsl, "%s", VSB_data(vsb));
 	else {
 		ALLOC_OBJ(query, VSLQ_QUERY_MAGIC);
 		query->vex = vex;
diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index 4cf5c71..8c2f90c 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -207,7 +207,7 @@ VUT_Setup(void)
 	/* Create query */
 	VUT.vslq = VSLQ_New(VUT.vsl, &c, VUT.g_arg, VUT.q_arg);
 	if (VUT.vslq == NULL)
-		VUT_Error(1, "Query parse error (%s)", VSL_Error(VUT.vsl));
+		VUT_Error(1, "Query expression error:\n%s", VSL_Error(VUT.vsl));
 	AZ(c);
 
 	/* Signal handlers */



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] 4dfcedb Fix mgt uptime counter when suspended
Message-ID: <E1VQzND-0005ks-MD@project.varnish-software.com>

commit 4dfcedbde56df0a6ccfa1d096bf6bed61a0d0f5b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Sep 30 17:12:24 2013 +0200

    Fix mgt uptime counter when suspended
    
    Use VTIM_real instead of VTIM_mono for mgt uptime so that the time
    won't stop counting when the computer is suspended.

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index aa66c5a..2f81db1 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -694,7 +694,7 @@ mgt_uptime(const struct vev *e, int what)
 	(void)what;
 	AN(VSC_C_mgt);
 	VSC_C_mgt->uptime = static_VSC_C_mgt.uptime =
-	    VTIM_mono() - mgt_uptime_t0;
+	    VTIM_real() - mgt_uptime_t0;
 	if (heritage.vsm != NULL)
 		VSM_common_ageupdate(heritage.vsm);
 	return (0);
@@ -713,7 +713,7 @@ MGT_Run(void)
 	struct vev *e;
 	int i;
 
-	mgt_uptime_t0 = VTIM_mono();
+	mgt_uptime_t0 = VTIM_real();
 	e = vev_new();
 	XXXAN(e);
 	e->callback = mgt_uptime;



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] 872fb13 Reintroduce Ctrl-C and Ctrl-Z key handlers in
 varnishstat
Message-ID: <E1VQzND-0005kx-O7@project.varnish-software.com>

commit 872fb1372c2cb05a16b4f00ef499325eeeec1a4a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 09:54:16 2013 +0200

    Reintroduce Ctrl-C and Ctrl-Z key handlers in varnishstat

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 16df1b4..d870016 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -832,9 +832,15 @@ handle_keypress(int ch)
 	case 'Q':
 		keep_running = 0;
 		return;
+	case '\003':		/* Ctrl-C */
+		AZ(raise(SIGINT));
+		return;
 	case '\024':		/* Ctrl-T */
 		sample = 1;
 		return;
+	case '\032':		/* Ctrl-Z */
+		AZ(raise(SIGTSTP));
+		return;
 	default:
 		return;
 	}



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] d020188 Don't output error messages to stderr when in curses
 mode
Message-ID: <E1VQzND-0005l5-Pv@project.varnish-software.com>

commit d020188e6e330a202ed3a5abd4eed4338f63850f
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 10:09:01 2013 +0200

    Don't output error messages to stderr when in curses mode

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index d870016..ff69c0e 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -878,9 +878,7 @@ do_curses(struct VSM_data *vd, int delay)
 		if (VSM_Abandoned(vd)) {
 			delete_pt_list();
 			VSM_Close(vd);
-			if (VSM_Open(vd) < 0)
-				fprintf(stderr, "VSM_Open failed: %s\n",
-				    VSM_Error(vd));
+			VSM_Open(vd);
 		}
 		VSC_C_mgt = VSC_Mgt(vd, &f_mgt);
 		VSC_C_main = VSC_Main(vd, &f_main);



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] ddff9a2 Show a disconnected message in varnishstat curses
 when not having an open VSM connection
Message-ID: <E1VQzND-0005lD-RU@project.varnish-software.com>

commit ddff9a26bdf5a49e68761a700fb806b02069a61a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 10:09:26 2013 +0200

    Show a disconnected message in varnishstat curses when not having an
    open VSM connection

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index ff69c0e..cfffac5 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -497,6 +497,7 @@ draw_status(void)
 {
 	time_t up_mgt = 0;
 	time_t up_chld = 0;
+	static const char discon[] = "*** DISCONNECTED ***";
 
 	AN(w_status);
 
@@ -513,6 +514,9 @@ draw_status(void)
 	    up_chld / 86400, (up_chld % 86400) / 3600, (up_chld % 3600) / 60,
 	    up_chld % 60);
 
+	if (VSC_C_mgt == NULL)
+		mvwprintw(w_status, 0, COLS - strlen(discon), discon);
+
 	wnoutrefresh(w_status);
 }
 



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] 2b9d43c Properly initialize the fantom iterators before
 using them.
Message-ID: <E1VQzND-0005lL-TA@project.varnish-software.com>

commit 2b9d43cc84fec99ebf41bf91455dc85664ee34f2
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 11:06:04 2013 +0200

    Properly initialize the fantom iterators before using them.
    
    Spotted by: Coverity

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index cfffac5..ecd7620 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -860,7 +860,9 @@ do_curses(struct VSM_data *vd, int delay)
 	long t;
 	int ch;
 	double now;
-	struct VSM_fantom f_main, f_mgt, f_iter;
+	struct VSM_fantom f_main = VSM_FANTOM_NULL;
+	struct VSM_fantom f_mgt = VSM_FANTOM_NULL;
+	struct VSM_fantom f_iter = VSM_FANTOM_NULL;
 
 	(void)delay;
 



From martin at varnish-cache.org  Tue Oct  1 12:48:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:19 +0200
Subject: [master] 7b18d9a Fix up file reading byte sizes when reading VSL log
 records from a file.
Message-ID: <E1VQzND-0005lT-VZ@project.varnish-software.com>

commit 7b18d9a074f7dd1c6fbec42d68aaec10a57acc17
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 13:25:33 2013 +0200

    Fix up file reading byte sizes when reading VSL log records from a file.
    
    The code got word sizes confused, and failed to allocate a proper
    buffer based on the word size. Make the code use word sizes in local
    variables for clarity and make sure that the allocation routines get
    byte sizes.
    
    Spotted by: Coverity

diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index c4f3cf1..d1fcfe0 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -306,6 +306,7 @@ vslc_file_delete(struct VSL_cursor *cursor)
 	FREE_OBJ(c);
 }
 
+/* Read n bytes from fd into buf */
 static ssize_t
 vslc_file_readn(int fd, void *buf, size_t n)
 {
@@ -325,7 +326,8 @@ static int
 vslc_file_next(struct VSL_cursor *cursor)
 {
 	struct vslc_file *c;
-	ssize_t i, l;
+	ssize_t i;
+	size_t l;
 
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_FILE_MAGIC);
 	assert(&c->cursor == cursor);
@@ -335,27 +337,28 @@ vslc_file_next(struct VSL_cursor *cursor)
 
 	do {
 		c->cursor.rec.ptr = NULL;
-		assert(c->buflen >= VSL_BYTES(2));
+		assert(c->buflen >= 2);
 		i = vslc_file_readn(c->fd, c->buf, VSL_BYTES(2));
 		if (i < 0)
 			return (-4);	/* I/O error */
 		if (i == 0)
 			return (-1);	/* EOF */
 		assert(i == VSL_BYTES(2));
-		l = VSL_BYTES(2 + VSL_WORDS(VSL_LEN(c->buf)));
+		l = 2 + VSL_WORDS(VSL_LEN(c->buf));
 		if (c->buflen < l) {
-			c->buf = realloc(c->buf, 2 * l);
+			while (c->buflen < l)
+				c->buflen = 2 * l;
+			c->buf = realloc(c->buf, VSL_BYTES(c->buflen));
 			AN(c->buf);
-			c->buflen = 2 * l;
 		}
-		if (l > VSL_BYTES(2)) {
+		if (l > 2) {
 			i = vslc_file_readn(c->fd, c->buf + 2,
-			    l - VSL_BYTES(2));
+			    VSL_BYTES(l - 2));
 			if (i < 0)
 				return (-4);	/* I/O error */
 			if (i == 0)
 				return (-1);	/* EOF */
-			assert(i == l - VSL_BYTES(2));
+			assert(i == VSL_BYTES(l - 2));
 		}
 		c->cursor.rec.ptr = c->buf;
 	} while (VSL_TAG(c->cursor.rec.ptr) == SLT__Batch);
@@ -425,8 +428,8 @@ VSL_CursorFile(struct VSL_data *vsl, const char *name)
 	c->cursor.priv_data = c;
 
 	c->fd = fd;
-	c->buflen = BUFSIZ;
-	c->buf = malloc(c->buflen);
+	c->buflen = VSL_WORDS(BUFSIZ);
+	c->buf = malloc(VSL_BYTES(c->buflen));
 	AN(c->buf);
 
 	return (&c->cursor);



From martin at varnish-cache.org  Tue Oct  1 12:48:20 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:20 +0200
Subject: [master] e773852 Free temp VSM structure when finished with it in
 n_arg_sock
Message-ID: <E1VQzNE-0005lb-0m@project.varnish-software.com>

commit e773852f5dfa3d7b968aedfc0338d822a51d4008
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 13:35:08 2013 +0200

    Free temp VSM structure when finished with it in n_arg_sock
    
    Spotted by: Coverity

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index e7dc7ba..f356546 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -387,17 +387,21 @@ n_arg_sock(const char *n_arg)
 	struct VSM_fantom vt;
 
 	vsm = VSM_New();
+	AN(vsm);
 	if (VSM_n_Arg(vsm, n_arg) < 0) {
 		fprintf(stderr, "%s\n", VSM_Error(vsm));
+		VSM_Delete(vsm);
 		return (-1);
 	}
 	if (VSM_Open(vsm)) {
 		fprintf(stderr, "%s\n", VSM_Error(vsm));
+		VSM_Delete(vsm);
 		return (-1);
 	}
 
 	if (!VSM_Get(vsm, &vt, "Arg", "-T", "")) {
 		fprintf(stderr, "No -T arg in shared memory\n");
+		VSM_Delete(vsm);
 		return (-1);
 	}
 	AN(vt.b);
@@ -408,6 +412,8 @@ n_arg_sock(const char *n_arg)
 		S_arg = strdup(vt.b);
 	}
 
+	VSM_Delete(vsm);
+
 	sock = -1;
 	while (*T_arg) {
 		p = strchr(T_arg, '\n');



From martin at varnish-cache.org  Tue Oct  1 12:48:20 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:20 +0200
Subject: [master] c112f3f Keep a flag telling if the file cursor's fd should
 be closed on delete.
Message-ID: <E1VQzNE-0005lj-2g@project.varnish-software.com>

commit c112f3f46544f17e5bd6d1cc11054450c60ff572
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 13:42:09 2013 +0200

    Keep a flag telling if the file cursor's fd should be closed on delete.
    
    Spotted by: Coverity

diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index d1fcfe0..fe1f82e 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -288,6 +288,7 @@ struct vslc_file {
 
 	int				error;
 	int				fd;
+	int				close_fd;
 	ssize_t				buflen;
 	uint32_t			*buf;
 };
@@ -299,7 +300,7 @@ vslc_file_delete(struct VSL_cursor *cursor)
 
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_FILE_MAGIC);
 	assert(&c->cursor == cursor);
-	if (c->fd > STDIN_FILENO)
+	if (c->close_fd)
 		(void)close(c->fd);
 	if (c->buf != NULL)
 		free(c->buf);
@@ -387,6 +388,7 @@ VSL_CursorFile(struct VSL_data *vsl, const char *name)
 {
 	struct vslc_file *c;
 	int fd;
+	int close_fd = 0;
 	char buf[] = VSL_FILE_ID;
 	ssize_t i;
 
@@ -399,11 +401,12 @@ VSL_CursorFile(struct VSL_data *vsl, const char *name)
 			    strerror(errno));
 			return (NULL);
 		}
+		close_fd = 1;
 	}
 
 	i = vslc_file_readn(fd, buf, sizeof buf);
 	if (i <= 0) {
-		if (fd > STDIN_FILENO)
+		if (close_fd)
 			(void)close(fd);
 		vsl_diag(vsl, "VSL file read error: %s\n",
 		    i < 0 ? strerror(errno) : "EOF");
@@ -411,7 +414,7 @@ VSL_CursorFile(struct VSL_data *vsl, const char *name)
 	}
 	assert(i == sizeof buf);
 	if (memcmp(buf, VSL_FILE_ID, sizeof buf)) {
-		if (fd > STDIN_FILENO)
+		if (close_fd)
 			(void)close(fd);
 		vsl_diag(vsl, "Not a VSL file: %s\n", name);
 		return (NULL);
@@ -419,7 +422,7 @@ VSL_CursorFile(struct VSL_data *vsl, const char *name)
 
 	ALLOC_OBJ(c, VSLC_FILE_MAGIC);
 	if (c == NULL) {
-		if (fd > STDIN_FILENO)
+		if (close_fd)
 			(void)close(fd);
 		vsl_diag(vsl, "Out of memory\n");
 		return (NULL);
@@ -428,6 +431,7 @@ VSL_CursorFile(struct VSL_data *vsl, const char *name)
 	c->cursor.priv_data = c;
 
 	c->fd = fd;
+	c->close_fd = close_fd;
 	c->buflen = VSL_WORDS(BUFSIZ);
 	c->buf = malloc(VSL_BYTES(c->buflen));
 	AN(c->buf);



From martin at varnish-cache.org  Tue Oct  1 12:48:20 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:20 +0200
Subject: [master] 3c8d56a Free tmp var in vsc_f_arg when in error condition.
Message-ID: <E1VQzNE-0005lr-4M@project.varnish-software.com>

commit 3c8d56a8b0683b0c37bd9cfb84b57b76f9390e10
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 13:44:57 2013 +0200

    Free tmp var in vsc_f_arg when in error condition.
    
    Spotted by: Coverity

diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index f2f270c..c617520 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -184,8 +184,11 @@ vsc_f_arg(struct VSM_data *vd, const char *opt)
 
 	av = VAV_Parse(opt, NULL, ARGV_COMMA);
 	AN(av);
-	if (av[0] != NULL)
-		return (vsm_diag(vd, "Parse error: %s", av[0]));
+	if (av[0] != NULL) {
+		i = vsm_diag(vd, "Parse error: %s", av[0]);
+		VAV_Free(av);
+		return (i);
+	}
 	for (i = 1; av[i] != NULL; i++) {
 		ALLOC_OBJ(sf, VSC_SF_MAGIC);
 		AN(sf);



From martin at varnish-cache.org  Tue Oct  1 12:48:20 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 14:48:20 +0200
Subject: [master] 50bc002 Fix buffer overrun in varnishadm pass function
Message-ID: <E1VQzNE-0005lz-64@project.varnish-software.com>

commit 50bc002998c7040c56bbdbb994bad173e3f97784
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 14:32:44 2013 +0200

    Fix buffer overrun in varnishadm pass function
    
    Spotted by: Coverity

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index f356546..982fbc2 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -351,7 +351,7 @@ pass(int sock)
 			}
 		}
 		if (fds[1].revents & POLLIN || fds[1].revents & POLLHUP) {
-			n = read(fds[1].fd, buf, sizeof buf);
+			n = read(fds[1].fd, buf, sizeof buf - 1);
 			if (n == 0) {
 				AZ(shutdown(sock, SHUT_WR));
 				fds[1].fd = -1;



From martin at varnish-cache.org  Tue Oct  1 13:54:49 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 15:54:49 +0200
Subject: [master] d66fc3f Use spaces instead of tabs to keep python3 happy
Message-ID: <E1VR0PZ-0006j6-IJ@project.varnish-software.com>

commit d66fc3f9c777d670bdc5e76a6171dd96787dafd6
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 15:54:33 2013 +0200

    Use spaces instead of tabs to keep python3 happy

diff --git a/lib/libvarnishapi/generate.py b/lib/libvarnishapi/generate.py
index 87bd044..60ec07b 100755
--- a/lib/libvarnishapi/generate.py
+++ b/lib/libvarnishapi/generate.py
@@ -60,7 +60,7 @@ tokens = {
         # Boolean operators
         "T_AND":        "and",
         "T_OR":         "or",
-	"T_NOT":	"not",
+        "T_NOT":        "not",
 
         # Miscellaneous
         None:           "<>~[]{}():,",
@@ -69,8 +69,8 @@ tokens = {
         "VAL":          None,
         "EOI":          None,
 
-	# Special
-	"T_TRUE":	None,
+        # Special
+        "T_TRUE":       None,
 }
 
 #######################################################################
@@ -106,19 +106,19 @@ vxp_fixed_token(const char *p, const char **q)
                         if (j[0] != last_initial):
                                 continue
 
-			fo.write("\t\tif (")
-			k = 1
-			l = len(j)
-			while (k < l):
-				fo.write("p[%d] == '%s'" % (k, j[k]))
-				fo.write(" &&\n\t\t    ")
-				k += 1
-			fo.write("(isword(p[%d]) ? !isword(p[%d]) : 1)) {\n" %
-				 (l - 1, l))
-			fo.write("\t\t\t*q = p + %d;\n" % l)
-			fo.write("\t\t\treturn (%s);\n" % emit[j])
-			fo.write("\t\t}\n");
-		fo.write("\t\treturn (0);\n")
+                        fo.write("\t\tif (")
+                        k = 1
+                        l = len(j)
+                        while (k < l):
+                                fo.write("p[%d] == '%s'" % (k, j[k]))
+                                fo.write(" &&\n\t\t    ")
+                                k += 1
+                        fo.write("(isword(p[%d]) ? !isword(p[%d]) : 1)) {\n" %
+                                 (l - 1, l))
+                        fo.write("\t\t\t*q = p + %d;\n" % l)
+                        fo.write("\t\t\treturn (%s);\n" % emit[j])
+                        fo.write("\t\t}\n");
+                fo.write("\t\treturn (0);\n")
 
         fo.write("\tdefault:\n\t\treturn (0);\n\t}\n}\n")
 



From phk at varnish-cache.org  Tue Oct  1 14:05:47 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 16:05:47 +0200
Subject: [master] 282df8e Zero the sigaction structure,
 to make it obvious to Coverity that we know what we're doing.
Message-ID: <E1VR0aB-00085c-MF@project.varnish-software.com>

commit 282df8ede5daf018f17bff4171d4552c6340b622
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 14:03:00 2013 +0000

    Zero the sigaction structure, to make it obvious to Coverity that
    we know what we're doing.

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 2f81db1..5aad339 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -412,6 +412,7 @@ mgt_launch_child(struct cli *cli)
 #endif
 
 		if (mgt_param.sigsegv_handler) {
+			memset(&sa, 0, sizeof sa);
 			sa.sa_sigaction = mgt_sigsegv_handler;
 			sa.sa_flags = SA_SIGINFO;
 			(void)sigaction(SIGSEGV, &sa, NULL);



From phk at varnish-cache.org  Tue Oct  1 14:12:59 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 16:12:59 +0200
Subject: [master] e0ea91a Add comment to explain why we don't free av (The
 hasher gets to keep it)
Message-ID: <E1VR0h9-0000D7-EW@project.varnish-software.com>

commit e0ea91a01a4187b87fc3e661e559260ae7b65700
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 14:12:28 2013 +0000

    Add comment to explain why we don't free av (The hasher gets to keep it)

diff --git a/bin/varnishd/hash/hash_mgt.c b/bin/varnishd/hash/hash_mgt.c
index 30460b3..a8cc352 100644
--- a/bin/varnishd/hash/hash_mgt.c
+++ b/bin/varnishd/hash/hash_mgt.c
@@ -80,4 +80,5 @@ HSH_config(const char *h_arg)
 	else if (ac > 0)
 		ARGV_ERR("Hash method \"%s\" takes no arguments\n",
 		    hp->name);
+	/* NB: Don't free av, the hasher is allowed to keep it. */
 }



From phk at varnish-cache.org  Tue Oct  1 14:17:25 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 16:17:25 +0200
Subject: [master] 3998482 Try to make Coverity understand that this struct
 telnet is used.
Message-ID: <E1VR0lR-0000o4-Ji@project.varnish-software.com>

commit 39984825007b07e974305ab63358f11debb89937
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 14:17:09 2013 +0000

    Try to make Coverity understand that this struct telnet is used.

diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index 6f319f6..9ee037f 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -550,6 +550,7 @@ mgt_cli_telnet(const char *T_arg)
 		tn->ev->fd = sock;
 		tn->ev->fd_flags = POLLIN;
 		tn->ev->callback = telnet_accept;
+		tn->ev->priv = tn;;
 		AZ(vev_add(mgt_evb, tn->ev));
 		free(ta[i]);
 		ta[i] = NULL;



From phk at varnish-cache.org  Tue Oct  1 14:20:07 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 16:20:07 +0200
Subject: [master] 037fb7c Add comment to explain why we don't free av:
 Stevedore keeps it.
Message-ID: <E1VR0o3-00016a-Ad@project.varnish-software.com>

commit 037fb7c1365cbd8752000b797db1aaf408168263
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 14:19:50 2013 +0000

    Add comment to explain why we don't free av:  Stevedore keeps it.

diff --git a/bin/varnishd/storage/stevedore_mgt.c b/bin/varnishd/storage/stevedore_mgt.c
index 4251839..fd0cbec 100644
--- a/bin/varnishd/storage/stevedore_mgt.c
+++ b/bin/varnishd/storage/stevedore_mgt.c
@@ -164,6 +164,7 @@ STV_Config(const char *spec)
 	} else {
 		VTAILQ_INSERT_TAIL(&stv_stevedores, stv, list);
 	}
+	/* NB: Do not free av, stevedore gets to keep it */
 }
 
 /*--------------------------------------------------------------------*/



From phk at varnish-cache.org  Tue Oct  1 14:27:32 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 16:27:32 +0200
Subject: [master] ba37d41 Fix memory-leak spotted by Coverity
Message-ID: <E1VR0vE-0001n1-VC@project.varnish-software.com>

commit ba37d4136c6a02aad8ed92f0a304091a252e22a5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 14:27:22 2013 +0000

    Fix memory-leak spotted by Coverity

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index df311be..097a04b 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -543,6 +543,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 				st = NULL;
 		}
 	}
+	ObjIterEnd(&oi);
 	assert(al == bo->ims_obj->len);
 	assert(obj->len == al);
 	if (bo->state != BOS_FAILED)



From phk at varnish-cache.org  Tue Oct  1 14:43:53 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 16:43:53 +0200
Subject: [master] 89aa01f Plug an isignificant resource leak.
Message-ID: <E1VR1B3-0003E5-Ba@project.varnish-software.com>

commit 89aa01f4e2a322da8bb682042549f03c5ae9612a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 14:43:33 2013 +0000

    Plug an isignificant resource leak.
    
    Found by Coverity

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index e75191e..2c8b77a 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -607,6 +607,7 @@ tweak_poolparam(struct cli *cli, const struct parspec *par, const char *arg)
 			}
 			*pp = px;
 		} while(0);
+		VAV_Free(av);
 	}
 }
 



From phk at varnish-cache.org  Tue Oct  1 14:48:11 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 16:48:11 +0200
Subject: [master] e9cb722 Fix insignificant mem-leak
Message-ID: <E1VR1FD-0003em-SY@project.varnish-software.com>

commit e9cb722615106e762ab9661f1494d7a39ee4bf57
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 14:47:54 2013 +0000

    Fix insignificant mem-leak
    
    Spotted by: Coverity

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index bea1298..0aa4769 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -201,6 +201,7 @@ macro_expand(struct vtclog *vl, const char *text)
 			return (NULL);
 		}
 		VSB_printf(vsb, "%s", m);
+		free(m);
 		text = q + 1;
 	}
 	AZ(VSB_finish(vsb));



From phk at varnish-cache.org  Tue Oct  1 14:50:59 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 16:50:59 +0200
Subject: [master] 47be64f Plug an insignificant memleak.
Message-ID: <E1VR1Hv-0004NR-FK@project.varnish-software.com>

commit 47be64f7b434c17d56b0311caba378ed88ab23de
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 14:50:46 2013 +0000

    Plug an insignificant memleak.
    
    Spotted by: Coverity

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 3ac966b..8cb80fb 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -768,6 +768,7 @@ cmd_http_txresp(CMD_ARGS)
 	REPLACE(body, "");
 
 	av = http_tx_parse_args(av, vl, hp, body);
+	free(body);
 	if (*av != NULL)
 		vtc_log(hp->vl, 0, "Unknown http txresp spec: %s\n", *av);
 



From geoff at varnish-cache.org  Tue Oct  1 14:57:55 2013
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 01 Oct 2013 16:57:55 +0200
Subject: [master] 780d1ff varnishtest uses TMPDIR for temp vtc.* dirs if set,
 else /tmp
Message-ID: <E1VR1Od-00052H-4G@project.varnish-software.com>

commit 780d1ff228469922501e4d9850fc3695a4f055d9
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Oct 1 16:56:01 2013 +0200

    varnishtest uses TMPDIR for temp vtc.* dirs if set, else /tmp

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index 04aba0d..d168ec3 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -85,6 +85,7 @@ static int vtc_verbosity = 1;		/* Verbosity Level */
 static int vtc_good;
 static int vtc_fail;
 static int leave_temp;
+static char *tmppath;
 
 /**********************************************************************
  * Parse a -D option argument into a name/val pair, and insert
@@ -149,8 +150,8 @@ usage(void)
 	fprintf(stderr, FMT, "-i", "Find varnishd in build tree");
 	fprintf(stderr, FMT, "-j jobs", "Run this many tests in parallel");
 	fprintf(stderr, FMT, "-k", "Continue on test failure");
-	fprintf(stderr, FMT, "-l", "Leave /tmp/vtc.* if test fails");
-	fprintf(stderr, FMT, "-L", "Always leave /tmp/vtc.*");
+	fprintf(stderr, FMT, "-l", "Leave temporary vtc.* if test fails");
+	fprintf(stderr, FMT, "-L", "Always leave temporary vtc.*");
 	fprintf(stderr, FMT, "-n iterations", "Run tests this many times");
 	fprintf(stderr, FMT, "-q", "Quiet mode: report only failures");
 	fprintf(stderr, FMT, "-t duration", "Time tests out after this long");
@@ -268,7 +269,8 @@ start_test(void)
 	memset(jp->buf, 0, jp->bufsiz);
 
 	srandomdev();
-	bprintf(tmpdir, "/tmp/vtc.%d.%08x", (int)getpid(), (unsigned)random());
+	bprintf(tmpdir, "%s/vtc.%d.%08x", tmppath, (int)getpid(),
+		(unsigned)random());
 	AZ(mkdir(tmpdir, 0711));
 
 	tp = VTAILQ_FIRST(&tst_head);
@@ -332,6 +334,10 @@ main(int argc, char * const *argv)
 	char *p;
 
 	extmacro_def("varnishd", "varnishd"); /* Default to path lookup */
+	if (getenv("TMPDIR") != NULL)
+		tmppath = strdup(getenv("TMPDIR"));
+	else
+		tmppath = strdup("/tmp");
 
 	setbuf(stdout, NULL);
 	setbuf(stderr, NULL);
diff --git a/doc/sphinx/reference/varnishtest.rst b/doc/sphinx/reference/varnishtest.rst
index ff68337..a0df5e2 100644
--- a/doc/sphinx/reference/varnishtest.rst
+++ b/doc/sphinx/reference/varnishtest.rst
@@ -39,9 +39,9 @@ The following options are available:
 
 -k               Continue on test failure
 
--l               Leave /tmp/vtc.* if test fails
+-l               Leave temporary vtc.* if test fails
 
--L               Always leave /tmp/vtc.*
+-L               Always leave temporary vtc.*
 
 -n iterations    Run tests this many times
 
@@ -60,6 +60,8 @@ Macro definitions that can be overridden.
 
 varnishd         Path to varnishd to use [varnishd]
 
+If `TMPDIR` is set in the environment, varnishtest creates temporary
+`vtc.*` directories for each test in `$TMPDIR`, otherwise in `/tmp`.
 
 SCRIPTS
 =======



From phk at varnish-cache.org  Tue Oct  1 15:02:39 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 17:02:39 +0200
Subject: [master] e94de5d Close the fd on failure.
Message-ID: <E1VR1TD-0007oU-3t@project.varnish-software.com>

commit e94de5d976b5d397de7324b41fcff64260839c8f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 15:02:14 2013 +0000

    Close the fd on failure.
    
    Spotted by: Coverity

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index d168ec3..05adb14 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -127,6 +127,7 @@ read_file(const char *fn)
 	s = read(fd, buf, sz - 1);
 	if (s <= 0) {
 		free(buf);
+		(void)close(fd);
 		return (NULL);
 	}
 	AZ(close (fd));



From phk at varnish-cache.org  Tue Oct  1 15:02:39 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 17:02:39 +0200
Subject: [master] 69a521d Fix previous fix.
Message-ID: <E1VR1TD-0007oP-2D@project.varnish-software.com>

commit 69a521d11683a8d59437b3755813127c94551c49
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 15:01:51 2013 +0000

    Fix previous fix.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 8cb80fb..2349702 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -720,9 +720,11 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp,
 	if (body != NULL && !nolen)
 		VSB_printf(hp->vsb, "Content-Length: %d%s", bodylen, nl);
 	VSB_cat(hp->vsb, nl);
-	if (body != NULL)
+	if (body != NULL) {
 		VSB_bcat(hp->vsb, body, bodylen);
-	return av;
+		free(body);
+	}
+	return (av);
 }
 
 /**********************************************************************
@@ -768,7 +770,6 @@ cmd_http_txresp(CMD_ARGS)
 	REPLACE(body, "");
 
 	av = http_tx_parse_args(av, vl, hp, body);
-	free(body);
 	if (*av != NULL)
 		vtc_log(hp->vl, 0, "Unknown http txresp spec: %s\n", *av);
 



From phk at varnish-cache.org  Tue Oct  1 15:34:18 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 17:34:18 +0200
Subject: [master] 92f175d Fix old code which did not follow the times.
Message-ID: <E1VR1xq-0001ok-QG@project.varnish-software.com>

commit 92f175d94ee6acf3b5b21950fd05e52ed3c616f6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 15:33:59 2013 +0000

    Fix old code which did not follow the times.
    
    Spotted by: Coverity

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index c79a1bb..3258033 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -348,9 +348,7 @@ pan_sess(const struct sess *sp)
 	VSB_printf(pan_vsp, "  sp = %p {\n", sp);
 	VSB_printf(pan_vsp, "    fd = %d, vxid = %u,\n",
 	    sp->fd, sp->vxid & VSL_IDENTMASK);
-	VSB_printf(pan_vsp, "    client = %s %s,\n",
-	    sp->addr ? sp->addr : "?.?.?.?",
-	    sp->port ? sp->port : "?");
+	VSB_printf(pan_vsp, "    client = %s %s,\n", sp->addr, sp->port);
 	switch (sp->sess_step) {
 #define SESS_STEP(l, u) case S_STP_##u: stp = "S_STP_" #u; break;
 #include "tbl/steps.h"



From phk at varnish-cache.org  Tue Oct  1 15:48:02 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 17:48:02 +0200
Subject: [master] 81417a4 Swap two variable names, so we don't upset Coverity.
Message-ID: <E1VR2B8-000345-TN@project.varnish-software.com>

commit 81417a43d8186502a916346c81e8c5f7ca0947ff
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 15:47:42 2013 +0000

    Swap two variable names, so we don't upset Coverity.

diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 33a59b1..95e4a7a 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -535,20 +535,20 @@ vcc_func(struct vcc *tl, struct expr **e, const char *cfunc,
 	p = args;
 	if (extra == NULL)
 		extra = "";
-	e2 = vcc_mk_expr(vcc_arg_type(&p), "%s(ctx%s\v+", cfunc, extra);
+	e1 = vcc_mk_expr(vcc_arg_type(&p), "%s(ctx%s\v+", cfunc, extra);
 	while (*p != '\0') {
-		e1 = NULL;
+		e2 = NULL;
 		fmt = vcc_arg_type(&p);
 		if (fmt == VOID && !strcmp(p, "PRIV_VCL")) {
 			r = strchr(name, '.');
 			AN(r);
-			e1 = vcc_mk_expr(VOID, "&vmod_priv_%.*s",
+			e2 = vcc_mk_expr(VOID, "&vmod_priv_%.*s",
 			    (int) (r - name), name);
 			p += strlen(p) + 1;
 		} else if (fmt == VOID && !strcmp(p, "PRIV_CALL")) {
 			bprintf(buf, "vmod_priv_%u", tl->nvmodpriv++);
 			Fh(tl, 0, "struct vmod_priv %s;\n", buf);
-			e1 = vcc_mk_expr(VOID, "&%s", buf);
+			e2 = vcc_mk_expr(VOID, "&%s", buf);
 			p += strlen(p) + 1;
 		} else if (fmt == ENUM) {
 			ExpectErr(tl, ID);
@@ -569,7 +569,7 @@ vcc_func(struct vcc *tl, struct expr **e, const char *cfunc,
 				vcc_ErrWhere(tl, tl->t);
 				return;
 			}
-			e1 = vcc_mk_expr(VOID, "\"%.*s\"", PF(tl->t));
+			e2 = vcc_mk_expr(VOID, "\"%.*s\"", PF(tl->t));
 			while (*p != '\0')
 				p += strlen(p) + 1;
 			p++;
@@ -577,31 +577,31 @@ vcc_func(struct vcc *tl, struct expr **e, const char *cfunc,
 			if (*p != '\0')		/*lint !e448 */
 				SkipToken(tl, ',');
 		} else {
-			vcc_expr0(tl, &e1, fmt);
+			vcc_expr0(tl, &e2, fmt);
 			ERRCHK(tl);
-			if (e1->fmt != fmt) {
+			if (e2->fmt != fmt) {
 				VSB_printf(tl->sb, "Wrong argument type.");
 				VSB_printf(tl->sb, "  Expected %s.",
 					vcc_Type(fmt));
 				VSB_printf(tl->sb, "  Got %s.\n",
-					vcc_Type(e1->fmt));
-				vcc_ErrWhere2(tl, e1->t1, tl->t);
+					vcc_Type(e2->fmt));
+				vcc_ErrWhere2(tl, e2->t1, tl->t);
 				return;
 			}
-			assert(e1->fmt == fmt);
-			if (e1->fmt == STRING_LIST) {
-				e1 = vcc_expr_edit(STRING_LIST,
+			assert(e2->fmt == fmt);
+			if (e2->fmt == STRING_LIST) {
+				e2 = vcc_expr_edit(STRING_LIST,
 				    "\v+\n\v1,\nvrt_magic_string_end\v-",
-				    e1, NULL);
+				    e2, NULL);
 			}
 			if (*p != '\0')
 				SkipToken(tl, ',');
 		}
-		e2 = vcc_expr_edit(e2->fmt, "\v1,\n\v2", e2, e1);
+		e1 = vcc_expr_edit(e1->fmt, "\v1,\n\v2", e1, e2);
 	}
 	SkipToken(tl, ')');
-	e2 = vcc_expr_edit(e2->fmt, "\v1\n)\v-", e2, NULL);
-	*e = e2;
+	e1 = vcc_expr_edit(e1->fmt, "\v1\n)\v-", e1, NULL);
+	*e = e1;
 }
 
 /*--------------------------------------------------------------------



From phk at varnish-cache.org  Tue Oct  1 15:56:07 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 17:56:07 +0200
Subject: [master] 00762b5 Add an assert to show Coverity that we don't expect
 this to fail.
Message-ID: <E1VR2Ix-0003xB-TT@project.varnish-software.com>

commit 00762b593aa28a20a6abf9a2637840c0a741e7b8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 15:55:49 2013 +0000

    Add an assert to show Coverity that we don't expect this to fail.

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 9e9733f..12b6dd3 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -377,6 +377,7 @@ varnish_launch(struct varnish *v)
 	nap = VSS_resolve("127.0.0.1", "0", &ap);
 	AN(nap);
 	v->cli_fd = VSS_listen(ap[0], 1);
+	assert(v->cli_fd > 0);
 	VTCP_myname(v->cli_fd, abuf, sizeof abuf, pbuf, sizeof pbuf);
 
 	AZ(VSB_finish(v->args));



From martin at varnish-cache.org  Tue Oct  1 15:58:33 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 01 Oct 2013 17:58:33 +0200
Subject: [master] ad86d5b Fix potential buffer overflow in build_pt_list_cb
Message-ID: <E1VR2LJ-0004TQ-Ag@project.varnish-software.com>

commit ad86d5ba07921fc6aacf37b5dcb4e3391f8e5099
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 17:56:49 2013 +0200

    Fix potential buffer overflow in build_pt_list_cb
    
    Spotted by: Coverity

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index ecd7620..a355918 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -258,6 +258,7 @@ build_pt_list_cb(void *priv, const struct VSC_point *vpt)
 	struct pt_priv *pt_priv;
 	struct pt *pt;
 	char buf[128];
+	size_t l;
 
 	if (vpt == NULL)
 		return (0);
@@ -290,16 +291,17 @@ build_pt_list_cb(void *priv, const struct VSC_point *vpt)
 	pt->key = strdup(buf);
 	AN(pt->key);
 
+	l = sizeof buf;
 	*buf = '\0';
 	if (strcmp(vpt->section->type, "")) {
-		strcat(buf, vpt->section->type);
-		strcat(buf, ".");
+		strncat(buf, vpt->section->type, sizeof buf - strlen(buf) - 1);
+		strncat(buf, ".", sizeof buf - strlen(buf) - 1);
 	}
 	if (strcmp(vpt->section->ident, "")) {
-		strcat(buf, vpt->section->ident);
-		strcat(buf, ".");
+		strncat(buf, vpt->section->ident, sizeof buf - strlen(buf) - 1);
+		strncat(buf, ".", sizeof buf - strlen(buf) - 1);
 	}
-	strcat(buf, vpt->desc->name);
+	strncat(buf, vpt->desc->name, sizeof buf - strlen(buf) - 1);
 	pt->name = strdup(buf);
 	AN(pt->name);
 
@@ -667,7 +669,6 @@ draw_line_bitmap(WINDOW *w, int y, int x, int X, struct pt *pt)
 			}
 			break;
 		default:
-			x += COLW;
 			break;
 		}
 		col++;



From phk at varnish-cache.org  Tue Oct  1 16:04:23 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Oct 2013 18:04:23 +0200
Subject: [master] dba2321 Add an assert to show Coverity we know what we do.
Message-ID: <E1VR2Qx-0005Ip-8V@project.varnish-software.com>

commit dba232196619238be04bb43b2101d1af6eef4228
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 1 16:03:46 2013 +0000

    Add an assert to show Coverity we know what we do.

diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index 4cc03ce..53d1183 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -349,6 +349,7 @@ vbp_wrk_poll_backend(void *priv)
 
 		vt->req = VSB_data(vt->vsb);
 		vt->req_len = VSB_len(vt->vsb);
+		assert(vt->req_len > 0);
 
 		vbp_start_poke(vt);
 		vbp_poke(vt);



From phk at varnish-cache.org  Wed Oct  2 06:54:03 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 02 Oct 2013 08:54:03 +0200
Subject: [master] 432ebd5 Add a couple of asserts to explain stuff to Coverity
Message-ID: <E1VRGJv-0000OA-Qk@project.varnish-software.com>

commit 432ebd54043b3c0a74a2a1e03e7876c3ed9bb95d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 2 06:53:47 2013 +0000

    Add a couple of asserts to explain stuff to Coverity

diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c
index e173b93..880d3d2 100644
--- a/bin/varnishd/cache/cache_esi_parse.c
+++ b/bin/varnishd/cache/cache_esi_parse.c
@@ -926,6 +926,7 @@ VEP_Parse(const struct busyobj *bo, const char *p, size_t l)
 			 * Match against a table while split over input
 			 * sections.
 			 */
+			AN(vep->match);
 			do {
 				if (*p == '>') {
 					for (vm = vep->match;
diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c
index 0589af9..bfe6b40 100644
--- a/bin/varnishd/storage/storage_persistent_subr.c
+++ b/bin/varnishd/storage/storage_persistent_subr.c
@@ -139,6 +139,7 @@ smp_reset_sign(struct smp_signctx *ctx)
 {
 
 	memset(ctx->ss, 0, sizeof *ctx->ss);
+	assert(strlen(ctx->id) < sizeof *ctx->ss);
 	strcpy(ctx->ss->ident, ctx->id);
 	ctx->ss->unique = ctx->unique;
 	ctx->ss->mapped = (uintptr_t)ctx->ss;



From phk at varnish-cache.org  Wed Oct  2 07:52:34 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 02 Oct 2013 09:52:34 +0200
Subject: [master] c0fc574 VRY_Len() returns only the length of a single entry
 in the VRY spec, for conditional fetch-copying,
 we need the length of the full VRY-spec.
Message-ID: <E1VRHEY-00088P-Io@project.varnish-software.com>

commit c0fc574a530b7a66060e80b22bd91de8e2eb5b32
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 2 07:51:52 2013 +0000

    VRY_Len() returns only the length of a single entry in the VRY spec,
    for conditional fetch-copying, we need the length of the full VRY-spec.
    
    Fixes:	#1350

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7647e9c..a7ac6aa 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1125,11 +1125,10 @@ void VSL_Flush(struct vsl_log *, int overflow);
 /* cache_vary.c */
 int VRY_Create(struct busyobj *bo, struct vsb **psb);
 int VRY_Match(struct req *, const uint8_t *vary);
-void VRY_Validate(const uint8_t *vary);
+unsigned VRY_Validate(const uint8_t *vary);
 void VRY_Prep(struct req *);
 enum vry_finish_flag { KEEP, DISCARD };
 void VRY_Finish(struct req *req, enum vry_finish_flag);
-unsigned VRY_Len(const uint8_t *);
 
 /* cache_vcl.c */
 void VCL_Init(void);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 097a04b..24c6a6a 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -393,7 +393,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		obj->vary = (void *)WS_Copy(obj->http->ws,
 		    VSB_data(vary), varyl);
 		AN(obj->vary);
-		VRY_Validate(obj->vary);
+		(void)VRY_Validate(obj->vary);
 		VSB_delete(vary);
 	}
 
@@ -469,15 +469,18 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	struct object *obj;
 	struct objiter *oi;
 	void *sp;
-	ssize_t sl, al, tl;
+	ssize_t sl, al, tl, vl;
 	struct storage *st;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	l = 0;
-	if (bo->ims_obj->vary != NULL)
-		l += VRY_Len(bo->ims_obj->vary);
+	if (bo->ims_obj->vary != NULL) {
+		vl = VRY_Validate(bo->ims_obj->vary);
+		l += vl;
+	} else
+		vl = 0;
 	l += http_EstimateWS(bo->ims_obj->http, 0, &nhttp);
 
 	bo->stats = &wrk->stats;
@@ -499,9 +502,11 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 
 	/* XXX: ESI */
 
-	if (bo->ims_obj->vary != NULL)
+	if (bo->ims_obj->vary != NULL) {
 		obj->vary = (void *)WS_Copy(obj->http->ws,
-		    bo->ims_obj->vary, VRY_Len(bo->ims_obj->vary));
+		    bo->ims_obj->vary, vl);
+		assert(vl == VRY_Validate(obj->vary));
+	}
 
 	obj->vxid = bo->vsl->wid;
 
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 363e357..bfc8686 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -173,7 +173,7 @@ VRY_Create(struct busyobj *bo, struct vsb **psb)
 /*
  * Find length of a vary entry
  */
-unsigned
+static unsigned
 VRY_Len(const uint8_t *p)
 {
 	unsigned l = vbe16dec(p);
@@ -247,12 +247,12 @@ VRY_Finish(struct req *req, enum vry_finish_flag flg)
 {
 	uint8_t *p = NULL;
 
-	VRY_Validate(req->vary_b);
+	(void)VRY_Validate(req->vary_b);
 	if (flg == KEEP && req->vary_l != NULL) {
 		p = malloc(req->vary_l - req->vary_b);
 		if (p != NULL) {
 			memcpy(p, req->vary_b, req->vary_l - req->vary_b);
-			VRY_Validate(p);
+			(void)VRY_Validate(p);
 		}
 	}
 	WS_Release(req->ws, 0);
@@ -323,7 +323,7 @@ VRY_Match(struct req *req, const uint8_t *vary)
 			vsp[ln + 0] = 0xff;
 			vsp[ln + 1] = 0xff;
 			vsp[ln + 2] = 0;
-			VRY_Validate(vsp);
+			(void)VRY_Validate(vsp);
 			req->vary_l = vsp + ln + 3;
 
 			i = vry_cmp(vary, vsp);
@@ -352,12 +352,19 @@ VRY_Match(struct req *req, const uint8_t *vary)
 	}
 }
 
-void
+/*
+ * Check the validity of a Vary string and return its total length
+ */
+
+unsigned
 VRY_Validate(const uint8_t *vary)
 {
+	unsigned retval = 0;
 
 	while (vary[2] != 0) {
 		assert(strlen((const char*)vary+3) == vary[2]);
+		retval += VRY_Len(vary);
 		vary += VRY_Len(vary);
 	}
+	return (retval + 3);
 }
diff --git a/bin/varnishtest/tests/r01350.vtc b/bin/varnishtest/tests/r01350.vtc
new file mode 100644
index 0000000..ae3cfbe
--- /dev/null
+++ b/bin/varnishtest/tests/r01350.vtc
@@ -0,0 +1,31 @@
+varnishtest "IMS + Vary panic"
+
+server s1 {
+	rxreq
+	txresp -hdr "Vary: Accept-Encoding" -hdr "Last-Modified: Wed, 10 May 2006 07:22:58 GMT" -body "IMS"
+	rxreq
+	txresp -status 304 -body ""
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.ttl = 2s;
+		set beresp.grace = 1m;
+		set beresp.keep = 1m;
+		return(deliver);
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+delay 3
+	txreq
+	rxresp
+	expect resp.status == 200
+delay 1
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run



From phk at varnish-cache.org  Wed Oct  2 09:29:14 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 02 Oct 2013 11:29:14 +0200
Subject: [master] 1b4a6b5 Spot and handle 3-digit status from backend,
 where the first digit is zero.
Message-ID: <E1VRIk6-0005Xe-Mv@project.varnish-software.com>

commit 1b4a6b5ed8bf3389b4d920dc54579c754f475f36
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 2 09:28:37 2013 +0000

    Spot and handle 3-digit status from backend, where the first digit is zero.
    
    Fixes:	#1337

diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index d59c384..d03a437 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -461,7 +461,6 @@ HTTP1_DissectRequest(struct req *req)
 uint16_t
 HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 {
-	int j;
 	uint16_t retval = 0;
 	char *p;
 
@@ -481,16 +480,22 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 	if (retval == 0) {
 		hp->status = 0;
 		p = hp->hd[HTTP_HDR_STATUS].b;
-		for (j = 100; j != 0; j /= 10) {
-			if (!vct_isdigit(*p)) {
-				retval = 503;
-				break;
-			}
-			hp->status += (uint16_t)(j * (*p - '0'));
-			p++;
-		}
-		if (*p != '\0')
+
+		if (p[0] < '1' || p[0] > '9')
+			retval = 503;
+		else
+			hp->status += 100 * (p[0] - '0');
+
+		if (p[1] < '0' || p[1] > '9')
+			retval = 503;
+		else
+			hp->status += 10 * (p[1] - '0');
+
+		if (p[2] < '0' || p[2] > '9')
 			retval = 503;
+		else
+			hp->status += (p[2] - '0');
+		assert(hp->status <= 999);
 	}
 
 	if (retval != 0) {
diff --git a/bin/varnishtest/tests/r01337.vtc b/bin/varnishtest/tests/r01337.vtc
new file mode 100644
index 0000000..d954347
--- /dev/null
+++ b/bin/varnishtest/tests/r01337.vtc
@@ -0,0 +1,22 @@
+varnishtest "Bogus backend status"
+
+server s1 {
+	rxreq
+	expect req.url == /low
+	txresp -status 099
+	accept
+	rxreq
+	expect req.url == /high
+	txresp -status 1000
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+client c1 {
+	txreq -url /low
+	rxresp
+	expect resp.status == 503
+	txreq -url /high
+	rxresp
+	expect resp.status == 503
+} -run



From phk at varnish-cache.org  Wed Oct  2 09:39:27 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 02 Oct 2013 11:39:27 +0200
Subject: [master] df8ee4d Make this test a comprehensive test of the status
 parsing
Message-ID: <E1VRItz-00077y-UD@project.varnish-software.com>

commit df8ee4d8c1ca4fb62adac51d2cfefd6af94d6ee2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 2 09:39:16 2013 +0000

    Make this test a comprehensive test of the status parsing

diff --git a/bin/varnishtest/tests/r01337.vtc b/bin/varnishtest/tests/r01337.vtc
index d954347..1af5573 100644
--- a/bin/varnishtest/tests/r01337.vtc
+++ b/bin/varnishtest/tests/r01337.vtc
@@ -2,21 +2,78 @@ varnishtest "Bogus backend status"
 
 server s1 {
 	rxreq
+	expect req.url == /small
+	txresp -status 99
+
+	accept
+	rxreq
 	expect req.url == /low
 	txresp -status 099
+
 	accept
 	rxreq
 	expect req.url == /high
 	txresp -status 1000
+
+	accept
+	rxreq
+	expect req.url == /X
+	txresp -status X99
+
+	accept
+	rxreq
+	expect req.url == /Y
+	txresp -status 9X9
+
+	accept
+	rxreq
+	expect req.url == /Z
+	txresp -status 99X
+
+	accept
+	rxreq
+	expect req.url == /x
+	txresp -status %99
+
+	accept
+	rxreq
+	expect req.url == /y
+	txresp -status 9%9
+
+	accept
+	rxreq
+	expect req.url == /z
+	txresp -status 99%
 } -start
 
 varnish v1 -vcl+backend {} -start
 
 client c1 {
+	txreq -url /small
+	rxresp
+	expect resp.status == 503
 	txreq -url /low
 	rxresp
 	expect resp.status == 503
 	txreq -url /high
 	rxresp
 	expect resp.status == 503
+	txreq -url /X
+	rxresp
+	expect resp.status == 503
+	txreq -url /Y
+	rxresp
+	expect resp.status == 503
+	txreq -url /Z
+	rxresp
+	expect resp.status == 503
+	txreq -url /x
+	rxresp
+	expect resp.status == 503
+	txreq -url /y
+	rxresp
+	expect resp.status == 503
+	txreq -url /z
+	rxresp
+	expect resp.status == 503
 } -run



From martin at varnish-cache.org  Wed Oct  2 11:13:24 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 02 Oct 2013 13:13:24 +0200
Subject: [master] fdc2866 Break line longer than 80 characters
Message-ID: <E1VRKMu-0002bO-Ur@project.varnish-software.com>

commit fdc2866d53949e5b7a030ff72d8a283420070016
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 2 11:09:01 2013 +0200

    Break line longer than 80 characters

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 4371521..84ba7c6 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -293,7 +293,8 @@ int VSL_Print(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo);
 	 *     -5:	I/O write error - see errno
 	 */
 
-int VSL_PrintTerse(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo);
+int VSL_PrintTerse(const struct VSL_data *vsl, const struct VSL_cursor *c,
+    void *fo);
 	/*
 	 * Print the log record pointed to by cursor to stream.
 	 *



From martin at varnish-cache.org  Wed Oct  2 11:13:25 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 02 Oct 2013 13:13:25 +0200
Subject: [master] 7a28137 Fixup docs based on feedback.
Message-ID: <E1VRKMv-0002bT-0G@project.varnish-software.com>

commit 7a28137b694234e4f4ee8e2107c890fb618103ce
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 2 11:14:33 2013 +0200

    Fixup docs based on feedback.

diff --git a/doc/sphinx/reference/vsl-query.rst b/doc/sphinx/reference/vsl-query.rst
index 73f69c3..5aa5793 100644
--- a/doc/sphinx/reference/vsl-query.rst
+++ b/doc/sphinx/reference/vsl-query.rst
@@ -80,7 +80,8 @@ QUERY LANGUAGE
 ==============
 
 A query expression consists of a record selection criteria, and
-optionally an operator and a value to match against ::
+optionally an operator and a value to match against the selected
+records. ::
 
   <record selection criteria> <operator> <operand>
 
@@ -232,4 +233,4 @@ QUERY EXPRESSION EXAMPLES
 * Transactions that has backend failures or long delivery time on
   their ESI subrequests. (Assumes request grouping mode). ::
 
-    BerespStatus >= 500 or {1+}ReqEnd[5] > 1.
+    BerespStatus >= 500 or {2+}ReqEnd[5] > 1.



From martin at varnish-cache.org  Wed Oct  2 11:13:25 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 02 Oct 2013 13:13:25 +0200
Subject: [master] 8d5752d Fix error handling code and typo when parsing -IX
 flags
Message-ID: <E1VRKMv-0002bb-1g@project.varnish-software.com>

commit 8d5752d0e5cd6284ebc37e913c3d201a48f5965a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 2 13:01:35 2013 +0200

    Fix error handling code and typo when parsing -IX flags
    
    Spotted by: Coverity

diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 7795165..6babd06 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -278,7 +278,7 @@ vsl_IX_arg(struct VSL_data *vsl, int opt, const char *arg)
 			return (vsl_diag(vsl,
 				"-%c: \"%*.*s\" is ambiguous",
 				(char)opt, l, l, b));
-		else if (i == 3)
+		else if (i <= -3)
 			return (vsl_diag(vsl,
 				"-%c: Syntax error in \"%*.*s\"",
 				(char)opt, l, l, b));



From martin at varnish-cache.org  Wed Oct  2 11:13:25 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 02 Oct 2013 13:13:25 +0200
Subject: [master] 3fe20d0 Check return value from VSB_finish
Message-ID: <E1VRKMv-0002bj-35@project.varnish-software.com>

commit 3fe20d03f1d6d1167f0e4196579660200ba7117b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 2 13:04:14 2013 +0200

    Check return value from VSB_finish
    
    Spotted by: Coverity

diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 40780bb..6ec44b4 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -295,7 +295,7 @@ vslq_newquery(struct VSL_data *vsl, enum VSL_grouping_e grouping,
 	vsb = VSB_new_auto();
 	AN(vsb);
 	vex = vex_New(querystring, vsb);
-	VSB_finish(vsb);
+	AZ(VSB_finish(vsb));
 	if (vex == NULL)
 		vsl_diag(vsl, "%s", VSB_data(vsb));
 	else {



From martin at varnish-cache.org  Wed Oct  2 11:13:25 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 02 Oct 2013 13:13:25 +0200
Subject: [master] 503a72e Add break after WRONG() assertion to silence Coverity
Message-ID: <E1VRKMv-0002br-4h@project.varnish-software.com>

commit 503a72ed41fda543a16d07bcce8cf27ef3497ef0
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 2 13:05:31 2013 +0200

    Add break after WRONG() assertion to silence Coverity

diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index 83dbd52..99a7999 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -311,6 +311,7 @@ vxp_expr_cmp(struct vxp *vxp, struct vex **pvex)
 	switch((*pvex)->tok) {
 	case '\0':
 		WRONG("Missing token");
+		break;
 	case T_EQ:		/* == */
 	case '<':		/* < */
 	case '>':		/* > */



From martin at varnish-cache.org  Wed Oct  2 11:13:25 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 02 Oct 2013 13:13:25 +0200
Subject: [master] ced89f6 Rename variable to deconfuse Coverity
Message-ID: <E1VRKMv-0002bz-6K@project.varnish-software.com>

commit ced89f600db3f3e39e76b815ba23b9a938d164a9
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 2 13:09:09 2013 +0200

    Rename variable to deconfuse Coverity

diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index cfb424a..fb8e67a 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -100,14 +100,14 @@ VSL_New(void)
 }
 
 static void
-vsl_IX_free(vslf_list *list)
+vsl_IX_free(vslf_list *filters)
 {
 	struct vslf *vslf;
 
-	while (!VTAILQ_EMPTY(list)) {
-		vslf = VTAILQ_FIRST(list);
+	while (!VTAILQ_EMPTY(filters)) {
+		vslf = VTAILQ_FIRST(filters);
 		CHECK_OBJ_NOTNULL(vslf, VSLF_MAGIC);
-		VTAILQ_REMOVE(list, vslf, list);
+		VTAILQ_REMOVE(filters, vslf, list);
 		if (vslf->tags)
 			vbit_destroy(vslf->tags);
 		AN(vslf->vre);



From martin at varnish-cache.org  Wed Oct  2 12:11:09 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 02 Oct 2013 14:11:09 +0200
Subject: [3.0] 76516d9 Be more thorough the testing streaming+restart
Message-ID: <E1VRLGn-0002gl-LI@project.varnish-software.com>

commit 76516d9584ba7c1baffe4049a82f13f6e841f615
Author: Guillaume Quintard <guillaume.quintard at smartjog.com>
Date:   Thu Sep 19 17:49:07 2013 +0200

    Be more thorough the testing streaming+restart
    
    Reset vfp when restarting in deliver, in case it has be set in
    cnt_fetchbody already.
    
    Fixes: #979

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index d11d600..19eb2ce 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -234,6 +234,7 @@ cnt_prepresp(struct sess *sp)
 		AZ(sp->obj);
 		sp->restarts++;
 		sp->director = NULL;
+		sp->wrk->vfp = NULL;
 		sp->wrk->h_content_length = NULL;
 		http_Setup(sp->wrk->bereq, NULL);
 		http_Setup(sp->wrk->beresp, NULL);
diff --git a/bin/varnishtest/tests/r00979.vtc b/bin/varnishtest/tests/r00979.vtc
index bc72efc..d459115 100644
--- a/bin/varnishtest/tests/r00979.vtc
+++ b/bin/varnishtest/tests/r00979.vtc
@@ -2,7 +2,7 @@ varnishtest "r00979.vtc Test restart when do_stream in vcl_deliver"
 
 server s1 {
 	rxreq
-	txresp -status 200 -body "1"
+	txresp -status 200 -gzipbody "1"
 	expect_close
 
 	accept



From martin at varnish-cache.org  Wed Oct  2 12:11:19 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 02 Oct 2013 14:11:19 +0200
Subject: [master] 03d60dc Be more thorough the testing streaming+restart
Message-ID: <E1VRLGx-0002kk-0m@project.varnish-software.com>

commit 03d60dce9044d6dce650c7bb239e04e6089d2d77
Author: Guillaume Quintard <guillaume.quintard at smartjog.com>
Date:   Thu Sep 19 17:49:07 2013 +0200

    Be more thorough the testing streaming+restart
    
    Reset vfp when restarting in deliver, in case it has be set in
    cnt_fetchbody already.
    
    Fixes: #979
    
    Conflicts:
    	bin/varnishd/cache_center.c

diff --git a/bin/varnishtest/tests.disabled/r00979.vtc b/bin/varnishtest/tests.disabled/r00979.vtc
index 216f7cb..432aea5 100644
--- a/bin/varnishtest/tests.disabled/r00979.vtc
+++ b/bin/varnishtest/tests.disabled/r00979.vtc
@@ -2,7 +2,7 @@ varnishtest "r00979.vtc Test restart when do_stream in vcl_deliver"
 
 server s1 {
 	rxreq
-	txresp -status 200 -body "1"
+	txresp -status 200 -gzipbody "1"
 	expect_close
 
 	accept



From martin at varnish-cache.org  Wed Oct  2 12:24:10 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 02 Oct 2013 14:24:10 +0200
Subject: [3.0] 7c3e40f Handle input from stdin properly in varnishadm
Message-ID: <E1VRLTO-00043V-Cq@project.varnish-software.com>

commit 7c3e40fe2dae02028cf3094f459834e1656978f4
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Jun 12 13:07:36 2013 +0200

    Handle input from stdin properly in varnishadm
    
    readline doesn't really handle when input comes from a file or pipe,
    so only use readline if stdin is a tty.
    
    Thanks to johnnyrun for a patch which was used for inspiration here.
    
    Fixes #1314

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index c09a1c7..0abda8d 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -222,7 +222,7 @@ varnishadm_completion (const char *text, int start, int end)
  * Send a "banner" to varnish, to provoke a welcome message.
  */
 static void
-pass(int sock)
+interactive(int sock)
 {
 	struct pollfd fds[2];
 	char buf[1024];
@@ -231,11 +231,7 @@ pass(int sock)
 	unsigned u, status;
 	_line_sock = sock;
 	rl_already_prompted = 1;
-	if (isatty(0)) {
-		rl_callback_handler_install("varnish> ", send_line);
-	} else {
-		rl_callback_handler_install("", send_line);
-	}
+	rl_callback_handler_install("varnish> ", send_line);
 	rl_attempted_completion_function = varnishadm_completion;
 
 	fds[0].fd = sock;
@@ -305,6 +301,65 @@ pass(int sock)
 	}
 }
 
+/*
+ * No arguments given, simply pass bytes on stdin/stdout and CLI socket
+ */
+static void
+pass(int sock)
+{
+	struct pollfd fds[2];
+	char buf[1024];
+	int i;
+	char *answer = NULL;
+	unsigned u, status;
+	ssize_t n;
+
+	fds[0].fd = sock;
+	fds[0].events = POLLIN;
+	fds[1].fd = 0;
+	fds[1].events = POLLIN;
+	while (1) {
+		i = poll(fds, 2, -1);
+		if (i == -1 && errno == EINTR) {
+			continue;
+		}
+		assert(i > 0);
+		if (fds[0].revents & POLLIN) {
+			u = VCLI_ReadResult(fds[0].fd, &status, &answer,
+			    timeout);
+			if (u) {
+				if (status == CLIS_COMMS)
+					RL_EXIT(0);
+				if (answer)
+					fprintf(stderr, "%s\n", answer);
+				RL_EXIT(1);
+			}
+
+			sprintf(buf, "%u\n", status);
+			u = write(1, buf, strlen(buf));
+			if (answer) {
+				u = write(1, answer, strlen(answer));
+				u = write(1, "\n", 1);
+				free(answer);
+				answer = NULL;
+			}
+		}
+		if (fds[1].revents & POLLIN || fds[1].revents & POLLHUP) {
+			n = read(fds[1].fd, buf, sizeof buf);
+			if (n == 0) {
+				AZ(shutdown(sock, SHUT_WR));
+				fds[1].fd = -1;
+			} else if (n < 0) {
+				RL_EXIT(0);
+			} else {
+				buf[n] = '\0';
+				cli_write(sock, buf);
+			}
+		}
+	}
+}
+
+
 static void
 usage(void)
 {
@@ -404,8 +459,12 @@ main(int argc, char * const *argv)
 
 	if (argc > 0)
 		do_args(sock, argc, argv);
-	else
-		pass(sock);
-
+	else {
+		if (isatty(0)) {
+			interactive(sock);
+		} else {
+			pass(sock);
+		}
+	}
 	exit(0);
 }



From martin at varnish-cache.org  Wed Oct  2 12:24:10 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 02 Oct 2013 14:24:10 +0200
Subject: [3.0] 0531578 Fix buffer overrun in varnishadm pass function
Message-ID: <E1VRLTO-00043a-ER@project.varnish-software.com>

commit 0531578def6184919d04ebe171fce98f54f4333c
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 1 14:32:44 2013 +0200

    Fix buffer overrun in varnishadm pass function
    
    Spotted by: Coverity

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index 0abda8d..005c62f 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -345,7 +345,7 @@ pass(int sock)
 			}
 		}
 		if (fds[1].revents & POLLIN || fds[1].revents & POLLHUP) {
-			n = read(fds[1].fd, buf, sizeof buf);
+			n = read(fds[1].fd, buf, sizeof buf - 1);
 			if (n == 0) {
 				AZ(shutdown(sock, SHUT_WR));
 				fds[1].fd = -1;



From martin at varnish-cache.org  Wed Oct  2 15:05:01 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 02 Oct 2013 17:05:01 +0200
Subject: [master] 4f4b550 Enable -b and -c options to VSL
Message-ID: <E1VRNz3-0001Tf-2X@project.varnish-software.com>

commit 4f4b5503815484773a774fdd50c1503e4397c115
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 2 16:57:38 2013 +0200

    Enable -b and -c options to VSL

diff --git a/include/vapi/vapi_options.h b/include/vapi/vapi_options.h
index c9e6e02..fae7fdd 100644
--- a/include/vapi/vapi_options.h
+++ b/include/vapi/vapi_options.h
@@ -53,6 +53,18 @@
 	    " than overwrite it."					\
 	)
 
+#define VSL_OPT_b							\
+	VOPT("b", "[-b]", "Only display backend records",		\
+	    "Only display transactions and log records coming from"	\
+	    " backend communication."					\
+	)
+
+#define VSL_OPT_c							\
+	VOPT("c", "[-c]", "Only display client records",		\
+	    "Only display transactions and log records coming from"	\
+	    " client communication."					\
+	)
+
 #define VSL_OPT_d \
 	VOPT("d", "[-d]", "Process old log entries on startup",		\
 	    "Start processing log records at the head of the log"	\
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index fb8e67a..93e731a 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -188,6 +188,10 @@ VSL_Match(struct VSL_data *vsl, const struct VSL_cursor *c)
 	tag = VSL_TAG(c->rec.ptr);
 	if (tag <= SLT__Bogus || tag >= SLT__Reserved)
 		return (0);
+	if (vsl->c_opt && !VSL_CLIENT(c->rec.ptr))
+		return (0);
+	if (vsl->b_opt && !VSL_BACKEND(c->rec.ptr))
+		return (0);
 	if (!VTAILQ_EMPTY(&vsl->vslf_select) &&
 	    vsl_match_IX(vsl, &vsl->vslf_select, c))
 		return (1);
@@ -323,8 +327,25 @@ VSL_PrintTransactions(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 	if (pt[0] == NULL)
 		return (0);
 
-	t = pt[0];
-	while (t) {
+	for (t = pt[0]; t != NULL; t = *++pt) {
+		if (vsl->c_opt || vsl->b_opt) {
+			switch (t->type) {
+			case VSL_t_req:
+			case VSL_t_esireq:
+				if (!vsl->c_opt)
+					continue;
+				break;
+			case VSL_t_bereq:
+				if (!vsl->b_opt)
+					continue;
+				break;
+			case VSL_t_raw:
+				break;
+			default:
+				continue;
+			}
+		}
+
 		verbose = 0;
 		if (t->level == 0 || vsl->v_opt)
 			verbose = 1;
@@ -363,7 +384,6 @@ VSL_PrintTransactions(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			if (i != 0)
 				return (i);
 		}
-		t = *++pt;
 	}
 
 	if (delim)
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index 65007fd..53984c3 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -88,6 +88,8 @@ struct VSL_data {
 	vslf_list			vslf_select;
 	vslf_list			vslf_suppress;
 
+	int				b_opt;
+	int				c_opt;
 	int				v_opt;
 };
 
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 6babd06..dad558d 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -320,6 +320,8 @@ VSL_Arg(struct VSL_data *vsl, int opt, const char *arg)
 			vbit_set(vsl->vbm_supress, i);
 
 	switch (opt) {
+	case 'b': vsl->b_opt = 1; return (1);
+	case 'c': vsl->c_opt = 1; return (1);
 	case 'i': case 'x': return (vsl_ix_arg(vsl, opt, arg));
 	case 'I': case 'X': return (vsl_IX_arg(vsl, opt, arg));
 	case 'v': vsl->v_opt = 1; return (1);



From martin at varnish-cache.org  Wed Oct  2 15:05:01 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 02 Oct 2013 17:05:01 +0200
Subject: [master] 1fe4190 Enable -b and -c options for varnishlog
Message-ID: <E1VRNz3-0001Tx-4F@project.varnish-software.com>

commit 1fe4190d5012f2183ef68475b7a184f3f4d8f735
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 2 16:59:16 2013 +0200

    Enable -b and -c options for varnishlog

diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h
index 53519b7..add96b4 100644
--- a/bin/varnishlog/varnishlog_options.h
+++ b/bin/varnishlog/varnishlog_options.h
@@ -31,6 +31,8 @@
 #include "vut_options.h"
 
 VSL_OPT_a
+VSL_OPT_b
+VSL_OPT_c
 VSL_OPT_d
 VUT_OPT_D
 VSL_OPT_g
diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index 3a1f1f2..5014da8 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -29,20 +29,6 @@ The following options are available:
 
 .. include:: ../../../bin/varnishlog/varnishlog_options.rst
 
--b
-
-	Only show backend transactions. If neither -b nor -c is
-	specified, varnishlog acts as if they both were specified.
-
-	XXX: Not yet implemented
-
--c
-
-	Only show client transactions. If neither -b nor -c is
-	specified, varnishlog acts as if they both were present.
-
-	XXX: Not yet implemented
-
 -C
 
 	Ignore case when matching regular expressions.



From martin at varnish-cache.org  Wed Oct  2 15:10:20 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 02 Oct 2013 17:10:20 +0200
Subject: [master] e71ca0c Remove code that was added by mistake and is
 breaking the build
Message-ID: <E1VRO4C-0003zH-Ly@project.varnish-software.com>

commit e71ca0cb7174ec864980a8136a3d1470a2e1393a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 2 17:09:40 2013 +0200

    Remove code that was added by mistake and is breaking the build

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index a355918..5cf30a5 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -258,7 +258,6 @@ build_pt_list_cb(void *priv, const struct VSC_point *vpt)
 	struct pt_priv *pt_priv;
 	struct pt *pt;
 	char buf[128];
-	size_t l;
 
 	if (vpt == NULL)
 		return (0);
@@ -291,7 +290,6 @@ build_pt_list_cb(void *priv, const struct VSC_point *vpt)
 	pt->key = strdup(buf);
 	AN(pt->key);
 
-	l = sizeof buf;
 	*buf = '\0';
 	if (strcmp(vpt->section->type, "")) {
 		strncat(buf, vpt->section->type, sizeof buf - strlen(buf) - 1);



From phk at varnish-cache.org  Thu Oct  3 07:02:05 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 03 Oct 2013 09:02:05 +0200
Subject: [master] 95d70ea Improve the API for the obj-iter code a bit.
Message-ID: <E1VRcvF-0001kq-OI@project.varnish-software.com>

commit 95d70ea6441c03c8e8fbb536b437e5e89344dd0a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 3 07:00:46 2013 +0000

    Improve the API for the obj-iter code a bit.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a7ac6aa..36283bd 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1063,8 +1063,14 @@ void *MPL_Get(struct mempool *mpl, unsigned *size);
 void MPL_Free(struct mempool *mpl, void *item);
 
 /* cache_obj.c */
+enum objiter_status {
+	OIS_DONE,
+	OIS_DATA,
+	OIS_STREAM,
+	OIS_ERROR,
+};
 struct objiter *ObjIterBegin(struct worker *, struct object *);
-int ObjIter(struct objiter *, void **, ssize_t *);
+enum objiter_status ObjIter(struct objiter *, void **, ssize_t *);
 void ObjIterEnd(struct objiter **);
 
 /* cache_panic.c */
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 24c6a6a..10f1626 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -471,6 +471,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	void *sp;
 	ssize_t sl, al, tl, vl;
 	struct storage *st;
+	enum objiter_status ois;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -529,7 +530,8 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	al = 0;
 
 	oi = ObjIterBegin(wrk, bo->ims_obj);
-	while (1 == ObjIter(oi, &sp, &sl)) {
+	do {
+		ois = ObjIter(oi, &sp, &sl);
 		while (sl > 0) {
 			if (st == NULL) {
 				st = VFP_GetStorage(bo, bo->ims_obj->len - al);
@@ -547,7 +549,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 			if (st->len == st->space)
 				st = NULL;
 		}
-	}
+	} while (ois == OIS_DATA || ois == OIS_STREAM);
 	ObjIterEnd(&oi);
 	assert(al == bo->ims_obj->len);
 	assert(obj->len == al);
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 6ed3110..baf33d5 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -158,7 +158,7 @@ v1d_dorange(struct req *req, const char *r)
 static void
 v1d_WriteDirObj(struct req *req)
 {
-	int i;
+	enum objiter_status ois;
 	ssize_t len;
 	struct objiter *oi;
 	void *ptr;
@@ -168,12 +168,14 @@ v1d_WriteDirObj(struct req *req)
 	oi = ObjIterBegin(req->wrk, req->obj);
 	XXXAN(oi);
 
-	do {
-		i = ObjIter(oi, &ptr, &len);
-		if (i != 0)
-			if (VDP_bytes(req, i == 2 ? VDP_FLUSH : VDP_NULL,  ptr, len))
-				break;
-	} while (i);
+	while (1) {
+		ois = ObjIter(oi, &ptr, &len);
+		if (ois == OIS_DATA && !VDP_bytes(req, VDP_NULL,  ptr, len))
+			continue;
+		if (ois == OIS_STREAM && !VDP_bytes(req, VDP_FLUSH,  ptr, len))
+			continue;
+		break;
+	}
 	(void)VDP_bytes(req, VDP_FINISH,  NULL, 0);
 	ObjIterEnd(&oi);
 }
diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c
index 2ca0a01..282b996 100644
--- a/bin/varnishd/cache/cache_obj.c
+++ b/bin/varnishd/cache/cache_obj.c
@@ -58,7 +58,7 @@ ObjIterBegin(struct worker *wrk, struct object *obj)
 	return (oi);
 }
 
-int
+enum objiter_status
 ObjIter(struct objiter *oi, void **p, ssize_t *l)
 {
 	ssize_t ol;
@@ -67,6 +67,8 @@ ObjIter(struct objiter *oi, void **p, ssize_t *l)
 	CHECK_OBJ_NOTNULL(oi, OBJITER_MAGIC);
 	AN(p);
 	AN(l);
+	*p = NULL;
+	*l = 0;
 
 	if (oi->bo == NULL) {
 		if (oi->st == NULL)
@@ -76,15 +78,21 @@ ObjIter(struct objiter *oi, void **p, ssize_t *l)
 		if (oi->st != NULL) {
 			*p = oi->st->ptr;
 			*l = oi->st->len;
-			return (1);
+			return (OIS_DATA);
 		}
-		return (0);
+		return (OIS_DONE);
 	} else {
 		ol = oi->len;
-		nl = VBO_waitlen(oi->bo, ol);
-		VSL(SLT_Debug, 0, "STREAM %zd -> %zd", ol, nl);
-		if (nl == ol)
-			return (0);
+		while (1) {
+			nl = VBO_waitlen(oi->bo, ol);
+			VSL(SLT_Debug, 0, "STREAM %zd -> %zd", ol, nl);
+			if (nl != ol)
+				break;
+			if (oi->bo->state == BOS_FINISHED)
+				return (OIS_DONE);
+			if (oi->bo->state == BOS_FAILED)
+				return (OIS_ERROR);
+		}
 		VTAILQ_FOREACH(oi->st, &oi->obj->store, list) {
 			if (oi->st->len <= ol) {
 				ol -= oi->st->len;
@@ -92,8 +100,8 @@ ObjIter(struct objiter *oi, void **p, ssize_t *l)
 			} else {
 				*p = oi->st->ptr + ol;
 				*l = (nl - ol);
-				oi->len += (nl - ol);
-				return (2);
+				oi->len += *l;
+				return (OIS_STREAM);
 			}
 		}
 		WRONG("ran off end");



From phk at varnish-cache.org  Thu Oct  3 07:08:51 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 03 Oct 2013 09:08:51 +0200
Subject: [master] 96fe701 Style nits
Message-ID: <E1VRd1n-0002jk-8r@project.varnish-software.com>

commit 96fe701c0354ebe37920144683c8067e2b60a8b3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 3 07:08:42 2013 +0000

    Style nits

diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index 9ee037f..3e43709 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -550,7 +550,7 @@ mgt_cli_telnet(const char *T_arg)
 		tn->ev->fd = sock;
 		tn->ev->fd_flags = POLLIN;
 		tn->ev->callback = telnet_accept;
-		tn->ev->priv = tn;;
+		tn->ev->priv = tn;
 		AZ(vev_add(mgt_evb, tn->ev));
 		free(ta[i]);
 		ta[i] = NULL;
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 36c20ab..f54368d 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -476,7 +476,7 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size,
 			/* Render this smp_object mostly harmless */
 			(*so)->ttl = 0.;
 			(*so)->ban = 0.;
-			(*so)->ptr = 0;;
+			(*so)->ptr = 0;
 			sg->objs = *so;
 			*idx = ++sg->p.lobjlist;
 		}
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index 9a7c56a..63204a7 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -312,8 +312,8 @@ vcc_ParseVcl(struct vcc *tl)
  *	ACL definitions
  *	Function definitions
  *	Backend definitions
- * 	VMOD import directives
- * 	VCL version declarations
+ *	VMOD import directives
+ *	VCL version declarations
  *	End of input
  */
 



From phk at varnish-cache.org  Thu Oct  3 07:17:51 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 03 Oct 2013 09:17:51 +0200
Subject: [master] adc0193 Compact the status parsing code,
 it got a little too verbose for Federico's taste :-)
Message-ID: <E1VRdAV-00043Q-Mk@project.varnish-software.com>

commit adc019308015dedc1287fddb85d8251d2710e4ce
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 3 07:17:10 2013 +0000

    Compact the status parsing code, it got a little too verbose for
    Federico's taste :-)

diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index d03a437..6905ac2 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -478,24 +478,15 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 		retval = 503;
 
 	if (retval == 0) {
-		hp->status = 0;
 		p = hp->hd[HTTP_HDR_STATUS].b;
 
-		if (p[0] < '1' || p[0] > '9')
-			retval = 503;
-		else
-			hp->status += 100 * (p[0] - '0');
-
-		if (p[1] < '0' || p[1] > '9')
-			retval = 503;
+		if (p[0] >= '1' && p[0] <= '9' &&
+		    p[1] >= '0' && p[1] <= '9' &&
+		    p[2] >= '0' && p[2] <= '9')
+			hp->status =
+			    100 * (p[0] - '0') + 10 * (p[1] - '0') + p[2] - '0';
 		else
-			hp->status += 10 * (p[1] - '0');
-
-		if (p[2] < '0' || p[2] > '9')
 			retval = 503;
-		else
-			hp->status += (p[2] - '0');
-		assert(hp->status <= 999);
 	}
 
 	if (retval != 0) {



From phk at varnish-cache.org  Thu Oct  3 07:55:20 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 03 Oct 2013 09:55:20 +0200
Subject: [master] 39c0497 Protect the obj->storage list with bo->mtx during
 the fetching.
Message-ID: <E1VRdkm-00007u-8G@project.varnish-software.com>

commit 39c049726ffc11332a8dcd50058b40801e93b2e7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 3 07:54:59 2013 +0000

    Protect the obj->storage list with bo->mtx during the fetching.

diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 8f49b20..3891156 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -228,15 +228,13 @@ ssize_t
 VBO_waitlen(struct busyobj *bo, ssize_t l)
 {
 	Lck_Lock(&bo->mtx);
-	if (bo->state <= BOS_FINISHED)
-		assert(l <= bo->fetch_obj->len);
+	assert(l <= bo->fetch_obj->len || bo->state == BOS_FAILED);
 	while (1) {
-		if (bo->fetch_obj->len > l || bo->state >= BOS_FINISHED) {
-			l = bo->fetch_obj->len;
+		if (bo->fetch_obj->len > l || bo->state >= BOS_FINISHED)
 			break;
-		}
 		(void)Lck_CondWait(&bo->cond, &bo->mtx, NULL);
 	}
+	l = bo->fetch_obj->len;
 	Lck_Unlock(&bo->mtx);
 	return (l);
 }
diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c
index c7d1dcd..b1da1db 100644
--- a/bin/varnishd/cache/cache_fetch_proc.c
+++ b/bin/varnishd/cache/cache_fetch_proc.c
@@ -189,7 +189,9 @@ VFP_GetStorage(struct busyobj *bo, ssize_t sz)
 		return (NULL);
 	}
 	AZ(st->len);
+	Lck_Lock(&bo->mtx);
 	VTAILQ_INSERT_TAIL(&obj->store, st, list);
+	Lck_Unlock(&bo->mtx);
 	return (st);
 }
 
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index 2620912..b2627fd 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -334,7 +334,7 @@ V1F_fetch_body(struct worker *wrk, struct busyobj *bo)
 
 	AN(bo->vfp);
 	AZ(bo->vgz_rx);
-	AZ(VTAILQ_FIRST(&obj->store));
+	assert(VTAILQ_EMPTY(&obj->store));
 
 	/* XXX: pick up estimate from objdr ? */
 	cl = 0;
diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c
index 282b996..eadd358 100644
--- a/bin/varnishd/cache/cache_obj.c
+++ b/bin/varnishd/cache/cache_obj.c
@@ -93,18 +93,20 @@ ObjIter(struct objiter *oi, void **p, ssize_t *l)
 			if (oi->bo->state == BOS_FAILED)
 				return (OIS_ERROR);
 		}
+		Lck_Lock(&oi->bo->mtx);
 		VTAILQ_FOREACH(oi->st, &oi->obj->store, list) {
-			if (oi->st->len <= ol) {
-				ol -= oi->st->len;
-				nl -= oi->st->len;
-			} else {
+			if (oi->st->len > ol) {
 				*p = oi->st->ptr + ol;
 				*l = (nl - ol);
 				oi->len += *l;
-				return (OIS_STREAM);
+				break;
 			}
+			ol -= oi->st->len;
+			nl -= oi->st->len;
 		}
-		WRONG("ran off end");
+		Lck_Unlock(&oi->bo->mtx);
+		assert(*l > 0);
+		return (OIS_STREAM);
 	}
 }
 



From phk at varnish-cache.org  Thu Oct  3 09:51:01 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 03 Oct 2013 11:51:01 +0200
Subject: [master] 99a9c56 Pass the rx timestamp into RFC2616_Ttl() to collect
 all the time-math one place.
Message-ID: <E1VRfYj-0005Pe-8p@project.varnish-software.com>

commit 99a9c56455a8df4364e2757a3eb1e6786bc91423
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 3 09:50:32 2013 +0000

    Pass the rx timestamp into RFC2616_Ttl() to collect all the
    time-math one place.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 36283bd..4b7b90d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1195,7 +1195,7 @@ void *WS_Copy(struct ws *ws, const void *str, int len);
 char *WS_Snapshot(struct ws *ws);
 
 /* rfc2616.c */
-void RFC2616_Ttl(struct busyobj *);
+void RFC2616_Ttl(struct busyobj *, double now);
 enum body_status RFC2616_Body(struct busyobj *, struct dstat *);
 unsigned RFC2616_Req_Gzip(const struct http *);
 int RFC2616_Do_Cond(const struct req *sp);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 10f1626..5aca605 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -206,8 +206,7 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 	 * What does RFC2616 think about TTL ?
 	 */
 	EXP_Clr(&bo->exp);
-	bo->exp.entered = W_TIM_real(wrk);
-	RFC2616_Ttl(bo);
+	RFC2616_Ttl(bo, W_TIM_real(wrk));
 
 	/* private objects have negative TTL */
 	if (bo->fetch_objcore->flags & OC_F_PRIVATE)
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index 5b887a2..32887a2 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -63,7 +63,7 @@
  */
 
 void
-RFC2616_Ttl(struct busyobj *bo)
+RFC2616_Ttl(struct busyobj *bo, double now)
 {
 	unsigned max_age, age;
 	double h_date, h_expires;
@@ -76,7 +76,9 @@ RFC2616_Ttl(struct busyobj *bo)
 
 	hp = bo->beresp;
 
-	assert(expp->entered != 0.0 && !isnan(expp->entered));
+	assert(now != 0.0 && !isnan(now));
+	expp->entered = now;
+
 	/* If all else fails, cache using default ttl */
 	expp->ttl = cache_param->default_ttl;
 
@@ -93,6 +95,7 @@ RFC2616_Ttl(struct busyobj *bo)
 		age = strtoul(p, NULL, 0);
 		expp->age = age;
 	}
+
 	if (http_GetHdr(hp, H_Expires, &p))
 		h_expires = VTIM_parse(p);
 
@@ -144,17 +147,16 @@ RFC2616_Ttl(struct busyobj *bo)
 		}
 
 		if (h_date == 0 ||
-		    fabs(h_date - expp->entered) < cache_param->clock_skew) {
+		    fabs(h_date - now) < cache_param->clock_skew) {
 			/*
 			 * If we have no Date: header or if it is
 			 * sufficiently close to our clock we will
 			 * trust Expires: relative to our own clock.
 			 */
-			if (h_expires < expp->entered)
+			if (h_expires < now)
 				expp->ttl = 0;
 			else
-				expp->ttl = h_expires -
-				    expp->entered;
+				expp->ttl = h_expires - now;
 			break;
 		} else {
 			/*
@@ -170,7 +172,7 @@ RFC2616_Ttl(struct busyobj *bo)
 	/* calculated TTL, Our time, Date, Expires, max-age, age */
 	VSLb(bo->vsl, SLT_TTL,
 	    "RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u",
-	    expp->ttl, -1., -1., expp->entered,
+	    expp->ttl, -1., -1., now,
 	    expp->age, h_date, h_expires, max_age);
 }
 



From phk at varnish-cache.org  Thu Oct  3 10:32:28 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 03 Oct 2013 12:32:28 +0200
Subject: [master] eafe825 Fix a race (detected with Fryer and diagnosed by
 Martin) when we clear req->wrk on a busy-waiting request.
Message-ID: <E1VRgCq-0002m2-0j@project.varnish-software.com>

commit eafe8253d0d0ff04049b6da71b577a15bee9f893
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 3 10:31:47 2013 +0000

    Fix a race (detected with Fryer and diagnosed by Martin) when we
    clear req->wrk on a busy-waiting request.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 90af1fc..61b6621 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -492,7 +492,6 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 		    req, w_list);
 		if (DO_DEBUG(DBG_WAITINGLIST))
 			VSLb(req->vsl, SLT_Debug, "on waiting list <%p>", oh);
-		req->wrk = NULL;
 	} else {
 		if (DO_DEBUG(DBG_WAITINGLIST))
 			VSLb(req->vsl, SLT_Debug, "hit busy obj <%p>", oh);
@@ -506,6 +505,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 	 * calls us again
 	 */
 	req->hash_objhead = oh;
+	req->wrk = NULL;
 	Lck_Unlock(&oh->mtx);
 	return (HSH_BUSY);
 }
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 812759c..9807836 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -882,10 +882,10 @@ CNT_Request(struct worker *wrk, struct req *req)
 		 */
 		VSLb(req->vsl, SLT_End, "%s", "");
 		req->vsl->wid = 0;
+		req->wrk = NULL;
 	}
 
-	req->wrk = NULL;
-
+	AZ(req->wrk);
 	assert(WRW_IsReleased(wrk));
 	return (nxt);
 }



From phk at varnish-cache.org  Thu Oct  3 11:35:26 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 03 Oct 2013 13:35:26 +0200
Subject: [master] 160927b Collaps ->age and ->entered into ->t_origin. This is
 the time this object was created,
 as best we can determine it from the headers etc.
Message-ID: <E1VRhBm-0000z4-RX@project.varnish-software.com>

commit 160927b690dd076702601b7407eb71bd423c62dd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 3 11:34:30 2013 +0000

    Collaps ->age and ->entered into ->t_origin.  This is the time this
    object was created, as best we can determine it from the headers etc.
    
    This is also the point from which ttls should be counted.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 4b7b90d..cd71bc6 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -292,8 +292,7 @@ struct exp {
 	double			ttl;
 	double			grace;
 	double			keep;
-	double			age;
-	double			entered;
+	double			t_origin;
 };
 
 /*--------------------------------------------------------------------*/
@@ -610,6 +609,8 @@ struct object {
 
 	/* VCL only variables */
 	long			hits;
+	double			last_use;
+
 	double			last_modified;
 
 	struct http		*http;
@@ -618,7 +619,6 @@ struct object {
 
 	struct storage		*esidata;
 
-	double			last_use;
 };
 
 /*--------------------------------------------------------------------*/
@@ -905,7 +905,7 @@ void EXP_Set_keep(struct exp *e, double v);
 
 double EXP_Ttl(const struct req *, const struct object*);
 double EXP_Grace(const struct req *, const struct object*);
-void EXP_Insert(const struct object *o);
+void EXP_Insert(const struct object *o, double now);
 void EXP_Inject(struct objcore *oc, struct lru *lru, double when);
 void EXP_Init(void);
 void EXP_Rearm(const struct object *o);
@@ -1195,7 +1195,7 @@ void *WS_Copy(struct ws *ws, const void *str, int len);
 char *WS_Snapshot(struct ws *ws);
 
 /* rfc2616.c */
-void RFC2616_Ttl(struct busyobj *, double now);
+void RFC2616_Ttl(struct busyobj *);
 enum body_status RFC2616_Body(struct busyobj *, struct dstat *);
 unsigned RFC2616_Req_Gzip(const struct http *);
 int RFC2616_Do_Cond(const struct req *sp);
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index d764856..e59e26a 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -77,8 +77,7 @@ EXP_Clr(struct exp *e)
 	e->ttl = -1;
 	e->grace = -1;
 	e->keep = -1;
-	e->age = 0;
-	e->entered = 0;
+	e->t_origin = 0;
 }
 
 #define EXP_ACCESS(fld, low_val, extra)				\
@@ -142,7 +141,7 @@ EXP_Ttl(const struct req *req, const struct object *o)
 	r = o->exp.ttl;
 	if (req != NULL && req->exp.ttl > 0. && req->exp.ttl < r)
 		r = req->exp.ttl;
-	return (o->exp.entered + r);
+	return (o->exp.t_origin + r);
 }
 
 /*--------------------------------------------------------------------
@@ -216,7 +215,7 @@ EXP_Inject(struct objcore *oc, struct lru *lru, double when)
  */
 
 void
-EXP_Insert(const struct object *o)
+EXP_Insert(const struct object *o, double now)
 {
 	struct objcore *oc;
 	struct lru *lru;
@@ -226,8 +225,8 @@ EXP_Insert(const struct object *o)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	HSH_Ref(oc);
 
-	assert(o->exp.entered != 0 && !isnan(o->exp.entered));
-	oc->last_lru = o->exp.entered;
+	assert(o->exp.t_origin != 0 && !isnan(o->exp.t_origin));
+	oc->last_lru = now;
 
 	lru = oc_getlru(oc);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 5aca605..d65d181 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -206,7 +206,7 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 	 * What does RFC2616 think about TTL ?
 	 */
 	EXP_Clr(&bo->exp);
-	RFC2616_Ttl(bo, W_TIM_real(wrk));
+	RFC2616_Ttl(bo);
 
 	/* private objects have negative TTL */
 	if (bo->fetch_objcore->flags & OC_F_PRIVATE)
@@ -411,7 +411,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (http_GetHdr(hp, H_Last_Modified, &b))
 		obj->last_modified = VTIM_parse(b);
 	else
-		obj->last_modified = floor(bo->exp.entered);
+		obj->last_modified = floor(bo->exp.t_origin);
 
 	assert(WRW_IsReleased(wrk));
 
@@ -422,7 +422,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	assert(bo->refcount >= 1);
 
 	if (!(bo->fetch_obj->objcore->flags & OC_F_PRIVATE)) {
-		EXP_Insert(obj);
+		EXP_Insert(obj, bo->t_fetch);
 		AN(obj->objcore->ban);
 	}
 
@@ -517,7 +517,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 
 
 	if (!(bo->fetch_obj->objcore->flags & OC_F_PRIVATE)) {
-		EXP_Insert(obj);
+		EXP_Insert(obj, bo->t_fetch);
 		AN(obj->objcore->ban);
 	}
 
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 61b6621..9851584 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -343,7 +343,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 	struct objcore *oc;
 	struct objcore *exp_oc;
 	struct object *o, *exp_o;
-	double exp_entered;
+	double exp_t_origin;
 	int busy_found;
 	enum lookup_e retval;
 
@@ -393,7 +393,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 	busy_found = 0;
 	exp_o = NULL;
 	exp_oc = NULL;
-	exp_entered = 0.0;
+	exp_t_origin = 0.0;
 	o = NULL;
 	VTAILQ_FOREACH(oc, &oh->objcs, list) {
 		/* Must be at least our own ref + the objcore we examine */
@@ -438,12 +438,12 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 			*ocp = oc;
 			return (HSH_HIT);
 		}
-		if (o->exp.entered > exp_entered &&
+		if (o->exp.t_origin > exp_t_origin &&
 		    !(oc->flags & OC_F_PASS)) {
 			/* record the newest object */
 			exp_oc = oc;
 			exp_o = o;
-			exp_entered = o->exp.entered;
+			exp_t_origin = o->exp.t_origin;
 		}
 	}
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 9807836..647ef49 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -128,7 +128,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
 
 	http_PrintfHeader(req->resp, "Age: %.0f",
-	    req->obj->exp.age + req->t_resp - req->obj->exp.entered);
+	    req->t_resp - req->obj->exp.t_origin);
 
 	http_SetHeader(req->resp, "Via: 1.1 varnish");
 
@@ -217,7 +217,7 @@ cnt_error(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 	AZ(req->objcore);
 	req->obj->vxid = bo->vsl->wid;
-	req->obj->exp.entered = req->t_req;
+	req->obj->exp.t_origin = req->t_req;
 
 	h = req->obj->http;
 
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index 32887a2..69735f3 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -63,7 +63,7 @@
  */
 
 void
-RFC2616_Ttl(struct busyobj *bo, double now)
+RFC2616_Ttl(struct busyobj *bo)
 {
 	unsigned max_age, age;
 	double h_date, h_expires;
@@ -76,8 +76,8 @@ RFC2616_Ttl(struct busyobj *bo, double now)
 
 	hp = bo->beresp;
 
-	assert(now != 0.0 && !isnan(now));
-	expp->entered = now;
+	assert(bo->t_fetch != 0.0 && !isnan(bo->t_fetch));
+	expp->t_origin = bo->t_fetch;
 
 	/* If all else fails, cache using default ttl */
 	expp->ttl = cache_param->default_ttl;
@@ -92,8 +92,13 @@ RFC2616_Ttl(struct busyobj *bo, double now)
 	 */
 
 	if (http_GetHdr(hp, H_Age, &p)) {
-		age = strtoul(p, NULL, 0);
-		expp->age = age;
+		/*
+		 * We deliberately run with partial results, rather than
+		 * reject the Age: header outright.  This will be future
+		 * compatible with fractional seconds.
+		 */
+		age = strtoul(p, NULL, 10);
+		expp->t_origin -= age;
 	}
 
 	if (http_GetHdr(hp, H_Expires, &p))
@@ -147,16 +152,16 @@ RFC2616_Ttl(struct busyobj *bo, double now)
 		}
 
 		if (h_date == 0 ||
-		    fabs(h_date - now) < cache_param->clock_skew) {
+		    fabs(h_date - bo->t_fetch) < cache_param->clock_skew) {
 			/*
 			 * If we have no Date: header or if it is
 			 * sufficiently close to our clock we will
 			 * trust Expires: relative to our own clock.
 			 */
-			if (h_expires < now)
+			if (h_expires < bo->t_fetch)
 				expp->ttl = 0;
 			else
-				expp->ttl = h_expires - now;
+				expp->ttl = h_expires - bo->t_fetch;
 			break;
 		} else {
 			/*
@@ -172,8 +177,8 @@ RFC2616_Ttl(struct busyobj *bo, double now)
 	/* calculated TTL, Our time, Date, Expires, max-age, age */
 	VSLb(bo->vsl, SLT_TTL,
 	    "RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u",
-	    expp->ttl, -1., -1., now,
-	    expp->age, h_date, h_expires, max_age);
+	    expp->ttl, -1., -1., bo->t_fetch,
+	    expp->t_origin, h_date, h_expires, max_age);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 3d691a5..9afb9a3 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -467,9 +467,9 @@ vrt_wsp_exp(struct vsl_log *vsl, double now, const struct exp *e)
 {
 	double dt;
 
-	dt = now - e->entered;
+	dt = now - e->t_origin;
 	VSLb(vsl, SLT_TTL, "VCL %.0f %.0f %.0f %.0f %.0f",
-	    e->ttl - dt, e->grace, e->keep, now, e->age + dt);
+	    e->ttl - dt, e->grace, e->keep, now, dt);
 }
 
 VRT_DO_EXP(req, ctx->req->exp, ttl, 0, )
@@ -480,7 +480,7 @@ VRT_DO_EXP(obj, ctx->req->obj->exp, grace, 0,
    EXP_Rearm(ctx->req->obj);
    vrt_wsp_exp(ctx->vsl, ctx->req->t_req, &ctx->req->obj->exp);)
 VRT_DO_EXP(obj, ctx->req->obj->exp, ttl,
-   (ctx->req->t_req - ctx->req->obj->exp.entered),
+   (ctx->req->t_req - ctx->req->obj->exp.t_origin),
    EXP_Rearm(ctx->req->obj);
    vrt_wsp_exp(ctx->vsl, ctx->req->t_req, &ctx->req->obj->exp);)
 VRT_DO_EXP(obj, ctx->req->obj->exp, keep, 0,
@@ -488,11 +488,11 @@ VRT_DO_EXP(obj, ctx->req->obj->exp, keep, 0,
    vrt_wsp_exp(ctx->vsl, ctx->req->t_req, &ctx->req->obj->exp);)
 
 VRT_DO_EXP(beresp, ctx->bo->exp, grace, 0,
-   vrt_wsp_exp(ctx->vsl, ctx->bo->exp.entered, &ctx->bo->exp);)
+   vrt_wsp_exp(ctx->vsl, ctx->bo->exp.t_origin, &ctx->bo->exp);)
 VRT_DO_EXP(beresp, ctx->bo->exp, ttl, 0,
-   vrt_wsp_exp(ctx->vsl, ctx->bo->exp.entered, &ctx->bo->exp);)
+   vrt_wsp_exp(ctx->vsl, ctx->bo->exp.t_origin, &ctx->bo->exp);)
 VRT_DO_EXP(beresp, ctx->bo->exp, keep, 0,
-   vrt_wsp_exp(ctx->vsl, ctx->bo->exp.entered, &ctx->bo->exp);)
+   vrt_wsp_exp(ctx->vsl, ctx->bo->exp.t_origin, &ctx->bo->exp);)
 
 /*--------------------------------------------------------------------
  * req.xid
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index a8203c4..aa5d7eb 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -284,7 +284,7 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo,
 	HTTP_Setup(o->http, bo->ws_o, bo->vsl, HTTP_Obj);
 	o->http->magic = HTTP_MAGIC;
 	o->exp = bo->exp;
-	o->last_use = bo->exp.entered;
+	o->last_use = bo->t_fetch;
 	VTAILQ_INIT(&o->store);
 	bo->stats->n_object++;
 
@@ -376,9 +376,9 @@ STV_NewObject(struct busyobj *bo, const char *hint,
 		}
 	}
 
-	if (o == NULL) {
+	if (o == NULL)
 		return (NULL);
-	}
+
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(o->objstore, STORAGE_MAGIC);
 	return (o);



From martin at varnish-cache.org  Thu Oct  3 12:00:41 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 03 Oct 2013 14:00:41 +0200
Subject: [master] be7aa6d Implement the vxid_parent field of reported VSL
 transactions
Message-ID: <E1VRhaD-0004Jd-Is@project.varnish-software.com>

commit be7aa6df26ad6e081835f46f7eec326b03bcef9d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 2 22:03:49 2013 +0200

    Implement the vxid_parent field of reported VSL transactions

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 84ba7c6..cee9307 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -69,7 +69,7 @@ enum VSL_transaction_e {
 struct VSL_transaction {
 	unsigned		level;
 	int32_t			vxid;
-	/* int32_t		vxid_parent; /\* XXX: Implement this *\/ */
+	int32_t			vxid_parent;
 	enum VSL_transaction_e	type;
 	struct VSL_cursor	*c;
 };
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 055eefe..b6d77ff 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -750,6 +750,7 @@ vslq_callback(const struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
 	vtxs[0] = vtx;
 	trans[0].level = 1;
 	trans[0].vxid = vtx->key.vxid;
+	trans[0].vxid_parent = 0;
 	trans[0].type = vtx->type;
 	trans[0].c = &vtx->c.cursor;
 	i = 1;
@@ -761,6 +762,7 @@ vslq_callback(const struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
 			vtxs[i] = vtx;
 			trans[i].level = trans[j].level + 1;
 			trans[i].vxid = vtx->key.vxid;
+			trans[i].vxid_parent = trans[j].vxid;
 			trans[i].type = vtx->type;
 			trans[i].c = &vtx->c.cursor;
 			i++;



From martin at varnish-cache.org  Thu Oct  3 12:00:41 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 03 Oct 2013 14:00:41 +0200
Subject: [master] 6d6979f Set the client/backend bit when synthesizing log
 records
Message-ID: <E1VRhaD-0004JU-Ev@project.varnish-software.com>

commit 6d6979f0e224a0338392d8fa8bdc1b7cb97629be
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 2 21:48:54 2013 +0200

    Set the client/backend bit when synthesizing log records

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 1544604..055eefe 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -803,6 +803,17 @@ vtx_diag(struct vtx *vtx, const char *fmt, ...)
 		l = buflen - 1;
 	buf[l++] = '\0';	/* NUL-terminated */
 	diag->chunk[1] = vtx->key.vxid;
+	switch (vtx->type) {
+	case VSL_t_req:
+	case VSL_t_esireq:
+		diag->chunk[1] |= VSL_CLIENTMARKER;
+		break;
+	case VSL_t_bereq:
+		diag->chunk[1] |= VSL_BACKENDMARKER;
+		break;
+	default:
+		break;
+	}
 	diag->chunk[0] = ((((unsigned)SLT_VSL & 0xff) << 24) | l);
 	VTAILQ_INSERT_TAIL(&vtx->diag, diag, list);
 



From martin at varnish-cache.org  Thu Oct  3 12:00:41 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 03 Oct 2013 14:00:41 +0200
Subject: [master] 5e251ab Fix Ctrl-C handling in VUT_main
Message-ID: <E1VRhaD-0004Jl-Kh@project.varnish-software.com>

commit 5e251abf50e2c02b84231e81a94b1c8cf6faecd5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 2 22:47:04 2013 +0200

    Fix Ctrl-C handling in VUT_main
    
    Left over code in VUT_main was preventing Ctrl-C handling when looping
    trying to reconnect. Remove offending code.

diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index 8c2f90c..0d331b8 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -259,6 +259,8 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 	struct VSL_cursor *c;
 	int i;
 
+	AN(VUT.vslq);
+
 	if (func == NULL) {
 		if (VUT.w_arg)
 			func = VSL_WriteTransactions;
@@ -281,6 +283,7 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 		}
 
 		if (VUT.vslq == NULL) {
+			/* Reconnect VSM */
 			AZ(VUT.r_arg);
 			AN(VUT.vsm);
 			VTIM_sleep(0.1);
@@ -291,6 +294,7 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 			c = VSL_CursorVSM(VUT.vsl, VUT.vsm, 1);
 			if (c == NULL) {
 				VSL_ResetError(VUT.vsl);
+				VSM_Close(VUT.vsm);
 				continue;
 			}
 			VUT.vslq = VSLQ_New(VUT.vsl, &c, VUT.g_arg, VUT.q_arg);
@@ -328,25 +332,6 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 			/* Overrun */
 			VUT_Error(0, "Log overrun");
 		}
-
-		/* Reconnect VSM */
-		while (VUT.vslq == NULL) {
-			AZ(VUT.r_arg);
-			AN(VUT.vsm);
-			VTIM_sleep(0.1);
-			if (VSM_Open(VUT.vsm)) {
-				VSM_ResetError(VUT.vsm);
-				continue;
-			}
-			c = VSL_CursorVSM(VUT.vsl, VUT.vsm, 1);
-			if (c == NULL) {
-				VSL_ResetError(VUT.vsl);
-				continue;
-			}
-			VUT.vslq = VSLQ_New(VUT.vsl, &c, VUT.g_arg, VUT.q_arg);
-			AN(VUT.vslq);
-			AZ(c);
-		}
 	}
 
 	if (VUT.vslq != NULL)



From martin at varnish-cache.org  Thu Oct  3 12:00:41 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 03 Oct 2013 14:00:41 +0200
Subject: [master] 4830d6b Make transaction limit and timeout into cmd line
 options
Message-ID: <E1VRhaD-0004Jx-QL@project.varnish-software.com>

commit 4830d6b9cbfc4b48c43702fa3f80e6372b69d96c
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 3 10:36:29 2013 +0200

    Make transaction limit and timeout into cmd line options

diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h
index add96b4..1c1df7c 100644
--- a/bin/varnishlog/varnishlog_options.h
+++ b/bin/varnishlog/varnishlog_options.h
@@ -38,11 +38,13 @@ VUT_OPT_D
 VSL_OPT_g
 VSL_OPT_i
 VSL_OPT_I
+VSL_OPT_L
 VSM_OPT_n
 VSM_OPT_N
 VUT_OPT_P
 VUT_OPT_q
 VSL_OPT_r
+VSL_OPT_T
 VSL_OPT_u
 VSL_OPT_v
 VSL_OPT_w
diff --git a/include/vapi/vapi_options.h b/include/vapi/vapi_options.h
index fae7fdd..344b3ed 100644
--- a/include/vapi/vapi_options.h
+++ b/include/vapi/vapi_options.h
@@ -95,11 +95,29 @@
 	    VSL_iI_PS							\
 	)
 
+#define VSL_OPT_L							\
+	VOPT("L:", "[-L limit]", "Incomplete transaction limit",	\
+	    "Sets the upper limit of incomplete transactions kept"	\
+	    " before the oldest transaction is force completed. A"	\
+	    " warning record is synthesized when this happens. This"	\
+	    " setting keeps an upper bound on the memory usage of"	\
+	    " running queries. Defaults to 1000 transactions."		\
+	)
+
 #define VSL_OPT_r							\
 	VOPT("r:", "[-r filename]", "Binary file input",		\
 	    "Read log in binary file format from this file."		\
 	)
 
+#define VSL_OPT_T							\
+	VOPT("T:", "[-T seconds]", "Transaction end timeout",		\
+	    "Sets the transaction timeout in seconds. This defines the"	\
+	    " maximum number of seconds elapsed between a Begin tag"	\
+	    " and the End tag. If the timeout expires, a warning"	\
+	    " record is synthesized and the transaction is force"	\
+	    " completed. Defaults to 120 seconds."			\
+	)
+
 #define VSL_OPT_u							\
 	VOPT("u", "[-u]", "Binary file output unbuffered",		\
 	    "Unbuffered binary file output mode."			\
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 93e731a..7c5dafa 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -91,6 +91,8 @@ VSL_New(void)
 	if (vsl == NULL)
 		return (NULL);
 
+	vsl->L_opt = 1000;
+	vsl->T_opt = 120.;
 	vsl->vbm_select = vbit_init(SLT__MAX);
 	vsl->vbm_supress = vbit_init(SLT__MAX);
 	VTAILQ_INIT(&vsl->vslf_select);
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index 53984c3..40e3438 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -90,6 +90,8 @@ struct VSL_data {
 
 	int				b_opt;
 	int				c_opt;
+	int				L_opt;
+	double				T_opt;
 	int				v_opt;
 };
 
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index dad558d..9370ed9 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -41,6 +41,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include <limits.h>
 
 #include "miniobj.h"
 #include "vas.h"
@@ -312,6 +313,9 @@ int
 VSL_Arg(struct VSL_data *vsl, int opt, const char *arg)
 {
 	int i;
+	char *p;
+	double d;
+	long l;
 
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 	/* If first option is 'i', set all bits for supression */
@@ -324,6 +328,26 @@ VSL_Arg(struct VSL_data *vsl, int opt, const char *arg)
 	case 'c': vsl->c_opt = 1; return (1);
 	case 'i': case 'x': return (vsl_ix_arg(vsl, opt, arg));
 	case 'I': case 'X': return (vsl_IX_arg(vsl, opt, arg));
+	case 'L':
+		l = strtol(arg, &p, 0);
+		while (isspace(*p))
+			p++;
+		if (*p != '\0')
+			return (vsl_diag(vsl, "-L: Syntax error"));
+		if (l < 0 || l > INT_MAX)
+			return (vsl_diag(vsl, "-L: Range error"));
+		vsl->L_opt = (int)l;
+		return (1);
+	case 'T':
+		d = strtod(arg, &p);
+		while (isspace(*p))
+			p++;
+		if (*p != '\0')
+			return (vsl_diag(vsl, "-P: Syntax error"));
+		if (d < 0.)
+			return (vsl_diag(vsl, "-L: Range error"));
+		vsl->T_opt = d;
+		return (1);
 	case 'v': vsl->v_opt = 1; return (1);
 	default:
 		return (0);
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index b6d77ff..db92c7f 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -1009,9 +1009,7 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 
 	now = VTIM_mono();
 	while ((vtx = VTAILQ_FIRST(&vslq->incomplete)) &&
-	    now - vtx->t_start > 120.) {
-		/* XXX: Make timeout configurable through options and
-		   provide a sane default */
+	    now - vtx->t_start > vslq->vsl->T_opt) {
 		AZ(vtx->flags & VTX_F_COMPLETE);
 		vtx = vtx_force(vslq, vtx, "incomplete - timeout");
 		if (vtx) {
@@ -1026,9 +1024,7 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 	if (i)
 		return (i);
 
-	while (vslq->n_incomplete > 1000) {
-		/* XXX: Make limit configurable through options and
-		   provide a sane default */
+	while (vslq->n_incomplete > vslq->vsl->L_opt) {
 		vtx = VTAILQ_FIRST(&vslq->incomplete);
 		AN(vtx);
 		AZ(vtx->flags & VTX_F_COMPLETE);



From martin at varnish-cache.org  Thu Oct  3 12:00:41 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 03 Oct 2013 14:00:41 +0200
Subject: [master] 85a2eec Flush file output on exit
Message-ID: <E1VRhaD-0004K8-U6@project.varnish-software.com>

commit 85a2eecf956ed857d4c17b75d6d211fa2f056e66
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 3 13:58:53 2013 +0200

    Flush file output on exit
    
    Make sure to flush the file stream on exit in VUT

diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index 0d331b8..03127ec 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -240,6 +240,9 @@ VUT_Fini(void)
 	free(VUT.r_arg);
 	free(VUT.P_arg);
 
+	if (VUT.fo != NULL)
+		fflush(VUT.fo);
+
 	vut_vpf_remove();
 	AZ(VUT.pfh);
 



From phk at varnish-cache.org  Thu Oct  3 14:02:19 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 03 Oct 2013 16:02:19 +0200
Subject: [master] 1aa28e5 Adjust test-cases for slightly slimmer exp structure
Message-ID: <E1VRjTv-0002Ub-Nn@project.varnish-software.com>

commit 1aa28e5ff9776e5bd7c6fbf322cb886c3d9e74ff
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 3 14:01:48 2013 +0000

    Adjust test-cases for slightly slimmer exp structure

diff --git a/bin/varnishtest/tests/c00045.vtc b/bin/varnishtest/tests/c00045.vtc
index 73d87a6..0e3f4fd 100644
--- a/bin/varnishtest/tests/c00045.vtc
+++ b/bin/varnishtest/tests/c00045.vtc
@@ -2,11 +2,11 @@ varnishtest	"Object/LRU/Stevedores with hinting"
 
 server s1 {
 	rxreq
-	txresp -bodylen 1048172
+	txresp -bodylen 1048180
 	rxreq
-	txresp -bodylen 1048173
+	txresp -bodylen 1048181
 	rxreq
-	txresp -bodylen 1048174
+	txresp -bodylen 1048182
 } -start
 
 varnish v1 -storage "-smalloc,1m -smalloc,1m, -smalloc,1m" -vcl+backend {
@@ -21,7 +21,7 @@ client c1 {
 	txreq -url /foo
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048172
+	expect resp.bodylen == 1048180
 } -run
 
 varnish v1 -expect SMA.Transient.g_bytes == 0
@@ -36,7 +36,7 @@ client c1 {
 	txreq -url /bar
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048173
+	expect resp.bodylen == 1048181
 } -run
 
 varnish v1 -expect n_lru_nuked == 1
@@ -52,7 +52,7 @@ client c1 {
 	txreq -url /foo
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048174
+	expect resp.bodylen == 1048182
 } -run
 
 varnish v1 -expect n_lru_nuked == 2
diff --git a/bin/varnishtest/tests/r01140.vtc b/bin/varnishtest/tests/r01140.vtc
index f18de6c..1ab3bee 100644
--- a/bin/varnishtest/tests/r01140.vtc
+++ b/bin/varnishtest/tests/r01140.vtc
@@ -4,7 +4,7 @@ server s1 {
 	# This response should almost completely fill the storage
 	rxreq
 	expect req.url == /url1
-	txresp -bodylen 1048100
+	txresp -bodylen 1048108
 
 	# The next one should not fit in the storage, ending up in transient
 	# with zero ttl (=shortlived)
@@ -29,7 +29,7 @@ client c1 {
 	txreq -url /url1
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048100
+	expect resp.bodylen == 1048108
 } -run
 
 delay .1
diff --git a/bin/varnishtest/tests/r01283.vtc b/bin/varnishtest/tests/r01283.vtc
index c81da5a..f7fc759 100644
--- a/bin/varnishtest/tests/r01283.vtc
+++ b/bin/varnishtest/tests/r01283.vtc
@@ -2,7 +2,7 @@ varnishtest "#1283 - Test failure to allocate object for error (transient full)"
 
 server s1 {
 	rxreq
-	txresp -bodylen 1048140
+	txresp -bodylen 1048148
 } -start
 
 varnish v1 -arg "-p nuke_limit=0" -storage "-sTransient=malloc,1m" -vcl+backend {
diff --git a/bin/varnishtest/tests/r01284.vtc b/bin/varnishtest/tests/r01284.vtc
index 74f1306..4348f01 100644
--- a/bin/varnishtest/tests/r01284.vtc
+++ b/bin/varnishtest/tests/r01284.vtc
@@ -6,7 +6,7 @@ server s1 {
 	txresp -bodylen 1048140
 	rxreq
 	expect req.url == "/obj2"
-	txresp -hdr "Long: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -hdr "Long2: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	txresp -hdr "Long: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -hdr "Long2: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 } -start
 
 varnish v1 -arg "-p nuke_limit=0" -storage "-sTransient=malloc,1m" -vcl+backend {



From phk at varnish-cache.org  Fri Oct  4 06:27:17 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 04 Oct 2013 08:27:17 +0200
Subject: [master] e0b02b8 Kill req.grace (grace is now a VCL decision,
 so it's unneeded). Kill req.keep (it never made any sense in the
 first place).
Message-ID: <E1VRyr7-0005RQ-5B@project.varnish-software.com>

commit e0b02b84a9cacd042203cac14fa903afa7ec0b9b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 4 06:26:41 2013 +0000

    Kill req.grace (grace is now a VCL decision, so it's unneeded).
    Kill req.keep (it never made any sense in the first place).

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index e59e26a..dd805fd 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -115,8 +115,6 @@ EXP_Keep(const struct req *req, const struct object *o)
 	r = (double)cache_param->default_keep;
 	if (o->exp.keep > 0.)
 		r = o->exp.keep;
-	if (req != NULL && req->exp.keep > 0. && req->exp.keep < r)
-		r = req->exp.keep;
 	return (EXP_Ttl(req, o) + r);
 }
 
@@ -128,8 +126,6 @@ EXP_Grace(const struct req *req, const struct object *o)
 	r = (double)cache_param->default_grace;
 	if (o->exp.grace >= 0.)
 		r = o->exp.grace;
-	if (req != NULL && req->exp.grace > 0. && req->exp.grace < r)
-		r = req->exp.grace;
 	return (EXP_Ttl(req, o) + r);
 }
 
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 9afb9a3..a605ad1 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -473,8 +473,6 @@ vrt_wsp_exp(struct vsl_log *vsl, double now, const struct exp *e)
 }
 
 VRT_DO_EXP(req, ctx->req->exp, ttl, 0, )
-VRT_DO_EXP(req, ctx->req->exp, grace, 0, )
-VRT_DO_EXP(req, ctx->req->exp, keep, 0, )
 
 VRT_DO_EXP(obj, ctx->req->obj->exp, grace, 0,
    EXP_Rearm(ctx->req->obj);
diff --git a/bin/varnishtest/tests/b00030.vtc b/bin/varnishtest/tests/b00030.vtc
index 9703cc7..8b945e1 100644
--- a/bin/varnishtest/tests/b00030.vtc
+++ b/bin/varnishtest/tests/b00030.vtc
@@ -10,7 +10,7 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_recv {
-		return (error(200,req.grace));
+		return (error(200,req.ttl));
 	}
 	sub vcl_error {
 		set obj.http.x-timestamp = now;
@@ -20,5 +20,5 @@ varnish v1 -vcl+backend {
 client c1 {
 	txreq
 	rxresp
-	expect resp.msg == "0.000"
+	expect resp.msg == "-1.000"
 } -run
diff --git a/bin/varnishtest/tests/v00000.vtc b/bin/varnishtest/tests/v00000.vtc
index 4041d61..b2676b6 100644
--- a/bin/varnishtest/tests/v00000.vtc
+++ b/bin/varnishtest/tests/v00000.vtc
@@ -10,7 +10,7 @@ server s1 -start
 
 varnish v1 -vcl+backend {
 	sub vcl_recv {
-		set req.grace += 1 s;
+		set req.ttl += 1 s;
 	}
 	sub vcl_backend_response {
 		set beresp.ttl += 1 m;
@@ -22,10 +22,5 @@ client c1 {
 	txreq -url "/"
 	rxresp
 	expect resp.status == 200
-}
-
-client c1 -run
-
-server s1 -wait
+} -run
 
-varnish v1 -stop
diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc
index 2a98d30..e63be3f 100644
--- a/bin/varnishtest/tests/v00020.vtc
+++ b/bin/varnishtest/tests/v00020.vtc
@@ -52,7 +52,7 @@ varnish v1 -errvcl {Symbol not found: 'vcl_recv' (expected type STRING_LIST)} {
 # XXX: not obvious if this actually fails for the desired reason ?
 varnish v1 -errvcl {Unknown token '-' when looking for DURATION} {
 	sub vcl_recv {
-		if (req.grace < -3s || req.grace) {
+		if (req.ttl < -3s || req.ttl) {
 			set req.http.foo = vcl_recv;
 		}
 	}
@@ -60,7 +60,7 @@ varnish v1 -errvcl {Unknown token '-' when looking for DURATION} {
 
 varnish v1 -errvcl {'&&' must be followed by BOOL, found DURATION.} {
 	sub vcl_recv {
-		if (req.grace < 3s && req.grace) {
+		if (req.ttl < 3s && req.ttl) {
 			set req.http.foo = vcl_recv;
 		}
 	}
@@ -74,13 +74,13 @@ varnish v1 -errvcl {Operator * not possible on type STRING.} {
 
 varnish v1 -errvcl {DURATION + INT not possible.} {
 	sub vcl_backend_response {
-		set req.http.foo = req.grace + beresp.status;
+		set req.http.foo = req.ttl + beresp.status;
 	}
 }
 
 varnish v1 -errvcl {'!' must be followed by BOOL, found DURATION.} {
 	sub vcl_backend_response {
-		if (! req.grace) {
+		if (! req.ttl) {
 		}
 	}
 }
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 3f20b08..6b5b365 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -172,16 +172,6 @@ sp_variables = [
 		( 'client',),
 		( 'client',),
 	),
-	('req.grace',
-		'DURATION',
-		( 'client',),
-		( 'client',),
-	),
-	('req.keep',
-		'DURATION',
-		( 'client',),
-		( 'client',),
-	),
 	('req.xid',
 		'STRING',
 		( 'client',),



From phk at varnish-cache.org  Fri Oct  4 06:43:08 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 04 Oct 2013 08:43:08 +0200
Subject: [master] 4670ed0 Eliminate req->exp, there's just req.d_ttl left now.
Message-ID: <E1VRz6S-00074C-9f@project.varnish-software.com>

commit 4670ed03fc88fc8793f683f52669ae65acd7d636
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 4 06:42:51 2013 +0000

    Eliminate req->exp, there's just req.d_ttl left now.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index cd71bc6..52b8d93 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -658,7 +658,7 @@ struct req {
 	uint8_t			digest[DIGEST_LEN];
 
 	enum sess_close		doclose;
-	struct exp		exp;
+	double			d_ttl;
 
 	unsigned char		wantbody;
 	uint64_t		req_bodybytes;
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index dd805fd..d9225bf 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -135,8 +135,8 @@ EXP_Ttl(const struct req *req, const struct object *o)
 	double r;
 
 	r = o->exp.ttl;
-	if (req != NULL && req->exp.ttl > 0. && req->exp.ttl < r)
-		r = req->exp.ttl;
+	if (req != NULL && req->d_ttl > 0. && req->d_ttl < r)
+		r = req->d_ttl;
 	return (o->exp.t_origin + r);
 }
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 647ef49..9e1609b 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -684,8 +684,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 	req->director = req->vcl->director[0];
 	AN(req->director);
 
-	EXP_Clr(&req->exp);
-
+	req->d_ttl = -1;
 	req->disable_esi = 0;
 	req->hash_always_miss = 0;
 	req->hash_ignore_busy = 0;
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index a605ad1..9c5ba19 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -304,24 +304,32 @@ VRT_l_beresp_storage(const struct vrt_ctx *ctx, const char *str, ...)
 
 /*--------------------------------------------------------------------*/
 
-void
-VRT_l_req_backend(const struct vrt_ctx *ctx, struct director *be)
-{
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	ctx->req->director = be;
+#define REQ_VAR_L(nm, elem, type,extra)					\
+									\
+void									\
+VRT_l_req_##nm(const struct vrt_ctx *ctx, type arg)			\
+{									\
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);				\
+	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);				\
+	extra;								\
+	ctx->req->elem = arg;						\
 }
 
-struct director *
-VRT_r_req_backend(const struct vrt_ctx *ctx)
-{
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	return (ctx->req->director);
+#define REQ_VAR_R(nm, elem, type)					\
+									\
+type									\
+VRT_r_req_##nm(const struct vrt_ctx *ctx)				\
+{									\
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);				\
+	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);				\
+	return(ctx->req->elem);						\
 }
 
+REQ_VAR_L(backend, director, struct director *,)
+REQ_VAR_R(backend, director, struct director *)
+REQ_VAR_L(ttl, d_ttl, double, if (!(arg>0.0)) arg = 0;)
+REQ_VAR_R(ttl, d_ttl, double)
+
 unsigned
 VRT_r_req_backend_healthy(const struct vrt_ctx *ctx)
 {
@@ -472,8 +480,6 @@ vrt_wsp_exp(struct vsl_log *vsl, double now, const struct exp *e)
 	    e->ttl - dt, e->grace, e->keep, now, dt);
 }
 
-VRT_DO_EXP(req, ctx->req->exp, ttl, 0, )
-
 VRT_DO_EXP(obj, ctx->req->obj->exp, grace, 0,
    EXP_Rearm(ctx->req->obj);
    vrt_wsp_exp(ctx->vsl, ctx->req->t_req, &ctx->req->obj->exp);)



From phk at varnish-cache.org  Fri Oct  4 06:54:57 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 04 Oct 2013 08:54:57 +0200
Subject: [master] 6cb9465 Revamp descriptions of $default_(ttl,grace,keep)
Message-ID: <E1VRzHt-0008Ap-Gn@project.varnish-software.com>

commit 6cb9465478b798c9664898aa9733aaeb40865144
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 4 06:54:39 2013 +0000

    Revamp descriptions of $default_(ttl,grace,keep)

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 2c8b77a..8284a14 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -618,6 +618,10 @@ tweak_poolparam(struct cli *cli, const struct parspec *par, const char *arg)
  * formatting will go haywire.
  */
 
+#define OBJ_STICKY_TEXT \
+	"\nNB: This parameter is evaluated only when objects are created." \
+	"To change it for all objects, restart or ban everything."
+
 #define DELAYED_EFFECT_TEXT \
 	"\nNB: This parameter may take quite some time to take (full) effect."
 
@@ -701,6 +705,8 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv)
 			VCLI_Out(cli, "%-*s Default is %s\n",
 			    margin, "", pp->def);
 			mcf_wrap(cli, pp->descr);
+			if (pp->flags & OBJ_STICKY)
+				mcf_wrap(cli, OBJ_STICKY_TEXT);
 			if (pp->flags & DELAYED_EFFECT)
 				mcf_wrap(cli, DELAYED_EFFECT_TEXT);
 			if (pp->flags & EXPERIMENTAL)
diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h
index 4026bb9..64fccf1 100644
--- a/bin/varnishd/mgt/mgt_param.h
+++ b/bin/varnishd/mgt/mgt_param.h
@@ -46,6 +46,7 @@ struct parspec {
 #define MUST_RELOAD	(1<<3)
 #define WIZARD		(1<<4)
 #define PROTECTED	(1<<5)
+#define OBJ_STICKY	(1<<6)
 	const char	*def;
 	const char	*units;
 };
diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index 6086c9c..d2aca5c 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -56,13 +56,25 @@ const struct parspec mgt_parspec[] = {
 	{ "default_ttl", tweak_timeout_double, &mgt_param.default_ttl,
 		0, UINT_MAX,
 		"The TTL assigned to objects if neither the backend nor "
-		"the VCL code assigns one.\n"
-		"Objects already cached will not be affected by changes "
-		"made until they are fetched from the backend again.\n"
-		"To force an immediate effect at the expense of a total "
-		"flush of the cache use \"ban obj.http.date ~ .\"",
-		0,
+		"the VCL code assigns one.\n",
+		OBJ_STICKY,
 		"120", "seconds" },
+	{ "default_grace", tweak_timeout_double, &mgt_param.default_grace,
+		0, UINT_MAX,
+		"Default grace period.  We will deliver an object "
+		"this long after it has expired, provided another thread "
+		"is attempting to get a new copy.\n",
+		OBJ_STICKY,
+		"10", "seconds" },
+	{ "default_keep", tweak_timeout_double, &mgt_param.default_keep,
+		0, UINT_MAX,
+		"Default keep period.  We will keep a useless object "
+		"around this long, making it available for conditional "
+		"backend fetches.  "
+		"That means that the object will be removed from the "
+		"cache at the end of ttl+grace+keep.",
+		OBJ_STICKY,
+		"0", "seconds" },
 	{ "workspace_client",
 		tweak_bytes_u, &mgt_param.workspace_client, 3072, UINT_MAX,
 		"Bytes of HTTP protocol workspace for clients HTTP req/resp."
@@ -148,24 +160,6 @@ const struct parspec mgt_parspec[] = {
 		"Maximum is 65535 bytes.",
 		0,
 		"255", "bytes" },
-	{ "default_grace", tweak_timeout_double, &mgt_param.default_grace,
-		0, UINT_MAX,
-		"Default grace period.  We will deliver an object "
-		"this long after it has expired, provided another thread "
-		"is attempting to get a new copy.\n"
-		"Objects already cached will not be affected by changes "
-		"made until they are fetched from the backend again.\n",
-		DELAYED_EFFECT,
-		"10", "seconds" },
-	{ "default_keep", tweak_timeout_double, &mgt_param.default_keep,
-		0, UINT_MAX,
-		"Default keep period.  We will keep a useless object "
-		"around this long, making it available for conditional "
-		"backend fetches.  "
-		"That means that the object will be removed from the "
-		"cache at the end of ttl+grace+keep.",
-		DELAYED_EFFECT,
-		"0", "seconds" },
 	{ "timeout_idle", tweak_timeout_double, &mgt_param.timeout_idle,
 		0, UINT_MAX,
 		"Idle timeout for client connections.\n"



From phk at varnish-cache.org  Fri Oct  4 07:17:35 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 04 Oct 2013 09:17:35 +0200
Subject: [master] e0acf69 Assign obj.keep and obj.grace from defaults on obj
 creation.
Message-ID: <E1VRzdn-0002Cb-5D@project.varnish-software.com>

commit e0acf69d44c23acaf698ebf11e77ef02dc7d775f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 4 07:17:03 2013 +0000

    Assign obj.keep and obj.grace from defaults on obj creation.
    
    Losse EXP_Grace() and EXP_Keep() functions

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 52b8d93..7880481 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -904,7 +904,6 @@ void EXP_Set_grace(struct exp *e, double v);
 void EXP_Set_keep(struct exp *e, double v);
 
 double EXP_Ttl(const struct req *, const struct object*);
-double EXP_Grace(const struct req *, const struct object*);
 void EXP_Insert(const struct object *o, double now);
 void EXP_Inject(struct objcore *oc, struct lru *lru, double when);
 void EXP_Init(void);
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index d9225bf..de4e3e9 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -107,28 +107,6 @@ EXP_ACCESS(keep, 0.,)
  * adjusted for defaults and by per-session limits.
  */
 
-static double
-EXP_Keep(const struct req *req, const struct object *o)
-{
-	double r;
-
-	r = (double)cache_param->default_keep;
-	if (o->exp.keep > 0.)
-		r = o->exp.keep;
-	return (EXP_Ttl(req, o) + r);
-}
-
-double
-EXP_Grace(const struct req *req, const struct object *o)
-{
-	double r;
-
-	r = (double)cache_param->default_grace;
-	if (o->exp.grace >= 0.)
-		r = o->exp.grace;
-	return (EXP_Ttl(req, o) + r);
-}
-
 double
 EXP_Ttl(const struct req *req, const struct object *o)
 {
@@ -148,17 +126,14 @@ static int
 update_object_when(const struct object *o)
 {
 	struct objcore *oc;
-	double when, w2;
+	double when;
 
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	oc = o->objcore;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	Lck_AssertHeld(&exp_mtx);
 
-	when = EXP_Keep(NULL, o);
-	w2 = EXP_Grace(NULL, o);
-	if (w2 > when)
-		when = w2;
+	when = o->exp.t_origin + o->exp.ttl + o->exp.grace + o->exp.keep;
 	assert(!isnan(when));
 	if (when == oc->timer_when)
 		return (0);
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index 69735f3..66724ab 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -79,8 +79,9 @@ RFC2616_Ttl(struct busyobj *bo)
 	assert(bo->t_fetch != 0.0 && !isnan(bo->t_fetch));
 	expp->t_origin = bo->t_fetch;
 
-	/* If all else fails, cache using default ttl */
 	expp->ttl = cache_param->default_ttl;
+	expp->grace = cache_param->default_grace;
+	expp->keep = cache_param->default_keep;
 
 	max_age = age = 0;
 	h_expires = 0;
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index f54368d..df90d7e 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -547,7 +547,7 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo,
 	/* We have to do this somewhere, might as well be here... */
 	assert(sizeof so->hash == DIGEST_LEN);
 	memcpy(so->hash, oc->objhead->digest, DIGEST_LEN);
-	so->ttl = EXP_Grace(NULL, o);
+	so->ttl = oc->timer_when;
 	so->ptr = (uint8_t*)o - sc->base;
 	so->ban = BAN_Time(oc->ban);
 
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index 7ca79d6..75dee88 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -480,7 +480,6 @@ smp_oc_updatemeta(struct objcore *oc)
 	struct object *o;
 	struct smp_seg *sg;
 	struct smp_object *so;
-	double mttl;
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	o = smp_oc_getobj(NULL, oc);
@@ -490,17 +489,15 @@ smp_oc_updatemeta(struct objcore *oc)
 	CHECK_OBJ_NOTNULL(sg->sc, SMP_SC_MAGIC);
 	so = smp_find_so(sg, oc->priv2);
 
-	mttl = EXP_Grace(NULL, o);
-
 	if (sg == sg->sc->cur_seg) {
 		/* Lock necessary, we might race close_seg */
 		Lck_Lock(&sg->sc->mtx);
 		so->ban = BAN_Time(oc->ban);
-		so->ttl = mttl;
+		so->ttl = oc->timer_when;
 		Lck_Unlock(&sg->sc->mtx);
 	} else {
 		so->ban = BAN_Time(oc->ban);
-		so->ttl = mttl;
+		so->ttl = oc->timer_when;
 	}
 }
 
diff --git a/bin/varnishtest/tests/p00001.vtc b/bin/varnishtest/tests/p00001.vtc
index ac95149..e8c4aa5 100644
--- a/bin/varnishtest/tests/p00001.vtc
+++ b/bin/varnishtest/tests/p00001.vtc
@@ -13,6 +13,9 @@ varnish v1 \
 	-storage "-spersistent,${tmpdir}/_.per,10m" \
 	-vcl+backend { } -start 
 
+varnish v1 -cliok "param.set default_grace 0"
+varnish v1 -cliok "param.set default_keep 0"
+
 client c1 {
 	txreq -url "/"
 	rxresp
diff --git a/bin/varnishtest/tests/s00000.vtc b/bin/varnishtest/tests/s00000.vtc
index b5090ff..e04c960 100644
--- a/bin/varnishtest/tests/s00000.vtc
+++ b/bin/varnishtest/tests/s00000.vtc
@@ -11,6 +11,8 @@ server s1 {
 } -start
 
 varnish v1 -vcl+backend { } -start
+varnish v1 -cliok "param.set default_grace 0"
+varnish v1 -cliok "param.set default_keep 0"
 
 client c1 {
 	txreq -url "/"
diff --git a/bin/varnishtest/tests/s00001.vtc b/bin/varnishtest/tests/s00001.vtc
index 44ea211..0223916 100644
--- a/bin/varnishtest/tests/s00001.vtc
+++ b/bin/varnishtest/tests/s00001.vtc
@@ -12,6 +12,9 @@ server s1 {
 
 varnish v1 -vcl+backend { } -start
 
+varnish v1 -cliok "param.set default_keep 0"
+varnish v1 -cliok "param.set default_grace 0"
+
 client c1 {
 	txreq -url "/"
 	rxresp



From phk at varnish-cache.org  Fri Oct  4 07:58:40 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 04 Oct 2013 09:58:40 +0200
Subject: [master] 56ea5e3 Go over the entire struct exp-thing and simplify it
 a lot.
Message-ID: <E1VS0HY-0006pb-UN@project.varnish-software.com>

commit 56ea5e3cf99cc90a80d0f561ad9eecfd2ed0b8f3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 4 07:58:06 2013 +0000

    Go over the entire struct exp-thing and simplify it a lot.
    
    We now kill objects when t_origin + ttl + grace + keep has elapsed.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7880481..bd28089 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -289,10 +289,10 @@ typedef int vdp_bytes(struct req *, enum vdp_action, const void *ptr,
 /*--------------------------------------------------------------------*/
 
 struct exp {
-	double			ttl;
-	double			grace;
-	double			keep;
 	double			t_origin;
+	float			ttl;
+	float			grace;
+	float			keep;
 };
 
 /*--------------------------------------------------------------------*/
@@ -896,12 +896,6 @@ extern pthread_t cli_thread;
 
 /* cache_expiry.c */
 void EXP_Clr(struct exp *e);
-double EXP_Get_ttl(const struct exp *e);
-double EXP_Get_grace(const struct exp *e);
-double EXP_Get_keep(const struct exp *e);
-void EXP_Set_ttl(struct exp *e, double v);
-void EXP_Set_grace(struct exp *e, double v);
-void EXP_Set_keep(struct exp *e, double v);
 
 double EXP_Ttl(const struct req *, const struct object*);
 void EXP_Insert(const struct object *o, double now);
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index de4e3e9..0703050 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -35,18 +35,6 @@
  *
  * We hold a single object reference for both data structures.
  *
- * An attempted overview:
- *
- *	                        EXP_Ttl()      EXP_Grace()   EXP_Keep()
- *				   |                |            |
- *      entered                    v                v            |
- *         |                       +--------------->+            |
- *         v                       |      grace                  |
- *         +---------------------->+                             |
- *                  ttl            |                             v
- *                                 +---------------------------->+
- *                                     keep
- *
  */
 
 #include "config.h"
@@ -65,9 +53,6 @@ static struct lock exp_mtx;
 
 /*--------------------------------------------------------------------
  * struct exp manipulations
- *
- * The Get/Set functions encapsulate the mutual magic between the
- * fields in one single place.
  */
 
 void
@@ -75,36 +60,14 @@ EXP_Clr(struct exp *e)
 {
 
 	e->ttl = -1;
-	e->grace = -1;
-	e->keep = -1;
+	e->grace = 0;
+	e->keep = 0;
 	e->t_origin = 0;
 }
 
-#define EXP_ACCESS(fld, low_val, extra)				\
-	double							\
-	EXP_Get_##fld(const struct exp *e)			\
-	{							\
-		return (e->fld > 0. ? e->fld : low_val);	\
-	}							\
-								\
-	void							\
-	EXP_Set_##fld(struct exp *e, double v)			\
-	{							\
-		if (v > 0.)					\
-			e->fld = v;				\
-		else {						\
-			e->fld = -1.;				\
-			extra;					\
-		}						\
-	}							\
-
-EXP_ACCESS(ttl, -1., (e->grace = e->keep = -1.))
-EXP_ACCESS(grace, 0., )
-EXP_ACCESS(keep, 0.,)
-
 /*--------------------------------------------------------------------
- * Calculate an objects effective keep, grace or ttl time, suitably
- * adjusted for defaults and by per-session limits.
+ * Calculate an objects effective ttl time, taking req.ttl into account
+ * if it is available.
  */
 
 double
@@ -482,7 +445,7 @@ EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru)
 			VSLb(vsl, SLT_ExpKill, "%u %.0f LRU",
 			    oc_getxid(&wrk->stats, oc) & VSL_IDENTMASK,
 			    EXP_Ttl(NULL, o) - t);
-			EXP_Set_ttl(&o->exp, 0.);
+			o->exp.ttl = 0.0;
 			(void)HSH_DerefObjCore(&wrk->stats, &oc);
 		}
 
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 9c5ba19..3e0a73e 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -449,17 +449,22 @@ VRT_r_bereq_retries(const struct vrt_ctx *ctx)
  * keep are relative to ttl.
  */
 
-#define VRT_DO_EXP(which, exp, fld, offset, extra)		\
+#define VRT_DO_EXP(which, sexp, fld, offset, now, extra)	\
 								\
 void								\
 VRT_l_##which##_##fld(const struct vrt_ctx *ctx, double a)	\
 {								\
+	double dt;						\
 								\
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);			\
-	if (a > 0.)						\
-		a += offset;					\
-	EXP_Set_##fld(&exp, a);					\
+	if (a > 0.0)						\
+		sexp.fld = a + offset;				\
+	else							\
+		sexp.fld = 0;					\
 	extra;							\
+	dt = now - sexp.t_origin;				\
+	VSLb(ctx->vsl, SLT_TTL, "VCL %.0f %.0f %.0f %.0f %.0f",	\
+	    sexp.ttl - dt, sexp.grace, sexp.keep, now, dt);	\
 }								\
 								\
 double								\
@@ -467,36 +472,22 @@ VRT_r_##which##_##fld(const struct vrt_ctx *ctx)		\
 {								\
 								\
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);			\
-	return(EXP_Get_##fld(&exp) - offset);			\
+	if (sexp.fld > 0.0)					\
+		return(sexp.fld - offset);			\
+	return(0.0);						\
 }
 
-static void
-vrt_wsp_exp(struct vsl_log *vsl, double now, const struct exp *e)
-{
-	double dt;
-
-	dt = now - e->t_origin;
-	VSLb(vsl, SLT_TTL, "VCL %.0f %.0f %.0f %.0f %.0f",
-	    e->ttl - dt, e->grace, e->keep, now, dt);
-}
-
-VRT_DO_EXP(obj, ctx->req->obj->exp, grace, 0,
-   EXP_Rearm(ctx->req->obj);
-   vrt_wsp_exp(ctx->vsl, ctx->req->t_req, &ctx->req->obj->exp);)
+VRT_DO_EXP(obj, ctx->req->obj->exp, grace, 0, ctx->req->t_req,
+   EXP_Rearm(ctx->req->obj);)
 VRT_DO_EXP(obj, ctx->req->obj->exp, ttl,
-   (ctx->req->t_req - ctx->req->obj->exp.t_origin),
-   EXP_Rearm(ctx->req->obj);
-   vrt_wsp_exp(ctx->vsl, ctx->req->t_req, &ctx->req->obj->exp);)
-VRT_DO_EXP(obj, ctx->req->obj->exp, keep, 0,
-   EXP_Rearm(ctx->req->obj);
-   vrt_wsp_exp(ctx->vsl, ctx->req->t_req, &ctx->req->obj->exp);)
-
-VRT_DO_EXP(beresp, ctx->bo->exp, grace, 0,
-   vrt_wsp_exp(ctx->vsl, ctx->bo->exp.t_origin, &ctx->bo->exp);)
-VRT_DO_EXP(beresp, ctx->bo->exp, ttl, 0,
-   vrt_wsp_exp(ctx->vsl, ctx->bo->exp.t_origin, &ctx->bo->exp);)
-VRT_DO_EXP(beresp, ctx->bo->exp, keep, 0,
-   vrt_wsp_exp(ctx->vsl, ctx->bo->exp.t_origin, &ctx->bo->exp);)
+   (ctx->req->t_req - ctx->req->obj->exp.t_origin), ctx->req->t_req,
+   EXP_Rearm(ctx->req->obj);)
+VRT_DO_EXP(obj, ctx->req->obj->exp, keep, 0, ctx->req->t_req,
+   EXP_Rearm(ctx->req->obj);)
+
+VRT_DO_EXP(beresp, ctx->bo->exp, grace, 0, ctx->bo->exp.t_origin,)
+VRT_DO_EXP(beresp, ctx->bo->exp, ttl, 0, ctx->bo->exp.t_origin,)
+VRT_DO_EXP(beresp, ctx->bo->exp, keep, 0, ctx->bo->exp.t_origin,)
 
 /*--------------------------------------------------------------------
  * req.xid
diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index 5402e1f..e203cb8 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -170,4 +170,4 @@
 -e840	// Use of nul character in a string literal (see: vcc_if.c)
 -e663	// Suspicious array to pointer conversion
 -e778   // Constant expression evaluates to 0 in operation '___'
-
+-e736	// Loss of precision (___) (___ bits to ___ bits)
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index 75dee88..c6f8a2f 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -460,7 +460,7 @@ smp_oc_getobj(struct dstat *ds, struct objcore *oc)
 			bad |= 0x100;
 
 		if(bad) {
-			EXP_Set_ttl(&o->exp, -1);
+			o->exp.ttl = -1;
 			so->ttl = 0;
 		}
 



From phk at varnish-cache.org  Fri Oct  4 08:34:19 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 04 Oct 2013 10:34:19 +0200
Subject: [master] 1e94b37 This AZ introduced the opposite race of the one it
 was supposed to check: The req might get rescheduled before we get there.
Message-ID: <E1VS0q3-0002VS-54@project.varnish-software.com>

commit 1e94b37816156667d7471a3df6e74f1275923bbc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 4 08:33:51 2013 +0000

    This AZ introduced the opposite race of the one it was supposed
    to check:  The req might get rescheduled before we get there.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 9e1609b..c20aa83 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -884,7 +884,6 @@ CNT_Request(struct worker *wrk, struct req *req)
 		req->wrk = NULL;
 	}
 
-	AZ(req->wrk);
 	assert(WRW_IsReleased(wrk));
 	return (nxt);
 }



From phk at varnish-cache.org  Fri Oct  4 10:59:09 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 04 Oct 2013 12:59:09 +0200
Subject: [master] bb6137b Polish, yet again,
 specific sizes to make this pass on both 32 and 64 bit architectures.
Message-ID: <E1VS36D-0007Fe-T2@project.varnish-software.com>

commit bb6137b66f7540d9d54938e1a89acfbc838b10f4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 4 10:58:20 2013 +0000

    Polish, yet again, specific sizes to make this pass on both
    32 and 64 bit architectures.
    
    The good news is that this is necessary because our obj overhead
    decreases.

diff --git a/bin/varnishtest/tests/b00018.vtc b/bin/varnishtest/tests/b00018.vtc
index 02d4a93..2ad8cd2 100644
--- a/bin/varnishtest/tests/b00018.vtc
+++ b/bin/varnishtest/tests/b00018.vtc
@@ -16,12 +16,15 @@ varnish v1 -vcl+backend {
 	}
 } -start
 
+varnish v1 -cliok "param.set default_grace 0"
+varnish v1 -cliok "param.set default_keep 0"
+
 client c1 {
 	txreq -url "/"
 	rxresp
 	expect resp.status == 523
 } -run
 
-delay 10
+delay 4
 
 varnish v1 -expect n_object == 0
diff --git a/bin/varnishtest/tests/c00044.vtc b/bin/varnishtest/tests/c00044.vtc
index 2aac160..6723a3a 100644
--- a/bin/varnishtest/tests/c00044.vtc
+++ b/bin/varnishtest/tests/c00044.vtc
@@ -2,17 +2,17 @@ varnishtest	"Object/LRU/Stevedores"
 
 server s1 {
 	rxreq
-	txresp -bodylen 1048182
+	txresp -bodylen 1048190
 	rxreq
-	txresp -bodylen 1048183
+	txresp -bodylen 1048191
 	rxreq
-	txresp -bodylen 1048184
+	txresp -bodylen 1048192
 
 	rxreq
-	txresp -bodylen 1048185
+	txresp -bodylen 1048193
 
 	rxreq
-	txresp -bodylen 1048186
+	txresp -bodylen 1048194
 } -start
 
 varnish v1 -storage "-smalloc,1m -smalloc,1m, -smalloc,1m" -vcl+backend {
@@ -27,7 +27,7 @@ client c1 {
 	txreq -url /foo
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048182
+	expect resp.bodylen == 1048190
 } -run
 
 varnish v1 -expect SMA.Transient.g_bytes == 0
@@ -42,7 +42,7 @@ client c1 {
 	txreq -url /bar
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048183
+	expect resp.bodylen == 1048191
 } -run
 
 varnish v1 -expect SMA.Transient.g_bytes == 0
@@ -57,7 +57,7 @@ client c1 {
 	txreq -url /burp
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048184
+	expect resp.bodylen == 1048192
 } -run
 
 varnish v1 -expect SMA.Transient.g_bytes == 0
@@ -72,7 +72,7 @@ client c1 {
 	txreq -url /foo1
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048185
+	expect resp.bodylen == 1048193
 } -run
 
 varnish v1 -expect n_lru_nuked == 1
@@ -81,7 +81,7 @@ client c1 {
 	txreq -url /foo
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048186
+	expect resp.bodylen == 1048194
 } -run
 
 varnish v1 -expect n_lru_nuked == 2
diff --git a/bin/varnishtest/tests/c00045.vtc b/bin/varnishtest/tests/c00045.vtc
index 0e3f4fd..aa2897e 100644
--- a/bin/varnishtest/tests/c00045.vtc
+++ b/bin/varnishtest/tests/c00045.vtc
@@ -2,11 +2,11 @@ varnishtest	"Object/LRU/Stevedores with hinting"
 
 server s1 {
 	rxreq
-	txresp -bodylen 1048180
+	txresp -bodylen 1048188
 	rxreq
-	txresp -bodylen 1048181
+	txresp -bodylen 1048189
 	rxreq
-	txresp -bodylen 1048182
+	txresp -bodylen 1048190
 } -start
 
 varnish v1 -storage "-smalloc,1m -smalloc,1m, -smalloc,1m" -vcl+backend {
@@ -21,7 +21,7 @@ client c1 {
 	txreq -url /foo
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048180
+	expect resp.bodylen == 1048188
 } -run
 
 varnish v1 -expect SMA.Transient.g_bytes == 0
@@ -36,7 +36,7 @@ client c1 {
 	txreq -url /bar
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048181
+	expect resp.bodylen == 1048189
 } -run
 
 varnish v1 -expect n_lru_nuked == 1
@@ -52,7 +52,7 @@ client c1 {
 	txreq -url /foo
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048182
+	expect resp.bodylen == 1048190
 } -run
 
 varnish v1 -expect n_lru_nuked == 2
diff --git a/bin/varnishtest/tests/r01140.vtc b/bin/varnishtest/tests/r01140.vtc
index 1ab3bee..e9d1eef 100644
--- a/bin/varnishtest/tests/r01140.vtc
+++ b/bin/varnishtest/tests/r01140.vtc
@@ -4,7 +4,7 @@ server s1 {
 	# This response should almost completely fill the storage
 	rxreq
 	expect req.url == /url1
-	txresp -bodylen 1048108
+	txresp -bodylen 1048208
 
 	# The next one should not fit in the storage, ending up in transient
 	# with zero ttl (=shortlived)
@@ -29,7 +29,7 @@ client c1 {
 	txreq -url /url1
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048108
+	expect resp.bodylen == 1048208
 } -run
 
 delay .1
diff --git a/bin/varnishtest/tests/r01283.vtc b/bin/varnishtest/tests/r01283.vtc
index f7fc759..fab7ee2 100644
--- a/bin/varnishtest/tests/r01283.vtc
+++ b/bin/varnishtest/tests/r01283.vtc
@@ -2,7 +2,7 @@ varnishtest "#1283 - Test failure to allocate object for error (transient full)"
 
 server s1 {
 	rxreq
-	txresp -bodylen 1048148
+	txresp -bodylen 1048198
 } -start
 
 varnish v1 -arg "-p nuke_limit=0" -storage "-sTransient=malloc,1m" -vcl+backend {
diff --git a/bin/varnishtest/tests/r01284.vtc b/bin/varnishtest/tests/r01284.vtc
index 4348f01..92c1a51 100644
--- a/bin/varnishtest/tests/r01284.vtc
+++ b/bin/varnishtest/tests/r01284.vtc
@@ -3,7 +3,7 @@ varnishtest "#1284 - Test resource cleanup after STV_NewObject fail in fetch"
 server s1 {
 	rxreq
 	expect req.url == "/obj1"
-	txresp -bodylen 1048140
+	txresp -bodylen 1048190
 	rxreq
 	expect req.url == "/obj2"
 	txresp -hdr "Long: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -hdr "Long2: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"



From phk at varnish-cache.org  Mon Oct  7 08:58:43 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 07 Oct 2013 10:58:43 +0200
Subject: [master] 1e60f81 Don't return OIS_STREAM as long as we know more data
 is forthcoming
Message-ID: <E1VT6eJ-0002Qj-Qo@project.varnish-software.com>

commit 1e60f8195b1fc377a6f4e83c403211cc05a3e13e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 7 08:58:26 2013 +0000

    Don't return OIS_STREAM as long as we know more data is forthcoming

diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c
index eadd358..6da3348 100644
--- a/bin/varnishd/cache/cache_obj.c
+++ b/bin/varnishd/cache/cache_obj.c
@@ -104,9 +104,12 @@ ObjIter(struct objiter *oi, void **p, ssize_t *l)
 			ol -= oi->st->len;
 			nl -= oi->st->len;
 		}
+		oi->st = VTAILQ_NEXT(oi->st, list);
+		if (oi->st != NULL && oi->st->len == 0)
+			oi->st = NULL;
 		Lck_Unlock(&oi->bo->mtx);
 		assert(*l > 0);
-		return (OIS_STREAM);
+		return (oi->st ? OIS_DATA : OIS_STREAM);
 	}
 }
 



From phk at varnish-cache.org  Tue Oct  8 06:58:04 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 08 Oct 2013 08:58:04 +0200
Subject: [master] 39921cd Remove the diagnostic macro, we don't use it.
Message-ID: <E1VTRF6-0004No-6U@project.varnish-software.com>

commit 39921cd7e8ab8495e7cf0eefb5d3455689c33f93
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 8 06:57:45 2013 +0000

    Remove the diagnostic macro, we don't use it.

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index f9d0372..b6f3916 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -261,7 +261,7 @@ http_GetHdr(const struct http *hp, const char *hdr, char **ptr)
 	char *p;
 
 	l = hdr[0];
-	diagnostic(l == strlen(hdr + 1));
+	assert(l == strlen(hdr + 1));
 	assert(hdr[l] == ':');
 	hdr++;
 	u = http_findhdr(hp, l - 1, hdr);



From phk at varnish-cache.org  Tue Oct  8 11:55:55 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 08 Oct 2013 13:55:55 +0200
Subject: [master] 842e400 Pack the binheap into a structure which will allow
 us to have multiple instances at a later date.
Message-ID: <E1VTVtL-0005A4-Sb@project.varnish-software.com>

commit 842e400840650acdb9474d99168a62cff23ab5ce
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 8 10:12:02 2013 +0000

    Pack the binheap into a structure which will allow us to have
    multiple instances at a later date.
    
    Put the "LRU_DONTMOVE" flag on the lru, rather than the objcore.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index bd28089..0bd75c9 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -377,6 +377,10 @@ struct lru {
 #define LRU_MAGIC		0x3fec7bb0
 	VTAILQ_HEAD(,objcore)	lru_head;
 	struct lock		mtx;
+	unsigned		flags;
+#define LRU_F_DONTMOVE		(1<<1)
+#define LRU_F_CONDEMMED		(1<<2)
+	unsigned		n_objcore;
 };
 
 /* Storage -----------------------------------------------------------*/
@@ -430,7 +434,7 @@ struct objcore {
 	unsigned		flags;
 #define OC_F_BUSY		(1<<1)
 #define OC_F_PASS		(1<<2)
-#define OC_F_LRUDONTMOVE	(1<<4)
+// #define OC_F_LRUDONTMOVE	(1<<4)
 #define OC_F_PRIV		(1<<5)		/* Stevedore private flag */
 #define OC_F_LURK		(3<<6)		/* Ban-lurker-color */
 #define OC_F_PRIVATE		(1<<8)
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 0703050..00ad4a4 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -40,6 +40,7 @@
 #include "config.h"
 
 #include <math.h>
+#include <stdlib.h>
 
 #include "cache.h"
 
@@ -47,9 +48,15 @@
 #include "hash/hash_slinger.h"
 #include "vtim.h"
 
-static pthread_t exp_thread;
-static struct binheap *exp_heap;
-static struct lock exp_mtx;
+struct exp_priv {
+	unsigned			magic;
+#define EXP_PRIV_MAGIC			0x9db22482
+	struct lock			mtx;
+	VTAILQ_HEAD(,objcore)		inbox;
+	struct binheap			*heap;
+};
+
+static struct exp_priv *exphdl;
 
 /*--------------------------------------------------------------------
  * struct exp manipulations
@@ -94,7 +101,7 @@ update_object_when(const struct object *o)
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	oc = o->objcore;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	Lck_AssertHeld(&exp_mtx);
+	Lck_AssertHeld(&exphdl->mtx);
 
 	when = o->exp.t_origin + o->exp.ttl + o->exp.grace + o->exp.keep;
 	assert(!isnan(when));
@@ -113,9 +120,9 @@ exp_insert(struct objcore *oc, struct lru *lru)
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 
 	Lck_AssertHeld(&lru->mtx);
-	Lck_AssertHeld(&exp_mtx);
+	Lck_AssertHeld(&exphdl->mtx);
 	assert(oc->timer_idx == BINHEAP_NOIDX);
-	binheap_insert(exp_heap, oc);
+	binheap_insert(exphdl->heap, oc);
 	assert(oc->timer_idx != BINHEAP_NOIDX);
 	VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
 }
@@ -134,10 +141,10 @@ EXP_Inject(struct objcore *oc, struct lru *lru, double when)
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 
 	Lck_Lock(&lru->mtx);
-	Lck_Lock(&exp_mtx);
+	Lck_Lock(&exphdl->mtx);
 	oc->timer_when = when;
 	exp_insert(oc, lru);
-	Lck_Unlock(&exp_mtx);
+	Lck_Unlock(&exphdl->mtx);
 	Lck_Unlock(&lru->mtx);
 }
 
@@ -165,10 +172,10 @@ EXP_Insert(const struct object *o, double now)
 	lru = oc_getlru(oc);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 	Lck_Lock(&lru->mtx);
-	Lck_Lock(&exp_mtx);
+	Lck_Lock(&exphdl->mtx);
 	(void)update_object_when(o);
 	exp_insert(oc, lru);
-	Lck_Unlock(&exp_mtx);
+	Lck_Unlock(&exphdl->mtx);
 	Lck_Unlock(&lru->mtx);
 	oc_updatemeta(oc);
 }
@@ -176,7 +183,7 @@ EXP_Insert(const struct object *o, double now)
 /*--------------------------------------------------------------------
  * Object was used, move to tail of LRU list.
  *
- * To avoid the exp_mtx becoming a hotspot, we only attempt to move
+ * To avoid the exphdl->mtx becoming a hotspot, we only attempt to move
  * objects if they have not been moved recently and if the lock is available.
  * This optimization obviously leaves the LRU list imperfectly sorted.
  */
@@ -188,19 +195,16 @@ EXP_Touch(struct objcore *oc)
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
+	lru = oc_getlru(oc);
+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+
 	/*
-	 * For -spersistent we don't move objects on the lru list.  Each
-	 * segment has its own LRU list, and the order on it is not material
-	 * for anything.  The code below would move the objects to the
-	 * LRU list of the currently open segment, which would prevent
-	 * the cleaner from doing its job.
+	 * For -spersistent (and possibly other stevedores, we don't move
+	 * objects on the lru list, since LRU doesn't really help much.
 	 */
-	if (oc->flags & OC_F_LRUDONTMOVE)
+	if (lru->flags & LRU_F_DONTMOVE)
 		return (0);
 
-	lru = oc_getlru(oc);
-	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
-
 	/*
 	 * We only need the LRU lock here.  The locking order is LRU->EXP
 	 * so we can trust the content of the oc->timer_idx without the
@@ -241,17 +245,17 @@ EXP_Rearm(const struct object *o)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	lru = oc_getlru(oc);
 	Lck_Lock(&lru->mtx);
-	Lck_Lock(&exp_mtx);
+	Lck_Lock(&exphdl->mtx);
 	/*
 	 * The hang-man might have this object of the binheap while
 	 * tending to a timer.  If so, we do not muck with it here.
 	 */
 	if (oc->timer_idx != BINHEAP_NOIDX && update_object_when(o)) {
 		assert(oc->timer_idx != BINHEAP_NOIDX);
-		binheap_reorder(exp_heap, oc->timer_idx);
+		binheap_reorder(exphdl->heap, oc->timer_idx);
 		assert(oc->timer_idx != BINHEAP_NOIDX);
 	}
-	Lck_Unlock(&exp_mtx);
+	Lck_Unlock(&exphdl->mtx);
 	Lck_Unlock(&lru->mtx);
 	oc_updatemeta(oc);
 }
@@ -269,8 +273,10 @@ exp_timer(struct worker *wrk, void *priv)
 	double t;
 	struct object *o;
 	struct vsl_log vsl;
+	struct exp_priv *ep;
 
-	(void)priv;
+
+	CAST_OBJ_NOTNULL(ep, priv, EXP_PRIV_MAGIC);
 	VSL_Setup(&vsl, NULL, 0);
 	t = VTIM_real();
 	oc = NULL;
@@ -282,10 +288,10 @@ exp_timer(struct worker *wrk, void *priv)
 			t = VTIM_real();
 		}
 
-		Lck_Lock(&exp_mtx);
-		oc = binheap_root(exp_heap);
+		Lck_Lock(&exphdl->mtx);
+		oc = binheap_root(exphdl->heap);
 		if (oc == NULL) {
-			Lck_Unlock(&exp_mtx);
+			Lck_Unlock(&exphdl->mtx);
 			continue;
 		}
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
@@ -297,22 +303,22 @@ exp_timer(struct worker *wrk, void *priv)
 		if (oc->timer_when > t)
 			t = VTIM_real();
 		if (oc->timer_when > t) {
-			Lck_Unlock(&exp_mtx);
+			Lck_Unlock(&exphdl->mtx);
 			oc = NULL;
 			continue;
 		}
 
 		/* If the object is busy, we have to wait for it */
 		if (oc->flags & OC_F_BUSY) {
-			Lck_Unlock(&exp_mtx);
+			Lck_Unlock(&exphdl->mtx);
 			oc = NULL;
 			continue;
 		}
 
 		/*
 		 * It's time...
-		 * Technically we should drop the exp_mtx, get the lru->mtx
-		 * get the exp_mtx again and then check that the oc is still
+		 * Technically we should drop the exphdl->mtx, get the lru->mtx
+		 * get the exphdl->mtx again and then check that the oc is still
 		 * on the binheap.  We take the shorter route and try to
 		 * get the lru->mtx and punt if we fail.
 		 */
@@ -320,21 +326,21 @@ exp_timer(struct worker *wrk, void *priv)
 		lru = oc_getlru(oc);
 		CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 		if (Lck_Trylock(&lru->mtx)) {
-			Lck_Unlock(&exp_mtx);
+			Lck_Unlock(&exphdl->mtx);
 			oc = NULL;
 			continue;
 		}
 
 		/* Remove from binheap */
 		assert(oc->timer_idx != BINHEAP_NOIDX);
-		binheap_delete(exp_heap, oc->timer_idx);
+		binheap_delete(exphdl->heap, oc->timer_idx);
 		assert(oc->timer_idx == BINHEAP_NOIDX);
 
 		/* And from LRU */
 		lru = oc_getlru(oc);
 		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
 
-		Lck_Unlock(&exp_mtx);
+		Lck_Unlock(&exphdl->mtx);
 		Lck_Unlock(&lru->mtx);
 
 		VSC_C_main->n_expired++;
@@ -362,7 +368,7 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 
 	/* Find the first currently unused object on the LRU.  */
 	Lck_Lock(&lru->mtx);
-	Lck_Lock(&exp_mtx);
+	Lck_Lock(&exphdl->mtx);
 	VTAILQ_FOREACH(oc, &lru->lru_head, lru_list) {
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 		assert(oc->timer_idx != BINHEAP_NOIDX);
@@ -377,11 +383,11 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 	}
 	if (oc != NULL) {
 		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
-		binheap_delete(exp_heap, oc->timer_idx);
+		binheap_delete(exphdl->heap, oc->timer_idx);
 		assert(oc->timer_idx == BINHEAP_NOIDX);
 		VSC_C_main->n_lru_nuked++;
 	}
-	Lck_Unlock(&exp_mtx);
+	Lck_Unlock(&exphdl->mtx);
 	Lck_Unlock(&lru->mtx);
 
 	if (oc == NULL)
@@ -417,7 +423,7 @@ EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru)
 	t = VTIM_real();
 	Lck_Lock(&lru->mtx);
 	while (!VTAILQ_EMPTY(&lru->lru_head)) {
-		Lck_Lock(&exp_mtx);
+		Lck_Lock(&exphdl->mtx);
 		n = 0;
 		while (n < NUKEBUF) {
 			oc = VTAILQ_FIRST(&lru->lru_head);
@@ -429,14 +435,14 @@ EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru)
 			/* Remove from the LRU and binheap */
 			VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
 			assert(oc->timer_idx != BINHEAP_NOIDX);
-			binheap_delete(exp_heap, oc->timer_idx);
+			binheap_delete(exphdl->heap, oc->timer_idx);
 			assert(oc->timer_idx == BINHEAP_NOIDX);
 
 			oc_array[n++] = oc;
 			VSC_C_main->n_lru_nuked++;
 		}
 		assert(n > 0);
-		Lck_Unlock(&exp_mtx);
+		Lck_Unlock(&exphdl->mtx);
 		Lck_Unlock(&lru->mtx);
 
 		for (i = 0; i < n; i++) {
@@ -486,9 +492,16 @@ object_update(void *priv, void *p, unsigned u)
 void
 EXP_Init(void)
 {
-
-	Lck_New(&exp_mtx, lck_exp);
-	exp_heap = binheap_new(NULL, object_cmp, object_update);
-	XXXAN(exp_heap);
-	WRK_BgThread(&exp_thread, "cache-timeout", exp_timer, NULL);
+	struct exp_priv *ep;
+	pthread_t pt;
+
+	ALLOC_OBJ(ep, EXP_PRIV_MAGIC);
+	AN(ep);
+
+	Lck_New(&ep->mtx, lck_exp);
+	ep->heap = binheap_new(NULL, object_cmp, object_update);
+	AN(ep->heap);
+	VTAILQ_INIT(&ep->inbox);
+	exphdl = ep;
+	WRK_BgThread(&pt, "cache-timeout", exp_timer, ep);
 }
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index df90d7e..487a77e 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -538,7 +538,6 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo,
 
 	oc = o->objcore;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	oc->flags |= OC_F_LRUDONTMOVE;
 
 	Lck_Lock(&sc->mtx);
 	sg->nfixed++;
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index c6f8a2f..5bdba03 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -152,7 +152,7 @@ smp_load_seg(struct worker *wrk, const struct smp_sc *sc,
 			continue;
 		ALLOC_OBJ(oc, OBJCORE_MAGIC);
 		AN(oc);
-		oc->flags |= OC_F_NEEDFIXUP | OC_F_LRUDONTMOVE;
+		oc->flags |= OC_F_NEEDFIXUP;
 		oc->flags &= ~OC_F_BUSY;
 		smp_init_oc(oc, sg, no);
 		oc->ban = BAN_RefBan(oc, so->ban, sc->tailban);
@@ -215,6 +215,7 @@ smp_new_seg(struct smp_sc *sc)
 	*sg = tmpsg;
 	sg->lru = LRU_Alloc();
 	CHECK_OBJ_NOTNULL(sg->lru, LRU_MAGIC);
+	sg->lru->flags |= LRU_F_DONTMOVE;
 
 	sg->p.offset = IRNUP(sc, sg->p.offset);
 	sg->p.length = IRNDN(sc, sg->p.length);



From phk at varnish-cache.org  Tue Oct  8 11:55:55 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 08 Oct 2013 13:55:55 +0200
Subject: [master] 255f02c Move inserts into the binheap into the expiry-thread.
Message-ID: <E1VTVtL-0005AD-VH@project.varnish-software.com>

commit 255f02cfcfb50b24efab29616208bb22459bd271
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 8 11:55:01 2013 +0000

    Move inserts into the binheap into the expiry-thread.
    
    This should make object creation faster.
    
    As a sideeffect, we loose the $expiry_sleep parameter:  Now we always
    sleep optimally.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 0bd75c9..5f68e22 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -434,7 +434,7 @@ struct objcore {
 	unsigned		flags;
 #define OC_F_BUSY		(1<<1)
 #define OC_F_PASS		(1<<2)
-// #define OC_F_LRUDONTMOVE	(1<<4)
+#define OC_F_OFFLRU		(1<<4)
 #define OC_F_PRIV		(1<<5)		/* Stevedore private flag */
 #define OC_F_LURK		(3<<6)		/* Ban-lurker-color */
 #define OC_F_PRIVATE		(1<<8)
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 00ad4a4..a340af9 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -54,6 +54,7 @@ struct exp_priv {
 	struct lock			mtx;
 	VTAILQ_HEAD(,objcore)		inbox;
 	struct binheap			*heap;
+	pthread_cond_t			condvar;
 };
 
 static struct exp_priv *exphdl;
@@ -111,22 +112,6 @@ update_object_when(const struct object *o)
 	return (1);
 }
 
-/*--------------------------------------------------------------------*/
-
-static void
-exp_insert(struct objcore *oc, struct lru *lru)
-{
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
-
-	Lck_AssertHeld(&lru->mtx);
-	Lck_AssertHeld(&exphdl->mtx);
-	assert(oc->timer_idx == BINHEAP_NOIDX);
-	binheap_insert(exphdl->heap, oc);
-	assert(oc->timer_idx != BINHEAP_NOIDX);
-	VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
-}
-
 /*--------------------------------------------------------------------
  * Object has been added to cache, record in lru & binheap.
  *
@@ -141,11 +126,15 @@ EXP_Inject(struct objcore *oc, struct lru *lru, double when)
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 
 	Lck_Lock(&lru->mtx);
+	lru->n_objcore++;
+	Lck_Unlock(&lru->mtx);
+
 	Lck_Lock(&exphdl->mtx);
 	oc->timer_when = when;
-	exp_insert(oc, lru);
+	oc->flags |= OC_F_OFFLRU;
+	VTAILQ_INSERT_TAIL(&exphdl->inbox, oc, lru_list);
+	AZ(pthread_cond_signal(&exphdl->condvar));
 	Lck_Unlock(&exphdl->mtx);
-	Lck_Unlock(&lru->mtx);
 }
 
 /*--------------------------------------------------------------------
@@ -172,11 +161,16 @@ EXP_Insert(const struct object *o, double now)
 	lru = oc_getlru(oc);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 	Lck_Lock(&lru->mtx);
+	lru->n_objcore++;
+	Lck_Unlock(&lru->mtx);
+
 	Lck_Lock(&exphdl->mtx);
 	(void)update_object_when(o);
-	exp_insert(oc, lru);
+	oc->flags |= OC_F_OFFLRU;
+	VTAILQ_INSERT_TAIL(&exphdl->inbox, oc, lru_list);
+	AZ(pthread_cond_signal(&exphdl->condvar));
 	Lck_Unlock(&exphdl->mtx);
-	Lck_Unlock(&lru->mtx);
+
 	oc_updatemeta(oc);
 }
 
@@ -214,7 +208,9 @@ EXP_Touch(struct objcore *oc)
 	if (Lck_Trylock(&lru->mtx))
 		return (0);
 
-	if (oc->timer_idx != BINHEAP_NOIDX) {
+	if (oc->flags & OC_F_OFFLRU) {
+		/* Cannot move it while it's in the inbox */
+	} else if (oc->timer_idx != BINHEAP_NOIDX) {
 		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
 		VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
 		VSC_C_main->n_lru_moved++;
@@ -254,6 +250,7 @@ EXP_Rearm(const struct object *o)
 		assert(oc->timer_idx != BINHEAP_NOIDX);
 		binheap_reorder(exphdl->heap, oc->timer_idx);
 		assert(oc->timer_idx != BINHEAP_NOIDX);
+		AZ(pthread_cond_signal(&exphdl->condvar)); // XXX
 	}
 	Lck_Unlock(&exphdl->mtx);
 	Lck_Unlock(&lru->mtx);
@@ -274,24 +271,55 @@ exp_timer(struct worker *wrk, void *priv)
 	struct object *o;
 	struct vsl_log vsl;
 	struct exp_priv *ep;
-
+	struct timespec ts;
+	int idle;
 
 	CAST_OBJ_NOTNULL(ep, priv, EXP_PRIV_MAGIC);
 	VSL_Setup(&vsl, NULL, 0);
 	t = VTIM_real();
 	oc = NULL;
+	idle = 0;
 	while (1) {
-		if (oc == NULL) {
+		Lck_Lock(&ep->mtx);
+
+		if (idle) {
 			VSL_Flush(&vsl, 0);
 			WRK_SumStat(wrk);
-			VTIM_sleep(cache_param->expiry_sleep);
+			oc = binheap_root(ep->heap);
+			if (oc != NULL && oc->timer_when > 0.0) {
+				ts.tv_nsec = modf(oc->timer_when, &t) * 1e9;
+				ts.tv_sec = t;
+				(void)Lck_CondWait(&ep->condvar, &ep->mtx, &ts);
+			} else if (oc == NULL) {
+				(void)Lck_CondWait(&ep->condvar, &ep->mtx,NULL);
+			} else {
+				/* We're behind, don't sleep */
+			}
 			t = VTIM_real();
 		}
 
-		Lck_Lock(&exphdl->mtx);
-		oc = binheap_root(exphdl->heap);
+		oc = VTAILQ_FIRST(&ep->inbox);
+		if (oc != NULL) {
+			VTAILQ_REMOVE(&ep->inbox, oc, lru_list);
+			assert(oc->timer_idx == BINHEAP_NOIDX);
+			binheap_insert(exphdl->heap, oc);
+			assert(oc->timer_idx != BINHEAP_NOIDX);
+			Lck_Unlock(&ep->mtx);
+
+			lru = oc_getlru(oc);
+			Lck_Lock(&lru->mtx);
+			VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
+			oc->flags &= ~OC_F_OFFLRU;
+			oc->last_lru = t;
+			Lck_Unlock(&lru->mtx);
+			idle = 0;
+			continue;
+		}
+
+		oc = binheap_root(ep->heap);
 		if (oc == NULL) {
-			Lck_Unlock(&exphdl->mtx);
+			Lck_Unlock(&ep->mtx);
+			idle = 1;
 			continue;
 		}
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
@@ -303,22 +331,22 @@ exp_timer(struct worker *wrk, void *priv)
 		if (oc->timer_when > t)
 			t = VTIM_real();
 		if (oc->timer_when > t) {
-			Lck_Unlock(&exphdl->mtx);
-			oc = NULL;
+			Lck_Unlock(&ep->mtx);
+			idle = 1;
 			continue;
 		}
 
 		/* If the object is busy, we have to wait for it */
 		if (oc->flags & OC_F_BUSY) {
-			Lck_Unlock(&exphdl->mtx);
-			oc = NULL;
+			Lck_Unlock(&ep->mtx);
+			idle = 1;
 			continue;
 		}
 
 		/*
 		 * It's time...
-		 * Technically we should drop the exphdl->mtx, get the lru->mtx
-		 * get the exphdl->mtx again and then check that the oc is still
+		 * Technically we should drop the ep->mtx, get the lru->mtx
+		 * get the ep->mtx again and then check that the oc is still
 		 * on the binheap.  We take the shorter route and try to
 		 * get the lru->mtx and punt if we fail.
 		 */
@@ -326,21 +354,21 @@ exp_timer(struct worker *wrk, void *priv)
 		lru = oc_getlru(oc);
 		CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 		if (Lck_Trylock(&lru->mtx)) {
-			Lck_Unlock(&exphdl->mtx);
-			oc = NULL;
+			Lck_Unlock(&ep->mtx);
+			idle = 1;
 			continue;
 		}
 
 		/* Remove from binheap */
 		assert(oc->timer_idx != BINHEAP_NOIDX);
-		binheap_delete(exphdl->heap, oc->timer_idx);
+		binheap_delete(ep->heap, oc->timer_idx);
 		assert(oc->timer_idx == BINHEAP_NOIDX);
 
 		/* And from LRU */
 		lru = oc_getlru(oc);
 		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
 
-		Lck_Unlock(&exphdl->mtx);
+		Lck_Unlock(&ep->mtx);
 		Lck_Unlock(&lru->mtx);
 
 		VSC_C_main->n_expired++;
@@ -499,6 +527,7 @@ EXP_Init(void)
 	AN(ep);
 
 	Lck_New(&ep->mtx, lck_exp);
+	AZ(pthread_cond_init(&ep->condvar, NULL));
 	ep->heap = binheap_new(NULL, object_cmp, object_update);
 	AN(ep->heap);
 	VTAILQ_INIT(&ep->inbox);
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index f990354..4a16304 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -174,9 +174,6 @@ struct params {
 	/* Acceptable clockskew with backends */
 	unsigned		clock_skew;
 
-	/* Expiry pacer parameters */
-	double			expiry_sleep;
-
 	/* Acceptor pacer parameters */
 	double			acceptor_sleep_max;
 	double			acceptor_sleep_incr;
diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index d2aca5c..f123661 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -173,11 +173,6 @@ const struct parspec mgt_parspec[] = {
 		" from first non-white-space character to double CRNL.",
 		0,
 		"2", "seconds" },
-	{ "expiry_sleep", tweak_timeout_double, &mgt_param.expiry_sleep, 0, 60,
-		"How long the expiry thread sleeps when there is nothing "
-		"for it to do.\n",
-		0,
-		"1", "seconds" },
 	{ "pipe_timeout", tweak_timeout, &mgt_param.pipe_timeout, 0, 0,
 		"Idle timeout for PIPE sessions. "
 		"If nothing have been received in either direction for "
diff --git a/bin/varnishtest/tests/r01140.vtc b/bin/varnishtest/tests/r01140.vtc
index e9d1eef..98abc61 100644
--- a/bin/varnishtest/tests/r01140.vtc
+++ b/bin/varnishtest/tests/r01140.vtc
@@ -18,7 +18,7 @@ server s1 {
 	txresp -bodylen 1025
 } -start
 
-varnish v1 -arg "-p expiry_sleep=0.01 -p nuke_limit=0 -p shortlived=0" \
+varnish v1 -arg "-p nuke_limit=0 -p shortlived=0" \
 	-storage "-smalloc,1m" -vcl+backend { 
 	sub vcl_backend_response {
 		set beresp.do_stream = false;



From phk at varnish-cache.org  Tue Oct  8 22:40:14 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 09 Oct 2013 00:40:14 +0200
Subject: [master] ef0666a Totally overhaul the EXP code.
Message-ID: <E1VTfws-0002Ai-Lw@project.varnish-software.com>

commit ef0666a099528e336f60044bff3036a6ffef2386
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 8 22:36:50 2013 +0000

    Totally overhaul the EXP code.
    
    In general new/changed/killed objects are posted in the exp_thread
    mailbox, and suitably handled by the exp_thread at its convenience.
    
    The binheap is only manipulated by exp_thread, and it doesn't
    need to hold any locks to do so.
    
    The exp_mtx only protects the inbox to the exp_thread.
    
    oc->flags and the lru-list are protected by the lru->mtx.
    
    EXP_NukeLRU() is temporarily disabled.
    
    When LRU-Nuking to make space, we mark the object as dying, so HSH
    will ignore it and steal the storage from it right away.  We post the
    rest of the object to exp_thread to clean up later.
    
    The LRU change makes the 3-4 LRU test-cases unstable, (c00044, c00045,
    r01140...) because they were used to the full object being reaped,
    rather than only the storage segments.  I'm to tired to fix/tune those
    cases tonight.
    
    But apart from that, it seems to run the test-cases.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 5f68e22..12118fd 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -437,8 +437,11 @@ struct objcore {
 #define OC_F_OFFLRU		(1<<4)
 #define OC_F_PRIV		(1<<5)		/* Stevedore private flag */
 #define OC_F_LURK		(3<<6)		/* Ban-lurker-color */
+#define OC_F_DYING		(1<<7)
 #define OC_F_PRIVATE		(1<<8)
 #define OC_F_FAILED		(1<<9)
+#define OC_F_MOVE		(1<<10)
+#define OC_F_INSERT		(1<<11)
 	unsigned		timer_idx;
 	VTAILQ_ENTRY(objcore)	list;
 	VTAILQ_ENTRY(objcore)	lru_list;
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index a340af9..92565df 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -28,13 +28,6 @@
  *
  * LRU and object timer handling.
  *
- * We have two data structures, a LRU-list and a binary heap for the timers
- * and two ways to kill objects: TTL-timeouts and LRU cleanups.
- *
- * Any object on the LRU is also on the binheap and vice versa.
- *
- * We hold a single object reference for both data structures.
- *
  */
 
 #include "config.h"
@@ -52,6 +45,10 @@ struct exp_priv {
 	unsigned			magic;
 #define EXP_PRIV_MAGIC			0x9db22482
 	struct lock			mtx;
+
+	struct worker			*wrk;
+	struct vsl_log			vsl;
+
 	VTAILQ_HEAD(,objcore)		inbox;
 	struct binheap			*heap;
 	pthread_cond_t			condvar;
@@ -90,26 +87,19 @@ EXP_Ttl(const struct req *req, const struct object *o)
 }
 
 /*--------------------------------------------------------------------
- * When & why does the timer fire for this object ?
+ * Calculate when we should wake up for this object
  */
 
-static int
-update_object_when(const struct object *o)
+static double
+exp_when(const struct object *o)
 {
-	struct objcore *oc;
 	double when;
 
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	oc = o->objcore;
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	Lck_AssertHeld(&exphdl->mtx);
 
 	when = o->exp.t_origin + o->exp.ttl + o->exp.grace + o->exp.keep;
 	assert(!isnan(when));
-	if (when == oc->timer_when)
-		return (0);
-	oc->timer_when = when;
-	return (1);
+	return (when);
 }
 
 /*--------------------------------------------------------------------
@@ -125,13 +115,16 @@ EXP_Inject(struct objcore *oc, struct lru *lru, double when)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 
+	AZ(oc->flags & OC_F_OFFLRU);
+
 	Lck_Lock(&lru->mtx);
 	lru->n_objcore++;
+	oc->flags |= OC_F_OFFLRU | OC_F_INSERT;
 	Lck_Unlock(&lru->mtx);
 
-	Lck_Lock(&exphdl->mtx);
 	oc->timer_when = when;
-	oc->flags |= OC_F_OFFLRU;
+
+	Lck_Lock(&exphdl->mtx);
 	VTAILQ_INSERT_TAIL(&exphdl->inbox, oc, lru_list);
 	AZ(pthread_cond_signal(&exphdl->condvar));
 	Lck_Unlock(&exphdl->mtx);
@@ -158,20 +151,23 @@ EXP_Insert(const struct object *o, double now)
 	assert(o->exp.t_origin != 0 && !isnan(o->exp.t_origin));
 	oc->last_lru = now;
 
+	AZ(oc->flags & OC_F_OFFLRU);
+
 	lru = oc_getlru(oc);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+
 	Lck_Lock(&lru->mtx);
 	lru->n_objcore++;
+	oc->flags |= OC_F_OFFLRU | OC_F_INSERT;
 	Lck_Unlock(&lru->mtx);
 
+	oc->timer_when = exp_when(o);
+	oc_updatemeta(oc);
+
 	Lck_Lock(&exphdl->mtx);
-	(void)update_object_when(o);
-	oc->flags |= OC_F_OFFLRU;
 	VTAILQ_INSERT_TAIL(&exphdl->inbox, oc, lru_list);
 	AZ(pthread_cond_signal(&exphdl->condvar));
 	Lck_Unlock(&exphdl->mtx);
-
-	oc_updatemeta(oc);
 }
 
 /*--------------------------------------------------------------------
@@ -199,18 +195,11 @@ EXP_Touch(struct objcore *oc)
 	if (lru->flags & LRU_F_DONTMOVE)
 		return (0);
 
-	/*
-	 * We only need the LRU lock here.  The locking order is LRU->EXP
-	 * so we can trust the content of the oc->timer_idx without the
-	 * EXP lock.   Since each lru list has its own lock, this should
-	 * reduce contention a fair bit
-	 */
 	if (Lck_Trylock(&lru->mtx))
 		return (0);
 
-	if (oc->flags & OC_F_OFFLRU) {
-		/* Cannot move it while it's in the inbox */
-	} else if (oc->timer_idx != BINHEAP_NOIDX) {
+	if (!(oc->flags & OC_F_OFFLRU)) {
+		/* Can only it while it's actually on the LRU list */
 		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
 		VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
 		VSC_C_main->n_lru_moved++;
@@ -220,12 +209,8 @@ EXP_Touch(struct objcore *oc)
 }
 
 /*--------------------------------------------------------------------
- * We have changed one or more of the object timers, shuffle it
- * accordingly in the binheap
+ * We have changed one or more of the object timers, tell the exp_thread
  *
- * The VCL code can send us here on a non-cached object, just return.
- *
- * XXX: special case check for ttl = 0 ?
  */
 
 void
@@ -233,28 +218,173 @@ EXP_Rearm(const struct object *o)
 {
 	struct objcore *oc;
 	struct lru *lru;
+	double when;
 
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	oc = o->objcore;
 	if (oc == NULL)
 		return;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	when = exp_when(o);
+
+	VSL(SLT_ExpKill, 0, "EXP_Rearm %p %.9f %.9f 0x%x", oc,
+	    oc->timer_when, when, oc->flags);
+
+	if (oc->timer_when == when)
+		return;
+
 	lru = oc_getlru(oc);
+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+
 	Lck_Lock(&lru->mtx);
-	Lck_Lock(&exphdl->mtx);
+
+	if (when < 0)
+		oc->flags |= OC_F_DYING;
+	else
+		oc->flags |= OC_F_MOVE;
+
+	if (oc->flags & OC_F_OFFLRU) {
+		oc = NULL;
+	} else {
+		oc->flags |= OC_F_OFFLRU;
+		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
+	}
+	Lck_Unlock(&lru->mtx);
+
+	if (oc != NULL) {
+		Lck_Lock(&exphdl->mtx);
+		if (oc->flags & OC_F_DYING)
+			VTAILQ_INSERT_HEAD(&exphdl->inbox, oc, lru_list);
+		else
+			VTAILQ_INSERT_TAIL(&exphdl->inbox, oc, lru_list);
+		AZ(pthread_cond_signal(&exphdl->condvar));
+		Lck_Unlock(&exphdl->mtx);
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Handle stuff in the inbox
+ */
+
+static void
+exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
+{
+	unsigned flags;
+	struct lru *lru;
+	struct object *o;
+
+	CHECK_OBJ_NOTNULL(ep, EXP_PRIV_MAGIC);
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	VSLb(&ep->vsl, SLT_ExpKill, "EXP_INBOX %p %.9f 0x%x", oc,
+	    oc->timer_when, oc->flags);
+	if (oc->flags & OC_F_DYING) {
+		/* Remove from binheap */
+		assert(oc->timer_idx != BINHEAP_NOIDX);
+		binheap_delete(ep->heap, oc->timer_idx);
+		assert(oc->timer_idx == BINHEAP_NOIDX);
+		(void)HSH_DerefObjCore(&ep->wrk->stats, &oc);
+		return;
+	}
+
+	lru = oc_getlru(oc);
+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+
+	Lck_Lock(&lru->mtx);
+	flags = oc->flags;
+	AN(flags & OC_F_OFFLRU);
+	oc->flags &= ~(OC_F_INSERT | OC_F_MOVE | OC_F_OFFLRU);
+	oc->last_lru = now;
+	VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
+	Lck_Unlock(&lru->mtx);
+
+	if (flags & OC_F_MOVE) {
+		o = oc_getobj(&ep->wrk->stats, oc);
+		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+		oc->timer_when = exp_when(o);
+		oc_updatemeta(oc);
+	}
+
+	VSLb(&ep->vsl, SLT_ExpKill, "EXP_WHEN %p %.9f 0x%x", oc,
+	    oc->timer_when, oc->flags);
+
 	/*
-	 * The hang-man might have this object of the binheap while
-	 * tending to a timer.  If so, we do not muck with it here.
+	 * XXX: There are some pathological cases here, were we
+	 * XXX: insert or move an expired object, only to find out
+	 * XXX: the next moment and rip them out again.
 	 */
-	if (oc->timer_idx != BINHEAP_NOIDX && update_object_when(o)) {
+
+	if (flags & OC_F_INSERT) {
+		assert(oc->timer_idx == BINHEAP_NOIDX);
+		binheap_insert(exphdl->heap, oc);
+		assert(oc->timer_idx != BINHEAP_NOIDX);
+	} else if (flags & OC_F_MOVE) {
 		assert(oc->timer_idx != BINHEAP_NOIDX);
 		binheap_reorder(exphdl->heap, oc->timer_idx);
 		assert(oc->timer_idx != BINHEAP_NOIDX);
-		AZ(pthread_cond_signal(&exphdl->condvar)); // XXX
+	} else {
+		WRONG("Objcore state wrong in inbox");
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Expire stuff from the binheap
+ */
+
+static double
+exp_expire(struct exp_priv *ep, double now)
+{
+	struct lru *lru;
+	struct objcore *oc;
+	struct object *o;
+
+	CHECK_OBJ_NOTNULL(ep, EXP_PRIV_MAGIC);
+
+	oc = binheap_root(ep->heap);
+	if (oc == NULL)
+		return (now + 1.0);
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	/* Ready ? */
+	if (oc->timer_when > now)
+		return (oc->timer_when);
+
+	/* If the object is busy, we have to wait for it */
+	if (oc->flags & OC_F_BUSY)
+		return (now + 0.01);		// XXX ?
+
+	lru = oc_getlru(oc);
+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+	Lck_Lock(&lru->mtx);
+	oc->flags |= OC_F_DYING;
+	if (oc->flags & OC_F_OFFLRU)
+		oc = NULL;
+	else {
+		oc->flags |= OC_F_OFFLRU;
+		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
 	}
-	Lck_Unlock(&exphdl->mtx);
 	Lck_Unlock(&lru->mtx);
-	oc_updatemeta(oc);
+
+	if (oc == NULL)
+		return (now + 1e-3);		// XXX ?
+
+	/* Remove from binheap */
+	assert(oc->timer_idx != BINHEAP_NOIDX);
+	binheap_delete(ep->heap, oc->timer_idx);
+	assert(oc->timer_idx == BINHEAP_NOIDX);
+
+	VSC_C_main->n_expired++;
+
+	CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
+	o = oc_getobj(&ep->wrk->stats, oc);
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	VSLb(&ep->vsl, SLT_ExpKill, "%u %.0f",
+	    oc_getxid(&ep->wrk->stats, oc) & VSL_IDENTMASK,
+	    EXP_Ttl(NULL, o) - now);
+	(void)HSH_DerefObjCore(&ep->wrk->stats, &oc);
+	return (0);
 }
 
 /*--------------------------------------------------------------------
@@ -263,122 +393,39 @@ EXP_Rearm(const struct object *o)
  */
 
 static void * __match_proto__(bgthread_t)
-exp_timer(struct worker *wrk, void *priv)
+exp_thread(struct worker *wrk, void *priv)
 {
 	struct objcore *oc;
-	struct lru *lru;
-	double t;
-	struct object *o;
-	struct vsl_log vsl;
+	double t, tnext;
 	struct exp_priv *ep;
 	struct timespec ts;
-	int idle;
 
 	CAST_OBJ_NOTNULL(ep, priv, EXP_PRIV_MAGIC);
-	VSL_Setup(&vsl, NULL, 0);
-	t = VTIM_real();
-	oc = NULL;
-	idle = 0;
+	ep->wrk = wrk;
+	VSL_Setup(&ep->vsl, NULL, 0);
+	tnext = 0;
 	while (1) {
-		Lck_Lock(&ep->mtx);
-
-		if (idle) {
-			VSL_Flush(&vsl, 0);
-			WRK_SumStat(wrk);
-			oc = binheap_root(ep->heap);
-			if (oc != NULL && oc->timer_when > 0.0) {
-				ts.tv_nsec = modf(oc->timer_when, &t) * 1e9;
-				ts.tv_sec = t;
-				(void)Lck_CondWait(&ep->condvar, &ep->mtx, &ts);
-			} else if (oc == NULL) {
-				(void)Lck_CondWait(&ep->condvar, &ep->mtx,NULL);
-			} else {
-				/* We're behind, don't sleep */
-			}
-			t = VTIM_real();
-		}
 
+		Lck_Lock(&ep->mtx);
 		oc = VTAILQ_FIRST(&ep->inbox);
 		if (oc != NULL) {
 			VTAILQ_REMOVE(&ep->inbox, oc, lru_list);
-			assert(oc->timer_idx == BINHEAP_NOIDX);
-			binheap_insert(exphdl->heap, oc);
-			assert(oc->timer_idx != BINHEAP_NOIDX);
-			Lck_Unlock(&ep->mtx);
-
-			lru = oc_getlru(oc);
-			Lck_Lock(&lru->mtx);
-			VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
-			oc->flags &= ~OC_F_OFFLRU;
-			oc->last_lru = t;
-			Lck_Unlock(&lru->mtx);
-			idle = 0;
-			continue;
-		}
-
-		oc = binheap_root(ep->heap);
-		if (oc == NULL) {
-			Lck_Unlock(&ep->mtx);
-			idle = 1;
-			continue;
-		}
-		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-
-		/*
-		 * We may have expired so many objects that our timestamp
-		 * got out of date, refresh it and check again.
-		 */
-		if (oc->timer_when > t)
-			t = VTIM_real();
-		if (oc->timer_when > t) {
-			Lck_Unlock(&ep->mtx);
-			idle = 1;
-			continue;
-		}
-
-		/* If the object is busy, we have to wait for it */
-		if (oc->flags & OC_F_BUSY) {
-			Lck_Unlock(&ep->mtx);
-			idle = 1;
-			continue;
-		}
-
-		/*
-		 * It's time...
-		 * Technically we should drop the ep->mtx, get the lru->mtx
-		 * get the ep->mtx again and then check that the oc is still
-		 * on the binheap.  We take the shorter route and try to
-		 * get the lru->mtx and punt if we fail.
-		 */
-
-		lru = oc_getlru(oc);
-		CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
-		if (Lck_Trylock(&lru->mtx)) {
-			Lck_Unlock(&ep->mtx);
-			idle = 1;
-			continue;
+		} else if (tnext > 0) {
+			VSL_Flush(&ep->vsl, 0);
+			WRK_SumStat(wrk);
+			ts.tv_nsec = (long)(modf(tnext, &t) * 1e9);
+			ts.tv_sec = (long)t;
+			(void)Lck_CondWait(&ep->condvar, &ep->mtx, &ts);
 		}
-
-		/* Remove from binheap */
-		assert(oc->timer_idx != BINHEAP_NOIDX);
-		binheap_delete(ep->heap, oc->timer_idx);
-		assert(oc->timer_idx == BINHEAP_NOIDX);
-
-		/* And from LRU */
-		lru = oc_getlru(oc);
-		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
-
 		Lck_Unlock(&ep->mtx);
-		Lck_Unlock(&lru->mtx);
 
-		VSC_C_main->n_expired++;
+		t = VTIM_real();
 
-		CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
-		o = oc_getobj(&wrk->stats, oc);
-		VSLb(&vsl, SLT_ExpKill, "%u %.0f",
-		    oc_getxid(&wrk->stats, oc) & VSL_IDENTMASK,
-		    EXP_Ttl(NULL, o) - t);
-		(void)HSH_DerefObjCore(&wrk->stats, &oc);
+		if (oc != NULL) {
+			exp_inbox(ep, oc, t);
+			tnext = 0;
+		} else
+			tnext = exp_expire(ep, t);
 	}
 	NEEDLESS_RETURN(NULL);
 }
@@ -393,36 +440,58 @@ int
 EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 {
 	struct objcore *oc;
+	struct objhead *oh;
+	struct object *o;
 
 	/* Find the first currently unused object on the LRU.  */
 	Lck_Lock(&lru->mtx);
-	Lck_Lock(&exphdl->mtx);
 	VTAILQ_FOREACH(oc, &lru->lru_head, lru_list) {
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-		assert(oc->timer_idx != BINHEAP_NOIDX);
+
+		AZ(oc->flags & OC_F_DYING);
+
 		/*
 		 * It wont release any space if we cannot release the last
 		 * reference, besides, if somebody else has a reference,
 		 * it's a bad idea to nuke this object anyway. Also do not
 		 * touch busy objects.
 		 */
-		if (oc->refcnt == 1 && !(oc->flags & OC_F_BUSY))
+		if (oc->flags & OC_F_BUSY)
+			continue;
+		if (oc->refcnt > 1)
+			continue;
+		oh = oc->objhead;
+		CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+		if (Lck_Trylock(&oh->mtx))
+			continue;
+		if (oc->refcnt == 1) {
+			oc->flags |= OC_F_DYING | OC_F_OFFLRU;
+			oc->refcnt++;
+			VSC_C_main->n_lru_nuked++; // XXX per lru ?
+			VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
+		}
+		Lck_Unlock(&oh->mtx);
+		if (oc->flags & OC_F_DYING)
 			break;
 	}
-	if (oc != NULL) {
-		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
-		binheap_delete(exphdl->heap, oc->timer_idx);
-		assert(oc->timer_idx == BINHEAP_NOIDX);
-		VSC_C_main->n_lru_nuked++;
-	}
-	Lck_Unlock(&exphdl->mtx);
 	Lck_Unlock(&lru->mtx);
 
-	if (oc == NULL)
+	if (oc == NULL) {
+		VSLb(bo->vsl, SLT_ExpKill, "LRU failed");
 		return (-1);
+	}
+
+	/* XXX: We could grab and return one storage segment to our caller */
+	o = oc_getobj(bo->stats, oc);
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	STV_Freestore(o);
+
+	Lck_Lock(&exphdl->mtx);
+	VTAILQ_INSERT_HEAD(&exphdl->inbox, oc, lru_list);
+	AZ(pthread_cond_signal(&exphdl->condvar));
+	Lck_Unlock(&exphdl->mtx);
 
-	/* XXX: bad idea for -spersistent */
-	VSLb(bo->vsl, SLT_ExpKill, "%u LRU",
+	VSLb(bo->vsl, SLT_ExpKill, "LRU %u",
 	    oc_getxid(bo->stats, oc) & VSL_IDENTMASK);
 	(void)HSH_DerefObjCore(bo->stats, &oc);
 	return (1);
@@ -432,6 +501,8 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
  * Nukes an entire LRU
  */
 
+#if 0		// Not yet
+
 #define NUKEBUF 10	/* XXX: Randomly chosen to be bigger than one */
 
 void
@@ -490,6 +561,8 @@ EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru)
 	WRK_SumStat(wrk);
 }
 
+#endif
+
 /*--------------------------------------------------------------------
  * BinHeap helper functions for objcore.
  */
@@ -532,5 +605,5 @@ EXP_Init(void)
 	AN(ep->heap);
 	VTAILQ_INIT(&ep->inbox);
 	exphdl = ep;
-	WRK_BgThread(&pt, "cache-timeout", exp_timer, ep);
+	WRK_BgThread(&pt, "cache-timeout", exp_thread, ep);
 }
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 9851584..ce98926 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -401,7 +401,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 		assert(oc->objhead == oh);
 
-		if (oc->flags & OC_F_FAILED)
+		if (oc->flags & (OC_F_FAILED | OC_F_DYING))
 			continue;
 
 		if (oc->flags & OC_F_BUSY || oc->busyobj != NULL) {



From phk at varnish-cache.org  Wed Oct  9 09:36:52 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 09 Oct 2013 11:36:52 +0200
Subject: [master] 18fb08c Cleanup & Polishing
Message-ID: <E1VTqCK-0002Zq-GI@project.varnish-software.com>

commit 18fb08cd2889e76375d3dd95672e3a8fa8033c2a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 9 09:36:09 2013 +0000

    Cleanup & Polishing

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 92565df..381266f 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -103,6 +103,25 @@ exp_when(const struct object *o)
 }
 
 /*--------------------------------------------------------------------
+ * Post an objcore to the exp_thread's inbox.
+ */
+
+static void
+exp_mail_it(struct objcore *oc)
+{
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	AN(oc->flags & OC_F_OFFLRU);
+	Lck_Lock(&exphdl->mtx);
+	if (oc->flags & OC_F_DYING)
+		VTAILQ_INSERT_HEAD(&exphdl->inbox, oc, lru_list);
+	else
+		VTAILQ_INSERT_TAIL(&exphdl->inbox, oc, lru_list);
+	AZ(pthread_cond_signal(&exphdl->condvar));
+	Lck_Unlock(&exphdl->mtx);
+}
+
+/*--------------------------------------------------------------------
  * Object has been added to cache, record in lru & binheap.
  *
  * The objcore comes with a reference, which we inherit.
@@ -124,10 +143,7 @@ EXP_Inject(struct objcore *oc, struct lru *lru, double when)
 
 	oc->timer_when = when;
 
-	Lck_Lock(&exphdl->mtx);
-	VTAILQ_INSERT_TAIL(&exphdl->inbox, oc, lru_list);
-	AZ(pthread_cond_signal(&exphdl->condvar));
-	Lck_Unlock(&exphdl->mtx);
+	exp_mail_it(oc);
 }
 
 /*--------------------------------------------------------------------
@@ -164,10 +180,7 @@ EXP_Insert(const struct object *o, double now)
 	oc->timer_when = exp_when(o);
 	oc_updatemeta(oc);
 
-	Lck_Lock(&exphdl->mtx);
-	VTAILQ_INSERT_TAIL(&exphdl->inbox, oc, lru_list);
-	AZ(pthread_cond_signal(&exphdl->condvar));
-	Lck_Unlock(&exphdl->mtx);
+	exp_mail_it(oc);
 }
 
 /*--------------------------------------------------------------------
@@ -252,182 +265,8 @@ EXP_Rearm(const struct object *o)
 	}
 	Lck_Unlock(&lru->mtx);
 
-	if (oc != NULL) {
-		Lck_Lock(&exphdl->mtx);
-		if (oc->flags & OC_F_DYING)
-			VTAILQ_INSERT_HEAD(&exphdl->inbox, oc, lru_list);
-		else
-			VTAILQ_INSERT_TAIL(&exphdl->inbox, oc, lru_list);
-		AZ(pthread_cond_signal(&exphdl->condvar));
-		Lck_Unlock(&exphdl->mtx);
-	}
-}
-
-/*--------------------------------------------------------------------
- * Handle stuff in the inbox
- */
-
-static void
-exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
-{
-	unsigned flags;
-	struct lru *lru;
-	struct object *o;
-
-	CHECK_OBJ_NOTNULL(ep, EXP_PRIV_MAGIC);
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-
-	VSLb(&ep->vsl, SLT_ExpKill, "EXP_INBOX %p %.9f 0x%x", oc,
-	    oc->timer_when, oc->flags);
-	if (oc->flags & OC_F_DYING) {
-		/* Remove from binheap */
-		assert(oc->timer_idx != BINHEAP_NOIDX);
-		binheap_delete(ep->heap, oc->timer_idx);
-		assert(oc->timer_idx == BINHEAP_NOIDX);
-		(void)HSH_DerefObjCore(&ep->wrk->stats, &oc);
-		return;
-	}
-
-	lru = oc_getlru(oc);
-	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
-
-	Lck_Lock(&lru->mtx);
-	flags = oc->flags;
-	AN(flags & OC_F_OFFLRU);
-	oc->flags &= ~(OC_F_INSERT | OC_F_MOVE | OC_F_OFFLRU);
-	oc->last_lru = now;
-	VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
-	Lck_Unlock(&lru->mtx);
-
-	if (flags & OC_F_MOVE) {
-		o = oc_getobj(&ep->wrk->stats, oc);
-		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-		oc->timer_when = exp_when(o);
-		oc_updatemeta(oc);
-	}
-
-	VSLb(&ep->vsl, SLT_ExpKill, "EXP_WHEN %p %.9f 0x%x", oc,
-	    oc->timer_when, oc->flags);
-
-	/*
-	 * XXX: There are some pathological cases here, were we
-	 * XXX: insert or move an expired object, only to find out
-	 * XXX: the next moment and rip them out again.
-	 */
-
-	if (flags & OC_F_INSERT) {
-		assert(oc->timer_idx == BINHEAP_NOIDX);
-		binheap_insert(exphdl->heap, oc);
-		assert(oc->timer_idx != BINHEAP_NOIDX);
-	} else if (flags & OC_F_MOVE) {
-		assert(oc->timer_idx != BINHEAP_NOIDX);
-		binheap_reorder(exphdl->heap, oc->timer_idx);
-		assert(oc->timer_idx != BINHEAP_NOIDX);
-	} else {
-		WRONG("Objcore state wrong in inbox");
-	}
-}
-
-/*--------------------------------------------------------------------
- * Expire stuff from the binheap
- */
-
-static double
-exp_expire(struct exp_priv *ep, double now)
-{
-	struct lru *lru;
-	struct objcore *oc;
-	struct object *o;
-
-	CHECK_OBJ_NOTNULL(ep, EXP_PRIV_MAGIC);
-
-	oc = binheap_root(ep->heap);
-	if (oc == NULL)
-		return (now + 1.0);
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-
-	/* Ready ? */
-	if (oc->timer_when > now)
-		return (oc->timer_when);
-
-	/* If the object is busy, we have to wait for it */
-	if (oc->flags & OC_F_BUSY)
-		return (now + 0.01);		// XXX ?
-
-	lru = oc_getlru(oc);
-	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
-	Lck_Lock(&lru->mtx);
-	oc->flags |= OC_F_DYING;
-	if (oc->flags & OC_F_OFFLRU)
-		oc = NULL;
-	else {
-		oc->flags |= OC_F_OFFLRU;
-		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
-	}
-	Lck_Unlock(&lru->mtx);
-
-	if (oc == NULL)
-		return (now + 1e-3);		// XXX ?
-
-	/* Remove from binheap */
-	assert(oc->timer_idx != BINHEAP_NOIDX);
-	binheap_delete(ep->heap, oc->timer_idx);
-	assert(oc->timer_idx == BINHEAP_NOIDX);
-
-	VSC_C_main->n_expired++;
-
-	CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
-	o = oc_getobj(&ep->wrk->stats, oc);
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	VSLb(&ep->vsl, SLT_ExpKill, "%u %.0f",
-	    oc_getxid(&ep->wrk->stats, oc) & VSL_IDENTMASK,
-	    EXP_Ttl(NULL, o) - now);
-	(void)HSH_DerefObjCore(&ep->wrk->stats, &oc);
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * This thread monitors the root of the binary heap and whenever an
- * object expires, accounting also for graceability, it is killed.
- */
-
-static void * __match_proto__(bgthread_t)
-exp_thread(struct worker *wrk, void *priv)
-{
-	struct objcore *oc;
-	double t, tnext;
-	struct exp_priv *ep;
-	struct timespec ts;
-
-	CAST_OBJ_NOTNULL(ep, priv, EXP_PRIV_MAGIC);
-	ep->wrk = wrk;
-	VSL_Setup(&ep->vsl, NULL, 0);
-	tnext = 0;
-	while (1) {
-
-		Lck_Lock(&ep->mtx);
-		oc = VTAILQ_FIRST(&ep->inbox);
-		if (oc != NULL) {
-			VTAILQ_REMOVE(&ep->inbox, oc, lru_list);
-		} else if (tnext > 0) {
-			VSL_Flush(&ep->vsl, 0);
-			WRK_SumStat(wrk);
-			ts.tv_nsec = (long)(modf(tnext, &t) * 1e9);
-			ts.tv_sec = (long)t;
-			(void)Lck_CondWait(&ep->condvar, &ep->mtx, &ts);
-		}
-		Lck_Unlock(&ep->mtx);
-
-		t = VTIM_real();
-
-		if (oc != NULL) {
-			exp_inbox(ep, oc, t);
-			tnext = 0;
-		} else
-			tnext = exp_expire(ep, t);
-	}
-	NEEDLESS_RETURN(NULL);
+	if (oc != NULL)
+		exp_mail_it(oc);
 }
 
 /*--------------------------------------------------------------------
@@ -486,10 +325,7 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	STV_Freestore(o);
 
-	Lck_Lock(&exphdl->mtx);
-	VTAILQ_INSERT_HEAD(&exphdl->inbox, oc, lru_list);
-	AZ(pthread_cond_signal(&exphdl->condvar));
-	Lck_Unlock(&exphdl->mtx);
+	exp_mail_it(oc);
 
 	VSLb(bo->vsl, SLT_ExpKill, "LRU %u",
 	    oc_getxid(bo->stats, oc) & VSL_IDENTMASK);
@@ -563,8 +399,136 @@ EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru)
 
 #endif
 
+
+/*--------------------------------------------------------------------
+ * Handle stuff in the inbox
+ */
+
+static void
+exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
+{
+	unsigned flags;
+	struct lru *lru;
+	struct object *o;
+
+	CHECK_OBJ_NOTNULL(ep, EXP_PRIV_MAGIC);
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	VSLb(&ep->vsl, SLT_ExpKill, "EXP_INBOX %p %.9f 0x%x", oc,
+	    oc->timer_when, oc->flags);
+
+	lru = oc_getlru(oc);
+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+
+	/* Evacuate our action-flags, and put it back on the LRU list */
+	Lck_Lock(&lru->mtx);
+	flags = oc->flags;
+	AN(flags & OC_F_OFFLRU);
+	oc->flags &= ~(OC_F_INSERT | OC_F_MOVE | OC_F_OFFLRU);
+	oc->last_lru = now;
+	if (!(flags & OC_F_DYING))
+		VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
+	Lck_Unlock(&lru->mtx);
+
+	if (flags & OC_F_DYING) {
+		assert(oc->timer_idx != BINHEAP_NOIDX);
+		binheap_delete(ep->heap, oc->timer_idx);
+		assert(oc->timer_idx == BINHEAP_NOIDX);
+		(void)HSH_DerefObjCore(&ep->wrk->stats, &oc);
+		return;
+	}
+
+	if (flags & OC_F_MOVE) {
+		o = oc_getobj(&ep->wrk->stats, oc);
+		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+		oc->timer_when = exp_when(o);
+		oc_updatemeta(oc);
+	}
+
+	VSLb(&ep->vsl, SLT_ExpKill, "EXP_WHEN %p %.9f 0x%x", oc,
+	    oc->timer_when, oc->flags);
+
+	/*
+	 * XXX: There are some pathological cases here, were we
+	 * XXX: insert or move an expired object, only to find out
+	 * XXX: the next moment and rip them out again.
+	 */
+
+	if (flags & OC_F_INSERT) {
+		assert(oc->timer_idx == BINHEAP_NOIDX);
+		binheap_insert(exphdl->heap, oc);
+		assert(oc->timer_idx != BINHEAP_NOIDX);
+	} else if (flags & OC_F_MOVE) {
+		assert(oc->timer_idx != BINHEAP_NOIDX);
+		binheap_reorder(exphdl->heap, oc->timer_idx);
+		assert(oc->timer_idx != BINHEAP_NOIDX);
+	} else {
+		WRONG("Objcore state wrong in inbox");
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Expire stuff from the binheap
+ */
+
+static double
+exp_expire(struct exp_priv *ep, double now)
+{
+	struct lru *lru;
+	struct objcore *oc;
+	struct object *o;
+
+	CHECK_OBJ_NOTNULL(ep, EXP_PRIV_MAGIC);
+
+	oc = binheap_root(ep->heap);
+	if (oc == NULL)
+		return (now + 355./113.);
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	/* Ready ? */
+	if (oc->timer_when > now)
+		return (oc->timer_when);
+
+	/* If the object is busy, we have to wait for it */
+	if (oc->flags & OC_F_BUSY)
+		return (now + 0.01);		// XXX ?
+
+	VSC_C_main->n_expired++;
+
+	lru = oc_getlru(oc);
+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+	Lck_Lock(&lru->mtx);
+	oc->flags |= OC_F_DYING;
+	if (oc->flags & OC_F_OFFLRU)
+		oc = NULL;
+	else {
+		oc->flags |= OC_F_OFFLRU;
+		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
+	}
+	Lck_Unlock(&lru->mtx);
+
+	if (oc == NULL)
+		return (now + 1e-3);		// XXX ?
+
+	/* Remove from binheap */
+	assert(oc->timer_idx != BINHEAP_NOIDX);
+	binheap_delete(ep->heap, oc->timer_idx);
+	assert(oc->timer_idx == BINHEAP_NOIDX);
+
+	CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
+	o = oc_getobj(&ep->wrk->stats, oc);
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	VSLb(&ep->vsl, SLT_ExpKill, "%u %.0f",
+	    oc_getxid(&ep->wrk->stats, oc) & VSL_IDENTMASK,
+	    EXP_Ttl(NULL, o) - now);
+	(void)HSH_DerefObjCore(&ep->wrk->stats, &oc);
+	return (0);
+}
+
 /*--------------------------------------------------------------------
- * BinHeap helper functions for objcore.
+ * This thread monitors the root of the binary heap and whenever an
+ * object expires, accounting also for graceability, it is killed.
  */
 
 static int
@@ -588,6 +552,45 @@ object_update(void *priv, void *p, unsigned u)
 	oc->timer_idx = u;
 }
 
+static void * __match_proto__(bgthread_t)
+exp_thread(struct worker *wrk, void *priv)
+{
+	struct objcore *oc;
+	double t = 0, tnext = 0;
+	struct exp_priv *ep;
+	struct timespec ts;
+
+	CAST_OBJ_NOTNULL(ep, priv, EXP_PRIV_MAGIC);
+	ep->wrk = wrk;
+	VSL_Setup(&ep->vsl, NULL, 0);
+	ep->heap = binheap_new(NULL, object_cmp, object_update);
+	AN(ep->heap);
+	while (1) {
+
+		Lck_Lock(&ep->mtx);
+		oc = VTAILQ_FIRST(&ep->inbox);
+		if (oc != NULL) {
+			VTAILQ_REMOVE(&ep->inbox, oc, lru_list);
+			tnext = 0;
+		} else if (tnext > t) {
+			VSL_Flush(&ep->vsl, 0);
+			WRK_SumStat(wrk);
+			ts.tv_nsec = (long)(modf(tnext, &t) * 1e9);
+			ts.tv_sec = (long)t;
+			(void)Lck_CondWait(&ep->condvar, &ep->mtx, &ts);
+		}
+		Lck_Unlock(&ep->mtx);
+
+		t = VTIM_real();
+
+		if (oc != NULL)
+			exp_inbox(ep, oc, t);
+		else
+			tnext = exp_expire(ep, t);
+	}
+	NEEDLESS_RETURN(NULL);
+}
+
 /*--------------------------------------------------------------------*/
 
 void
@@ -601,8 +604,6 @@ EXP_Init(void)
 
 	Lck_New(&ep->mtx, lck_exp);
 	AZ(pthread_cond_init(&ep->condvar, NULL));
-	ep->heap = binheap_new(NULL, object_cmp, object_update);
-	AN(ep->heap);
 	VTAILQ_INIT(&ep->inbox);
 	exphdl = ep;
 	WRK_BgThread(&pt, "cache-timeout", exp_thread, ep);



From phk at varnish-cache.org  Wed Oct  9 10:55:51 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 09 Oct 2013 12:55:51 +0200
Subject: [master] 9f39566 More polishing
Message-ID: <E1VTrQl-0003AN-C9@project.varnish-software.com>

commit 9f39566cf0bbbc1636d203da8d726664e7720488
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 9 10:55:38 2013 +0000

    More polishing

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 12118fd..5c249f5 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -442,6 +442,7 @@ struct objcore {
 #define OC_F_FAILED		(1<<9)
 #define OC_F_MOVE		(1<<10)
 #define OC_F_INSERT		(1<<11)
+#define OC_F_EXP		(1<<12)
 	unsigned		timer_idx;
 	VTAILQ_ENTRY(objcore)	list;
 	VTAILQ_ENTRY(objcore)	lru_list;
@@ -905,7 +906,7 @@ extern pthread_t cli_thread;
 void EXP_Clr(struct exp *e);
 
 double EXP_Ttl(const struct req *, const struct object*);
-void EXP_Insert(const struct object *o, double now);
+void EXP_Insert(struct objcore *oc);
 void EXP_Inject(struct objcore *oc, struct lru *lru, double when);
 void EXP_Init(void);
 void EXP_Rearm(const struct object *o);
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 381266f..dced5bb 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -122,9 +122,10 @@ exp_mail_it(struct objcore *oc)
 }
 
 /*--------------------------------------------------------------------
- * Object has been added to cache, record in lru & binheap.
+ * Inject an object with a reference into the lru/binheap.
  *
- * The objcore comes with a reference, which we inherit.
+ * This can either come from a stevedore (persistent) during startup
+ * or from EXP_Insert() below.
  */
 
 void
@@ -132,55 +133,40 @@ EXP_Inject(struct objcore *oc, struct lru *lru, double when)
 {
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 
 	AZ(oc->flags & OC_F_OFFLRU);
+	AZ(oc->flags & OC_F_BUSY);
+
+	if (lru == NULL)
+		lru = oc_getlru(oc);
+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 
 	Lck_Lock(&lru->mtx);
 	lru->n_objcore++;
-	oc->flags |= OC_F_OFFLRU | OC_F_INSERT;
+	oc->flags |= OC_F_OFFLRU | OC_F_INSERT | OC_F_EXP;
+	if (when < 0)
+		oc->flags |= OC_F_MOVE;
+	else
+		oc->timer_when = when;
 	Lck_Unlock(&lru->mtx);
 
-	oc->timer_when = when;
-
 	exp_mail_it(oc);
 }
 
 /*--------------------------------------------------------------------
- * Object has been added to cache, record in lru & binheap.
+ * Insert new object.
  *
  * We grab a reference to the object, which will keep it around until
  * we decide its time to let it go.
  */
 
 void
-EXP_Insert(const struct object *o, double now)
+EXP_Insert(struct objcore *oc)
 {
-	struct objcore *oc;
-	struct lru *lru;
 
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	oc = o->objcore;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	HSH_Ref(oc);
-
-	assert(o->exp.t_origin != 0 && !isnan(o->exp.t_origin));
-	oc->last_lru = now;
-
-	AZ(oc->flags & OC_F_OFFLRU);
-
-	lru = oc_getlru(oc);
-	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
-
-	Lck_Lock(&lru->mtx);
-	lru->n_objcore++;
-	oc->flags |= OC_F_OFFLRU | OC_F_INSERT;
-	Lck_Unlock(&lru->mtx);
-
-	oc->timer_when = exp_when(o);
-	oc_updatemeta(oc);
-
-	exp_mail_it(oc);
+	EXP_Inject(oc, NULL, -1);
 }
 
 /*--------------------------------------------------------------------
@@ -211,6 +197,8 @@ EXP_Touch(struct objcore *oc)
 	if (Lck_Trylock(&lru->mtx))
 		return (0);
 
+	AN(oc->flags & OC_F_EXP);
+
 	if (!(oc->flags & OC_F_OFFLRU)) {
 		/* Can only it while it's actually on the LRU list */
 		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
@@ -251,6 +239,7 @@ EXP_Rearm(const struct object *o)
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 
 	Lck_Lock(&lru->mtx);
+	AN(oc->flags & OC_F_EXP);
 
 	if (when < 0)
 		oc->flags |= OC_F_DYING;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index d65d181..3d75780 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -421,14 +421,14 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	assert(bo->refcount >= 1);
 
-	if (!(bo->fetch_obj->objcore->flags & OC_F_PRIVATE)) {
-		EXP_Insert(obj, bo->t_fetch);
-		AN(obj->objcore->ban);
-	}
-
 	AZ(bo->ws_o->overflow);
-	if (bo->do_stream)
+	if (bo->do_stream) {
 		HSH_Unbusy(&wrk->stats, obj->objcore);
+		if (!(obj->objcore->flags & OC_F_PRIVATE)) {
+			EXP_Insert(obj->objcore);
+			AN(obj->objcore->ban);
+		}
+	}
 
 	if (bo->vfp == NULL)
 		bo->vfp = &VFP_nop;
@@ -437,8 +437,13 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	VBO_setstate(bo, BOS_FETCHING);
 
 	V1F_fetch_body(wrk, bo);
-	if (!bo->do_stream)
+	if (!bo->do_stream) {
 		HSH_Unbusy(&wrk->stats, obj->objcore);
+		if (!(bo->fetch_obj->objcore->flags & OC_F_PRIVATE)) {
+			EXP_Insert(obj->objcore);
+			AN(obj->objcore->ban);
+		}
+	}
 	HSH_Complete(obj->objcore);
 
 	assert(bo->refcount >= 1);
@@ -516,15 +521,15 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	http_CopyHome(obj->http);
 
 
-	if (!(bo->fetch_obj->objcore->flags & OC_F_PRIVATE)) {
-		EXP_Insert(obj, bo->t_fetch);
-		AN(obj->objcore->ban);
-	}
-
 	AZ(bo->ws_o->overflow);
 	VBO_setstate(bo, BOS_FETCHING);
 	HSH_Unbusy(&wrk->stats, obj->objcore);
 
+	if (!(obj->objcore->flags & OC_F_PRIVATE)) {
+		EXP_Insert(obj->objcore);
+		AN(obj->objcore->ban);
+	}
+
 	st = NULL;
 	al = 0;
 
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index b2627fd..e0b35d8 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -411,8 +411,6 @@ V1F_fetch_body(struct worker *wrk, struct busyobj *bo)
 	if (bo->state == BOS_FAILED) {
 		wrk->stats.fetch_failed++;
 		obj->len = 0;
-		EXP_Clr(&obj->exp);
-		EXP_Rearm(obj);
 	} else {
 		assert(bo->state == BOS_FETCHING);
 



From phk at varnish-cache.org  Wed Oct  9 11:03:11 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 09 Oct 2013 13:03:11 +0200
Subject: [master] a44b3a8 Don't tell EXP about failed fetches
Message-ID: <E1VTrXr-0003tm-FM@project.varnish-software.com>

commit a44b3a8165f3f6ea63f32cb04280bab2f5e50046
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 9 11:02:49 2013 +0000

    Don't tell EXP about failed fetches

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 3d75780..f5d7d8f 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -437,7 +437,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	VBO_setstate(bo, BOS_FETCHING);
 
 	V1F_fetch_body(wrk, bo);
-	if (!bo->do_stream) {
+	if (!bo->do_stream && bo->state != BOS_FAILED) {
 		HSH_Unbusy(&wrk->stats, obj->objcore);
 		if (!(bo->fetch_obj->objcore->flags & OC_F_PRIVATE)) {
 			EXP_Insert(obj->objcore);



From phk at varnish-cache.org  Wed Oct  9 11:41:09 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 09 Oct 2013 13:41:09 +0200
Subject: [master] fd238e4 Fix these two test-cases to work with the new LRU
 strategy.
Message-ID: <E1VTs8b-0007xS-8N@project.varnish-software.com>

commit fd238e4f2c6921020df51a26a432df73a0c8f3bb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 9 11:40:43 2013 +0000

    Fix these two test-cases to work with the new LRU strategy.
    
    Log a little bit more debugging information for now.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index dced5bb..99e915b 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -276,6 +276,9 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 	VTAILQ_FOREACH(oc, &lru->lru_head, lru_list) {
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
+		VSLb(bo->vsl, SLT_ExpKill, "LRU_Cand %p f=0x%x r=%d",
+		    oc, oc->flags, oc->refcnt);
+
 		AZ(oc->flags & OC_F_DYING);
 
 		/*
diff --git a/bin/varnishtest/tests/c00044.vtc b/bin/varnishtest/tests/c00044.vtc
index 6723a3a..f325ae8 100644
--- a/bin/varnishtest/tests/c00044.vtc
+++ b/bin/varnishtest/tests/c00044.vtc
@@ -3,16 +3,18 @@ varnishtest	"Object/LRU/Stevedores"
 server s1 {
 	rxreq
 	txresp -bodylen 1048190
+
 	rxreq
 	txresp -bodylen 1048191
+
 	rxreq
 	txresp -bodylen 1048192
 
 	rxreq
-	txresp -bodylen 1048193
+	txresp -bodylen 1047193
 
 	rxreq
-	txresp -bodylen 1048194
+	txresp -bodylen 1047194
 } -start
 
 varnish v1 -storage "-smalloc,1m -smalloc,1m, -smalloc,1m" -vcl+backend {
@@ -72,7 +74,7 @@ client c1 {
 	txreq -url /foo1
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048193
+	expect resp.bodylen == 1047193
 } -run
 
 varnish v1 -expect n_lru_nuked == 1
@@ -81,7 +83,7 @@ client c1 {
 	txreq -url /foo
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048194
+	expect resp.bodylen == 1047194
 } -run
 
 varnish v1 -expect n_lru_nuked == 2
diff --git a/bin/varnishtest/tests/c00045.vtc b/bin/varnishtest/tests/c00045.vtc
index aa2897e..bfb71c7 100644
--- a/bin/varnishtest/tests/c00045.vtc
+++ b/bin/varnishtest/tests/c00045.vtc
@@ -4,9 +4,9 @@ server s1 {
 	rxreq
 	txresp -bodylen 1048188
 	rxreq
-	txresp -bodylen 1048189
+	txresp -bodylen 1047189
 	rxreq
-	txresp -bodylen 1048190
+	txresp -bodylen 1047190
 } -start
 
 varnish v1 -storage "-smalloc,1m -smalloc,1m, -smalloc,1m" -vcl+backend {
@@ -36,13 +36,13 @@ client c1 {
 	txreq -url /bar
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048189
+	expect resp.bodylen == 1047189
 } -run
 
 varnish v1 -expect n_lru_nuked == 1
 varnish v1 -expect SMA.Transient.g_bytes == 0
 varnish v1 -expect SMA.s0.g_bytes > 1000000
-varnish v1 -expect SMA.s0.g_space < 170
+varnish v1 -expect SMA.s0.g_space < 1171
 varnish v1 -expect SMA.s1.g_bytes == 0
 varnish v1 -expect SMA.s1.g_space > 1000000
 varnish v1 -expect SMA.s2.g_bytes == 0
@@ -52,13 +52,13 @@ client c1 {
 	txreq -url /foo
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 1048190
+	expect resp.bodylen == 1047190
 } -run
 
 varnish v1 -expect n_lru_nuked == 2
 varnish v1 -expect SMA.Transient.g_bytes == 0
 varnish v1 -expect SMA.s0.g_bytes > 1000000
-varnish v1 -expect SMA.s0.g_space < 170
+varnish v1 -expect SMA.s0.g_space < 1172
 varnish v1 -expect SMA.s1.g_bytes == 0
 varnish v1 -expect SMA.s1.g_space > 1000000
 varnish v1 -expect SMA.s2.g_bytes == 0



From phk at varnish-cache.org  Wed Oct  9 13:14:24 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 09 Oct 2013 15:14:24 +0200
Subject: [master] 32ffec8 We don't have BUSY objects in EXP any more
Message-ID: <E1VTtaq-0000hZ-2O@project.varnish-software.com>

commit 32ffec8053c73e9a454e139969e5cb7f6ea5a05d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 9 13:14:14 2013 +0000

    We don't have BUSY objects in EXP any more

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 99e915b..794f6c7 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -284,11 +284,8 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 		/*
 		 * It wont release any space if we cannot release the last
 		 * reference, besides, if somebody else has a reference,
-		 * it's a bad idea to nuke this object anyway. Also do not
-		 * touch busy objects.
+		 * it's a bad idea to nuke this object anyway. 
 		 */
-		if (oc->flags & OC_F_BUSY)
-			continue;
 		if (oc->refcnt > 1)
 			continue;
 		oh = oc->objhead;
@@ -409,6 +406,8 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 	VSLb(&ep->vsl, SLT_ExpKill, "EXP_INBOX %p %.9f 0x%x", oc,
 	    oc->timer_when, oc->flags);
 
+	AZ(oc->flags & OC_F_BUSY);
+
 	lru = oc_getlru(oc);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 
@@ -482,15 +481,12 @@ exp_expire(struct exp_priv *ep, double now)
 	if (oc->timer_when > now)
 		return (oc->timer_when);
 
-	/* If the object is busy, we have to wait for it */
-	if (oc->flags & OC_F_BUSY)
-		return (now + 0.01);		// XXX ?
-
 	VSC_C_main->n_expired++;
 
 	lru = oc_getlru(oc);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 	Lck_Lock(&lru->mtx);
+	AZ(oc->flags & OC_F_BUSY);
 	oc->flags |= OC_F_DYING;
 	if (oc->flags & OC_F_OFFLRU)
 		oc = NULL;



From martin at varnish-cache.org  Wed Oct  9 14:03:03 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 09 Oct 2013 16:03:03 +0200
Subject: [master] 19a5542 Rework VSL_Dispatch
Message-ID: <E1VTuLv-0005eH-KW@project.varnish-software.com>

commit 19a5542d7b1795e3fcfd6f0697ecace41ddba4d2
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 8 10:22:12 2013 +0200

    Rework VSL_Dispatch
    
    Highlights:
    
    * Improved buffer handling. A vtx can now have a mix of buffered and
      shm chunks. And as shm refs are used and later buffered, they can be
      reused on the same vtx. This benefits long running transactions.
    
    * Synth records are now returned by the cursor at the position the
      cursor was at when it was created. This solves a problem when
      writing to a file and a vtx times out, the reading of that file
      would also cause a time out delay.
    
    * Begin records are now strictly honored, and any records before a
      Begin record are now ignored.
    
    * More asserts on the structures
    
    * More comments, should be easier to understand now
    
    * Various other changes

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index db92c7f..a47d018 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -46,7 +46,16 @@
 
 #define VTX_CACHE 10
 #define VTX_BUFSIZE_MIN 64
-#define VTX_CHUNKS 3
+#define VTX_SHMCHUNKS 3
+
+static const char * const vsl_t_names[VSL_t__MAX] = {
+	[VSL_t_unknown]	= "unknown",
+	[VSL_t_sess]	= "sess",
+	[VSL_t_req]	= "req",
+	[VSL_t_esireq]	= "esireq",
+	[VSL_t_bereq]	= "bereq",
+	[VSL_t_raw]	= "raw",
+};
 
 struct vtx;
 
@@ -61,20 +70,41 @@ struct vslc_raw {
 	const uint32_t		*next;
 };
 
-struct vtx_chunk {
-	struct VSLC_ptr		start;
-	ssize_t			len;
-	ssize_t			offset;
-};
-
-struct vtx_diag {
+struct synth {
 	unsigned		magic;
-#define VTX_DIAG_MAGIC		0xC654479F
+#define SYNTH_MAGIC		0xC654479F
 
-	VTAILQ_ENTRY(vtx_diag)	list;
-	uint32_t		chunk[2 + 256 / sizeof (uint32_t)];
+	VTAILQ_ENTRY(synth)	list;
+	size_t			offset;
+	uint32_t		data[2 + 64 / sizeof (uint32_t)];
 };
+VTAILQ_HEAD(synthhead, synth);
 
+enum chunk_t {
+	chunk_t__unassigned,
+	chunk_t_shm,
+	chunk_t_buf,
+};
+
+struct chunk {
+	unsigned				magic;
+#define CHUNK_MAGIC				0x48DC0194
+	enum chunk_t				type;
+	union {
+		struct {
+			struct VSLC_ptr		start;
+			VTAILQ_ENTRY(chunk)	shmref;
+		} shm;
+		struct {
+			uint32_t		*data;
+			size_t			space;
+		} buf;
+	};
+	size_t					len;
+	struct vtx				*vtx;
+	VTAILQ_ENTRY(chunk)			list;
+};
+VTAILQ_HEAD(chunkhead, chunk);
 
 struct vslc_vtx {
 	unsigned		magic;
@@ -83,11 +113,10 @@ struct vslc_vtx {
 	struct VSL_cursor	cursor;
 
 	struct vtx		*vtx;
-
-	struct vtx_diag		*diag;	/* Current diag message pointer */
-
-	unsigned		chunk;	/* Current chunk */
-	ssize_t			offset; /* Offset of next record */
+	struct synth		*synth;
+	struct chunk		*chunk;
+	size_t			chunkstart;
+	size_t			offset;
 };
 
 struct vtx_key {
@@ -102,12 +131,15 @@ struct vtx {
 #define VTX_MAGIC		0xACC21D09
 	VTAILQ_ENTRY(vtx)	list_child;
 	VTAILQ_ENTRY(vtx)	list_incomplete;
-	VTAILQ_ENTRY(vtx)	list_shm;
 
 	double			t_start;
 	unsigned		flags;
-#define VTX_F_COMPLETE		0x1
-#define VTX_F_READY		0x2
+#define VTX_F_BEGIN		0x1 /* Begin record processed */
+#define VTX_F_END		0x2 /* End record processed */
+#define VTX_F_COMPLETE		0x4 /* Marked complete. No new children
+				       should be appended */
+#define VTX_F_READY		0x8 /* This vtx and all it's children are
+				       complete */
 
 	enum VSL_transaction_e	type;
 
@@ -117,17 +149,15 @@ struct vtx {
 	unsigned		n_childready;
 	unsigned		n_descend;
 
-	struct vslc_vtx		c;
-
-	VTAILQ_HEAD(,vtx_diag)	diag;
+	VTAILQ_HEAD(,synth)	synth;
 
-	struct vtx_chunk	chunk[VTX_CHUNKS];
-	unsigned		n_chunk;
+	struct chunk		shmchunks[VTX_SHMCHUNKS];
+	struct chunkhead	shmchunks_free;
 
-	uint32_t		*buf;
-	ssize_t			bufsize;
+	struct chunkhead	chunks;
+	size_t			len;
 
-	ssize_t			len;
+	struct vslc_vtx		c;
 };
 
 struct VSLQ {
@@ -145,14 +175,15 @@ struct VSLQ {
 	VTAILQ_HEAD(,vtx)	incomplete;
 	unsigned		n_incomplete;
 
-	VTAILQ_HEAD(,vtx)	shmlist;
+	struct chunkhead	shmrefs;
 
 	VTAILQ_HEAD(,vtx)	cache;
 	unsigned		n_cache;
 };
 
+static void vtx_synth_rec(struct vtx *vtx, unsigned tag, const char *fmt, ...);
 /*lint -esym(534, vtx_diag) */
-static int vtx_diag(struct vtx *vtx, const char *fmt, ...);
+static int vtx_diag(struct vtx *vtx, const char *msg);
 /*lint -esym(534, vtx_diag_tag) */
 static int vtx_diag_tag(struct vtx *vtx, const uint32_t *ptr,
     const char *reason);
@@ -217,49 +248,49 @@ static int
 vslc_vtx_next(struct VSL_cursor *cursor)
 {
 	struct vslc_vtx *c;
-	struct vtx_chunk *chunk;
+	const uint32_t *ptr;
 
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VTX_MAGIC);
 	assert(&c->cursor == cursor);
 	CHECK_OBJ_NOTNULL(c->vtx, VTX_MAGIC);
 
-	if (c->diag == NULL && VTAILQ_FIRST(&c->vtx->diag) != NULL) {
-		/* Send first diag msg */
-		c->diag = VTAILQ_FIRST(&c->vtx->diag);
-		c->cursor.rec.ptr = c->diag->chunk;
-		return (1);
-	} else if (c->diag != NULL && VTAILQ_NEXT(c->diag, list) != NULL) {
-		/* Send next diag msg */
-		c->diag = VTAILQ_NEXT(c->diag, list);
-		c->cursor.rec.ptr = c->diag->chunk;
-		return (1);
-	}
-
-	assert (c->offset <= c->vtx->len);
-	if (c->offset == c->vtx->len)
-		return (0);
-
-	if (c->vtx->n_chunk == 0) {
-		/* Buffer */
-		AN(c->vtx->buf);
-		assert(c->offset < c->vtx->bufsize);
-		c->cursor.rec.ptr = c->vtx->buf + c->offset;
-		c->offset += VSL_NEXT(c->cursor.rec.ptr) - c->cursor.rec.ptr;
-		return (1);
-	}
-
-	/* Shmptr chunks */
-	assert(c->chunk < c->vtx->n_chunk);
-	chunk = &c->vtx->chunk[c->chunk];
-	assert(c->offset >= chunk->offset);
-	assert(c->offset <= chunk->offset + chunk->len);
-	if (c->offset == chunk->offset + chunk->len) {
-		c->chunk++;
-		chunk = &c->vtx->chunk[c->chunk];
-	}
-	AN(chunk->start.ptr);
-	c->cursor.rec.ptr = chunk->start.ptr + c->offset - chunk->offset;
-	c->offset += VSL_NEXT(c->cursor.rec.ptr) - c->cursor.rec.ptr;
+	do {
+		CHECK_OBJ_ORNULL(c->synth, SYNTH_MAGIC);
+		if (c->synth != NULL && c->synth->offset == c->offset) {
+			/* We're at the offset of the next synth record,
+			   point to it and advance the pointer */
+			c->cursor.rec.ptr = c->synth->data;
+			c->synth = VTAILQ_NEXT(c->synth, list);
+		} else {
+			assert(c->offset <= c->vtx->len);
+			if (c->offset == c->vtx->len)
+				/* End of cursor */
+				return (0);
+
+			/* Advance chunk pointer */
+			if (c->chunk == NULL) {
+				c->chunk = VTAILQ_FIRST(&c->vtx->chunks);
+				c->chunkstart = 0;
+			}
+			CHECK_OBJ_NOTNULL(c->chunk, CHUNK_MAGIC);
+			while (c->offset >= c->chunkstart + c->chunk->len) {
+				c->chunkstart += c->chunk->len;
+				c->chunk = VTAILQ_NEXT(c->chunk, list);
+				CHECK_OBJ_NOTNULL(c->chunk, CHUNK_MAGIC);
+			}
+
+			/* Point to the next stored record */
+			if (c->chunk->type == chunk_t_shm)
+				ptr = c->chunk->shm.start.ptr;
+			else {
+				assert(c->chunk->type == chunk_t_buf);
+				ptr = c->chunk->buf.data;
+			}
+			c->cursor.rec.ptr = ptr + c->offset - c->chunkstart;
+			c->offset += VSL_NEXT(c->cursor.rec.ptr) -
+			    c->cursor.rec.ptr;
+		}
+	} while (VSL_TAG(c->cursor.rec.ptr) == SLT__Batch);
 
 	return (1);
 }
@@ -271,8 +302,10 @@ vslc_vtx_reset(struct VSL_cursor *cursor)
 
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VTX_MAGIC);
 	assert(&c->cursor == cursor);
-	c->diag = NULL;
-	c->chunk = 0;
+	CHECK_OBJ_NOTNULL(c->vtx, VTX_MAGIC);
+	c->synth = VTAILQ_FIRST(&c->vtx->synth);
+	c->chunk = NULL;
+	c->chunkstart = 0;
 	c->offset = 0;
 	c->cursor.rec.ptr = NULL;
 
@@ -288,10 +321,136 @@ static const struct vslc_tbl vslc_vtx_tbl = {
 	.check	= NULL,
 };
 
+/* Create a buf chunk */
+static struct chunk *
+chunk_newbuf(struct vtx *vtx, const uint32_t *ptr, size_t len)
+{
+	struct chunk *chunk;
+
+	ALLOC_OBJ(chunk, CHUNK_MAGIC);
+	chunk->type = chunk_t_buf;
+	chunk->vtx = vtx;
+	chunk->buf.space = VTX_BUFSIZE_MIN;
+	while (chunk->buf.space < len)
+		chunk->buf.space *= 2;
+	chunk->buf.data = malloc(sizeof (uint32_t) * chunk->buf.space);
+	AN(chunk->buf.data);
+	memcpy(chunk->buf.data, ptr, sizeof (uint32_t) * len);
+	chunk->len = len;
+	return (chunk);
+}
+
+/* Free a buf chunk */
+static void
+chunk_freebuf(struct chunk **pchunk)
+{
+
+	CHECK_OBJ_NOTNULL(*pchunk, CHUNK_MAGIC);
+	assert((*pchunk)->type == chunk_t_buf);
+	free((*pchunk)->buf.data);
+	FREE_OBJ(*pchunk);
+	*pchunk = NULL;
+}
+
+/* Append a set of records to a chunk */
+static void
+chunk_appendbuf(struct chunk *chunk, const uint32_t *ptr, size_t len)
+{
+
+	CHECK_OBJ_NOTNULL(chunk, CHUNK_MAGIC);
+	assert(chunk->type == chunk_t_buf);
+	if (chunk->buf.space < chunk->len + len) {
+		while (chunk->buf.space < chunk->len + len)
+			chunk->buf.space *= 2;
+		chunk->buf.data = realloc(chunk->buf.data,
+		    sizeof (uint32_t) * chunk->buf.space);
+	}
+	memcpy(chunk->buf.data + chunk->len, ptr, sizeof (uint32_t) * len);
+	chunk->len += len;
+}
+
+/* Transform a shm chunk to a buf chunk */
+static void
+chunk_shm_to_buf(struct VSLQ *vslq, struct chunk *chunk)
+{
+	struct vtx *vtx;
+	struct chunk *buf;
+
+	CHECK_OBJ_NOTNULL(chunk, CHUNK_MAGIC);
+	assert(chunk->type == chunk_t_shm);
+	vtx = chunk->vtx;
+	CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
+
+	buf = VTAILQ_PREV(chunk, chunkhead, list);
+	if (buf != NULL && buf->type == chunk_t_buf)
+		/* Previous is a buf chunk, append to it */
+		chunk_appendbuf(buf, chunk->shm.start.ptr, chunk->len);
+	else {
+		/* Create a new buf chunk and insert it before this */
+		buf = chunk_newbuf(vtx, chunk->shm.start.ptr, chunk->len);
+		AN(buf);
+		VTAILQ_INSERT_BEFORE(chunk, buf, list);
+	}
+
+	/* Reset cursor chunk pointer, vslc_vtx_next will set it correctly */
+	vtx->c.chunk = NULL;
+
+	/* Remove from the shmref list and vtx, and put chunk back
+	   on the free list */
+	VTAILQ_REMOVE(&vslq->shmrefs, chunk, shm.shmref);
+	VTAILQ_REMOVE(&vtx->chunks, chunk, list);
+	VTAILQ_INSERT_HEAD(&vtx->shmchunks_free, chunk, list);
+}
+
+/* Append a set of records to a vtx structure */
+static void
+vtx_append(struct VSLQ *vslq, struct vtx *vtx, const struct VSLC_ptr *start,
+    size_t len)
+{
+	struct chunk *chunk;
+
+	AN(vtx);
+	if (len == 0)
+		return;
+	AN(start);
+
+	if (VSL_Check(vslq->c, start) == 2 &&
+	    !VTAILQ_EMPTY(&vtx->shmchunks_free)) {
+		/* Shmref it */
+		chunk = VTAILQ_FIRST(&vtx->shmchunks_free);
+		CHECK_OBJ_NOTNULL(chunk, CHUNK_MAGIC);
+		assert(chunk->type == chunk_t_shm);
+		assert(chunk->vtx == vtx);
+		VTAILQ_REMOVE(&vtx->shmchunks_free, chunk, list);
+		chunk->shm.start = *start;
+		chunk->len = len;
+		VTAILQ_INSERT_TAIL(&vtx->chunks, chunk, list);
+
+		/* Append to shmref list */
+		VTAILQ_INSERT_TAIL(&vslq->shmrefs, chunk, shm.shmref);
+	} else {
+		/* Buffer it */
+		chunk = VTAILQ_LAST(&vtx->chunks, chunkhead);
+		CHECK_OBJ_ORNULL(chunk, CHUNK_MAGIC);
+		if (chunk != NULL && chunk->type == chunk_t_buf) {
+			/* Tail is a buf chunk, append to that */
+			chunk_appendbuf(chunk, start->ptr, len);
+		} else {
+			/* Append new buf chunk */
+			chunk = chunk_newbuf(vtx, start->ptr, len);
+			AN(chunk);
+			VTAILQ_INSERT_TAIL(&vtx->chunks, chunk, list);
+		}
+	}
+	vtx->len += len;
+}
+
+/* Allocate a new vtx structure */
 static struct vtx *
 vtx_new(struct VSLQ *vslq)
 {
 	struct vtx *vtx;
+	int i;
 
 	AN(vslq);
 	if (vslq->n_cache) {
@@ -302,52 +461,48 @@ vtx_new(struct VSLQ *vslq)
 	} else {
 		ALLOC_OBJ(vtx, VTX_MAGIC);
 		AN(vtx);
+
+		VTAILQ_INIT(&vtx->child);
+		VTAILQ_INIT(&vtx->shmchunks_free);
+		for (i = 0; i < VTX_SHMCHUNKS; i++) {
+			vtx->shmchunks[i].magic = CHUNK_MAGIC;
+			vtx->shmchunks[i].type = chunk_t_shm;
+			vtx->shmchunks[i].vtx = vtx;
+			VTAILQ_INSERT_TAIL(&vtx->shmchunks_free,
+			    &vtx->shmchunks[i], list);
+		}
+		VTAILQ_INIT(&vtx->chunks);
+		VTAILQ_INIT(&vtx->synth);
 		vtx->c.magic = VSLC_VTX_MAGIC;
 		vtx->c.vtx = vtx;
 		vtx->c.cursor.priv_tbl = &vslc_vtx_tbl;
 		vtx->c.cursor.priv_data = &vtx->c;
 	}
 
+	CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
 	vtx->key.vxid = 0;
 	vtx->t_start = VTIM_mono();
 	vtx->flags = 0;
 	vtx->type = VSL_t_unknown;
 	vtx->parent = NULL;
-	VTAILQ_INIT(&vtx->child);
 	vtx->n_child = 0;
 	vtx->n_childready = 0;
 	vtx->n_descend = 0;
-	VTAILQ_INIT(&vtx->diag);
-	memset(vtx->chunk, 0, sizeof vtx->chunk);
-	vtx->n_chunk = 0;
 	vtx->len = 0;
 	(void)vslc_vtx_reset(&vtx->c.cursor);
 
-	VTAILQ_INSERT_TAIL(&vslq->incomplete, vtx, list_incomplete);
-	vslq->n_incomplete++;
-
 	return (vtx);
 }
 
-static void
-vtx_free(struct vtx **pvtx)
-{
-	struct vtx *vtx;
-
-	AN(pvtx);
-	vtx = *pvtx;
-	*pvtx = NULL;
-
-	free(vtx->buf);
-	FREE_OBJ(vtx);
-}
-
+/* Disuse a vtx and all it's children, freeing any resources held. Free or
+   cache the vtx for later use */
 static void
 vtx_retire(struct VSLQ *vslq, struct vtx **pvtx)
 {
 	struct vtx *vtx;
 	struct vtx *child;
-	struct vtx_diag *diag;
+	struct synth *synth;
+	struct chunk *chunk;
 
 	AN(vslq);
 	AN(pvtx);
@@ -374,177 +529,161 @@ vtx_retire(struct VSLQ *vslq, struct vtx **pvtx)
 	}
 	AZ(vtx->n_child);
 	AZ(vtx->n_descend);
+	vtx->n_childready = 0;
 	AN(VRB_REMOVE(vtx_tree, &vslq->tree, &vtx->key));
+	vtx->key.vxid = 0;
+	vtx->flags = 0;
 
-	while (!VTAILQ_EMPTY(&vtx->diag)) {
-		diag = VTAILQ_FIRST(&vtx->diag);
-		VTAILQ_REMOVE(&vtx->diag, diag, list);
-		FREE_OBJ(diag);
+	while (!VTAILQ_EMPTY(&vtx->synth)) {
+		synth = VTAILQ_FIRST(&vtx->synth);
+		CHECK_OBJ_NOTNULL(synth, SYNTH_MAGIC);
+		VTAILQ_REMOVE(&vtx->synth, synth, list);
+		FREE_OBJ(synth);
 	}
 
-	if (vtx->n_chunk)
-		VTAILQ_REMOVE(&vslq->shmlist, vtx, list_shm);
+	while (!VTAILQ_EMPTY(&vtx->chunks)) {
+		chunk = VTAILQ_FIRST(&vtx->chunks);
+		CHECK_OBJ_NOTNULL(chunk, CHUNK_MAGIC);
+		VTAILQ_REMOVE(&vtx->chunks, chunk, list);
+		if (chunk->type == chunk_t_shm) {
+			VTAILQ_REMOVE(&vslq->shmrefs, chunk, shm.shmref);
+			VTAILQ_INSERT_HEAD(&vtx->shmchunks_free, chunk, list);
+		} else {
+			assert(chunk->type == chunk_t_buf);
+			chunk_freebuf(&chunk);
+			AZ(chunk);
+		}
+	}
+	vtx->len = 0;
 
 	if (vslq->n_cache < VTX_CACHE) {
 		VTAILQ_INSERT_HEAD(&vslq->cache, vtx, list_child);
 		vslq->n_cache++;
 	} else {
-		vtx_free(&vtx);
-		AZ(vtx);
+		FREE_OBJ(vtx);
+		vtx = NULL;
 	}
 }
 
+/* Lookup a vtx by vxid from the managed list */
 static struct vtx *
-vtx_lori(struct VSLQ *vslq, unsigned vxid)
+vtx_lookup(struct VSLQ *vslq, unsigned vxid)
 {
-	struct vtx *vtx;
 	struct vtx_key lkey, *key;
+	struct vtx *vtx;
 
 	AN(vslq);
 	lkey.vxid = vxid;
 	key = VRB_FIND(vtx_tree, &vslq->tree, &lkey);
-	if (key != NULL) {
-		CAST_OBJ_NOTNULL(vtx, (void *)key, VTX_MAGIC);
-		return (vtx);
-	}
-
-	vtx = vtx_new(vslq);
-	AN(vtx);
-	vtx->key.vxid = vxid;
-	AZ(VRB_INSERT(vtx_tree, &vslq->tree, &vtx->key));
+	if (key == NULL)
+		return (NULL);
+	CAST_OBJ_NOTNULL(vtx, (void *)key, VTX_MAGIC);
 	return (vtx);
 }
 
-static void
-vtx_set_bufsize(struct vtx *vtx, ssize_t len)
-{
-
-	AN(vtx);
-	assert(len >= 0);
-	if (vtx->bufsize >= len)
-		return;
-	if (vtx->bufsize == 0)
-		vtx->bufsize = VTX_BUFSIZE_MIN;
-	while (vtx->bufsize < len)
-		vtx->bufsize *= 2;
-	vtx->buf = realloc(vtx->buf, sizeof (uint32_t) * vtx->bufsize);
-	AN(vtx->buf);
-}
-
-static void
-vtx_buffer(struct VSLQ *vslq, struct vtx *vtx)
-{
-	int i;
-
-	AN(vtx->n_chunk);
-	AN(vtx->len);
-
-	vtx_set_bufsize(vtx, vtx->len);
-	AN(vtx->buf);
-	assert(vtx->bufsize >= vtx->len);
-
-	for (i = 0; i < vtx->n_chunk; i++)
-		memcpy(vtx->buf + vtx->chunk[i].offset, vtx->chunk[i].start.ptr,
-		    sizeof (uint32_t) * vtx->chunk[i].len);
-
-	memset(vtx->chunk, 0, sizeof vtx->chunk);
-	VTAILQ_REMOVE(&vslq->shmlist, vtx, list_shm);
-	vtx->n_chunk = 0;
-}
-
-static void
-vtx_append(struct VSLQ *vslq, struct vtx *vtx, const struct VSLC_ptr *start,
-    ssize_t len, int copy)
+/* Insert a new vtx into the managed list */
+static struct vtx *
+vtx_add(struct VSLQ *vslq, unsigned vxid)
 {
+	struct vtx *vtx;
 
+	AN(vslq);
+	vtx = vtx_new(vslq);
 	AN(vtx);
-	if (len == 0)
-		return;
-	AN(start);
-
-	if (vtx->len > 0 && vtx->n_chunk == 0)
-		/* Can't mix buffer and shmptr */
-		copy = 1;
-
-	if (!copy && vtx->n_chunk < VTX_CHUNKS) {
-		/* Add shmptr chunk */
-		AZ(vtx->chunk[vtx->n_chunk].len);
-		vtx->chunk[vtx->n_chunk].start = *start;
-		vtx->chunk[vtx->n_chunk].len = len;
-		vtx->chunk[vtx->n_chunk].offset = vtx->len;
-		vtx->len += len;
-		if (vtx->n_chunk == 0)
-			VTAILQ_INSERT_TAIL(&vslq->shmlist, vtx, list_shm);
-		vtx->n_chunk++;
-		return;
-	}
-
-	/* Append to buffer */
-	vtx_set_bufsize(vtx, vtx->len + len);
-	if (vtx->n_chunk)
-		vtx_buffer(vslq, vtx);
-	AZ(vtx->n_chunk);
-	AN(vtx->buf);
-	memcpy(vtx->buf + vtx->len, start->ptr, sizeof (uint32_t) * len);
-	vtx->len += len;
+	vtx->key.vxid = vxid;
+	AZ(VRB_INSERT(vtx_tree, &vslq->tree, &vtx->key));
+	VTAILQ_INSERT_TAIL(&vslq->incomplete, vtx, list_incomplete);
+	vslq->n_incomplete++;
+	return (vtx);
 }
 
+/* Mark a vtx complete, update child counters and if possible push it or
+   it's top parent to the ready state */
 static struct vtx *
-vtx_check_ready(struct VSLQ *vslq, struct vtx *vtx)
+vtx_mark_complete(struct VSLQ *vslq, struct vtx *vtx)
 {
-	struct vtx *ready;
 
 	AN(vslq);
-	AN(vtx->flags & VTX_F_COMPLETE);
-	AZ(vtx->flags & VTX_F_READY);
+	AN(vtx->flags & VTX_F_END);
+	AZ(vtx->flags & VTX_F_COMPLETE);
 
 	if (vtx->type == VSL_t_unknown)
 		vtx_diag(vtx, "vtx of unknown type marked complete");
 
-	ready = vtx;
+	vtx->flags |= VTX_F_COMPLETE;
+	VTAILQ_REMOVE(&vslq->incomplete, vtx, list_incomplete);
+	AN(vslq->n_incomplete);
+	vslq->n_incomplete--;
+
 	while (1) {
-		if (ready->flags & VTX_F_COMPLETE &&
-		    ready->n_child == ready->n_childready)
-			ready->flags |= VTX_F_READY;
+		AZ(vtx->flags & VTX_F_READY);
+		if (vtx->flags & VTX_F_COMPLETE &&
+		    vtx->n_child == vtx->n_childready)
+			vtx->flags |= VTX_F_READY;
 		else
-			break;
-		if (ready->parent == NULL)
-			break;
-		ready = ready->parent;
-		ready->n_childready++;
-		assert(ready->n_child >= ready->n_childready);
+			return (NULL);
+		if (vtx->parent == NULL) {
+			/* Top level vtx ready */
+			return (vtx);
+		}
+		vtx = vtx->parent;
+		vtx->n_childready++;
+		assert(vtx->n_child >= vtx->n_childready);
 	}
 
-	if (ready->flags & VTX_F_READY && ready->parent == NULL)
-		/* Top level vtx ready */
-		return (ready);
+	if (vtx->flags & VTX_F_READY && vtx->parent == NULL)
+		return (vtx);
 
 	return (NULL);
 }
 
+/* Add a child to a parent, and update child counters */
+static void
+vtx_set_parent(struct vtx *parent, struct vtx *child)
+{
+
+	CHECK_OBJ_NOTNULL(parent, VTX_MAGIC);
+	CHECK_OBJ_NOTNULL(child, VTX_MAGIC);
+	AZ(parent->flags & VTX_F_COMPLETE);
+	AZ(child->flags & VTX_F_COMPLETE);
+	AZ(child->parent);
+	child->parent = parent;
+	VTAILQ_INSERT_TAIL(&parent->child, child, list_child);
+	parent->n_child++;
+	do
+		parent->n_descend += 1 + child->n_descend;
+	while ((parent = parent->parent));
+}
+
+/* Parse a begin or link record. Returns the number of elements that was
+   successfully parsed. */
 static int
-vtx_parsetag_bl(const char *str, enum VSL_transaction_e *ptype,
+vtx_parse_beginlink(const char *str, enum VSL_transaction_e *ptype,
     unsigned *pvxid)
 {
-	char buf[7];
+	char buf[8];
 	unsigned vxid;
-	int i;
+	int i, j;
 	enum VSL_transaction_e type = VSL_t_unknown;
 
 	AN(str);
-	i = sscanf(str, "%6s %u", buf, &vxid);
+	i = sscanf(str, "%7s %u", buf, &vxid);
 	if (i < 1)
 		return (-1);
-	if (!strcmp(buf, "sess"))
-		type = VSL_t_sess;
-	else if (!strcmp(buf, "req"))
-		type = VSL_t_req;
-	else if (!strcmp(buf, "esireq"))
-		type = VSL_t_esireq;
-	else if (!strcmp(buf, "bereq"))
-		type = VSL_t_bereq;
-	else
+	for (j = 0; j < VSL_t__MAX; j++)
+		if (!strcmp(buf, vsl_t_names[j]))
+			break;
+	switch (j) {
+	case VSL_t_sess:
+	case VSL_t_req:
+	case VSL_t_esireq:
+	case VSL_t_bereq:
+		/* Valid types */
+		type = j;
+		break;
+	default:
 		return (-1);
+	}
 	if (i == 1)
 		vxid = 0;
 	if (ptype)
@@ -554,23 +693,9 @@ vtx_parsetag_bl(const char *str, enum VSL_transaction_e *ptype,
 	return (i);
 }
 
-static void
-vtx_set_parent(struct vtx *parent, struct vtx *child)
-{
-
-	AN(parent);
-	AN(child);
-	AZ(child->parent);
-	child->parent = parent;
-	VTAILQ_INSERT_TAIL(&parent->child, child, list_child);
-	parent->n_child++;
-	do
-		parent->n_descend += 1 + child->n_descend;
-	while ((parent = parent->parent));
-}
-
+/* Parse and process a begin record */
 static int
-vtx_scan_begintag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
+vtx_scan_begin(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 {
 	int i;
 	enum VSL_transaction_e type;
@@ -579,17 +704,19 @@ vtx_scan_begintag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 
 	assert(VSL_TAG(ptr) == SLT_Begin);
 
-	if (vtx->flags & VTX_F_READY)
-		return (vtx_diag_tag(vtx, ptr, "link too late"));
+	AZ(vtx->flags & VTX_F_READY);
 
-	i = vtx_parsetag_bl(VSL_CDATA(ptr), &type, &p_vxid);
+	i = vtx_parse_beginlink(VSL_CDATA(ptr), &type, &p_vxid);
 	if (i < 1)
 		return (vtx_diag_tag(vtx, ptr, "parse error"));
 
 	/* Check/set vtx type */
 	assert(type != VSL_t_unknown);
-	if (vtx->type != VSL_t_unknown && vtx->type != type)
+	if (vtx->type != VSL_t_unknown && vtx->type != type) {
+		/* Type not matching the one previously set by a link
+		   record */
 		return (vtx_diag_tag(vtx, ptr, "type mismatch"));
+	}
 	vtx->type = type;
 
 	if (i == 1 || p_vxid == 0)
@@ -600,25 +727,38 @@ vtx_scan_begintag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 	if (vslq->grouping == VSL_g_request && vtx->type == VSL_t_req)
 		return (0);	/* No links */
 
-	/* Lookup and check parent vtx */
-	p_vtx = vtx_lori(vslq, p_vxid);
-	AN(p_vtx);
-	if (vtx->parent == p_vtx)
-		/* Link already exists */
+	if (vtx->parent != NULL) {
+		if (vtx->parent->key.vxid != p_vxid) {
+			/* This vtx already belongs to a different
+			   parent */
+			return (vtx_diag_tag(vtx, ptr, "link mismatch"));
+		} else
+			/* Link already exists */
+			return (0);
+	}
+
+	p_vtx = vtx_lookup(vslq, p_vxid);
+	if (p_vtx == NULL) {
+		/* Not seen parent yet. Insert it and create link. */
+		p_vtx = vtx_add(vslq, p_vxid);
+		AN(p_vtx);
+		vtx_set_parent(p_vtx, vtx);
 		return (0);
+	}
 
-	if (vtx->parent != NULL)
-		return (vtx_diag_tag(vtx, ptr, "duplicate link"));
-	if (p_vtx->flags & VTX_F_READY)
+	CHECK_OBJ_NOTNULL(p_vtx, VTX_MAGIC);
+	if (p_vtx->flags & VTX_F_COMPLETE)
 		return (vtx_diag_tag(vtx, ptr, "link too late"));
 
+	/* Create link */
 	vtx_set_parent(p_vtx, vtx);
 
 	return (0);
 }
 
+/* Parse and process a link record */
 static int
-vtx_scan_linktag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
+vtx_scan_link(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 {
 	int i;
 	enum VSL_transaction_e c_type;
@@ -627,13 +767,13 @@ vtx_scan_linktag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 
 	assert(VSL_TAG(ptr) == SLT_Link);
 
-	if (vtx->flags & VTX_F_READY)
-		return (vtx_diag_tag(vtx, ptr, "link too late"));
+	AZ(vtx->flags & VTX_F_READY);
 
-	i = vtx_parsetag_bl(VSL_CDATA(ptr), &c_type, &c_vxid);
+	i = vtx_parse_beginlink(VSL_CDATA(ptr), &c_type, &c_vxid);
 	if (i < 2)
 		return (vtx_diag_tag(vtx, ptr, "parse error"));
 	assert(i == 2);
+	assert(c_type != VSL_t_unknown);
 
 	if (vslq->grouping == VSL_g_vxid)
 		return (0);	/* No links */
@@ -641,88 +781,92 @@ vtx_scan_linktag(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 		return (0);	/* No links */
 
 	/* Lookup and check child vtx */
-	c_vtx = vtx_lori(vslq, c_vxid);
-	AN(c_vtx);
+	c_vtx = vtx_lookup(vslq, c_vxid);
+	if (c_vtx == NULL) {
+		/* Child not seen before. Insert it and create link */
+		c_vtx = vtx_add(vslq, c_vxid);
+		AN(c_vtx);
+		c_vtx->type = c_type;
+		vtx_set_parent(vtx, c_vtx);
+		return (0);
+	}
+
+	CHECK_OBJ_NOTNULL(c_vtx, VTX_MAGIC);
 	if (c_vtx->parent == vtx)
 		/* Link already exists */
 		return (0);
-	if (c_vtx->parent != NULL)
+	if (c_vtx->parent != vtx)
 		return (vtx_diag_tag(vtx, ptr, "duplicate link"));
-	if (c_vtx->flags & VTX_F_READY)
+	if (c_vtx->flags & VTX_F_COMPLETE)
 		return (vtx_diag_tag(vtx, ptr, "link too late"));
 	if (c_vtx->type != VSL_t_unknown && c_vtx->type != c_type)
 		return (vtx_diag_tag(vtx, ptr, "type mismatch"));
-	c_vtx->type = c_type;
 
+	c_vtx->type = c_type;
 	vtx_set_parent(vtx, c_vtx);
-
 	return (0);
 }
 
+/* Scan the records of a vtx, performing processing actions on specific
+   records */
 static struct vtx *
 vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
 {
 	const uint32_t *ptr;
 	enum VSL_tag_e tag;
-	struct vtx *ret = NULL;
+
+	if (vtx->flags & VTX_F_END)
+		return (NULL);
 
 	while (vslc_vtx_next(&vtx->c.cursor) == 1) {
 		ptr = vtx->c.cursor.rec.ptr;
 		tag = VSL_TAG(ptr);
-
-		if (tag == SLT__Batch || tag == SLT_VSL)
-			continue;
-
-		if (vtx->flags & VTX_F_COMPLETE) {
-			vtx_diag_tag(vtx, ptr, "late log rec");
-			continue;
-		}
-
-		if (vtx->type == VSL_t_unknown && tag != SLT_Begin)
-			vtx_diag_tag(vtx, ptr, "early log rec");
+		assert(tag != SLT__Batch);
 
 		switch (tag) {
 		case SLT_Begin:
-			(void)vtx_scan_begintag(vslq, vtx, ptr);
+			(void)vtx_scan_begin(vslq, vtx, ptr);
+			vtx->flags |= VTX_F_BEGIN;
 			break;
 
 		case SLT_Link:
-			(void)vtx_scan_linktag(vslq, vtx, ptr);
+			(void)vtx_scan_link(vslq, vtx, ptr);
 			break;
 
 		case SLT_End:
-			AZ(vtx->flags & VTX_F_COMPLETE);
-			AZ(ret);
-			VTAILQ_REMOVE(&vslq->incomplete, vtx, list_incomplete);
-			vtx->flags |= VTX_F_COMPLETE;
-			AN(vslq->n_incomplete);
-			vslq->n_incomplete--;
-			ret = vtx_check_ready(vslq, vtx);
-			break;
+			vtx->flags |= VTX_F_END;
+			return (vtx_mark_complete(vslq, vtx));
 
 		default:
 			break;
 		}
 	}
 
-	return (ret);
+	return (NULL);
 }
 
+/* Force a vtx into complete status by synthing the necessary outstanding
+   records */
 static struct vtx *
 vtx_force(struct VSLQ *vslq, struct vtx *vtx, const char *reason)
 {
+	struct vtx *ret;
+
 	AZ(vtx->flags & VTX_F_COMPLETE);
 	AZ(vtx->flags & VTX_F_READY);
+	if (!(vtx->flags & VTX_F_BEGIN))
+		vtx_synth_rec(vtx, SLT_Begin, "%s %u synth",
+		    vsl_t_names[vtx->type], vtx->key.vxid);
 	vtx_diag(vtx, reason);
-
-	VTAILQ_REMOVE(&vslq->incomplete, vtx, list_incomplete);
-	vtx->flags |= VTX_F_COMPLETE;
-	AN(vslq->n_incomplete);
-	vslq->n_incomplete--;
-
-	return (vtx_check_ready(vslq, vtx));
+	if (!(vtx->flags & VTX_F_END))
+		vtx_synth_rec(vtx, SLT_End, "synth");
+	ret = vtx_scan(vslq, vtx);
+	AN(vtx->flags & VTX_F_COMPLETE);
+	return (ret);
 }
 
+/* Build transaction array, do the query and callback. Returns 0 or the
+   return value from func */
 static int
 vslq_callback(const struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
     void *priv)
@@ -735,6 +879,7 @@ vslq_callback(const struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
 
 	AN(vslq);
 	CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
+	AN(vtx->flags & VTX_F_READY);
 
 	if (func == NULL)
 		return (0);
@@ -784,19 +929,21 @@ vslq_callback(const struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
 	return ((func)(vslq->vsl, ptrans, priv));
 }
 
-static int
-vtx_diag(struct vtx *vtx, const char *fmt, ...)
+/* Create a synthetic log record. The record will be inserted at the
+   current cursor offset */
+static void
+vtx_synth_rec(struct vtx *vtx, unsigned tag, const char *fmt, ...)
 {
-	struct vtx_diag *diag;
+	struct synth *synth, *it;
 	va_list ap;
 	char *buf;
 	int l, buflen;
 
-	ALLOC_OBJ(diag, VTX_DIAG_MAGIC);
-	AN(diag);
+	ALLOC_OBJ(synth, SYNTH_MAGIC);
+	AN(synth);
 
-	buf = (char *)&diag->chunk[2];
-	buflen = sizeof (diag->chunk) - 2 * sizeof (uint32_t);
+	buf = (char *)&synth->data[2];
+	buflen = sizeof (synth->data) - 2 * sizeof (uint32_t);
 	va_start(ap, fmt);
 	l = vsnprintf(buf, buflen, fmt, ap);
 	assert(l >= 0);
@@ -804,29 +951,56 @@ vtx_diag(struct vtx *vtx, const char *fmt, ...)
 	if (l > buflen - 1)
 		l = buflen - 1;
 	buf[l++] = '\0';	/* NUL-terminated */
-	diag->chunk[1] = vtx->key.vxid;
+	synth->data[1] = vtx->key.vxid;
 	switch (vtx->type) {
 	case VSL_t_req:
 	case VSL_t_esireq:
-		diag->chunk[1] |= VSL_CLIENTMARKER;
+		synth->data[1] |= VSL_CLIENTMARKER;
 		break;
 	case VSL_t_bereq:
-		diag->chunk[1] |= VSL_BACKENDMARKER;
+		synth->data[1] |= VSL_BACKENDMARKER;
 		break;
 	default:
 		break;
 	}
-	diag->chunk[0] = ((((unsigned)SLT_VSL & 0xff) << 24) | l);
-	VTAILQ_INSERT_TAIL(&vtx->diag, diag, list);
+	synth->data[0] = (((tag & 0xff) << 24) | l);
+	synth->offset = vtx->c.offset;
+
+	VTAILQ_FOREACH_REVERSE(it, &vtx->synth, synthhead, list) {
+		/* Make sure the synth list is sorted on offset */
+		CHECK_OBJ_NOTNULL(it, SYNTH_MAGIC);
+		if (synth->offset >= it->offset)
+			break;
+	}
+	if (it != NULL)
+		VTAILQ_INSERT_AFTER(&vtx->synth, it, synth, list);
+	else
+		VTAILQ_INSERT_HEAD(&vtx->synth, synth, list);
+
+	/* Update cursor */
+	CHECK_OBJ_ORNULL(vtx->c.synth, SYNTH_MAGIC);
+	if (vtx->c.synth == NULL || vtx->c.synth->offset > synth->offset)
+		vtx->c.synth = synth;
+}
+
+/* Add a diagnostic SLT_VSL synth record to the vtx. */
+static int
+vtx_diag(struct vtx *vtx, const char *msg)
+{
 
+	vtx_synth_rec(vtx, SLT_VSL, msg);
 	return (-1);
 }
 
+/* Add a SLT_VSL diag synth record to the vtx. Takes an offending record
+   that will be included in the log record */
 static int
 vtx_diag_tag(struct vtx *vtx, const uint32_t *ptr, const char *reason)
 {
-	return (vtx_diag(vtx, "%s (%s: %.*s)", reason,
-		VSL_tags[VSL_TAG(ptr)],	(int)VSL_LEN(ptr), VSL_CDATA(ptr)));
+
+	vtx_synth_rec(vtx, SLT_VSL, "%s (%s: %.*s)", reason,
+	    VSL_tags[VSL_TAG(ptr)], (int)VSL_LEN(ptr), VSL_CDATA(ptr));
+	return (-1);
 }
 
 struct VSLQ *
@@ -858,7 +1032,7 @@ VSLQ_New(struct VSL_data *vsl, struct VSL_cursor **cp,
 	vslq->query = query;
 	VRB_INIT(&vslq->tree);
 	VTAILQ_INIT(&vslq->incomplete);
-	VTAILQ_INIT(&vslq->shmlist);
+	VTAILQ_INIT(&vslq->shmrefs);
 	VTAILQ_INIT(&vslq->cache);
 
 	return (vslq);
@@ -889,13 +1063,14 @@ VSLQ_Delete(struct VSLQ **pvslq)
 		vtx = VTAILQ_FIRST(&vslq->cache);
 		VTAILQ_REMOVE(&vslq->cache, vtx, list_child);
 		vslq->n_cache--;
-		vtx_free(&vtx);
-		AZ(vtx);
+		FREE_OBJ(vtx);
 	}
 
 	FREE_OBJ(vslq);
 }
 
+/* Regard each log line as a single transaction, feed it through the query
+   and do the callback */
 static int
 vslq_raw(const struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 {
@@ -944,11 +1119,45 @@ vslq_raw(const struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 	return (i);
 }
 
+/* Check the beginning of the shmref list, and buffer refs that are at
+ * warning level.
+ *
+ * Returns:
+ *    0:	OK
+ *   -3:	Failure
+ */
+static int
+vslq_shmref_check(struct VSLQ *vslq)
+{
+	struct chunk *chunk;
+	int i;
+
+	while ((chunk = VTAILQ_FIRST(&vslq->shmrefs))) {
+		CHECK_OBJ_NOTNULL(chunk, CHUNK_MAGIC);
+		assert(chunk->type == chunk_t_shm);
+		i = VSL_Check(vslq->c, &chunk->shm.start);
+		if (i == 2)
+			/* First on list is OK, refs behind it must also
+			   be OK */
+			return (0);
+		else if (i == 1)
+			/* Warning level. Buffer this chunk */
+			chunk_shm_to_buf(vslq, chunk);
+		else
+			/* Too late to buffer */
+			return (-3);
+	}
+
+	return (0);
+}
+
+/* Process the input cursor, calling the callback function on matching
+   transaction sets */
 int
 VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 {
 	struct VSL_cursor *c;
-	int i;
+	int i, batch;
 	enum VSL_tag_e tag;
 	ssize_t len;
 	unsigned vxid;
@@ -962,39 +1171,45 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 
 	c = vslq->c;
 	while (1) {
-		/* Check shmlist and buffer on warning */
-		while ((vtx = VTAILQ_FIRST(&vslq->shmlist))) {
-			AN(vtx->n_chunk);
-			i = VSL_Check(c, &vtx->chunk[0].start);
-			if (i == 2)
-				break;
-			else if (i == 1)
-				vtx_buffer(vslq, vtx);
-			else
-				/* Too late */
-				return (-3);
-		}
+		/* Check shmref list */
+		i = vslq_shmref_check(vslq);
+		if (i)
+			break;
 
 		i = VSL_Next(c);
 		if (i != 1)
 			break;
 		tag = VSL_TAG(c->rec.ptr);
 		if (tag == SLT__Batch) {
+			batch = 1;
 			vxid = VSL_BATCHID(c->rec.ptr);
 			len = VSL_END(c->rec.ptr, VSL_BATCHLEN(c->rec.ptr)) -
 			    c->rec.ptr;
+			if (len == 0)
+				continue;
+			tag = VSL_TAG(VSL_NEXT(c->rec.ptr));
 		} else {
+			batch = 0;
 			vxid = VSL_ID(c->rec.ptr);
 			len = VSL_NEXT(c->rec.ptr) - c->rec.ptr;
 		}
+
+		assert(len > 0);
 		if (vxid == 0)
+			/* Skip non-transactional records */
 			continue;
-		vtx = vtx_lori(vslq, vxid);
-		AN(vtx);
-		vtx_append(vslq, vtx, &c->rec, len, VSL_Check(c, &c->rec) != 2);
-		if (tag == SLT__Batch)
+
+		vtx = vtx_lookup(vslq, vxid);
+		if (vtx == NULL && tag == SLT_Begin) {
+			vtx = vtx_add(vslq, vxid);
+			AN(vtx);
+		}
+		if (vtx != NULL) {
+			vtx_append(vslq, vtx, &c->rec, len);
+			vtx = vtx_scan(vslq, vtx);
+		}
+		if (batch)
 			AZ(vsl_skip(c, VSL_WORDS(VSL_BATCHLEN(c->rec.ptr))));
-		vtx = vtx_scan(vslq, vtx);
 		if (vtx) {
 			AN(vtx->flags & VTX_F_READY);
 			i = vslq_callback(vslq, vtx, func, priv);
@@ -1042,6 +1257,8 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 	return (i);
 }
 
+/* Flush incomplete any incomplete vtx held on to. Do callbacks if func !=
+   NULL */
 int
 VSLQ_Flush(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 {



From martin at varnish-cache.org  Wed Oct  9 14:03:03 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 09 Oct 2013 16:03:03 +0200
Subject: [master] 3105a4c Tune error messages given by varnishlog when loosing
 the log connection.
Message-ID: <E1VTuLv-0005eW-OK@project.varnish-software.com>

commit 3105a4c95337ddcb096260a76d2bd580c5337d49
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Oct 8 23:35:33 2013 +0200

    Tune error messages given by varnishlog when loosing the log
    connection.

diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index 03127ec..7583c77 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -303,6 +303,7 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 			VUT.vslq = VSLQ_New(VUT.vsl, &c, VUT.g_arg, VUT.q_arg);
 			AN(VUT.vslq);
 			AZ(c);
+			VUT_Error(0, "Log reaquired");
 		}
 
 		i = VSLQ_Dispatch(VUT.vslq, func, priv);
@@ -329,7 +330,7 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 
 		if (i == -2) {
 			/* Abandoned */
-			VUT_Error(0, "Log abandoned - reopening");
+			VUT_Error(0, "Log abandoned");
 			VSM_Close(VUT.vsm);
 		} else if (i < -2) {
 			/* Overrun */



From martin at varnish-cache.org  Wed Oct  9 14:03:03 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 09 Oct 2013 16:03:03 +0200
Subject: [master] 53ee8e5 VSLQ_Dispatch now returns sooner.
Message-ID: <E1VTuLv-0005ee-Sb@project.varnish-software.com>

commit 53ee8e5c6baa2c4cfb6b9c948c209cebc2cfa77c
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 9 12:53:42 2013 +0200

    VSLQ_Dispatch now returns sooner.
    
    Change VSLQ_Dispatch to always return after processing one record or
    one batch of records. It will return 1 if it should be called again
    without sleeping.
    
    This fixes the situation where a calling program is lagging behind,
    and VSLQ_Dispatch never returns because there is a constant stream of
    new log records.

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index cee9307..f88e20f 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -423,8 +423,9 @@ int VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv);
 	 *  priv: An argument passed to func
 	 *
 	 * Return values:
+	 *     1: Call again
 	 *     0: No more log records available
-	 *   !=0: The return value from either VSL_Next() or func
+	 *   !=0: The error code from VSL_Next() or func returned non-zero
 	 */
 
 int VSLQ_Flush(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv);
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index a47d018..6912943 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -65,9 +65,7 @@ struct vslc_raw {
 
 	struct VSL_cursor	cursor;
 
-	const uint32_t		*start;
-	ssize_t			len;
-	const uint32_t		*next;
+	const uint32_t		*ptr;
 };
 
 struct synth {
@@ -130,7 +128,7 @@ struct vtx {
 	unsigned		magic;
 #define VTX_MAGIC		0xACC21D09
 	VTAILQ_ENTRY(vtx)	list_child;
-	VTAILQ_ENTRY(vtx)	list_incomplete;
+	VTAILQ_ENTRY(vtx)	list_vtx;
 
 	double			t_start;
 	unsigned		flags;
@@ -170,15 +168,24 @@ struct VSLQ {
 
 	enum VSL_grouping_e	grouping;
 
+	/* Structured mode */
 	struct vtx_tree		tree;
-
+	VTAILQ_HEAD(,vtx)	ready;
 	VTAILQ_HEAD(,vtx)	incomplete;
 	unsigned		n_incomplete;
-
 	struct chunkhead	shmrefs;
-
 	VTAILQ_HEAD(,vtx)	cache;
 	unsigned		n_cache;
+
+	/* Raw mode */
+	struct {
+		struct vslc_raw		c;
+		struct VSL_transaction	trans;
+		struct VSL_transaction	*ptrans[2];
+		struct VSLC_ptr		start;
+		ssize_t			len;
+		size_t			offset;
+	} raw;
 };
 
 static void vtx_synth_rec(struct vtx *vtx, unsigned tag, const char *fmt, ...);
@@ -209,14 +216,14 @@ vslc_raw_next(struct VSL_cursor *cursor)
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_RAW_MAGIC);
 	assert(&c->cursor == cursor);
 
-	assert(c->next >= c->start);
-	assert(c->next <= c->start + c->len);
-	if (c->next < c->start + c->len) {
-		c->cursor.rec.ptr = c->next;
-		c->next = VSL_NEXT(c->next);
+	AN(c->ptr);
+	if (c->cursor.rec.ptr == NULL) {
+		c->cursor.rec.ptr = c->ptr;
 		return (1);
+	} else {
+		c->cursor.rec.ptr = NULL;
+		return (0);
 	}
-	return (0);
 }
 
 static int
@@ -227,9 +234,7 @@ vslc_raw_reset(struct VSL_cursor *cursor)
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_RAW_MAGIC);
 	assert(&c->cursor == cursor);
 
-	assert(c->next >= c->start);
-	assert(c->next <= c->start + c->len);
-	c->next = c->start;
+	AN(c->ptr);
 	c->cursor.rec.ptr = NULL;
 
 	return (0);
@@ -592,14 +597,14 @@ vtx_add(struct VSLQ *vslq, unsigned vxid)
 	AN(vtx);
 	vtx->key.vxid = vxid;
 	AZ(VRB_INSERT(vtx_tree, &vslq->tree, &vtx->key));
-	VTAILQ_INSERT_TAIL(&vslq->incomplete, vtx, list_incomplete);
+	VTAILQ_INSERT_TAIL(&vslq->incomplete, vtx, list_vtx);
 	vslq->n_incomplete++;
 	return (vtx);
 }
 
 /* Mark a vtx complete, update child counters and if possible push it or
    it's top parent to the ready state */
-static struct vtx *
+static void
 vtx_mark_complete(struct VSLQ *vslq, struct vtx *vtx)
 {
 
@@ -611,7 +616,7 @@ vtx_mark_complete(struct VSLQ *vslq, struct vtx *vtx)
 		vtx_diag(vtx, "vtx of unknown type marked complete");
 
 	vtx->flags |= VTX_F_COMPLETE;
-	VTAILQ_REMOVE(&vslq->incomplete, vtx, list_incomplete);
+	VTAILQ_REMOVE(&vslq->incomplete, vtx, list_vtx);
 	AN(vslq->n_incomplete);
 	vslq->n_incomplete--;
 
@@ -621,20 +626,16 @@ vtx_mark_complete(struct VSLQ *vslq, struct vtx *vtx)
 		    vtx->n_child == vtx->n_childready)
 			vtx->flags |= VTX_F_READY;
 		else
-			return (NULL);
+			return;
 		if (vtx->parent == NULL) {
 			/* Top level vtx ready */
-			return (vtx);
+			VTAILQ_INSERT_TAIL(&vslq->ready, vtx, list_vtx);
+			return;
 		}
 		vtx = vtx->parent;
 		vtx->n_childready++;
 		assert(vtx->n_child >= vtx->n_childready);
 	}
-
-	if (vtx->flags & VTX_F_READY && vtx->parent == NULL)
-		return (vtx);
-
-	return (NULL);
 }
 
 /* Add a child to a parent, and update child counters */
@@ -809,14 +810,14 @@ vtx_scan_link(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr)
 
 /* Scan the records of a vtx, performing processing actions on specific
    records */
-static struct vtx *
+static void
 vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
 {
 	const uint32_t *ptr;
 	enum VSL_tag_e tag;
 
 	if (vtx->flags & VTX_F_END)
-		return (NULL);
+		return;
 
 	while (vslc_vtx_next(&vtx->c.cursor) == 1) {
 		ptr = vtx->c.cursor.rec.ptr;
@@ -841,28 +842,25 @@ vtx_scan(struct VSLQ *vslq, struct vtx *vtx)
 			break;
 		}
 	}
-
-	return (NULL);
 }
 
 /* Force a vtx into complete status by synthing the necessary outstanding
    records */
-static struct vtx *
+static void
 vtx_force(struct VSLQ *vslq, struct vtx *vtx, const char *reason)
 {
-	struct vtx *ret;
 
 	AZ(vtx->flags & VTX_F_COMPLETE);
 	AZ(vtx->flags & VTX_F_READY);
+	vtx_scan(vslq, vtx);
 	if (!(vtx->flags & VTX_F_BEGIN))
 		vtx_synth_rec(vtx, SLT_Begin, "%s %u synth",
 		    vsl_t_names[vtx->type], vtx->key.vxid);
 	vtx_diag(vtx, reason);
 	if (!(vtx->flags & VTX_F_END))
 		vtx_synth_rec(vtx, SLT_End, "synth");
-	ret = vtx_scan(vslq, vtx);
+	vtx_scan(vslq, vtx);
 	AN(vtx->flags & VTX_F_COMPLETE);
-	return (ret);
 }
 
 /* Build transaction array, do the query and callback. Returns 0 or the
@@ -880,9 +878,8 @@ vslq_callback(const struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
 	AN(vslq);
 	CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
 	AN(vtx->flags & VTX_F_READY);
+	AN(func);
 
-	if (func == NULL)
-		return (0);
 	if (vslq->grouping == VSL_g_session &&
 	    vtx->type != VSL_t_sess)
 		return (0);
@@ -1030,11 +1027,24 @@ VSLQ_New(struct VSL_data *vsl, struct VSL_cursor **cp,
 	*cp = NULL;
 	vslq->grouping = grouping;
 	vslq->query = query;
+
+	/* Setup normal mode */
 	VRB_INIT(&vslq->tree);
+	VTAILQ_INIT(&vslq->ready);
 	VTAILQ_INIT(&vslq->incomplete);
 	VTAILQ_INIT(&vslq->shmrefs);
 	VTAILQ_INIT(&vslq->cache);
 
+	/* Setup raw mode */
+	vslq->raw.c.magic = VSLC_RAW_MAGIC;
+	vslq->raw.c.cursor.priv_tbl = &vslc_raw_tbl;
+	vslq->raw.c.cursor.priv_data = &vslq->raw.c;
+	vslq->raw.trans.level = 0;
+	vslq->raw.trans.type = VSL_t_raw;
+	vslq->raw.trans.c = &vslq->raw.c.cursor;
+	vslq->raw.ptrans[0] = &vslq->raw.trans;
+	vslq->raw.ptrans[1] = NULL;
+
 	return (vslq);
 }
 
@@ -1051,6 +1061,15 @@ VSLQ_Delete(struct VSLQ **pvslq)
 
 	(void)VSLQ_Flush(vslq, NULL, NULL);
 	AZ(vslq->n_incomplete);
+
+	while (!VTAILQ_EMPTY(&vslq->ready)) {
+		vtx = VTAILQ_FIRST(&vslq->ready);
+		CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
+		VTAILQ_REMOVE(&vslq->ready, vtx, list_vtx);
+		AN(vtx->flags & VTX_F_READY);
+		vtx_retire(vslq, &vtx);
+	}
+
 	VSL_DeleteCursor(vslq->c);
 	vslq->c = NULL;
 
@@ -1072,49 +1091,43 @@ VSLQ_Delete(struct VSLQ **pvslq)
 /* Regard each log line as a single transaction, feed it through the query
    and do the callback */
 static int
-vslq_raw(const struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
+vslq_raw(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 {
-	struct vslc_raw rawc;
-	struct VSL_transaction trans;
-	struct VSL_transaction *ptrans[2];
-	struct VSL_cursor *c;
-	int i;
+	int i = 1;
+	int r;
 
 	assert(vslq->grouping == VSL_g_raw);
-	c = vslq->c;
 
-	memset(&rawc, 0, sizeof rawc);
-	rawc.magic = VSLC_RAW_MAGIC;
-	rawc.cursor.priv_tbl = &vslc_raw_tbl;
-	rawc.cursor.priv_data = &rawc;
-	trans.level = 0;
-	trans.type = VSL_t_raw;
-	trans.c = &rawc.cursor;
-	ptrans[0] = &trans;
-	ptrans[1] = NULL;
+	assert(vslq->raw.offset <= vslq->raw.len);
+	do {
+		if (vslq->raw.offset == vslq->raw.len) {
+			i = VSL_Next(vslq->c);
+			if (i <= 0)
+				return (i);
+			AN(vslq->c->rec.ptr);
+			vslq->raw.start = vslq->c->rec;
+			vslq->raw.len = VSL_NEXT(vslq->raw.start.ptr) -
+			    vslq->raw.start.ptr;
+			assert(vslq->raw.len > 0);
+			vslq->raw.offset = 0;
+		}
 
-	while (1) {
-		i = VSL_Next(c);
-		if (i <= 0)
-			break;
-		AN(c->rec.ptr);
-		if (func == NULL)
-			continue;
-		rawc.start = c->rec.ptr;
-		rawc.len = VSL_NEXT(c->rec.ptr) - c->rec.ptr;
-		rawc.next = rawc.start;
-		rawc.cursor.rec.ptr = NULL;
-		trans.vxid = VSL_ID(c->rec.ptr);
-
-		/* Query check goes here */
-		if (vslq->query != NULL && !vslq_runquery(vslq->query, ptrans))
-			continue;
-
-		/* Callback */
-		i = (func)(vslq->vsl, ptrans, priv);
-		if (i)
-			break;
-	}
+		vslq->raw.c.ptr = vslq->raw.start.ptr + vslq->raw.offset;
+		vslq->raw.c.cursor.rec.ptr = NULL;
+		vslq->raw.trans.vxid = VSL_ID(vslq->raw.c.ptr);
+		vslq->raw.offset += VSL_NEXT(vslq->raw.c.ptr) - vslq->raw.c.ptr;
+	} while (VSL_TAG(vslq->raw.c.ptr) == SLT__Batch);
+
+	if (func == NULL)
+		return (i);
+
+	if (vslq->query != NULL &&
+	    !vslq_runquery(vslq->query, vslq->raw.ptrans))
+		return (i);
+
+	r = (func)(vslq->vsl, vslq->raw.ptrans, priv);
+	if (r)
+		return (r);
 
 	return (i);
 }
@@ -1151,10 +1164,9 @@ vslq_shmref_check(struct VSLQ *vslq)
 	return (0);
 }
 
-/* Process the input cursor, calling the callback function on matching
-   transaction sets */
-int
-VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
+/* Process next input record */
+static int
+vslq_next(struct VSLQ *vslq)
 {
 	struct VSL_cursor *c;
 	int i, batch;
@@ -1162,98 +1174,120 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 	ssize_t len;
 	unsigned vxid;
 	struct vtx *vtx;
-	double now;
 
-	CHECK_OBJ_NOTNULL(vslq, VSLQ_MAGIC);
+	c = vslq->c;
+	i = VSL_Next(c);
+	if (i != 1)
+		return (i);
 
-	if (vslq->grouping == VSL_g_raw)
-		return (vslq_raw(vslq, func, priv));
+	tag = VSL_TAG(c->rec.ptr);
+	if (tag == SLT__Batch) {
+		batch = 1;
+		vxid = VSL_BATCHID(c->rec.ptr);
+		len = VSL_END(c->rec.ptr, VSL_BATCHLEN(c->rec.ptr)) -
+		    c->rec.ptr;
+		if (len == 0)
+			return (i);
+		tag = VSL_TAG(VSL_NEXT(c->rec.ptr));
+	} else {
+		batch = 0;
+		vxid = VSL_ID(c->rec.ptr);
+		len = VSL_NEXT(c->rec.ptr) - c->rec.ptr;
+	}
+	assert(len > 0);
+	if (vxid == 0)
+		/* Skip non-transactional records */
+		return (i);
 
-	c = vslq->c;
-	while (1) {
-		/* Check shmref list */
-		i = vslq_shmref_check(vslq);
-		if (i)
-			break;
+	vtx = vtx_lookup(vslq, vxid);
+	if (vtx == NULL && tag == SLT_Begin) {
+		vtx = vtx_add(vslq, vxid);
+		AN(vtx);
+	}
+	if (vtx != NULL) {
+		vtx_append(vslq, vtx, &c->rec, len);
+		vtx_scan(vslq, vtx);
+	}
+	if (batch)
+		AZ(vsl_skip(c, VSL_WORDS(VSL_BATCHLEN(c->rec.ptr))));
 
-		i = VSL_Next(c);
-		if (i != 1)
-			break;
-		tag = VSL_TAG(c->rec.ptr);
-		if (tag == SLT__Batch) {
-			batch = 1;
-			vxid = VSL_BATCHID(c->rec.ptr);
-			len = VSL_END(c->rec.ptr, VSL_BATCHLEN(c->rec.ptr)) -
-			    c->rec.ptr;
-			if (len == 0)
-				continue;
-			tag = VSL_TAG(VSL_NEXT(c->rec.ptr));
-		} else {
-			batch = 0;
-			vxid = VSL_ID(c->rec.ptr);
-			len = VSL_NEXT(c->rec.ptr) - c->rec.ptr;
-		}
+	return (i);
+}
 
-		assert(len > 0);
-		if (vxid == 0)
-			/* Skip non-transactional records */
-			continue;
+/* Test query and report any ready transactions */
+static int
+vslq_process_ready(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
+{
+	struct vtx *vtx;
+	int i = 0;
 
-		vtx = vtx_lookup(vslq, vxid);
-		if (vtx == NULL && tag == SLT_Begin) {
-			vtx = vtx_add(vslq, vxid);
-			AN(vtx);
-		}
-		if (vtx != NULL) {
-			vtx_append(vslq, vtx, &c->rec, len);
-			vtx = vtx_scan(vslq, vtx);
-		}
-		if (batch)
-			AZ(vsl_skip(c, VSL_WORDS(VSL_BATCHLEN(c->rec.ptr))));
-		if (vtx) {
-			AN(vtx->flags & VTX_F_READY);
+	AN(vslq);
+
+	while (!VTAILQ_EMPTY(&vslq->ready)) {
+		vtx = VTAILQ_FIRST(&vslq->ready);
+		CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
+		VTAILQ_REMOVE(&vslq->ready, vtx, list_vtx);
+		AN(vtx->flags & VTX_F_READY);
+		if (func != NULL)
 			i = vslq_callback(vslq, vtx, func, priv);
-			vtx_retire(vslq, &vtx);
-			AZ(vtx);
-			if (i)
-				break;
-		}
+		vtx_retire(vslq, &vtx);
+		AZ(vtx);
+		if (i)
+			return (i);
 	}
+
+	return (0);
+}
+
+/* Process the input cursor, calling the callback function on matching
+   transaction sets */
+int
+VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
+{
+	int i, r;
+	double now;
+	struct vtx *vtx;
+
+	CHECK_OBJ_NOTNULL(vslq, VSLQ_MAGIC);
+
+	if (vslq->grouping == VSL_g_raw)
+		return (vslq_raw(vslq, func, priv));
+
+	/* Check shmref list and buffer if necessary */
+	r = vslq_shmref_check(vslq);
+	if (r)
+		return (r);
+
+	/* Process next cursor input */
+	i = vslq_next(vslq);
 	if (i)
 		return (i);
 
+	/* Check vtx timeout */
 	now = VTIM_mono();
-	while ((vtx = VTAILQ_FIRST(&vslq->incomplete)) &&
-	    now - vtx->t_start > vslq->vsl->T_opt) {
-		AZ(vtx->flags & VTX_F_COMPLETE);
-		vtx = vtx_force(vslq, vtx, "incomplete - timeout");
-		if (vtx) {
-			AN(vtx->flags & VTX_F_READY);
-			i = vslq_callback(vslq, vtx, func, priv);
-			vtx_retire(vslq, &vtx);
-			AZ(vtx);
-			if (i)
-				break;
-		}
+	while (!VTAILQ_EMPTY(&vslq->incomplete)) {
+		vtx = VTAILQ_FIRST(&vslq->incomplete);
+		CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
+		if (now - vtx->t_start < vslq->vsl->T_opt)
+			break;
+		vtx_force(vslq, vtx, "timeout");
+		AN(vtx->flags & VTX_F_COMPLETE);
 	}
-	if (i)
-		return (i);
 
+	/* Check store limit */
 	while (vslq->n_incomplete > vslq->vsl->L_opt) {
 		vtx = VTAILQ_FIRST(&vslq->incomplete);
-		AN(vtx);
-		AZ(vtx->flags & VTX_F_COMPLETE);
-		vtx = vtx_force(vslq, vtx, "incomplete - store overflow");
-		if (vtx) {
-			AN(vtx->flags & VTX_F_READY);
-			i = vslq_callback(vslq, vtx, func, priv);
-			vtx_retire(vslq, &vtx);
-			AZ(vtx);
-			if (i)
-				break;
-		}
+		CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
+		vtx_force(vslq, vtx, "store overflow");
+		AN(vtx->flags & VTX_F_COMPLETE);
 	}
 
+	/* Check ready list */
+	if (!VTAILQ_EMPTY(&vslq->ready))
+		r = vslq_process_ready(vslq, func, priv);
+	if (r)
+		return (r);
+
 	return (i);
 }
 
@@ -1263,23 +1297,15 @@ int
 VSLQ_Flush(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 {
 	struct vtx *vtx;
-	int i = 0;
 
 	CHECK_OBJ_NOTNULL(vslq, VSLQ_MAGIC);
 
 	while (vslq->n_incomplete) {
 		vtx = VTAILQ_FIRST(&vslq->incomplete);
-		AN(vtx);
+		CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
 		AZ(vtx->flags & VTX_F_COMPLETE);
-		vtx = vtx_force(vslq, vtx, "incomplete - flushing");
-		if (vtx) {
-			AN(vtx->flags & VTX_F_READY);
-			i = vslq_callback(vslq, vtx, func, priv);
-			vtx_retire(vslq, &vtx);
-			AZ(vtx);
-			if (i)
-				break;
-		}
+		vtx_force(vslq, vtx, "flush");
 	}
-	return (i);
+
+	return (vslq_process_ready(vslq, func, priv));
 }
diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index 7583c77..5e24af4 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -307,14 +307,16 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 		}
 
 		i = VSLQ_Dispatch(VUT.vslq, func, priv);
-		if (i == 0) {
+		if (i == 1)
+			/* Call again */
+			continue;
+		else if (i == 0) {
 			/* Nothing to do but wait */
 			if (VUT.fo)
 				fflush(VUT.fo);
 			VTIM_sleep(0.01);
 			continue;
-		}
-		if (i == -1) {
+		} else if (i == -1) {
 			/* EOF */
 			break;
 		}



From martin at varnish-cache.org  Wed Oct  9 14:03:04 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 09 Oct 2013 16:03:04 +0200
Subject: [master] a88cef8 Add an option argument to the cursors
Message-ID: <E1VTuLw-0005en-16@project.varnish-software.com>

commit a88cef86ff8da2d2da5264bf2b54904f393c5421
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 9 13:25:41 2013 +0200

    Add an option argument to the cursors

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index f88e20f..5f30c0c 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -201,24 +201,30 @@ void VSL_ResetError(struct VSL_data *vsl);
 	 * Reset any error message.
 	 */
 
+#define VSL_COPT_TAIL	(1 << 0)
 struct VSL_cursor *VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm,
-    int tail);
+    unsigned options);
        /*
         * Set the cursor pointed to by cursor up as a raw cursor in the
-        * log. If tail is non-zero, it will point to the tail of the
-        * log. Is tail is zero, it will point close to the head of the
-        * log, at least 2 segments away from the head.
+        * log. Cursor points at the current log head.
+	*
+	* Options:
+	*   VSL_COPT_TAIL	Start cursor at log tail
 	*
 	* Return values:
 	* non-NULL: Pointer to cursor
 	*     NULL: Error, see VSL_Error
         */
 
-struct VSL_cursor *VSL_CursorFile(struct VSL_data *vsl, const char *name);
+struct VSL_cursor *VSL_CursorFile(struct VSL_data *vsl, const char *name,
+    unsigned options);
 	/*
 	 * Create a cursor pointing to the beginning of the binary VSL log
 	 * in file name. If name is '-' reads from stdin.
 	 *
+	 * Options:
+	 *   NONE
+	 *
 	 * Return values:
 	 * non-NULL: Pointer to cursor
 	 *     NULL: Error, see VSL_Error
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index fe1f82e..e030776 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -53,6 +53,8 @@ struct vslc_vsm {
 
 	struct VSL_cursor		cursor;
 
+	unsigned			options;
+
 	struct VSM_data			*vsm;
 	struct VSM_fantom		vf;
 
@@ -228,7 +230,7 @@ static const struct vslc_tbl vslc_vsm_tbl = {
 };
 
 struct VSL_cursor *
-VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, int tail)
+VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, unsigned options)
 {
 	struct vslc_vsm *c;
 	struct VSM_fantom vf;
@@ -260,13 +262,14 @@ VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, int tail)
 	c->cursor.priv_tbl = &vslc_vsm_tbl;
 	c->cursor.priv_data = c;
 
+	c->options = options;
 	c->vsm = vsm;
 	c->vf = vf;
 	c->head = head;
 	c->end = vf.e;
 	c->segsize = (c->end - c->head->log) / VSL_SEGMENTS;
 
-	if (tail) {
+	if (c->options & VSL_COPT_TAIL) {
 		/* Locate tail of log */
 		c->next.ptr = c->head->log +
 		    c->head->segments[c->head->segment];
@@ -384,7 +387,7 @@ static const struct vslc_tbl vslc_file_tbl = {
 };
 
 struct VSL_cursor *
-VSL_CursorFile(struct VSL_data *vsl, const char *name)
+VSL_CursorFile(struct VSL_data *vsl, const char *name, unsigned options)
 {
 	struct vslc_file *c;
 	int fd;
@@ -392,6 +395,10 @@ VSL_CursorFile(struct VSL_data *vsl, const char *name)
 	char buf[] = VSL_FILE_ID;
 	ssize_t i;
 
+	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
+	AN(name);
+	(void)options;
+
 	if (!strcmp(name, "-"))
 		fd = STDIN_FILENO;
 	else {
diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index 5e24af4..6510256 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -181,7 +181,7 @@ VUT_Setup(void)
 	if (VUT.r_arg && VUT.vsm)
 		VUT_Error(1, "Can't have both -n and -r options");
 	if (VUT.r_arg)
-		c = VSL_CursorFile(VUT.vsl, VUT.r_arg);
+		c = VSL_CursorFile(VUT.vsl, VUT.r_arg, 0);
 	else {
 		if (VUT.vsm == NULL)
 			/* Default uses VSM with n=hostname */
@@ -190,7 +190,8 @@ VUT_Setup(void)
 		if (VSM_Open(VUT.vsm))
 			VUT_Error(1, "Can't open VSM file (%s)",
 			    VSM_Error(VUT.vsm));
-		c = VSL_CursorVSM(VUT.vsl, VUT.vsm, !VUT.d_opt);
+		c = VSL_CursorVSM(VUT.vsl, VUT.vsm,
+		    VUT.d_opt ? 0 : VSL_COPT_TAIL);
 	}
 	if (c == NULL)
 		VUT_Error(1, "Can't open log (%s)", VSL_Error(VUT.vsl));
@@ -294,7 +295,7 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 				VSM_ResetError(VUT.vsm);
 				continue;
 			}
-			c = VSL_CursorVSM(VUT.vsl, VUT.vsm, 1);
+			c = VSL_CursorVSM(VUT.vsl, VUT.vsm, VSL_COPT_TAIL);
 			if (c == NULL) {
 				VSL_ResetError(VUT.vsl);
 				VSM_Close(VUT.vsm);



From martin at varnish-cache.org  Wed Oct  9 14:03:04 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 09 Oct 2013 16:03:04 +0200
Subject: [master] a03e139 Add batch records as a cursor option to the VSM
 cursor
Message-ID: <E1VTuLw-0005ew-7b@project.varnish-software.com>

commit a03e139afc50f782681e2c6485814316ba3f2c78
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 9 13:46:26 2013 +0200

    Add batch records as a cursor option to the VSM cursor
    
    If the option isn't set, batch records are never returned from the cursor.
    
    If the option is set, batch records are reported, and the next call to
    the cursor will return the first record after the batch. Calling
    application will then need to know how to use VSL_NEXT pointer macros
    to extract the records in the batch.

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 5f30c0c..03d6e6c 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -202,6 +202,7 @@ void VSL_ResetError(struct VSL_data *vsl);
 	 */
 
 #define VSL_COPT_TAIL	(1 << 0)
+#define VSL_COPT_BATCH	(1 << 1)
 struct VSL_cursor *VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm,
     unsigned options);
        /*
@@ -210,6 +211,7 @@ struct VSL_cursor *VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm,
 	*
 	* Options:
 	*   VSL_COPT_TAIL	Start cursor at log tail
+	*   VSL_COPT_BATCH	Return batch records
 	*
 	* Return values:
 	* non-NULL: Pointer to cursor
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index e030776..14634fe 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -172,6 +172,15 @@ vslc_vsm_next(struct VSL_cursor *cursor)
 
 		c->cursor.rec = c->next;
 		c->next.ptr = VSL_NEXT(c->next.ptr);
+		if (VSL_TAG(c->cursor.rec.ptr) == SLT__Batch) {
+			if (!(c->options & VSL_COPT_BATCH))
+				/* Skip the batch record */
+				continue;
+			/* Next call will point to the first record past
+			   the batch */
+			c->next.ptr +=
+			    VSL_WORDS(VSL_BATCHLEN(c->cursor.rec.ptr));
+		}
 		return (1);
 	}
 }
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 6912943..5038a89 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -1116,6 +1116,7 @@ vslq_raw(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 		vslq->raw.c.cursor.rec.ptr = NULL;
 		vslq->raw.trans.vxid = VSL_ID(vslq->raw.c.ptr);
 		vslq->raw.offset += VSL_NEXT(vslq->raw.c.ptr) - vslq->raw.c.ptr;
+		assert(VSL_TAG(vslq->raw.c.ptr) != SLT__Batch);
 	} while (VSL_TAG(vslq->raw.c.ptr) == SLT__Batch);
 
 	if (func == NULL)
@@ -1181,6 +1182,7 @@ vslq_next(struct VSLQ *vslq)
 		return (i);
 
 	tag = VSL_TAG(c->rec.ptr);
+	assert(tag != SLT__Batch);
 	if (tag == SLT__Batch) {
 		batch = 1;
 		vxid = VSL_BATCHID(c->rec.ptr);



From martin at varnish-cache.org  Wed Oct  9 14:03:04 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 09 Oct 2013 16:03:04 +0200
Subject: [master] 2da2ff7 Use VSL_COPT_BATCH in VSL_Dispatch
Message-ID: <E1VTuLw-0005f4-Bt@project.varnish-software.com>

commit 2da2ff753d853b97010879938073594af53f9515
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 9 14:32:05 2013 +0200

    Use VSL_COPT_BATCH in VSL_Dispatch

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 5038a89..2007f09 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -1106,8 +1106,13 @@ vslq_raw(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 				return (i);
 			AN(vslq->c->rec.ptr);
 			vslq->raw.start = vslq->c->rec;
-			vslq->raw.len = VSL_NEXT(vslq->raw.start.ptr) -
-			    vslq->raw.start.ptr;
+			if (VSL_TAG(vslq->c->rec.ptr) == SLT__Batch)
+				vslq->raw.len = VSL_END(vslq->c->rec.ptr,
+				    VSL_BATCHLEN(vslq->c->rec.ptr)) -
+				    vslq->c->rec.ptr;
+			else
+				vslq->raw.len = VSL_NEXT(vslq->raw.start.ptr) -
+				    vslq->raw.start.ptr;
 			assert(vslq->raw.len > 0);
 			vslq->raw.offset = 0;
 		}
@@ -1116,7 +1121,6 @@ vslq_raw(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 		vslq->raw.c.cursor.rec.ptr = NULL;
 		vslq->raw.trans.vxid = VSL_ID(vslq->raw.c.ptr);
 		vslq->raw.offset += VSL_NEXT(vslq->raw.c.ptr) - vslq->raw.c.ptr;
-		assert(VSL_TAG(vslq->raw.c.ptr) != SLT__Batch);
 	} while (VSL_TAG(vslq->raw.c.ptr) == SLT__Batch);
 
 	if (func == NULL)
@@ -1170,7 +1174,7 @@ static int
 vslq_next(struct VSLQ *vslq)
 {
 	struct VSL_cursor *c;
-	int i, batch;
+	int i;
 	enum VSL_tag_e tag;
 	ssize_t len;
 	unsigned vxid;
@@ -1182,9 +1186,7 @@ vslq_next(struct VSLQ *vslq)
 		return (i);
 
 	tag = VSL_TAG(c->rec.ptr);
-	assert(tag != SLT__Batch);
 	if (tag == SLT__Batch) {
-		batch = 1;
 		vxid = VSL_BATCHID(c->rec.ptr);
 		len = VSL_END(c->rec.ptr, VSL_BATCHLEN(c->rec.ptr)) -
 		    c->rec.ptr;
@@ -1192,7 +1194,6 @@ vslq_next(struct VSLQ *vslq)
 			return (i);
 		tag = VSL_TAG(VSL_NEXT(c->rec.ptr));
 	} else {
-		batch = 0;
 		vxid = VSL_ID(c->rec.ptr);
 		len = VSL_NEXT(c->rec.ptr) - c->rec.ptr;
 	}
@@ -1210,8 +1211,6 @@ vslq_next(struct VSLQ *vslq)
 		vtx_append(vslq, vtx, &c->rec, len);
 		vtx_scan(vslq, vtx);
 	}
-	if (batch)
-		AZ(vsl_skip(c, VSL_WORDS(VSL_BATCHLEN(c->rec.ptr))));
 
 	return (i);
 }
diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index 6510256..8061722 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -191,7 +191,7 @@ VUT_Setup(void)
 			VUT_Error(1, "Can't open VSM file (%s)",
 			    VSM_Error(VUT.vsm));
 		c = VSL_CursorVSM(VUT.vsl, VUT.vsm,
-		    VUT.d_opt ? 0 : VSL_COPT_TAIL);
+		    (VUT.d_opt ? 0 : VSL_COPT_TAIL) | VSL_COPT_BATCH);
 	}
 	if (c == NULL)
 		VUT_Error(1, "Can't open log (%s)", VSL_Error(VUT.vsl));
@@ -295,7 +295,8 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 				VSM_ResetError(VUT.vsm);
 				continue;
 			}
-			c = VSL_CursorVSM(VUT.vsl, VUT.vsm, VSL_COPT_TAIL);
+			c = VSL_CursorVSM(VUT.vsl, VUT.vsm,
+			    VSL_COPT_TAIL | VSL_COPT_BATCH);
 			if (c == NULL) {
 				VSL_ResetError(VUT.vsl);
 				VSM_Close(VUT.vsm);



From martin at varnish-cache.org  Wed Oct  9 14:03:04 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 09 Oct 2013 16:03:04 +0200
Subject: [master] 924d0a9 Remove badly defined,
 hidden and now unused cursor skip functionality
Message-ID: <E1VTuLw-0005fC-Ft@project.varnish-software.com>

commit 924d0a9c8a44ed838180b187f7a24911167dc105
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 9 14:32:35 2013 +0200

    Remove badly defined, hidden and now unused cursor skip functionality

diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index 40e3438..782fed8 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -39,14 +39,12 @@
 /*lint -esym(534, vsl_diag) */
 int vsl_diag(struct VSL_data *vsl, const char *fmt, ...)
     __printflike(2, 3);
-int vsl_skip(struct VSL_cursor *c, ssize_t words);
 void vsl_vbm_bitset(int bit, void *priv);
 void vsl_vbm_bitclr(int bit, void *priv);
 
 typedef void vslc_delete_f(struct VSL_cursor *);
 typedef int vslc_next_f(struct VSL_cursor *);
 typedef int vslc_reset_f(struct VSL_cursor *);
-typedef int vslc_skip_f(struct VSL_cursor *, ssize_t words);
 typedef int vslc_check_f(const struct VSL_cursor *, const struct VSLC_ptr *);
 
 struct vslc_tbl {
@@ -56,7 +54,6 @@ struct vslc_tbl {
 	vslc_delete_f			*delete;
 	vslc_next_f			*next;
 	vslc_reset_f			*reset;
-	vslc_skip_f			*skip;
 	vslc_check_f			*check;
 };
 
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index 14634fe..f15c686 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -211,30 +211,11 @@ vslc_vsm_reset(struct VSL_cursor *cursor)
 	return (0);
 }
 
-static int
-vslc_vsm_skip(struct VSL_cursor *cursor, ssize_t words)
-{
-	struct vslc_vsm *c;
-
-	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VSM_MAGIC);
-	assert(&c->cursor == cursor);
-	if (words < 0)
-		return (-1);
-
-	c->next.ptr += words;
-	assert(c->next.ptr >= c->head->log);
-	assert(c->next.ptr < c->end);
-	c->cursor.rec.ptr = NULL;
-
-	return (0);
-}
-
 static const struct vslc_tbl vslc_vsm_tbl = {
 	.magic		= VSLC_TBL_MAGIC,
 	.delete		= vslc_vsm_delete,
 	.next		= vslc_vsm_next,
 	.reset		= vslc_vsm_reset,
-	.skip		= vslc_vsm_skip,
 	.check		= vslc_vsm_check,
 };
 
@@ -391,7 +372,6 @@ static const struct vslc_tbl vslc_file_tbl = {
 	.delete		= vslc_file_delete,
 	.next		= vslc_file_next,
 	.reset		= vslc_file_reset,
-	.skip		= NULL,
 	.check		= NULL,
 };
 
@@ -488,17 +468,6 @@ VSL_Next(struct VSL_cursor *cursor)
 }
 
 int
-vsl_skip(struct VSL_cursor *cursor, ssize_t words)
-{
-	const struct vslc_tbl *tbl;
-
-	CAST_OBJ_NOTNULL(tbl, cursor->priv_tbl, VSLC_TBL_MAGIC);
-	if (tbl->skip == NULL)
-		return (-1);
-	return ((tbl->skip)(cursor, words));
-}
-
-int
 VSL_Check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr)
 {
 	const struct vslc_tbl *tbl;
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 2007f09..3a572d6 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -245,7 +245,6 @@ static const struct vslc_tbl vslc_raw_tbl = {
 	.delete	= NULL,
 	.next	= vslc_raw_next,
 	.reset	= vslc_raw_reset,
-	.skip	= NULL,
 	.check	= NULL,
 };
 
@@ -322,7 +321,6 @@ static const struct vslc_tbl vslc_vtx_tbl = {
 	.delete	= NULL,
 	.next	= vslc_vtx_next,
 	.reset	= vslc_vtx_reset,
-	.skip	= NULL,
 	.check	= NULL,
 };
 



From martin at varnish-cache.org  Wed Oct  9 14:03:04 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 09 Oct 2013 16:03:04 +0200
Subject: [master] f3c3e0e Also flush stdout before waiting for more data
Message-ID: <E1VTuLw-0005fK-Hg@project.varnish-software.com>

commit f3c3e0ee4681f2480b638993780a14396dea5fcd
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 9 14:48:17 2013 +0200

    Also flush stdout before waiting for more data

diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index 8061722..caf80e6 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -203,7 +203,8 @@ VUT_Setup(void)
 		if (VUT.fo == NULL)
 			VUT_Error(1, "Can't open output file (%s)",
 			    VSL_Error(VUT.vsl));
-	}
+	} else
+		VUT.fo = stdout;
 
 	/* Create query */
 	VUT.vslq = VSLQ_New(VUT.vsl, &c, VUT.g_arg, VUT.q_arg);
@@ -270,6 +271,7 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 			func = VSL_WriteTransactions;
 		else
 			func = VSL_PrintTransactions;
+		AN(VUT.fo);
 		priv = VUT.fo;
 	}
 
@@ -344,6 +346,8 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 
 	if (VUT.vslq != NULL)
 		VSLQ_Flush(VUT.vslq, func, priv);
+	if (VUT.fo)
+		fflush(VUT.fo);
 
 	return (i);
 }



From martin at varnish-cache.org  Wed Oct  9 14:03:04 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 09 Oct 2013 16:03:04 +0200
Subject: [master] f268e93 Add a SIGUSR1 handler to flush outstanding
 transactions
Message-ID: <E1VTuLw-0005fS-Jf@project.varnish-software.com>

commit f268e93bb8636201f046ae2cbdaca560e74e47b5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 9 15:08:55 2013 +0200

    Add a SIGUSR1 handler to flush outstanding transactions
    
    Also don't flush on SIGINT exit

diff --git a/include/vut.h b/include/vut.h
index 2ba148c..51f2552 100644
--- a/include/vut.h
+++ b/include/vut.h
@@ -51,6 +51,7 @@ struct VUT {
 	struct vpf_fh	*pfh;
 	int		sighup;
 	int		sigint;
+	int		sigusr1;
 };
 
 extern struct VUT VUT;
diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index caf80e6..d318c45 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -76,6 +76,13 @@ vut_sigint(int sig)
 	VUT.sigint = 1;
 }
 
+static void
+vut_sigusr1(int sig)
+{
+	(void)sig;
+	VUT.sigusr1 = 1;
+}
+
 void
 VUT_Error(int status, const char *fmt, ...)
 {
@@ -216,6 +223,7 @@ VUT_Setup(void)
 	(void)signal(SIGHUP, vut_sighup);
 	(void)signal(SIGINT, vut_sigint);
 	(void)signal(SIGTERM, vut_sigint);
+	(void)signal(SIGUSR1, vut_sigusr1);
 
 	/* Open PID file */
 	if (VUT.P_arg) {
@@ -288,6 +296,13 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 				    VSL_Error(VUT.vsl));
 		}
 
+		if (VUT.sigusr1) {
+			/* Flush and report any incomplete records */
+			VUT.sigusr1 = 0;
+			if (VUT.vslq != NULL)
+				VSLQ_Flush(VUT.vslq, func, priv);
+		}
+
 		if (VUT.vslq == NULL) {
 			/* Reconnect VSM */
 			AZ(VUT.r_arg);
@@ -344,8 +359,6 @@ VUT_Main(VSLQ_dispatch_f *func, void *priv)
 		}
 	}
 
-	if (VUT.vslq != NULL)
-		VSLQ_Flush(VUT.vslq, func, priv);
 	if (VUT.fo)
 		fflush(VUT.fo);
 



From martin at varnish-cache.org  Wed Oct  9 14:03:04 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 09 Oct 2013 16:03:04 +0200
Subject: [master] 5b63ab7 Document the signals varnishlog supports
Message-ID: <E1VTuLw-0005fa-MB@project.varnish-software.com>

commit 5b63ab758fcfe2a6b519308828024368b45f3286
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 9 15:10:03 2013 +0200

    Document the signals varnishlog supports

diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index 5014da8..0127d3d 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -55,6 +55,17 @@ The following options are available:
 
 	XXX: Not yet implemented
 
+SIGNALS
+=======
+
+* SIGHUP
+
+  Rotate the log file (see -w option)
+
+* SIGUSR1
+
+  Flush any outstanding transactions
+
 TAGS
 ====
 The following log entry tags are currently defined:



From phk at varnish-cache.org  Wed Oct  9 21:54:46 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 09 Oct 2013 23:54:46 +0200
Subject: [master] 2fc2e84 Even more cleanup.
Message-ID: <E1VU1iQ-0005IQ-Gn@project.varnish-software.com>

commit 2fc2e842fa475da846a947840a18bc310f37a034
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 9 21:53:11 2013 +0000

    Even more cleanup.
    
    Now EXP_Rearm() can take the timers to change.
    
    Close a race where we could have an object be both born and killed
    in the same slot on the inbox.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 5c249f5..b606f49 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -909,7 +909,8 @@ double EXP_Ttl(const struct req *, const struct object*);
 void EXP_Insert(struct objcore *oc);
 void EXP_Inject(struct objcore *oc, struct lru *lru, double when);
 void EXP_Init(void);
-void EXP_Rearm(const struct object *o);
+void EXP_Rearm(struct object *o, double now, double ttl, double grace,
+    double keep);
 int EXP_Touch(struct objcore *oc);
 int EXP_NukeOne(struct busyobj *, struct lru *lru);
 void EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru);
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index dbad851..3fd7bb2 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -62,6 +62,7 @@
 
 #include "config.h"
 
+#include <math.h>
 #include <pcre.h>
 #include <stdio.h>
 
@@ -907,13 +908,9 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 		oc_updatemeta(oc);
 		return (0);
 	} else {
-		EXP_Clr(&o->exp);
 		oc->ban = NULL;
-		oc_updatemeta(oc);
-		/* BAN also changed, but that is not important any more */
-		/* XXX: no req in lurker */
 		VSLb(vsl, SLT_ExpBan, "%u was banned", o->vxid);
-		EXP_Rearm(o);
+		EXP_Rearm(o, o->exp.t_origin, 0, 0, 0);	// XXX fake now
 		return (1);
 	}
 }
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 794f6c7..a6c802a 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -215,7 +215,7 @@ EXP_Touch(struct objcore *oc)
  */
 
 void
-EXP_Rearm(const struct object *o)
+EXP_Rearm(struct object *o, double now, double ttl, double grace, double keep)
 {
 	struct objcore *oc;
 	struct lru *lru;
@@ -227,6 +227,13 @@ EXP_Rearm(const struct object *o)
 		return;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
+	if (!isnan(ttl))
+		o->exp.ttl = now + ttl - o->exp.t_origin;
+	if (!isnan(grace))
+		o->exp.grace = grace;
+	if (!isnan(keep))
+		o->exp.keep = keep;
+
 	when = exp_when(o);
 
 	VSL(SLT_ExpKill, 0, "EXP_Rearm %p %.9f %.9f 0x%x", oc,
@@ -241,7 +248,7 @@ EXP_Rearm(const struct object *o)
 	Lck_Lock(&lru->mtx);
 	AN(oc->flags & OC_F_EXP);
 
-	if (when < 0)
+	if (!isnan(now) && when <= now)
 		oc->flags |= OC_F_DYING;
 	else
 		oc->flags |= OC_F_MOVE;
@@ -422,9 +429,13 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 	Lck_Unlock(&lru->mtx);
 
 	if (flags & OC_F_DYING) {
-		assert(oc->timer_idx != BINHEAP_NOIDX);
-		binheap_delete(ep->heap, oc->timer_idx);
-		assert(oc->timer_idx == BINHEAP_NOIDX);
+		VSLb(&ep->vsl, SLT_ExpKill, "EXP_KILL %p %.9f 0x%x", oc,
+		    oc->timer_when, oc->flags);
+		if (!(flags & OC_F_INSERT)) {
+			assert(oc->timer_idx != BINHEAP_NOIDX);
+			binheap_delete(ep->heap, oc->timer_idx);
+			assert(oc->timer_idx == BINHEAP_NOIDX);
+		}
 		(void)HSH_DerefObjCore(&ep->wrk->stats, &oc);
 		return;
 	}
@@ -437,7 +448,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 	}
 
 	VSLb(&ep->vsl, SLT_ExpKill, "EXP_WHEN %p %.9f 0x%x", oc,
-	    oc->timer_when, oc->flags);
+	    oc->timer_when, flags);
 
 	/*
 	 * XXX: There are some pathological cases here, were we
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index ce98926..5b90f94 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -61,6 +61,7 @@
 
 #include "hash/hash_slinger.h"
 #include "vsha256.h"
+#include "vtim.h"
 
 static const struct hash_slinger *hash;
 static struct objhead *private_oh;
@@ -558,6 +559,7 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
 	struct objcore *oc, **ocp;
 	unsigned spc, nobj, n;
 	struct object *o;
+	double now;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
@@ -566,10 +568,11 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
 	Lck_Lock(&oh->mtx);
 	assert(oh->refcnt > 0);
 	nobj = 0;
+	now = VTIM_real();
 	VTAILQ_FOREACH(oc, &oh->objcs, list) {
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 		assert(oc->objhead == oh);
-		if (oc->flags & OC_F_BUSY) {
+		if (oc->flags & (OC_F_BUSY|OC_F_DYING)) {
 			/*
 			 * We cannot purge busy objects here, because their
 			 * owners have special rights to them, and may nuke
@@ -578,6 +581,7 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
 			 */
 			continue;
 		}
+		xxxassert(spc > sizeof *ocp);
 
 		(void)oc_getobj(&wrk->stats, oc);
 		    /* XXX: still needed ? */
@@ -589,11 +593,6 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
 	}
 	Lck_Unlock(&oh->mtx);
 
-	/* NB: inverse test to catch NAN also */
-	if (!(ttl > 0.))
-		ttl = -1.;
-	if (!(grace > 0.))
-		grace = -1.;
 	for (n = 0; n < nobj; n++) {
 		oc = ocp[n];
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
@@ -601,9 +600,7 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
 		if (o == NULL)
 			continue;
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-		o->exp.ttl = ttl;
-		o->exp.grace = grace;
-		EXP_Rearm(o);
+		EXP_Rearm(o, now, ttl, grace, NAN);	// XXX: Keep ?
 		(void)HSH_DerefObj(&wrk->stats, &o);
 	}
 	WS_Release(wrk->aws, 0);
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 3e0a73e..ec97408 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -30,6 +30,7 @@
  */
 #include "config.h"
 
+#include <math.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -478,12 +479,12 @@ VRT_r_##which##_##fld(const struct vrt_ctx *ctx)		\
 }
 
 VRT_DO_EXP(obj, ctx->req->obj->exp, grace, 0, ctx->req->t_req,
-   EXP_Rearm(ctx->req->obj);)
+   EXP_Rearm(ctx->req->obj, ctx->req->t_req, NAN, NAN, NAN);)
 VRT_DO_EXP(obj, ctx->req->obj->exp, ttl,
    (ctx->req->t_req - ctx->req->obj->exp.t_origin), ctx->req->t_req,
-   EXP_Rearm(ctx->req->obj);)
+   EXP_Rearm(ctx->req->obj, ctx->req->t_req, NAN, NAN, NAN);)
 VRT_DO_EXP(obj, ctx->req->obj->exp, keep, 0, ctx->req->t_req,
-   EXP_Rearm(ctx->req->obj);)
+   EXP_Rearm(ctx->req->obj, ctx->req->t_req, NAN, NAN, NAN);)
 
 VRT_DO_EXP(beresp, ctx->bo->exp, grace, 0, ctx->bo->exp.t_origin,)
 VRT_DO_EXP(beresp, ctx->bo->exp, ttl, 0, ctx->bo->exp.t_origin,)
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index 5bdba03..e57a758 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -34,6 +34,7 @@
 
 #include "config.h"
 
+#include <math.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -471,7 +472,7 @@ smp_oc_getobj(struct dstat *ds, struct objcore *oc)
 		oc->flags &= ~OC_F_NEEDFIXUP;
 	}
 	Lck_Unlock(&sg->sc->mtx);
-	EXP_Rearm(o);
+	EXP_Rearm(o, NAN, NAN, NAN, NAN); 	// XXX: Shouldn't be needed
 	return (o);
 }
 



From phk at varnish-cache.org  Thu Oct 10 09:44:10 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 10 Oct 2013 11:44:10 +0200
Subject: [master] 9c81b8e Use exit() rather than _exit() so we can dump gcov
 info.
Message-ID: <E1VUCmw-0004QW-SX@project.varnish-software.com>

commit 9c81b8e0ccfada542ef5c44083b8a11f18dd05d4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 10 09:43:51 2013 +0000

    Use exit() rather than _exit() so we can dump gcov info.

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index 05adb14..c4ef35c 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -301,7 +301,7 @@ start_test(void)
 			(void)close(sfd);
 		retval = exec_file(jp->tst->filename, jp->tst->script,
 		    jp->tmpdir, jp->buf, jp->bufsiz);
-		_exit(retval);
+		exit(retval);
 	}
 	AZ(close(p[1]));
 



From martin at varnish-cache.org  Thu Oct 10 14:48:39 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 10 Oct 2013 16:48:39 +0200
Subject: [master] 60d5c6d Move VUT options from vapi_options.h to vut_options.h
Message-ID: <E1VUHXb-0003Qm-91@project.varnish-software.com>

commit 60d5c6dad665c6c9944ccaef4d0a4f82b77bbbee
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 9 18:23:21 2013 +0200

    Move VUT options from vapi_options.h to vut_options.h
    
    Some options were defined in the incorrect file. Move them where they
    should be.

diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h
index 1c1df7c..4ca0dd5 100644
--- a/bin/varnishlog/varnishlog_options.h
+++ b/bin/varnishlog/varnishlog_options.h
@@ -30,23 +30,23 @@
 #include "vapi/vapi_options.h"
 #include "vut_options.h"
 
-VSL_OPT_a
+VUT_OPT_a
 VSL_OPT_b
 VSL_OPT_c
-VSL_OPT_d
+VUT_OPT_d
 VUT_OPT_D
-VSL_OPT_g
+VUT_OPT_g
 VSL_OPT_i
 VSL_OPT_I
 VSL_OPT_L
-VSM_OPT_n
-VSM_OPT_N
+VUT_OPT_n
+VUT_OPT_N
 VUT_OPT_P
 VUT_OPT_q
-VSL_OPT_r
+VUT_OPT_r
 VSL_OPT_T
-VSL_OPT_u
+VUT_OPT_u
 VSL_OPT_v
-VSL_OPT_w
+VUT_OPT_w
 VSL_OPT_x
 VSL_OPT_X
diff --git a/include/vapi/vapi_options.h b/include/vapi/vapi_options.h
index 344b3ed..e2ddcfa 100644
--- a/include/vapi/vapi_options.h
+++ b/include/vapi/vapi_options.h
@@ -27,32 +27,12 @@
  * SUCH DAMAGE.
  */
 
-/* VSM options */
-
-#define VSM_OPT_n							\
-	VOPT("n:", "[-n name]", "Varnish instance name",		\
-	    "Specify the name of the varnishd instance to get logs"	\
-	    " from. If -n is not specified, the host name is used."	\
-	)
-
-#define VSM_OPT_N							\
-	VOPT("N:", "[-N filename]", "VSM filename",			\
-	    "Specify the filename of a stale VSM instance. When using"	\
-	    " this option the abandonment checking is disabled."	\
-	)
-
 /* VSL options */
 
 #define VSL_iI_PS							\
 	"If a tag include option is the first of any tag selection"	\
 	" options, all tags are first marked excluded."
 
-#define VSL_OPT_a							\
-	VOPT("a", "[-a]", "Append binary file output",			\
-	    "When writing binary output to a file, append to it rather"	\
-	    " than overwrite it."					\
-	)
-
 #define VSL_OPT_b							\
 	VOPT("b", "[-b]", "Only display backend records",		\
 	    "Only display transactions and log records coming from"	\
@@ -65,18 +45,6 @@
 	    " client communication."					\
 	)
 
-#define VSL_OPT_d \
-	VOPT("d", "[-d]", "Process old log entries on startup",		\
-	    "Start processing log records at the head of the log"	\
-	    " instead of the tail."					\
-	)
-
-#define VSL_OPT_g							\
-	VOPT("g:", "[-g <session|request|vxid|raw>]", "Grouping mode",	\
-	    "The grouping of the log records. The default is to group"	\
-	    " by request."						\
-	)
-
 #define VSL_OPT_i							\
 	VOPT("i:", "[-i taglist]", "Include tags",			\
 	    "Include log records of these tags in output. Taglist is"   \
@@ -104,11 +72,6 @@
 	    " running queries. Defaults to 1000 transactions."		\
 	)
 
-#define VSL_OPT_r							\
-	VOPT("r:", "[-r filename]", "Binary file input",		\
-	    "Read log in binary file format from this file."		\
-	)
-
 #define VSL_OPT_T							\
 	VOPT("T:", "[-T seconds]", "Transaction end timeout",		\
 	    "Sets the transaction timeout in seconds. This defines the"	\
@@ -118,11 +81,6 @@
 	    " completed. Defaults to 120 seconds."			\
 	)
 
-#define VSL_OPT_u							\
-	VOPT("u", "[-u]", "Binary file output unbuffered",		\
-	    "Unbuffered binary file output mode."			\
-	)
-
 #define VSL_OPT_v							\
 	VOPT("v", "[-v]", "Verbose record printing",			\
 	    "Use verbose output on record set printing, giving the"	\
@@ -130,15 +88,6 @@
 	    " will only be given on the header of that transaction."	\
 	)
 
-#define VSL_OPT_w							\
-	VOPT("w:", "[-w filename]", "Binary output filename",		\
-	    "Write log entries to this file instead of displaying"	\
-	    " them. The file will be overwritten unless the -a option"	\
-	    " was specified. If the application receives a SIGHUP"	\
-	    " while writing to a file, it will reopen the file"		\
-	    " allowing the old one to be rotated away."			\
-	)
-
 #define VSL_OPT_x							\
 	VOPT("x:", "[-x taglist]", "Exclude tags",			\
 	    "Exclude log records of these tags in output. Taglist is"   \
diff --git a/include/vut_options.h b/include/vut_options.h
index 7328b91..cc25ae8 100644
--- a/include/vut_options.h
+++ b/include/vut_options.h
@@ -29,11 +29,41 @@
 
 /* VUT options */
 
+#define VUT_OPT_a							\
+	VOPT("a", "[-a]", "Append binary file output",			\
+	    "When writing binary output to a file, append to it rather"	\
+	    " than overwrite it."					\
+	)
+
+#define VUT_OPT_d \
+	VOPT("d", "[-d]", "Process old log entries on startup",		\
+	    "Start processing log records at the head of the log"	\
+	    " instead of the tail."					\
+	)
+
 #define VUT_OPT_D							\
 	VOPT("D", "[-D]", "Daemonize",					\
 	    "Daemonize."						\
 	)
 
+#define VUT_OPT_g							\
+	VOPT("g:", "[-g <session|request|vxid|raw>]", "Grouping mode",	\
+	    "The grouping of the log records. The default is to group"	\
+	    " by request."						\
+	)
+
+#define VUT_OPT_n							\
+	VOPT("n:", "[-n name]", "Varnish instance name",		\
+	    "Specify the name of the varnishd instance to get logs"	\
+	    " from. If -n is not specified, the host name is used."	\
+	)
+
+#define VUT_OPT_N							\
+	VOPT("N:", "[-N filename]", "VSM filename",			\
+	    "Specify the filename of a stale VSM instance. When using"	\
+	    " this option the abandonment checking is disabled."	\
+	)
+
 #define VUT_OPT_P							\
 	VOPT("P:", "[-P file]", "PID file",				\
 		"Write the process' PID to the specified file."		\
@@ -43,3 +73,22 @@
 	VOPT("q:", "[-q query]", "VSL query",				\
 		"Specifies the VSL query to use."			\
 	)
+
+#define VUT_OPT_r							\
+	VOPT("r:", "[-r filename]", "Binary file input",		\
+	    "Read log in binary file format from this file."		\
+	)
+
+#define VUT_OPT_u							\
+	VOPT("u", "[-u]", "Binary file output unbuffered",		\
+	    "Unbuffered binary file output mode."			\
+	)
+
+#define VUT_OPT_w							\
+	VOPT("w:", "[-w filename]", "Binary output filename",		\
+	    "Write log entries to this file instead of displaying"	\
+	    " them. The file will be overwritten unless the -a option"	\
+	    " was specified. If the application receives a SIGHUP"	\
+	    " while writing to a file, it will reopen the file"		\
+	    " allowing the old one to be rotated away."			\
+	)



From martin at varnish-cache.org  Thu Oct 10 14:48:39 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 10 Oct 2013 16:48:39 +0200
Subject: [master] 0de1ebe Enable -N option in VUT
Message-ID: <E1VUHXb-0003Qt-Ar@project.varnish-software.com>

commit 0de1ebe013ae87c65fe85d563fc8a60eedae0e5f
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 10 12:26:03 2013 +0200

    Enable -N option in VUT

diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 8862129..34f3851 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -117,5 +117,6 @@ LIBVARNISHAPI_1.3 {
 	VSLQ_Name2Grouping;
 	VSL_Glob2Tags;
 	VSL_List2Tags;
+	VSM_N_Arg;
 	# Variables:
 } LIBVARNISHAPI_1.0;
diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index d318c45..709b6bd 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -139,6 +139,15 @@ VUT_Arg(int opt, const char *arg)
 		if (VSM_n_Arg(VUT.vsm, arg) <= 0)
 			VUT_Error(1, "%s", VSM_Error(VUT.vsm));
 		return (1);
+	case 'N':
+		/* Varnish stale VSM file */
+		if (VUT.vsm == NULL)
+			VUT.vsm = VSM_New();
+		AN(VUT.vsm);
+		if (VSM_N_Arg(VUT.vsm, arg) <= 0)
+			VUT_Error(1, "%s", VSM_Error(VUT.vsm));
+		VUT.d_opt = 1;	/* Enforces -d */
+		return (1);
 	case 'P':
 		/* PID file */
 		REPLACE(VUT.P_arg, arg);



From martin at varnish-cache.org  Thu Oct 10 14:48:39 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 10 Oct 2013 16:48:39 +0200
Subject: [master] b2a04a0 Add support for caseless string comparison and
 regular expression evaluation.
Message-ID: <E1VUHXb-0003R1-Ci@project.varnish-software.com>

commit b2a04a0aa6b738bf6e879e92f444b02016f6b14e
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 10 14:17:06 2013 +0200

    Add support for caseless string comparison and regular expression
    evaluation.

diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h
index 4ca0dd5..f03b2ce 100644
--- a/bin/varnishlog/varnishlog_options.h
+++ b/bin/varnishlog/varnishlog_options.h
@@ -33,6 +33,7 @@
 VUT_OPT_a
 VSL_OPT_b
 VSL_OPT_c
+VSL_OPT_C
 VUT_OPT_d
 VUT_OPT_D
 VUT_OPT_g
diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index 0127d3d..50329e1 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -29,12 +29,6 @@ The following options are available:
 
 .. include:: ../../../bin/varnishlog/varnishlog_options.rst
 
--C
-
-	Ignore case when matching regular expressions.
-
-	XXX: Not yet implemented
-
 -k num
 
 	Only show the first num log transactions (or log records
diff --git a/include/vapi/vapi_options.h b/include/vapi/vapi_options.h
index e2ddcfa..c3cbaa6 100644
--- a/include/vapi/vapi_options.h
+++ b/include/vapi/vapi_options.h
@@ -45,6 +45,11 @@
 	    " client communication."					\
 	)
 
+#define VSL_OPT_C							\
+	VOPT("C", "[-C]", "Caseless regular expressions",		\
+	    "Do all regular expression and string matching caseless."	\
+	)
+
 #define VSL_OPT_i							\
 	VOPT("i:", "[-i taglist]", "Include tags",			\
 	    "Include log records of these tags in output. Taglist is"   \
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index 782fed8..c7996a9 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -87,6 +87,7 @@ struct VSL_data {
 
 	int				b_opt;
 	int				c_opt;
+	int				C_opt;
 	int				L_opt;
 	double				T_opt;
 	int				v_opt;
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 9370ed9..eb0eda1 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -286,7 +286,7 @@ vsl_IX_arg(struct VSL_data *vsl, int opt, const char *arg)
 		b = e + 1;
 	}
 
-	vre = VRE_compile(b, 0, &err, &off);
+	vre = VRE_compile(b, vsl->C_opt ? VRE_CASELESS : 0, &err, &off);
 	if (vre == NULL) {
 		if (tags)
 			vbit_destroy(tags);
@@ -326,6 +326,10 @@ VSL_Arg(struct VSL_data *vsl, int opt, const char *arg)
 	switch (opt) {
 	case 'b': vsl->b_opt = 1; return (1);
 	case 'c': vsl->c_opt = 1; return (1);
+	case 'C':
+		/* Caseless regular expressions */
+		vsl->C_opt = 1;
+		return (1);
 	case 'i': case 'x': return (vsl_ix_arg(vsl, opt, arg));
 	case 'I': case 'X': return (vsl_IX_arg(vsl, opt, arg));
 	case 'L':
diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index 6ec44b4..ba8b173 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -168,15 +168,27 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 		VSLQ_TEST_NUMOP(rhs->type, lhs, >=, rhs->val);
 	case T_SEQ:		/* eq */
 		assert(rhs->type == VEX_STRING);
-		if (e - b == rhs->val_stringlen &&
-		    !strncmp(b, rhs->val_string, e - b))
-			return (1);
-		return (0);
+		if (e - b != rhs->val_stringlen)
+			return (0);
+		if (vex->options & VEX_OPT_CASELESS) {
+			if (strncasecmp(b, rhs->val_string, e - b))
+				return (0);
+		} else {
+			if (strncmp(b, rhs->val_string, e - b))
+				return (0);
+		}
+		return (1);
 	case T_SNEQ:		/* ne */
 		assert(rhs->type == VEX_STRING);
-		if (e - b != rhs->val_stringlen ||
-		    strncmp(b, rhs->val_string, e - b))
+		if (e - b != rhs->val_stringlen)
 			return (1);
+		if (vex->options & VEX_OPT_CASELESS) {
+			if (strncasecmp(b, rhs->val_string, e - b))
+				return (1);
+		} else {
+			if (strncmp(b, rhs->val_string, e - b))
+				return (1);
+		}
 		return (0);
 	case '~':		/* ~ */
 		assert(rhs->type == VEX_REGEX && rhs->val_regex != NULL);
@@ -294,7 +306,7 @@ vslq_newquery(struct VSL_data *vsl, enum VSL_grouping_e grouping,
 
 	vsb = VSB_new_auto();
 	AN(vsb);
-	vex = vex_New(querystring, vsb);
+	vex = vex_New(querystring, vsb, vsl->C_opt ? VEX_OPT_CASELESS : 0);
 	AZ(VSB_finish(vsb));
 	if (vex == NULL)
 		vsl_diag(vsl, "%s", VSB_data(vsb));
diff --git a/lib/libvarnishapi/vxp.c b/lib/libvarnishapi/vxp.c
index fffa550..75a7614 100644
--- a/lib/libvarnishapi/vxp.c
+++ b/lib/libvarnishapi/vxp.c
@@ -195,7 +195,7 @@ vxp_Delete(struct vxp **pvxp)
 }
 
 struct vex *
-vex_New(const char *query, struct vsb *sb)
+vex_New(const char *query, struct vsb *sb, unsigned options)
 {
 	struct vxp *vxp;
 	struct vex *vex;
@@ -205,6 +205,9 @@ vex_New(const char *query, struct vsb *sb)
 	vxp = vxp_New(sb);
 	vxp->b = query;
 	vxp->e = query + strlen(query);
+	vxp->vex_options = options;
+	if (options & VEX_OPT_CASELESS)
+		vxp->vre_options |= VRE_CASELESS;
 
 	vxp_Lexer(vxp);
 
diff --git a/lib/libvarnishapi/vxp.h b/lib/libvarnishapi/vxp.h
index 20df7ed..1e773a5 100644
--- a/lib/libvarnishapi/vxp.h
+++ b/lib/libvarnishapi/vxp.h
@@ -70,6 +70,9 @@ struct vxp {
 	VTAILQ_HEAD(, membit)	membits;
 	struct token		*t;
 
+	unsigned		vex_options;
+	int			vre_options;
+
 	struct vsb		*sb;
 	int			err;
 };
@@ -115,6 +118,7 @@ struct vex {
 	unsigned		magic;
 #define VEX_MAGIC		0xC7DB792D
 	unsigned		tok;
+	unsigned		options;
 	struct vex		*a, *b;
 	struct vex_lhs		*lhs;
 	struct vex_rhs		*rhs;
@@ -137,8 +141,8 @@ void vxp_Lexer(struct vxp *vxp);
 struct vex * vxp_Parse(struct vxp *vxp);
 
 /* API internal interface */
-
-struct vex * vex_New(const char *query, struct vsb *sb);
+#define VEX_OPT_CASELESS	(1 << 0)
+struct vex * vex_New(const char *query, struct vsb *sb, unsigned options);
 void vex_Free(struct vex **pvex);
 
 /* Debug routines */
diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index 99a7999..01f5272 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -49,6 +49,17 @@
 
 static void vxp_expr_or(struct vxp *vxp, struct vex **pvex);
 
+static struct vex *
+vex_alloc(struct vxp *vxp)
+{
+	struct vex *vex;
+
+	ALLOC_OBJ(vex, VEX_MAGIC);
+	AN(vex);
+	vex->options = vxp->vex_options;
+	return (vex);
+}
+
 static void
 vxp_expr_lhs(struct vxp *vxp, struct vex_lhs **plhs)
 {
@@ -244,7 +255,8 @@ vxp_expr_regex(struct vxp *vxp, struct vex_rhs **prhs)
 	AN(*prhs);
 	(*prhs)->type = VEX_REGEX;
 	(*prhs)->val_string = strdup(vxp->t->dec);
-	(*prhs)->val_regex = VRE_compile(vxp->t->dec, 0, &errptr, &erroff);
+	(*prhs)->val_regex = VRE_compile(vxp->t->dec, vxp->vre_options,
+	    &errptr, &erroff);
 	if ((*prhs)->val_regex == NULL) {
 		AN(errptr);
 		VSB_printf(vxp->sb, "Regular expression error: %s ", errptr);
@@ -267,7 +279,7 @@ vxp_expr_cmp(struct vxp *vxp, struct vex **pvex)
 
 	AN(pvex);
 	AZ(*pvex);
-	ALLOC_OBJ(*pvex, VEX_MAGIC);
+	*pvex = vex_alloc(vxp);
 	AN(*pvex);
 	vxp_expr_lhs(vxp, &(*pvex)->lhs);
 	ERRCHK(vxp);
@@ -373,7 +385,7 @@ vxp_expr_not(struct vxp *vxp, struct vex **pvex)
 	AZ(*pvex);
 
 	if (vxp->t->tok == T_NOT) {
-		ALLOC_OBJ(*pvex, VEX_MAGIC);
+		*pvex = vex_alloc(vxp);
 		AN(*pvex);
 		(*pvex)->tok = vxp->t->tok;
 		vxp_NextToken(vxp);
@@ -402,7 +414,7 @@ vxp_expr_and(struct vxp *vxp, struct vex **pvex)
 	ERRCHK(vxp);
 	while (vxp->t->tok == T_AND) {
 		a = *pvex;
-		ALLOC_OBJ(*pvex, VEX_MAGIC);
+		*pvex = vex_alloc(vxp);
 		AN(*pvex);
 		(*pvex)->tok = vxp->t->tok;
 		(*pvex)->a = a;
@@ -430,7 +442,7 @@ vxp_expr_or(struct vxp *vxp, struct vex **pvex)
 	ERRCHK(vxp);
 	while (vxp->t->tok == T_OR) {
 		a = *pvex;
-		ALLOC_OBJ(*pvex, VEX_MAGIC);
+		*pvex = vex_alloc(vxp);
 		AN(*pvex);
 		(*pvex)->tok = vxp->t->tok;
 		(*pvex)->a = a;



From martin at varnish-cache.org  Thu Oct 10 14:48:39 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 10 Oct 2013 16:48:39 +0200
Subject: [master] 075e6e6 Implement -V (version) for vut (varnishlog)
Message-ID: <E1VUHXb-0003RA-Eh@project.varnish-software.com>

commit 075e6e64060ce0be47cd3b8b36dfac780386e791
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 10 14:32:35 2013 +0200

    Implement -V (version) for vut (varnishlog)

diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index a10c595..84a23ae 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -50,11 +50,13 @@
 #include "vtim.h"
 #include "vut.h"
 
+static const char progname[] = "varnishlog";
+
 static void
 usage(void)
 {
 	const char **opt;
-	fprintf(stderr, "Usage: varnishlog <options>\n\n");
+	fprintf(stderr, "Usage: %s <options>\n\n", progname);
 	fprintf(stderr, "Options:\n");
 	for (opt = vopt_usage; *opt != NULL; opt += 2)
 		fprintf(stderr, "  %-25s %s\n", *opt, *(opt + 1));
@@ -66,7 +68,7 @@ main(int argc, char * const *argv)
 {
 	char opt;
 
-	VUT_Init();
+	VUT_Init(progname);
 
 	while ((opt = getopt(argc, argv, vopt_optstring)) != -1) {
 		switch (opt) {
diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h
index f03b2ce..b58c54f 100644
--- a/bin/varnishlog/varnishlog_options.h
+++ b/bin/varnishlog/varnishlog_options.h
@@ -48,6 +48,7 @@ VUT_OPT_r
 VSL_OPT_T
 VUT_OPT_u
 VSL_OPT_v
+VUT_OPT_V
 VUT_OPT_w
 VSL_OPT_x
 VSL_OPT_X
diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index 50329e1..6021fd4 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -43,12 +43,6 @@ The following options are available:
 
 	XXX: Not yet implemented
 
--V
-
-	Display the version number and exit.
-
-	XXX: Not yet implemented
-
 SIGNALS
 =======
 
diff --git a/include/vut.h b/include/vut.h
index 51f2552..5ba134b 100644
--- a/include/vut.h
+++ b/include/vut.h
@@ -32,6 +32,8 @@
 #include "vdef.h"
 
 struct VUT {
+	const char	*progname;
+
 	/* Options */
 	int		a_opt;
 	int		d_opt;
@@ -65,7 +67,7 @@ int VUT_Arg(int opt, const char *arg);
 
 void VUT_Setup(void);
 
-void VUT_Init(void);
+void VUT_Init(const char *progname);
 
 void VUT_Fini(void);
 
diff --git a/include/vut_options.h b/include/vut_options.h
index cc25ae8..feab4ab 100644
--- a/include/vut_options.h
+++ b/include/vut_options.h
@@ -84,6 +84,11 @@
 	    "Unbuffered binary file output mode."			\
 	)
 
+#define VUT_OPT_V							\
+	VOPT("V", "[-V]", "Version",					\
+	    "Print version information and exit."			\
+	)
+
 #define VUT_OPT_w							\
 	VOPT("w:", "[-w filename]", "Binary output filename",		\
 	    "Write log entries to this file instead of displaying"	\
diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index 709b6bd..8f454ca 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -48,6 +48,7 @@
 #include "vtim.h"
 #include "vas.h"
 #include "miniobj.h"
+#include "vcs.h"
 
 #include "vut.h"
 
@@ -164,6 +165,10 @@ VUT_Arg(int opt, const char *arg)
 		/* Unbuffered binary output */
 		VUT.u_opt = 1;
 		return (1);
+	case 'V':
+		/* Print version number and exit */
+		VCS_Message(VUT.progname);
+		exit(1);
 	case 'w':
 		/* Binary file output */
 		REPLACE(VUT.w_arg, arg);
@@ -178,8 +183,9 @@ VUT_Arg(int opt, const char *arg)
 }
 
 void
-VUT_Init(void)
+VUT_Init(const char *progname)
 {
+	VUT.progname = progname;
 	VUT.g_arg = VSL_g_vxid;
 	AZ(VUT.vsl);
 	VUT.vsl = VSL_New();



From martin at varnish-cache.org  Thu Oct 10 14:48:39 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 10 Oct 2013 16:48:39 +0200
Subject: [master] fa0f8b4 Add -h option to print usage and exit
Message-ID: <E1VUHXb-0003RI-GS@project.varnish-software.com>

commit fa0f8b407595212ca987dde08be808caab5a3c83
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 10 14:54:52 2013 +0200

    Add -h option to print usage and exit

diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index 84a23ae..571c7d7 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -53,14 +53,14 @@
 static const char progname[] = "varnishlog";
 
 static void
-usage(void)
+usage(int status)
 {
 	const char **opt;
 	fprintf(stderr, "Usage: %s <options>\n\n", progname);
 	fprintf(stderr, "Options:\n");
 	for (opt = vopt_usage; *opt != NULL; opt += 2)
 		fprintf(stderr, "  %-25s %s\n", *opt, *(opt + 1));
-	exit(1);
+	exit(status);
 }
 
 int
@@ -72,14 +72,16 @@ main(int argc, char * const *argv)
 
 	while ((opt = getopt(argc, argv, vopt_optstring)) != -1) {
 		switch (opt) {
+		case 'h':
+			usage(0);
 		default:
 			if (!VUT_Arg(opt, optarg))
-				usage();
+				usage(1);
 		}
 	}
 
 	if (optind != argc)
-		usage();
+		usage(1);
 
 	VUT_Setup();
 	VUT_Main(NULL, NULL);
diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h
index b58c54f..b1e4c67 100644
--- a/bin/varnishlog/varnishlog_options.h
+++ b/bin/varnishlog/varnishlog_options.h
@@ -37,6 +37,7 @@ VSL_OPT_C
 VUT_OPT_d
 VUT_OPT_D
 VUT_OPT_g
+VUT_OPT_h
 VSL_OPT_i
 VSL_OPT_I
 VSL_OPT_L
diff --git a/include/vut_options.h b/include/vut_options.h
index feab4ab..730fe8a 100644
--- a/include/vut_options.h
+++ b/include/vut_options.h
@@ -52,6 +52,11 @@
 	    " by request."						\
 	)
 
+#define VUT_OPT_h							\
+	VOPT("h", "[-h]", "Usage help",					\
+	    "Print program usage and exit"				\
+	)
+
 #define VUT_OPT_n							\
 	VOPT("n:", "[-n name]", "Varnish instance name",		\
 	    "Specify the name of the varnishd instance to get logs"	\



From martin at varnish-cache.org  Thu Oct 10 14:48:39 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 10 Oct 2013 16:48:39 +0200
Subject: [master] 7aec15c Add a VSL_COPT_TAILSTOP cursor option to the VSM
 cursor
Message-ID: <E1VUHXb-0003RQ-I3@project.varnish-software.com>

commit 7aec15caa27fc9d3cdd6ffe43d91367e5d42c447
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 10 15:36:29 2013 +0200

    Add a VSL_COPT_TAILSTOP cursor option to the VSM cursor
    
    If this option is set, the cursor will return EOF when reaching the log tail.

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 03d6e6c..a68e41f 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -201,8 +201,9 @@ void VSL_ResetError(struct VSL_data *vsl);
 	 * Reset any error message.
 	 */
 
-#define VSL_COPT_TAIL	(1 << 0)
-#define VSL_COPT_BATCH	(1 << 1)
+#define VSL_COPT_TAIL		(1 << 0)
+#define VSL_COPT_BATCH		(1 << 1)
+#define VSL_COPT_TAILSTOP	(1 << 2)
 struct VSL_cursor *VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm,
     unsigned options);
        /*
@@ -212,6 +213,7 @@ struct VSL_cursor *VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm,
 	* Options:
 	*   VSL_COPT_TAIL	Start cursor at log tail
 	*   VSL_COPT_BATCH	Return batch records
+	*   VSL_COPT_TAILSTOP	Return EOF when reaching the log tail
 	*
 	* Return values:
 	* non-NULL: Pointer to cursor
@@ -267,7 +269,7 @@ int VSL_Next(struct VSL_cursor *c);
 	 * Return values:
 	 *	1:	Cursor points to next log record
 	 *	0:	End of log
-	 *     -1:	End of file (-r) (XXX / -k arg exhausted / "done")
+	 *     -1:	End of file
 	 *     -2:	Remote abandoned or closed
 	 *     -3:	Overrun
 	 *     -4:	I/O read error - see errno
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index f15c686..d37a454 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -164,7 +164,11 @@ vslc_vsm_next(struct VSL_cursor *cursor)
 				c->next.ptr = c->head->log;
 				continue;
 			}
-			return (0);
+			if (c->options & VSL_COPT_TAILSTOP)
+				/* EOF */
+				return (-1);
+			else
+				return (0);
 		}
 
 		if (c->next.ptr == c->head->log)



From martin at varnish-cache.org  Thu Oct 10 14:48:39 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 10 Oct 2013 16:48:39 +0200
Subject: [master] 047c517 Set VSL_COPT_TAILSTOP when -d is in effect
Message-ID: <E1VUHXb-0003RY-KT@project.varnish-software.com>

commit 047c517f210348d631eec949df9231c27ce7b759
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 10 15:37:10 2013 +0200

    Set VSL_COPT_TAILSTOP when -d is in effect

diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index 8f454ca..e8d022d 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -213,7 +213,8 @@ VUT_Setup(void)
 			VUT_Error(1, "Can't open VSM file (%s)",
 			    VSM_Error(VUT.vsm));
 		c = VSL_CursorVSM(VUT.vsl, VUT.vsm,
-		    (VUT.d_opt ? 0 : VSL_COPT_TAIL) | VSL_COPT_BATCH);
+		    (VUT.d_opt ? VSL_COPT_TAILSTOP : VSL_COPT_TAIL)
+		    | VSL_COPT_BATCH);
 	}
 	if (c == NULL)
 		VUT_Error(1, "Can't open log (%s)", VSL_Error(VUT.vsl));



From martin at varnish-cache.org  Sat Oct 12 13:00:55 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Sat, 12 Oct 2013 15:00:55 +0200
Subject: [master] 947eb3b correcting documented default value of -g option
Message-ID: <E1VUyoR-0008Ro-Hd@project.varnish-software.com>

commit 947eb3bb161ebacad694ad4a86ae1828096f4789
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sat Oct 12 14:57:44 2013 +0200

    correcting documented default value of -g option

diff --git a/include/vut_options.h b/include/vut_options.h
index 730fe8a..3437305 100644
--- a/include/vut_options.h
+++ b/include/vut_options.h
@@ -49,7 +49,7 @@
 #define VUT_OPT_g							\
 	VOPT("g:", "[-g <session|request|vxid|raw>]", "Grouping mode",	\
 	    "The grouping of the log records. The default is to group"	\
-	    " by request."						\
+	    " by vxid."							\
 	)
 
 #define VUT_OPT_h							\



From tfheen at varnish-cache.org  Mon Oct 14 11:02:34 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 14 Oct 2013 13:02:34 +0200
Subject: [master] f86feda Add missing BuildRequires on libedit-devel
Message-ID: <E1VVfv0-0003ym-PZ@project.varnish-software.com>

commit f86feda38bc2bd471908d21bc4bf8233717001e3
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 14 13:01:15 2013 +0200

    Add missing BuildRequires on libedit-devel

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index df56f62..2f96907 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -15,7 +15,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 # To build from git, start with a make dist, see redhat/README.redhat 
 # You will need at least automake autoconf libtool python-docutils
 #BuildRequires: automake autoconf libtool python-docutils
-BuildRequires: ncurses-devel groff pcre-devel pkgconfig
+BuildRequires: ncurses-devel groff pcre-devel pkgconfig libedit-devel
 Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses



From tfheen at varnish-cache.org  Mon Oct 14 11:02:55 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 14 Oct 2013 13:02:55 +0200
Subject: [3.0] 34bfadb Add missing BuildRequires on libedit-devel
Message-ID: <E1VVfvL-00042Y-4C@project.varnish-software.com>

commit 34bfadb159dc4e6356b21545df227235123312a0
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 14 13:01:15 2013 +0200

    Add missing BuildRequires on libedit-devel

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 7f53b98..0f69cc6 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -11,7 +11,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 # To build from git, start with a make dist, see redhat/README.redhat 
 # You will need at least automake autoconf libtool python-docutils
 #BuildRequires: automake autoconf libtool python-docutils
-BuildRequires: ncurses-devel libxslt groff pcre-devel pkgconfig
+BuildRequires: ncurses-devel libxslt groff pcre-devel pkgconfig libedit-devel
 Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses



From tfheen at varnish-cache.org  Mon Oct 14 12:23:28 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 14 Oct 2013 14:23:28 +0200
Subject: [master] a3bfe81 Bump version number
Message-ID: <E1VVhBI-0000QW-Fv@project.varnish-software.com>

commit a3bfe81129c32feb4ec2687f895a830f033e3202
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 14 14:23:12 2013 +0200

    Bump version number

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 2f96907..8a75cd6 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -3,8 +3,8 @@
 
 Summary: High-performance HTTP accelerator
 Name: varnish
-Version: 3.1.0
-Release: 0.20111006%{?v_rc}%{?dist}
+Version: 4.0.0
+Release: 0.20131014%{?v_rc}%{?dist}
 License: BSD
 Group: System Environment/Daemons
 URL: http://www.varnish-cache.org/



From tfheen at varnish-cache.org  Mon Oct 14 12:23:28 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 14 Oct 2013 14:23:28 +0200
Subject: [master] ec522bb Package up more of the tree
Message-ID: <E1VVhBI-0000Qb-I9@project.varnish-software.com>

commit ec522bb1de392b8d8df6b0608bb742d95e6b6419
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 14 14:23:26 2013 +0200

    Package up more of the tree

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 8a75cd6..a489486 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -234,6 +234,9 @@ rm -rf %{buildroot}
 %dir %{_includedir}/varnish
 %{_includedir}/varnish/*
 %{_libdir}/pkgconfig/varnishapi.pc
+/usr/share/varnish
+/usr/share/aclocal
+
 %doc LICENSE
 
 %files docs



From martin at varnish-cache.org  Tue Oct 15 09:08:33 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 15 Oct 2013 11:08:33 +0200
Subject: [master] adb21d8 Don't return early from VSL_Dispatch when cursor
 reports more data available
Message-ID: <E1VW0cD-0005W9-If@project.varnish-software.com>

commit adb21d8e5f0b10923fce47ab4f8875d0597a9456
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Oct 14 12:54:06 2013 +0200

    Don't return early from VSL_Dispatch when cursor reports more data available
    
    VSL_Dispatch would return with (1) when the cursor reports more data
    available, causing the processing of ready transaction not to be
    run. Would cause 'varnishlog -d' not to report any data in some
    situations.
    
    Add some comments about the different return points in VSL_Dispatch.

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 3a572d6..2351244 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -1255,11 +1255,13 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 	/* Check shmref list and buffer if necessary */
 	r = vslq_shmref_check(vslq);
 	if (r)
+		/* Buffering of shm ref failed */
 		return (r);
 
 	/* Process next cursor input */
 	i = vslq_next(vslq);
-	if (i)
+	if (i < 0)
+		/* Cursor reports error condition */
 		return (i);
 
 	/* Check vtx timeout */
@@ -1285,8 +1287,10 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 	if (!VTAILQ_EMPTY(&vslq->ready))
 		r = vslq_process_ready(vslq, func, priv);
 	if (r)
+		/* User return code */
 		return (r);
 
+	/* Return cursor return value */
 	return (i);
 }
 



From martin at varnish-cache.org  Tue Oct 15 09:08:33 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 15 Oct 2013 11:08:33 +0200
Subject: [master] ee212f9 Count transaction limit towards the number of
 outstanding transactions instead of the number of incomplete transactions.
Message-ID: <E1VW0cD-0005WE-Mq@project.varnish-software.com>

commit ee212f949f524cc9babcb0a0633deb0119e145f6
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Oct 14 14:45:11 2013 +0200

    Count transaction limit towards the number of outstanding transactions
    instead of the number of incomplete transactions.
    
    Session grouping mode could grab an unintended amount of memory
    because the transaction limit was counted towards the incomplete
    rather that not yet reported transactions. A long lasting session
    would only count as 1 towards the limit even if it had thousands of
    complete children that should put it above the limit.

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 2351244..43a8243 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -172,7 +172,7 @@ struct VSLQ {
 	struct vtx_tree		tree;
 	VTAILQ_HEAD(,vtx)	ready;
 	VTAILQ_HEAD(,vtx)	incomplete;
-	unsigned		n_incomplete;
+	unsigned		n_outstanding;
 	struct chunkhead	shmrefs;
 	VTAILQ_HEAD(,vtx)	cache;
 	unsigned		n_cache;
@@ -558,6 +558,8 @@ vtx_retire(struct VSLQ *vslq, struct vtx **pvtx)
 		}
 	}
 	vtx->len = 0;
+	AN(vslq->n_outstanding);
+	vslq->n_outstanding--;
 
 	if (vslq->n_cache < VTX_CACHE) {
 		VTAILQ_INSERT_HEAD(&vslq->cache, vtx, list_child);
@@ -596,7 +598,7 @@ vtx_add(struct VSLQ *vslq, unsigned vxid)
 	vtx->key.vxid = vxid;
 	AZ(VRB_INSERT(vtx_tree, &vslq->tree, &vtx->key));
 	VTAILQ_INSERT_TAIL(&vslq->incomplete, vtx, list_vtx);
-	vslq->n_incomplete++;
+	vslq->n_outstanding++;
 	return (vtx);
 }
 
@@ -615,8 +617,6 @@ vtx_mark_complete(struct VSLQ *vslq, struct vtx *vtx)
 
 	vtx->flags |= VTX_F_COMPLETE;
 	VTAILQ_REMOVE(&vslq->incomplete, vtx, list_vtx);
-	AN(vslq->n_incomplete);
-	vslq->n_incomplete--;
 
 	while (1) {
 		AZ(vtx->flags & VTX_F_READY);
@@ -1058,7 +1058,7 @@ VSLQ_Delete(struct VSLQ **pvslq)
 	CHECK_OBJ_NOTNULL(vslq, VSLQ_MAGIC);
 
 	(void)VSLQ_Flush(vslq, NULL, NULL);
-	AZ(vslq->n_incomplete);
+	AN(VTAILQ_EMPTY(&vslq->incomplete));
 
 	while (!VTAILQ_EMPTY(&vslq->ready)) {
 		vtx = VTAILQ_FIRST(&vslq->ready);
@@ -1067,6 +1067,7 @@ VSLQ_Delete(struct VSLQ **pvslq)
 		AN(vtx->flags & VTX_F_READY);
 		vtx_retire(vslq, &vtx);
 	}
+	AZ(vslq->n_outstanding);
 
 	VSL_DeleteCursor(vslq->c);
 	vslq->c = NULL;
@@ -1276,19 +1277,24 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 	}
 
 	/* Check store limit */
-	while (vslq->n_incomplete > vslq->vsl->L_opt) {
+	while (vslq->n_outstanding > vslq->vsl->L_opt) {
 		vtx = VTAILQ_FIRST(&vslq->incomplete);
 		CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
 		vtx_force(vslq, vtx, "store overflow");
 		AN(vtx->flags & VTX_F_COMPLETE);
+		r = vslq_process_ready(vslq, func, priv);
+		if (r)
+			/* User return code */
+			return (r);
 	}
 
 	/* Check ready list */
-	if (!VTAILQ_EMPTY(&vslq->ready))
+	if (!VTAILQ_EMPTY(&vslq->ready)) {
 		r = vslq_process_ready(vslq, func, priv);
-	if (r)
-		/* User return code */
-		return (r);
+		if (r)
+			/* User return code */
+			return (r);
+	}
 
 	/* Return cursor return value */
 	return (i);
@@ -1303,7 +1309,7 @@ VSLQ_Flush(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 
 	CHECK_OBJ_NOTNULL(vslq, VSLQ_MAGIC);
 
-	while (vslq->n_incomplete) {
+	while (!VTAILQ_EMPTY(&vslq->incomplete)) {
 		vtx = VTAILQ_FIRST(&vslq->incomplete);
 		CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
 		AZ(vtx->flags & VTX_F_COMPLETE);



From martin at varnish-cache.org  Tue Oct 15 10:53:00 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 15 Oct 2013 12:53:00 +0200
Subject: [master] 5858dea varnishtest: logexpect only attempts regex matches
 for matching tags
Message-ID: <E1VW2FI-0001kp-QU@project.varnish-software.com>

commit 5858deaae774b935d7389d08a767e7c517c57b1c
Author: Geoff Simmons <geoff at uplex.de>
Date:   Thu Oct 10 10:58:37 2013 +0200

    varnishtest: logexpect only attempts regex matches for matching tags

diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c
index 8a0f384..cbee6c0 100644
--- a/bin/varnishtest/vtc_logexp.c
+++ b/bin/varnishtest/vtc_logexp.c
@@ -217,6 +217,8 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 					ok = 0;
 			}
 			if (le->test->vre &&
+			    le->test->tag >= 0 &&
+			    le->test->tag == tag &&
 			    VRE_ERROR_NOMATCH == VRE_exec(le->test->vre, data,
 				len, 0, 0, NULL, 0, NULL))
 				ok = 0;



From perbu at varnish-cache.org  Wed Oct 16 08:00:00 2013
From: perbu at varnish-cache.org (Per Buer)
Date: Wed, 16 Oct 2013 10:00:00 +0200
Subject: [master] 656cabe Expand on backslashes and quoting
Message-ID: <E1VWM1Q-0001B2-HQ@project.varnish-software.com>

commit 656cabe3de433012949be1e28419ad9034fc8ae4
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Oct 16 09:59:57 2013 +0200

    Expand on backslashes and quoting

diff --git a/doc/sphinx/users-guide/vcl-syntax.rst b/doc/sphinx/users-guide/vcl-syntax.rst
index 8277033..ce476b2 100644
--- a/doc/sphinx/users-guide/vcl-syntax.rst
+++ b/doc/sphinx/users-guide/vcl-syntax.rst
@@ -15,12 +15,21 @@ Strings
 
 Basic strings are enclosed in " ... ", and may not contain newlines.
 
+Backslash is not special, so for instance in regsub() you do not need
+to do the "count-the-backslashes" polka:::
+
+  regsub("barf", "(b)(a)(r)(f)", "\4\3\2p") -> "frap"
+
+Given that we have never been able to come up with a valid need for
+any escaped characters apart from that, we decided that %22 for " and
+%25 for %, was much less suffering than doubling all backslashes in
+regexp/regsub contexts.
+
 Long strings are enclosed in {" ... "}. They may contain any character
 including ", newline and other control characters except for the NUL
 (0x00) character. If you really want NUL characters in a string there
 is a VMOD that makes it possible to create such strings.
 
-
 Access control lists (ACLs)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 



From perbu at varnish-cache.org  Wed Oct 16 08:09:07 2013
From: perbu at varnish-cache.org (Per Buer)
Date: Wed, 16 Oct 2013 10:09:07 +0200
Subject: [master] c6e0cd0 Oops. Last commit was partially wrong. Thanks scoof
Message-ID: <E1VWMAF-0004Qa-Rs@project.varnish-software.com>

commit c6e0cd03ea8c67cbf1ae849f7e2a7b0b4fbf9ac3
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Oct 16 10:09:04 2013 +0200

    Oops. Last commit was partially wrong.
    Thanks scoof

diff --git a/doc/sphinx/users-guide/vcl-syntax.rst b/doc/sphinx/users-guide/vcl-syntax.rst
index ce476b2..c979bb5 100644
--- a/doc/sphinx/users-guide/vcl-syntax.rst
+++ b/doc/sphinx/users-guide/vcl-syntax.rst
@@ -20,11 +20,6 @@ to do the "count-the-backslashes" polka:::
 
   regsub("barf", "(b)(a)(r)(f)", "\4\3\2p") -> "frap"
 
-Given that we have never been able to come up with a valid need for
-any escaped characters apart from that, we decided that %22 for " and
-%25 for %, was much less suffering than doubling all backslashes in
-regexp/regsub contexts.
-
 Long strings are enclosed in {" ... "}. They may contain any character
 including ", newline and other control characters except for the NUL
 (0x00) character. If you really want NUL characters in a string there



From martin at varnish-cache.org  Wed Oct 16 12:39:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 16 Oct 2013 14:39:13 +0200
Subject: [master] 1cd1d7a Generate the VSL record tag documentation from the
 header tables.
Message-ID: <E1VWQNd-0002PF-JW@project.varnish-software.com>

commit 1cd1d7a53ce4deae18bf4bacbf29cd1a7b54ec5c
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 16 12:12:51 2013 +0200

    Generate the VSL record tag documentation from the header tables.
    
    This patch generates an rst file containing the record tag
    documentation from the corresponding header tables.
    
    It also adds a man-page from the vsl.rst document and includes the tag
    descriptions.
    
    The old list of tags in varnishlog.rst is removed.
    
    Fixes: #1338

diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index 6021fd4..f4cb33c 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -54,62 +54,6 @@ SIGNALS
 
   Flush any outstanding transactions
 
-TAGS
-====
-The following log entry tags are currently defined:
-
-* Backend
-* BackendClose
-* BackendOpen
-* BackendReuse
-* BackendXID
-* CLI
-* ClientAddr
-* Debug
-* Error
-* ExpBan
-* ExpKill
-* ExpPick
-* Hit
-* HitPass
-* HttpError
-* HttpGarbage
-* Length
-* ObjHeader
-* ObjLostHeader
-* ObjProtocol
-* ObjRequest
-* ObjResponse
-* ObjStatus
-* ObjURL
-* ReqEnd
-* ReqStart
-* RxHeader
-* RxLostHeader
-* RxProtocol
-* RxRequest
-* RxResponse
-* RxStatus
-* RxURL
-* SessionClose
-* SessionOpen
-* StatAddr
-* StatSess
-* TTL
-* TxHeader
-* TxLostHeader
-* TxProtocol
-* TxRequest
-* TxResponse
-* TxStatus
-* TxURL
-* VCL_acl
-* VCL_call
-* VCL_return
-* VCL_trace
-* WorkThread
-
-
 SEE ALSO
 ========
 * varnishd(1)
@@ -117,6 +61,7 @@ SEE ALSO
 * varnishncsa(1)
 * varnishstat(1)
 * varnishtop(1)
+* vsl(7)
 * vsl-query(7)
 
 HISTORY
diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst
index e9f2ff3..dbe789d 100644
--- a/doc/sphinx/reference/vsl.rst
+++ b/doc/sphinx/reference/vsl.rst
@@ -1,8 +1,18 @@
 .. _reference-vsl:
 
-=====================
+===
+VSL
+===
+
+---------------------
 Shared Memory Logging
-=====================
+---------------------
+
+:Author: Poul-Henning Kamp
+:Author: Martin Blix Grydeland
+:Date: 2013-10-16
+:Version: 1.0
+:Manual section: 7
 
 TTL records
 ~~~~~~~~~~~
@@ -61,3 +71,10 @@ Examples::
 
 	U F E 182 159 80 80 1392
 	G F E 159 173 80 1304 1314
+
+VSL tags
+~~~~~~~~
+
+Alphabetical list of all the VSL log records with description:
+
+.. include:: ../../../lib/libvarnishapi/vsl-tags.rst
diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index 6f36ef6..6316612 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -51,16 +51,31 @@ libvarnishapi_la_LDFLAGS += -export-symbols-regex '^V'
 endif
 
 EXTRA_DIST = \
-	generate.py
+	generate.py \
+	vsl-tags.rst
 
 BUILT_SOURCES = \
 	vxp_fixed_token.c \
-	vxp_tokens.h
+	vxp_tokens.h \
+	vsl-tags.rst
 
 CLEANFILES = \
 	$(builddir)/vxp_fixed_token.c \
 	$(builddir)/vxp_tokens.h
 
+MAINTAINERCLEANFILES = \
+	vsl-tags.rst
+
+noinst_PROGRAMS = vsl2rst
+
+vsl2rst_SOURCES = \
+	vsl2rst.c \
+	$(top_srcdir)/include/tbl/vsl_tags.h \
+	$(top_srcdir)/include/tbl/vsl_tags_http.h
+
+vsl-tags.rst: vsl2rst
+	./vsl2rst > $@
+
 vxp_fixed_token.c vxp_tokens.h: \
 	$(srcdir)/generate.py
 	@PYTHON@ $(srcdir)/generate.py $(srcdir) $(top_builddir)
diff --git a/lib/libvarnishapi/vsl2rst.c b/lib/libvarnishapi/vsl2rst.c
new file mode 100644
index 0000000..41ee69c
--- /dev/null
+++ b/lib/libvarnishapi/vsl2rst.c
@@ -0,0 +1,113 @@
+/*-
+ * Copyright (c) 2011-2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#include "vapi/vsl.h"
+
+struct SLT {
+	unsigned	tag;
+	const char	*name;
+	const char	*sdesc;
+	const char	*ldesc;
+};
+
+static struct SLT tags[SLT__MAX] = {
+#define SLTM(name, sdesc, ldesc) \
+	[SLT_##name] = { SLT_##name, #name, sdesc, ldesc },
+#include "tbl/vsl_tags.h"
+#undef SLTM
+};
+
+static int
+ptag_cmp(const void *va, const void *vb)
+{
+	const struct SLT *a, *b;
+
+	a = *(const struct SLT * const *)va;
+	b = *(const struct SLT * const *)vb;
+	if (a->name == NULL && b->name != NULL)
+		return (1);
+	else if (a->name != NULL && b->name == NULL)
+		return (-1);
+	else if (a->name == NULL && b->name == NULL)
+		return (0);
+	return (strcmp(a->name, b->name));
+}
+
+static void
+print_tabbed(const char *string, int tabs)
+{
+	int i;
+	const char *c;
+
+	for (c = string; *c; c++) {
+		if (c == string || *(c - 1) == '\n')
+			for (i = 0; i < tabs; i++)
+				printf("\t");
+		printf("%c", *c);
+	}
+}
+
+int
+main(int argc, char *argv[])
+{
+	int i;
+	struct SLT *ptags[SLT__MAX];
+
+	(void)argc;
+	(void)argv;
+
+	for (i = 0; i < SLT__MAX; i++)
+		ptags[i] = &tags[i];
+
+	qsort(&ptags, SLT__MAX, sizeof *ptags, &ptag_cmp);
+
+	for (i = 0; i < SLT__MAX; i++) {
+		if (ptags[i]->name == NULL || !strcmp(ptags[i]->name, ""))
+			continue;
+		printf("* %s", ptags[i]->name);
+		if (ptags[i]->sdesc != NULL && strcmp(ptags[i]->sdesc, ""))
+			printf(" (%s)", ptags[i]->sdesc);
+		printf("\n\n");
+		if (ptags[i]->ldesc != NULL && strcmp(ptags[i]->ldesc, "")) {
+			printf("\t::\n\n");
+			print_tabbed(ptags[i]->ldesc, 2);
+			printf("\n\n");
+		}
+	}
+
+	return (0);
+}
diff --git a/man/Makefile.am b/man/Makefile.am
index cc32a4b..878e7f4 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -8,7 +8,7 @@ vsc2rst_SOURCES = vsc2rst.c \
 
 AM_CPPFLAGS = -I$(top_srcdir)/include
 
-dist_man_MANS = vcl.7 varnish-cli.7 varnish-counters.7
+dist_man_MANS = vcl.7 varnish-cli.7 varnish-counters.7 vsl.7
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 
 vcl.7: $(top_srcdir)/doc/sphinx/reference/vcl.rst \
@@ -42,3 +42,14 @@ varnish-counters.7: $(top_srcdir)/include/tbl/vsc_fields.h
 	@echo "========================================"
 	@false
 endif
+
+vsl.7: $(top_srcdir)/doc/sphinx/reference/vsl.rst \
+	$(top_srcdir)/lib/libvarnishapi/vsl-tags.rst
+if HAVE_RST2MAN
+	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vsl.rst $@
+else
+	@echo "========================================"
+	@echo "You need rst2man installed to make dist"
+	@echo "========================================"
+	@false
+endif



From martin at varnish-cache.org  Wed Oct 16 12:39:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 16 Oct 2013 14:39:13 +0200
Subject: [master] 52f5eff Remove xid field from ReqEnd description
Message-ID: <E1VWQNd-0002PL-Km@project.varnish-software.com>

commit 52f5eff2aefa5ca67d4145605b975a207727d22d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 16 12:39:13 2013 +0200

    Remove xid field from ReqEnd description
    
    The xid field is not present in ReqEnd any more, as it is not needed
    now that the vxid is used as the record ID. Remove it from the
    description.

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index c2fb6d9..2bfc145 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -55,7 +55,6 @@ SLTM(CLI, "CLI communication",
 
 SLTM(ReqEnd, "Client request end",
 	"Marks the end of client request.\n\n"
-	"xid\n    Transaction id.\n\n"
 	"Trxd\n     Timestamp when the request started.\n\n"
 	"Tidle\n    Timestamp when the request ended.\n\n"
 	"dTrx\n    Time to receive request\n\n"



From martin at varnish-cache.org  Wed Oct 16 12:39:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 16 Oct 2013 14:39:13 +0200
Subject: [master] 7608168 Reformat the log record long descriptions.
Message-ID: <E1VWQNd-0002PT-Lv@project.varnish-software.com>

commit 7608168d548f1f4c835cc914f8c7e91fa31d74c2
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 16 13:07:29 2013 +0200

    Reformat the log record long descriptions.
    
    Format the long log record descriptions for use with pre-formatted
    text style.

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 2bfc145..6b3a921 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -55,11 +55,12 @@ SLTM(CLI, "CLI communication",
 
 SLTM(ReqEnd, "Client request end",
 	"Marks the end of client request.\n\n"
-	"Trxd\n     Timestamp when the request started.\n\n"
-	"Tidle\n    Timestamp when the request ended.\n\n"
-	"dTrx\n    Time to receive request\n\n"
-	"dTproc\n    Time to process request\n\n"
-	"dTtx\n    Time to transmit response\n\n"
+	"Fields:\n"
+	"  Trxd     Timestamp when the request started.\n"
+	"  Tidle    Timestamp when the request ended.\n"
+	"  dTrx     Time to receive request.\n"
+	"  dTproc   Time to process request.\n"
+	"  dTtx     Time to transmit response.\n"
 )
 
 /*---------------------------------------------------------------------*/
@@ -67,12 +68,13 @@ SLTM(ReqEnd, "Client request end",
 SLTM(SessOpen, "Client connection opened",
 	"The first record for a client connection, with the\n"
 	"socket-endpoints of the connection.\n\n"
-	"caddr\n    Client IPv4/6 address\n\n"
-	"cport\n    Client TCP port\n\n"
-	"lsock\n    Listen socket\n\n"
-	"laddr\n    Local IPv4/6 address ('-' if !$log_local_addr)\n\n"
-	"lport\n    Local TCP port ('-' if !$log_local_addr)\n\n"
-	"fd\n    File descriptor number"
+	"Fields:\n"
+	"  caddr    Client IPv4/6 address.\n"
+	"  cport    Client TCP port.\n"
+	"  lsock    Listen socket.\n"
+	"  laddr    Local IPv4/6 address ('-' if !$log_local_addr).\n"
+	"  lport    Local TCP port ('-' if !$log_local_addr).\n"
+	"  fd       File descriptor number.\n"
 )
 
 /*
@@ -87,14 +89,15 @@ SLTM(SessOpen, "Client connection opened",
 
 SLTM(SessClose, "Client connection closed",
 	"SessionClose is the last record for any client connection.\n\n"
-	"reason\n    Why the connection closed.\n\n"
-	"duration\n    How long the session were open.\n\n"
-	"Nreq\n    How many requests on session.\n\n"
-	"Npipe\n    If 'pipe' were used on session.\n\n"
-	"Npass\n    Requests handled with pass.\n\n"
-	"Nfetch\n    Backend fetches by session.\n\n"
-	"Bhdr\n    Header bytes sent on session.\n\n"
-	"Bbody\n    Body bytes sent on session.\n\n"
+	"Fields:\n"
+	"  reason   Why the connection closed.\n"
+	"  duration How long the session were open.\n"
+	"  Nreq     How many requests on session.\n"
+	"  Npipe    If 'pipe' were used on session.\n"
+	"  Npass    Requests handled with pass.\n"
+	"  Nfetch   Backend fetches by session.\n"
+	"  Bhdr     Header bytes sent on session.\n"
+	"  Bbody    Body bytes sent on session.\n"
 )
 
 /*---------------------------------------------------------------------*/
@@ -130,8 +133,9 @@ SLTM(FetchError, "Error while fetching object", "")
 #undef SLTH
 
 SLTM(BogoHeader, "Bogus HTTP received",
-	"Contains the first 20 characters of received HTTP headers we could"
-	" not make sense of.  Applies to both req.http and beres.http."
+	"Contains the first 20 characters of received HTTP headers we\n"
+	"could not make sense of.  Applies to both req.http and\n"
+	"beres.http.\n"
 )
 SLTM(LostHeader, "Failed attempt to set HTTP header", "")
 
@@ -161,30 +165,24 @@ SLTM(VCL_Error, "", "")
 SLTM(Gzip, "G(un)zip performed on object", "")
 
 SLTM(Link, "Links to a child VXID",
-	"Links this VXID to any child VXID it initiates\n"
-	"The first field gives the type of the child:\n"
-	"    req     Request\n"
-	"    bereq   Backend request\n"
-	"    esireq  ESI subrequest\n"
-	"The second field gives the VXID of the child.\n"
+	"Links this VXID to any child VXID it initiates\n\n"
+	"Fields:\n"
+	"  ctype    Child type (req, bereq or esireq).\n"
+	"  cvxid    Child vxid.\n"
 )
 
 SLTM(Begin, "Marks the start of a VXID",
-    "The first record of a VXID transaction.\n"
-    "The first field gives the type of the transaction:\n"
-    "    sess	Session\n"
-    "    req	Request\n"
-    "    bereq	Backend request\n"
-    "    esireq	ESI subrequest\n"
-    "The second field gives the VXID of the parent that initiated this"
-    " transaction. For Session transactions this field is blank.\n"
+	"The first record of a VXID transaction.\n\n"
+	"Fields:\n"
+	"  type     Transaction type (sess, req, bereq or esireq).\n"
+	"  pvxid    Parent vxid.\n"
 )
 
 SLTM(End, "Marks the end of a VXID",
-    "The last record of a VXID transaction.\n"
+	"The last record of a VXID transaction.\n"
 )
 
-SLTM(VSL, "Internally generated VSL API warnings and error message",
-    "Warnings and error messages genererated by the VSL API while reading the"
-    " shared memory log"
+SLTM(VSL, "VSL API warnings and error message",
+	"Warnings and error messages genererated by the VSL API while\n"
+	"reading the shared memory log.\n"
 )



From martin at varnish-cache.org  Wed Oct 16 12:39:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 16 Oct 2013 14:39:13 +0200
Subject: [master] 36087a5 Move the TTL and Gzip log record description to
 vsl_tags.h
Message-ID: <E1VWQNd-0002Pc-NF@project.varnish-software.com>

commit 36087a57cff71354554bc1e4a27705818a4935a3
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 16 13:30:59 2013 +0200

    Move the TTL and Gzip log record description to vsl_tags.h

diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst
index dbe789d..c51648a 100644
--- a/doc/sphinx/reference/vsl.rst
+++ b/doc/sphinx/reference/vsl.rst
@@ -14,64 +14,6 @@ Shared Memory Logging
 :Version: 1.0
 :Manual section: 7
 
-TTL records
-~~~~~~~~~~~
-
-A TTL record is emitted whenever the ttl, grace or keep values for an
-object is set.
-
-The format is::
-
-	%u %s %d %d %d %d %d [ %d %u %u ]
-	|  |  |  |  |  |  |    |  |  |
-	|  |  |  |  |  |  |    |  |  +- Max-Age from Cache-Control header
-	|  |  |  |  |  |  |    |  +---- Expires header
-	|  |  |  |  |  |  |    +------- Date header
-	|  |  |  |  |  |  +------------ Age (incl Age: header value)
-	|  |  |  |  |  +--------------- Reference time for TTL
-	|  |  |  |  +------------------ Keep
-	|  |  |  +--------------------- Grace
-	|  |  +------------------------ TTL
-	|  +--------------------------- "RFC" or "VCL"
-	+------------------------------ object XID
-
-The last three fields are only present in "RFC" headers.
-
-Examples::
-
-	1001 RFC 19 -1 -1 1312966109 4 0 0 23
-	1001 VCL 10 -1 -1 1312966109 4
-	1001 VCL 7 -1 -1 1312966111 6
-	1001 VCL 7 120 -1 1312966111 6
-	1001 VCL 7 120 3600 1312966111 6
-	1001 VCL 12 120 3600 1312966113 8
-
-Gzip records
-~~~~~~~~~~~~
-
-A Gzip record is emitted for each instance of gzip or gunzip work
-performed.
-Worst case, an ESI transaction stored in gzip'ed objects but delivered
-gunziped, will run into many of these.
-
-The format is::
-
-	%c %c %c %d %d %d %d %d
-	|  |  |  |  |  |  |  |
-	|  |  |  |  |  |  |  +- Bit length of compressed data
-	|  |  |  |  |  |  +---- Bit location of 'last' bit
-	|  |  |  |  |  +------- Bit location of first deflate block
-	|  |  |  |  +---------- Bytes output
-	|  |  |  +------------- Bytes input
-	|  |  +---------------- 'E' = ESI, '-' = Plain object
-	|  +------------------- 'F' = Fetch, 'D' = Deliver
-	+---------------------- 'G' = Gzip, 'U' = Gunzip, 'u' = Gunzip-test
-
-Examples::
-
-	U F E 182 159 80 80 1392
-	G F E 159 173 80 1304 1314
-
 VSL tags
 ~~~~~~~~
 
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 6b3a921..cad62b1 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -139,7 +139,36 @@ SLTM(BogoHeader, "Bogus HTTP received",
 )
 SLTM(LostHeader, "Failed attempt to set HTTP header", "")
 
-SLTM(TTL, "TTL set on object", "")
+SLTM(TTL, "TTL set on object",
+	"A TTL record is emitted whenever the ttl, grace or keep\n"
+	"values for an object is set.\n"
+	"\n"
+	"The format is:\n"
+	"\n"
+	"%u %s %d %d %d %d %d [ %d %u %u ]\n"
+	"|  |  |  |  |  |  |    |  |  |\n"
+	"|  |  |  |  |  |  |    |  |  +- Max-Age from Cache-Control header\n"
+	"|  |  |  |  |  |  |    |  +---- Expires header\n"
+	"|  |  |  |  |  |  |    +------- Date header\n"
+	"|  |  |  |  |  |  +------------ Age (incl Age: header value)\n"
+	"|  |  |  |  |  +--------------- Reference time for TTL\n"
+	"|  |  |  |  +------------------ Keep\n"
+	"|  |  |  +--------------------- Grace\n"
+	"|  |  +------------------------ TTL\n"
+	"|  +--------------------------- \"RFC\" or \"VCL\"\n"
+	"+------------------------------ object XID\n"
+	"\n"
+	"The last three fields are only present in \"RFC\" headers.\n"
+	"\n"
+	"Examples:\n"
+	"\n"
+	"1001 RFC 19 -1 -1 1312966109 4 0 0 23\n"
+	"1001 VCL 10 -1 -1 1312966109 4\n"
+	"1001 VCL 7 -1 -1 1312966111 6\n"
+	"1001 VCL 7 120 -1 1312966111 6\n"
+	"1001 VCL 7 120 3600 1312966111 6\n"
+	"1001 VCL 12 120 3600 1312966113 8\n"
+)
 SLTM(Fetch_Body, "Body fetched from backend", "")
 SLTM(VCL_acl, "", "")
 SLTM(VCL_call, "VCL method called", "")
@@ -162,7 +191,30 @@ SLTM(VCL_Debug, "Unused", "")
 SLTM(VCL_Log, "Log statement from VCL", "")
 SLTM(VCL_Error, "", "")
 
-SLTM(Gzip, "G(un)zip performed on object", "")
+SLTM(Gzip, "G(un)zip performed on object",
+	"A Gzip record is emitted for each instance of gzip or gunzip\n"
+	"work performed. Worst case, an ESI transaction stored in\n"
+	"gzip'ed objects but delivered gunziped, will run into many of\n"
+	"these.\n"
+	"\n"
+	"The format is:\n"
+	"\n"
+	"%c %c %c %d %d %d %d %d\n"
+	"|  |  |  |  |  |  |  |\n"
+	"|  |  |  |  |  |  |  +- Bit length of compressed data\n"
+	"|  |  |  |  |  |  +---- Bit location of 'last' bit\n"
+	"|  |  |  |  |  +------- Bit location of first deflate block\n"
+	"|  |  |  |  +---------- Bytes output\n"
+	"|  |  |  +------------- Bytes input\n"
+	"|  |  +---------------- 'E' = ESI, '-' = Plain object\n"
+	"|  +------------------- 'F' = Fetch, 'D' = Deliver\n"
+	"+---------------------- 'G' = Gzip, 'U' = Gunzip, 'u' = Gunzip-test\n"
+	"\n"
+	"Examples:\n"
+	"\n"
+	"U F E 182 159 80 80 1392\n"
+	"G F E 159 173 80 1304 1314\n"
+)
 
 SLTM(Link, "Links to a child VXID",
 	"Links this VXID to any child VXID it initiates\n\n"



From martin at varnish-cache.org  Wed Oct 16 12:39:13 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 16 Oct 2013 14:39:13 +0200
Subject: [master] c723806 Honor RST definition list format in long description
 of log records.
Message-ID: <E1VWQNd-0002Pl-Oj@project.varnish-software.com>

commit c723806d9588bdb1597b8d0f4806d3f99304b9fc
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 16 14:37:38 2013 +0200

    Honor RST definition list format in long description of log records.

diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst
index c51648a..e254719 100644
--- a/doc/sphinx/reference/vsl.rst
+++ b/doc/sphinx/reference/vsl.rst
@@ -17,6 +17,4 @@ Shared Memory Logging
 VSL tags
 ~~~~~~~~
 
-Alphabetical list of all the VSL log records with description:
-
 .. include:: ../../../lib/libvarnishapi/vsl-tags.rst
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index cad62b1..eb9b3c0 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -42,39 +42,45 @@
  */
 
 SLTM(Debug, "Debug messages",
-	"Debug messages can normally be ignored, but are sometimes\n"
-	"helpful during trouble-shooting.  Most debug messages must\n"
-	"be explicitly enabled with parameters."
+	"Debug messages can normally be ignored, but are sometimes"
+	" helpful during trouble-shooting.  Most debug messages must"
+	" be explicitly enabled with parameters.\n\n"
 )
 SLTM(Error, "Error messages",
-	"Error messages are stuff you probably want to know."
+	"Error messages are stuff you probably want to know.\n\n"
 )
 SLTM(CLI, "CLI communication",
-	"CLI communication between master and child process."
+	"CLI communication between master and child process.\n\n"
 )
 
 SLTM(ReqEnd, "Client request end",
 	"Marks the end of client request.\n\n"
-	"Fields:\n"
-	"  Trxd     Timestamp when the request started.\n"
-	"  Tidle    Timestamp when the request ended.\n"
-	"  dTrx     Time to receive request.\n"
-	"  dTproc   Time to process request.\n"
-	"  dTtx     Time to transmit response.\n"
+	"The format is::\n\n"
+	"\t%f %f %f %f %f\n"
+	"\t|  |  |  |  |\n"
+	"\t|  |  |  |  +- Time to transmit response\n"
+	"\t|  |  |  +---- Time to process request\n"
+	"\t|  |  +------- Time to receive request\n"
+	"\t|  +---------- Timestamp (since epoch) when the request ended\n"
+	"\t+------------- Timestamp (since epoch) when the request started\n"
+	"\n"
 )
 
 /*---------------------------------------------------------------------*/
 
 SLTM(SessOpen, "Client connection opened",
-	"The first record for a client connection, with the\n"
-	"socket-endpoints of the connection.\n\n"
-	"Fields:\n"
-	"  caddr    Client IPv4/6 address.\n"
-	"  cport    Client TCP port.\n"
-	"  lsock    Listen socket.\n"
-	"  laddr    Local IPv4/6 address ('-' if !$log_local_addr).\n"
-	"  lport    Local TCP port ('-' if !$log_local_addr).\n"
-	"  fd       File descriptor number.\n"
+	"The first record for a client connection, with the socket-endpoints"
+	" of the connection.\n\n"
+	"The format is::\n\n"
+	"\t%s %d %s %s %s %d\n"
+	"\t|  |  |  |  |  |\n"
+	"\t|  |  |  |  |  +- File descriptor number\n"
+	"\t|  |  |  |  +---- Local TCP port ('-' if !$log_local_addr)\n"
+	"\t|  |  |  +------- Local IPv4/6 address ('-' if !$log_local_addr)\n"
+	"\t|  |  +---------- Listen socket\n"
+	"\t|  +------------- Client TCP socket\n"
+	"\t+---------------- Client IPv4/6 address\n"
+	"\n"
 )
 
 /*
@@ -89,15 +95,18 @@ SLTM(SessOpen, "Client connection opened",
 
 SLTM(SessClose, "Client connection closed",
 	"SessionClose is the last record for any client connection.\n\n"
-	"Fields:\n"
-	"  reason   Why the connection closed.\n"
-	"  duration How long the session were open.\n"
-	"  Nreq     How many requests on session.\n"
-	"  Npipe    If 'pipe' were used on session.\n"
-	"  Npass    Requests handled with pass.\n"
-	"  Nfetch   Backend fetches by session.\n"
-	"  Bhdr     Header bytes sent on session.\n"
-	"  Bbody    Body bytes sent on session.\n"
+	"The format is::\n\n"
+	"\t%s %f %u %u %u %u %u %u\n"
+	"\t|  |  |  |  |  |  |  |\n"
+	"\t|  |  |  |  |  |  |  +- Body bytes sent on session\n"
+	"\t|  |  |  |  |  |  +---- Header bytes sent on session\n"
+	"\t|  |  |  |  |  +------- Backend fetches by session\n"
+	"\t|  |  |  |  +---------- Requests handled with pass\n"
+	"\t|  |  |  +------------- If 'pipe' were used on session\n"
+	"\t|  |  +---------------- How many requests on session\n"
+	"\t|  +------------------- How long the session was open\n"
+	"\t+---------------------- Why the connection closed\n"
+	"\n"
 )
 
 /*---------------------------------------------------------------------*/
@@ -133,41 +142,37 @@ SLTM(FetchError, "Error while fetching object", "")
 #undef SLTH
 
 SLTM(BogoHeader, "Bogus HTTP received",
-	"Contains the first 20 characters of received HTTP headers we\n"
-	"could not make sense of.  Applies to both req.http and\n"
-	"beres.http.\n"
+	"Contains the first 20 characters of received HTTP headers we could"
+	" not make sense of.  Applies to both req.http and beres.http.\n\n"
 )
 SLTM(LostHeader, "Failed attempt to set HTTP header", "")
 
 SLTM(TTL, "TTL set on object",
-	"A TTL record is emitted whenever the ttl, grace or keep\n"
-	"values for an object is set.\n"
-	"\n"
-	"The format is:\n"
-	"\n"
-	"%u %s %d %d %d %d %d [ %d %u %u ]\n"
-	"|  |  |  |  |  |  |    |  |  |\n"
-	"|  |  |  |  |  |  |    |  |  +- Max-Age from Cache-Control header\n"
-	"|  |  |  |  |  |  |    |  +---- Expires header\n"
-	"|  |  |  |  |  |  |    +------- Date header\n"
-	"|  |  |  |  |  |  +------------ Age (incl Age: header value)\n"
-	"|  |  |  |  |  +--------------- Reference time for TTL\n"
-	"|  |  |  |  +------------------ Keep\n"
-	"|  |  |  +--------------------- Grace\n"
-	"|  |  +------------------------ TTL\n"
-	"|  +--------------------------- \"RFC\" or \"VCL\"\n"
-	"+------------------------------ object XID\n"
+	"A TTL record is emitted whenever the ttl, grace or keep"
+	" values for an object is set.\n\n"
+	"The format is::\n\n"
+	"\t%u %s %d %d %d %d %d [ %d %u %u ]\n"
+	"\t|  |  |  |  |  |  |    |  |  |\n"
+	"\t|  |  |  |  |  |  |    |  |  +- Max-Age from Cache-Control header\n"
+	"\t|  |  |  |  |  |  |    |  +---- Expires header\n"
+	"\t|  |  |  |  |  |  |    +------- Date header\n"
+	"\t|  |  |  |  |  |  +------------ Age (incl Age: header value)\n"
+	"\t|  |  |  |  |  +--------------- Reference time for TTL\n"
+	"\t|  |  |  |  +------------------ Keep\n"
+	"\t|  |  |  +--------------------- Grace\n"
+	"\t|  |  +------------------------ TTL\n"
+	"\t|  +--------------------------- \"RFC\" or \"VCL\"\n"
+	"\t+------------------------------ object XID\n"
 	"\n"
-	"The last three fields are only present in \"RFC\" headers.\n"
+	"The last three fields are only present in \"RFC\" headers.\n\n"
+	"Examples::\n\n"
+	"\t1001 RFC 19 -1 -1 1312966109 4 0 0 23\n"
+	"\t1001 VCL 10 -1 -1 1312966109 4\n"
+	"\t1001 VCL 7 -1 -1 1312966111 6\n"
+	"\t1001 VCL 7 120 -1 1312966111 6\n"
+	"\t1001 VCL 7 120 3600 1312966111 6\n"
+	"\t1001 VCL 12 120 3600 1312966113 8\n"
 	"\n"
-	"Examples:\n"
-	"\n"
-	"1001 RFC 19 -1 -1 1312966109 4 0 0 23\n"
-	"1001 VCL 10 -1 -1 1312966109 4\n"
-	"1001 VCL 7 -1 -1 1312966111 6\n"
-	"1001 VCL 7 120 -1 1312966111 6\n"
-	"1001 VCL 7 120 3600 1312966111 6\n"
-	"1001 VCL 12 120 3600 1312966113 8\n"
 )
 SLTM(Fetch_Body, "Body fetched from backend", "")
 SLTM(VCL_acl, "", "")
@@ -192,49 +197,53 @@ SLTM(VCL_Log, "Log statement from VCL", "")
 SLTM(VCL_Error, "", "")
 
 SLTM(Gzip, "G(un)zip performed on object",
-	"A Gzip record is emitted for each instance of gzip or gunzip\n"
-	"work performed. Worst case, an ESI transaction stored in\n"
-	"gzip'ed objects but delivered gunziped, will run into many of\n"
-	"these.\n"
-	"\n"
-	"The format is:\n"
-	"\n"
-	"%c %c %c %d %d %d %d %d\n"
-	"|  |  |  |  |  |  |  |\n"
-	"|  |  |  |  |  |  |  +- Bit length of compressed data\n"
-	"|  |  |  |  |  |  +---- Bit location of 'last' bit\n"
-	"|  |  |  |  |  +------- Bit location of first deflate block\n"
-	"|  |  |  |  +---------- Bytes output\n"
-	"|  |  |  +------------- Bytes input\n"
-	"|  |  +---------------- 'E' = ESI, '-' = Plain object\n"
-	"|  +------------------- 'F' = Fetch, 'D' = Deliver\n"
-	"+---------------------- 'G' = Gzip, 'U' = Gunzip, 'u' = Gunzip-test\n"
+	"A Gzip record is emitted for each instance of gzip or gunzip"
+	" work performed. Worst case, an ESI transaction stored in"
+	" gzip'ed objects but delivered gunziped, will run into many of"
+	" these.\n\n"
+	"The format is::\n\n"
+	"\t%c %c %c %d %d %d %d %d\n"
+	"\t|  |  |  |  |  |  |  |\n"
+	"\t|  |  |  |  |  |  |  +- Bit length of compressed data\n"
+	"\t|  |  |  |  |  |  +---- Bit location of 'last' bit\n"
+	"\t|  |  |  |  |  +------- Bit location of first deflate block\n"
+	"\t|  |  |  |  +---------- Bytes output\n"
+	"\t|  |  |  +------------- Bytes input\n"
+	"\t|  |  +---------------- 'E': ESI, '-': Plain object\n"
+	"\t|  +------------------- 'F': Fetch, 'D': Deliver\n"
+	"\t+---------------------- 'G': Gzip, 'U': Gunzip, 'u': Gunzip-test\n"
 	"\n"
-	"Examples:\n"
+	"Examples::\n\n"
+	"\tU F E 182 159 80 80 1392\n"
+	"\tG F E 159 173 80 1304 1314\n"
 	"\n"
-	"U F E 182 159 80 80 1392\n"
-	"G F E 159 173 80 1304 1314\n"
 )
 
 SLTM(Link, "Links to a child VXID",
-	"Links this VXID to any child VXID it initiates\n\n"
-	"Fields:\n"
-	"  ctype    Child type (req, bereq or esireq).\n"
-	"  cvxid    Child vxid.\n"
+	"Links this VXID to any child VXID it initiates.\n\n"
+	"The format is::\n\n"
+	"\t%s %d\n"
+	"\t|  |\n"
+	"\t|  +- Child vxid\n"
+	"\t+---- Child type (\"req\", \"bereq\" or \"esireq\")\n"
+	"\n"
 )
 
 SLTM(Begin, "Marks the start of a VXID",
 	"The first record of a VXID transaction.\n\n"
-	"Fields:\n"
-	"  type     Transaction type (sess, req, bereq or esireq).\n"
-	"  pvxid    Parent vxid.\n"
+	"The format is::\n\n"
+	"\t%s %d\n"
+	"\t|  |\n"
+	"\t|  +- Parent vxid\n"
+	"\t+---- Type (\"sess\", \"req\", \"bereq\" or \"esireq\")\n"
+	"\n"
 )
 
 SLTM(End, "Marks the end of a VXID",
-	"The last record of a VXID transaction.\n"
+	"The last record of a VXID transaction.\n\n"
 )
 
 SLTM(VSL, "VSL API warnings and error message",
-	"Warnings and error messages genererated by the VSL API while\n"
-	"reading the shared memory log.\n"
+	"Warnings and error messages genererated by the VSL API while"
+	" reading the shared memory log.\n\n"
 )
diff --git a/lib/libvarnishapi/vsl2rst.c b/lib/libvarnishapi/vsl2rst.c
index 41ee69c..26ee3ee 100644
--- a/lib/libvarnishapi/vsl2rst.c
+++ b/lib/libvarnishapi/vsl2rst.c
@@ -98,15 +98,14 @@ main(int argc, char *argv[])
 	for (i = 0; i < SLT__MAX; i++) {
 		if (ptags[i]->name == NULL || !strcmp(ptags[i]->name, ""))
 			continue;
-		printf("* %s", ptags[i]->name);
+		printf("%s", ptags[i]->name);
 		if (ptags[i]->sdesc != NULL && strcmp(ptags[i]->sdesc, ""))
-			printf(" (%s)", ptags[i]->sdesc);
-		printf("\n\n");
+			printf(" - %s", ptags[i]->sdesc);
+		printf("\n");
 		if (ptags[i]->ldesc != NULL && strcmp(ptags[i]->ldesc, "")) {
-			printf("\t::\n\n");
-			print_tabbed(ptags[i]->ldesc, 2);
-			printf("\n\n");
+			print_tabbed(ptags[i]->ldesc, 1);
 		}
+		printf("\n\n");
 	}
 
 	return (0);



From martin at varnish-cache.org  Wed Oct 16 14:51:50 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 16 Oct 2013 16:51:50 +0200
Subject: [master] 4502870 Silence compiler warning about possibly
 uninitialized variables.
Message-ID: <E1VWSRy-00069E-4Y@project.varnish-software.com>

commit 4502870118f6583139bb059f9208bcc2338f9c08
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 16 15:02:17 2013 +0200

    Silence compiler warning about possibly uninitialized variables.

diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c
index ba8b173..5cddbde 100644
--- a/lib/libvarnishapi/vsl_query.c
+++ b/lib/libvarnishapi/vsl_query.c
@@ -72,8 +72,8 @@ static int
 vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec)
 {
 	const struct vex_rhs *rhs;
-	long long lhs_int;
-	double lhs_float;
+	long long lhs_int = 0;
+	double lhs_float = 0.;
 	const char *b, *e;
 	char *p;
 	int i;



From martin at varnish-cache.org  Wed Oct 16 14:51:50 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 16 Oct 2013 16:51:50 +0200
Subject: [master] 06a23d7 Don't list unused log record tags in the
 documentation.
Message-ID: <E1VWSRy-00069J-6I@project.varnish-software.com>

commit 06a23d75a8b5a713fb3f0fb010df0d7f346a7fb5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 16 16:08:36 2013 +0200

    Don't list unused log record tags in the documentation.
    
    If the short description of a log record contains the string
    "(unused)" it will not be listed in the documentation.

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index eb9b3c0..347e2ec 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -192,7 +192,7 @@ SLTM(Hash, "Value added to hash", "")
 
 SLTM(Backend_health, "Backend health check", "")
 
-SLTM(VCL_Debug, "Unused", "")
+SLTM(VCL_Debug, "(unused)", "")
 SLTM(VCL_Log, "Log statement from VCL", "")
 SLTM(VCL_Error, "", "")
 
diff --git a/lib/libvarnishapi/vsl2rst.c b/lib/libvarnishapi/vsl2rst.c
index 26ee3ee..44f1f7c 100644
--- a/lib/libvarnishapi/vsl2rst.c
+++ b/lib/libvarnishapi/vsl2rst.c
@@ -98,6 +98,11 @@ main(int argc, char *argv[])
 	for (i = 0; i < SLT__MAX; i++) {
 		if (ptags[i]->name == NULL || !strcmp(ptags[i]->name, ""))
 			continue;
+		if (ptags[i]->sdesc != NULL &&
+		    strstr(ptags[i]->sdesc, "(unused)"))
+			/* Don't list tags where the short description
+			   contains the string "(unused)" */
+			continue;
 		printf("%s", ptags[i]->name);
 		if (ptags[i]->sdesc != NULL && strcmp(ptags[i]->sdesc, ""))
 			printf(" - %s", ptags[i]->sdesc);



From martin at varnish-cache.org  Wed Oct 16 14:51:50 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 16 Oct 2013 16:51:50 +0200
Subject: [master] d78f82d Provide short and long descriptions for the request
 response log records.
Message-ID: <E1VWSRy-00069R-7X@project.varnish-software.com>

commit d78f82ddc1484dcc81ed031816f73ec4d739d531
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 16 16:09:44 2013 +0200

    Provide short and long descriptions for the request response log records.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b606f49..1ce24fa 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -98,7 +98,7 @@ enum sess_close {
 
 enum {
 	/* Fields from the first line of HTTP proto */
-#define SLTH(aa, bb)	bb,
+#define SLTH(tag, ind, req, resp, sdesc, ldesc)	ind,
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 };
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 347e2ec..d33247e 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -121,23 +121,29 @@ SLTM(Length, "Size of object body", "")
 
 SLTM(FetchError, "Error while fetching object", "")
 
-#define SLTH(aa, bb)	SLTM(Req##aa, "", "")
+#define SLTH(tag, ind, req, resp, sdesc, ldesc) \
+	SLTM(Req##tag, (req ? "Client request " sdesc : "(unused)"), ldesc)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
-#define SLTH(aa, bb)	SLTM(Resp##aa, "", "")
+#define SLTH(tag, ind, req, resp, sdesc, ldesc) \
+	SLTM(Resp##tag, (resp ? "Client response " sdesc : "(unused)"), ldesc)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
-#define SLTH(aa, bb)	SLTM(Bereq##aa, "", "")
+#define SLTH(tag, ind, req, resp, sdesc, ldesc) \
+	SLTM(Bereq##tag, (req ? "Backend request " sdesc : "(unused)"), ldesc)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
-#define SLTH(aa, bb)	SLTM(Beresp##aa, "", "")
+#define SLTH(tag, ind, req, resp, sdesc, ldesc) \
+	SLTM(Beresp##tag, (resp ? "Backend response " sdesc : "(unused)"), \
+	    ldesc)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
-#define SLTH(aa, bb)	SLTM(Obj##aa, "", "")
+#define SLTH(tag, ind, req, resp, sdesc, ldesc) \
+	SLTM(Obj##tag, (resp ? "Object  " sdesc : "(unused)"), ldesc)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
diff --git a/include/tbl/vsl_tags_http.h b/include/tbl/vsl_tags_http.h
index f515492..55b98fc 100644
--- a/include/tbl/vsl_tags_http.h
+++ b/include/tbl/vsl_tags_http.h
@@ -30,12 +30,40 @@
  * The order of this table is not random, do not resort.
  * In particular, the FIRST and LOST entries must be last, in that order.
  *
+ * Arguments:
+ *	Tag-Name
+ *	struct http header index
+ *	1 if this header is used in requests
+ *	1 if this header is used in responses
+ *	short description postfix
+ *	long description (in RST "definition list" format)
+ *
  */
 
-SLTH(Method,	HTTP_HDR_METHOD)
-SLTH(URL,	HTTP_HDR_URL)
-SLTH(Protocol,	HTTP_HDR_PROTO)
-SLTH(Status,	HTTP_HDR_STATUS)
-SLTH(Response,	HTTP_HDR_RESPONSE)
-SLTH(Header,	HTTP_HDR_FIRST)
-SLTH(Lost,	HTTP_HDR_LOST)
+SLTH(Method,	HTTP_HDR_METHOD,	1, 0, "method",
+	"The HTTP request method used.\n\n"
+)
+SLTH(URL,	HTTP_HDR_URL,		1, 0, "URL",
+	"The HTTP request URL.\n\n"
+)
+SLTH(Protocol,	HTTP_HDR_PROTO,		1, 1, "protocol",
+	"The HTTP protocol version information.\n\n"
+)
+SLTH(Status,	HTTP_HDR_STATUS,	0, 1, "status",
+	"The HTTP status code received.\n\n"
+)
+SLTH(Response,	HTTP_HDR_RESPONSE,	0, 1, "response",
+	"The HTTP response string received.\n\n"
+)
+SLTH(Header,	HTTP_HDR_FIRST,		1, 1, "header",
+	"HTTP header contents.\n\n"
+	"The format is::\n\n"
+	"\t%s: %s\n"
+	"\t|   |\n"
+	"\t|   +- Header value\n"
+	"\t+----- Header name\n"
+	"\n"
+)
+SLTH(Lost,	HTTP_HDR_LOST,		1, 1, "(unused)",
+	""
+)



From martin at varnish-cache.org  Wed Oct 16 14:51:50 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 16 Oct 2013 16:51:50 +0200
Subject: [master] 823c986 Document more log records
Message-ID: <E1VWSRy-00069Z-8n@project.varnish-software.com>

commit 823c986cdbad88042353705c5f56086ade26ca6b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 16 16:48:08 2013 +0200

    Document more log records

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index d33247e..6bf6646 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -180,7 +180,16 @@ SLTM(TTL, "TTL set on object",
 	"\t1001 VCL 12 120 3600 1312966113 8\n"
 	"\n"
 )
-SLTM(Fetch_Body, "Body fetched from backend", "")
+SLTM(Fetch_Body, "Body fetched from backend",
+	"Finished fetching body from backend.\n\n"
+	"The format is::\n\n"
+	"\t%d(%s) cls %d\n"
+	"\t|  |       |\n"
+	"\t|  |       +- 1 if the backend connection was closed\n"
+	"\t|  +--------- Text description of body status\n"
+	"\t+------------ Body status\n"
+	"\n"
+)
 SLTM(VCL_acl, "", "")
 SLTM(VCL_call, "VCL method called", "")
 SLTM(VCL_trace, "VCL trace data", "")
@@ -192,14 +201,35 @@ SLTM(ExpBan, "Object evicted due to ban", "")
 SLTM(ExpKill, "Object expired", "")
 SLTM(WorkThread, "", "")
 
-SLTM(ESI_xmlerror, "Error while parsing ESI tags", "")
+SLTM(ESI_xmlerror, "ESI parser error or warning message",
+	"An error or warning was generated during parsing of an ESI object."
+	" The log record describes the problem encountered."
+)
 
-SLTM(Hash, "Value added to hash", "")
+SLTM(Hash, "Value added to hash",
+	"This value was added to the object lookup hash."
+)
 
-SLTM(Backend_health, "Backend health check", "")
+SLTM(Backend_health, "Backend health check",
+	"The result of a backend health probe.\n\n"
+	"The format is::\n\n"
+	"\t%s %s %s %u %u %u %f %f %s\n"
+	"\t|  |  |  |  |  |  |  |  |\n"
+	"\t|  |  |  |  |  |  |  |  +- Probe HTTP response\n"
+	"\t|  |  |  |  |  |  |  +---- Average response time\n"
+	"\t|  |  |  |  |  |  +------- Response time\n"
+	"\t|  |  |  |  |  +---------- Probe window size\n"
+	"\t|  |  |  |  +------------- Probe threshold level\n"
+	"\t|  |  |  +---------------- Number of good probes in window\n"
+	"\t|  |  +------------------- Probe window bits\n"
+	"\t|  +---------------------- Status message\n"
+	"\t+------------------------- Backend name\n"
+)
 
 SLTM(VCL_Debug, "(unused)", "")
-SLTM(VCL_Log, "Log statement from VCL", "")
+SLTM(VCL_Log, "Log statement from VCL",
+	"User generated log messages insert from VCL through std.log()"
+)
 SLTM(VCL_Error, "", "")
 
 SLTM(Gzip, "G(un)zip performed on object",



From phk at varnish-cache.org  Thu Oct 17 08:28:50 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 17 Oct 2013 10:28:50 +0200
Subject: [master] 6610974 Polishing
Message-ID: <E1VWiws-0002LP-Mr@project.varnish-software.com>

commit 6610974463078569cec5555b404af2eb8f359647
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 17 08:28:38 2013 +0000

    Polishing

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index a6c802a..193ddb0 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -291,7 +291,7 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 		/*
 		 * It wont release any space if we cannot release the last
 		 * reference, besides, if somebody else has a reference,
-		 * it's a bad idea to nuke this object anyway. 
+		 * it's a bad idea to nuke this object anyway.
 		 */
 		if (oc->refcnt > 1)
 			continue;
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 5031eab..1e0d7e4 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -91,6 +91,7 @@ mgt_sltm(const char *tag, const char *sdesc, const char *ldesc)
 
 }
 
+/*lint -e{506} constant value boolean */
 static void
 mgt_DumpRstVsl(void)
 {
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index e57a758..35af470 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -472,7 +472,7 @@ smp_oc_getobj(struct dstat *ds, struct objcore *oc)
 		oc->flags &= ~OC_F_NEEDFIXUP;
 	}
 	Lck_Unlock(&sg->sc->mtx);
-	EXP_Rearm(o, NAN, NAN, NAN, NAN); 	// XXX: Shouldn't be needed
+	EXP_Rearm(o, NAN, NAN, NAN, NAN);	// XXX: Shouldn't be needed
 	return (o);
 }
 



From phk at varnish-cache.org  Thu Oct 17 10:07:19 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 17 Oct 2013 12:07:19 +0200
Subject: [master] 5c03d65 Add a feature 'esi_remove_bom' which will make ESI
 ignore and remove UTF-8 BOM's at the start of an ESI-object.
Message-ID: <E1VWkUB-0005fC-2T@project.varnish-software.com>

commit 5c03d65b8f63adccabce33d827203cf9d259bb6d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 17 09:36:55 2013 +0000

    Add a feature 'esi_remove_bom' which will make ESI ignore and remove
    UTF-8 BOM's at the start of an ESI-object.
    
    Notice that the removal only happens if the file is actually ESI
    processed on delivery, so to get BOM removal for non-XML files,
    you may have to disable the XML test and insert a dummy ESI directive
    such as <esi:remove>BOMs Be Gone!<esi:remove/> or similar.
    
    Fixes   #1355

diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c
index 880d3d2..48482c9 100644
--- a/bin/varnishd/cache/cache_esi_parse.c
+++ b/bin/varnishd/cache/cache_esi_parse.c
@@ -121,6 +121,7 @@ struct vep_state {
 /*---------------------------------------------------------------------*/
 
 static const char * const VEP_START =		"[Start]";
+static const char * const VEP_BOM =		"[BOM]";
 static const char * const VEP_TESTXML =		"[TestXml]";
 static const char * const VEP_NOTXML =		"[NotXml]";
 
@@ -175,6 +176,13 @@ static struct vep_match vep_match_attr_include[] = {
 	{ NULL,		&VEP_SKIPATTR }
 };
 
+/*---------------------------------------------------------------------*/
+
+static struct vep_match vep_match_bom[] = {
+	{ "\xeb\xbb\xbf",	&VEP_START },
+	{ NULL,			&VEP_BOM }
+};
+
 /*--------------------------------------------------------------------
  * Report a parsing error
  */
@@ -594,6 +602,13 @@ VEP_Parse(const struct busyobj *bo, const char *p, size_t l)
 		 */
 
 		if (vep->state == VEP_START) {
+			if (FEATURE(FEATURE_ESI_REMOVE_BOM) && *p == '\xeb') {
+				vep->match = vep_match_bom;
+				vep->state = VEP_MATCH;
+			} else
+				vep->state = VEP_BOM;
+		} else if (vep->state == VEP_BOM) {
+			vep_mark_skip(vep, p);
 			if (FEATURE(FEATURE_ESI_DISABLE_XML_CHECK))
 				vep->state = VEP_NEXTTAG;
 			else
@@ -609,6 +624,12 @@ VEP_Parse(const struct busyobj *bo, const char *p, size_t l)
 			if (p < e && *p == '<') {
 				p++;
 				vep->state = VEP_STARTTAG;
+			} else if (p < e && *p == '\xeb') {
+				VSLb(vep->bo->vsl, SLT_ESI_xmlerror,
+				    "No ESI processing, first char not '<'"
+				    " (BOM? see feature esi_remove_bom)"
+				);
+				vep->state = VEP_NOTXML;
 			} else if (p < e) {
 				VSLb(vep->bo->vsl, SLT_ESI_xmlerror,
 				    "No ESI processing, first char not '<'");
diff --git a/bin/varnishtest/tests/r01355.vtc b/bin/varnishtest/tests/r01355.vtc
new file mode 100644
index 0000000..0aa132b
--- /dev/null
+++ b/bin/varnishtest/tests/r01355.vtc
@@ -0,0 +1,52 @@
+varnishtest "Test ESI ignoring BOMs"
+
+server s1 {
+	rxreq
+	expect req.url == /1
+	txresp -body "\xeb\xbb\xbf    <html>   <esi:remove>blabla</esi:remove>"
+	rxreq
+	expect req.url == /2
+	txresp -body "\xeb\xbb\xbf    <html>   <esi:remove>blabla</esi:remove>"
+	rxreq
+	expect req.url == /3
+	txresp -body "\xeb\xbb\xbf\xeb\xbb\xbf    <html>   <esi:remove>blabla</esi:remove>"
+	rxreq
+	expect req.url == /4
+	txresp -body "\xeb\xbc    <html>   <esi:remove>blabla</esi:remove>"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
+} -start
+
+client c1 {
+	# No ESI processing
+	txreq -url /1
+	rxresp
+	expect resp.bodylen == 47
+} -run
+
+varnish v1 -cliok "param.set feature +esi_remove_bom"
+
+client c1 {
+	# BOM removed, ESI processing
+	txreq -url /2
+	rxresp
+	expect resp.bodylen == 13
+} -run
+
+client c1 {
+	# BOMs removed, ESI processing
+	txreq -url /3
+	rxresp
+	expect resp.bodylen == 13
+} -run
+
+client c1 {
+	# Not a BOM, no ESI processing
+	txreq -url /4
+	rxresp
+	expect resp.bodylen == 46
+} -run
diff --git a/include/tbl/feature_bits.h b/include/tbl/feature_bits.h
index 3e4ecba..2f30e58 100644
--- a/include/tbl/feature_bits.h
+++ b/include/tbl/feature_bits.h
@@ -53,3 +53,8 @@ FEATURE_BIT(ESI_IGNORE_OTHER_ELEMENTS,	esi_ignore_other_elements, "",
     "Ignore non-esi XML-elements",
     "Allows syntax errors in the XML"
 )
+FEATURE_BIT(ESI_REMOVE_BOM,		esi_remove_bom, "",
+    "Remove UTF-8 BOM",
+    "Remove UTF-8 BOM from front of object."
+    "Ignore and remove the UTF-8 BOM (0xeb 0xbb 0xbf) from front of object."
+)



From phk at varnish-cache.org  Thu Oct 17 10:14:49 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 17 Oct 2013 12:14:49 +0200
Subject: [master] 1cc4f78 Fix prototype for user defined subroutines.
Message-ID: <E1VWkbR-0006QC-F4@project.varnish-software.com>

commit 1cc4f78ba39735b763669dd1022ca6fb05fbd1fc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 17 10:14:30 2013 +0000

    Fix prototype for user defined subroutines.
    
    Fixes #1357

diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index 63204a7..e161b89 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -251,9 +251,9 @@ vcc_ParseFunction(struct vcc *tl)
 		}
 		tl->curproc = vcc_AddProc(tl, tl->t);
 		Fh(tl, 0, "static int VGC_function_%.*s "
-		    "(struct vrt_ctx *ctx);\n", PF(tl->t));
+		    "(const struct vrt_ctx *ctx);\n", PF(tl->t));
 		Fc(tl, 1, "\nstatic int __match_proto__(vcl_func_t)\n");
-		Fc(tl, 1, "VGC_function_%.*s(struct vrt_ctx *ctx)\n",
+		Fc(tl, 1, "VGC_function_%.*s(const struct vrt_ctx *ctx)\n",
 		    PF(tl->t));
 	}
 	vcc_NextToken(tl);



From martin at varnish-cache.org  Thu Oct 17 17:54:36 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 17 Oct 2013 19:54:36 +0200
Subject: [master] fbabb9a Unbreak build of dist tarball when no rst2man
Message-ID: <E1VWrmO-00062r-GS@project.varnish-software.com>

commit fbabb9aebff4f6e6d3b113c7288c5911356f56ef
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 17 12:13:12 2013 +0200

    Unbreak build of dist tarball when no rst2man

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 3152943..ed24bae 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -129,6 +129,7 @@ EXTRA_DIST = \
 	reference/vmod_std.rst \
 	reference/vsm.rst \
 	reference/vsl-query.rst \
+	reference/vsl.rst \
 	tutorial/index.rst \
 	tutorial/introduction.rst \
 	tutorial/starting_varnish.rst \
diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index 6316612..fce096d 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -73,8 +73,10 @@ vsl2rst_SOURCES = \
 	$(top_srcdir)/include/tbl/vsl_tags.h \
 	$(top_srcdir)/include/tbl/vsl_tags_http.h
 
+if HAVE_RST2MAN
 vsl-tags.rst: vsl2rst
 	./vsl2rst > $@
+endif
 
 vxp_fixed_token.c vxp_tokens.h: \
 	$(srcdir)/generate.py



From tfheen at varnish-cache.org  Mon Oct 21 09:39:08 2013
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 21 Oct 2013 11:39:08 +0200
Subject: [master] a3549d8 Update README to point out how to contribute patches
 and bug reports.
Message-ID: <E1VYBx6-00035p-Tl@project.varnish-software.com>

commit a3549d8cdc7279f6ec3fff847b8a776def025257
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 21 11:37:06 2013 +0200

    Update README to point out how to contribute patches and bug reports.
    
    Thanks to James Pearson for the suggestion.

diff --git a/README b/README
index 2538063..3ce0b45 100644
--- a/README
+++ b/README
@@ -1,10 +1,12 @@
 This is Varnish Cache, the high-performance HTTP accelerator.
 
 Documentation and additional information about Varnish is available on
-http://www.varnish-cache.org/
+https://www.varnish-cache.org/
 
 Technical questions about Varnish and this release should be addressed
-to <varnish-misc at varnish-cache.org>.
+to <varnish-misc at varnish-cache.org>.  Please see
+https://www.varnish-cache.org/trac/wiki/Contributing for how to
+contribute patches and report bugs.
 
 Questions about commercial support and services related to Varnish
 should be addressed to <sales at varnish-software.com>.



From phk at varnish-cache.org  Thu Oct 24 08:09:51 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 24 Oct 2013 10:09:51 +0200
Subject: [master] b86f2e5 Style nit
Message-ID: <E1VZFzL-0002qk-GN@project.varnish-software.com>

commit b86f2e5d09be7a6d28ff597034688af712f88c27
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 24 08:09:46 2013 +0000

    Style nit

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 5aad339..f5ff211 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -316,7 +316,8 @@ child_poker(const struct vev *e, int what)
  */
 
 static void __match_proto__()
-mgt_sigsegv_handler(int s, siginfo_t *si, void *c) {
+mgt_sigsegv_handler(int s, siginfo_t *si, void *c)
+{
 	char buf[1024];
 
 	(void)s;



From phk at varnish-cache.org  Thu Oct 24 08:29:04 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 24 Oct 2013 10:29:04 +0200
Subject: [master] 32cab68 The assert that triggered in #1363 was to
 restrictive.
Message-ID: <E1VZGHw-0006Sy-1W@project.varnish-software.com>

commit 32cab6890b04a4f95a743f7c00228b33fd577e24
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 24 08:28:04 2013 +0000

    The assert that triggered in #1363 was to restrictive.
    
    Fixes #1363

diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c
index 6da3348..95862ae 100644
--- a/bin/varnishd/cache/cache_obj.c
+++ b/bin/varnishd/cache/cache_obj.c
@@ -97,7 +97,7 @@ ObjIter(struct objiter *oi, void **p, ssize_t *l)
 		VTAILQ_FOREACH(oi->st, &oi->obj->store, list) {
 			if (oi->st->len > ol) {
 				*p = oi->st->ptr + ol;
-				*l = (nl - ol);
+				*l = oi->st->len - ol;
 				oi->len += *l;
 				break;
 			}
@@ -108,8 +108,8 @@ ObjIter(struct objiter *oi, void **p, ssize_t *l)
 		if (oi->st != NULL && oi->st->len == 0)
 			oi->st = NULL;
 		Lck_Unlock(&oi->bo->mtx);
-		assert(*l > 0);
-		return (oi->st ? OIS_DATA : OIS_STREAM);
+		assert(*l > 0 || oi->bo->state == BOS_FINISHED);
+		return (oi->st != NULL ? OIS_DATA : OIS_STREAM);
 	}
 }
 



From phk at varnish-cache.org  Thu Oct 24 08:30:18 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 24 Oct 2013 10:30:18 +0200
Subject: [master] e686898 Try to explain to static checkers that we know what
 we're doing.
Message-ID: <E1VZGJ8-0006oL-Ht@project.varnish-software.com>

commit e6868989fa4bf97921f6f95a7ba70ca84dc25c90
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 24 08:30:01 2013 +0000

    Try to explain to static checkers that we know what we're doing.

diff --git a/include/vct.h b/include/vct.h
index 52530a6..4f9e2d4 100644
--- a/include/vct.h
+++ b/include/vct.h
@@ -44,9 +44,10 @@
 extern const uint16_t vct_typtab[256];
 
 static inline int
-vct_is(unsigned char x, uint16_t y)
+vct_is(int x, uint16_t y)
 {
 
+	x &= 0xff;
 	return (vct_typtab[x] & (y));
 }
 



From phk at varnish-cache.org  Thu Oct 24 09:54:27 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 24 Oct 2013 11:54:27 +0200
Subject: [master] 6f0e274 Inspired by #1356: Make handling of req.body much
 more robust and RFC-compliant.
Message-ID: <E1VZHcZ-0007mG-G6@project.varnish-software.com>

commit 6f0e274716b8eac91936fab63b98cb798dd56e72
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 24 09:53:39 2013 +0000

    Inspired by #1356:  Make handling of req.body much more robust and
    RFC-compliant.
    
    Fixes #1356

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 1ce24fa..b51354d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -652,7 +652,6 @@ struct req {
 	struct {
 		ssize_t			bytes_done;
 		ssize_t			bytes_yet;
-		enum {CL, CHUNKED}	mode;
 	}				h1;	/* HTTP1 specific */
 
 	/* The busy objhead we sleep on */
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index e0b35d8..50c421a 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -240,6 +240,13 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 		i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk);
 		if (req->req_body_status == REQ_BODY_DONE)
 			retry = -1;
+		if (req->req_body_status == REQ_BODY_FAIL) {
+			VSLb(bo->vsl, SLT_FetchError,
+			    "req.body read error: %d (%s)",
+			    errno, strerror(errno));
+			req->doclose = SC_RX_BODY;
+			retry = -1;
+		}
 	}
 
 	if (WRW_FlushRelease(wrk) || i != 0) {
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index 85e3022..fcdbebb 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -255,16 +255,11 @@ http1_req_body_status(struct req *req)
 			return (REQ_BODY_FAIL);
 		if (req->req_bodybytes == 0)
 			return (REQ_BODY_NONE);
-		req->h1.mode = CL;
 		req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
 		return (REQ_BODY_PRESENT);
 	}
-
-	if (http_GetHdr(req->http, H_Transfer_Encoding, NULL)) {
-		req->h1.mode = CHUNKED;
-		VSLb(req->vsl, SLT_Debug, "Transfer-Encoding in request");
+	if (http_GetHdr(req->http, H_Transfer_Encoding, NULL)) 
 		return (REQ_BODY_FAIL);
-	}
 	return (REQ_BODY_NONE);
 }
 
@@ -274,7 +269,8 @@ http1_req_body_status(struct req *req)
 static enum req_fsm_nxt
 http1_dissect(struct worker *wrk, struct req *req)
 {
-	const char *r = "HTTP/1.1 100 Continue\r\n\r\n";
+	const char *r_100 = "HTTP/1.1 100 Continue\r\n\r\n";
+	const char *r_411 = "HTTP/1.1 411 Length Required\r\n\r\n";
 	char *p;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -308,6 +304,19 @@ http1_dissect(struct worker *wrk, struct req *req)
 		SES_Close(req->sp, SC_RX_JUNK);
 		return (REQ_FSM_DONE);
 	}
+
+	if (req->req_body_status == REQ_BODY_INIT)
+		req->req_body_status = http1_req_body_status(req);
+	else
+		assert(req->req_body_status == REQ_BODY_NONE);	// ESI
+
+	if (req->req_body_status == REQ_BODY_FAIL) {
+		wrk->stats.client_req_411++;
+		(void)write(req->sp->fd, r_411, strlen(r_411));
+		SES_Close(req->sp, SC_RX_JUNK);
+		return (REQ_FSM_DONE);
+	}
+
 	req->acct_req.req++;
 
 	req->ws_req = WS_Snapshot(req->ws);
@@ -318,7 +327,8 @@ http1_dissect(struct worker *wrk, struct req *req)
 		if (strcasecmp(p, "100-continue")) {
 			wrk->stats.client_req_417++;
 			req->err_code = 417;
-		} else if (strlen(r) != write(req->sp->fd, r, strlen(r))) {
+		} else if (strlen(r_100) !=
+		    write(req->sp->fd, r_100, strlen(r_100))) {
 			SES_Close(req->sp, SC_REM_CLOSE);
 			return (REQ_FSM_DONE);
 		}
@@ -328,10 +338,6 @@ http1_dissect(struct worker *wrk, struct req *req)
 		wrk->stats.client_req++;
 
 	http_Unset(req->http, H_Expect);
-	if (req->req_body_status == REQ_BODY_INIT)
-		req->req_body_status = http1_req_body_status(req);
-	else
-		assert(req->req_body_status == REQ_BODY_NONE);
 
 	assert(req->req_body_status != REQ_BODY_INIT);
 
@@ -437,26 +443,23 @@ http1_iter_req_body(struct req *req, void *buf, ssize_t len)
 {
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	if (req->h1.mode == CL) {
-		AN(req->req_bodybytes);
-		AN(len);
-		AN(buf);
-		if (len > req->req_bodybytes - req->h1.bytes_done)
-			len = req->req_bodybytes - req->h1.bytes_done;
-		if (len == 0) {
-			req->req_body_status = REQ_BODY_DONE;
-			return (0);
-		}
-		len = HTTP1_Read(req->htc, buf, len);
-		if (len <= 0) {
-			req->req_body_status = REQ_BODY_FAIL;
-			return (-1);
-		}
-		req->h1.bytes_done += len;
-		req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
-		return (len);
+	AN(req->req_bodybytes);
+	AN(len);
+	AN(buf);
+	if (len > req->req_bodybytes - req->h1.bytes_done)
+		len = req->req_bodybytes - req->h1.bytes_done;
+	if (len == 0) {
+		req->req_body_status = REQ_BODY_DONE;
+		return (0);
 	}
-	INCOMPL();
+	len = HTTP1_Read(req->htc, buf, len);
+	if (len <= 0) {
+		req->req_body_status = REQ_BODY_FAIL;
+		return (-1);
+	}
+	req->h1.bytes_done += len;
+	req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
+	return (len);
 }
 
 /*----------------------------------------------------------------------
@@ -513,6 +516,7 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv)
 	do {
 		l = http1_iter_req_body(req, buf, sizeof buf);
 		if (l < 0) {
+			req->doclose = SC_RX_BODY;
 			return (l);
 		}
 		if (l > 0) {
@@ -548,6 +552,8 @@ HTTP1_DiscardReqBody(struct req *req)
 
 	if (req->req_body_status == REQ_BODY_DONE)
 		return(0);
+	if (req->req_body_status == REQ_BODY_FAIL)
+		return(0);
 	if (req->req_body_status == REQ_BODY_TAKEN)
 		return(0);
 	return(HTTP1_IterateReqBody(req, httpq_req_body_discard, NULL));
@@ -568,8 +574,11 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
+	assert (req->req_step == R_STP_RECV);
 	switch(req->req_body_status) {
 	case REQ_BODY_CACHED:
+	case REQ_BODY_FAIL:
+		return (-1);
 	case REQ_BODY_NONE:
 		return (0);
 	case REQ_BODY_PRESENT:
@@ -599,8 +608,10 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 
 		l = st->space - st->len;
 		l = http1_iter_req_body(req, st->ptr + st->len, l);
-		if (l < 0)
+		if (l < 0) {
+			req->doclose = SC_RX_BODY;
 			return (l);
+		}
 		if (req->req_bodybytes > maxsize) {
 			req->req_body_status = REQ_BODY_FAIL;
 			return (-1);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c20aa83..fe089cd 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -693,6 +693,11 @@ cnt_recv(struct worker *wrk, struct req *req)
 	http_CollectHdr(req->http, H_Cache_Control);
 
 	VCL_recv_method(req->vcl, wrk, req, NULL, req->http->ws);
+
+	/* Attempts to cache req.body may fail */
+	if (req->req_body_status == REQ_BODY_FAIL) {
+		return (REQ_FSM_DONE);
+	}
 	recv_handling = wrk->handling;
 
 	if (cache_param->http_gzip_support &&
diff --git a/bin/varnishtest/tests/c00055.vtc b/bin/varnishtest/tests/c00055.vtc
index ff13f66..4ecbe35 100644
--- a/bin/varnishtest/tests/c00055.vtc
+++ b/bin/varnishtest/tests/c00055.vtc
@@ -30,3 +30,19 @@ client c1 {
 	expect resp.http.Foo == "Foo"
 	expect resp.bodylen == 2
 } -run
+
+client c1 {
+	txreq -req POST -nolen -hdr "Content-Length: 52"
+	delay .3
+} -run
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+client c1 {
+	txreq -url "/is_varnish_still_running"
+	rxresp
+	expect resp.status == 200
+} -run
diff --git a/bin/varnishtest/tests/r01356.vtc b/bin/varnishtest/tests/r01356.vtc
new file mode 100644
index 0000000..f0c00b2
--- /dev/null
+++ b/bin/varnishtest/tests/r01356.vtc
@@ -0,0 +1,37 @@
+varnishtest "#1356, req.body failure"
+
+server s1 {
+	rxhdrs
+	expect_close
+} -start
+
+varnish v1 -vcl+backend { } -start
+
+client c1 {
+	txreq -req POST -nolen -hdr "Transfer-Encoding: carrier-pigeon"
+	rxresp
+	expect resp.status == 411
+} -run
+
+client c1 {
+	txreq -req POST -nolen -hdr "Content-Length: carrier-pigeon"
+	rxresp
+	expect resp.status == 411
+} -run
+
+client c1 {
+	txreq -req POST -nolen -hdr "Content-Length: 56"
+} -run
+
+# Check that varnishd still runs
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+client c1 {
+	txreq 
+	rxresp
+	expect resp.status == 200
+} -run
diff --git a/include/tbl/sess_close.h b/include/tbl/sess_close.h
index 6246a51..b132b39 100644
--- a/include/tbl/sess_close.h
+++ b/include/tbl/sess_close.h
@@ -32,6 +32,7 @@
 SESS_CLOSE(REM_CLOSE,	"Client Closed")
 SESS_CLOSE(REQ_CLOSE,	"Client requested close")
 SESS_CLOSE(REQ_HTTP10,	"proto < HTTP.1.1")
+SESS_CLOSE(RX_BODY,	"Failure receiving req.body")
 SESS_CLOSE(RX_JUNK,	"Received junk data")
 SESS_CLOSE(RX_OVERFLOW,	"Received buffer overflow")
 SESS_CLOSE(RX_TIMEOUT,	"Receive timeout")
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 51ad0f6..825d73e 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -101,6 +101,11 @@ VSC_F(client_req_400,		uint64_t, 1, 'a', info,
 	" malformed in some drastic way."
 )
 
+VSC_F(client_req_411,		uint64_t, 1, 'a', info,
+    "Client requests received, subject to 411 errors",
+	"411 means the client did not send a Content-Lenght for the req.body."
+)
+
 VSC_F(client_req_413,		uint64_t, 1, 'a', info,
     "Client requests received, subject to 413 errors",
 	"413 means that HTTP headers execeeded length or count limits."



From phk at varnish-cache.org  Thu Oct 24 13:12:53 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 24 Oct 2013 15:12:53 +0200
Subject: [master] 68b9177 Integrate (with minor polishing) patch from Geoff to
 add a SLT_BereqEnd record with timing info about the fetch.
Message-ID: <E1VZKib-0005p7-9t@project.varnish-software.com>

commit 68b9177c6fc0748f86d6ca7224a04a090a614098
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 24 13:12:37 2013 +0000

    Integrate (with minor polishing) patch from Geoff to add a SLT_BereqEnd
    record with timing info about the fetch.
    
    Submitted by: Geoff Simmons @ UPLEX

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b51354d..662adbd 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -579,6 +579,13 @@ struct busyobj {
 	double			first_byte_timeout;
 	double			between_bytes_timeout;
 
+	/* Timers */
+	double			t_start;
+	double			t_send;
+	double			t_sent;
+	double			t_hdr;
+	double			t_body;
+
 	const char		*storage_hint;
 	struct director		*director;
 	struct VCL_conf		*vcl;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index f5d7d8f..6a177b5 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -586,6 +586,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 {
 	struct busyobj *bo;
 	enum fetch_step stp;
+	double t_hdr, t_body;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
@@ -593,6 +594,11 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 
 	THR_SetBusyobj(bo);
 	stp = F_STP_MKBEREQ;
+	bo->t_start = VTIM_real();
+	bo->t_send = NAN;
+	bo->t_sent = NAN;
+	bo->t_hdr = NAN;
+	bo->t_body = NAN;
 
 	while (stp != F_STP_DONE) {
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -618,6 +624,14 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	if (bo->ims_obj != NULL)
 		(void)HSH_DerefObj(&wrk->stats, &bo->ims_obj);
 
+	t_hdr = bo->t_hdr - bo->t_sent;
+	t_body = bo->t_body - bo->t_hdr;
+	VSLb(bo->vsl, SLT_BereqEnd, "%.9f %.9f %.9f %.9f %.9f %.9f",
+	     bo->t_start,
+	     VTIM_real(),
+	     bo->t_sent - bo->t_send,
+	     t_hdr, t_body, t_hdr + t_body);
+
 	VBO_DerefBusyObj(wrk, &bo);
 	THR_SetBusyobj(NULL);
 }
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index 50c421a..57384f3 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -42,6 +42,7 @@
 #include "vcli_priv.h"
 #include "vct.h"
 #include "vtcp.h"
+#include "vtim.h"
 
 /*--------------------------------------------------------------------
  * Convert a string to a size_t safely
@@ -231,6 +232,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
 	WRW_Reserve(wrk, &vc->fd, bo->vsl, bo->t_fetch);
+	bo->t_send = VTIM_mono();
 	(void)HTTP1_Write(wrk, hp, 0);	/* XXX: stats ? */
 
 	/* Deal with any message-body the request might (still) have */
@@ -257,8 +259,8 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 		return (retry);
 	}
 
-	/* XXX is this the right place? */
 	VSC_C_main->backend_req++;
+	bo->t_sent = VTIM_mono();
 
 	/* Receive response */
 
@@ -293,6 +295,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 			    vc->between_bytes_timeout);
 		}
 	} while (hs != HTTP1_COMPLETE);
+	bo->t_hdr = VTIM_mono();
 
 	hp = bo->beresp;
 
@@ -381,6 +384,7 @@ V1F_fetch_body(struct worker *wrk, struct busyobj *bo)
 	default:
 		INCOMPL();
 	}
+	bo->t_body = VTIM_mono();
 	AZ(bo->vgz_rx);
 
 	/*
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index fcdbebb..6a92bc5 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -258,7 +258,7 @@ http1_req_body_status(struct req *req)
 		req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
 		return (REQ_BODY_PRESENT);
 	}
-	if (http_GetHdr(req->http, H_Transfer_Encoding, NULL)) 
+	if (http_GetHdr(req->http, H_Transfer_Encoding, NULL))
 		return (REQ_BODY_FAIL);
 	return (REQ_BODY_NONE);
 }
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 6bf6646..67548d5 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -119,6 +119,16 @@ SLTM(HttpGarbage, "", "")
 SLTM(Backend, "Backend selected", "")
 SLTM(Length, "Size of object body", "")
 
+SLTM(BereqEnd, "Backend request end",
+	"Marks the end of a backend request.\n\n"
+	"Tstart\n    Timestamp when the fetch started (epoch)\n\n"
+	"Tend\n    Timestamp when the fetch ended (epoch)\n\n"
+	"dTsend\n    Time to send the backend request\n\n"
+	"dThdr\n    Time to receive the backend response headers\n\n"
+	"dTbody\n    Time to receive the backend response body\n\n"
+	"dTresp\n    Time to receive the backend response (dThdr + dTbody)\n\n"
+)
+
 SLTM(FetchError, "Error while fetching object", "")
 
 #define SLTH(tag, ind, req, resp, sdesc, ldesc) \



From phk at varnish-cache.org  Sat Oct 26 08:14:27 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 26 Oct 2013 10:14:27 +0200
Subject: [master] 174552e Avoid inet_ntop(3), use VTC_name() instead.
Message-ID: <E1VZz0t-0008MY-Bj@project.varnish-software.com>

commit 174552e37ad591c39a46afadb08522d0c746df29
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 26 07:08:39 2013 +0000

    Avoid inet_ntop(3), use VTC_name() instead.

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 850e7a3..46ceccf 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -31,10 +31,6 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -47,6 +43,8 @@
 #include "vcl.h"
 #include "vrt.h"
 #include "vrt_obj.h"
+#include "vsa.h"
+#include "vtcp.h"
 #include "vtim.h"
 
 const void * const vrt_magic_string_end = &vrt_magic_string_end;
@@ -301,31 +299,13 @@ char *
 VRT_IP_string(const struct vrt_ctx *ctx, const VCL_IP ip)
 {
 	char *p;
-	const struct sockaddr *sa;
-	const struct sockaddr_in *si4;
-	const struct sockaddr_in6 *si6;
-	const void *addr;
-	int len;
+	unsigned len;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	sa = ip;
-	switch (sa->sa_family) {
-	case AF_INET:
-		len = INET_ADDRSTRLEN;
-		si4 = ip;
-		addr = &(si4->sin_addr);
-		break;
-	case AF_INET6:
-		len = INET6_ADDRSTRLEN;
-		si6 = ip;
-		addr = &(si6->sin6_addr);
-		break;
-	default:
-		INCOMPL();
-	}
-	XXXAN(len);
-	AN(p = WS_Alloc(ctx->ws, len));
-	AN(inet_ntop(sa->sa_family, addr, p, len));
+	len = WS_Reserve(ctx->ws, 0);
+	p = ctx->ws->r;
+	VTCP_name(ip, VSA_Len(ip), p, len, NULL, 0);
+	WS_Release(ctx->ws, strlen(p) + 1);
 	return (p);
 }
 



From phk at varnish-cache.org  Sat Oct 26 08:14:27 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 26 Oct 2013 10:14:27 +0200
Subject: [master] 2f4e3f5 Add a test for bans on obj.status.
Message-ID: <E1VZz0t-0008MT-A4@project.varnish-software.com>

commit 2f4e3f5a78d1adda6926796a2360b37e8862353c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 26 06:40:10 2013 +0000

    Add a test for bans on obj.status.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 3fd7bb2..bb22fa4 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -791,7 +791,7 @@ ban_evaluate(const uint8_t *bs, const struct http *objhttp,
 			sprintf(buf, "%d", objhttp->status);
 			break;
 		default:
-			INCOMPL();
+			WRONG("Wrong BAN_ARG code");
 		}
 
 		switch (bt.oper) {
@@ -816,7 +816,7 @@ ban_evaluate(const uint8_t *bs, const struct http *objhttp,
 				return (0);
 			break;
 		default:
-			INCOMPL();
+			WRONG("Wrong BAN_OPER code");
 		}
 	}
 	return (1);
@@ -1199,8 +1199,11 @@ ban_render(struct cli *cli, const uint8_t *bs)
 			VCLI_Out(cli, "obj.http.%.*s",
 			    bt.arg1_spec[0] - 1, bt.arg1_spec + 1);
 			break;
+		case BANS_ARG_OBJSTATUS:
+			VCLI_Out(cli, "obj.status");
+			break;
 		default:
-			INCOMPL();
+			WRONG("Wrong BANS_ARG");
 		}
 		switch (bt.oper) {
 		case BANS_OPER_EQ:	VCLI_Out(cli, " == "); break;
@@ -1208,7 +1211,7 @@ ban_render(struct cli *cli, const uint8_t *bs)
 		case BANS_OPER_MATCH:	VCLI_Out(cli, " ~ "); break;
 		case BANS_OPER_NMATCH:	VCLI_Out(cli, " !~ "); break;
 		default:
-			INCOMPL();
+			WRONG("Wrong BANS_OPER");
 		}
 		VCLI_Out(cli, "%s", bt.arg2);
 		if (bs < be)
diff --git a/bin/varnishtest/tests/c00059.vtc b/bin/varnishtest/tests/c00059.vtc
new file mode 100644
index 0000000..db0e6d2
--- /dev/null
+++ b/bin/varnishtest/tests/c00059.vtc
@@ -0,0 +1,26 @@
+varnishtest "test ban obj.status"
+
+server s1 {
+	rxreq
+	txresp -bodylen 5
+	rxreq
+	txresp -bodylen 3
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.bodylen == 5
+} -run
+
+varnish v1 -cliok "ban obj.status == 201"
+varnish v1 -cliok "ban obj.status == 200"
+varnish v1 -cliok "ban.list"
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.bodylen == 3
+} -run



From phk at varnish-cache.org  Sun Oct 27 12:43:45 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 27 Oct 2013 13:43:45 +0100
Subject: [master] a32c827 Put some "section headers" in the compiled VCL code.
Message-ID: <E1VaPh3-0008A1-L4@project.varnish-software.com>

commit a32c82723ebc1c81287646b515705405a75af83e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 27 12:43:30 2013 +0000

    Put some "section headers" in the compiled VCL code.

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 6b5b365..0d21382 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -542,7 +542,9 @@ def emit_vcl_tnames(fo, tokens):
 #######################################################################
 # Read a C-source file and spit out code that outputs it with VSB_cat()
 
-def emit_file(fo, fn):
+def emit_file(fo, fd, bn):
+	fn = fd + "/" + bn
+
 	fi = open(fn)
 	fc = fi.read()
 	fi.close()
@@ -553,6 +555,7 @@ def emit_file(fo, fn):
 	x = 0
 	l = 0
 	fo.write("\n\t/* %s */\n\n" % fn)
+	fo.write('\tVSB_cat(sb, "/* ---===### %s ###===--- */\\n");\n' % bn)
 	for c in fc:
 		if l == 0:
 			fo.write("\tVSB_cat(sb, \"")
@@ -878,9 +881,9 @@ vcl_output_lang_h(struct vsb *sb)
 {
 """)
 
-emit_file(fo, buildroot + "/include/vcl.h")
-emit_file(fo, srcroot + "/include/vrt.h")
-emit_file(fo, buildroot + "/include/vrt_obj.h")
+emit_file(fo, buildroot, "include/vcl.h")
+emit_file(fo, srcroot, "include/vrt.h")
+emit_file(fo, buildroot, "include/vrt_obj.h")
 
 fo.write("""
 }
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 1b3ee4e..41b8ede 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -626,7 +626,7 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 	sym->wildcard = vcc_Stv_Wildcard;
 
 	vcl_output_lang_h(tl->fh);
-	Fh(tl, 0, "\n/* ---===### VCC generated below here ###===---*/\n");
+	Fh(tl, 0, "\n/* ---===### VCC generated code ###===---*/\n");
 	Fh(tl, 0, "\nextern const struct VCL_conf VCL_conf;\n");
 
 	/* Macro for accessing directors */



From phk at varnish-cache.org  Sun Oct 27 20:46:54 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 27 Oct 2013 21:46:54 +0100
Subject: [master] 7b21e7e Start introducing "struct suckaddr" which will
 become our new VCL_IP type.
Message-ID: <E1VaXEc-0005Mr-0k@project.varnish-software.com>

commit 7b21e7e3a735d472162f7f9e05e89a44bf9855ef
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 27 20:46:25 2013 +0000

    Start introducing "struct suckaddr" which will become our new VCL_IP type.

diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index e203cb8..bfa3ccb 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -1,4 +1,5 @@
 -d__flexelint_v9__=1
++fan
 
 -printf(3, VSL)
 -printf(2, http_PrintfHeader)
diff --git a/include/vrt.h b/include/vrt.h
index e203168..18b0fba 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -42,6 +42,7 @@ struct cli;
 struct director;
 struct VCL_conf;
 struct sockaddr_storage;
+struct suckaddr;
 
 /***********************************************************************
  * This is the central definition of the mapping from VCL types to
@@ -225,6 +226,9 @@ void VRT_synth_page(const struct vrt_ctx *, unsigned flags, const char *, ...);
 void VRT_init_dir(struct cli *, struct director **, int idx, const void *priv);
 void VRT_fini_dir(struct cli *, struct director *);
 
+/* Suckaddr related */
+int VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst);
+
 /* VMOD/Modules related */
 int VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
     const char *path, struct cli *cli);
diff --git a/include/vsa.h b/include/vsa.h
index 3ac8eb2..c1e5509 100644
--- a/include/vsa.h
+++ b/include/vsa.h
@@ -30,6 +30,9 @@
 #ifndef VSA_H_INCLUDED
 #define VSA_H_INCLUDED
 
+struct suckaddr;
+extern const int vsa_suckaddr_len;
+
 int VSA_Sane(const void *);
 socklen_t VSA_Len(const void *);
 unsigned VSA_Port(const void *);
diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c
index 1f2a92f..000678d 100644
--- a/lib/libvarnish/vsa.c
+++ b/lib/libvarnish/vsa.c
@@ -40,6 +40,162 @@
 
 #include "vas.h"
 #include "vsa.h"
+#include "vrt.h"
+
+/*
+ * Struct sockaddr{|_in|_in6|_storage} is absolutely the worst data
+ * structure I have ever seen gold-plated in international standards.
+ *
+ * Network addresses have multiple different forms, many fewer today
+ * than in last century, but imagine that in addition to IPv4 and IPv6
+ * we had 40 other protocols.  Actually, you don't need to imagine that
+ * just count the AF_* macros in /usr/include/sys/socket.h.
+ *
+ * So what do we pass the kernel API for an address to bind(2), connect(2) &
+ * listen(2) etc. etc ?
+ *
+ * We could define a struct which is big enough to hold any and all
+ * of these addresses.  That would make it a fixed size argument.
+ * obviously the struct would have to be something like:
+ *	struct bla {
+ *		int family;
+ *		char address[MAX_ADDR_LEN];
+ *	}
+ * and MAX_ADDR_LEN would have to be quite large, 128 byte or so.
+ *
+ * Back in last century that was TOTALLY unacceptable waste of space.
+ *
+ * The way which was chosen instead, was to make a "generic" address,
+ * and have per protocol "specific" addresses, and pass the length
+ * argument explicitly to the KPI functions.
+ *
+ * The generic address was called "struct sockaddr", and the specific
+ * were called "struct sockaddr_${whatever}".  All of these must have
+ * a "family" field as first element, so the kernel can figure out
+ * which protocol it is.
+ *
+ * The generic struct sockaddr was made big enough for all protocols
+ * supported in the kernel, so it would have different sizes depending
+ * on your machine and kernel configuration.
+ *
+ * However, that allowed you to write protocol-agnostic programs, by
+ * using "struct sockaddr" throughout, and relying on libray APIs for
+ * things like name to address (and vice versa) resolution, and since
+ * nobody were in the business of shipping random UNIX binaries around
+ * the lack of binary portability didn't matter.
+ *
+ * Along the way the BSD people figured out that it was a bother
+ * to carry the length argument separately, and added that to the
+ * format of sockaddr, but other groups found this unclean, as
+ * the length was already an explicit paramter.
+ *
+ * The net result of this is that your "portable" code, must take
+ * care to handle the "sa_len" member on kernels which have it,
+ * while still tracking the separate length argument for all other
+ * kernels.
+ *
+ * Needless to say, there were no neat #define to tell you which
+ * was which, so each programmer found a different heuristic to
+ * decide, often not understanding it fully, which caused the kind
+ * of portability issues which lead to the autocrap tools.
+ *
+ * Then all the other protocols died, we were left with IP and
+ * life were good, the dot-com madness multiplied the IT-business
+ * by a factor 1000, by making any high-school student who had
+ * programmed PERL for 6 weeks a "senior web-programmer".
+ *
+ * Next IPv6 happened, in a rush even, (no seriously, I'm not kidding!),
+ * and since IPv6 addresses were HUGE, like 16 bytes HUGE, the generic
+ * struct sockaddr was not increased in size.
+ *
+ * At least "not yet", because it would break all the shitty code written
+ * by the dot-com generation.
+ *
+ * Nobody used IPv6 anyway so that didn't matter that much.
+ *
+ * Then people actually started using IPv6 and its struct sockaddr_in6,
+ * and realized that all the code which used "struct sockaddr" to allocate
+ * space at compile time were broken.
+ *
+ * Some people took to using sockaddr_in6, since that was known to
+ * be big enough for both IPv4 and IPv6, but "purist" found that
+ * ugly and "prone to future trouble".
+ *
+ * So instead they came up with a "clean solution":  The added
+ * "struct sockaddr_storage" which is defined to be "Large enough
+ * to accommodate all supported protocol-specific address structures".
+ *
+ * Since we cannot possibly know what zany protocols will exist in
+ * the future, and since some people think that we will add future
+ * protocols, while retaining ABI compatibility, (totally overlooking
+ * the fact that no code for name-resolution supports that) it is
+ * usually defined so it can cope with 128 byte addresses.
+ *
+ * Does that ring a bell ?
+ *
+ * Only, not quite:  Remember that all APIs require you to track
+ * the address and the length separately, so you only get the
+ * size of the specific protocols sockaddr_${whatever} from API
+ * functions, not a full sockaddr_storage, and besides the
+ * prototype for the KPI is still "struct sockaddr *", so you
+ * cannot gain C type-safety back by using sockaddr_storage
+ * as the "generic network address" type.
+ *
+ * So we have come full circle, while causing maximum havoc along
+ * the way and for the forseeable future.
+ *
+ * Do I need to tell you that static code analysis tools have a
+ * really hard time coping with this, and that they give a lot of
+ * false negatives which confuse people ?
+ *
+ * I have decided to try to contain this crap in this single
+ * source-file, with only minimum leakage into the rest of Varnish,
+ * which will only know of pointers to "struct suckaddr", the naming
+ * of which is my of the historical narrative above.
+ *
+ * And you don't need to take my word for this, you can see it all
+ * in various #include files on your own system.   If you are on
+ * a Solaris derivative, don't miss the beautiful horror hidden in the
+ * variant definition of IPv6 addresses between kernel and userland.
+ *
+ */
+
+struct suckaddr {
+	union {
+		struct sockaddr		sa;
+		struct sockaddr_in	sa4;
+		struct sockaddr_in6	sa6;
+	};
+};
+
+const int vsa_suckaddr_len = sizeof(struct suckaddr);
+
+/*
+ * This VRT interface is for the VCC generated ACL code, which needs
+ * to know the address family and a pointer to the actual address.
+ */
+
+int
+VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst)
+{
+	AN(dst);
+	if (sua == NULL)
+		return (-1);
+
+	switch(sua->sa.sa_family) {
+		case PF_INET:
+			assert(sua->sa.sa_family == sua->sa4.sin_family);
+			*dst = (const unsigned char *)&sua->sa4.sin_addr;
+			return (sua->sa4.sin_family);
+		case PF_INET6:
+			assert(sua->sa.sa_family == sua->sa6.sin6_family);
+			*dst = (const unsigned char *)&sua->sa6.sin6_addr;
+			return (sua->sa6.sin6_family);
+		default:
+			*dst = NULL;
+			return (-1);
+	}
+}
 
 int
 VSA_Sane(const void *s)
diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index 433d848..182dbec 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -320,34 +320,6 @@ vcc_acl_entry(struct vcc *tl)
  * Emit a function to match the ACL we have collected
  */
 
-/*
- * XXX: this is semi-silly.  We try really hard to not depend in the
- * XXX: systems include files while compiling VCL, but we need to know
- * XXX: the size of the sa_familiy member.
- * XXX: FlexeLint complains about these antics, so isolate it in a
- * XXX: separate function.
- */
-
-/*lint -save -e506 -e774 -e550 */
-static void
-c_is_a_silly_language(const struct vcc *tl)
-{
-	struct sockaddr sa;
-
-	assert(sizeof (unsigned char) == 1);
-	assert(sizeof (unsigned short) == 2);
-	assert(sizeof (unsigned int) == 4);
-	if (sizeof sa.sa_family == 1)
-		Fh(tl, 0, "\tunsigned char fam;\n");
-	else if (sizeof sa.sa_family == 2)
-		Fh(tl, 0, "\tunsigned short fam;\n");
-	else if (sizeof sa.sa_family == 4)
-		Fh(tl, 0, "\tunsigned int fam;\n");
-	else
-		assert(0 == __LINE__);
-}
-/*lint -restore */
-
 static void
 vcc_acl_emit(const struct vcc *tl, const char *acln, int anon)
 {
@@ -357,21 +329,14 @@ vcc_acl_emit(const struct vcc *tl, const char *acln, int anon)
 	const char *oc;
 
 	Fh(tl, 0, "\nstatic int\n");
-	Fh(tl, 0, "match_acl_%s_%s(const struct vrt_ctx *ctx, const void *p)\n",
+	Fh(tl, 0, "match_acl_%s_%s(const struct vrt_ctx *ctx, const VCL_IP p)\n",
 	    anon ? "anon" : "named", acln);
 	Fh(tl, 0, "{\n");
 	Fh(tl, 0, "\tconst unsigned char *a;\n");
-	c_is_a_silly_language(tl);
-
+	Fh(tl, 0, "\tint fam;\n");
 	Fh(tl, 0, "\n");
-	Fh(tl, 0, "\ta = p;\n");
-	Fh(tl, 0, "\tVRT_memmove(&fam, a + %zd, sizeof fam);\n",
-	    offsetof(struct sockaddr, sa_family));
-	Fh(tl, 0, "\tif (fam == %d)\n", PF_INET);
-	Fh(tl, 0, "\t\ta += %zd;\n", offsetof(struct sockaddr_in, sin_addr));
-	Fh(tl, 0, "\telse if (fam == %d)\n", PF_INET6);
-	Fh(tl, 0, "\t\ta += %zd;\n", offsetof(struct sockaddr_in6, sin6_addr));
-	Fh(tl, 0, "\telse {\n");
+	Fh(tl, 0, "\tfam = VRT_VSA_GetPtr(p, &a);\n");
+	Fh(tl, 0, "\tif (fam < 0) {\n");
 	Fh(tl, 0, "\t\tVRT_acl_log(ctx, \"NO_FAM %s\");\n", acln);
 	Fh(tl, 0, "\t\treturn(0);\n");
 	Fh(tl, 0, "\t}\n\n");



From phk at varnish-cache.org  Sun Oct 27 21:23:26 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 27 Oct 2013 22:23:26 +0100
Subject: [master] c6fdc39 Emit backend IP's as encoded struct suckaddr in the
 VGC code.
Message-ID: <E1VaXny-0001iE-BW@project.varnish-software.com>

commit c6fdc3980f7c0d9b1cd15c2e7c5651305edfff23
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 27 21:12:40 2013 +0000

    Emit backend IP's as encoded struct suckaddr in the VGC code.

diff --git a/include/vsa.h b/include/vsa.h
index c1e5509..7293547 100644
--- a/include/vsa.h
+++ b/include/vsa.h
@@ -38,4 +38,6 @@ socklen_t VSA_Len(const void *);
 unsigned VSA_Port(const void *);
 int VSA_Compare(const void *, const void *);
 
+struct suckaddr *VSA_Malloc(const void *s, unsigned  sal);
+
 #endif
diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c
index 000678d..5aadb4f 100644
--- a/lib/libvarnish/vsa.c
+++ b/lib/libvarnish/vsa.c
@@ -35,6 +35,7 @@
 
 #include <errno.h>
 #include <string.h>
+#include <stdlib.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
 
@@ -197,6 +198,37 @@ VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst)
 	}
 }
 
+/*
+ * Malloc a suckaddr from a sockaddr of some kind.
+ */
+
+struct suckaddr *
+VSA_Malloc(const void *s, unsigned  sal)
+{
+	struct suckaddr *sua = NULL;
+	const struct sockaddr *sa = s;
+	unsigned l = 0;
+
+	switch(sa->sa_family) {
+		case PF_INET:
+			if (sal == sizeof sua->sa4)
+				l = sal;
+			break;
+		case PF_INET6:
+			if (sal == sizeof sua->sa6)
+				l = sal;
+			break;
+		default:
+			break;
+	}
+	if (l != 0) {
+		sua = calloc(1, sizeof *sua);
+		if (sua != NULL)
+			memcpy(&sua->sa, s, l);
+	}
+	return (sua);
+}
+
 int
 VSA_Sane(const void *s)
 {
diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c
index 5870af5..aa82bc8 100644
--- a/lib/libvcc/vcc_utils.c
+++ b/lib/libvcc/vcc_utils.c
@@ -31,6 +31,7 @@
 
 #include <stdio.h>
 #include <string.h>
+#include <stdlib.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netdb.h>
@@ -77,42 +78,47 @@ vcc_regexp(struct vcc *tl)
 
 /*
  * The IPv6 crew royally screwed up the entire idea behind
- * struct sockaddr, and combined with various other incomptency
- * in the OS business, that means that there is no sane or even
- * remotely portable way to initialize a sockaddr at compile time.
+ * struct sockaddr, see libvarnish/vsa.c for blow-by-blow account.
+ *
+ * There is no sane or even remotely portable way to initialize
+ * a sockaddr for random protocols at compile time.
  *
  * In our case it is slightly more tricky than that, because we don't
  * even want to #include the struct sockaddr* definitions.
  *
- * Instead we make sure the sockaddr is sane (for our values of sane)
- * and dump it in binary, using a 64 bit integertype, hoping that this
- * will ensure good enough alignment.
+ * Instead we make sure the sockaddr is sane (for our values of
+ * sane) and dump it as our own "struct suckaddr" type, in binary,
+ * using the widest integertype, hoping that this will ensure sufficient
+ * alignment.
  */
 
 static const char *
 vcc_sockaddr(struct vcc *tl, const void *sa, unsigned sal)
 {
-	unsigned n = (sal + 7) / 8, len;
-	uint64_t b[n];
+	unsigned n = (vsa_suckaddr_len + 7) / 8, len;
+	unsigned long long b[n];
+	struct suckaddr *sua;
 	char *p;
 
+	assert(sizeof(unsigned long long) == 8);
+
 	assert(VSA_Sane(sa));
 	AN(sa);
 	AN(sal);
-	assert(sal < sizeof(struct sockaddr_storage));
-	assert(sizeof(unsigned long long) == 8);
+
+	sua = VSA_Malloc(sa, sal);
+	AN(sua);
 
 	p = TlAlloc(tl, 20);
+	AN(p);
 	sprintf(p, "sockaddr_%u", tl->unique++);
 
 	Fh(tl, 0, "static const unsigned long long");
 	Fh(tl, 0, " %s[%d] = {\n", p, n);
-	memcpy(b, sa, sal);
-	for (len = 0; len <n; len++) {
-		Fh(tl, 0, "%s    0x%016jx",
-		    len ? ",\n" : "",
-		    (uintmax_t)b[len]);
-	}
+	memcpy(b, sua, vsa_suckaddr_len);
+	free(sua);
+	for (len = 0; len < n; len++)
+		Fh(tl, 0, "%s    0x%016llx", len ? ",\n" : "", b[len]);
 	Fh(tl, 0, "\n};\n");
 	return (p);
 }



From phk at varnish-cache.org  Sun Oct 27 21:23:26 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 27 Oct 2013 22:23:26 +0100
Subject: [master] 33f3880 Start percolating struct suckaddr out of the VGC
 code to VRT.
Message-ID: <E1VaXny-0001iJ-Cr@project.varnish-software.com>

commit 33f388036f7e86262397fd15d0404888ef024654
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 27 21:23:05 2013 +0000

    Start percolating struct suckaddr out of the VGC code to VRT.

diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index 1fdf795..9bd1ff0 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -174,7 +174,7 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 	char buf[128];
 
 	AN(vb->vcl_name);
-	assert(vb->ipv4_sockaddr != NULL || vb->ipv6_sockaddr != NULL);
+	assert(vb->ipv4_suckaddr != NULL || vb->ipv6_suckaddr != NULL);
 	(void)cli;
 	ASSERT_CLI();
 
@@ -183,11 +183,11 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
 		if (strcmp(b->vcl_name, vb->vcl_name))
 			continue;
-		if (vb->ipv4_sockaddr != NULL &&
-		    VSA_Compare(b->ipv4, vb->ipv4_sockaddr))
+		if (vb->ipv4_suckaddr != NULL &&
+		    VSA_Compare(b->ipv4, vb->ipv4_suckaddr))
 			continue;
-		if (vb->ipv6_sockaddr != NULL &&
-		    VSA_Compare(b->ipv6, vb->ipv6_sockaddr))
+		if (vb->ipv6_suckaddr != NULL &&
+		    VSA_Compare(b->ipv6, vb->ipv6_suckaddr))
 			continue;
 		b->refcount++;
 		b->vsc->vcls++;
@@ -223,10 +223,10 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 	/*
 	 * Copy over the sockaddrs
 	 */
-	if (vb->ipv4_sockaddr != NULL)
-		copy_sockaddr(&b->ipv4, vb->ipv4_sockaddr);
-	if (vb->ipv6_sockaddr != NULL)
-		copy_sockaddr(&b->ipv6, vb->ipv6_sockaddr);
+	if (vb->ipv4_suckaddr != NULL)
+		copy_sockaddr(&b->ipv4, vb->ipv4_suckaddr);
+	if (vb->ipv6_suckaddr != NULL)
+		copy_sockaddr(&b->ipv6, vb->ipv6_suckaddr);
 
 	assert(b->ipv4 != NULL || b->ipv6 != NULL);
 
diff --git a/include/vrt.h b/include/vrt.h
index 18b0fba..0d52fc6 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -125,8 +125,8 @@ struct vrt_backend {
 	const char			*ipv6_addr;
 	const char			*port;
 
-	const void			*ipv4_sockaddr;
-	const void			*ipv6_sockaddr;
+	const struct suckaddr		*ipv4_suckaddr;
+	const struct suckaddr		*ipv6_suckaddr;
 
 	const char			*hosthdr;
 
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index b548d94..6982fb2 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -65,11 +65,13 @@ Emit_Sockaddr(struct vcc *tl, const struct token *t_host, const char *port)
 	    &ipv4, &ipv4a, &ipv6, &ipv6a, &pa, 2, t_host, "Backend host");
 	ERRCHK(tl);
 	if (ipv4 != NULL) {
-		Fb(tl, 0, "\t.ipv4_sockaddr = %s,\n", ipv4);
+		Fb(tl, 0, "\t.ipv4_suckaddr = (const struct suckaddr *)%s,\n",
+		    ipv4);
 		Fb(tl, 0, "\t.ipv4_addr = \"%s\",\n", ipv4a);
 	}
 	if (ipv6 != NULL) {
-		Fb(tl, 0, "\t.ipv6_sockaddr = %s,\n", ipv6);
+		Fb(tl, 0, "\t.ipv6_suckaddr = (const struct suckaddr *)%s,\n",
+		    ipv6);
 		Fb(tl, 0, "\t.ipv6_addr = \"%s\",\n", ipv6a);
 	}
 	Fb(tl, 0, "\t.port = \"%s\",\n", pa);



From phk at varnish-cache.org  Sun Oct 27 21:45:30 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 27 Oct 2013 22:45:30 +0100
Subject: [master] 63d6667 Move suckaddr all the way through backends (I
 think...)
Message-ID: <E1VaY9K-0004JV-Tv@project.varnish-software.com>

commit 63d66674a84ff86ff957444dccb2e341403a6362
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 27 21:45:10 2013 +0000

    Move suckaddr all the way through backends (I think...)

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index f626552..9e8a050 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -108,7 +108,7 @@ VBE_ReleaseConn(struct vbc *vc)
 
 static int
 vbe_TryConnect(const struct busyobj *bo, int pf,
-    const struct sockaddr_storage *sa, const struct vdi_simple *vs)
+    const struct suckaddr *sa, const struct vdi_simple *vs)
 {
 	int s, i, tmo;
 	double tmod;
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 265073b..9324dfb 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -113,8 +113,8 @@ struct backend {
 	char			*ipv6_addr;
 	char			*port;
 
-	struct sockaddr_storage	*ipv4;
-	struct sockaddr_storage	*ipv6;
+	struct suckaddr		*ipv4;
+	struct suckaddr		*ipv6;
 
 	unsigned		n_conn;
 	VTAILQ_HEAD(, vbc)	connlist;
@@ -139,7 +139,7 @@ struct vbc {
 	struct vsl_log		*vsl;
 	int			fd;
 
-	struct sockaddr_storage	*addr;
+	struct suckaddr		*addr;
 
 	uint8_t			recycled;
 
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index 9bd1ff0..63ab402 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -151,13 +151,13 @@ VBE_DropRefConn(struct backend *b)
  */
 
 static void
-copy_sockaddr(struct sockaddr_storage **sa, const void *src)
+copy_sockaddr(struct suckaddr **sa, const struct suckaddr *src)
 {
 
 	assert(VSA_Sane(src));
-	*sa = calloc(sizeof **sa, 1);
+	*sa = calloc(1, vsa_suckaddr_len);
 	XXXAN(*sa);
-	memcpy(*sa, src, VSA_Len(src));
+	memcpy(*sa, src, vsa_suckaddr_len);
 	assert(VSA_Sane(*sa));
 }
 
diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index 53d1183..4773785 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -105,7 +105,7 @@ static struct lock			vbp_mtx;
  */
 
 static int
-vbp_connect(int pf, const struct sockaddr_storage *sa, int tmo)
+vbp_connect(int pf, const struct suckaddr *sa, int tmo)
 {
 	int s, i;
 
diff --git a/include/vsa.h b/include/vsa.h
index 7293547..1e430d6 100644
--- a/include/vsa.h
+++ b/include/vsa.h
@@ -36,7 +36,7 @@ extern const int vsa_suckaddr_len;
 int VSA_Sane(const void *);
 socklen_t VSA_Len(const void *);
 unsigned VSA_Port(const void *);
-int VSA_Compare(const void *, const void *);
+int VSA_Compare(const struct suckaddr *, const struct suckaddr *);
 
 struct suckaddr *VSA_Malloc(const void *s, unsigned  sal);
 
diff --git a/include/vtcp.h b/include/vtcp.h
index 205d9e1..842d897 100644
--- a/include/vtcp.h
+++ b/include/vtcp.h
@@ -28,6 +28,8 @@
  *
  */
 
+struct suckaddr;
+
 /* from libvarnish/tcp.c */
 /* NI_MAXHOST and NI_MAXSERV are ridiculously long for numeric format */
 #define VTCP_ADDRBUFSIZE		64
@@ -67,7 +69,7 @@ int VTCP_check_hup(int sock);
 #ifdef SOL_SOCKET
 void VTCP_name(const struct sockaddr_storage *addr, unsigned l, char *abuf,
     unsigned alen, char *pbuf, unsigned plen);
-int VTCP_connect(int s, const struct sockaddr_storage *name, int msec);
+int VTCP_connect(int s, const struct suckaddr *name, int msec);
 void VTCP_close(int *s);
 void VTCP_set_read_timeout(int s, double seconds);
 #endif
diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c
index 5aadb4f..8bdc73f 100644
--- a/lib/libvarnish/vsa.c
+++ b/lib/libvarnish/vsa.c
@@ -259,17 +259,11 @@ VSA_Len(const void *s)
 }
 
 int
-VSA_Compare(const void *s1, const void *s2)
+VSA_Compare(const struct suckaddr *s1, const struct suckaddr *s2)
 {
-	const struct sockaddr *sa = s1;
-
-	switch(sa->sa_family) {
-		case PF_INET:
-		case PF_INET6:
-			return (memcmp(s1, s2, VSA_Len(s1)));
-		default:
-			return (-1);
-	}
+	AN(s1);
+	AN(s2);
+	return (memcmp(s1, s2, vsa_suckaddr_len));
 }
 
 unsigned
diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index b33d306..81a5f27 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -195,7 +195,7 @@ VTCP_nonblocking(int sock)
  */
 
 int
-VTCP_connect(int s, const struct sockaddr_storage *name, int msec)
+VTCP_connect(int s, const struct suckaddr *name, int msec)
 {
 	int i, k;
 	socklen_t l;



From phk at varnish-cache.org  Sun Oct 27 21:58:57 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 27 Oct 2013 22:58:57 +0100
Subject: [master] 8a769e6 Convert std.ip() implementation to struct suckaddr
Message-ID: <E1VaYML-0005lN-Ki@project.varnish-software.com>

commit 8a769e6eff70e463eeaac8392beeee8144f2032e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 27 21:58:43 2013 +0000

    Convert std.ip() implementation to struct suckaddr

diff --git a/include/vrt.h b/include/vrt.h
index 0d52fc6..cebe8e5 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -56,6 +56,7 @@ typedef double				VCL_DURATION;
 typedef const char *			VCL_ENUM;
 typedef const char *			VCL_HEADER;
 typedef long				VCL_INT;
+//typedef const struct suckaddr *		VCL_IP;
 typedef const void *			VCL_IP;
 typedef double				VCL_REAL;
 typedef const char *			VCL_STRING;
diff --git a/include/vsa.h b/include/vsa.h
index 1e430d6..9bad737 100644
--- a/include/vsa.h
+++ b/include/vsa.h
@@ -38,6 +38,15 @@ socklen_t VSA_Len(const void *);
 unsigned VSA_Port(const void *);
 int VSA_Compare(const struct suckaddr *, const struct suckaddr *);
 
+/*
+ * 's' is a sockaddr of some kind, 'sal' is its length
+ */
 struct suckaddr *VSA_Malloc(const void *s, unsigned  sal);
 
+/*
+ * 'd' SHALL point to vsa_suckaddr_len aligned bytes of storage,
+ * 's' is a sockaddr of some kind, 'sal' is its length.
+ */
+struct suckaddr *VSA_Build(void *d, const void *s, unsigned sal);
+
 #endif
diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c
index 8bdc73f..4dd32b6 100644
--- a/lib/libvarnish/vsa.c
+++ b/lib/libvarnish/vsa.c
@@ -209,6 +209,7 @@ VSA_Malloc(const void *s, unsigned  sal)
 	const struct sockaddr *sa = s;
 	unsigned l = 0;
 
+	AN(s);
 	switch(sa->sa_family) {
 		case PF_INET:
 			if (sal == sizeof sua->sa4)
@@ -229,6 +230,33 @@ VSA_Malloc(const void *s, unsigned  sal)
 	return (sua);
 }
 
+/* 'd' SHALL point to vsa_suckaddr_len aligned bytes of storage */
+struct suckaddr *
+VSA_Build(void *d, const void *s, unsigned sal)
+{
+	struct suckaddr *sua = d;
+	const struct sockaddr *sa = s;
+	unsigned l = 0;
+
+	AN(d);
+	AN(s);
+	switch(sa->sa_family) {
+		case PF_INET:
+			if (sal == sizeof sua->sa4)
+				l = sal;
+			break;
+		case PF_INET6:
+			if (sal == sizeof sua->sa6)
+				l = sal;
+			break;
+		default:
+			break;
+	}
+	if (l != 0)
+		memcpy(&sua->sa, s, l);
+	return (sua);
+}
+
 int
 VSA_Sane(const void *s)
 {
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index 68af3ad..42325d8 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -127,9 +127,10 @@ VCL_IP
 vmod_ip(const struct vrt_ctx *ctx, VCL_STRING s, VCL_IP d)
 {
 	struct addrinfo hints, *res0 = NULL;
-	const struct addrinfo *res;
+	const struct addrinfo *res, *best = NULL;
 	int error;
-	char *p;
+	void *p;
+	struct suckaddr *r;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(d);
@@ -144,17 +145,21 @@ vmod_ip(const struct vrt_ctx *ctx, VCL_STRING s, VCL_IP d)
 			for (res = res0; res != NULL; res = res->ai_next) {
 				if (VSA_Sane(res->ai_addr) &&
 				    res->ai_addrlen >= VSA_Len(res->ai_addr)) {
-					d = res->ai_addr;
+					best = res;
 					break;
 				}
 			}
 		}
 	}
-	AN(d);
-	p = WS_Alloc(ctx->ws, VSA_Len(d));
+	p = WS_Alloc(ctx->ws, vsa_suckaddr_len);
 	AN(p);
-	memcpy(p, d, VSA_Len(d));
+	if (best != NULL)
+		r = VSA_Build(p, best->ai_addr, best->ai_addrlen);
+	else {
+		r = p;
+		memcpy(r, d, vsa_suckaddr_len);
+	}
 	if (res0 != NULL)
 		freeaddrinfo(res0);
-	return (p);
+	return (r);
 }



From phk at varnish-cache.org  Sun Oct 27 22:23:44 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 27 Oct 2013 23:23:44 +0100
Subject: [master] b8c2aa5 Add a function to extract a sockaddr and a socklen_t
 from a suckaddr.
Message-ID: <E1VaYkK-0000PS-EA@project.varnish-software.com>

commit b8c2aa5c5b98c7fe43dc378e9f0964845456b278
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 27 22:23:24 2013 +0000

    Add a function to extract a sockaddr and a socklen_t from a suckaddr.

diff --git a/include/vsa.h b/include/vsa.h
index 9bad737..d21f3c0 100644
--- a/include/vsa.h
+++ b/include/vsa.h
@@ -38,6 +38,8 @@ socklen_t VSA_Len(const void *);
 unsigned VSA_Port(const void *);
 int VSA_Compare(const struct suckaddr *, const struct suckaddr *);
 
+const struct sockaddr *VSA_Get_Sockaddr(const struct suckaddr *, socklen_t *sl);
+
 /*
  * 's' is a sockaddr of some kind, 'sal' is its length
  */
diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c
index 4dd32b6..97ff0fa 100644
--- a/lib/libvarnish/vsa.c
+++ b/lib/libvarnish/vsa.c
@@ -257,6 +257,24 @@ VSA_Build(void *d, const void *s, unsigned sal)
 	return (sua);
 }
 
+const struct sockaddr *
+VSA_Get_Sockaddr(const struct suckaddr *sua, socklen_t *sl)
+{
+	AN(sua);
+	AN(sl);
+	switch(sua->sa.sa_family) {
+		case PF_INET:
+			*sl = sizeof sua->sa4;
+			break;
+		case PF_INET6:
+			*sl = sizeof sua->sa6;
+			break;
+		default:
+			return (NULL);
+	}
+	return (&sua->sa);
+}
+
 int
 VSA_Sane(const void *s)
 {



From phk at varnish-cache.org  Sun Oct 27 22:38:51 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 27 Oct 2013 23:38:51 +0100
Subject: [master] e5cbbed Use VCL typedefs for generated VRT prototypes,
 and also for implementations which return VCL_IP
Message-ID: <E1VaYyx-0002K4-3u@project.varnish-software.com>

commit e5cbbedd2d9342595817c1b66b22d1c90e8c60fc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 27 22:38:28 2013 +0000

    Use VCL typedefs for generated VRT prototypes, and also for implementations
    which return VCL_IP

diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index ec97408..2526d3b 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -258,7 +258,7 @@ VRT_r_beresp_backend_name(const struct vrt_ctx *ctx)
 	return(ctx->bo->vbc->backend->vcl_name);
 }
 
-const void *
+VCL_IP
 VRT_r_beresp_backend_ip(const struct vrt_ctx *ctx)
 {
 
@@ -535,7 +535,7 @@ REQ_BOOL(hash_always_miss)
 
 /*--------------------------------------------------------------------*/
 
-const void *
+VCL_IP
 VRT_r_client_ip(const struct vrt_ctx *ctx)
 {
 
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 0d21382..f0224e3 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -815,6 +815,7 @@ const struct var vcc_vars[] = {
 
 sp_variables.sort()
 for i in sp_variables:
+	fh.write("\n")
 	typ = i[1]
 	cnam = i[0].replace(".", "_")
 	ctyp = vcltypes[typ]
@@ -829,7 +830,7 @@ for i in sp_variables:
 		fo.write('",\n')
 	else:
 		fo.write('\t    "VRT_r_%s(ctx)",\n' % cnam)
-		fh.write(ctyp + " VRT_r_%s(const struct vrt_ctx *);\n" % cnam )
+		fh.write("VCL_" + typ + " VRT_r_%s(const struct vrt_ctx *);\n" % cnam )
 	restrict(fo, i[2])
 
 	if len(i[3]) == 0:
@@ -842,7 +843,7 @@ for i in sp_variables:
 		fo.write('\t    "VRT_l_%s(ctx, ",\n' % cnam)
 		fh.write("void VRT_l_%s(const struct vrt_ctx *, " % cnam)
 		if typ != "STRING":
-			fh.write(ctyp + ");\n")
+			fh.write("VCL_" + typ + ");\n")
 		else:
 			fh.write(ctyp + ", ...);\n")
 	restrict(fo, i[3])
@@ -851,6 +852,7 @@ for i in sp_variables:
 
 fo.write("\t{ NULL }\n};\n")
 
+fh.write("\n")
 for i in stv_variables:
 	fh.write(vcltypes[i[1]] + " VRT_Stv_" + i[0] + "(const char *);\n")
 



From phk at varnish-cache.org  Mon Oct 28 07:25:58 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 28 Oct 2013 08:25:58 +0100
Subject: [master] ff05032 Convert the sessions local address to be a suckaddr.
 This shaves 92 bytes of struct sess.
Message-ID: <E1VahD4-0002Vz-PQ@project.varnish-software.com>

commit ff05032e678d070833ed3829495f46097250c6f0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 28 07:25:13 2013 +0000

    Convert the sessions local address to be a suckaddr.  This shaves
    92 bytes of struct sess.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 662adbd..069a80a 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -768,9 +768,8 @@ struct sess {
 	/* Session related fields ------------------------------------*/
 
 	socklen_t		sockaddrlen;
-	socklen_t		mysockaddrlen;
-	struct sockaddr_storage	sockaddr;
-	struct sockaddr_storage	mysockaddr;
+	struct sockaddr_storage sockaddr;
+	struct suckaddr		*our_addr;
 
 	/* formatted ascii client address */
 	char			addr[ADDR_BUFSIZE];
@@ -1117,6 +1116,8 @@ struct req *SES_GetReq(struct worker *, struct sess *);
 void SES_Handle(struct sess *sp, double now);
 void SES_ReleaseReq(struct req *);
 pool_func_t SES_pool_accept_task;
+void SES_Get_Our_Addr(struct sess *sp);
+
 
 /* cache_shmlog.c */
 extern struct VSC_C_main *VSC_C_main;
diff --git a/bin/varnishd/cache/cache_cli.c b/bin/varnishd/cache/cache_cli.c
index a2927fa..55145cd 100644
--- a/bin/varnishd/cache/cache_cli.c
+++ b/bin/varnishd/cache/cache_cli.c
@@ -43,6 +43,7 @@
 
 #include "cache_backend.h"		// struct vbc
 #include "hash/hash_slinger.h"		// struct objhead
+#include "vsa.h"
 #include "vcli.h"
 #include "vcli_common.h"
 #include "vcli_priv.h"
@@ -139,6 +140,8 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv)
 	SZOF(struct VSC_C_main);
 	SZOF(struct lock);
 	SZOF(struct dstat);
+	VCLI_Out(cli, "sizeof(struct suckaddr) = %d = 0x%x\n",
+	    vsa_suckaddr_len, vsa_suckaddr_len);
 #if 0
 #define OFOF(foo, bar)	{ foo __foo; VCLI_Out(cli, \
     "%-30s = 0x%4zx @ 0x%4zx\n", \
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 8d80160..92221f0 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -47,10 +47,11 @@
 #include "cache.h"
 
 #include "waiter/waiter.h"
+#include "vsa.h"
 #include "vtcp.h"
 #include "vtim.h"
 
-static unsigned ses_size = sizeof (struct sess);
+static unsigned ses_size;
 
 /*--------------------------------------------------------------------*/
 
@@ -89,23 +90,6 @@ SES_Charge(struct worker *wrk, struct req *req)
 }
 
 /*--------------------------------------------------------------------
- * This prepares a session for use, based on its sessmem structure.
- */
-
-static void
-ses_setup(struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	sp->sockaddrlen = sizeof(sp->sockaddr);
-	sp->mysockaddrlen = sizeof(sp->mysockaddr);
-	sp->sockaddr.ss_family = sp->mysockaddr.ss_family = PF_UNSPEC;
-	sp->t_open = NAN;
-	sp->t_idle = NAN;
-	Lck_New(&sp->mtx, lck_sess);
-}
-
-/*--------------------------------------------------------------------
  * Get a new session, preferably by recycling an already ready one
  */
 
@@ -118,7 +102,12 @@ ses_new(struct sesspool *pp)
 	sp = MPL_Get(pp->mpl_sess, NULL);
 	sp->magic = SESS_MAGIC;
 	sp->sesspool = pp;
-	ses_setup(sp);
+	sp->sockaddrlen = sizeof(sp->sockaddr);
+	sp->sockaddr.ss_family = PF_UNSPEC;
+	sp->t_open = NAN;
+	sp->t_idle = NAN;
+	sp->our_addr = NULL;
+	Lck_New(&sp->mtx, lck_sess);
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	return (sp);
 }
@@ -167,6 +156,29 @@ ses_sess_pool_task(struct worker *wrk, void *arg)
 }
 
 /*--------------------------------------------------------------------
+ * Get the local socket address
+ */
+
+void
+SES_Get_Our_Addr(struct sess *sp)
+{
+	char *s;
+	struct sockaddr_storage ss;
+	socklen_t sl;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	if (sp->our_addr != NULL)
+		return;
+
+	sl = sizeof ss;
+	AZ(getsockname(sp->fd, (void*)&ss, &sl));
+	s = (char *)sp;
+	s += sizeof *sp;
+	sp->our_addr = VSA_Build(s, &ss, sl);
+	assert(VSA_Sane(sp->our_addr));
+}
+
+/*--------------------------------------------------------------------
  * VSL log the endpoints of the TCP connection.
  *
  * We use VSL() to get the sessions vxid and to make sure tha this
@@ -188,9 +200,8 @@ ses_vsl_socket(struct sess *sp, const char *lsockname)
 	VTCP_name(&sp->sockaddr, sp->sockaddrlen,
 	    sp->addr, sizeof sp->addr, sp->port, sizeof sp->port);
 	if (cache_param->log_local_addr) {
-		AZ(getsockname(sp->fd, (void*)&sp->mysockaddr,
-		    &sp->mysockaddrlen));
-		VTCP_name(&sp->mysockaddr, sp->mysockaddrlen,
+		SES_Get_Our_Addr(sp);
+		VTCP_name((const void*)sp->our_addr, vsa_suckaddr_len,
 		    laddr, sizeof laddr, lport, sizeof lport);
 	} else {
 		strcpy(laddr, "-");
@@ -447,6 +458,7 @@ SES_NewPool(struct pool *wp, unsigned pool_no)
 	pp->mpl_req = MPL_New(nb, &cache_param->req_pool,
 	    &cache_param->workspace_client);
 	bprintf(nb, "sess%u", pool_no);
+	ses_size = sizeof (struct sess) + vsa_suckaddr_len;
 	pp->mpl_sess = MPL_New(nb, &cache_param->sess_pool, &ses_size);
 	return (pp);
 }
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 46ceccf..280118b 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -296,7 +296,7 @@ VRT_r_now(const struct vrt_ctx *ctx)
 /*--------------------------------------------------------------------*/
 
 char *
-VRT_IP_string(const struct vrt_ctx *ctx, const VCL_IP ip)
+VRT_IP_string(const struct vrt_ctx *ctx, VCL_IP ip)
 {
 	char *p;
 	unsigned len;
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 2526d3b..d14de37 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -41,7 +41,6 @@
 #include "vrt.h"
 #include "vrt_obj.h"
 #include "vsa.h"
-#include "vtcp.h"
 
 static char vrt_hostname[255] = "";
 
@@ -544,21 +543,14 @@ VRT_r_client_ip(const struct vrt_ctx *ctx)
 	return (&ctx->req->sp->sockaddr);
 }
 
-const void *
+VCL_IP
 VRT_r_server_ip(const struct vrt_ctx *ctx)
 {
-	int i;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	if (ctx->req->sp->mysockaddr.ss_family == AF_UNSPEC) {
-		i = getsockname(ctx->req->sp->fd,
-		    (void*)&ctx->req->sp->mysockaddr,
-		    &ctx->req->sp->mysockaddrlen);
-		assert(VTCP_Check(i));
-	}
-
-	return (&ctx->req->sp->mysockaddr);
+	SES_Get_Our_Addr(ctx->req->sp);
+	return (ctx->req->sp->our_addr);
 }
 
 const char*
@@ -589,17 +581,11 @@ VRT_r_server_hostname(const struct vrt_ctx *ctx)
 long
 VRT_r_server_port(const struct vrt_ctx *ctx)
 {
-	int i;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	if (ctx->req->sp->mysockaddr.ss_family == AF_UNSPEC) {
-		i = getsockname(ctx->req->sp->fd,
-		    (void*)&ctx->req->sp->mysockaddr,
-		    &ctx->req->sp->mysockaddrlen);
-		assert(VTCP_Check(i));
-	}
-	return (VSA_Port(&ctx->req->sp->mysockaddr));
+	SES_Get_Our_Addr(ctx->req->sp);
+	return (VSA_Port(ctx->req->sp->our_addr));
 }
 
 /*--------------------------------------------------------------------*/



From phk at varnish-cache.org  Mon Oct 28 07:38:00 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 28 Oct 2013 08:38:00 +0100
Subject: [master] ab5b0c8 Convert the remote address of the session to
 suckaddr too.
Message-ID: <E1VahOi-00046S-8K@project.varnish-software.com>

commit ab5b0c8388eb0698bd7d90f1df0657cc8a5287e0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 28 07:37:44 2013 +0000

    Convert the remote address of the session to suckaddr too.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 069a80a..17f7f1f 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -767,8 +767,7 @@ struct sess {
 
 	/* Session related fields ------------------------------------*/
 
-	socklen_t		sockaddrlen;
-	struct sockaddr_storage sockaddr;
+	struct suckaddr		*their_addr;
 	struct suckaddr		*our_addr;
 
 	/* formatted ascii client address */
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index bfaabca..17fe100 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -51,6 +51,7 @@
 
 #include "vcli.h"
 #include "vcli_priv.h"
+#include "vsa.h"
 #include "vtcp.h"
 #include "vtim.h"
 
@@ -350,9 +351,9 @@ VCA_SetupSess(struct worker *wrk, struct sess *sp)
 	sp->fd = wa->acceptsock;
 	wa->acceptsock = -1;
 	retval = wa->acceptlsock->name;
-	assert(wa->acceptaddrlen <= sp->sockaddrlen);
-	memcpy(&sp->sockaddr, &wa->acceptaddr, wa->acceptaddrlen);
-	sp->sockaddrlen = wa->acceptaddrlen;
+	assert(wa->acceptaddrlen <= vsa_suckaddr_len);
+	sp->their_addr = VSA_Build(sp->their_addr,
+	    &wa->acceptaddr, wa->acceptaddrlen);
 	vca_pace_good();
 	wrk->stats.sess_conn++;
 	WS_Release(wrk->aws, 0);
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 92221f0..e27ed68 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -97,13 +97,19 @@ static struct sess *
 ses_new(struct sesspool *pp)
 {
 	struct sess *sp;
+	char *s;
 
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 	sp = MPL_Get(pp->mpl_sess, NULL);
 	sp->magic = SESS_MAGIC;
 	sp->sesspool = pp;
-	sp->sockaddrlen = sizeof(sp->sockaddr);
-	sp->sockaddr.ss_family = PF_UNSPEC;
+
+	s = (char *)sp;
+	s += sizeof *sp;
+	s += vsa_suckaddr_len;
+	memset(s, 0, vsa_suckaddr_len);
+	sp->their_addr = (void*)s;
+
 	sp->t_open = NAN;
 	sp->t_idle = NAN;
 	sp->our_addr = NULL;
@@ -197,7 +203,7 @@ ses_vsl_socket(struct sess *sp, const char *lsockname)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AN(lsockname);
 
-	VTCP_name(&sp->sockaddr, sp->sockaddrlen,
+	VTCP_name((const void *)sp->their_addr, vsa_suckaddr_len,
 	    sp->addr, sizeof sp->addr, sp->port, sizeof sp->port);
 	if (cache_param->log_local_addr) {
 		SES_Get_Our_Addr(sp);
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index d14de37..0ef4a05 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -540,7 +540,7 @@ VRT_r_client_ip(const struct vrt_ctx *ctx)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	return (&ctx->req->sp->sockaddr);
+	return (ctx->req->sp->their_addr);
 }
 
 VCL_IP



From phk at varnish-cache.org  Mon Oct 28 07:54:18 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 28 Oct 2013 08:54:18 +0100
Subject: [master] c73544e Make VTCP_name() take a suckaddr.
Message-ID: <E1VaheU-00064N-Pj@project.varnish-software.com>

commit c73544e44cdbaaaec6396f5f66b47f32a486d326
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 28 07:54:02 2013 +0000

    Make VTCP_name() take a suckaddr.

diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index e27ed68..3d1964a 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -203,11 +203,11 @@ ses_vsl_socket(struct sess *sp, const char *lsockname)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AN(lsockname);
 
-	VTCP_name((const void *)sp->their_addr, vsa_suckaddr_len,
+	VTCP_name(sp->their_addr,
 	    sp->addr, sizeof sp->addr, sp->port, sizeof sp->port);
 	if (cache_param->log_local_addr) {
 		SES_Get_Our_Addr(sp);
-		VTCP_name((const void*)sp->our_addr, vsa_suckaddr_len,
+		VTCP_name(sp->our_addr,
 		    laddr, sizeof laddr, lport, sizeof lport);
 	} else {
 		strcpy(laddr, "-");
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 280118b..1f58a1d 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -43,7 +43,6 @@
 #include "vcl.h"
 #include "vrt.h"
 #include "vrt_obj.h"
-#include "vsa.h"
 #include "vtcp.h"
 #include "vtim.h"
 
@@ -304,7 +303,7 @@ VRT_IP_string(const struct vrt_ctx *ctx, VCL_IP ip)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	len = WS_Reserve(ctx->ws, 0);
 	p = ctx->ws->r;
-	VTCP_name(ip, VSA_Len(ip), p, len, NULL, 0);
+	VTCP_name(ip, p, len, NULL, 0);
 	WS_Release(ctx->ws, strlen(p) + 1);
 	return (p);
 }
diff --git a/include/vtcp.h b/include/vtcp.h
index 842d897..d89d383 100644
--- a/include/vtcp.h
+++ b/include/vtcp.h
@@ -67,8 +67,8 @@ int VTCP_linger(int sock, int linger);
 int VTCP_check_hup(int sock);
 
 #ifdef SOL_SOCKET
-void VTCP_name(const struct sockaddr_storage *addr, unsigned l, char *abuf,
-    unsigned alen, char *pbuf, unsigned plen);
+void VTCP_name(const struct suckaddr *addr, char *abuf, unsigned alen,
+    char *pbuf, unsigned plen);
 int VTCP_connect(int s, const struct suckaddr *name, int msec);
 void VTCP_close(int *s);
 void VTCP_set_read_timeout(int s, double seconds);
diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 81a5f27..282d5f5 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -55,14 +55,13 @@
 #include "vtcp.h"
 
 /*--------------------------------------------------------------------*/
-
-void
-VTCP_name(const struct sockaddr_storage *addr, unsigned l,
-    char *abuf, unsigned alen, char *pbuf, unsigned plen)
+static void
+vtcp_sa_to_ascii(const void *sa, socklen_t l, char *abuf, unsigned alen,
+    char *pbuf, unsigned plen)
 {
 	int i;
 
-	i = getnameinfo((const void *)addr, l, abuf, alen, pbuf, plen,
+	i = getnameinfo(sa, l, abuf, alen, pbuf, plen,
 	   NI_NUMERICHOST | NI_NUMERICSERV);
 	if (i) {
 		/*
@@ -85,6 +84,19 @@ VTCP_name(const struct sockaddr_storage *addr, unsigned l,
 /*--------------------------------------------------------------------*/
 
 void
+VTCP_name(const struct suckaddr *addr, char *abuf, unsigned alen,
+    char *pbuf, unsigned plen)
+{
+	const struct sockaddr *sa;
+	socklen_t sl;
+
+	sa = VSA_Get_Sockaddr(addr, &sl);
+	vtcp_sa_to_ascii(sa, sl, abuf, alen, pbuf, plen);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
 VTCP_myname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen)
 {
 	struct sockaddr_storage addr_s;
@@ -92,8 +104,9 @@ VTCP_myname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen)
 
 	l = sizeof addr_s;
 	AZ(getsockname(sock, (void *)&addr_s, &l));
-	VTCP_name(&addr_s, l, abuf, alen, pbuf, plen);
+	vtcp_sa_to_ascii(&addr_s, l, abuf, alen, pbuf, plen);
 }
+
 /*--------------------------------------------------------------------*/
 
 void
@@ -104,7 +117,7 @@ VTCP_hisname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen)
 
 	l = sizeof addr_s;
 	if (!getpeername(sock, (void*)&addr_s, &l))
-		VTCP_name(&addr_s, l, abuf, alen, pbuf, plen);
+		vtcp_sa_to_ascii(&addr_s, l, abuf, alen, pbuf, plen);
 	else {
 		(void)snprintf(abuf, alen, "<none>");
 		(void)snprintf(pbuf, plen, "<none>");



From phk at varnish-cache.org  Mon Oct 28 08:10:33 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 28 Oct 2013 09:10:33 +0100
Subject: [master] 6662b1b Pull the switch and give VCL_IP a proper type.
Message-ID: <E1VahuD-00083a-F7@project.varnish-software.com>

commit 6662b1bbb8895f2961e6ed43393d9b6250b07e08
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 28 08:10:11 2013 +0000

    Pull the switch and give VCL_IP a proper type.

diff --git a/include/vrt.h b/include/vrt.h
index cebe8e5..cab38de 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -56,8 +56,7 @@ typedef double				VCL_DURATION;
 typedef const char *			VCL_ENUM;
 typedef const char *			VCL_HEADER;
 typedef long				VCL_INT;
-//typedef const struct suckaddr *		VCL_IP;
-typedef const void *			VCL_IP;
+typedef const struct suckaddr *		VCL_IP;
 typedef double				VCL_REAL;
 typedef const char *			VCL_STRING;
 typedef double				VCL_TIME;
diff --git a/include/vsa.h b/include/vsa.h
index d21f3c0..26f6419 100644
--- a/include/vsa.h
+++ b/include/vsa.h
@@ -33,9 +33,9 @@
 struct suckaddr;
 extern const int vsa_suckaddr_len;
 
-int VSA_Sane(const void *);
-socklen_t VSA_Len(const void *);
-unsigned VSA_Port(const void *);
+int VSA_Sane(const struct suckaddr *);
+socklen_t VSA_Len(const struct suckaddr *);
+unsigned VSA_Port(const struct suckaddr *);
 int VSA_Compare(const struct suckaddr *, const struct suckaddr *);
 
 const struct sockaddr *VSA_Get_Sockaddr(const struct suckaddr *, socklen_t *sl);
diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c
index 97ff0fa..4517549 100644
--- a/lib/libvarnish/vsa.c
+++ b/lib/libvarnish/vsa.c
@@ -252,9 +252,11 @@ VSA_Build(void *d, const void *s, unsigned sal)
 		default:
 			break;
 	}
-	if (l != 0)
+	if (l != 0) {
 		memcpy(&sua->sa, s, l);
-	return (sua);
+		return (sua);
+	}
+	return (NULL);
 }
 
 const struct sockaddr *
@@ -276,11 +278,10 @@ VSA_Get_Sockaddr(const struct suckaddr *sua, socklen_t *sl)
 }
 
 int
-VSA_Sane(const void *s)
+VSA_Sane(const struct suckaddr *s)
 {
-	const struct sockaddr *sa = s;
 
-	switch(sa->sa_family) {
+	switch(s->sa.sa_family) {
 		case PF_INET:
 		case PF_INET6:
 			return (1);
@@ -290,17 +291,16 @@ VSA_Sane(const void *s)
 }
 
 socklen_t
-VSA_Len(const void *s)
+VSA_Len(const struct suckaddr *s)
 {
-	const struct sockaddr *sa = s;
 
-	switch(sa->sa_family) {
+	switch(s->sa.sa_family) {
 		case PF_INET:
-			return (sizeof(struct sockaddr_in));
+			return (sizeof(s->sa4));
 		case PF_INET6:
-			return (sizeof(struct sockaddr_in6));
+			return (sizeof(s->sa6));
 		default:
-			WRONG("Illegal socket family");
+			return (0);
 	}
 }
 
@@ -313,22 +313,15 @@ VSA_Compare(const struct suckaddr *s1, const struct suckaddr *s2)
 }
 
 unsigned
-VSA_Port(const void *s)
+VSA_Port(const struct suckaddr *s)
 {
-	const struct sockaddr *sa = s;
 
-	switch(sa->sa_family) {
+	switch(s->sa.sa_family) {
 		case PF_INET:
-			{
-			const struct sockaddr_in *ain = s;
-			return (ntohs((ain->sin_port)));
-			}
+			return (ntohs(s->sa4.sin_port));
 		case PF_INET6:
-			{
-			const struct sockaddr_in6 *ain = s;
-			return (ntohs((ain->sin6_port)));
-			}
+			return (ntohs(s->sa6.sin6_port));
 		default:
-			WRONG("Illegal socket family");
+			return (0);
 	}
 }
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index 42325d8..822bb75 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -127,7 +127,7 @@ VCL_IP
 vmod_ip(const struct vrt_ctx *ctx, VCL_STRING s, VCL_IP d)
 {
 	struct addrinfo hints, *res0 = NULL;
-	const struct addrinfo *res, *best = NULL;
+	const struct addrinfo *res;
 	int error;
 	void *p;
 	struct suckaddr *r;
@@ -136,6 +136,10 @@ vmod_ip(const struct vrt_ctx *ctx, VCL_STRING s, VCL_IP d)
 	AN(d);
 	assert(VSA_Sane(d));
 
+	p = WS_Alloc(ctx->ws, vsa_suckaddr_len);
+	AN(p);
+	r = NULL;
+
 	if (s != NULL) {
 		memset(&hints, 0, sizeof(hints));
 		hints.ai_family = PF_UNSPEC;
@@ -143,19 +147,13 @@ vmod_ip(const struct vrt_ctx *ctx, VCL_STRING s, VCL_IP d)
 		error = getaddrinfo(s, "80", &hints, &res0);
 		if (!error) {
 			for (res = res0; res != NULL; res = res->ai_next) {
-				if (VSA_Sane(res->ai_addr) &&
-				    res->ai_addrlen >= VSA_Len(res->ai_addr)) {
-					best = res;
+				r = VSA_Build(p, res->ai_addr, res->ai_addrlen);
+				if (r != NULL)
 					break;
-				}
 			}
 		}
 	}
-	p = WS_Alloc(ctx->ws, vsa_suckaddr_len);
-	AN(p);
-	if (best != NULL)
-		r = VSA_Build(p, best->ai_addr, best->ai_addrlen);
-	else {
+	if (r == NULL) {
 		r = p;
 		memcpy(r, d, vsa_suckaddr_len);
 	}



From phk at varnish-cache.org  Mon Oct 28 08:45:36 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 28 Oct 2013 09:45:36 +0100
Subject: [master] b1b8114 Clean up a couple of places which assume to much
 about suckaddr.
Message-ID: <E1VaiS8-00070k-OT@project.varnish-software.com>

commit b1b811400ddb275aa6aac3831e95200ad6539e0f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 28 08:43:37 2013 +0000

    Clean up a couple of places which assume to much about suckaddr.

diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 282d5f5..989c4c0 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -213,6 +213,8 @@ VTCP_connect(int s, const struct suckaddr *name, int msec)
 	int i, k;
 	socklen_t l;
 	struct pollfd fds[1];
+	const struct sockaddr *sa;
+	socklen_t sl;
 
 	assert(s >= 0);
 
@@ -221,8 +223,9 @@ VTCP_connect(int s, const struct suckaddr *name, int msec)
 		(void)VTCP_nonblocking(s);
 
 	/* Attempt the connect */
-	assert(VSA_Sane(name));
-	i = connect(s, (const void *)name, VSA_Len(name));
+	AN(VSA_Sane(name));
+	sa = VSA_Get_Sockaddr(name, &sl);
+	i = connect(s, sa, sl);
 	if (i == 0 || errno != EINPROGRESS)
 		return (i);
 
diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c
index aa82bc8..e756360 100644
--- a/lib/libvcc/vcc_utils.c
+++ b/lib/libvcc/vcc_utils.c
@@ -102,7 +102,6 @@ vcc_sockaddr(struct vcc *tl, const void *sa, unsigned sal)
 
 	assert(sizeof(unsigned long long) == 8);
 
-	assert(VSA_Sane(sa));
 	AN(sa);
 	AN(sal);
 



From phk at varnish-cache.org  Mon Oct 28 08:45:36 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 28 Oct 2013 09:45:36 +0100
Subject: [master] 9beaead OK,
 that was sort of embarrasing: Allocate enough space for *both* the
 two sess-> suckaddrs.
Message-ID: <E1VaiS8-00070p-Pl@project.varnish-software.com>

commit 9beaead5064f0a10731cac64372d28dc4082a7af
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 28 08:45:09 2013 +0000

    OK, that was sort of embarrasing:  Allocate enough space for *both*
    the two sess-> suckaddrs.

diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 3d1964a..915aea6 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -464,7 +464,7 @@ SES_NewPool(struct pool *wp, unsigned pool_no)
 	pp->mpl_req = MPL_New(nb, &cache_param->req_pool,
 	    &cache_param->workspace_client);
 	bprintf(nb, "sess%u", pool_no);
-	ses_size = sizeof (struct sess) + vsa_suckaddr_len;
+	ses_size = sizeof (struct sess) + vsa_suckaddr_len * 2;
 	pp->mpl_sess = MPL_New(nb, &cache_param->sess_pool, &ses_size);
 	return (pp);
 }



From phk at varnish-cache.org  Mon Oct 28 08:45:56 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 28 Oct 2013 09:45:56 +0100
Subject: [master] 36ad06e Turn suckaddr into a miniobj
Message-ID: <E1VaiSS-00076h-13@project.varnish-software.com>

commit 36ad06ef25f2f0a6c0202e3e55253b67052c9d65
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 28 08:45:45 2013 +0000

    Turn suckaddr into a miniobj

diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c
index 4517549..c2838d6 100644
--- a/lib/libvarnish/vsa.c
+++ b/lib/libvarnish/vsa.c
@@ -42,6 +42,7 @@
 #include "vas.h"
 #include "vsa.h"
 #include "vrt.h"
+#include "miniobj.h"
 
 /*
  * Struct sockaddr{|_in|_in6|_storage} is absolutely the worst data
@@ -162,6 +163,8 @@
  */
 
 struct suckaddr {
+	unsigned			magic;
+#define SUCKADDR_MAGIC			0x4b1e9335
 	union {
 		struct sockaddr		sa;
 		struct sockaddr_in	sa4;
@@ -179,9 +182,11 @@ const int vsa_suckaddr_len = sizeof(struct suckaddr);
 int
 VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst)
 {
+
 	AN(dst);
 	if (sua == NULL)
 		return (-1);
+	CHECK_OBJ_NOTNULL(sua, SUCKADDR_MAGIC);
 
 	switch(sua->sa.sa_family) {
 		case PF_INET:
@@ -223,7 +228,7 @@ VSA_Malloc(const void *s, unsigned  sal)
 			break;
 	}
 	if (l != 0) {
-		sua = calloc(1, sizeof *sua);
+		ALLOC_OBJ(sua, SUCKADDR_MAGIC);
 		if (sua != NULL)
 			memcpy(&sua->sa, s, l);
 	}
@@ -253,6 +258,8 @@ VSA_Build(void *d, const void *s, unsigned sal)
 			break;
 	}
 	if (l != 0) {
+		memset(sua, 0, sizeof *sua);
+		sua->magic = SUCKADDR_MAGIC;
 		memcpy(&sua->sa, s, l);
 		return (sua);
 	}
@@ -262,7 +269,8 @@ VSA_Build(void *d, const void *s, unsigned sal)
 const struct sockaddr *
 VSA_Get_Sockaddr(const struct suckaddr *sua, socklen_t *sl)
 {
-	AN(sua);
+
+	CHECK_OBJ_NOTNULL(sua, SUCKADDR_MAGIC);
 	AN(sl);
 	switch(sua->sa.sa_family) {
 		case PF_INET:
@@ -278,10 +286,11 @@ VSA_Get_Sockaddr(const struct suckaddr *sua, socklen_t *sl)
 }
 
 int
-VSA_Sane(const struct suckaddr *s)
+VSA_Sane(const struct suckaddr *sua)
 {
+	CHECK_OBJ_NOTNULL(sua, SUCKADDR_MAGIC);
 
-	switch(s->sa.sa_family) {
+	switch(sua->sa.sa_family) {
 		case PF_INET:
 		case PF_INET6:
 			return (1);
@@ -291,36 +300,39 @@ VSA_Sane(const struct suckaddr *s)
 }
 
 socklen_t
-VSA_Len(const struct suckaddr *s)
+VSA_Len(const struct suckaddr *sua)
 {
+	CHECK_OBJ_NOTNULL(sua, SUCKADDR_MAGIC);
 
-	switch(s->sa.sa_family) {
+	switch(sua->sa.sa_family) {
 		case PF_INET:
-			return (sizeof(s->sa4));
+			return (sizeof(sua->sa4));
 		case PF_INET6:
-			return (sizeof(s->sa6));
+			return (sizeof(sua->sa6));
 		default:
 			return (0);
 	}
 }
 
 int
-VSA_Compare(const struct suckaddr *s1, const struct suckaddr *s2)
+VSA_Compare(const struct suckaddr *sua1, const struct suckaddr *sua2)
 {
-	AN(s1);
-	AN(s2);
-	return (memcmp(s1, s2, vsa_suckaddr_len));
+
+	CHECK_OBJ_NOTNULL(sua1, SUCKADDR_MAGIC);
+	CHECK_OBJ_NOTNULL(sua2, SUCKADDR_MAGIC);
+	return (memcmp(sua1, sua2, vsa_suckaddr_len));
 }
 
 unsigned
-VSA_Port(const struct suckaddr *s)
+VSA_Port(const struct suckaddr *sua)
 {
 
-	switch(s->sa.sa_family) {
+	CHECK_OBJ_NOTNULL(sua, SUCKADDR_MAGIC);
+	switch(sua->sa.sa_family) {
 		case PF_INET:
-			return (ntohs(s->sa4.sin_port));
+			return (ntohs(sua->sa4.sin_port));
 		case PF_INET6:
-			return (ntohs(s->sa6.sin6_port));
+			return (ntohs(sua->sa6.sin6_port));
 		default:
 			return (0);
 	}



From phk at varnish-cache.org  Tue Oct 29 12:50:07 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 29 Oct 2013 13:50:07 +0100
Subject: [master] 037e9dc Minor nits.
Message-ID: <E1Vb8kJ-0006FT-3P@project.varnish-software.com>

commit 037e9dc2666590a7070dd92d1b16d58fed16ac7b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 29 12:49:59 2013 +0000

    Minor nits.

diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 915aea6..4006ea3 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -464,7 +464,7 @@ SES_NewPool(struct pool *wp, unsigned pool_no)
 	pp->mpl_req = MPL_New(nb, &cache_param->req_pool,
 	    &cache_param->workspace_client);
 	bprintf(nb, "sess%u", pool_no);
-	ses_size = sizeof (struct sess) + vsa_suckaddr_len * 2;
+	ses_size = sizeof (struct sess) + vsa_suckaddr_len * 2L;
 	pp->mpl_sess = MPL_New(nb, &cache_param->sess_pool, &ses_size);
 	return (pp);
 }
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index f5ff211..6d58eeb 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -316,7 +316,7 @@ child_poker(const struct vev *e, int what)
  */
 
 static void __match_proto__()
-mgt_sigsegv_handler(int s, siginfo_t *si, void *c)
+child_sigsegv_handler(int s, siginfo_t *si, void *c)
 {
 	char buf[1024];
 
@@ -414,7 +414,7 @@ mgt_launch_child(struct cli *cli)
 
 		if (mgt_param.sigsegv_handler) {
 			memset(&sa, 0, sizeof sa);
-			sa.sa_sigaction = mgt_sigsegv_handler;
+			sa.sa_sigaction = child_sigsegv_handler;
 			sa.sa_flags = SA_SIGINFO;
 			(void)sigaction(SIGSEGV, &sa, NULL);
 		}



From phk at varnish-cache.org  Tue Oct 29 13:27:27 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 29 Oct 2013 14:27:27 +0100
Subject: [master] 5d383b0 size req->vsl using the proper parameter.
Message-ID: <E1Vb9KR-0008TE-6Z@project.varnish-software.com>

commit 5d383b0ba48b816a106bc96ec62cd4656c496f0c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 29 13:27:13 2013 +0000

    size req->vsl using the proper parameter.

diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 4006ea3..886b3de 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -401,7 +401,7 @@ SES_GetReq(struct worker *wrk, struct sess *sp)
 	p = (void*)PRNDUP(p);
 	assert(p < e);
 
-	sz = cache_param->workspace_thread;
+	sz = cache_param->vsl_buffer;
 	VSL_Setup(req->vsl, p, sz);
 	req->vsl->wid = VXID_Get(&wrk->vxid_pool) | VSL_CLIENTMARKER;
 	VSLb(req->vsl, SLT_Begin, "req %u", sp->vxid & VSL_IDENTMASK);



From phk at varnish-cache.org  Tue Oct 29 13:33:57 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 29 Oct 2013 14:33:57 +0100
Subject: [master] a9531d1 Update test-case now that req->VSL is sized correctly
Message-ID: <E1Vb9Qj-0000lO-AG@project.varnish-software.com>

commit a9531d1e1b02369e3844c7eab26737bd6c189639
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 29 13:33:44 2013 +0000

    Update test-case now that req->VSL is sized correctly

diff --git a/bin/varnishtest/tests/r01120.vtc b/bin/varnishtest/tests/r01120.vtc
index 97bc2e5..c55064a 100644
--- a/bin/varnishtest/tests/r01120.vtc
+++ b/bin/varnishtest/tests/r01120.vtc
@@ -8,7 +8,7 @@ server s1 {
 } -start
 
 varnish v1 \
-	-cliok "param.set workspace_client 8k" \
+	-cliok "param.set workspace_client 10k" \
 	-cliok "param.set workspace_backend 200k" \
 	-vcl+backend {
 } -start



From phk at varnish-cache.org  Tue Oct 29 14:00:56 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 29 Oct 2013 15:00:56 +0100
Subject: [master] 3fb23a2 Fix my typo that meant that we trampled on unowned
 memory.
Message-ID: <E1Vb9qq-0004bH-M5@project.varnish-software.com>

commit 3fb23a2edd675a5dd63d0fa0e591c036cd7cbed1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 29 14:00:22 2013 +0000

    Fix my typo that meant that we trampled on unowned memory.
    
    Almost pinpointed by Martin using Valgrind.

diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 886b3de..cdfcc9d 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -98,9 +98,10 @@ ses_new(struct sesspool *pp)
 {
 	struct sess *sp;
 	char *s;
+	unsigned sz;
 
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
-	sp = MPL_Get(pp->mpl_sess, NULL);
+	sp = MPL_Get(pp->mpl_sess, &sz);
 	sp->magic = SESS_MAGIC;
 	sp->sesspool = pp;
 
@@ -109,6 +110,8 @@ ses_new(struct sesspool *pp)
 	s += vsa_suckaddr_len;
 	memset(s, 0, vsa_suckaddr_len);
 	sp->their_addr = (void*)s;
+	s += vsa_suckaddr_len;
+	assert((char *)sp + sz == s);
 
 	sp->t_open = NAN;
 	sp->t_idle = NAN;
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 1f58a1d..563b2bc 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -302,7 +302,7 @@ VRT_IP_string(const struct vrt_ctx *ctx, VCL_IP ip)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	len = WS_Reserve(ctx->ws, 0);
-	p = ctx->ws->r;
+	p = ctx->ws->f;
 	VTCP_name(ip, p, len, NULL, 0);
 	WS_Release(ctx->ws, strlen(p) + 1);
 	return (p);



From martin at varnish-cache.org  Wed Oct 30 09:35:16 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 30 Oct 2013 10:35:16 +0100
Subject: [3.0] 1b61e9d Remove a -trunk file that got merged into 3.0 by
 mistake.
Message-ID: <E1VbSBI-00071u-44@project.varnish-software.com>

commit 1b61e9dbb5f13800214f0d16550ca74f672c3bd2
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 30 10:34:38 2013 +0100

    Remove a -trunk file that got merged into 3.0 by mistake.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
deleted file mode 100644
index e1fd343..0000000
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ /dev/null
@@ -1,1294 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * This file contains the request-handling state engine, which is intended to
- * (over time) be(ome) protocol agnostic.
- * We already use this now with ESI:includes, which are for all relevant
- * purposes a different "protocol"
- *
- * A special complication is the fact that we can suspend processing of
- * a request when hash-lookup finds a busy objhdr.
- *
- * Since the states are rather nasty in detail, I have decided to embedd
- * a dot(1) graph in the source code comments.  So to see the big picture,
- * extract the DOT lines and run though dot(1), for instance with the
- * command:
- *	sed -n '/^DOT/s///p' cache/cache_req_fsm.c | dot -Tps > /tmp/_.ps
- */
-
-/*
-DOT digraph vcl_center {
-xDOT	page="8.2,11.5"
-DOT	size="7.2,10.5"
-DOT	margin="0.5"
-DOT	center="1"
-DOT acceptor [
-DOT	shape=hexagon
-DOT	label="Request received"
-DOT ]
-DOT ESI_REQ [ shape=hexagon ]
-DOT ESI_REQ -> recv
-DOT ERROR [shape=plaintext]
-DOT RESTART [shape=plaintext]
-DOT acceptor -> recv [style=bold,color=green]
- */
-
-#include "config.h"
-
-#include <math.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "hash/hash_slinger.h"
-#include "vcl.h"
-#include "vsha256.h"
-#include "vtim.h"
-
-#ifndef HAVE_SRANDOMDEV
-#include "compat/srandomdev.h"
-#endif
-
-/*--------------------------------------------------------------------
- * We have a refcounted object on the session, and possibly the busyobj
- * which is fetching it, prepare a response.
- *
-DOT subgraph xcluster_prepresp {
-DOT	prepresp [
-DOT		shape=record
-DOT		label="{cnt_prepresp:|Filter obj.-\>resp.|{vcl_deliver\{\}|{req.|resp.}}|{error?|restart?}|stream ?}"
-DOT	]
-DOT	prepresp -> deliver [style=bold,color=green,label=deliver]
-DOT	prepresp -> deliver [style=bold,color=red]
-DOT	prepresp -> deliver [style=bold,color=blue]
-DOT }
- *
- */
-
-static enum req_fsm_nxt
-cnt_prepresp(struct worker *wrk, struct req *req)
-{
-	struct busyobj *bo;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	bo = req->busyobj;
-	CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC);
-
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
-
-	req->res_mode = 0;
-
-	if (bo == NULL) {
-		if (!req->disable_esi && req->obj->esidata != NULL) {
-			/* In ESI mode, we can't know the aggregate length */
-			req->res_mode &= ~RES_LEN;
-			req->res_mode |= RES_ESI;
-		} else {
-			req->res_mode |= RES_LEN;
-		}
-	} else {
-		AZ(bo->do_esi);
-	}
-
-	if (req->esi_level > 0) {
-		/* Included ESI object, always CHUNKED or EOF */
-		req->res_mode &= ~RES_LEN;
-		req->res_mode |= RES_ESI_CHILD;
-	}
-
-	if (cache_param->http_gzip_support && req->obj->gziped &&
-	    !RFC2616_Req_Gzip(req->http)) {
-		/*
-		 * We don't know what it uncompresses to
-		 * XXX: we could cache that
-		 */
-		req->res_mode &= ~RES_LEN;
-		req->res_mode |= RES_GUNZIP;
-	}
-
-	if (!(req->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
-		/* We havn't chosen yet, do so */
-		if (!req->wantbody) {
-			/* Nothing */
-		} else if (req->http->protover >= 11) {
-			req->res_mode |= RES_CHUNKED;
-		} else {
-			req->res_mode |= RES_EOF;
-			req->doclose = SC_TX_EOF;
-		}
-	}
-
-	req->t_resp = W_TIM_real(wrk);
-	if (req->obj->objcore->objhead != NULL) {
-		if ((req->t_resp - req->obj->last_lru) >
-		    cache_param->lru_timeout &&
-		    EXP_Touch(req->obj->objcore))
-			req->obj->last_lru = req->t_resp;
-		if (!cache_param->obj_readonly)
-			req->obj->last_use = req->t_resp; /* XXX: locking ? */
-	}
-	HTTP_Setup(req->resp, req->ws, req->vsl, HTTP_Resp);
-	RES_BuildHttp(req);
-
-	VCL_deliver_method(req);
-	switch (req->handling) {
-	case VCL_RET_DELIVER:
-		break;
-	case VCL_RET_RESTART:
-		if (req->restarts >= cache_param->max_restarts)
-			break;
-		if (bo != NULL) {
-			AN(bo->do_stream);
-			(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
-			VBO_DerefBusyObj(wrk, &req->busyobj);
-		} else {
-			(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
-		}
-		AZ(req->obj);
-		http_Teardown(req->resp);
-		req->req_step = R_STP_RESTART;
-		return (REQ_FSM_MORE);
-	default:
-		WRONG("Illegal action in vcl_deliver{}");
-	}
-	req->req_step = R_STP_DELIVER;
-	return (REQ_FSM_MORE);
-}
-
-/*--------------------------------------------------------------------
- * Deliver an already stored object
- *
-DOT subgraph xcluster_deliver {
-DOT	deliver [
-DOT		shape=record
-DOT		label="{cnt_deliver:|Send body}"
-DOT	]
-DOT }
-DOT deliver -> DONE [style=bold,color=green]
-DOT deliver -> DONE [style=bold,color=red]
-DOT deliver -> DONE [style=bold,color=blue]
- *
- */
-
-static enum req_fsm_nxt
-cnt_deliver(struct worker *wrk, struct req *req)
-{
-	struct busyobj *bo;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-	bo = req->busyobj;
-	CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC);
-
-	if (bo != NULL) {
-		while (bo->state < BOS_FAILED)
-			(void)usleep(10000);
-		assert(bo->state >= BOS_FAILED);
-
-		if (bo->state == BOS_FAILED) {
-			(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
-			VBO_DerefBusyObj(wrk, &req->busyobj);
-			req->err_code = 503;
-			req->req_step = R_STP_ERROR;
-			return (REQ_FSM_MORE);
-		}
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-	}
-
-	AZ(req->busyobj);
-	req->director = NULL;
-	req->restarts = 0;
-
-	RES_WriteObj(req);
-
-	/* No point in saving the body if it is hit-for-pass */
-	if (req->obj->objcore->flags & OC_F_PASS)
-		STV_Freestore(req->obj);
-
-	assert(WRW_IsReleased(wrk));
-	(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
-	http_Teardown(req->resp);
-	return (REQ_FSM_DONE);
-}
-/*--------------------------------------------------------------------
- * Emit an error
- *
-DOT subgraph xcluster_error {
-DOT	vcl_error [
-DOT		shape=record
-DOT		label="vcl_error()|resp."
-DOT	]
-DOT	ERROR -> vcl_error
-DOT	vcl_error-> prepresp [label=deliver]
-DOT }
-DOT vcl_error-> rsterr [label="restart",color=purple]
-DOT rsterr [label="RESTART",shape=plaintext]
- */
-
-static enum req_fsm_nxt
-cnt_error(struct worker *wrk, struct req *req)
-{
-	struct http *h;
-	struct busyobj *bo;
-	char date[40];
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	AZ(req->objcore);
-	AZ(req->obj);
-	AZ(req->busyobj);
-
-	req->acct_req.error++;
-	bo = VBO_GetBusyObj(wrk, req);
-	req->busyobj = bo;
-	AZ(bo->stats);
-	bo->stats = &wrk->stats;
-	req->objcore = HSH_NewObjCore(wrk);
-	req->obj = STV_NewObject(bo, &req->objcore,
-	    TRANSIENT_STORAGE, cache_param->http_resp_size,
-	    (uint16_t)cache_param->http_max_hdr);
-	bo->stats = NULL;
-	if (req->obj == NULL) {
-		req->doclose = SC_OVERLOAD;
-		req->director = NULL;
-		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
-		req->objcore = NULL;
-		http_Teardown(bo->beresp);
-		http_Teardown(bo->bereq);
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-		AZ(req->busyobj);
-		return (REQ_FSM_DONE);
-	}
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-	AZ(req->objcore);
-	req->obj->vxid = bo->vsl->wid;
-	req->obj->exp.entered = req->t_req;
-
-	h = req->obj->http;
-
-	if (req->err_code < 100 || req->err_code > 999)
-		req->err_code = 501;
-
-	http_PutProtocol(h, "HTTP/1.1");
-	http_PutStatus(h, req->err_code);
-	VTIM_format(W_TIM_real(wrk), date);
-	http_PrintfHeader(h, "Date: %s", date);
-	http_SetHeader(h, "Server: Varnish");
-
-	if (req->err_reason != NULL)
-		http_PutResponse(h, req->err_reason);
-	else
-		http_PutResponse(h, http_StatusMessage(req->err_code));
-	VCL_error_method(req);
-
-	if (req->handling == VCL_RET_RESTART &&
-	    req->restarts <  cache_param->max_restarts) {
-		HSH_Drop(wrk, &req->obj);
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-		req->req_step = R_STP_RESTART;
-		return (REQ_FSM_MORE);
-	} else if (req->handling == VCL_RET_RESTART)
-		req->handling = VCL_RET_DELIVER;
-
-
-	/* We always close when we take this path */
-	req->doclose = SC_TX_ERROR;
-	req->wantbody = 1;
-
-	assert(req->handling == VCL_RET_DELIVER);
-	req->err_code = 0;
-	req->err_reason = NULL;
-	http_Teardown(bo->bereq);
-	VBO_DerefBusyObj(wrk, &req->busyobj);
-	req->req_step = R_STP_PREPRESP;
-	return (REQ_FSM_MORE);
-}
-
-/*--------------------------------------------------------------------
- * Fetch response headers from the backend
- *
-DOT subgraph xcluster_fetch {
-DOT	fetch [
-DOT		shape=record
-DOT		label="{cnt_fetch:|fetch hdr\nfrom backend|(find obj.ttl)|{vcl_fetch\{\}|{req.|bereq.|beresp.}}|{<err>error?|<rst>restart?}}"
-DOT	]
-DOT }
-DOT fetch -> fetchbody [style=bold,color=red]
-DOT fetch -> fetchbody [style=bold,color=blue]
- */
-
-static enum req_fsm_nxt
-cnt_fetch(struct worker *wrk, struct req *req)
-{
-	int i, need_host_hdr;
-	struct busyobj *bo;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
-	bo = req->busyobj;
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-
-	AN(req->director);
-	AZ(bo->vbc);
-	AZ(bo->should_close);
-	AZ(req->storage_hint);
-
-	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
-
-	need_host_hdr = !http_GetHdr(bo->bereq, H_Host, NULL);
-
-	req->acct_req.fetch++;
-
-	i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL);
-	/*
-	 * If we recycle a backend connection, there is a finite chance
-	 * that the backend closed it before we get a request to it.
-	 * Do a single retry in that case.
-	 */
-	if (i == 1) {
-		VSC_C_main->backend_retry++;
-		i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL);
-	}
-
-	if (i) {
-		req->handling = VCL_RET_ERROR;
-		req->err_code = 503;
-	} else {
-		/*
-		 * These two headers can be spread over multiple actual headers
-		 * and we rely on their content outside of VCL, so collect them
-		 * into one line here.
-		 */
-		http_CollectHdr(bo->beresp, H_Cache_Control);
-		http_CollectHdr(bo->beresp, H_Vary);
-
-		/*
-		 * Figure out how the fetch is supposed to happen, before the
-		 * headers are adultered by VCL
-		 * NB: Also sets other wrk variables
-		 */
-		bo->htc.body_status = RFC2616_Body(bo, &wrk->stats);
-
-		req->err_code = http_GetStatus(bo->beresp);
-
-		/*
-		 * What does RFC2616 think about TTL ?
-		 */
-		EXP_Clr(&bo->exp);
-		bo->exp.entered = W_TIM_real(wrk);
-		RFC2616_Ttl(bo);
-
-		/* pass from vclrecv{} has negative TTL */
-		if (req->objcore->objhead == NULL)
-			bo->exp.ttl = -1.;
-
-		AZ(bo->do_esi);
-		AZ(bo->do_pass);
-
-		VCL_fetch_method(req);
-
-		if (bo->do_pass)
-			req->objcore->flags |= OC_F_PASS;
-
-		switch (req->handling) {
-		case VCL_RET_DELIVER:
-			req->req_step = R_STP_FETCHBODY;
-			return (REQ_FSM_MORE);
-		default:
-			break;
-		}
-
-		/* We are not going to fetch the body, Close the connection */
-		VDI_CloseFd(&bo->vbc);
-	}
-
-	/* Clean up partial fetch */
-	AZ(bo->vbc);
-
-	if (req->objcore->objhead != NULL ||
-	    req->handling == VCL_RET_RESTART ||
-	    req->handling == VCL_RET_ERROR) {
-		CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
-		req->objcore = NULL;
-	}
-	assert(bo->refcount == 2);
-	VBO_DerefBusyObj(wrk, &bo);
-	VBO_DerefBusyObj(wrk, &req->busyobj);
-	req->director = NULL;
-	req->storage_hint = NULL;
-
-	switch (req->handling) {
-	case VCL_RET_RESTART:
-		req->req_step = R_STP_RESTART;
-		return (REQ_FSM_MORE);
-	case VCL_RET_ERROR:
-		req->req_step = R_STP_ERROR;
-		return (REQ_FSM_MORE);
-	default:
-		WRONG("Illegal action in vcl_fetch{}");
-	}
-}
-
-/*--------------------------------------------------------------------
- * Prepare to fetch body from backend
- *
-DOT subgraph xcluster_body {
-DOT	fetchbody [
-DOT		shape=record
-DOT		label="{cnt_fetchbody:|start fetch_thread}"
-DOT	]
-DOT }
-DOT fetchbody:out -> prepresp [style=bold,color=red]
-DOT fetchbody:out -> prepresp [style=bold,color=blue]
- */
-
-static enum req_fsm_nxt
-cnt_fetchbody(struct worker *wrk, struct req *req)
-{
-	struct http *hp, *hp2;
-	char *b;
-	uint16_t nhttp;
-	unsigned l;
-	struct vsb *vary = NULL;
-	int varyl = 0, pass;
-	struct busyobj *bo;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-	bo = req->busyobj;
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-
-	assert(req->handling == VCL_RET_DELIVER);
-
-	if (req->objcore->objhead == NULL) {
-		/* This is a pass from vcl_recv */
-		pass = 1;
-		/* VCL may have fiddled this, but that doesn't help */
-		bo->exp.ttl = -1.;
-	} else if (bo->do_pass) {
-		pass = 1;
-	} else {
-		/* regular object */
-		pass = 0;
-	}
-
-	/*
-	 * The VCL variables beresp.do_g[un]zip tells us how we want the
-	 * object processed before it is stored.
-	 *
-	 * The backend Content-Encoding header tells us what we are going
-	 * to receive, which we classify in the following three classes:
-	 *
-	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
-	 *	no Content-Encoding		--> object is not gzip'ed.
-	 *	anything else			--> do nothing wrt gzip
-	 *
-	 */
-
-	/* We do nothing unless the param is set */
-	if (!cache_param->http_gzip_support)
-		bo->do_gzip = bo->do_gunzip = 0;
-
-	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
-
-	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
-
-	/* It can't be both */
-	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
-
-	/* We won't gunzip unless it is gzip'ed */
-	if (bo->do_gunzip && !bo->is_gzip)
-		bo->do_gunzip = 0;
-
-	/* If we do gunzip, remove the C-E header */
-	if (bo->do_gunzip)
-		http_Unset(bo->beresp, H_Content_Encoding);
-
-	/* We wont gzip unless it is ungziped */
-	if (bo->do_gzip && !bo->is_gunzip)
-		bo->do_gzip = 0;
-
-	/* If we do gzip, add the C-E header */
-	if (bo->do_gzip)
-		http_SetHeader(bo->beresp, "Content-Encoding: gzip");
-
-	/* But we can't do both at the same time */
-	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
-
-	/* ESI takes precedence and handles gzip/gunzip itself */
-	if (bo->do_esi)
-		bo->vfp = &vfp_esi;
-	else if (bo->do_gunzip)
-		bo->vfp = &vfp_gunzip;
-	else if (bo->do_gzip)
-		bo->vfp = &vfp_gzip;
-	else if (bo->is_gzip)
-		bo->vfp = &vfp_testgzip;
-
-	if (bo->do_esi || req->esi_level > 0)
-		bo->do_stream = 0;
-	if (!req->wantbody)
-		bo->do_stream = 0;
-
-	/* No reason to try streaming a non-existing body */
-	if (bo->htc.body_status == BS_NONE)
-		bo->do_stream = 0;
-
-	l = http_EstimateWS(bo->beresp,
-	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
-
-	/* Create Vary instructions */
-	if (req->objcore->objhead != NULL) {
-		varyl = VRY_Create(req, bo->beresp, &vary);
-		if (varyl > 0) {
-			AN(vary);
-			assert(varyl == VSB_len(vary));
-			l += varyl;
-		} else if (varyl < 0) {
-			/* Vary parse error */
-			AZ(vary);
-			req->err_code = 503;
-			req->req_step = R_STP_ERROR;
-			AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
-			req->objcore = NULL;
-			VDI_CloseFd(&bo->vbc);
-			VBO_DerefBusyObj(wrk, &req->busyobj);
-			return (REQ_FSM_MORE);
-		} else
-			/* No vary */
-			AZ(vary);
-	}
-
-	/*
-	 * Space for producing a Content-Length: header including padding
-	 * A billion gigabytes is enough for anybody.
-	 */
-	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
-
-	if (bo->exp.ttl < cache_param->shortlived ||
-	    req->objcore == NULL)
-		req->storage_hint = TRANSIENT_STORAGE;
-
-	AZ(bo->stats);
-	bo->stats = &wrk->stats;
-	req->obj = STV_NewObject(bo, &req->objcore, req->storage_hint, l,
-	    nhttp);
-	if (req->obj == NULL) {
-		/*
-		 * Try to salvage the transaction by allocating a
-		 * shortlived object on Transient storage.
-		 */
-		if (bo->exp.ttl > cache_param->shortlived)
-			bo->exp.ttl = cache_param->shortlived;
-		bo->exp.grace = 0.0;
-		bo->exp.keep = 0.0;
-		req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE,
-		    l, nhttp);
-	}
-	bo->stats = NULL;
-	if (req->obj == NULL) {
-		req->err_code = 503;
-		req->req_step = R_STP_ERROR;
-		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
-		req->objcore = NULL;
-		VDI_CloseFd(&bo->vbc);
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-		return (REQ_FSM_MORE);
-	}
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-
-	req->storage_hint = NULL;
-
-	AZ(bo->fetch_obj);
-	bo->fetch_obj = req->obj;
-
-	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
-		req->obj->gziped = 1;
-
-	if (vary != NULL) {
-		req->obj->vary = (void *)WS_Copy(req->obj->http->ws,
-		    VSB_data(vary), varyl);
-		AN(req->obj->vary);
-		VRY_Validate(req->obj->vary);
-		VSB_delete(vary);
-	}
-
-	req->obj->vxid = bo->vsl->wid;
-	req->obj->response = req->err_code;
-	WS_Assert(req->obj->ws_o);
-
-	/* Filter into object */
-	hp = bo->beresp;
-	hp2 = req->obj->http;
-
-	hp2->logtag = HTTP_Obj;
-	http_FilterResp(hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS);
-	http_CopyHome(hp2);
-
-	if (http_GetHdr(hp, H_Last_Modified, &b))
-		req->obj->last_modified = VTIM_parse(b);
-	else
-		req->obj->last_modified = floor(bo->exp.entered);
-
-	assert(WRW_IsReleased(wrk));
-
-	/*
-	 * If we can deliver a 304 reply, we don't bother streaming.
-	 * Notice that vcl_deliver{} could still nuke the headers
-	 * that allow the 304, in which case we return 200 non-stream.
-	 */
-	if (req->obj->response == 200 &&
-	    req->http->conds &&
-	    RFC2616_Do_Cond(req))
-		bo->do_stream = 0;
-
-	/*
-	 * Ready to fetch the body
-	 */
-	bo->fetch_task.func = FetchBody;
-	bo->fetch_task.priv = bo;
-
-	assert(bo->refcount == 2);	/* one for each thread */
-
-	if (req->obj->objcore->objhead != NULL) {
-		EXP_Insert(req->obj);
-		AN(req->obj->objcore->ban);
-		AZ(req->obj->ws_o->overflow);
-		HSH_Unbusy(&wrk->stats, req->obj->objcore);
-	}
-
-	if (!bo->do_stream ||
-	    Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE))
-		FetchBody(wrk, bo);
-
-	if (req->obj->objcore->objhead != NULL)
-		HSH_Ref(req->obj->objcore);
-
-	if (bo->state == BOS_FINISHED) {
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-	} else if (bo->state == BOS_FAILED) {
-		/* handle early failures */
-		(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-		req->err_code = 503;
-		req->req_step = R_STP_ERROR;
-		return (REQ_FSM_MORE);
-	}
-
-	assert(WRW_IsReleased(wrk));
-	req->req_step = R_STP_PREPRESP;
-	return (REQ_FSM_MORE);
-}
-
-/*--------------------------------------------------------------------
- * HIT
- * We had a cache hit.  Ask VCL, then march off as instructed.
- *
-DOT subgraph xcluster_hit {
-DOT	hit [
-DOT		shape=record
-DOT		label="{cnt_hit:|{vcl_hit()|{req.|obj.}}|{<err>error?|<rst>restart?}|{<del>deliver?|<pass>pass?}}"
-DOT	]
-DOT }
-XDOT hit:err -> err_hit [label="error"]
-XDOT err_hit [label="ERROR",shape=plaintext]
-XDOT hit:rst -> rst_hit [label="restart",color=purple]
-XDOT rst_hit [label="RESTART",shape=plaintext]
-DOT hit:pass -> pass [label=pass,style=bold,color=red]
-DOT hit:del -> prepresp [label="deliver",style=bold,color=green]
- */
-
-static enum req_fsm_nxt
-cnt_hit(struct worker *wrk, struct req *req)
-{
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-
-	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
-	AZ(req->objcore);
-	AZ(req->busyobj);
-
-	assert(!(req->obj->objcore->flags & OC_F_PASS));
-
-	VCL_hit_method(req);
-
-	if (req->handling == VCL_RET_DELIVER) {
-		//AZ(req->busyobj->bereq->ws);
-		//AZ(req->busyobj->beresp->ws);
-		(void)HTTP1_DiscardReqBody(req);	// XXX: handle err
-		req->req_step = R_STP_PREPRESP;
-		return (REQ_FSM_MORE);
-	}
-
-	/* Drop our object, we won't need it */
-	(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
-	req->objcore = NULL;
-
-	switch(req->handling) {
-	case VCL_RET_PASS:
-		req->req_step = R_STP_PASS;
-		return (REQ_FSM_MORE);
-	case VCL_RET_ERROR:
-		req->req_step = R_STP_ERROR;
-		return (REQ_FSM_MORE);
-	case VCL_RET_RESTART:
-		req->req_step = R_STP_RESTART;
-		return (REQ_FSM_MORE);
-	default:
-		WRONG("Illegal action in vcl_hit{}");
-	}
-}
-
-/*--------------------------------------------------------------------
- * LOOKUP
- * Hash things together and look object up in hash-table.
- *
- * LOOKUP consists of two substates so that we can reenter if we
- * encounter a busy object.
- *
-DOT subgraph xcluster_lookup {
-DOT	lookup [
-DOT		shape=record
-DOT		label="{<top>cnt_lookup:|hash lookup|{<busy>busy ?|<miss>miss ?}|{<no>no|obj.f.pass?|<yes>yes}}"
-DOT	]
-DOT }
-DOT lookup:busy -> lookup:top [label="(waitinglist)"]
-DOT lookup:miss -> miss [style=bold,color=blue]
-DOT lookup:no -> hit [style=bold,color=green]
-DOT lookup:yes -> pass [style=bold,color=red]
- */
-
-static enum req_fsm_nxt
-cnt_lookup(struct worker *wrk, struct req *req)
-{
-	struct objcore *oc;
-	struct object *o;
-	struct objhead *oh;
-	struct busyobj *bo;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	AZ(req->objcore);
-
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
-	AZ(req->busyobj);
-
-	VRY_Prep(req);
-
-	AZ(req->objcore);
-	oc = HSH_Lookup(req);
-	if (oc == NULL) {
-		/*
-		 * We lost the session to a busy object, disembark the
-		 * worker thread.   We return to STP_LOOKUP when the busy
-		 * object has been unbusied, and still have the objhead
-		 * around to restart the lookup with.
-		 */
-		return (REQ_FSM_DISEMBARK);
-	}
-	AZ(req->objcore);
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	oh = oc->objhead;
-	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-
-	/* If we inserted a new object it's a miss */
-	if (oc->flags & OC_F_BUSY) {
-		AZ(req->busyobj);
-		bo = VBO_GetBusyObj(wrk, req);
-		req->busyobj = bo;
-		/* One ref for req, one for FetchBody */
-		bo->refcount = 2;
-		VRY_Finish(req, bo);
-
-		oc->busyobj = bo;
-		wrk->stats.cache_miss++;
-
-		req->objcore = oc;
-		req->req_step = R_STP_MISS;
-		return (REQ_FSM_MORE);
-	}
-
-	/* We are not prepared to do streaming yet */
-	XXXAZ(req->busyobj);
-
-	o = oc_getobj(&wrk->stats, oc);
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	req->obj = o;
-
-	VRY_Finish(req, NULL);
-
-	if (oc->flags & OC_F_PASS) {
-		wrk->stats.cache_hitpass++;
-		VSLb(req->vsl, SLT_HitPass, "%u", req->obj->vxid);
-		(void)HSH_Deref(&wrk->stats, NULL, &req->obj);
-		AZ(req->objcore);
-		req->req_step = R_STP_PASS;
-		return (REQ_FSM_MORE);
-	}
-
-	wrk->stats.cache_hit++;
-	VSLb(req->vsl, SLT_Hit, "%u", req->obj->vxid);
-	req->req_step = R_STP_HIT;
-	return (REQ_FSM_MORE);
-}
-
-/*--------------------------------------------------------------------
- * We had a miss, ask VCL, proceed as instructed
- *
-DOT subgraph xcluster_miss {
-DOT	miss [
-DOT		shape=record
-DOT		label="{cnt_miss:|filter req.-\>bereq.|{vcl_miss\{\}|{req.*|bereq.*}}|{<err>error?|<rst>restart?}|{<pass>pass?|<fetch>fetch?}}"
-DOT	]
-DOT }
-DOT miss:fetch -> fetch [label="fetch",style=bold,color=blue]
-DOT miss:pass -> pass [label="pass",style=bold,color=red]
-DOT
- */
-
-static enum req_fsm_nxt
-cnt_miss(struct worker *wrk, struct req *req)
-{
-	struct busyobj *bo;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
-	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-	bo = req->busyobj;
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	AZ(req->obj);
-
-	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
-	http_FilterReq(req, HTTPH_R_FETCH);
-	http_ForceGet(bo->bereq);
-	if (cache_param->http_gzip_support) {
-		/*
-		 * We always ask the backend for gzip, even if the
-		 * client doesn't grok it.  We will uncompress for
-		 * the minority of clients which don't.
-		 */
-		http_Unset(bo->bereq, H_Accept_Encoding);
-		http_SetHeader(bo->bereq, "Accept-Encoding: gzip");
-	}
-
-	VCL_miss_method(req);
-
-	if (req->handling == VCL_RET_FETCH) {
-		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-		req->req_step = R_STP_FETCH;
-		return (REQ_FSM_MORE);
-	}
-
-	AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
-	req->objcore = NULL;
-	http_Teardown(bo->bereq);
-	VBO_DerefBusyObj(wrk, &req->busyobj);
-
-	switch(req->handling) {
-	case VCL_RET_ERROR:
-		req->req_step = R_STP_ERROR;
-		break;
-	case VCL_RET_PASS:
-		req->req_step = R_STP_PASS;
-		break;
-	case VCL_RET_RESTART:
-		req->req_step = R_STP_RESTART;
-		break;
-	default:
-		WRONG("Illegal action in vcl_miss{}");
-	}
-	return (REQ_FSM_MORE);
-}
-
-/*--------------------------------------------------------------------
- * Start pass processing by getting headers from backend, then
- * continue in passbody.
- *
-DOT subgraph xcluster_pass {
-DOT	pass [
-DOT		shape=record
-DOT		label="{cnt_pass:|(XXX: deref obj.)|filter req.*-\>bereq.|{vcl_pass\{\}|{req.*|bereq.*}}|{<err>error?|<rst>restart?}|<pass>create anon obj}"
-DOT	]
-DOT }
-DOT pass:pass -> fetch [style=bold, color=red]
-XDOT pass:rst -> rst_pass [label="restart",color=purple]
-XDOT rst_pass [label="RESTART",shape=plaintext]
-XDOT pass:err -> err_pass [label="error"]
-XDOT err_pass [label="ERROR",shape=plaintext]
- */
-
-static enum req_fsm_nxt
-cnt_pass(struct worker *wrk, struct req *req)
-{
-	struct busyobj *bo;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
-	AZ(req->objcore);
-	AZ(req->obj);
-	AZ(req->busyobj);
-
-	req->busyobj = VBO_GetBusyObj(wrk, req);
-	bo = req->busyobj;
-	bo->refcount = 2;
-	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
-	http_FilterReq(req, HTTPH_R_PASS);
-
-	VCL_pass_method(req);
-
-	if (req->handling == VCL_RET_ERROR) {
-		http_Teardown(bo->bereq);
-		VBO_DerefBusyObj(wrk, &req->busyobj);
-		req->req_step = R_STP_ERROR;
-		return (REQ_FSM_MORE);
-	}
-	assert(req->handling == VCL_RET_PASS);
-	req->acct_req.pass++;
-	req->req_step = R_STP_FETCH;
-
-	req->objcore = HSH_NewObjCore(wrk);
-	req->objcore->busyobj = bo;
-	return (REQ_FSM_MORE);
-}
-
-/*--------------------------------------------------------------------
- * Ship the request header to the backend unchanged, then pipe
- * until one of the ends close the connection.
- *
-DOT subgraph xcluster_pipe {
-DOT	pipe [
-DOT		shape=ellipse
-DOT		label="Filter req.->bereq."
-DOT	]
-DOT	vcl_pipe [
-DOT		shape=record
-DOT		label="vcl_pipe()|req.\nbereq\."
-DOT	]
-DOT	pipe_do [
-DOT		shape=ellipse
-DOT		label="send bereq.\npipe until close"
-DOT	]
-DOT	vcl_pipe -> pipe_do [label="pipe",style=bold,color=orange]
-DOT	pipe -> vcl_pipe [style=bold,color=orange]
-DOT }
-DOT pipe_do -> DONE [style=bold,color=orange]
-DOT vcl_pipe -> err_pipe [label="error"]
-DOT err_pipe [label="ERROR",shape=plaintext]
- */
-
-static enum req_fsm_nxt
-cnt_pipe(struct worker *wrk, struct req *req)
-{
-	struct busyobj *bo;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
-	AZ(req->busyobj);
-
-	req->acct_req.pipe++;
-	req->busyobj = VBO_GetBusyObj(wrk, req);
-	bo = req->busyobj;
-	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
-	http_FilterReq(req, 0);
-
-	VCL_pipe_method(req);
-
-	if (req->handling == VCL_RET_ERROR)
-		INCOMPL();
-	assert(req->handling == VCL_RET_PIPE);
-
-	PipeRequest(req);
-	assert(WRW_IsReleased(wrk));
-	http_Teardown(bo->bereq);
-	VBO_DerefBusyObj(wrk, &req->busyobj);
-	return (REQ_FSM_DONE);
-}
-
-/*--------------------------------------------------------------------
- *
-DOT subgraph xcluster_restart {
-DOT	restart [
-DOT		shape=record
-DOT		label="{cnt_restart}"
-DOT	]
-DOT }
-DOT RESTART -> restart [color=purple]
-DOT restart -> recv [color=purple]
-DOT restart -> err_restart
-DOT err_restart [label="ERROR",shape=plaintext]
- */
-
-static enum req_fsm_nxt
-cnt_restart(const struct worker *wrk, struct req *req)
-{
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-
-	req->director = NULL;
-	if (++req->restarts >= cache_param->max_restarts) {
-		req->err_code = 503;
-		req->req_step = R_STP_ERROR;
-	} else {
-		req->err_code = 0;
-		req->req_step = R_STP_RECV;
-	}
-	return (REQ_FSM_MORE);
-}
-
-/*--------------------------------------------------------------------
- * RECV
- * We have a complete request, set everything up and start it.
- * We can come here both with a request from the client and with
- * a interior request during ESI delivery.
- *
-DOT subgraph xcluster_recv {
-DOT	recv [
-DOT		shape=record
-DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}|{<pipe>pipe?|<pass>pass?|<error>error?|<lookup>lookup?}}"
-DOT	]
-DOT }
-DOT subgraph xcluster_hash {
-DOT	hash [
-DOT		shape=record
-DOT		label="{cnt_recv:|{vcl_hash\{\}|req.*}}"
-DOT	]
-DOT }
-DOT recv:pipe -> pipe [style=bold,color=orange]
-DOT recv:pass -> pass [style=bold,color=red]
-#DOT recv:error -> err_recv
-#DOT err_recv [label="ERROR",shape=plaintext]
-DOT recv:lookup -> hash [style=bold,color=green]
-DOT hash -> lookup [label="hash",style=bold,color=green]
- */
-
-static enum req_fsm_nxt
-cnt_recv(const struct worker *wrk, struct req *req)
-{
-	unsigned recv_handling;
-	struct SHA256Context sha256ctx;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
-	AZ(req->objcore);
-	AZ(req->obj);
-	AZ(req->objcore);
-	AZ(req->busyobj);
-
-	VSLb(req->vsl, SLT_ReqStart, "%s %s", req->sp->addr, req->sp->port);
-
-	if (req->err_code) {
-		req->req_step = R_STP_ERROR;
-		return (REQ_FSM_MORE);
-	}
-
-	/* By default we use the first backend */
-	AZ(req->director);
-	req->director = req->vcl->director[0];
-	AN(req->director);
-
-	EXP_Clr(&req->exp);
-
-	req->disable_esi = 0;
-	req->hash_always_miss = 0;
-	req->hash_ignore_busy = 0;
-	req->client_identity = NULL;
-
-	http_CollectHdr(req->http, H_Cache_Control);
-
-	VCL_recv_method(req);
-	recv_handling = req->handling;
-
-	if (cache_param->http_gzip_support &&
-	     (recv_handling != VCL_RET_PIPE) &&
-	     (recv_handling != VCL_RET_PASS)) {
-		if (RFC2616_Req_Gzip(req->http)) {
-			http_Unset(req->http, H_Accept_Encoding);
-			http_SetHeader(req->http, "Accept-Encoding: gzip");
-		} else {
-			http_Unset(req->http, H_Accept_Encoding);
-		}
-	}
-
-	req->sha256ctx = &sha256ctx;	/* so HSH_AddString() can find it */
-	SHA256_Init(req->sha256ctx);
-	VCL_hash_method(req);
-	assert(req->handling == VCL_RET_HASH);
-	SHA256_Final(req->digest, req->sha256ctx);
-	req->sha256ctx = NULL;
-
-	if (!strcmp(req->http->hd[HTTP_HDR_METHOD].b, "HEAD"))
-		req->wantbody = 0;
-	else
-		req->wantbody = 1;
-
-	switch(recv_handling) {
-	case VCL_RET_LOOKUP:
-		req->req_step = R_STP_LOOKUP;
-		return (REQ_FSM_MORE);
-	case VCL_RET_PIPE:
-		if (req->esi_level > 0) {
-			/* XXX: VSL something */
-			INCOMPL();
-			return (REQ_FSM_DONE);
-		}
-		req->req_step = R_STP_PIPE;
-		return (REQ_FSM_MORE);
-	case VCL_RET_PASS:
-		req->req_step = R_STP_PASS;
-		return (REQ_FSM_MORE);
-	case VCL_RET_ERROR:
-		req->req_step = R_STP_ERROR;
-		return (REQ_FSM_MORE);
-	default:
-		WRONG("Illegal action in vcl_recv{}");
-	}
-}
-
-/*--------------------------------------------------------------------
- * Central state engine dispatcher.
- *
- * Kick the session around until it has had enough.
- *
- */
-
-static void
-cnt_diag(struct req *req, const char *state)
-{
-
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-
-	VSLb(req->vsl,  SLT_Debug, "vxid %u STP_%s sp %p obj %p vcl %p",
-	    req->vsl->wid, state, req->sp, req->obj, req->vcl);
-	VSL_Flush(req->vsl, 0);
-}
-
-enum req_fsm_nxt
-CNT_Request(struct worker *wrk, struct req *req)
-{
-	enum req_fsm_nxt nxt;
-	struct storage *st;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-
-	/*
-	 * Possible entrance states
-	 */
-	assert(
-	    req->req_step == R_STP_LOOKUP ||
-	    req->req_step == R_STP_RECV);
-
-	AN(req->vsl->wid & VSL_CLIENTMARKER);
-
-	req->wrk = wrk;
-
-	for (nxt = REQ_FSM_MORE; nxt == REQ_FSM_MORE; ) {
-		/*
-		 * This is a good place to be paranoid about the various
-		 * pointers still pointing to the things we expect.
-		 */
-		CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-		CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC);
-		CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-
-		/*
-		 * We don't want the thread workspace to be used for
-		 * anything of long duration, so mandate that it be
-		 * empty on state-transitions.
-		 */
-		WS_Assert(wrk->aws);
-		assert(wrk->aws->s == wrk->aws->f);
-
-		switch (req->req_step) {
-#define REQ_STEP(l,u,arg) \
-		    case R_STP_##u: \
-			if (DO_DEBUG(DBG_REQ_STATE)) \
-				cnt_diag(req, #u); \
-			nxt = cnt_##l arg; \
-		        break;
-#include "tbl/steps.h"
-#undef REQ_STEP
-		default:
-			WRONG("State engine misfire");
-		}
-		WS_Assert(wrk->aws);
-		CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC);
-	}
-	if (nxt == REQ_FSM_DONE) {
-		/* XXX: Workaround for pipe */
-		if (req->sp->fd >= 0) {
-			VSLb(req->vsl, SLT_Length, "%ju",
-			    (uintmax_t)req->resp_bodybytes);
-		}
-		VSLb(req->vsl, SLT_ReqEnd, "%.9f %.9f %.9f %.9f %.9f",
-		    req->t_req,
-		    req->sp->t_idle,
-		    req->sp->t_idle - req->t_resp,
-		    req->t_resp - req->t_req,
-		    req->sp->t_idle - req->t_resp);
-
-		while (!VTAILQ_EMPTY(&req->body)) {
-			st = VTAILQ_FIRST(&req->body);
-			VTAILQ_REMOVE(&req->body, st, list);
-			STV_free(st);
-		}
-
-		/* done == 2 was charged by cache_hash.c */
-		SES_Charge(wrk, req);
-
-		/*
-		 * Nuke the VXID, cache_http1_fsm.c::http1_dissect() will
-		 * allocate a new one when necessary.
-		 */
-		req->vsl->wid = 0;
-	}
-
-	req->wrk = NULL;
-
-	assert(WRW_IsReleased(wrk));
-	return (nxt);
-}
-
-/*
-DOT }
-*/



From phk at varnish-cache.org  Wed Oct 30 10:19:09 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 30 Oct 2013 11:19:09 +0100
Subject: [master] e40679e Allow expect_close to be used by clients
Message-ID: <E1VbSrl-0002G8-92@project.varnish-software.com>

commit e40679e45e72b924ca7045157e294868afc5ca8f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 30 10:16:32 2013 +0000

    Allow expect_close to be used by clients

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 2349702..dcfe023 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1050,7 +1050,6 @@ cmd_http_expect_close(CMD_ARGS)
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	AZ(av[1]);
-	assert(hp->sfd != NULL);
 
 	vtc_log(vl, 4, "Expecting close (fd = %d)", hp->fd);
 	while (1) {



From phk at varnish-cache.org  Wed Oct 30 10:19:09 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 30 Oct 2013 11:19:09 +0100
Subject: [master] 6be44e1 We can do the last part of the test properly now.
Message-ID: <E1VbSrl-0002GP-AP@project.varnish-software.com>

commit 6be44e1ef4cc9fedbfec8738c159d8be71cfce9d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 30 10:18:42 2013 +0000

    We can do the last part of the test properly now.

diff --git a/bin/varnishtest/tests/c00039.vtc b/bin/varnishtest/tests/c00039.vtc
index 9339b59..780bec2 100644
--- a/bin/varnishtest/tests/c00039.vtc
+++ b/bin/varnishtest/tests/c00039.vtc
@@ -45,7 +45,13 @@ client c1 {
 		-hdr "1...5: ..0....5\n ..0...." 
 	rxresp
 	expect resp.status == 200
-	# XXX: Varnish test does not allow us to test for the fact
-	# XXX: that the backend summarily closes on us.  Adding one
-	# XXX: char to the above test, should cause that.
+
+	txreq -url "/1" \
+		-hdr "1...5: ..0....5\n ..0....5....0....5....0" \
+		-hdr "1...5: ..0....5\n ..0....5....0....5....0" \
+		-hdr "1...5: ..0....5\n ..0....5....0....5....0" \
+		-hdr "1...5: ..0....5\n ..0....5....0....5....0" \
+		-hdr "1...5: ..0....5\n ..0....5....0....5....0" \
+		-hdr "1...5: ..0....5\n ..0....5" 
+	expect_close
 } -run



From phk at varnish-cache.org  Wed Oct 30 10:26:26 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 30 Oct 2013 11:26:26 +0100
Subject: [master] 9c9a990 Make up our mind: Any req.* we receive from the
 client with fundamental trouble gets failed back without VCL involvement.
Message-ID: <E1VbSyo-0002wG-Sj@project.varnish-software.com>

commit 9c9a9904bdb56b62017f338baf9c8e906b88dcac
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 30 10:24:58 2013 +0000

    Make up our mind:  Any req.* we receive from the client with fundamental
    trouble gets failed back without VCL involvement.
    
    Fixes	#1367

diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index 6a92bc5..c1564be 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -270,7 +270,9 @@ static enum req_fsm_nxt
 http1_dissect(struct worker *wrk, struct req *req)
 {
 	const char *r_100 = "HTTP/1.1 100 Continue\r\n\r\n";
+	const char *r_400 = "HTTP/1.1 400 Bad Request\r\n\r\n";
 	const char *r_411 = "HTTP/1.1 411 Length Required\r\n\r\n";
+	const char *r_417 = "HTTP/1.1 417 Expectation Failed\r\n\r\n";
 	char *p;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -299,8 +301,9 @@ http1_dissect(struct worker *wrk, struct req *req)
 	req->err_code = HTTP1_DissectRequest(req);
 
 	/* If we could not even parse the request, just close */
-	if (req->err_code == 400) {
+	if (req->err_code != 0) {
 		wrk->stats.client_req_400++;
+		(void)write(req->sp->fd, r_400, strlen(r_400));
 		SES_Close(req->sp, SC_RX_JUNK);
 		return (REQ_FSM_DONE);
 	}
@@ -317,25 +320,27 @@ http1_dissect(struct worker *wrk, struct req *req)
 		return (REQ_FSM_DONE);
 	}
 
-	req->acct_req.req++;
-
-	req->ws_req = WS_Snapshot(req->ws);
-	req->doclose = http_DoConnection(req->http);
-
-	/* XXX: Expect headers are a mess */
-	if (req->err_code == 0 && http_GetHdr(req->http, H_Expect, &p)) {
+	if (http_GetHdr(req->http, H_Expect, &p)) {
 		if (strcasecmp(p, "100-continue")) {
 			wrk->stats.client_req_417++;
 			req->err_code = 417;
-		} else if (strlen(r_100) !=
-		    write(req->sp->fd, r_100, strlen(r_100))) {
+			(void)write(req->sp->fd, r_417, strlen(r_417));
+			SES_Close(req->sp, SC_RX_JUNK);
+			return (REQ_FSM_DONE);
+		}
+		if (strlen(r_100) != write(req->sp->fd, r_100, strlen(r_100))) {
+			// XXX: stats counter ?
 			SES_Close(req->sp, SC_REM_CLOSE);
 			return (REQ_FSM_DONE);
 		}
-	} else if (req->err_code == 413)
-		wrk->stats.client_req_413++;
-	else
-		wrk->stats.client_req++;
+	}
+
+	wrk->stats.client_req++;
+
+	AZ(req->err_code);
+	req->acct_req.req++;
+	req->ws_req = WS_Snapshot(req->ws);
+	req->doclose = http_DoConnection(req->http);
 
 	http_Unset(req->http, H_Expect);
 
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 6905ac2..583d2e1 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -251,9 +251,9 @@ htc_dissect_hdrs(struct http *hp, char *p, const struct http_conn *htc)
 		}
 
 		if (q - p > htc->maxhdr) {
-			VSLb(hp->vsl, SLT_BogoHeader, "%.*s",
+			VSLb(hp->vsl, SLT_BogoHeader, "Header too long: %.*s",
 			    (int)(q - p > 20 ? 20 : q - p), p);
-			return (413);
+			return (400);
 		}
 
 		/* Empty header = end of headers */
@@ -276,9 +276,9 @@ htc_dissect_hdrs(struct http *hp, char *p, const struct http_conn *htc)
 			http_VSLH(hp, hp->nhd);
 			hp->nhd++;
 		} else {
-			VSLb(hp->vsl, SLT_BogoHeader, "%.*s",
+			VSLb(hp->vsl, SLT_BogoHeader, "Too many headers: %.*s",
 			    (int)(q - p > 20 ? 20 : q - p), p);
-			return (413);
+			return (400);
 		}
 	}
 	return (0);
@@ -339,7 +339,7 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 	hp->hd[h2].e = p;
 
 	if (!Tlen(hp->hd[h2]))
-		return (413);
+		return (400);
 
 	/* Skip SP */
 	for (; vct_issp(*p); p++) {
diff --git a/bin/varnishtest/tests/c00039.vtc b/bin/varnishtest/tests/c00039.vtc
index 780bec2..a9d35d8 100644
--- a/bin/varnishtest/tests/c00039.vtc
+++ b/bin/varnishtest/tests/c00039.vtc
@@ -23,7 +23,7 @@ client c1 {
 	expect resp.status == 200
 	txreq -url "/1" -hdr "1...5....0....5....0....5....0....5....0."
 	rxresp
-	expect resp.status == 413
+	expect resp.status == 400
 } -run
 
 client c1 {
@@ -32,7 +32,7 @@ client c1 {
 	expect resp.status == 200
 	txreq -url "/2" -hdr "1...5....0....5\n ..0....5....0....5....0."
 	rxresp
-	expect resp.status == 413
+	expect resp.status == 400
 } -run
 
 client c1 {
diff --git a/bin/varnishtest/tests/r01367.vtc b/bin/varnishtest/tests/r01367.vtc
new file mode 100644
index 0000000..cb2b1f8
--- /dev/null
+++ b/bin/varnishtest/tests/r01367.vtc
@@ -0,0 +1,24 @@
+varnishtest "blank GET"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend { 
+	sub vcl_error {
+		return (restart);
+	}
+} -start
+
+client c1 {
+	send "GET    \nHost: example.com\n\n"
+	rxresp
+	expect resp.status == 400
+} -run
+
+client c1 {
+	txreq -hdr "Expect: Santa-Claus"
+	rxresp
+	expect resp.status == 417
+} -run



From martin at varnish-cache.org  Wed Oct 30 12:53:08 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Wed, 30 Oct 2013 13:53:08 +0100
Subject: [3.0] 4bd5b79 Make up our mind: Any req.* we receive from the client
 with fundamental trouble gets failed back without VCL involvement.
Message-ID: <E1VbVGm-0006sp-Lu@project.varnish-software.com>

commit 4bd5b7991bf602a6c46dd0d65fc04d4b8d9667a6
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 30 13:48:20 2013 +0100

    Make up our mind:  Any req.* we receive from the client with fundamental
    trouble gets failed back without VCL involvement.
    
    Fixes	#1367

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 19eb2ce..fdf7cee 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -1474,9 +1474,12 @@ DOT start -> recv [style=bold,color=green]
 static int
 cnt_start(struct sess *sp)
 {
-	uint16_t done;
+	uint16_t err_code;
 	char *p;
-	const char *r = "HTTP/1.1 100 Continue\r\n\r\n";
+	const char *r_100 = "HTTP/1.1 100 Continue\r\n\r\n";
+	const char *r_400 = "HTTP/1.1 400 Bad Request\r\n\r\n";
+	const char *r_413 = "HTTP/1.1 413 Request Entity Too Large\r\n\r\n";
+	const char *r_417 = "HTTP/1.1 417 Expectation Failed\r\n\r\n";
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->restarts);
@@ -1499,10 +1502,14 @@ cnt_start(struct sess *sp)
 	sp->wrk->vcl = NULL;
 
 	http_Setup(sp->http, sp->ws);
-	done = http_DissectRequest(sp);
+	err_code = http_DissectRequest(sp);
 
 	/* If we could not even parse the request, just close */
-	if (done == 400) {
+	if (err_code == 400)
+		(void)write(sp->fd, r_400, strlen(r_400));
+	else if (err_code == 413)
+		(void)write(sp->fd, r_413, strlen(r_413));
+	if (err_code != 0) {
 		sp->step = STP_DONE;
 		vca_close_session(sp, "junk");
 		return (0);
@@ -1514,12 +1521,6 @@ cnt_start(struct sess *sp)
 	/* Catch original request, before modification */
 	HTTP_Copy(sp->http0, sp->http);
 
-	if (done != 0) {
-		sp->err_code = done;
-		sp->step = STP_ERROR;
-		return (0);
-	}
-
 	sp->doclose = http_DoConnection(sp->http);
 
 	/* XXX: Handle TRACE & OPTIONS of Max-Forwards = 0 */
@@ -1529,13 +1530,14 @@ cnt_start(struct sess *sp)
 	 */
 	if (http_GetHdr(sp->http, H_Expect, &p)) {
 		if (strcasecmp(p, "100-continue")) {
-			sp->err_code = 417;
-			sp->step = STP_ERROR;
+			(void)write(sp->fd, r_417, strlen(r_417));
+			sp->step = STP_DONE;
+			vca_close_session(sp, "junk");
 			return (0);
 		}
 
 		/* XXX: Don't bother with write failures for now */
-		(void)write(sp->fd, r, strlen(r));
+		(void)write(sp->fd, r_100, strlen(r_100));
 		/* XXX: When we do ESI includes, this is not removed
 		 * XXX: because we use http0 as our basis.  Believed
 		 * XXX: safe, but potentially confusing.
diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index 8753acc..605975b 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -601,7 +601,7 @@ http_splitline(struct worker *w, int fd, struct http *hp,
 	hp->hd[h2].e = p;
 
 	if (!Tlen(hp->hd[h2]))
-		return (413);
+		return (400);
 
 	/* Skip SP */
 	for (; vct_issp(*p); p++) {
diff --git a/bin/varnishtest/tests/r01367.vtc b/bin/varnishtest/tests/r01367.vtc
new file mode 100644
index 0000000..e1de20a
--- /dev/null
+++ b/bin/varnishtest/tests/r01367.vtc
@@ -0,0 +1,30 @@
+varnishtest "blank GET"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend { 
+	sub vcl_error {
+		return (restart);
+	}
+} -start
+
+client c1 {
+	send "GET    \nHost: example.com\n\n"
+	rxresp
+	expect resp.status == 400
+} -run
+
+client c1 {
+	txreq -hdr "Expect: Santa-Claus"
+	rxresp
+	expect resp.status == 417
+} -run
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run



From phk at varnish-cache.org  Thu Oct 31 09:19:23 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 31 Oct 2013 10:19:23 +0100
Subject: [master] 319497d Inspired by yesterdays code reading,
 polish up the HTTP protocol parser a little bit,
 and nail a couple of corner cases.
Message-ID: <E1VboPT-0002KM-1X@project.varnish-software.com>

commit 319497d64c9c5e75bd8c1c3bef4c2ebffe8b02bd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 31 09:18:49 2013 +0000

    Inspired by yesterdays code reading, polish up the HTTP protocol
    parser a little bit, and nail a couple of corner cases.

diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 583d2e1..db0a1c5 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -37,6 +37,9 @@
  * and stops when we see the magic marker (double [CR]NL), and if we overshoot,
  * it keeps track of the "pipelined" data.
  *
+ * Until we see the magic marker, we have to keep the rxbuf NUL terminated
+ * because we use strchr(3) on it.
+ *
  * We use this both for client and backend connections.
  */
 
@@ -97,17 +100,17 @@ HTTP1_Reinit(struct http_conn *htc)
 
 /*--------------------------------------------------------------------
  * Check if we have a complete HTTP request or response yet
- *
  */
 
 enum htc_status_e
 HTTP1_Complete(struct http_conn *htc)
 {
-	int i;
-	const char *p;
+	char *p;
 	txt *t;
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	AZ(htc->pipeline.b);
+	AZ(htc->pipeline.e);
 
 	t = &htc->rxbuf;
 	Tcheck(*t);
@@ -133,14 +136,11 @@ HTTP1_Complete(struct http_conn *htc)
 			break;
 	}
 	p++;
-	i = p - t->b;
-	WS_ReleaseP(htc->ws, htc->rxbuf.e);
-	AZ(htc->pipeline.b);
-	AZ(htc->pipeline.e);
-	if (htc->rxbuf.b + i < htc->rxbuf.e) {
-		htc->pipeline.b = htc->rxbuf.b + i;
-		htc->pipeline.e = htc->rxbuf.e;
-		htc->rxbuf.e = htc->pipeline.b;
+	WS_ReleaseP(htc->ws, t->e);
+	if (p < t->e) {
+		htc->pipeline.b = p;
+		htc->pipeline.e = t->e;
+		t->e = p;
 	}
 	return (HTTP1_COMPLETE);
 }
@@ -156,6 +156,8 @@ HTTP1_Rx(struct http_conn *htc)
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	AN(htc->ws->r);
+	AZ(htc->pipeline.b);
+	AZ(htc->pipeline.e);
 	i = (htc->ws->r - htc->rxbuf.e) - 1;	/* space for NUL */
 	if (i <= 0) {
 		WS_ReleaseP(htc->ws, htc->rxbuf.b);
@@ -221,9 +223,6 @@ htc_dissect_hdrs(struct http *hp, char *p, const struct http_conn *htc)
 	char *q, *r;
 	txt t = htc->rxbuf;
 
-	if (*p == '\r')
-		p++;
-
 	hp->nhd = HTTP_HDR_FIRST;
 	hp->conds = 0;
 	r = NULL;		/* For FlexeLint */
@@ -232,7 +231,7 @@ htc_dissect_hdrs(struct http *hp, char *p, const struct http_conn *htc)
 		/* Find end of next header */
 		q = r = p;
 		while (r < t.e) {
-			if (!vct_iscrlf(*r)) {
+			if (!vct_iscrlf(r)) {
 				r++;
 				continue;
 			}
@@ -260,6 +259,13 @@ htc_dissect_hdrs(struct http *hp, char *p, const struct http_conn *htc)
 		if (p == q)
 			break;
 
+		if (vct_islws(*p)) {
+			VSLb(hp->vsl, SLT_BogoHeader,
+			    "1st header has white space: %.*s",
+			    (int)(q - p > 20 ? 20 : q - p), p);
+			return (400);
+		}
+
 		if ((p[0] == 'i' || p[0] == 'I') &&
 		    (p[1] == 'f' || p[1] == 'F') &&
 		    p[2] == '-')
@@ -291,9 +297,13 @@ htc_dissect_hdrs(struct http *hp, char *p, const struct http_conn *htc)
 static uint16_t
 htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 {
-	char *p, *q;
+	char *p;
 	int h1, h2, h3;
 
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+	Tcheck(htc->rxbuf);
+
 	if (req) {
 		h1 = HTTP_HDR_METHOD;
 		h2 = HTTP_HDR_URL;
@@ -304,38 +314,31 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 		h3 = HTTP_HDR_RESPONSE;
 	}
 
-	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
-
-	/* XXX: Assert a NUL at rx.e ? */
-	Tcheck(htc->rxbuf);
-
 	/* Skip leading LWS */
 	for (p = htc->rxbuf.b ; vct_islws(*p); p++)
 		continue;
+	hp->hd[h1].b = p;
 
-	/* First field cannot contain SP, CRLF or CTL */
-	q = p;
+	/* First field cannot contain SP or CTL */
 	for (; !vct_issp(*p); p++) {
 		if (vct_isctl(*p))
 			return (400);
 	}
-	hp->hd[h1].b = q;
 	hp->hd[h1].e = p;
+	assert(Tlen(hp->hd[h1]));
 
 	/* Skip SP */
 	for (; vct_issp(*p); p++) {
 		if (vct_isctl(*p))
 			return (400);
 	}
+	hp->hd[h2].b = p;
 
 	/* Second field cannot contain LWS or CTL */
-	q = p;
 	for (; !vct_islws(*p); p++) {
 		if (vct_isctl(*p))
 			return (400);
 	}
-	hp->hd[h2].b = q;
 	hp->hd[h2].e = p;
 
 	if (!Tlen(hp->hd[h2]))
@@ -346,15 +349,15 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 		if (vct_isctl(*p))
 			return (400);
 	}
+	hp->hd[h3].b = p;
 
-	/* Third field is optional and cannot contain CTL */
-	q = p;
-	if (!vct_iscrlf(*p)) {
-		for (; !vct_iscrlf(*p); p++)
-			if (!vct_issep(*p) && vct_isctl(*p))
-				return (400);
+	/* Third field is optional and cannot contain CTL except TAB */
+	for (; !vct_iscrlf(p); p++) {
+		if (vct_isctl(*p) && !vct_issp(*p)) {
+			hp->hd[h3].b = NULL;
+			return (400);
+		}
 	}
-	hp->hd[h3].b = q;
 	hp->hd[h3].e = p;
 
 	/* Skip CRLF */
diff --git a/bin/varnishtest/tests/b00040.vtc b/bin/varnishtest/tests/b00040.vtc
new file mode 100644
index 0000000..7ce7615
--- /dev/null
+++ b/bin/varnishtest/tests/b00040.vtc
@@ -0,0 +1,43 @@
+varnishtest "test certain mailformed reqests"
+
+server s1 {
+	rxreq
+	# expect req.url == /3
+	txresp
+} -start
+
+varnish v1 -vcl+backend { } -start
+
+client c1 {
+	send "GET /1 HTTP/1.1\r\n"
+	send " Host: foo\r\n"
+	send "\r\n"
+	rxresp
+	expect resp.status == 400
+} -run
+delay .1
+
+client c1 {
+	send "GET /2 HTTP/1.1\r\n"
+	send " Host: foo\r\n"
+	send "\r\n"
+	rxresp
+	expect resp.status == 400
+} -run
+delay .1
+
+client c1 {
+	send "GET /3 HTTP/1.1\r\n"
+	send "\rHost: foo\r\n"
+	send "\r\n"
+	rxresp
+	expect resp.status == 400
+} -run
+delay .1
+
+client c1 {
+	send "GET /4 HTTP/1.1\r\n"
+	send "\r\n"
+	rxresp
+	expect resp.status == 200
+} -run
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index dcfe023..5686ff0 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -294,17 +294,17 @@ http_splitheader(struct http *hp, int req)
 	hh[n++] = p;
 	while (!vct_islws(*p))
 		p++;
-	assert(!vct_iscrlf(*p));
+	assert(!vct_iscrlf(p));
 	*p++ = '\0';
 
 	/* URL/STATUS */
 	while (vct_issp(*p))		/* XXX: H space only */
 		p++;
-	assert(!vct_iscrlf(*p));
+	assert(!vct_iscrlf(p));
 	hh[n++] = p;
 	while (!vct_islws(*p))
 		p++;
-	if (vct_iscrlf(*p)) {
+	if (vct_iscrlf(p)) {
 		hh[n++] = NULL;
 		q = p;
 		p += vct_skipcrlf(p);
@@ -315,7 +315,7 @@ http_splitheader(struct http *hp, int req)
 		while (vct_issp(*p))		/* XXX: H space only */
 			p++;
 		hh[n++] = p;
-		while (!vct_iscrlf(*p))
+		while (!vct_iscrlf(p))
 			p++;
 		q = p;
 		p += vct_skipcrlf(p);
@@ -325,10 +325,10 @@ http_splitheader(struct http *hp, int req)
 
 	while (*p != '\0') {
 		assert(n < MAX_HDR);
-		if (vct_iscrlf(*p))
+		if (vct_iscrlf(p))
 			break;
 		hh[n++] = p++;
-		while (*p != '\0' && !vct_iscrlf(*p))
+		while (*p != '\0' && !vct_iscrlf(p))
 			p++;
 		q = p;
 		p += vct_skipcrlf(p);
@@ -419,11 +419,11 @@ http_rxchunk(struct http *hp)
 	}
 	l = hp->prxbuf;
 	(void)http_rxchar(hp, 2, 0);
-	if(!vct_iscrlf(hp->rxbuf[l]))
+	if(!vct_iscrlf(hp->rxbuf + l))
 		vtc_log(hp->vl, hp->fatal,
 		    "Wrong chunk tail[0] = %02x",
 		    hp->rxbuf[l] & 0xff);
-	if(!vct_iscrlf(hp->rxbuf[l + 1]))
+	if(!vct_iscrlf(hp->rxbuf + l + 1))
 		vtc_log(hp->vl, hp->fatal,
 		    "Wrong chunk tail[1] = %02x",
 		    hp->rxbuf[l + 1] & 0xff);
diff --git a/include/vct.h b/include/vct.h
index 4f9e2d4..3c41dd7 100644
--- a/include/vct.h
+++ b/include/vct.h
@@ -53,7 +53,6 @@ vct_is(int x, uint16_t y)
 
 #define vct_issp(x) vct_is(x, VCT_SP)
 #define vct_ishex(x) vct_is(x, VCT_HEX)
-#define vct_iscrlf(x) vct_is(x, VCT_CRLF)
 #define vct_islws(x) vct_is(x, VCT_LWS)
 #define vct_isctl(x) vct_is(x, VCT_CTL)
 #define vct_isdigit(x) vct_is(x, VCT_DIGIT)
@@ -63,5 +62,7 @@ vct_is(int x, uint16_t y)
 #define vct_isxmlnamestart(x) vct_is(x, VCT_XMLNAMESTART)
 #define vct_isxmlname(x) vct_is(x, VCT_XMLNAMESTART | VCT_XMLNAME)
 
+#define vct_iscrlf(p) (((p)[0] == '\r' && (p)[1] == '\n') || (p)[0] == '\n')
+
 /* NB: VCT always operate in ASCII, don't replace 0x0d with \r etc. */
-#define vct_skipcrlf(p) (p[0] == 0x0d && p[1] == 0x0a ? 2 : 1)
+#define vct_skipcrlf(p) ((p)[0] == 0x0d && (p)[1] == 0x0a ? 2 : 1)



From phk at varnish-cache.org  Thu Oct 31 10:16:29 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 31 Oct 2013 11:16:29 +0100
Subject: [master] 5655325 Use VTCP_Check() when expecting close.
Message-ID: <E1VbpIj-0006Wm-4m@project.varnish-software.com>

commit 565532550aa4888690d1d12518db7dbe6735b691
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 31 10:16:15 2013 +0000

    Use VTCP_Check() when expecting close.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 5686ff0..63b02e6 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1064,7 +1064,7 @@ cmd_http_expect_close(CMD_ARGS)
 			    "Expected close: poll = %d, revents = 0x%x",
 			    i, fds[0].revents);
 		i = read(hp->fd, &c, 1);
-		if (i == 0)
+		if (VTCP_Check(i))
 			break;
 		if (i == 1 && vct_islws(c))
 			continue;



From phk at varnish-cache.org  Thu Oct 31 10:50:31 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 31 Oct 2013 11:50:31 +0100
Subject: [master] 703fc52 Mark this write-failure as non-fatal in the server.
Message-ID: <E1Vbppg-0000Z8-01@project.varnish-software.com>

commit 703fc52bd80d5e889fe02439512d3f0ce5dd62e7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 31 10:50:13 2013 +0000

    Mark this write-failure as non-fatal in the server.

diff --git a/bin/varnishtest/tests/r00387.vtc b/bin/varnishtest/tests/r00387.vtc
index 879657c..5edb18a 100644
--- a/bin/varnishtest/tests/r00387.vtc
+++ b/bin/varnishtest/tests/r00387.vtc
@@ -2,6 +2,7 @@ varnishtest "Regression test for #387: too long chunk header"
 
 server s1 {
 	rxreq
+	non-fatal
 	send "HTTP/1.1 200 Ok\r\n"
 	send "Transfer-encoding: chunked\r\n"
 	send "\r\n"



From martin at varnish-cache.org  Thu Oct 31 11:21:08 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 31 Oct 2013 12:21:08 +0100
Subject: [master] 81ac141 Document more VSL tags
Message-ID: <E1VbqJI-0003D0-FV@project.varnish-software.com>

commit 81ac141c52d8a0615ecffa791cfee352a0453c2d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 23 15:01:09 2013 +0200

    Document more VSL tags

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 67548d5..18ada44 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -200,14 +200,35 @@ SLTM(Fetch_Body, "Body fetched from backend",
 	"\t+------------ Body status\n"
 	"\n"
 )
-SLTM(VCL_acl, "", "")
+SLTM(VCL_acl, "VSL ACL check results",
+	"Logs VCL ACL evaluation results.\n\n"
+)
 SLTM(VCL_call, "VCL method called", "")
 SLTM(VCL_trace, "VCL trace data", "")
-SLTM(VCL_return, "VCL method return value", "")
-SLTM(ReqStart, "Client request start", "")
-SLTM(Hit, "Hit object in cache", "")
-SLTM(HitPass, "Hit for pass object in cache", "")
-SLTM(ExpBan, "Object evicted due to ban", "")
+SLTM(VCL_return, "VCL method return value",
+	"Logs the VCL method terminating statement.\n\n"
+)
+SLTM(ReqStart, "Client request start",
+	"Start of request processing. Logs the client IP address and port"
+	" number.\n\n"
+	"The format is::\n\n"
+	"\t%s %s\n"
+	"\t|  |\n"
+	"\t|  +- Port number\n"
+	"\t+---- IP address\n"
+	"\n"
+)
+
+SLTM(Hit, "Hit object in cache",
+	"Object looked up in cache. Shows the VXID of the object.\n\n"
+)
+
+SLTM(HitPass, "Hit for pass object in cache (unused)", "")
+
+SLTM(ExpBan, "Object evicted due to ban",
+	"Logs the VXID when an object is banned.\n\n"
+)
+
 SLTM(ExpKill, "Object expired", "")
 SLTM(WorkThread, "", "")
 



From martin at varnish-cache.org  Thu Oct 31 11:21:08 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 31 Oct 2013 12:21:08 +0100
Subject: [master] 781e823 Document SLT_ExpKill
Message-ID: <E1VbqJI-0003D6-Jr@project.varnish-software.com>

commit 781e8233c8b82871b71b701727be91f4262aa231
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 23 15:01:24 2013 +0200

    Document SLT_ExpKill
    
    Use an attribute like syntax for these log records to make the
    documentation more readable.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 193ddb0..40f9c33 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -236,7 +236,7 @@ EXP_Rearm(struct object *o, double now, double ttl, double grace, double keep)
 
 	when = exp_when(o);
 
-	VSL(SLT_ExpKill, 0, "EXP_Rearm %p %.9f %.9f 0x%x", oc,
+	VSL(SLT_ExpKill, 0, "EXP_Rearm p=%p E=%.9f e=%.9f f=0x%x", oc,
 	    oc->timer_when, when, oc->flags);
 
 	if (oc->timer_when == when)
@@ -283,7 +283,7 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 	VTAILQ_FOREACH(oc, &lru->lru_head, lru_list) {
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
-		VSLb(bo->vsl, SLT_ExpKill, "LRU_Cand %p f=0x%x r=%d",
+		VSLb(bo->vsl, SLT_ExpKill, "LRU_Cand p=%p f=0x%x r=%d",
 		    oc, oc->flags, oc->refcnt);
 
 		AZ(oc->flags & OC_F_DYING);
@@ -312,7 +312,7 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 	Lck_Unlock(&lru->mtx);
 
 	if (oc == NULL) {
-		VSLb(bo->vsl, SLT_ExpKill, "LRU failed");
+		VSLb(bo->vsl, SLT_ExpKill, "LRU_Fail");
 		return (-1);
 	}
 
@@ -323,7 +323,7 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 
 	exp_mail_it(oc);
 
-	VSLb(bo->vsl, SLT_ExpKill, "LRU %u",
+	VSLb(bo->vsl, SLT_ExpKill, "LRU x=%u",
 	    oc_getxid(bo->stats, oc) & VSL_IDENTMASK);
 	(void)HSH_DerefObjCore(bo->stats, &oc);
 	return (1);
@@ -379,7 +379,8 @@ EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru)
 		for (i = 0; i < n; i++) {
 			oc = oc_array[i];
 			o = oc_getobj(&wrk->stats, oc);
-			VSLb(vsl, SLT_ExpKill, "%u %.0f LRU",
+			/* XXX: Not documented in vsl_tags.h */
+			VSLb(vsl, SLT_ExpKill, "x=%u t=%.0f LRU",
 			    oc_getxid(&wrk->stats, oc) & VSL_IDENTMASK,
 			    EXP_Ttl(NULL, o) - t);
 			o->exp.ttl = 0.0;
@@ -410,7 +411,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 	CHECK_OBJ_NOTNULL(ep, EXP_PRIV_MAGIC);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
-	VSLb(&ep->vsl, SLT_ExpKill, "EXP_INBOX %p %.9f 0x%x", oc,
+	VSLb(&ep->vsl, SLT_ExpKill, "EXP_Inbox p=%p e=%.9f f=0x%x", oc,
 	    oc->timer_when, oc->flags);
 
 	AZ(oc->flags & OC_F_BUSY);
@@ -429,7 +430,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 	Lck_Unlock(&lru->mtx);
 
 	if (flags & OC_F_DYING) {
-		VSLb(&ep->vsl, SLT_ExpKill, "EXP_KILL %p %.9f 0x%x", oc,
+		VSLb(&ep->vsl, SLT_ExpKill, "EXP_Kill p=%p e=%.9f f=0x%x", oc,
 		    oc->timer_when, oc->flags);
 		if (!(flags & OC_F_INSERT)) {
 			assert(oc->timer_idx != BINHEAP_NOIDX);
@@ -447,7 +448,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 		oc_updatemeta(oc);
 	}
 
-	VSLb(&ep->vsl, SLT_ExpKill, "EXP_WHEN %p %.9f 0x%x", oc,
+	VSLb(&ep->vsl, SLT_ExpKill, "EXP_When p=%p e=%.9f f=0x%x", oc,
 	    oc->timer_when, flags);
 
 	/*
@@ -518,7 +519,7 @@ exp_expire(struct exp_priv *ep, double now)
 	CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
 	o = oc_getobj(&ep->wrk->stats, oc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	VSLb(&ep->vsl, SLT_ExpKill, "%u %.0f",
+	VSLb(&ep->vsl, SLT_ExpKill, "EXP_Expired x=%u t=%.0f",
 	    oc_getxid(&ep->wrk->stats, oc) & VSL_IDENTMASK,
 	    EXP_Ttl(NULL, o) - now);
 	(void)HSH_DerefObjCore(&ep->wrk->stats, &oc);
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 18ada44..72c3264 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -229,7 +229,47 @@ SLTM(ExpBan, "Object evicted due to ban",
 	"Logs the VXID when an object is banned.\n\n"
 )
 
-SLTM(ExpKill, "Object expired", "")
+SLTM(ExpKill, "Object expiry event",
+	"Logs events related to object expiry. The events are:\n\n"
+	"EXP_Rearm\n"
+	"\tLogged when the expiry time of an object changes.\n\n"
+	"EXP_Inbox\n"
+	"\tLogged when the expiry thread picks an object from the inbox for"
+	" processing.\n\n"
+	"EXP_Kill\n"
+	"\tLogged when the expiry thread kills an object from the inbox.\n\n"
+	"EXP_When\n"
+	"\tLogged when the expiry thread moves an object on the binheap.\n\n"
+	"EXP_Expired\n"
+	"\tLogged when the expiry thread expires an object.\n\n"
+	"LRU_Cand\n"
+	"\tLogged when an object is evaluated for LRU force expiry.\n\n"
+	"LRU\n"
+	"\tLogged when an object is force expired due to LRU.\n\n"
+	"LRU_Fail\n"
+	"\tLogged when no suitable candidate object is found for LRU force"
+	" expiry.\n\n"
+	"The format is::\n\n"
+	"\tEXP_Rearm p=%p E=%f e=%f f=0x%x\n"
+	"\tEXP_Inbox p=%p e=%f f=0x%x\n"
+	"\tEXP_Kill p=%p e=%f f=0x%x\n"
+	"\tEXP_When p=%p e=%f f=0x%x\n"
+	"\tEXP_Expired x=%u t=%f\n"
+	"\tLRU_Cand p=%p f=0x%x r=%d\n"
+	"\tLRU x=%u\n"
+	"\tLRU_Fail\n"
+	"\t\n"
+	"\tLegend:\n"
+	"\tp=%p         Objcore pointer\n"
+	"\tt=%f         Remaining TTL (s)\n"
+	"\te=%f         Expiry time (unix epoch)\n"
+	"\tE=%f         Old expiry time (unix epoch)\n"
+	"\tf=0x%x       Objcore flags\n"
+	"\tr=%d         Objcore refcount\n"
+	"\tx=%u         Object VXID\n"
+	"\n"
+)
+
 SLTM(WorkThread, "", "")
 
 SLTM(ESI_xmlerror, "ESI parser error or warning message",
@@ -255,6 +295,7 @@ SLTM(Backend_health, "Backend health check",
 	"\t|  |  +------------------- Probe window bits\n"
 	"\t|  +---------------------- Status message\n"
 	"\t+------------------------- Backend name\n"
+	"\n"
 )
 
 SLTM(VCL_Debug, "(unused)", "")



From martin at varnish-cache.org  Thu Oct 31 11:21:08 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 31 Oct 2013 12:21:08 +0100
Subject: [master] 4558240 Document SLT_WorkThread log records
Message-ID: <E1VbqJI-0003DF-Mi@project.varnish-software.com>

commit 4558240ff8eeb308defcf7da061a32ac976f3ee8
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 24 14:54:57 2013 +0200

    Document SLT_WorkThread log records

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 72c3264..af3d52f 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -270,7 +270,14 @@ SLTM(ExpKill, "Object expiry event",
 	"\n"
 )
 
-SLTM(WorkThread, "", "")
+SLTM(WorkThread, "Logs thread start/stop events",
+	"Logs worker thread creation and termination events.\n\n"
+	"The format is::\n\n"
+	"\t%p %s\n"
+	"\t|  |\n"
+	"\t|  +- [start|end]\n"
+	"\t+---- Worker struct pointer"
+)
 
 SLTM(ESI_xmlerror, "ESI parser error or warning message",
 	"An error or warning was generated during parsing of an ESI object."



From martin at varnish-cache.org  Thu Oct 31 11:21:08 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 31 Oct 2013 12:21:08 +0100
Subject: [master] f069ab7 Add masked by default notes to appropriate log
 records.
Message-ID: <E1VbqJI-0003DN-Qf@project.varnish-software.com>

commit f069ab7f941967a7b3f97948bc2862564f90839e
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 24 14:56:14 2013 +0200

    Add masked by default notes to appropriate log records.

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index af3d52f..eb4d407 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -41,6 +41,8 @@
  *	Long Description (in RST "definition list" format)
  */
 
+#define NODEF_NOTICE "Note: This log record is masked by default\n\n"
+
 SLTM(Debug, "Debug messages",
 	"Debug messages can normally be ignored, but are sometimes"
 	" helpful during trouble-shooting.  Most debug messages must"
@@ -277,6 +279,8 @@ SLTM(WorkThread, "Logs thread start/stop events",
 	"\t|  |\n"
 	"\t|  +- [start|end]\n"
 	"\t+---- Worker struct pointer"
+	"\n"
+	NODEF_NOTICE
 )
 
 SLTM(ESI_xmlerror, "ESI parser error or warning message",
@@ -285,7 +289,8 @@ SLTM(ESI_xmlerror, "ESI parser error or warning message",
 )
 
 SLTM(Hash, "Value added to hash",
-	"This value was added to the object lookup hash."
+	"This value was added to the object lookup hash.\n\n"
+	NODEF_NOTICE
 )
 
 SLTM(Backend_health, "Backend health check",
@@ -362,3 +367,5 @@ SLTM(VSL, "VSL API warnings and error message",
 	"Warnings and error messages genererated by the VSL API while"
 	" reading the shared memory log.\n\n"
 )
+
+#undef NODEF_NOTICE



From martin at varnish-cache.org  Thu Oct 31 11:21:08 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 31 Oct 2013 12:21:08 +0100
Subject: [master] 51ca237 Document a couple of VSL log records
Message-ID: <E1VbqJI-0003DV-Uw@project.varnish-software.com>

commit 51ca2372261a1990896993dd484244aa6d44427a
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 24 15:39:18 2013 +0200

    Document a couple of VSL log records

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index eb4d407..884a884 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -205,8 +205,20 @@ SLTM(Fetch_Body, "Body fetched from backend",
 SLTM(VCL_acl, "VSL ACL check results",
 	"Logs VCL ACL evaluation results.\n\n"
 )
-SLTM(VCL_call, "VCL method called", "")
-SLTM(VCL_trace, "VCL trace data", "")
+SLTM(VCL_call, "VCL method called",
+	"Logs the VCL method name when a VCL method is called.\n\n"
+)
+SLTM(VCL_trace, "VCL trace data",
+	"Logs VCL execution trace data.\n\n"
+	"The format is::\n\n"
+	"\t%u %u.%u\n"
+	"\t|  |  |\n"
+	"\t|  |  +- VCL program line position\n"
+	"\t|  +---- VCL program line number\n"
+	"\t+------- VCL trace point index\n"
+	"\n"
+	NODEF_NOTICE
+)
 SLTM(VCL_return, "VCL method return value",
 	"Logs the VCL method terminating statement.\n\n"
 )
@@ -314,7 +326,9 @@ SLTM(VCL_Debug, "(unused)", "")
 SLTM(VCL_Log, "Log statement from VCL",
 	"User generated log messages insert from VCL through std.log()"
 )
-SLTM(VCL_Error, "", "")
+SLTM(VCL_Error, "VCL execution error message",
+	"Logs error messages generated during VCL execution.\n\n"
+)
 
 SLTM(Gzip, "G(un)zip performed on object",
 	"A Gzip record is emitted for each instance of gzip or gunzip"



From martin at varnish-cache.org  Thu Oct 31 11:21:09 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 31 Oct 2013 12:21:09 +0100
Subject: [master] d43eebc Document the backend related VSL records.
Message-ID: <E1VbqJJ-0003Dd-1q@project.varnish-software.com>

commit d43eebc3de4df1fe04a1056521bd352ad3c13aff
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 24 15:58:13 2013 +0200

    Document the backend related VSL records.
    
    Change what is logged in a couple of places for consistency.

diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c
index 2e10dcc..b3f17cf 100644
--- a/bin/varnishd/cache/cache_dir.c
+++ b/bin/varnishd/cache/cache_dir.c
@@ -54,7 +54,7 @@ VDI_CloseFd(struct vbc **vbp)
 
 	bp = vc->backend;
 
-	VSLb(vc->vsl, SLT_BackendClose, "%s", bp->display_name);
+	VSLb(vc->vsl, SLT_BackendClose, "%d %s", vc->fd, bp->display_name);
 
 	/*
 	 * Checkpoint log to flush all info related to this connection
@@ -86,7 +86,7 @@ VDI_RecycleFd(struct vbc **vbp)
 
 	bp = vc->backend;
 
-	VSLb(vc->vsl, SLT_BackendReuse, "%s", bp->display_name);
+	VSLb(vc->vsl, SLT_BackendReuse, "%d %s", vc->fd, bp->display_name);
 
 	/* XXX: revisit this hack */
 	VSL_Flush(vc->vsl, 0);
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 884a884..01ced35 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -113,12 +113,56 @@ SLTM(SessClose, "Client connection closed",
 
 /*---------------------------------------------------------------------*/
 
-SLTM(BackendOpen, "Backend connection opened", "")
-SLTM(BackendXID, "The unique ID of the backend transaction", "")
-SLTM(BackendReuse, "Backend connection reused", "")
-SLTM(BackendClose, "Backend connection closed", "")
+SLTM(BackendOpen, "Backend connection opened",
+	"Logged when a new backend connection is opened.\n\n"
+	"The format is::\n\n"
+	"\t%d %s %s %s\n"
+	"\t|  |  |  |\n"
+	"\t|  |  |  +- Remote port\n"
+	"\t|  |  +---- Remote address\n"
+	"\t|  +------- Backend display name\n"
+	"\t+---------- Connection file descriptor\n"
+	"\n"
+)
+
+SLTM(BackendXID, "The unique ID of the backend transaction (unused)", "")
+
+SLTM(BackendReuse, "Backend connection put up for reuse",
+	"Logged when a backend connection is put up for reuse by a later"
+	" connection.\n\n"
+	"The format is::\n\n"
+	"%d %s\n"
+	"|  |\n"
+	"|  +- Backend display name\n"
+	"+---- Connection file descriptor\n"
+	"\n"
+)
+
+SLTM(BackendClose, "Backend connection closed",
+	"Logged when a backend connection is closed.\n\n"
+	"The format is::\n\n"
+	"%d %s [ %s ]\n"
+	"|  |    |\n"
+	"|  |    +- Optional reason\n"
+	"|  +------ Backend display name\n"
+	"+--------- Connection file descriptor\n"
+	"\n"
+)
+
 SLTM(HttpGarbage, "", "")
-SLTM(Backend, "Backend selected", "")
+
+SLTM(Backend, "Backend selected",
+	"Logged when a connection is selected for handling a backend"
+	" request.\n\n"
+	"The format is::\n\n"
+	"\t%d %s %s\n"
+	"\t|  |  |\n"
+	"\t|  |  +- Backend display name\n"
+	"\t|  +---- VCL name\n"
+	"\t+------- Connection file descriptor\n"
+	"\n"
+)
+
 SLTM(Length, "Size of object body", "")
 
 SLTM(BereqEnd, "Backend request end",



From martin at varnish-cache.org  Thu Oct 31 11:21:09 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 31 Oct 2013 12:21:09 +0100
Subject: [master] 77e525b Document the last of the VSL records
Message-ID: <E1VbqJJ-0003Do-6W@project.varnish-software.com>

commit 77e525b355793eb62bc48bbb8fb7c868c4c4d7b3
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 24 16:18:00 2013 +0200

    Document the last of the VSL records

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 01ced35..e6bf5a3 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -131,25 +131,27 @@ SLTM(BackendReuse, "Backend connection put up for reuse",
 	"Logged when a backend connection is put up for reuse by a later"
 	" connection.\n\n"
 	"The format is::\n\n"
-	"%d %s\n"
-	"|  |\n"
-	"|  +- Backend display name\n"
-	"+---- Connection file descriptor\n"
+	"\t%d %s\n"
+	"\t|  |\n"
+	"\t|  +- Backend display name\n"
+	"\t+---- Connection file descriptor\n"
 	"\n"
 )
 
 SLTM(BackendClose, "Backend connection closed",
 	"Logged when a backend connection is closed.\n\n"
 	"The format is::\n\n"
-	"%d %s [ %s ]\n"
-	"|  |    |\n"
-	"|  |    +- Optional reason\n"
-	"|  +------ Backend display name\n"
-	"+--------- Connection file descriptor\n"
+	"\t%d %s [ %s ]\n"
+	"\t|  |    |\n"
+	"\t|  |    +- Optional reason\n"
+	"\t|  +------ Backend display name\n"
+	"\t+--------- Connection file descriptor\n"
 	"\n"
 )
 
-SLTM(HttpGarbage, "", "")
+SLTM(HttpGarbage, "Unparseable HTTP request",
+	"Logs the content of unparseable HTTP requests.\n\n"
+)
 
 SLTM(Backend, "Backend selected",
 	"Logged when a connection is selected for handling a backend"
@@ -163,7 +165,9 @@ SLTM(Backend, "Backend selected",
 	"\n"
 )
 
-SLTM(Length, "Size of object body", "")
+SLTM(Length, "Size of object body",
+	"Logs the size of a fetch object body.\n\n"
+)
 
 SLTM(BereqEnd, "Backend request end",
 	"Marks the end of a backend request.\n\n"
@@ -175,7 +179,9 @@ SLTM(BereqEnd, "Backend request end",
 	"dTresp\n    Time to receive the backend response (dThdr + dTbody)\n\n"
 )
 
-SLTM(FetchError, "Error while fetching object", "")
+SLTM(FetchError, "Error while fetching object",
+	"Logs the error message of a failed fetch operation.\n\n"
+)
 
 #define SLTH(tag, ind, req, resp, sdesc, ldesc) \
 	SLTM(Req##tag, (req ? "Client request " sdesc : "(unused)"), ldesc)
@@ -207,7 +213,10 @@ SLTM(BogoHeader, "Bogus HTTP received",
 	"Contains the first 20 characters of received HTTP headers we could"
 	" not make sense of.  Applies to both req.http and beres.http.\n\n"
 )
-SLTM(LostHeader, "Failed attempt to set HTTP header", "")
+SLTM(LostHeader, "Failed attempt to set HTTP header",
+	"Logs the header name of a failed HTTP header operation due to"
+	" resource exhaustion.\n\n"
+)
 
 SLTM(TTL, "TTL set on object",
 	"A TTL record is emitted whenever the ttl, grace or keep"



From martin at varnish-cache.org  Thu Oct 31 11:21:09 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 31 Oct 2013 12:21:09 +0100
Subject: [master] 2157bd5 Format BereqEnd in the same way as the other tags
Message-ID: <E1VbqJJ-0003Dt-C5@project.varnish-software.com>

commit 2157bd562ebad8be4c8c4d9945fe7e5b023166cc
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Oct 28 14:56:45 2013 +0100

    Format BereqEnd in the same way as the other tags

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index e6bf5a3..7457066 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -171,12 +171,16 @@ SLTM(Length, "Size of object body",
 
 SLTM(BereqEnd, "Backend request end",
 	"Marks the end of a backend request.\n\n"
-	"Tstart\n    Timestamp when the fetch started (epoch)\n\n"
-	"Tend\n    Timestamp when the fetch ended (epoch)\n\n"
-	"dTsend\n    Time to send the backend request\n\n"
-	"dThdr\n    Time to receive the backend response headers\n\n"
-	"dTbody\n    Time to receive the backend response body\n\n"
-	"dTresp\n    Time to receive the backend response (dThdr + dTbody)\n\n"
+	"The format is::\n\n"
+	"\t%f %f %f %f %f %f\n"
+	"\t|  |  |  |  |  |\n"
+	"\t|  |  |  |  |  +- Time to receive response (hdr + body)\n"
+	"\t|  |  |  |  +---- Time to receive body\n"
+	"\t|  |  |  +------- Time to receive headers\n"
+	"\t|  |  +---------- Time to send request body\n"
+	"\t|  +------------- Timestamp (since epoch) when the request ended\n"
+	"\t+---------------- Timestamp (since epoch) when the request started\n"
+	"\n"
 )
 
 SLTM(FetchError, "Error while fetching object",



From martin at varnish-cache.org  Thu Oct 31 11:21:09 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 31 Oct 2013 12:21:09 +0100
Subject: [master] ee1b415 Emit SLT_HitPass log records
Message-ID: <E1VbqJJ-0003E1-GZ@project.varnish-software.com>

commit ee1b415fab5131fc11f0d69a0fc4ef3a620cfd59
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Oct 28 17:00:07 2013 +0100

    Emit SLT_HitPass log records

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index fe089cd..00bc090 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -392,6 +392,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	if (oc->flags & OC_F_PASS) {
 		/* Found a hit-for-pass */
 		VSLb(req->vsl, SLT_Debug, "XXXX HIT-FOR-PASS");
+		VSLb(req->vsl, SLT_HitPass, "%u", req->obj->vxid);
 		AZ(boc);
 		(void)HSH_DerefObjCore(&wrk->stats, &oc);
 		req->objcore = NULL;
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 7457066..4edea09 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -294,7 +294,10 @@ SLTM(Hit, "Hit object in cache",
 	"Object looked up in cache. Shows the VXID of the object.\n\n"
 )
 
-SLTM(HitPass, "Hit for pass object in cache (unused)", "")
+SLTM(HitPass, "Hit for pass object in cache.\n\n",
+	"Hit-for-pass object looked up in cache. Shows the VXID of the"
+	" hit-for-pass object.\n\n"
+)
 
 SLTM(ExpBan, "Object evicted due to ban",
 	"Logs the VXID when an object is banned.\n\n"



From martin at varnish-cache.org  Thu Oct 31 11:21:09 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 31 Oct 2013 12:21:09 +0100
Subject: [master] 6725221 Flag unused VSL tags in the definition table.
Message-ID: <E1VbqJJ-0003E9-Js@project.varnish-software.com>

commit 6725221947288cd02005bdef5fa58a76a63a807b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 31 10:54:58 2013 +0100

    Flag unused VSL tags in the definition table.
    
    Add a flags field to the VSL tag definition table, and use this to
    flag unused records.

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 1e0d7e4..e3368df 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -100,7 +100,7 @@ mgt_DumpRstVsl(void)
 	    "\n.. The following is autogenerated output from "
 	    "varnishd -x dumprstvsl\n\n");
 
-#define SLTM(tag, sdesc, ldesc) mgt_sltm(#tag, sdesc, ldesc);
+#define SLTM(tag, flags, sdesc, ldesc) mgt_sltm(#tag, sdesc, ldesc);
 #include "tbl/vsl_tags.h"
 #undef SLTM
 }
diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c
index 1816e1b..247eeea 100644
--- a/bin/varnishd/mgt/mgt_param_bits.c
+++ b/bin/varnishd/mgt/mgt_param_bits.c
@@ -115,7 +115,7 @@ bit_tweak(struct cli *cli, uint8_t *p, unsigned l, const char *arg,
  */
 
 static const char * const VSL_tags[256] = {
-#  define SLTM(foo,sdesc,ldesc) [SLT_##foo] = #foo,
+#  define SLTM(foo,flags,sdesc,ldesc) [SLT_##foo] = #foo,
 #  include "tbl/vsl_tags.h"
 #  undef SLTM
 	NULL
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 4edea09..ca07ed5 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -37,25 +37,28 @@
  *
  * Arguments:
  *	Tag-Name
+ *	Flags
  *	Short Description (1 line, max ? chars)
  *	Long Description (in RST "definition list" format)
  */
 
 #define NODEF_NOTICE "Note: This log record is masked by default\n\n"
 
-SLTM(Debug, "Debug messages",
+SLTM(Debug, SLT_F_BINARY, "Debug messages",
 	"Debug messages can normally be ignored, but are sometimes"
 	" helpful during trouble-shooting.  Most debug messages must"
 	" be explicitly enabled with parameters.\n\n"
 )
-SLTM(Error, "Error messages",
+
+SLTM(Error, 0, "Error messages",
 	"Error messages are stuff you probably want to know.\n\n"
 )
-SLTM(CLI, "CLI communication",
+
+SLTM(CLI, 0, "CLI communication",
 	"CLI communication between master and child process.\n\n"
 )
 
-SLTM(ReqEnd, "Client request end",
+SLTM(ReqEnd, 0, "Client request end",
 	"Marks the end of client request.\n\n"
 	"The format is::\n\n"
 	"\t%f %f %f %f %f\n"
@@ -70,7 +73,7 @@ SLTM(ReqEnd, "Client request end",
 
 /*---------------------------------------------------------------------*/
 
-SLTM(SessOpen, "Client connection opened",
+SLTM(SessOpen, 0, "Client connection opened",
 	"The first record for a client connection, with the socket-endpoints"
 	" of the connection.\n\n"
 	"The format is::\n\n"
@@ -95,7 +98,7 @@ SLTM(SessOpen, "Client connection opened",
 #undef SESS_CLOSE
 */
 
-SLTM(SessClose, "Client connection closed",
+SLTM(SessClose, 0, "Client connection closed",
 	"SessionClose is the last record for any client connection.\n\n"
 	"The format is::\n\n"
 	"\t%s %f %u %u %u %u %u %u\n"
@@ -113,7 +116,7 @@ SLTM(SessClose, "Client connection closed",
 
 /*---------------------------------------------------------------------*/
 
-SLTM(BackendOpen, "Backend connection opened",
+SLTM(BackendOpen, 0, "Backend connection opened",
 	"Logged when a new backend connection is opened.\n\n"
 	"The format is::\n\n"
 	"\t%d %s %s %s\n"
@@ -125,9 +128,9 @@ SLTM(BackendOpen, "Backend connection opened",
 	"\n"
 )
 
-SLTM(BackendXID, "The unique ID of the backend transaction (unused)", "")
+SLTM(BackendXID, SLT_F_UNUSED, "The unique ID of the backend transaction", "")
 
-SLTM(BackendReuse, "Backend connection put up for reuse",
+SLTM(BackendReuse, 0, "Backend connection put up for reuse",
 	"Logged when a backend connection is put up for reuse by a later"
 	" connection.\n\n"
 	"The format is::\n\n"
@@ -138,7 +141,7 @@ SLTM(BackendReuse, "Backend connection put up for reuse",
 	"\n"
 )
 
-SLTM(BackendClose, "Backend connection closed",
+SLTM(BackendClose, 0, "Backend connection closed",
 	"Logged when a backend connection is closed.\n\n"
 	"The format is::\n\n"
 	"\t%d %s [ %s ]\n"
@@ -149,11 +152,11 @@ SLTM(BackendClose, "Backend connection closed",
 	"\n"
 )
 
-SLTM(HttpGarbage, "Unparseable HTTP request",
+SLTM(HttpGarbage, SLT_F_BINARY, "Unparseable HTTP request",
 	"Logs the content of unparseable HTTP requests.\n\n"
 )
 
-SLTM(Backend, "Backend selected",
+SLTM(Backend, 0, "Backend selected",
 	"Logged when a connection is selected for handling a backend"
 	" request.\n\n"
 	"The format is::\n\n"
@@ -165,11 +168,11 @@ SLTM(Backend, "Backend selected",
 	"\n"
 )
 
-SLTM(Length, "Size of object body",
+SLTM(Length, 0, "Size of object body",
 	"Logs the size of a fetch object body.\n\n"
 )
 
-SLTM(BereqEnd, "Backend request end",
+SLTM(BereqEnd, 0, "Backend request end",
 	"Marks the end of a backend request.\n\n"
 	"The format is::\n\n"
 	"\t%f %f %f %f %f %f\n"
@@ -183,46 +186,48 @@ SLTM(BereqEnd, "Backend request end",
 	"\n"
 )
 
-SLTM(FetchError, "Error while fetching object",
+SLTM(FetchError, 0, "Error while fetching object",
 	"Logs the error message of a failed fetch operation.\n\n"
 )
 
 #define SLTH(tag, ind, req, resp, sdesc, ldesc) \
-	SLTM(Req##tag, (req ? "Client request " sdesc : "(unused)"), ldesc)
+	SLTM(Req##tag, req ? 0 : SLT_F_UNUSED, "Client request " sdesc, ldesc)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
 #define SLTH(tag, ind, req, resp, sdesc, ldesc) \
-	SLTM(Resp##tag, (resp ? "Client response " sdesc : "(unused)"), ldesc)
+	SLTM(Resp##tag, resp ? 0 : SLT_F_UNUSED, "Client response " sdesc, \
+	    ldesc)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
 #define SLTH(tag, ind, req, resp, sdesc, ldesc) \
-	SLTM(Bereq##tag, (req ? "Backend request " sdesc : "(unused)"), ldesc)
+	SLTM(Bereq##tag, req ? 0 : SLT_F_UNUSED, "Backend request " sdesc, \
+	    ldesc)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
 #define SLTH(tag, ind, req, resp, sdesc, ldesc) \
-	SLTM(Beresp##tag, (resp ? "Backend response " sdesc : "(unused)"), \
+	SLTM(Beresp##tag, resp ? 0 : SLT_F_UNUSED, "Backend response " sdesc, \
 	    ldesc)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
 #define SLTH(tag, ind, req, resp, sdesc, ldesc) \
-	SLTM(Obj##tag, (resp ? "Object  " sdesc : "(unused)"), ldesc)
+	SLTM(Obj##tag, resp ? 0 : SLT_F_UNUSED, "Object  " sdesc, ldesc)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
-SLTM(BogoHeader, "Bogus HTTP received",
+SLTM(BogoHeader, 0, "Bogus HTTP received",
 	"Contains the first 20 characters of received HTTP headers we could"
 	" not make sense of.  Applies to both req.http and beres.http.\n\n"
 )
-SLTM(LostHeader, "Failed attempt to set HTTP header",
+SLTM(LostHeader, 0, "Failed attempt to set HTTP header",
 	"Logs the header name of a failed HTTP header operation due to"
 	" resource exhaustion.\n\n"
 )
 
-SLTM(TTL, "TTL set on object",
+SLTM(TTL, 0, "TTL set on object",
 	"A TTL record is emitted whenever the ttl, grace or keep"
 	" values for an object is set.\n\n"
 	"The format is::\n\n"
@@ -249,7 +254,8 @@ SLTM(TTL, "TTL set on object",
 	"\t1001 VCL 12 120 3600 1312966113 8\n"
 	"\n"
 )
-SLTM(Fetch_Body, "Body fetched from backend",
+
+SLTM(Fetch_Body, 0, "Body fetched from backend",
 	"Finished fetching body from backend.\n\n"
 	"The format is::\n\n"
 	"\t%d(%s) cls %d\n"
@@ -259,13 +265,16 @@ SLTM(Fetch_Body, "Body fetched from backend",
 	"\t+------------ Body status\n"
 	"\n"
 )
-SLTM(VCL_acl, "VSL ACL check results",
+
+SLTM(VCL_acl, 0, "VSL ACL check results",
 	"Logs VCL ACL evaluation results.\n\n"
 )
-SLTM(VCL_call, "VCL method called",
+
+SLTM(VCL_call, 0, "VCL method called",
 	"Logs the VCL method name when a VCL method is called.\n\n"
 )
-SLTM(VCL_trace, "VCL trace data",
+
+SLTM(VCL_trace, 0, "VCL trace data",
 	"Logs VCL execution trace data.\n\n"
 	"The format is::\n\n"
 	"\t%u %u.%u\n"
@@ -276,10 +285,12 @@ SLTM(VCL_trace, "VCL trace data",
 	"\n"
 	NODEF_NOTICE
 )
-SLTM(VCL_return, "VCL method return value",
+
+SLTM(VCL_return, 0, "VCL method return value",
 	"Logs the VCL method terminating statement.\n\n"
 )
-SLTM(ReqStart, "Client request start",
+
+SLTM(ReqStart, 0, "Client request start",
 	"Start of request processing. Logs the client IP address and port"
 	" number.\n\n"
 	"The format is::\n\n"
@@ -290,20 +301,20 @@ SLTM(ReqStart, "Client request start",
 	"\n"
 )
 
-SLTM(Hit, "Hit object in cache",
+SLTM(Hit, 0, "Hit object in cache",
 	"Object looked up in cache. Shows the VXID of the object.\n\n"
 )
 
-SLTM(HitPass, "Hit for pass object in cache.\n\n",
+SLTM(HitPass, 0, "Hit for pass object in cache.\n\n",
 	"Hit-for-pass object looked up in cache. Shows the VXID of the"
 	" hit-for-pass object.\n\n"
 )
 
-SLTM(ExpBan, "Object evicted due to ban",
+SLTM(ExpBan, 0, "Object evicted due to ban",
 	"Logs the VXID when an object is banned.\n\n"
 )
 
-SLTM(ExpKill, "Object expiry event",
+SLTM(ExpKill, 0, "Object expiry event",
 	"Logs events related to object expiry. The events are:\n\n"
 	"EXP_Rearm\n"
 	"\tLogged when the expiry time of an object changes.\n\n"
@@ -344,28 +355,28 @@ SLTM(ExpKill, "Object expiry event",
 	"\n"
 )
 
-SLTM(WorkThread, "Logs thread start/stop events",
+SLTM(WorkThread, 0, "Logs thread start/stop events",
 	"Logs worker thread creation and termination events.\n\n"
 	"The format is::\n\n"
 	"\t%p %s\n"
 	"\t|  |\n"
 	"\t|  +- [start|end]\n"
-	"\t+---- Worker struct pointer"
+	"\t+---- Worker struct pointer\n"
 	"\n"
 	NODEF_NOTICE
 )
 
-SLTM(ESI_xmlerror, "ESI parser error or warning message",
+SLTM(ESI_xmlerror, 0, "ESI parser error or warning message",
 	"An error or warning was generated during parsing of an ESI object."
 	" The log record describes the problem encountered."
 )
 
-SLTM(Hash, "Value added to hash",
+SLTM(Hash, 0, "Value added to hash",
 	"This value was added to the object lookup hash.\n\n"
 	NODEF_NOTICE
 )
 
-SLTM(Backend_health, "Backend health check",
+SLTM(Backend_health, 0, "Backend health check",
 	"The result of a backend health probe.\n\n"
 	"The format is::\n\n"
 	"\t%s %s %s %u %u %u %f %f %s\n"
@@ -382,15 +393,17 @@ SLTM(Backend_health, "Backend health check",
 	"\n"
 )
 
-SLTM(VCL_Debug, "(unused)", "")
-SLTM(VCL_Log, "Log statement from VCL",
+SLTM(VCL_Debug, SLT_F_UNUSED, "", "")
+
+SLTM(VCL_Log, 0, "Log statement from VCL",
 	"User generated log messages insert from VCL through std.log()"
 )
-SLTM(VCL_Error, "VCL execution error message",
+
+SLTM(VCL_Error, 0, "VCL execution error message",
 	"Logs error messages generated during VCL execution.\n\n"
 )
 
-SLTM(Gzip, "G(un)zip performed on object",
+SLTM(Gzip, 0, "G(un)zip performed on object",
 	"A Gzip record is emitted for each instance of gzip or gunzip"
 	" work performed. Worst case, an ESI transaction stored in"
 	" gzip'ed objects but delivered gunziped, will run into many of"
@@ -413,7 +426,7 @@ SLTM(Gzip, "G(un)zip performed on object",
 	"\n"
 )
 
-SLTM(Link, "Links to a child VXID",
+SLTM(Link, 0, "Links to a child VXID",
 	"Links this VXID to any child VXID it initiates.\n\n"
 	"The format is::\n\n"
 	"\t%s %d\n"
@@ -423,7 +436,7 @@ SLTM(Link, "Links to a child VXID",
 	"\n"
 )
 
-SLTM(Begin, "Marks the start of a VXID",
+SLTM(Begin, 0, "Marks the start of a VXID",
 	"The first record of a VXID transaction.\n\n"
 	"The format is::\n\n"
 	"\t%s %d\n"
@@ -433,11 +446,11 @@ SLTM(Begin, "Marks the start of a VXID",
 	"\n"
 )
 
-SLTM(End, "Marks the end of a VXID",
+SLTM(End, 0, "Marks the end of a VXID",
 	"The last record of a VXID transaction.\n\n"
 )
 
-SLTM(VSL, "VSL API warnings and error message",
+SLTM(VSL, 0, "VSL API warnings and error message",
 	"Warnings and error messages genererated by the VSL API while"
 	" reading the shared memory log.\n\n"
 )
diff --git a/include/tbl/vsl_tags_http.h b/include/tbl/vsl_tags_http.h
index 55b98fc..8d7e56d 100644
--- a/include/tbl/vsl_tags_http.h
+++ b/include/tbl/vsl_tags_http.h
@@ -64,6 +64,6 @@ SLTH(Header,	HTTP_HDR_FIRST,		1, 1, "header",
 	"\t+----- Header name\n"
 	"\n"
 )
-SLTH(Lost,	HTTP_HDR_LOST,		1, 1, "(unused)",
+SLTH(Lost,	HTTP_HDR_LOST,		0, 0, "lost header",
 	""
 )
diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h
index 02a9cb0..1a6f15a 100644
--- a/include/vapi/vsl_int.h
+++ b/include/vapi/vsl_int.h
@@ -107,11 +107,15 @@ struct VSL_head {
 #define SLT__MAX 256
 enum VSL_tag_e {
 	SLT__Bogus = 0,
-#define SLTM(foo,sdesc,ldesc)	SLT_##foo,
+#define SLTM(foo,flags,sdesc,ldesc)	SLT_##foo,
 #include "tbl/vsl_tags.h"
 #undef SLTM
 	SLT__Reserved = 254,
 	SLT__Batch = 255
 };
 
+/* VSL tag flags */
+#define SLT_F_UNUSED		(1 << 0)
+#define SLT_F_BINARY		(1 << 1)
+
 #endif /* VAPI_VSL_FMT_H_INCLUDED */
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 7c5dafa..886f5bf 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -58,7 +58,7 @@
 /*--------------------------------------------------------------------*/
 
 const char *VSL_tags[SLT__MAX] = {
-#  define SLTM(foo,sdesc,ldesc)       [SLT_##foo] = #foo,
+#  define SLTM(foo,flags,sdesc,ldesc)       [SLT_##foo] = #foo,
 #  include "tbl/vsl_tags.h"
 #  undef SLTM
 };
diff --git a/lib/libvarnishapi/vsl2rst.c b/lib/libvarnishapi/vsl2rst.c
index 44f1f7c..eacc79b 100644
--- a/lib/libvarnishapi/vsl2rst.c
+++ b/lib/libvarnishapi/vsl2rst.c
@@ -39,14 +39,15 @@
 
 struct SLT {
 	unsigned	tag;
+	unsigned	flags;
 	const char	*name;
 	const char	*sdesc;
 	const char	*ldesc;
 };
 
 static struct SLT tags[SLT__MAX] = {
-#define SLTM(name, sdesc, ldesc) \
-	[SLT_##name] = { SLT_##name, #name, sdesc, ldesc },
+#define SLTM(name, flags, sdesc, ldesc)				\
+	[SLT_##name] = { SLT_##name, flags, #name, sdesc, ldesc },
 #include "tbl/vsl_tags.h"
 #undef SLTM
 };
@@ -98,10 +99,7 @@ main(int argc, char *argv[])
 	for (i = 0; i < SLT__MAX; i++) {
 		if (ptags[i]->name == NULL || !strcmp(ptags[i]->name, ""))
 			continue;
-		if (ptags[i]->sdesc != NULL &&
-		    strstr(ptags[i]->sdesc, "(unused)"))
-			/* Don't list tags where the short description
-			   contains the string "(unused)" */
+		if (ptags[i]->flags & SLT_F_UNUSED)
 			continue;
 		printf("%s", ptags[i]->name);
 		if (ptags[i]->sdesc != NULL && strcmp(ptags[i]->sdesc, ""))



From martin at varnish-cache.org  Thu Oct 31 11:21:09 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 31 Oct 2013 12:21:09 +0100
Subject: [master] a4f9ef9 Use a tag flag to mark binary log records
Message-ID: <E1VbqJJ-0003EI-PM@project.varnish-software.com>

commit a4f9ef962707b896abf35984f6ace234d9479aed
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 31 12:15:36 2013 +0100

    Use a tag flag to mark binary log records

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index a68e41f..8fb008e 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -111,6 +111,11 @@ extern const char *VSL_tags[SLT__MAX];
 	 * Tag to string array.  Contains NULL for invalid tags.
 	 */
 
+extern unsigned VSL_tagflags[SLT__MAX];
+	/*
+	 * Tag flags array.
+	 */
+
 int VSL_Name2Tag(const char *name, int l);
 	/*
 	 * Convert string to tag number (= enum VSL_tag_e). Name can be a
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 886f5bf..8375a2c 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -63,6 +63,12 @@ const char *VSL_tags[SLT__MAX] = {
 #  undef SLTM
 };
 
+unsigned VSL_tagflags[SLT__MAX] = {
+#  define SLTM(foo, flags, sdesc, ldesc)	[SLT_##foo] = flags,
+#  include "tbl/vsl_tags.h"
+#  undef SLTM
+};
+
 int
 vsl_diag(struct VSL_data *vsl, const char *fmt, ...)
 {
@@ -246,7 +252,7 @@ VSL_Print(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
 	    'b' : '-';
 	data = VSL_CDATA(c->rec.ptr);
 
-	if (tag == SLT_Debug) {
+	if (VSL_tagflags[tag] & SLT_F_BINARY) {
 		VSL_PRINT(fo, "%10u %-14s %c \"", vxid, VSL_tags[tag], type);
 		while (len-- > 0) {
 			if (*data >= ' ' && *data <= '~')
@@ -279,7 +285,7 @@ VSL_PrintTerse(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo)
 	len = VSL_LEN(c->rec.ptr);
 	data = VSL_CDATA(c->rec.ptr);
 
-	if (tag == SLT_Debug) {
+	if (VSL_tagflags[tag] & SLT_F_BINARY) {
 		VSL_PRINT(fo, "%-14s \"", VSL_tags[tag]);
 		while (len-- > 0) {
 			if (*data >= ' ' && *data <= '~')



From martin at varnish-cache.org  Thu Oct 31 11:35:48 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 31 Oct 2013 12:35:48 +0100
Subject: [master] bca5e78 Remove a couple of unused VSL tag definitions
Message-ID: <E1VbqXU-0004kD-Ip@project.varnish-software.com>

commit bca5e78e0877588bab1056d3ba2c306438524f54
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 31 12:31:35 2013 +0100

    Remove a couple of unused VSL tag definitions
    
    Remove a couple of unused VSL tag definitions. This is safe to do now
    that we are making a major release.

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index ca07ed5..8ee6b2a 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -128,8 +128,6 @@ SLTM(BackendOpen, 0, "Backend connection opened",
 	"\n"
 )
 
-SLTM(BackendXID, SLT_F_UNUSED, "The unique ID of the backend transaction", "")
-
 SLTM(BackendReuse, 0, "Backend connection put up for reuse",
 	"Logged when a backend connection is put up for reuse by a later"
 	" connection.\n\n"
@@ -393,8 +391,6 @@ SLTM(Backend_health, 0, "Backend health check",
 	"\n"
 )
 
-SLTM(VCL_Debug, SLT_F_UNUSED, "", "")
-
 SLTM(VCL_Log, 0, "Log statement from VCL",
 	"User generated log messages insert from VCL through std.log()"
 )



From martin at varnish-cache.org  Thu Oct 31 12:24:36 2013
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 31 Oct 2013 13:24:36 +0100
Subject: [master] 645c1ca Get access to object before trying to use it
Message-ID: <E1VbrIi-0001uc-HT@project.varnish-software.com>

commit 645c1ca8e9d18afbbc4d9cf50c9e051a0ca2c573
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 31 13:22:31 2013 +0100

    Get access to object before trying to use it
    
    Fix issue where object was used before it was looked up with the
    stevedore.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 00bc090..37ec19e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -389,12 +389,16 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	AZ (oc->flags & OC_F_BUSY);
 	AZ(req->objcore);
 
+	o = oc_getobj(&wrk->stats, oc);
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	req->obj = o;
+
 	if (oc->flags & OC_F_PASS) {
 		/* Found a hit-for-pass */
 		VSLb(req->vsl, SLT_Debug, "XXXX HIT-FOR-PASS");
 		VSLb(req->vsl, SLT_HitPass, "%u", req->obj->vxid);
 		AZ(boc);
-		(void)HSH_DerefObjCore(&wrk->stats, &oc);
+		(void)HSH_DerefObj(&wrk->stats, &req->obj);
 		req->objcore = NULL;
 		wrk->stats.cache_hitpass++;
 		req->req_step = R_STP_PASS;
@@ -404,10 +408,6 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	oh = oc->objhead;
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 
-	o = oc_getobj(&wrk->stats, oc);
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	req->obj = o;
-
 	VSLb(req->vsl, SLT_Hit, "%u", req->obj->vxid);
 
 	VCL_hit_method(req->vcl, wrk, req, NULL, req->http->ws);