From nils.goroll at uplex.de Mon Dec 1 08:55:05 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 01 Dec 2014 09:55:05 +0100 Subject: [master] 6fbf247 Restore the default SIGSEGV handler during pan_ic Message-ID: commit 6fbf247c9b12b991c030a1e4f4942818e4c0927d Author: Nils Goroll Date: Mon Dec 1 09:28:54 2014 +0100 Restore the default SIGSEGV handler during pan_ic Leaving it enabled could hide panics. Fixes #1639 diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index bc1f992..0549029 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -37,6 +37,7 @@ #include #include +#include #include "cache.h" #include "cache_filter.h" @@ -472,10 +473,21 @@ pan_ic(const char *func, const char *file, int line, const char *cond, const char *q; struct req *req; struct busyobj *bo; + struct sigaction sa; AZ(pthread_mutex_lock(&panicstr_mtx)); /* Won't be released, we're going to die anyway */ + + /* + * should we trigger a SIGSEGV while handling a panic, our sigsegv + * handler would hide the panic, so we need to reset the handler to + * default + */ + memset(&sa, 0, sizeof sa); + sa.sa_handler = SIG_DFL; + (void)sigaction(SIGSEGV, &sa, NULL); + switch(kind) { case VAS_WRONG: VSB_printf(pan_vsp, From nils.goroll at uplex.de Mon Dec 1 08:55:05 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 01 Dec 2014 09:55:05 +0100 Subject: [master] fb25963 Check pointers before dereferencing - avoid SIGSEGV while panicking Message-ID: commit fb25963b9015cbf3c95dd62c921b18767bc2db0e Author: Nils Goroll Date: Mon Dec 1 09:42:03 2014 +0100 Check pointers before dereferencing - avoid SIGSEGV while panicking diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 0549029..295dfe5 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -324,14 +324,18 @@ pan_busyobj(const struct busyobj *bo) pan_ws(bo->ws, 4); VSB_printf(pan_vsp, " refcnt = %u\n", bo->refcount); VSB_printf(pan_vsp, " retries = %d\n", bo->retries); - VSB_printf(pan_vsp, " failed = %d\n", bo->vfc->failed); + if (bo->vfc != NULL) + VSB_printf(pan_vsp, " failed = %d\n", bo->vfc->failed); VSB_printf(pan_vsp, " state = %d\n", (int)bo->state); #define BO_FLAG(l, r, w, d) if(bo->l) VSB_printf(pan_vsp, " is_" #l "\n"); #include "tbl/bo_flags.h" #undef BO_FLAG - VSB_printf(pan_vsp, " bodystatus = %d (%s),\n", - bo->htc->body_status, body_status_2str(bo->htc->body_status)); + if (bo->htc != NULL) { + VSB_printf(pan_vsp, " bodystatus = %d (%s),\n", + bo->htc->body_status, + body_status_2str(bo->htc->body_status)); + } if (!VTAILQ_EMPTY(&bo->vfc->vfp)) { VSB_printf(pan_vsp, " filters ="); VTAILQ_FOREACH(vfe, &bo->vfc->vfp, list) @@ -340,11 +344,13 @@ pan_busyobj(const struct busyobj *bo) VSB_printf(pan_vsp, "\n"); } VSB_printf(pan_vsp, " },\n"); - if (VALID_OBJ(bo->htc->vbc, BACKEND_MAGIC)) + + if (bo->htc != NULL && bo->htc->vbc != NULL && + VALID_OBJ(bo->htc->vbc, BACKEND_MAGIC)) pan_vbc(bo->htc->vbc); - if (bo->bereq->ws != NULL) + if (bo->bereq != NULL && bo->bereq->ws != NULL) pan_http("bereq", bo->bereq, 4); - if (bo->beresp->ws != NULL) + if (bo->beresp != NULL && bo->beresp->ws != NULL) pan_http("beresp", bo->beresp, 4); if (bo->fetch_objcore) pan_objcore("FETCH", bo->fetch_objcore); From lkarsten at varnish-software.com Mon Dec 1 10:19:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 01 Dec 2014 11:19:24 +0100 Subject: [3.0] b69fff3 Use better criteria for determining if child CLI connection is hosed. Message-ID: commit b69fff3f3404ac8800d74657c7d3e2c04bf54eeb Author: Poul-Henning Kamp Date: Sun Nov 13 22:02:44 2011 +0000 Use better criteria for determining if child CLI connection is hosed. Backports commit 2144dc78 for Varnish 3.0.6 (by Geoff) diff --git a/bin/varnishd/mgt_cli.c b/bin/varnishd/mgt_cli.c index 7b27e98..da22660 100644 --- a/bin/varnishd/mgt_cli.c +++ b/bin/varnishd/mgt_cli.c @@ -172,7 +172,8 @@ mcf_askchild(struct cli *cli, const char * const *av, void *priv) return; } VSB_delete(vsb); - (void)VCLI_ReadResult(cli_i, &u, &q, params->cli_timeout); + if (VCLI_ReadResult(cli_i, &u, &q, params->cli_timeout)) + MGT_Child_Cli_Fail(); VCLI_SetResult(cli, u); VCLI_Out(cli, "%s", q); free(q); @@ -221,11 +222,10 @@ mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...) { return (CLIS_COMMS); } - (void)VCLI_ReadResult(cli_i, &u, resp, params->cli_timeout); + if (VCLI_ReadResult(cli_i, &u, resp, params->cli_timeout)) + MGT_Child_Cli_Fail(); if (status != NULL) *status = u; - if (u == CLIS_COMMS) - MGT_Child_Cli_Fail(); return (u == CLIS_OK ? 0 : u); } From phk at FreeBSD.org Mon Dec 1 11:19:59 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 01 Dec 2014 12:19:59 +0100 Subject: [master] cbb2f5d Initialize bo->vfc along with the rest of the busyobj so panics work. (slink's pointer test for bo->vfc is impotent because its bo->vfc[1]) Message-ID: commit cbb2f5d6ec722280d8f2e164c9cbe54f19814857 Author: Poul-Henning Kamp Date: Mon Dec 1 11:03:05 2014 +0000 Initialize bo->vfc along with the rest of the busyobj so panics work. (slink's pointer test for bo->vfc is impotent because its bo->vfc[1]) diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 16c67f2..a803942 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -39,6 +39,7 @@ #include "cache.h" #include "hash/hash_slinger.h" +#include "cache/cache_filter.h" static struct mempool *vbopool; @@ -156,6 +157,8 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req) VRTPRIV_init(bo->privs); + VFP_Setup(bo->vfc); + return (bo); } diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 08f253f..c44992b 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -389,7 +389,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) } else do_ims = 0; - VFP_Setup(bo->vfc); bo->vfc->bo = bo; bo->vfc->oc = bo->fetch_objcore; bo->vfc->wrk = bo->wrk; @@ -776,7 +775,6 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) assert(wrk->handling == VCL_RET_DELIVER); - VFP_Setup(bo->vfc); bo->vfc->bo = bo; bo->vfc->wrk = bo->wrk; bo->vfc->oc = bo->fetch_objcore; diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 295dfe5..d82e43f 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -324,8 +324,7 @@ pan_busyobj(const struct busyobj *bo) pan_ws(bo->ws, 4); VSB_printf(pan_vsp, " refcnt = %u\n", bo->refcount); VSB_printf(pan_vsp, " retries = %d\n", bo->retries); - if (bo->vfc != NULL) - VSB_printf(pan_vsp, " failed = %d\n", bo->vfc->failed); + VSB_printf(pan_vsp, " failed = %d\n", bo->vfc->failed); VSB_printf(pan_vsp, " state = %d\n", (int)bo->state); #define BO_FLAG(l, r, w, d) if(bo->l) VSB_printf(pan_vsp, " is_" #l "\n"); #include "tbl/bo_flags.h" From lkarsten at varnish-software.com Tue Dec 2 11:45:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Tue, 02 Dec 2014 12:45:24 +0100 Subject: [master] 0484c08 Permit ABI variables to be empty. Message-ID: commit 0484c0874c38ae39a8b24250c09eb7ce7af43fb4 Author: Lasse Karstensen Date: Tue Dec 2 12:43:23 2014 +0100 Permit ABI variables to be empty. It should be possible to build Debian packages that does not have the ABI provides set, but with the previous format dpkg-buildpackage errored out because of duplicate (empty) Provides. Moving to a single line and having varnish, which hopefully will be provided anyway, as the baseline. diff --git a/control b/control index ca29224..1c51426 100644 --- a/control +++ b/control @@ -29,8 +29,7 @@ Package: varnish Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends}, gcc (>= 3.3), libc6-dev | libc6.1-dev | libc-dev, adduser, libvarnishapi1 (>= ${binary:Version}) Suggests: varnish-doc -Provides: ${Varnish:ABI} -Provides: ${Varnish:strictABI} +Provides: varnish, ${Varnish:ABI}, ${Varnish:strictABI} Description: state of the art, high-performance web accelerator Varnish Cache is a state of the art web accelerator written with performance and flexibility in mind. From fgsch at lodoss.net Tue Dec 2 21:46:53 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 02 Dec 2014 22:46:53 +0100 Subject: [master] 6058ced Revert to binary output by default when using -w Message-ID: commit 6058cedf6ff5ff10f342a7e2728d709adc205a4e Author: Federico G. Schwindt Date: Tue Dec 2 21:15:16 2014 +0000 Revert to binary output by default when using -w Add -A to enable text output. Discussed with Martin. diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c index fbb2f4d..d73e45d 100644 --- a/bin/varnishlog/varnishlog.c +++ b/bin/varnishlog/varnishlog.c @@ -54,7 +54,7 @@ static const char progname[] = "varnishlog"; struct log { /* Options */ int a_opt; - int B_opt; + int A_opt; char *w_arg; /* State */ @@ -77,13 +77,13 @@ openout(int append) { AN(LOG.w_arg); - if (LOG.B_opt) - LOG.fo = VSL_WriteOpen(VUT.vsl, LOG.w_arg, append, 0); - else + if (LOG.A_opt) LOG.fo = fopen(LOG.w_arg, append ? "a" : "w"); + else + LOG.fo = VSL_WriteOpen(VUT.vsl, LOG.w_arg, append, 0); if (LOG.fo == NULL) VUT_Error(2, "Can't open output file (%s)", - LOG.B_opt ? VSL_Error(VUT.vsl) : strerror(errno)); + LOG.A_opt ? strerror(errno) : VSL_Error(VUT.vsl)); VUT.dispatch_priv = LOG.fo; } @@ -123,9 +123,9 @@ main(int argc, char * const *argv) /* Append to file */ LOG.a_opt = 1; break; - case 'B': - /* Binary output */ - LOG.B_opt = 1; + case 'A': + /* Text output */ + LOG.A_opt = 1; break; case 'h': /* Usage help */ @@ -145,10 +145,10 @@ main(int argc, char * const *argv) usage(1); /* Setup output */ - if (LOG.B_opt) - VUT.dispatch_f = VSL_WriteTransactions; - else + if (LOG.A_opt || !LOG.w_arg) VUT.dispatch_f = VSL_PrintTransactions; + else + VUT.dispatch_f = VSL_WriteTransactions; if (LOG.w_arg) { openout(LOG.a_opt); AN(LOG.fo); diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h index 55b30c7..1f2ab93 100644 --- a/bin/varnishlog/varnishlog_options.h +++ b/bin/varnishlog/varnishlog_options.h @@ -36,9 +36,9 @@ " overwrite it." \ ) -#define LOG_OPT_B \ - VOPT("B", "[-B]", "Binary output", \ - "Output binary data suitable for reading with -r." \ +#define LOG_OPT_A \ + VOPT("A", "[-A]", "Text output", \ + "Output data in ascii format." \ ) #define LOG_OPT_w \ @@ -50,8 +50,8 @@ ) LOG_OPT_a +LOG_OPT_A VSL_OPT_b -LOG_OPT_B VSL_OPT_c VSL_OPT_C VUT_OPT_d From nils.goroll at uplex.de Wed Dec 3 13:57:03 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 03 Dec 2014 14:57:03 +0100 Subject: [master] 5f345db bump the varnishtest CLI timeout from 20 to 30 seconds Message-ID: commit 5f345db5d49c6138cb1636ab00ddfc91fbc9b415 Author: Nils Goroll Date: Wed Dec 3 14:55:58 2014 +0100 bump the varnishtest CLI timeout from 20 to 30 seconds This avoids false negatives with slow name resolutions diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index c0be978..1a826ea 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -98,7 +98,7 @@ varnish_ask_cli(const struct varnish *v, const char *cmd, char **repl) i = write(v->cli_fd, "\n", 1); assert(i == 1); } - i = VCLI_ReadResult(v->cli_fd, &retval, &r, 20.0); + i = VCLI_ReadResult(v->cli_fd, &retval, &r, 30.0); if (i != 0) { vtc_log(v->vl, 0, "CLI failed (%s) = %d %u %s", cmd, i, retval, r); From lkarsten at varnish-software.com Wed Dec 3 15:32:40 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Wed, 03 Dec 2014 16:32:40 +0100 Subject: [master] 7e3370e Increase VRT_MINOR_VERSION due to VRT_selecthttp Message-ID: commit 7e3370eb0727264217aa98f873244ddb546f8314 Author: Lasse Karstensen Date: Mon Dec 1 10:06:52 2014 +0100 Increase VRT_MINOR_VERSION due to VRT_selecthttp See commit d192350fd. diff --git a/include/vrt.h b/include/vrt.h index 2e720a2..ff3e996 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -41,7 +41,7 @@ #define VRT_MAJOR_VERSION 2U -#define VRT_MINOR_VERSION 1U +#define VRT_MINOR_VERSION 2U /***********************************************************************/ From lkarsten at varnish-software.com Wed Dec 3 15:32:40 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Wed, 03 Dec 2014 16:32:40 +0100 Subject: [master] bb5fab8 White space OCD. Message-ID: commit bb5fab874a1a12aff9281ecf465ed1b75c837d74 Author: Lasse Karstensen Date: Wed Dec 3 16:32:18 2014 +0100 White space OCD. diff --git a/etc/example.vcl b/etc/example.vcl index a2987bb..eefdd9c 100644 --- a/etc/example.vcl +++ b/etc/example.vcl @@ -20,14 +20,14 @@ backend default { sub vcl_recv { # Happens before we check if we have this in cache already. - # + # # Typically you clean up the request here, removing cookies you don't need, # rewriting the request, etc. } sub vcl_backend_response { # Happens after we have read the response headers from the backend. - # + # # Here you clean the response headers, removing silly Set-Cookie headers # and other mistakes your backend does. } @@ -35,6 +35,6 @@ sub vcl_backend_response { sub vcl_deliver { # Happens when we have all the pieces we need, and are about to send the # response to the client. - # + # # You can do accounting or modifying the final object here. } From lkarsten at varnish-software.com Wed Dec 3 16:19:46 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Wed, 03 Dec 2014 17:19:46 +0100 Subject: [master] 100e377 Remove trailing white space. Message-ID: commit 100e377d30705d83f88af76c8108bce79ed6d7f6 Author: Lasse Karstensen Date: Wed Dec 3 17:19:35 2014 +0100 Remove trailing white space. diff --git a/redhat/varnish.params b/redhat/varnish.params index 636e975..27a14dd 100644 --- a/redhat/varnish.params +++ b/redhat/varnish.params @@ -1,35 +1,35 @@ # Varnish environment configuration description. This was derived from # the old style sysconfig/defaults settings -# Set this to 1 to make systemd reload try to switch vcl without restart. +# Set this to 1 to make systemd reload try to switch vcl without restart. RELOAD_VCL=1 # Main configuration file. You probably want to change it. -VARNISH_VCL_CONF=/etc/varnish/default.vcl +VARNISH_VCL_CONF=/etc/varnish/default.vcl # Default address and port to bind to. Blank address means all IPv4 # and IPv6 interfaces, otherwise specify a host name, an IPv4 dotted # quad, or an IPv6 address in brackets. # VARNISH_LISTEN_ADDRESS=192.168.1.5 -VARNISH_LISTEN_PORT=6081 +VARNISH_LISTEN_PORT=6081 # Admin interface listen address and port -VARNISH_ADMIN_LISTEN_ADDRESS=127.0.0.1 -VARNISH_ADMIN_LISTEN_PORT=6082 +VARNISH_ADMIN_LISTEN_ADDRESS=127.0.0.1 +VARNISH_ADMIN_LISTEN_PORT=6082 # Shared secret file for admin interface -VARNISH_SECRET_FILE=/etc/varnish/secret +VARNISH_SECRET_FILE=/etc/varnish/secret # Backend storage specification, see Storage Types in the varnishd(5) # man page for details. VARNISH_STORAGE="file,/var/lib/varnish/varnish_storage.bin,1G" # Default TTL used when the backend does not specify one -VARNISH_TTL=120 +VARNISH_TTL=120 # User and group for the varnishd worker processes -VARNISH_USER=varnish -VARNISH_GROUP=varnish +VARNISH_USER=varnish +VARNISH_GROUP=varnish # Other options, see the man page varnishd(1) #DAEMON_OPTS="-p thread_pool_min=5 -p thread_pool_max=500 -p thread_pool_timeout=300" From fgsch at lodoss.net Thu Dec 4 04:07:42 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 04 Dec 2014 05:07:42 +0100 Subject: [master] 165b4ca Typo Message-ID: commit 165b4caf8a522cd0ed5613ba5a06f0a37b828be7 Author: Federico G. Schwindt Date: Thu Dec 4 04:07:41 2014 +0000 Typo diff --git a/doc/sphinx/users-guide/vcl-variables.rst b/doc/sphinx/users-guide/vcl-variables.rst index 644c046..70862e5 100644 --- a/doc/sphinx/users-guide/vcl-variables.rst +++ b/doc/sphinx/users-guide/vcl-variables.rst @@ -22,7 +22,7 @@ objects can be accessed and manipulated using VCL. *beresp* The backend response object. It contains the headers of the object coming from the backend. If you want to modify the response coming from the - server you modify this object in `vcl_backend_reponse`. + server you modify this object in `vcl_backend_response`. *resp* The HTTP response right before it is delivered to the client. It is From fgsch at lodoss.net Mon Dec 8 11:49:59 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 08 Dec 2014 12:49:59 +0100 Subject: [master] 6066a2f Improve Range handling Message-ID: commit 6066a2f39b79bc42cab7991d98e06bfc7db6051a Author: Federico G. Schwindt Date: Mon Dec 8 09:30:09 2014 +0000 Improve Range handling Add support for 416 and fix some corner cases. diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index 2ba4587..be29ece 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -98,14 +98,12 @@ VRG_dorange(struct req *req, struct busyobj *bo, const char *r) else len = ObjGetLen(req->wrk, req->objcore); - if (strncmp(r, "bytes=", 6)) + if (strncasecmp(r, "bytes=", 6)) return; r += 6; /* The low end of range */ has_low = low = 0; - if (!vct_isdigit(*r) && *r != '-') - return; while (vct_isdigit(*r)) { has_low = 1; low *= 10; @@ -113,9 +111,6 @@ VRG_dorange(struct req *req, struct busyobj *bo, const char *r) r++; } - if (low >= len) - return; - if (*r != '-') return; r++; @@ -128,22 +123,31 @@ VRG_dorange(struct req *req, struct busyobj *bo, const char *r) high += *r - '0'; r++; } + if (high < low) + return; if (!has_low) { low = len - high; if (low < 0) low = 0; high = len - 1; - } - } else + } else if (high >= len) + high = len - 1; + } else if (has_low) high = len - 1; - if (*r != '\0') + else return; - if (high >= len) - high = len - 1; + if (*r != '\0') + return; - if (low > high) + if (low >= len) { + http_PrintfHeader(req->resp, "Content-Range: bytes */%jd", + (intmax_t)len); + http_Unset(req->resp, H_Content_Length); + http_PutResponse(req->resp, "HTTP/1.1", 416, NULL); + req->wantbody = 0; return; + } http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/%jd", (intmax_t)low, (intmax_t)high, (intmax_t)len); diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc index 23910de..9eeee15 100644 --- a/bin/varnishtest/tests/c00034.vtc +++ b/bin/varnishtest/tests/c00034.vtc @@ -5,13 +5,11 @@ server s1 { txresp -bodylen 100 } -start -varnish v1 -vcl+backend { - +varnish v1 -arg "-p http_range_support=off" -vcl+backend { sub vcl_backend_response { set beresp.do_stream = false; } } -start -varnish v1 -cliok "param.set http_range_support off" client c1 { txreq -hdr "Range: bytes=0-9" @@ -45,61 +43,71 @@ client c1 { expect resp.status == 200 expect resp.bodylen == 100 + txreq -hdr "Range: bytes=-" + rxresp + expect resp.status == 200 + expect resp.bodylen == 100 + + txreq -hdr "Range: bytes=5-2" + rxresp + expect resp.status == 200 + expect resp.bodylen == 100 } -run -varnish v1 -expect s_resp_bodybytes == 500 +varnish v1 -expect s_resp_bodybytes == 700 client c1 { - - txreq -hdr "Range: bytes=0-9" + txreq -hdr "Range: bytes=-0" rxresp - expect resp.status == 206 - expect resp.bodylen == 10 + expect resp.status == 416 + expect resp.bodylen == 0 + expect resp.http.content-range == "bytes */100" - txreq -hdr "Range: bytes=10-19" + txreq -hdr "Range: bytes=100-" rxresp - expect resp.status == 206 - expect resp.bodylen == 10 + expect resp.status == 416 + expect resp.bodylen == 0 + expect resp.http.content-range == "bytes */100" +} -run - txreq -hdr "Range: bytes=90-" - rxresp - expect resp.status == 206 - expect resp.bodylen == 10 +varnish v1 -expect s_resp_bodybytes == 700 - txreq -hdr "Range: bytes=-9" +client c1 { + txreq -hdr "Range: bytes=0-49" rxresp expect resp.status == 206 - expect resp.bodylen == 9 + expect resp.bodylen == 50 + expect resp.http.content-range == "bytes 0-49/100" - txreq -hdr "Range: bytes=-" + txreq -hdr "Range: bytes=50-99" rxresp expect resp.status == 206 - expect resp.bodylen == 100 - - txreq -hdr "Range: bytes=102-102" - rxresp - expect resp.status == 200 - expect resp.bodylen == 100 + expect resp.bodylen == 50 + expect resp.http.content-range == "bytes 50-99/100" - txreq -hdr "Range: bytes=99-" + txreq -hdr "Range: bytes=-50" rxresp expect resp.status == 206 - expect resp.bodylen == 1 + expect resp.bodylen == 50 + expect resp.http.content-range == "bytes 50-99/100" - txreq -hdr "Range: bytes=99-70" + txreq -hdr "Range: bytes=50-" rxresp - expect resp.status == 200 - expect resp.bodylen == 100 + expect resp.status == 206 + expect resp.bodylen == 50 + expect resp.http.content-range == "bytes 50-99/100" - txreq -hdr "Range: bytes=-" + txreq -hdr "Range: bytes=0-0" rxresp expect resp.status == 206 - expect resp.bodylen == 100 + expect resp.bodylen == 1 + expect resp.http.content-range == "bytes 0-0/100" - txreq -hdr "Range: bytes=-101" + txreq -hdr "Range: bytes=-2000" rxresp expect resp.status == 206 expect resp.bodylen == 100 + expect resp.http.content-range == "bytes 0-99/100" } -run -varnish v1 -expect s_resp_bodybytes == 1040 +varnish v1 -expect s_resp_bodybytes == 1001 From tfheen at err.no Mon Dec 8 13:22:19 2014 From: tfheen at err.no (Tollef Fog Heen) Date: Mon, 08 Dec 2014 14:22:19 +0100 Subject: [master] 635ff46 Minor typo in varnishsizes' -P documentation Message-ID: commit 635ff464330068749e9fc1678a399690edf04351 Author: Tollef Fog Heen Date: Mon Dec 8 14:22:11 2014 +0100 Minor typo in varnishsizes' -P documentation diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index fdd217e..f3223f3 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -47,7 +47,7 @@ #define HIS_OPT_P \ VOPT("P:", "[-P ]", \ "Profile definition", \ - "Either specify \"size\" or \"responstime\" profile or" \ + "Either specify \"size\" or \"responsetime\" profile or" \ " create a new one. Define the tag we'll look for, and the" \ " field number of the value we are interested in. min and" \ " max are the boundaries of the graph (these are power of" \ From tfheen at err.no Mon Dec 8 13:23:18 2014 From: tfheen at err.no (Tollef Fog Heen) Date: Mon, 08 Dec 2014 14:23:18 +0100 Subject: [master] 525885e Typo Message-ID: commit 525885e46d9ad2ea0e414b77c1d0002c76e1e3f5 Author: Tollef Fog Heen Date: Mon Dec 8 14:23:17 2014 +0100 Typo diff --git a/doc/changes.rst b/doc/changes.rst index c638014..f890b90 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1423,7 +1423,7 @@ varnishd varnishsizes -----------~ -- varnishsizes, which is like varnishhost, but for the length of +- varnishsizes, which is like varnishhist, but for the length of objects, has been added.. From fgsch at lodoss.net Mon Dec 8 13:58:41 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 08 Dec 2014 14:58:41 +0100 Subject: [master] 54fdb1c Remove varnishreplay from the index Message-ID: commit 54fdb1c9afd64fdbde3459c67ab07cbba5fbd172 Author: Federico G. Schwindt Date: Mon Dec 8 13:57:22 2014 +0000 Remove varnishreplay from the index Pointed out by geoff. diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst index fa69356..ec73438 100644 --- a/doc/sphinx/reference/index.rst +++ b/doc/sphinx/reference/index.rst @@ -15,7 +15,6 @@ The Varnish Reference Manual varnishhist.rst varnishlog.rst varnishncsa.rst - varnishreplay.rst varnishstat.rst varnishtest.rst varnishtop.rst From fgsch at lodoss.net Mon Dec 8 14:06:38 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 08 Dec 2014 15:06:38 +0100 Subject: [master] 7f48458 And remove rst file as well Message-ID: commit 7f484581eb4818f75f91dd3b66fedb3640dc1b0d Author: Federico G. Schwindt Date: Mon Dec 8 14:06:24 2014 +0000 And remove rst file as well sphinx complains otherwise. diff --git a/doc/sphinx/reference/varnishreplay.rst b/doc/sphinx/reference/varnishreplay.rst deleted file mode 100644 index 884e7b7..0000000 --- a/doc/sphinx/reference/varnishreplay.rst +++ /dev/null @@ -1,50 +0,0 @@ -============= -varnishreplay -============= - ------------------------- -HTTP traffic replay tool ------------------------- - -SYNOPSIS -======== -varnishreplay [-D] -a address:port -r file - -DESCRIPTION -=========== - -The varnishreplay utility parses Varnish logs and attempts to -reproduce the traffic. It is typically used to *warm* up caches or -various forms of testing. - -The following options are available: - --a backend Send the traffic over tcp to this server, specified by an - address and a port. This option is - mandatory. Only IPV4 is supported at this time. - --D Turn on debugging mode. - --r file Parse logs from this file. The input file has to be from - a varnishlog of the same version as the varnishreplay - binary. This option is mandatory. - -SEE ALSO -======== - -* varnishd(1) -* varnishlog(1) - -HISTORY -======= - -The varnishreplay utility and this manual page were written by Cecilie -Fritzvold and later updated by Per Buer. - -COPYRIGHT -========= - -This document is licensed under the same licence as Varnish -itself. See LICENCE for details. - -* Copyright (c) 2007-2014 Varnish Software AS From martin at varnish-software.com Wed Dec 10 10:14:09 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Wed, 10 Dec 2014 11:14:09 +0100 Subject: [master] 47e8292 Extend the type of argument and return value of V1L_Write Message-ID: commit 47e8292088626a59015152c23b3bc45acd579b5b Author: Martin Blix Grydeland Date: Tue Dec 9 15:58:24 2014 +0100 Extend the type of argument and return value of V1L_Write This allows adding up to ssize_t chunks diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 5ad4519..1db9644 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -964,7 +964,7 @@ void V1L_EndChunk(const struct worker *w); void V1L_Reserve(struct worker *, struct ws *, int *fd, struct vsl_log *, double t0); unsigned V1L_Flush(const struct worker *w); unsigned V1L_FlushRelease(struct worker *w); -unsigned V1L_Write(const struct worker *w, const void *ptr, int len); +size_t V1L_Write(const struct worker *w, const void *ptr, ssize_t len); /* cache_range.c [VRG] */ void VRG_dorange(struct req *req, struct busyobj *bo, const char *r); diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c index 271a369..5601ddc 100644 --- a/bin/varnishd/http1/cache_http1_line.c +++ b/bin/varnishd/http1/cache_http1_line.c @@ -221,8 +221,8 @@ V1L_Flush(const struct worker *wrk) return (v1l->werr); } -unsigned -V1L_Write(const struct worker *wrk, const void *ptr, int len) +size_t +V1L_Write(const struct worker *wrk, const void *ptr, ssize_t len) { struct v1l *v1l; From lkarsten at varnish-software.com Wed Dec 10 10:53:42 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Wed, 10 Dec 2014 11:53:42 +0100 Subject: [master] 74c3c3b Sync up with Debian downstream. Message-ID: commit 74c3c3bb037e1c54ed9ebd64da94090587c6f05d Author: Lasse Karstensen Date: Wed Dec 10 11:16:23 2014 +0100 Sync up with Debian downstream. They know this Debian packaging stuff better than us. I've reviewed each of these files manually, but I may have broken something still. Additional review is welcome. diff --git a/compat b/compat index 7f8f011..ec63514 100644 --- a/compat +++ b/compat @@ -1 +1 @@ -7 +9 diff --git a/control b/control index 1c51426..6cc69c2 100644 --- a/control +++ b/control @@ -8,10 +8,11 @@ Uploaders: Stig Sandbeck Mathisen , Tollef Fog Heen , Build-Depends: - autoconf, automake, autotools-dev, - debhelper (>= 7.0.50~), + debhelper (>= 9), + dh-autoreconf, + dh-systemd, libedit-dev, libjemalloc-dev, libncurses-dev, @@ -23,11 +24,15 @@ Build-Depends: Vcs-Browser: http://git.debian.org/?p=pkg-varnish/pkg-varnish.git;a=summary Vcs-Git: git://git.debian.org/pkg-varnish/pkg-varnish.git Homepage: http://varnish-cache.org/ -Standards-Version: 3.9.3 +Standards-Version: 3.9.6 Package: varnish Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends}, gcc (>= 3.3), libc6-dev | libc6.1-dev | libc-dev, adduser, libvarnishapi1 (>= ${binary:Version}) +Depends: ${shlibs:Depends}, ${misc:Depends}, + adduser, + gcc, + libc6-dev | libc6.1-dev | libc-dev, + libvarnishapi1 (>= ${binary:Version}) Suggests: varnish-doc Provides: varnish, ${Varnish:ABI}, ${Varnish:strictABI} Description: state of the art, high-performance web accelerator @@ -59,6 +64,7 @@ Section: libs Architecture: any Provides: libvarnish1 Conflicts: libvarnish1 +Pre-Depends: multiarch-support Depends: ${shlibs:Depends}, ${misc:Depends} Description: shared libraries for Varnish Shared libraries for the Varnish HTTP accelerator. @@ -66,7 +72,7 @@ Description: shared libraries for Varnish Package: libvarnishapi-dev Section: libdevel Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends}, libvarnishapi1 (= ${binary:Version}) +Depends: ${shlibs:Depends}, ${misc:Depends}, libvarnishapi1 (= ${binary:Version}), python Provides: libvarnish-dev Conflicts: libvarnish-dev Description: development files for Varnish diff --git a/libvarnishapi-dev.install b/libvarnishapi-dev.install index 7c5c870..3fe4930 100644 --- a/libvarnishapi-dev.install +++ b/libvarnishapi-dev.install @@ -1,5 +1,6 @@ usr/include -usr/lib/libvarnishapi.so -usr/lib/pkgconfig/*.pc usr/share/aclocal -usr/share/varnish +usr/share/varnish/include/* /usr/include/varnish +usr/share/varnish/vmodtool.py +/usr/lib/*/libvarnishapi.so +/usr/lib/*/pkgconfig/*.pc diff --git a/libvarnishapi1.install b/libvarnishapi1.install index d0dbfd1..d6a64d3 100644 --- a/libvarnishapi1.install +++ b/libvarnishapi1.install @@ -1 +1 @@ -usr/lib/lib*.so.* +/usr/lib/*/lib*.so.* diff --git a/rules b/rules index 20e921b..ebbacac 100755 --- a/rules +++ b/rules @@ -2,9 +2,8 @@ DH_VERBOSE=1 -# List of architectures that lack the *_2POW definitions in varnish, -# which is needed by jemalloc -DISABLE_JEMALLOC_ARCH_LIST := hppa s390 sparc m68k +# List of architectures where jemalloc is not available +DISABLE_JEMALLOC_ARCH_LIST := hppa m68k # Explicitly initialize a variable to select architecture, unless it has been # defined before. This is compared against the DISABLE_*_LIST variables later @@ -27,7 +26,7 @@ VMOD_ABI = $(shell printf '\#include "vrt.h"\nvarnishabi- VRT_MAJOR_VERSION . VR # Main build rule, leave everything to debhelper %: - dh $@ --parallel + dh $@ --parallel --with=systemd,autoreconf ifeq (,$(filter test,$(LOCAL_BUILD_OPTIONS))) # Disable automated build tests @@ -40,11 +39,8 @@ override_dh_auto_configure: override_dh_auto_install: dh_auto_install -a - @ # Install the example as default. - mkdir -p $(CURDIR)/debian/tmp/etc/varnish/ - install -o root -g root -m 644 $(CURDIR)/etc/example.vcl \ - $(CURDIR)/debian/tmp/etc/varnish/default.vcl - rm $(CURDIR)/debian/tmp/usr/share/doc/varnish/*vcl + install -d debian/tmp/etc/varnish + install -T -m 0644 etc/example.vcl debian/tmp/etc/varnish/default.vcl @ # Remove .la files @ # (See http://wiki.debian.org/ReleaseGoals/LAFileRemoval) find $(CURDIR)/debian/ -name \*.la -delete diff --git a/varnish.NEWS b/varnish.NEWS index 9daf13b..cdf0ce0 100644 --- a/varnish.NEWS +++ b/varnish.NEWS @@ -1,3 +1,15 @@ +varnish (4.0.0-1) unstable; urgency=medium + + The syntax of the VCL has changed. If upgrading from a previous version of + varnish, you will need to update your VCL before varnish will start. + + For instructions on rewriting your VCL, please see the file + /usr/share/doc/varnish-doc/html/whats-new/upgrading.html in the varnish-doc + package, or https://www.varnish-cache.org/docs/4.0/whats-new/upgrading.html + for online documentation + + -- Stig Sandbeck Mathisen Thu, 10 Apr 2014 13:53:53 +0200 + varnish (2.1.3-2) lucid; urgency=low Varnish will no longer start by default when installed. Edit diff --git a/varnish.init b/varnish.init index fd8cd4b..b86a4f8 100644 --- a/varnish.init +++ b/varnish.init @@ -20,7 +20,7 @@ NAME=varnishd DESC="HTTP accelerator" PATH=/sbin:/bin:/usr/sbin:/usr/bin DAEMON=/usr/sbin/varnishd -PIDFILE=/var/run/$NAME.pid +PIDFILE=/run/$NAME.pid test -x $DAEMON || exit 0 @@ -73,6 +73,15 @@ stop_varnishd() { else log_end_msg 1 fi + + if test -r $PIDFILE; then + read -r PID < $PIDFILE + if test ! -d /proc/$PID ; then + # stale pidfile + unset PID + rm -f $PIDFILE + fi + fi } reload_varnishd() { @@ -85,10 +94,18 @@ reload_varnishd() { } status_varnishd() { - status_of_proc -p "${PIDFILE}" "${DAEMON}" "${NAME}" + start-stop-daemon \ + --status --quiet --pidfile $PIDFILE \ + --exec $DAEMON exit $? } +configtest() { + $DAEMON ${DAEMON_OPTS} -C -n /tmp > /dev/null +} + + + case "$1" in start) case "${START:-}" in @@ -110,13 +127,20 @@ case "$1" in status_varnishd ;; restart|force-reload) + if status_of_proc -p "${PIDFILE}" "${DAEMON}" "${NAME}" 1>/dev/null; then + if ! configtest; then + log_failure_msg "Syntax check failed, not restarting" + exit 1 + fi + fi $0 stop $0 start ;; + configtest) + configtest && log_success_msg "Syntax ok" + ;; *) - log_success_msg "Usage: $0 {start|stop|restart|force-reload}" + log_success_msg "Usage: $0 {start|stop|restart|reload|force-reload|configtest}" exit 1 ;; esac - -exit 0 diff --git a/varnish.install b/varnish.install index 172753c..580430d 100644 --- a/varnish.install +++ b/varnish.install @@ -1,6 +1,7 @@ etc/varnish/default.vcl usr/bin/* usr/sbin/* -usr/lib/varnish +/usr/lib/*/varnish usr/share/man debian/*.service lib/systemd/system/ +usr/share/doc/varnish/*.vcl diff --git a/varnish.logrotate b/varnish.logrotate index 59ebe1b..010abe0 100644 --- a/varnish.logrotate +++ b/varnish.logrotate @@ -1,4 +1,4 @@ -/var/log/varnish/varnish.log /var/log/varnish/varnishncsa.log { +/var/log/varnish/varnish.log { daily rotate 7 missingok @@ -6,10 +6,18 @@ delaycompress missingok postrotate - for service in varnishlog varnishncsa; do - if /usr/bin/pgrep -P 1 $service >/dev/null; then - /usr/sbin/invoke-rc.d $service reload > /dev/null - fi - done + /usr/sbin/invoke-rc.d varnishlog reload > /dev/null + endscript +} + +/var/log/varnish/varnishncsa.log { + daily + rotate 7 + missingok + compress + delaycompress + missingok + postrotate + /usr/sbin/invoke-rc.d varnishncsa reload > /dev/null endscript } diff --git a/varnish.postinst b/varnish.postinst index 565f916..0bc7d0b 100644 --- a/varnish.postinst +++ b/varnish.postinst @@ -49,12 +49,6 @@ upgrade_change_log_permissions() { chown -Rhf ${log_user}: ${log_dir} } -# varnish version 2.1.3-1 and older started varnishd at boot, we keep -# this default for upgrading clients -upgrade_enable_varnishd() { - sed -i '/^START=/s/no/yes/g' /etc/default/varnish -} - case ${1:-} in configure) @@ -68,7 +62,6 @@ case ${1:-} in if dpkg --compare-versions "2.1.3-2" "gt-nl" "${2:-}" ; then upgrade_change_log_permissions - upgrade_enable_varnishd fi ;; esac diff --git a/varnish.postrm b/varnish.postrm index 8d852fa..026b5c5 100644 --- a/varnish.postrm +++ b/varnish.postrm @@ -9,25 +9,33 @@ case "$1" in remove|failed-upgrade|abort-install|abort-upgrade|disappear) if test -e /var/log/varnish ; then + rm -r /var/log/varnish > /dev/null 2>&1 || exit 78 + fi if test -e /var/lib/varnish; then + rm -r /var/lib/varnish > /dev/null 2>&1 || exit 78 fi + ;; purge) + if test -e /var/log/varnish; then - rm -r /var/log/varnish > /dev/null 2>&1 || exit 78 + + rm -r /var/log/varnish > /dev/null 2>&1 || exit 78 + fi if test -e /var/lib/varnish; then - rm -r /var/lib/varnish > /dev/null 2>&1 || exit 78 + + rm -r /var/lib/varnish > /dev/null 2>&1 || exit 78 fi - - rm -f /etc/varnish/secret - dpkg-statoverride --remove /var/log/varnish || true + + rm -f /etc/varnish/secret + ;; *) diff --git a/varnish.service b/varnish.service index e2eb99b..0e2050a 100644 --- a/varnish.service +++ b/varnish.service @@ -5,7 +5,9 @@ Description=Varnish HTTP accelerator Type=forking LimitNOFILE=131072 LimitMEMLOCK=82000 +ExecStartPre=/usr/sbin/varnishd -C -f /etc/varnish/default.vcl ExecStart=/usr/sbin/varnishd -a :6081 -T localhost:6082 -f /etc/varnish/default.vcl -S /etc/varnish/secret -s malloc,256m +ExecReload=/usr/share/varnish/reload-vcl [Install] WantedBy=multi-user.target diff --git a/varnish.varnishlog.init b/varnish.varnishlog.init index 1574ace..4597934 100644 --- a/varnish.varnishlog.init +++ b/varnish.varnishlog.init @@ -17,7 +17,7 @@ NAME=varnishlog DESC="HTTP accelerator log deamon" PATH=/sbin:/bin:/usr/sbin:/usr/bin DAEMON=/usr/bin/$NAME -PIDFILE=/var/run/$NAME/$NAME.pid +PIDFILE=/run/$NAME/$NAME.pid LOGFILE=/var/log/varnish/varnish.log USER=varnishlog DAEMON_OPTS="-a -w ${LOGFILE} -D -P $PIDFILE" @@ -103,5 +103,3 @@ case "$1" in exit 1 ;; esac - -exit 0 diff --git a/varnish.varnishncsa.default b/varnish.varnishncsa.default index e8ad83e..de36fb4 100644 --- a/varnish.varnishncsa.default +++ b/varnish.varnishncsa.default @@ -1,5 +1,7 @@ # Configuration file for varnishncsa # +# This shell script fragment is sourced by /etc/init.d/varnishncsa +# # Note: If systemd is installed, this file is obsolete and ignored. You will # need to copy /lib/systemd/system/varnishncsa.service to /etc/systemd/system/ # and edit that file. @@ -10,3 +12,8 @@ # # NCSA log format, to be used by HTTP log analyzers # VARNISHNCSA_ENABLED=1 +# +# If you want to add more arguments to varnishncsa, such as providing +# a different log format, you can override the DAEMON_OPTS variable +# from /etc/init.d/varnishncsa here. +# DAEMON_OPTS="-a -w ${LOGFILE} -D -P ${PIDFILE}" diff --git a/varnish.varnishncsa.init b/varnish.varnishncsa.init index 1d098ad..9799d5a 100644 --- a/varnish.varnishncsa.init +++ b/varnish.varnishncsa.init @@ -17,7 +17,7 @@ NAME=varnishncsa DESC="HTTP accelerator log deamon" PATH=/sbin:/bin:/usr/sbin:/usr/bin DAEMON=/usr/bin/$NAME -PIDFILE=/var/run/$NAME/$NAME.pid +PIDFILE=/run/$NAME/$NAME.pid LOGFILE=/var/log/varnish/varnishncsa.log USER=varnishlog DAEMON_OPTS="-a -w ${LOGFILE} -D -P ${PIDFILE}" @@ -103,5 +103,3 @@ case "$1" in exit 1 ;; esac - -exit 0 diff --git a/varnishlog.service b/varnishlog.service index 24cf0de..189aa2e 100644 --- a/varnishlog.service +++ b/varnishlog.service @@ -1,9 +1,11 @@ [Unit] Description=Varnish HTTP accelerator log daemon +After=varnish.service [Service] User=varnishlog ExecStart=/usr/bin/varnishlog -a -w /var/log/varnish/varnish.log +ExecReload=/bin/kill -HUP $MAINPID [Install] WantedBy=multi-user.target diff --git a/varnishncsa.service b/varnishncsa.service index 4542e50..c9fe95c 100644 --- a/varnishncsa.service +++ b/varnishncsa.service @@ -1,9 +1,11 @@ [Unit] Description=Varnish HTTP accelerator log daemon +After=varnish.service [Service] User=varnishlog ExecStart=/usr/bin/varnishncsa -a -w /var/log/varnish/varnishncsa.log +ExecReload=/bin/kill -HUP $MAINPID [Install] WantedBy=multi-user.target diff --git a/watch b/watch index 7f9d001..4ad9629 100644 --- a/watch +++ b/watch @@ -1,3 +1,3 @@ version=3 -opts="uversionmangle=s/-(beta|rc)/~$1/" http://repo.varnish-cache.org/source/varnish-(\d.*)\.tar\.gz +opts="uversionmangle=s/-(beta|rc|tp)/~$1/" http://repo.varnish-cache.org/source/varnish-(\d.*)\.tar\.gz From fgsch at lodoss.net Wed Dec 10 11:14:00 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 10 Dec 2014 12:14:00 +0100 Subject: [master] 09c93b4 Improve systemd service units Message-ID: commit 09c93b40c1820faf564ec7e39bfaccad6ca88343 Author: Federico G. Schwindt Date: Wed Dec 10 11:10:57 2014 +0000 Improve systemd service units Discussed with ingvar. diff --git a/redhat/varnish.service b/redhat/varnish.service index 659dba2..a4f3355 100644 --- a/redhat/varnish.service +++ b/redhat/varnish.service @@ -27,6 +27,7 @@ EnvironmentFile=/etc/varnish/varnish.params Type=forking PIDFile=/var/run/varnish.pid PrivateTmp=true +ExecStartPre=/usr/sbin/varnishd -C -f $VARNISH_VCL_CONF ExecStart=/usr/sbin/varnishd \ -P /var/run/varnish.pid \ -f $VARNISH_VCL_CONF \ diff --git a/redhat/varnishlog.service b/redhat/varnishlog.service index 1e3e274..c7a0193 100644 --- a/redhat/varnishlog.service +++ b/redhat/varnishlog.service @@ -1,6 +1,6 @@ [Unit] Description=Varnish HTTP accelerator logging daemon -After=network.target +After=varnish.service [Service] Type=forking diff --git a/redhat/varnishncsa.service b/redhat/varnishncsa.service index df5f19f..e2ebdcd 100644 --- a/redhat/varnishncsa.service +++ b/redhat/varnishncsa.service @@ -1,6 +1,6 @@ [Unit] -Description=Varnish NCSA logging -After=network.target +Description=Varnish HTTP accelerator NCSA daemon +After=varnish.service [Service] Type=forking From lkarsten at varnish-software.com Wed Dec 10 12:53:41 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Wed, 10 Dec 2014 13:53:41 +0100 Subject: [master] d8b86e8 No systemd on our wheezy systems. Message-ID: commit d8b86e8e72feafbf55998031bc1b7b3ff4b2c2d0 Author: Lasse Karstensen Date: Wed Dec 10 13:52:39 2014 +0100 No systemd on our wheezy systems. That definitively did not work on our wheezy build servers. (also remove extra comma/typo in control) diff --git a/control b/control index 6cc69c2..88eb6a6 100644 --- a/control +++ b/control @@ -12,7 +12,6 @@ Build-Depends: autotools-dev, debhelper (>= 9), dh-autoreconf, - dh-systemd, libedit-dev, libjemalloc-dev, libncurses-dev, @@ -20,7 +19,7 @@ Build-Depends: libtool, pkg-config, python-docutils, - python-sphinx, + python-sphinx Vcs-Browser: http://git.debian.org/?p=pkg-varnish/pkg-varnish.git;a=summary Vcs-Git: git://git.debian.org/pkg-varnish/pkg-varnish.git Homepage: http://varnish-cache.org/ diff --git a/rules b/rules index ebbacac..31f6842 100755 --- a/rules +++ b/rules @@ -26,7 +26,7 @@ VMOD_ABI = $(shell printf '\#include "vrt.h"\nvarnishabi- VRT_MAJOR_VERSION . VR # Main build rule, leave everything to debhelper %: - dh $@ --parallel --with=systemd,autoreconf + dh $@ --parallel with=autoreconf ifeq (,$(filter test,$(LOCAL_BUILD_OPTIONS))) # Disable automated build tests From lkarsten at varnish-software.com Wed Dec 10 13:03:03 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Wed, 10 Dec 2014 14:03:03 +0100 Subject: [master] 7b33c92 Also remove autoreconf. Message-ID: commit 7b33c927fbc956958a77016a85b1a856bb60c422 Author: Lasse Karstensen Date: Wed Dec 10 13:55:42 2014 +0100 Also remove autoreconf. diff --git a/control b/control index 88eb6a6..ae870f6 100644 --- a/control +++ b/control @@ -11,7 +11,6 @@ Build-Depends: automake, autotools-dev, debhelper (>= 9), - dh-autoreconf, libedit-dev, libjemalloc-dev, libncurses-dev, diff --git a/rules b/rules index 31f6842..386a809 100755 --- a/rules +++ b/rules @@ -26,7 +26,7 @@ VMOD_ABI = $(shell printf '\#include "vrt.h"\nvarnishabi- VRT_MAJOR_VERSION . VR # Main build rule, leave everything to debhelper %: - dh $@ --parallel with=autoreconf + dh $@ --parallel ifeq (,$(filter test,$(LOCAL_BUILD_OPTIONS))) # Disable automated build tests From phk at FreeBSD.org Wed Dec 10 14:18:57 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 10 Dec 2014 15:18:57 +0100 Subject: [master] 3eff253 Replace the separate init/fini functions for the compiled VCL with a single "event" function which takes the event as parameter. Message-ID: commit 3eff253ecc2591b7441cf90cbdb8164fc290aa96 Author: Poul-Henning Kamp Date: Wed Dec 10 13:13:37 2014 +0000 Replace the separate init/fini functions for the compiled VCL with a single "event" function which takes the event as parameter. diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 713abe6..d0fc13e 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -217,9 +217,9 @@ VCL_Load(const char *fn, const char *name, struct cli *cli) ctx.handling = &hand; ctx.cli = cli; - if (vcl->conf->init_vcl(&ctx)) { + if (vcl->conf->event_vcl(&ctx, VCL_EVENT_INIT)) { VCLI_Out(cli, "VCL \"%s\" Failed to initialize", name); - vcl->conf->fini_vcl(&ctx); + AZ(vcl->conf->event_vcl(&ctx, VCL_EVENT_FINI)); (void)dlclose(vcl->dlh); FREE_OBJ(vcl); return (1); @@ -229,7 +229,7 @@ VCL_Load(const char *fn, const char *name, struct cli *cli) VCLI_Out(cli, "VCL \"%s\" vcl_init{} failed", name); ctx.method = VCL_MET_FINI; (void)vcl->conf->fini_func(&ctx); - vcl->conf->fini_vcl(&ctx); + AZ(vcl->conf->event_vcl(&ctx, VCL_EVENT_FINI)); (void)dlclose(vcl->dlh); FREE_OBJ(vcl); return (1); @@ -268,7 +268,7 @@ VCL_Nuke(struct vcls *vcl) ctx.handling = &hand; (void)vcl->conf->fini_func(&ctx); assert(hand == VCL_RET_OK); - vcl->conf->fini_vcl(&ctx); + AZ(vcl->conf->event_vcl(&ctx, VCL_EVENT_FINI)); free(vcl->name); (void)dlclose(vcl->dlh); FREE_OBJ(vcl); diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index cf6bd66..e582c0f 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -945,6 +945,12 @@ struct ws; struct cli; struct worker; +enum vcl_event_e { + VCL_EVENT_INIT, + VCL_EVENT_FINI, +}; + +typedef int vcl_event_f(VRT_CTX, enum vcl_event_e); typedef int vcl_init_f(VRT_CTX); typedef void vcl_fini_f(VRT_CTX); typedef int vcl_func_f(VRT_CTX); @@ -992,8 +998,7 @@ struct VCL_conf { const char **srcname; const char **srcbody; - vcl_init_f *init_vcl; - vcl_fini_f *fini_vcl; + vcl_event_f *event_vcl; """) for i in returns: diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 237a07c..9aabb59 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -362,10 +362,21 @@ EmitStruct(const struct vcc *tl) Fc(tl, 0, "\nstatic struct director\t*directors[%d];\n", tl->ndirector); + Fc(tl, 0, "\nstatic int\n"); + Fc(tl, 0, "VGC_Event(VRT_CTX, enum vcl_event_e ev)\n"); + Fc(tl, 0, "{\n"); + Fc(tl, 0, "\tif (ev == VCL_EVENT_INIT)\n"); + Fc(tl, 0, "\t\treturn(VGC_Init(ctx));\n"); + Fc(tl, 0, "\telse if (ev == VCL_EVENT_FINI) {\n"); + Fc(tl, 0, "\t\tVGC_Fini(ctx);\n"); + Fc(tl, 0, "\t\treturn(0);\n"); + Fc(tl, 0, "\t} else\n"); + Fc(tl, 0, "\t\treturn (-1);\n"); + Fc(tl, 0, "}\n"); + Fc(tl, 0, "\nconst struct VCL_conf VCL_conf = {\n"); Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n"); - Fc(tl, 0, "\t.init_vcl = VGC_Init,\n"); - Fc(tl, 0, "\t.fini_vcl = VGC_Fini,\n"); + Fc(tl, 0, "\t.event_vcl = VGC_Event,\n"); Fc(tl, 0, "\t.ndirector = %d,\n", tl->ndirector); Fc(tl, 0, "\t.director = directors,\n"); Fc(tl, 0, "\t.ref = VGC_ref,\n"); From phk at FreeBSD.org Wed Dec 10 14:18:57 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 10 Dec 2014 15:18:57 +0100 Subject: [master] 263bb06 Expand inifin mechanism to also handle events Message-ID: commit 263bb0643ad99ba9a0cedc3e4e000a2ad74f24a4 Author: Poul-Henning Kamp Date: Wed Dec 10 13:27:44 2014 +0000 Expand inifin mechanism to also handle events diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 9aabb59..32dc8f2 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -134,6 +134,7 @@ New_IniFin(struct vcc *tl) p->magic = INIFIN_MAGIC; p->ini = VSB_new_auto(); p->fin = VSB_new_auto(); + p->event = VSB_new_auto(); p->n = ++tl->ninifin; VTAILQ_INSERT_TAIL(&tl->inifin, p, list); return (p); @@ -295,13 +296,22 @@ LocTable(const struct vcc *tl) Fc(tl, 0, "};\n"); } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * Init/Fini/Event + * + * We call Fini-s in the opposite order of init-s. + * Other events are called in same order as init-s, no matter which + * event it might be. + */ static void -EmitInitFunc(const struct vcc *tl) +EmitInitFini(const struct vcc *tl) { struct inifin *p; + /* + * INIT + */ Fc(tl, 0, "\nstatic int\nVGC_Init(VRT_CTX)\n{\n\n"); VTAILQ_FOREACH(p, &tl->inifin, list) { AZ(VSB_finish(p->ini)); @@ -312,14 +322,11 @@ EmitInitFunc(const struct vcc *tl) Fc(tl, 0, "\treturn(0);\n"); Fc(tl, 0, "}\n"); -} - -static void -EmitFiniFunc(const struct vcc *tl) -{ - struct inifin *p; - Fc(tl, 0, "\nstatic void\nVGC_Fini(VRT_CTX)\n{\n\n"); + /* + * FINI + */ + Fc(tl, 0, "\nstatic int\nVGC_Fini(VRT_CTX)\n{\n\n"); VTAILQ_FOREACH_REVERSE(p, &tl->inifin, inifinhead, list) { AZ(VSB_finish(p->fin)); @@ -328,6 +335,28 @@ EmitFiniFunc(const struct vcc *tl) VSB_delete(p->fin); } + Fc(tl, 0, "\treturn(0);\n"); + Fc(tl, 0, "}\n"); + + /* + * EVENTS + */ + Fc(tl, 0, "\nstatic int\n"); + Fc(tl, 0, "VGC_Event(VRT_CTX, enum vcl_event_e ev)\n"); + Fc(tl, 0, "{\n"); + Fc(tl, 0, "\tif (ev == VCL_EVENT_INIT)\n"); + Fc(tl, 0, "\t\treturn(VGC_Init(ctx));\n"); + Fc(tl, 0, "\telse if (ev == VCL_EVENT_FINI)\n"); + Fc(tl, 0, "\t\treturn(VGC_Fini(ctx));\n"); + Fc(tl, 0, "\telse {\n"); + VTAILQ_FOREACH(p, &tl->inifin, list) { + AZ(VSB_finish(p->event)); + if (VSB_len(p->event)) + Fc(tl, 0, "\t/* %u */\n%s\n", p->n, VSB_data(p->event)); + VSB_delete(p->event); + } + Fc(tl, 0, "\t\treturn (0);\n"); + Fc(tl, 0, "\t}\n"); Fc(tl, 0, "}\n"); } @@ -362,18 +391,6 @@ EmitStruct(const struct vcc *tl) Fc(tl, 0, "\nstatic struct director\t*directors[%d];\n", tl->ndirector); - Fc(tl, 0, "\nstatic int\n"); - Fc(tl, 0, "VGC_Event(VRT_CTX, enum vcl_event_e ev)\n"); - Fc(tl, 0, "{\n"); - Fc(tl, 0, "\tif (ev == VCL_EVENT_INIT)\n"); - Fc(tl, 0, "\t\treturn(VGC_Init(ctx));\n"); - Fc(tl, 0, "\telse if (ev == VCL_EVENT_FINI) {\n"); - Fc(tl, 0, "\t\tVGC_Fini(ctx);\n"); - Fc(tl, 0, "\t\treturn(0);\n"); - Fc(tl, 0, "\t} else\n"); - Fc(tl, 0, "\t\treturn (-1);\n"); - Fc(tl, 0, "}\n"); - Fc(tl, 0, "\nconst struct VCL_conf VCL_conf = {\n"); Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n"); Fc(tl, 0, "\t.event_vcl = VGC_Event,\n"); @@ -696,9 +713,7 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp) LocTable(tl); - EmitInitFunc(tl); - - EmitFiniFunc(tl); + EmitInitFini(tl); EmitStruct(tl); diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 128916b..9bc102c 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -150,6 +150,7 @@ struct inifin { unsigned n; struct vsb *ini; struct vsb *fin; + struct vsb *event; VTAILQ_ENTRY(inifin) list; }; From fgsch at lodoss.net Mon Dec 15 17:50:17 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 15 Dec 2014 18:50:17 +0100 Subject: [master] 938e872 Add STRING + REAL support Message-ID: commit 938e8724da75740ee037ffe08bae60bebc83b277 Author: Federico G. Schwindt Date: Sun Dec 14 17:59:42 2014 +0000 Add STRING + REAL support diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index f9be7ec..a9e9d3b 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -97,6 +97,7 @@ varnish v1 -vcl { sub vcl_recv { set req.http.foo = "foo" + "bar"; set req.http.foo = "foo" + 1; + set req.http.foo = "foo" + 1.5; set req.http.foo = "foo" + now; set req.http.foo = now + 1s; diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 49214cc..1e46792 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -785,6 +785,7 @@ vcc_expr4(struct vcc *tl, struct expr **e, enum var_type fmt) const char *ip; const struct symbol *sym; double d; + int i; *e = NULL; if (tl->t->tok == '(') { @@ -878,9 +879,15 @@ vcc_expr4(struct vcc *tl, struct expr **e, enum var_type fmt) } else if (fmt == REAL) { e1 = vcc_mk_expr(REAL, "%f", vcc_DoubleVal(tl)); ERRCHK(tl); - } else { + } else if (fmt == INT) { e1 = vcc_mk_expr(INT, "%.*s", PF(tl->t)); vcc_NextToken(tl); + } else { + vcc_NumVal(tl, &d, &i); + if (i) + e1 = vcc_mk_expr(REAL, "%f", d); + else + e1 = vcc_mk_expr(INT, "%ld", (long)d); } e1->constant = EXPR_CONST; *e = e1; From fgsch at lodoss.net Mon Dec 15 17:50:17 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 15 Dec 2014 18:50:17 +0100 Subject: [master] 65b1900 Permit to compare 2 times using standard operators Message-ID: commit 65b1900514b23ae859b53f5ee4a4884f2da36b5a Author: Federico G. Schwindt Date: Wed Dec 10 20:34:07 2014 +0000 Permit to compare 2 times using standard operators Useful with upcoming std function. diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 1e46792..7649e3e 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1104,6 +1104,7 @@ static const struct cmps { NUM_REL(DURATION), NUM_REL(BYTES), NUM_REL(REAL), + NUM_REL(TIME), {STRING, T_EQ, "!VRT_strcmp(\v1, \v2)" }, {STRING, T_NEQ, "VRT_strcmp(\v1, \v2)" }, From phk at FreeBSD.org Tue Dec 16 09:55:55 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Dec 2014 10:55:55 +0100 Subject: [master] 3dc48ce Continuously update our total object size estimate when cond-fetching a still-being-streamed object. Message-ID: commit 3dc48cec578ab1e9f99706b3508973f6b676b8e4 Author: Poul-Henning Kamp Date: Tue Dec 16 09:55:19 2014 +0000 Continuously update our total object size estimate when cond-fetching a still-being-streamed object. Fixes #1646 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index c44992b..f74a5c0 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -665,7 +665,6 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) void *sp; ssize_t sl, al, l; uint8_t *ptr; - uint64_t ol; enum objiter_status ois; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -686,12 +685,12 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) } al = 0; - ol = ObjGetLen(bo->wrk, bo->ims_oc); oi = ObjIterBegin(wrk, bo->ims_oc); do { ois = ObjIter(bo->ims_oc, oi, &sp, &sl); while (sl > 0) { - l = ol - al; + l = ObjGetLen(bo->wrk, bo->ims_oc) - al; + assert(l > 0); if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK) break; if (sl < l) @@ -710,7 +709,6 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) if (!bo->do_stream) HSH_Unbusy(wrk, bo->fetch_objcore); - assert(al == ol); assert(ObjGetLen(bo->wrk, bo->fetch_objcore) == al); EXP_Rearm(bo->ims_oc, bo->ims_oc->exp.t_origin, 0, 0, 0); diff --git a/bin/varnishtest/tests/r01646.vtc b/bin/varnishtest/tests/r01646.vtc new file mode 100644 index 0000000..951350e --- /dev/null +++ b/bin/varnishtest/tests/r01646.vtc @@ -0,0 +1,54 @@ +varnishtest "cond/stream race ?" + +server s1 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" -hdr "Etag: foo" + chunkedlen 32 + sema r1 sync 3 + chunkedlen 32 + sema r2 sync 2 + chunkedlen 32 + chunkedlen 0 +} -start + +server s2 { + rxreq + expect req.http.if-none-match == "foo" + txresp -status 304 +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_fetch { + if (bereq.http.foo == "s2") { + set bereq.backend = s2; + } + } + sub vcl_backend_response { + set beresp.ttl = 1s; + set beresp.grace = 0s; + set beresp.keep = 30s; + } +} -start + +varnish v1 -cliok "param.set debug +syncvsl" + +client c1 { + txreq + rxresphdrs + sema r1 sync 3 + rxrespbody + expect resp.bodylen == 96 +} -start + +client c2 { + sema r1 sync 3 + delay 2 + txreq -hdr "foo: s2" + rxresphdrs + sema r2 sync 2 + rxrespbody + expect resp.bodylen == 96 +} -start + +client c1 -wait +client c2 -wait From martin at varnish-software.com Tue Dec 16 10:15:00 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 16 Dec 2014 11:15:00 +0100 Subject: [master] f0c2d16 Fix indenting in pan_wrk part of panic message Message-ID: commit f0c2d1687a0c518940144ef299a7abefc2b41ef7 Author: Martin Blix Grydeland Date: Mon Dec 15 13:27:12 2014 +0100 Fix indenting in pan_wrk part of panic message diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index d82e43f..0a3bf1a 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -284,7 +284,7 @@ pan_wrk(const struct worker *wrk) pan_ws(wrk->aws, 4); m = wrk->cur_method; - VSB_printf(pan_vsp, " VCL::method = "); + VSB_printf(pan_vsp, " VCL::method = "); if (m == 0) { VSB_printf(pan_vsp, "none,\n"); return; @@ -299,10 +299,10 @@ pan_wrk(const struct worker *wrk) VSB_printf(pan_vsp, "0x%x,\n", m); hand = VCL_Return_Name(wrk->handling); if (hand != NULL) - VSB_printf(pan_vsp, " VCL::return = %s,\n", hand); + VSB_printf(pan_vsp, " VCL::return = %s,\n", hand); else - VSB_printf(pan_vsp, " VCL::return = 0x%x,\n", wrk->handling); - VSB_printf(pan_vsp, " VCL::methods = {"); + VSB_printf(pan_vsp, " VCL::return = 0x%x,\n", wrk->handling); + VSB_printf(pan_vsp, " VCL::methods = {"); m = wrk->seen_methods; p = ""; for (u = 1; m ; u <<= 1) { From martin at varnish-software.com Tue Dec 16 10:15:00 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 16 Dec 2014 11:15:00 +0100 Subject: [master] e0433a7 Add a missing ',' in pan_req panic string Message-ID: commit e0433a775d08ff65beeb348134df6e6199e163b5 Author: Martin Blix Grydeland Date: Mon Dec 15 13:27:31 2014 +0100 Add a missing ',' in pan_req panic string diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 0a3bf1a..476b252 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -389,7 +389,7 @@ pan_req(const struct req *req) " err_code = %d, err_reason = %s,\n", req->err_code, req->err_reason ? req->err_reason : "(null)"); - VSB_printf(pan_vsp, " restarts = %d, esi_level = %d\n", + VSB_printf(pan_vsp, " restarts = %d, esi_level = %d,\n", req->restarts, req->esi_level); if (req->sp != NULL) From phk at FreeBSD.org Tue Dec 16 10:43:28 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Dec 2014 11:43:28 +0100 Subject: [master] bfe5df8 If we cannot find nobody/nogroup, lookup current process uid/gid. Message-ID: commit bfe5df840f28a01b0ca9f69d58e4911284b66ee9 Author: Poul-Henning Kamp Date: Tue Dec 16 10:42:19 2014 +0000 If we cannot find nobody/nogroup, lookup current process uid/gid. If that fails to, bail at ARGV_ERR level. Fixes #1597 diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 6c6ba8a..10b037e 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -345,16 +345,40 @@ static void init_params(struct cli *cli) { ssize_t def, low; + struct passwd *pwd; + struct group *grp; MCF_CollectParams(); MCF_TcpParams(); - /* If we have nobody/nogroup, use them as defaults */ - if (getpwnam("nobody") != NULL) + /* + * If we have nobody/nogroup, use them as defaults for sandboxes, + * else fall back to whoever we run as. + */ + if (getpwnam("nobody") != NULL) { MCF_SetDefault("user", "nobody"); - if (getgrnam("nogroup") != NULL) + } else { + pwd = getpwuid(getuid()); + if (pwd == NULL) + ARGV_ERR("Neither user 'nobody' or my uid (%jd)" + " found in password database.\n", + (intmax_t)getuid()); + MCF_SetDefault("user", pwd->pw_name); + } + endpwent(); + + if (getgrnam("nogroup") != NULL) { MCF_SetDefault("group", "nogroup"); + } else { + grp = getgrgid(getgid()); + if (grp == NULL) + ARGV_ERR("Neither group 'nogroup' or my gid (%jd)" + " found in password database.\n", + (intmax_t)getgid()); + MCF_SetDefault("group", grp->gr_name); + } + endgrent(); if (sizeof(void *) < 8) { /* @@ -432,7 +456,6 @@ main(int argc, char * const *argv) /* for ASSERT_MGT() */ mgt_pid = getpid(); - /* * Run in UTC timezone, on the off-chance that this operating * system does not have a timegm() function, and translates diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index abfd86a..c8b7fe9 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -337,7 +337,7 @@ MCF_ParamSet(struct cli *cli, const char *param, const char *val) *heritage.param = mgt_param; if (cli->result != CLIS_OK) { - VCLI_Out(cli, "\n(attempting to set param %s to %s)", + VCLI_Out(cli, "\n(attempting to set param '%s' to '%s')", pp->name, val); } else if (child_pid >= 0 && pp->flags & MUST_RESTART) { VCLI_Out(cli, diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c index e757ec6..699fc79 100644 --- a/bin/varnishd/mgt/mgt_param_tweak.c +++ b/bin/varnishd/mgt/mgt_param_tweak.c @@ -384,17 +384,14 @@ tweak_user(struct vsb *vsb, const struct parspec *par, const char *arg) (void)par; if (arg != NULL) { - if (*arg != '\0') { - pw = getpwnam(arg); - if (pw == NULL) { - VSB_printf(vsb, "Unknown user"); - return(-1); - } - REPLACE(mgt_param.user, pw->pw_name); - mgt_param.uid = pw->pw_uid; - } else { - mgt_param.uid = getuid(); + pw = getpwnam(arg); + if (pw == NULL) { + VSB_printf(vsb, "Unknown user '%s'", arg); + return(-1); } + REPLACE(mgt_param.user, pw->pw_name); + mgt_param.uid = pw->pw_uid; + endpwent(); } else if (mgt_param.user) { VSB_printf(vsb, "%s (%d)", mgt_param.user, (int)mgt_param.uid); } else { @@ -414,17 +411,14 @@ tweak_group(struct vsb *vsb, const struct parspec *par, const char *arg) (void)par; if (arg != NULL) { - if (*arg != '\0') { - gr = getgrnam(arg); - if (gr == NULL) { - VSB_printf(vsb, "Unknown group"); - return(-1); - } - REPLACE(mgt_param.group, gr->gr_name); - mgt_param.gid = gr->gr_gid; - } else { - mgt_param.gid = getgid(); + gr = getgrnam(arg); + if (gr == NULL) { + VSB_printf(vsb, "Unknown group '%s'", arg); + return(-1); } + REPLACE(mgt_param.group, gr->gr_name); + mgt_param.gid = gr->gr_gid; + endgrent(); } else if (mgt_param.group) { VSB_printf(vsb, "%s (%d)", mgt_param.group, (int)mgt_param.gid); } else { From fgsch at lodoss.net Tue Dec 16 14:41:36 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 16 Dec 2014 15:41:36 +0100 Subject: [master] 60c98eb Add missing T mandated by ISO 8601 Message-ID: commit 60c98eb04e58fe2c3f51fefcba9b138abe2111de Author: Federico G. Schwindt Date: Mon Dec 15 23:17:33 2014 +0000 Add missing T mandated by ISO 8601 diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 50033c2..28b5f13 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -32,13 +32,14 @@ * In the highly unlikely event of performance trouble, handbuilt versions * would likely be faster than relying on the OS time functions. * - * We must parse three different formats: + * We must parse four different formats: * 000000000011111111112222222222 * 012345678901234567890123456789 * ------------------------------ * "Sun, 06 Nov 1994 08:49:37 GMT" RFC822 & RFC1123 * "Sunday, 06-Nov-94 08:49:37 GMT" RFC850 * "Sun Nov 6 08:49:37 1994" ANSI-C asctime() + * "1994-11-06T08:49:37" ISO 8601 * * And always output the RFC1123 format. * @@ -115,7 +116,7 @@ static const char *fmts[] = { "%a, %d %b %Y %T GMT", /* RFC 822 & RFC 1123 */ "%A, %d-%b-%y %T GMT", /* RFC 850 */ "%a %b %d %T %Y", /* ANSI-C asctime() */ - "%F %T", /* ISO 8601 */ + "%FT%T", /* ISO 8601 */ NULL }; From fgsch at lodoss.net Tue Dec 16 14:41:37 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 16 Dec 2014 15:41:37 +0100 Subject: [master] b916400 Add std.time() Message-ID: commit b916400776b26d6cac50add9b674a1cc97bc8ebc Author: Federico G. Schwindt Date: Tue Dec 16 11:05:56 2014 +0000 Add std.time() Converts an date in various formats to a VCL_TIME. diff --git a/bin/varnishtest/tests/m00020.vtc b/bin/varnishtest/tests/m00020.vtc new file mode 100644 index 0000000..3611cc4 --- /dev/null +++ b/bin/varnishtest/tests/m00020.vtc @@ -0,0 +1,40 @@ +varnishtest "Test std.time" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + import ${vmod_std}; + + sub vcl_deliver { + if (std.time(req.http.x-date, now) < now - 1y) { + set resp.http.x-past = 1; + } + if (std.time(req.http.x-date, now) > now + 1y) { + set resp.http.x-future = 1; + } + } +} -start + +client c1 { + txreq -hdr "X-Date: Mon, 20 Dec 2010 00:00:00 GMT" + rxresp + expect resp.http.x-past == 1 + txreq -hdr "X-Date: Monday, 20-Dec-30 00:00:00 GMT" + rxresp + expect resp.http.x-future == 1 + txreq -hdr "X-Date: Mon Dec 20 00:00:00 2010" + rxresp + expect resp.http.x-past == 1 + txreq -hdr "X-Date: 2030-12-20T00:00:00" + rxresp + expect resp.http.x-future == 1 + txreq -hdr "X-Date: 1292803200.00" + rxresp + expect resp.http.x-past == 1 + txreq -hdr "X-Date: 1923955200" + rxresp + expect resp.http.x-future == 1 +} -run diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index 5f21332..627ec14 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -227,6 +227,25 @@ Example | ... | } +$Function TIME time(STRING s, TIME fallback) + +Description + Converts the string *s* to a time. If conversion fails, + *fallback* will be returned. + + Supported formats: + + | "Sun, 06 Nov 1994 08:49:37 GMT" + | "Sunday, 06-Nov-94 08:49:37 GMT" + | "Sun Nov 6 08:49:37 1994" + | "1994-11-06T08:49:37" + | "784111777.00" + | "784111777" +Example + | if (std.time(resp.http.last-modified, now) < now - 1w) { + | ... + | } + SEE ALSO ======== diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index d050f43..d27ad44 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -41,6 +41,7 @@ #include "vrt.h" #include "vsa.h" +#include "vtim.h" #include "vcc_if.h" VCL_DURATION __match_proto__(td_std_duration) @@ -219,3 +220,18 @@ vmod_time2real(VRT_CTX, VCL_TIME t) return (t); } + +VCL_TIME __match_proto__(td_std_time) +vmod_time(VRT_CTX, VCL_STRING p, VCL_TIME d) +{ + double r; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + if (p == NULL) + return (d); + r = VTIM_parse(p); + if (r) + return (r); + return (vmod_real(ctx, p, d)); +} From fgsch at lodoss.net Tue Dec 16 14:41:37 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 16 Dec 2014 15:41:37 +0100 Subject: [master] de8cf10 Clarify comment Message-ID: commit de8cf10fc665b74692b56d630ab74eec4032f2bc Author: Federico G. Schwindt Date: Tue Dec 16 14:30:30 2014 +0000 Clarify comment diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 89949e4..3d66fe5 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -89,7 +89,7 @@ RFC2616_Ttl(struct busyobj *bo, double now) /* * Initial cacheability determination per [RFC2616, 13.4] - * We do not support ranges yet, so 206 is out. + * We do not support ranges to the backend yet, so 206 is out. */ if (http_GetHdr(hp, H_Age, &p)) { From phk at FreeBSD.org Tue Dec 16 21:36:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Dec 2014 22:36:45 +0100 Subject: [master] d057bf9 Nils patch for retrying partial fetches, with minor changes by me. Message-ID: commit d057bf951796de1acc0359613ebb50e4fc891a0f Author: Poul-Henning Kamp Date: Tue Dec 16 21:35:15 2014 +0000 Nils patch for retrying partial fetches, with minor changes by me. Fixes #1638 Fixes #1641 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index f74a5c0..a5b083f 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -213,17 +213,37 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo) /*-------------------------------------------------------------------- * Start a new VSL transaction and try again + * Prepare the busyobj and fetch processors */ static enum fetch_step vbf_stp_retry(struct worker *wrk, struct busyobj *bo) { + struct vfp_ctx *vfc; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + vfc = bo->vfc; + CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC); + + assert(bo->state == BOS_REQ_DONE); VSLb_ts_busyobj(bo, "Retry", W_TIM_real(wrk)); + /* VDI_Finish must have been called before */ + assert(bo->director_state == DIR_S_NULL); + bo->doclose = SC_NULL; + + /* reset other bo attributes - See VBO_GetBusyObj */ + bo->storage_hint = NULL; + bo->do_esi = 0; + bo->do_stream = 1; + + /* reset fetch processors */ + vfc->failed = 0; + VFP_Close(vfc); + VFP_Setup(vfc); + // XXX: BereqEnd + BereqAcct ? VSL_ChgId(bo->vsl, "bereq", "retry", VXID_Get(wrk, VSL_BACKENDMARKER)); VSLb_ts_busyobj(bo, "Start", bo->t_prev); @@ -405,11 +425,12 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) if (wrk->handling == VCL_RET_RETRY) { bo->doclose = SC_RESP_CLOSE; - VDI_Finish(bo->director_resp, bo->wrk, bo); - bo->doclose = SC_NULL; - bo->retries++; - if (bo->retries <= cache_param->max_retries) + if (bo->director_state != DIR_S_NULL) + VDI_Finish(bo->director_resp, bo->wrk, bo); + + if (bo->retries++ < cache_param->max_retries) return (F_STP_RETRY); + VSLb(bo->vsl, SLT_VCL_Error, "Too many retries, delivering 503"); assert(bo->director_state == DIR_S_NULL); @@ -608,7 +629,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) assert(bo->refcount >= 1); - assert (bo->state == BOS_REQ_DONE); + assert(bo->state == BOS_REQ_DONE); if (bo->do_stream) { HSH_Unbusy(wrk, bo->fetch_objcore); @@ -766,8 +787,14 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) if (wrk->handling == VCL_RET_RETRY) { VSB_delete(bo->synth_body); bo->synth_body = NULL; + + bo->doclose = SC_RESP_CLOSE; + if (bo->director_state != DIR_S_NULL) + VDI_Finish(bo->director_resp, bo->wrk, bo); + if (bo->retries++ < cache_param->max_retries) return (F_STP_RETRY); + return (F_STP_FAIL); } diff --git a/bin/varnishtest/tests/r01638.vtc b/bin/varnishtest/tests/r01638.vtc new file mode 100644 index 0000000..2ddebbb --- /dev/null +++ b/bin/varnishtest/tests/r01638.vtc @@ -0,0 +1,49 @@ +varnishtest "Test retry of straight insufficient bytes pass-fetch : do_stream = false" + +server s1 { + rxreq + txresp -hdr "Content-Length: 10000" -nolen -bodylen 5000 +} + +server s2 { + rxreq + txresp -bodylen 5 +} + +server s1 -start +server s2 -start + +varnish v1 -vcl+backend { + sub vcl_recv { + return (pass); + } + sub vcl_backend_fetch { + set bereq.between_bytes_timeout = 10s; + if (bereq.retries == 0) { + set bereq.backend = s1; + } else { + set bereq.backend = s2; + } + } + sub vcl_backend_response { + set beresp.do_stream = false; + } + sub vcl_backend_error { + if (bereq.retries == 0) { + return (retry); + } else { + return (deliver); + } + } +} -start + +client c1 { + timeout 10 + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.bodylen == 5 +} -run + +server s1 -wait +server s2 -wait diff --git a/bin/varnishtest/tests/r01641.vtc b/bin/varnishtest/tests/r01641.vtc new file mode 100644 index 0000000..1dddc85 --- /dev/null +++ b/bin/varnishtest/tests/r01641.vtc @@ -0,0 +1,49 @@ +varnishtest "Test retry of straight insufficient bytes pass-fetch : do_esi = true" + +server s1 { + rxreq + txresp -hdr "Content-Length: 10000" -nolen -bodylen 5000 +} + +server s2 { + rxreq + txresp -bodylen 5 +} + +server s1 -start +server s2 -start + +varnish v1 -vcl+backend { + sub vcl_recv { + return (pass); + } + sub vcl_backend_fetch { + set bereq.between_bytes_timeout = 10s; + if (bereq.retries == 0) { + set bereq.backend = s1; + } else { + set bereq.backend = s2; + } + } + sub vcl_backend_response { + set beresp.do_esi = true; + } + sub vcl_backend_error { + if (bereq.retries == 0) { + return (retry); + } else { + return (deliver); + } + } +} -start + +client c1 { + timeout 10 + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.bodylen == 5 +} -run + +server s1 -wait +server s2 -wait From martin at varnish-software.com Wed Dec 17 12:50:26 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Wed, 17 Dec 2014 13:50:26 +0100 Subject: [master] 298ffb6 Relax an assertion for the IMS update candidate object Message-ID: commit 298ffb63d3fb0c3e4a683acba0d9639d6901a655 Author: Martin Blix Grydeland Date: Tue Dec 16 18:05:54 2014 +0100 Relax an assertion for the IMS update candidate object The streaming fetch of an object can fail between lookup time and when we actually use it. Relax this assertion. Fixes: #1647 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 1078d34..b2e5aef 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -356,7 +356,7 @@ cnt_lookup(struct worker *wrk, struct req *req) switch (wrk->handling) { case VCL_RET_DELIVER: if (boc != NULL) { - AZ(oc->flags & (OC_F_FAILED|OC_F_PASS)); + AZ(oc->flags & OC_F_PASS); AZ(oc->exp_flags & OC_EF_DYING); AZ(boc->busyobj); VBF_Fetch(wrk, req, boc, oc, VBF_BACKGROUND); From martin at varnish-software.com Wed Dec 17 12:50:26 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Wed, 17 Dec 2014 13:50:26 +0100 Subject: [master] dba9daa Handle template object failures during condfetch Message-ID: commit dba9daad80db5cfefa43890fc463fbd42f4ee825 Author: Martin Blix Grydeland Date: Tue Dec 16 18:14:57 2014 +0100 Handle template object failures during condfetch Fixes: #1648 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index a5b083f..93d6ea7 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -709,6 +709,8 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) oi = ObjIterBegin(wrk, bo->ims_oc); do { ois = ObjIter(bo->ims_oc, oi, &sp, &sl); + if (ois == OIS_ERROR) + VFP_Error(bo->vfc, "Template object failed"); while (sl > 0) { l = ObjGetLen(bo->wrk, bo->ims_oc) - al; assert(l > 0); @@ -724,8 +726,12 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) } } while (!bo->vfc->failed && (ois == OIS_DATA || ois == OIS_STREAM)); ObjIterEnd(bo->ims_oc, &oi); - if (bo->vfc->failed) + if (bo->ims_oc->flags & OC_F_FAILED) + VFP_Error(bo->vfc, "Template object failed"); + if (bo->vfc->failed) { + VDI_Finish(bo->director_resp, bo->wrk, bo); return (F_STP_FAIL); + } if (!bo->do_stream) HSH_Unbusy(wrk, bo->fetch_objcore); diff --git a/bin/varnishtest/tests/r01648.vtc b/bin/varnishtest/tests/r01648.vtc new file mode 100644 index 0000000..de0c364 --- /dev/null +++ b/bin/varnishtest/tests/r01648.vtc @@ -0,0 +1,75 @@ +varnishtest "#1648 - corrupt object in cache through IMS update" + +# This server sends a broken response body +server s1 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" -hdr "Etag: \"foo\"" -hdr "Server: s1" + sema r1 sync 3 + delay 1 + chunked "abc" +} -start + +# This server validates the streaming response from s1 as it hasn't failed yet +server s2 { + rxreq + expect req.http.If-None-Match == "\"foo\"" + sema r1 sync 3 + txresp -status 304 -nolen -hdr "Server: s2" +} -start + +# This server sends the proper response body +server s3 { + rxreq + txresp -hdr "Server: s3" -body "abcdef" +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.http.client == "c1") { + set req.backend_hint = s1; + } else if (req.http.client == "c2") { + set req.backend_hint = s2; + } else if (req.http.client == "c3") { + set req.backend_hint = s3; + } + } + sub vcl_backend_response { + if (bereq.http.client == "c1") { + set beresp.ttl = 0.1s; + set beresp.grace = 0s; + set beresp.keep = 60s; + } + } +} -start + +varnish v1 -cliok "param.set debug +syncvsl" + +# This client gets a streaming failing result from s1 +client c1 { + txreq -hdr "Client: c1" + rxresphdrs + expect resp.status == 200 + expect resp.http.transfer-encoding == "chunked" +} -start + +delay 1 + +# This client gets a streaming failing result from s1 through +# IMS update by s2 +client c2 { + txreq -hdr "Client: c2" + sema r1 sync 3 + rxresphdrs + expect resp.status == 200 + expect resp.http.transfer-encoding == "chunked" +} -run + +delay 1 + +# This client should get a fresh fetch from s3 +client c3 { + txreq -hdr "Client: c3" + rxresp + expect resp.status == 200 + expect resp.body == "abcdef" +} -run From arianna.aondio at varnish-software.com Wed Dec 17 15:06:13 2014 From: arianna.aondio at varnish-software.com (arianna-aondio) Date: Wed, 17 Dec 2014 16:06:13 +0100 Subject: [master] d68863d Varnishncsa escapes non-printable chars. Message-ID: commit d68863d0d983685a286347824046f791bba58c55 Author: arianna-aondio Date: Wed Dec 17 15:57:43 2014 +0100 Varnishncsa escapes non-printable chars. Fixes: #1378 diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 162829d..dcda6e5 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -182,16 +182,61 @@ flushout(void) return (0); } +static int +vsb_esc_cat(struct vsb *sb, const char *b, const char *e) +{ + AN(b); + + for (; b < e; b++) { + if (isspace(*b)) { + switch (*b) { + case '\n': + VSB_cat(sb, "\\n"); + break; + case '\t': + VSB_cat(sb, "\\t"); + break; + case '\f': + VSB_cat(sb, "\\f"); + break; + case '\r': + VSB_cat(sb, "\\r"); + break; + case '\v': + VSB_cat(sb, "\\v"); + break; + default: + VSB_putc(sb, *b); + break; + } + } else if (isprint(*b)) { + switch (*b) { + case '"': + VSB_cat(sb, "\\\""); + break; + case '\\': + VSB_cat(sb, "\\\\"); + break; + default: + VSB_putc(sb, *b); + break; + } + } else + VSB_printf(sb, "\\x%02x", *b); + } + + return (VSB_error(sb)); +} + static inline int vsb_fcat(struct vsb *vsb, const struct fragment *f, const char *dflt) { - if (f->gen == CTX.gen) { assert(f->b <= f->e); - return (VSB_bcat(vsb, f->b, f->e - f->b)); + return (vsb_esc_cat(vsb, f->b, f->e)); } if (dflt) - return (VSB_cat(vsb, dflt)); + return (vsb_esc_cat(vsb, dflt, dflt + strlen(dflt))); return (-1); } @@ -226,10 +271,10 @@ format_fragment(const struct format *format) if (format->frag->gen != CTX.gen) { if (format->string == NULL) return (-1); - AZ(VSB_cat(CTX.vsb, format->string)); + AZ(vsb_esc_cat(CTX.vsb, format->string, + format->string + strlen(format->string))); return (0); } - AZ(vsb_fcat(CTX.vsb, format->frag, NULL)); return (1); } @@ -318,13 +363,14 @@ format_auth(const struct format *format) CTX.frag[F_auth].e)) { if (format->string == NULL) return (-1); - AZ(VSB_cat(CTX.vsb, format->string)); + AZ(vsb_esc_cat(CTX.vsb, format->string, + format->string + strlen(format->string))); return (0); } q = strchr(buf, ':'); if (q != NULL) *q = '\0'; - AZ(VSB_cat(CTX.vsb, buf)); + AZ(vsb_esc_cat(CTX.vsb, buf, buf + strlen(buf))); return (1); } From lasse at varnish-software.com Thu Dec 18 09:27:38 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 9a9a9cd typo Message-ID: commit 9a9a9cd9fe48668b31d7427c02896c34c45e9ee7 Author: Lasse Karstensen Date: Mon Feb 6 14:42:10 2012 +0100 typo diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c index c175f10..08d8eb6 100644 --- a/lib/libvarnishapi/vsm.c +++ b/lib/libvarnishapi/vsm.c @@ -157,7 +157,7 @@ VSM_Delete(struct VSM_data *vd) * The internal VSM open function * * Return: - * 0 = sucess + * 0 = success * <0 = failure * */ From lasse at varnish-software.com Thu Dec 18 09:27:38 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] db8b1d3 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit db8b1d33707e6bab430c983d22024f9a684e98a9 Merge: 9a9a9cd 512dddc Author: Lasse Karstensen Date: Tue Feb 7 11:06:12 2012 +0100 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] a9d35b4 Clarify comment, add assert + comment Message-ID: commit a9d35b4759edc0034e1241ec47943bb18937a9db Author: Poul-Henning Kamp Date: Sun Feb 19 07:44:10 2012 +0000 Clarify comment, add assert + comment diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 372ce08..a8b3dcb 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -572,9 +572,8 @@ FetchBody(struct worker *wrk, struct object *obj) AZ(bo->vgz_rx); /* - * It is OK for ->end to just leave the last storage segment - * sitting on wrk->storage, we will always call vfp_nop_end() - * to get it trimmed or thrown out if empty. + * We always call vfp_nop_end() to ditch or trim the last storage + * segment, to avoid having to replicate that code in all vfp's. */ AZ(vfp_nop_end(bo)); diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 2fce063..dd517be 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -177,6 +177,8 @@ stv_alloc(struct busyobj *bo, size_t size) if (size > cache_param->fetch_maxchunksize) size = cache_param->fetch_maxchunksize; + assert(size <= UINT_MAX); /* field limit in struct storage */ + for (;;) { /* try to allocate from it */ AN(stv->alloc); From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] d647764 Eliminate a single-use macro Message-ID: commit d647764b9c1657e329ab632d5e81f1dd5e127b33 Author: Poul-Henning Kamp Date: Sun Feb 19 07:51:30 2012 +0000 Eliminate a single-use macro diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index daa01f4..d441fca 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -935,9 +935,6 @@ void WSL_Flush(struct vsl_log *, int overflow); #define WSP(sess, tag, ...) \ WSL((sess)->wrk->vsl, tag, (sess)->vsl_id, __VA_ARGS__) -#define WSPR(sess, tag, txt) \ - WSLR((sess)->wrk->vsl, tag, (sess)->vsl_id, txt) - #define INCOMPL() do { \ VSL(SLT_Debug, 0, "INCOMPLETE AT: %s(%d)", __func__, __LINE__); \ fprintf(stderr, \ diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index de71ada..71533ef 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -677,7 +677,7 @@ http_DissectRequest(const struct sess *sp) retval = http_splitline(hp, htc, HTTP_HDR_REQ, HTTP_HDR_URL, HTTP_HDR_PROTO); if (retval != 0) { - WSPR(sp, SLT_HttpGarbage, htc->rxbuf); + WSLR(sp->req->vsl, SLT_HttpGarbage, -1, htc->rxbuf); return (retval); } http_ProtoVer(hp); From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 9a63616 Cleanup all the easy bits of the VSL api in varnishd. Message-ID: commit 9a63616cb869286a0a3f8139f8f0f1205ad87e80 Author: Poul-Henning Kamp Date: Sun Feb 19 08:40:55 2012 +0000 Cleanup all the easy bits of the VSL api in varnishd. We now have: VSL() -> send directly to global VSL buffer VSLb() -> send to buffered VSL VSLbt() -> send txt to buffered VSL diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index d441fca..2dd32bf 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -918,13 +918,13 @@ void VSM_Free(void *ptr); #ifdef VSL_ENDMARKER void VSL(enum VSL_tag_e tag, int id, const char *fmt, ...) __printflike(3, 4); -void WSLR(struct vsl_log *, enum VSL_tag_e tag, int id, txt t); void WSL(struct vsl_log *, enum VSL_tag_e tag, int id, const char *fmt, ...) __printflike(4, 5); -void VSLB(struct busyobj *, enum VSL_tag_e tag, const char *fmt, ...) +void VSLb(struct vsl_log *, enum VSL_tag_e tag, const char *fmt, ...) __printflike(3, 4); +void VSLbt(struct vsl_log *, enum VSL_tag_e tag, txt t); -void WSL_Flush(struct vsl_log *, int overflow); +void VSL_Flush(struct vsl_log *, int overflow); #define DSL(flag, tag, id, ...) \ do { \ @@ -932,9 +932,6 @@ void WSL_Flush(struct vsl_log *, int overflow); VSL((tag), (id), __VA_ARGS__); \ } while (0) -#define WSP(sess, tag, ...) \ - WSL((sess)->wrk->vsl, tag, (sess)->vsl_id, __VA_ARGS__) - #define INCOMPL() do { \ VSL(SLT_Debug, 0, "INCOMPLETE AT: %s(%d)", __func__, __LINE__); \ fprintf(stderr, \ diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 6e86538..085c7a7 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -183,7 +183,7 @@ bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs) } else { vc->vsl_id = s | VSL_BACKENDMARKER; VTCP_myname(s, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); - WSL(sp->wrk->vsl, SLT_BackendOpen, vc->vsl_id, "%s %s %s ", + WSL(sp->req->vsl, SLT_BackendOpen, vc->vsl_id, "%s %s %s ", vs->backend->display_name, abuf1, pbuf1); } @@ -346,7 +346,7 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs) if (vbe_CheckFd(vc->fd)) { /* XXX locking of stats */ VSC_C_main->backend_reuse += 1; - WSP(sp, SLT_Backend, "%d %s %s", + VSLb(sp->req->vsl, SLT_Backend, "%d %s %s", vc->fd, sp->req->director->vcl_name, bp->display_name); vc->vdis = vs; @@ -359,7 +359,7 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs) /* Checkpoint log to flush all info related to this connection before the OS reuses the FD */ - WSL_Flush(sp->wrk->vsl, 0); + VSL_Flush(sp->wrk->vsl, 0); VTCP_close(&vc->fd); VBE_DropRefConn(bp); @@ -389,7 +389,7 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs) } vc->backend = bp; VSC_C_main->backend_conn++; - WSP(sp, SLT_Backend, "%d %s %s", + VSLb(sp->req->vsl, SLT_Backend, "%d %s %s", vc->fd, sp->req->director->vcl_name, bp->display_name); vc->vdis = vs; return (vc); diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index 24a150e..a9ff554 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -761,7 +761,8 @@ ban_check_object(struct object *o, const struct sess *sp, int has_req) oc->ban = NULL; oc_updatemeta(oc); /* BAN also changed, but that is not important any more */ - WSP(sp, SLT_ExpBan, "%u was banned", o->xid); + /* XXX: no req in lurker */ + VSLb(sp->wrk->vsl, SLT_ExpBan, "%u was banned", o->xid); EXP_Rearm(o); return (1); } @@ -954,7 +955,7 @@ ban_lurker(struct sess *sp, void *priv) } i = ban_lurker_work(sp, pass); - WSL_Flush(sp->wrk->vsl, 0); + VSL_Flush(sp->wrk->vsl, 0); WRK_SumStat(sp->wrk); if (i) { pass += (1 << LURK_SHIFT); diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 8f0f31e..32ae7c8 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -183,7 +183,7 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo) if (r) return; - WSL_Flush(vbo->bo.vsl, 0); + VSL_Flush(vbo->bo.vsl, 0); /* XXX: Sanity checks & cleanup */ memset(&vbo->bo, 0, sizeof vbo->bo); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 4b0429a..9619a6e 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -394,14 +394,14 @@ cnt_done(struct sess *sp, struct worker *wrk, struct req *req) /* XXX: Add StatReq == StatSess */ /* XXX: Workaround for pipe */ if (sp->fd >= 0) { - WSP(sp, SLT_Length, "%ju", + VSLb(sp->req->vsl, SLT_Length, "%ju", (uintmax_t)req->req_bodybytes); } - WSP(sp, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f", + VSLb(sp->req->vsl, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f", req->xid, sp->t_req, sp->t_idle, dh, dp, da); } req->xid = 0; - WSL_Flush(wrk->vsl, 0); + VSL_Flush(wrk->vsl, 0); sp->t_req = NAN; req->t_resp = NAN; @@ -941,10 +941,12 @@ cnt_first(struct sess *sp, struct worker *wrk) &sp->mysockaddrlen)); VTCP_name(&sp->mysockaddr, sp->mysockaddrlen, laddr, sizeof laddr, lport, sizeof lport); - WSP(sp, SLT_SessionOpen, "%s %s %s %s", + /* XXX: have no req yet */ + VSLb(sp->wrk->vsl, SLT_SessionOpen, "%s %s %s %s", sp->addr, sp->port, laddr, lport); } else { - WSP(sp, SLT_SessionOpen, "%s %s %s", + /* XXX: have no req yet */ + VSLb(sp->wrk->vsl, SLT_SessionOpen, "%s %s %s", sp->addr, sp->port, sp->mylsock->name); } @@ -1116,7 +1118,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) if (oc->flags & OC_F_PASS) { wrk->stats.cache_hitpass++; - WSP(sp, SLT_HitPass, "%u", req->obj->xid); + VSLb(sp->req->vsl, SLT_HitPass, "%u", req->obj->xid); (void)HSH_Deref(&wrk->stats, NULL, &req->obj); AZ(req->objcore); sp->step = STP_PASS; @@ -1124,7 +1126,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) } wrk->stats.cache_hit++; - WSP(sp, SLT_Hit, "%u", req->obj->xid); + VSLb(sp->req->vsl, SLT_Hit, "%u", req->obj->xid); sp->step = STP_HIT; return (0); } @@ -1450,7 +1452,8 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) /* Assign XID and log */ req->xid = ++xids; /* XXX not locked */ - WSP(sp, SLT_ReqStart, "%s %s %u", sp->addr, sp->port, req->xid); + VSLb(sp->req->vsl, SLT_ReqStart, "%s %s %u", + sp->addr, sp->port, req->xid); /* Borrow VCL reference from worker thread */ VCL_Refresh(&wrk->vcl); @@ -1521,10 +1524,11 @@ cnt_diag(struct sess *sp, const char *state) obj = sp->req->obj; } - if (sp->wrk != NULL) { - WSP(sp, SLT_Debug, "vsl_id %u STP_%s sp %p obj %p vcl %p", + if (sp->req != NULL) { + VSLb(sp->req->vsl, SLT_Debug, + "vsl_id %u STP_%s sp %p obj %p vcl %p", sp->vsl_id, state, sp, obj, vcl); - WSL_Flush(sp->wrk->vsl, 0); + VSL_Flush(sp->req->vsl, 0); } else { VSL(SLT_Debug, sp->vsl_id, "vsl_id %u STP_%s sp %p obj %p vcl %p", @@ -1605,7 +1609,7 @@ CNT_Session(struct sess *sp) WS_Assert(wrk->aws); CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); } - WSL_Flush(wrk->vsl, 0); + VSL_Flush(wrk->vsl, 0); #define ACCT(foo) AZ(wrk->acct_tmp.foo); #include "tbl/acct_fields.h" #undef ACCT diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c index b1359e3..ab74b59 100644 --- a/bin/varnishd/cache/cache_dir.c +++ b/bin/varnishd/cache/cache_dir.c @@ -59,8 +59,8 @@ VDI_CloseFd(struct worker *wrk, struct vbc **vbp) /* Checkpoint log to flush all info related to this connection before the OS reuses the FD */ - WSL_Flush(wrk->vsl, 0); - WSL_Flush(vc->vsl, 0); + VSL_Flush(wrk->vsl, 0); + VSL_Flush(vc->vsl, 0); vc->vsl->wid = vc->orig_vsl_id; vc->vsl = NULL; vc->orig_vsl_id = 0; @@ -95,8 +95,8 @@ VDI_RecycleFd(struct worker *wrk, struct vbc **vbp) * Flush the shmlog, so that another session reusing this backend * will log chronologically later than our use of it. */ - WSL_Flush(wrk->vsl, 0); - WSL_Flush(vc->vsl, 0); + VSL_Flush(wrk->vsl, 0); + VSL_Flush(vc->vsl, 0); vc->vsl->wid = vc->orig_vsl_id; vc->vsl = NULL; vc->orig_vsl_id = 0; diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 54302c8..2c4d3d0 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -98,7 +98,7 @@ ved_include(struct sess *sp, const char *src, const char *host) if (sp->step == STP_DONE) break; AZ(sp->wrk); - WSL_Flush(w->vsl, 0); + VSL_Flush(w->vsl, 0); DSL(0x20, SLT_Debug, sp->vsl_id, "loop waiting for ESI"); (void)usleep(10000); } diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c index 053f614..6eac8d3 100644 --- a/bin/varnishd/cache/cache_esi_parse.c +++ b/bin/varnishd/cache/cache_esi_parse.c @@ -186,7 +186,7 @@ vep_error(const struct vep_state *vep, const char *p) VSC_C_main->esi_errors++; l = (intmax_t)(vep->ver_p - vep->hack_p); - VSLB(vep->bo, SLT_ESI_xmlerror, "ERR at %jd %s", l, p); + VSLb(vep->bo->vsl, SLT_ESI_xmlerror, "ERR at %jd %s", l, p); } @@ -202,7 +202,7 @@ vep_warn(const struct vep_state *vep, const char *p) VSC_C_main->esi_warnings++; l = (intmax_t)(vep->ver_p - vep->hack_p); printf("WARNING at %jd %s\n", l, p); - VSLB(vep->bo, SLT_ESI_xmlerror, "WARN at %jd %s", l, p); + VSLb(vep->bo->vsl, SLT_ESI_xmlerror, "WARN at %jd %s", l, p); } @@ -601,7 +601,7 @@ VEP_Parse(const struct busyobj *bo, const char *p, size_t l) p++; vep->state = VEP_STARTTAG; } else if (p < e) { - VSLB(vep->bo, SLT_ESI_xmlerror, + VSLb(vep->bo->vsl, SLT_ESI_xmlerror, "No ESI processing, first char not '<'"); vep->state = VEP_NOTXML; } diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index 060ed58..e0f3d27 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -342,7 +342,7 @@ exp_timer(struct sess *sp, void *priv) oc = NULL; while (1) { if (oc == NULL) { - WSL_Flush(sp->wrk->vsl, 0); + VSL_Flush(sp->wrk->vsl, 0); WRK_SumStat(sp->wrk); VTIM_sleep(cache_param->expiry_sleep); t = VTIM_real(); @@ -445,7 +445,7 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru) return (-1); /* XXX: bad idea for -spersistent */ - WSL(bo->vsl, SLT_ExpKill, -1, "%u LRU", oc_getxid(bo->stats, oc)); + VSLb(bo->vsl, SLT_ExpKill, "%u LRU", oc_getxid(bo->stats, oc)); (void)HSH_Deref(bo->stats, oc, NULL); return (1); } diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index a8b3dcb..48db272 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -58,9 +58,9 @@ FetchError2(struct busyobj *bo, const char *error, const char *more) CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); if (!bo->fetch_failed) { if (more == NULL) - VSLB(bo, SLT_FetchError, "%s", error); + VSLb(bo->vsl, SLT_FetchError, "%s", error); else - VSLB(bo, SLT_FetchError, "%s: %s", error, more); + VSLb(bo->vsl, SLT_FetchError, "%s: %s", error, more); } bo->fetch_failed = 1; return (-1); @@ -364,7 +364,7 @@ FetchReqBody(const struct sess *sp, int sendbody) } if (http_GetHdr(sp->req->http, H_Transfer_Encoding, NULL)) { /* XXX: Handle chunked encoding. */ - WSP(sp, SLT_Debug, "Transfer-Encoding in request"); + VSLb(sp->req->vsl, SLT_Debug, "Transfer-Encoding in request"); return (1); } return (0); @@ -408,7 +408,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) wrk->busyobj->vbc = VDI_GetFd(NULL, sp); if (wrk->busyobj->vbc == NULL) { - WSP(sp, SLT_FetchError, "no backend connection"); + VSLb(sp->req->vsl, SLT_FetchError, "no backend connection"); return (-1); } vc = wrk->busyobj->vbc; @@ -430,7 +430,8 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) /* Deal with any message-body the request might have */ i = FetchReqBody(sp, sendbody); if (WRW_FlushRelease(wrk) || i > 0) { - WSP(sp, SLT_FetchError, "backend write error: %d (%s)", + VSLb(sp->req->vsl, SLT_FetchError, + "backend write error: %d (%s)", errno, strerror(errno)); VDI_CloseFd(wrk, &wrk->busyobj->vbc); /* XXX: other cleanup ? */ @@ -438,7 +439,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) } /* Checkpoint the vsl.here */ - WSL_Flush(wrk->vsl, 0); + VSL_Flush(wrk->vsl, 0); /* XXX is this the right place? */ VSC_C_main->backend_req++; @@ -454,7 +455,8 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) i = HTC_Rx(htc); if (i < 0) { - WSP(sp, SLT_FetchError, "http first read error: %d %d (%s)", + VSLb(sp->req->vsl, SLT_FetchError, + "http first read error: %d %d (%s)", i, errno, strerror(errno)); VDI_CloseFd(wrk, &wrk->busyobj->vbc); /* XXX: other cleanup ? */ @@ -467,7 +469,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) while (i == 0) { i = HTC_Rx(htc); if (i < 0) { - WSP(sp, SLT_FetchError, + VSLb(sp->req->vsl, SLT_FetchError, "http first read error: %d %d (%s)", i, errno, strerror(errno)); VDI_CloseFd(wrk, &wrk->busyobj->vbc); @@ -479,7 +481,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) hp = wrk->busyobj->beresp; if (http_DissectResponse(hp, htc)) { - WSP(sp, SLT_FetchError, "http format error"); + VSLb(sp->req->vsl, SLT_FetchError, "http format error"); VDI_CloseFd(wrk, &wrk->busyobj->vbc); /* XXX: other cleanup ? */ return (-1); @@ -579,7 +581,7 @@ FetchBody(struct worker *wrk, struct object *obj) bo->fetch_obj = NULL; - VSLB(bo, SLT_Fetch_Body, "%u(%s) cls %d mklen %d", + VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s) cls %d mklen %d", bo->body_status, body_status(bo->body_status), cls, mklen); @@ -601,7 +603,7 @@ FetchBody(struct worker *wrk, struct object *obj) if (cls == 0 && bo->should_close) cls = 1; - VSLB(bo, SLT_Length, "%zd", obj->len); + VSLb(bo->vsl, SLT_Length, "%zd", obj->len); { /* Sanity check fetch methods accounting */ diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 530d43b..d4511bc 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -395,7 +395,7 @@ VGZ_Destroy(struct vgz **vgp) CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC); *vgp = NULL; - WSL(vg->vsl, SLT_Gzip, -1, "%s %jd %jd %jd %jd %jd", + VSLb(vg->vsl, SLT_Gzip, "%s %jd %jd %jd %jd %jd", vg->id, (intmax_t)vg->vz.total_in, (intmax_t)vg->vz.total_out, diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 7188ff4..3d44cc4 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -163,7 +163,7 @@ HSH_AddString(const struct sess *sp, const char *str) SHA256_Update(sp->req->sha256ctx, "#", 1); if (cache_param->log_hash) - WSP(sp, SLT_Hash, "%s", str); + VSLb(sp->req->vsl, SLT_Hash, "%s", str); } /*--------------------------------------------------------------------- @@ -431,7 +431,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) VTAILQ_INSERT_TAIL(&oh->waitinglist->list, sp, list); } if (cache_param->diag_bitmap & 0x20) - WSP(sp, SLT_Debug, + VSLb(sp->req->vsl, SLT_Debug, "on waiting list <%p>", oh); SES_Charge(sp); /* diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 71533ef..f721ea2 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -78,7 +78,7 @@ http_VSLH(const struct http *hp, unsigned hdr) AN(hp->vsl); AN(hp->vsl->wid & (VSL_CLIENTMARKER|VSL_BACKENDMARKER)); - WSLR(hp->vsl, http2shmlog(hp, hdr), -1, hp->hd[hdr]); + VSLbt(hp->vsl, http2shmlog(hp, hdr), hp->hd[hdr]); } /*--------------------------------------------------------------------*/ @@ -528,7 +528,7 @@ http_dissect_hdrs(struct http *hp, char *p, const struct http_conn *htc) if (q - p > htc->maxhdr) { VSC_C_main->losthdr++; - WSL(hp->vsl, SLT_LostHeader, -1, "%.*s", + VSLb(hp->vsl, SLT_LostHeader, "%.*s", (int)(q - p > 20 ? 20 : q - p), p); return (413); } @@ -554,7 +554,7 @@ http_dissect_hdrs(struct http *hp, char *p, const struct http_conn *htc) hp->nhd++; } else { VSC_C_main->losthdr++; - WSL(hp->vsl, SLT_LostHeader, -1, "%.*s", + VSLb(hp->vsl, SLT_LostHeader, "%.*s", (int)(q - p > 20 ? 20 : q - p), p); return (413); } @@ -677,7 +677,7 @@ http_DissectRequest(const struct sess *sp) retval = http_splitline(hp, htc, HTTP_HDR_REQ, HTTP_HDR_URL, HTTP_HDR_PROTO); if (retval != 0) { - WSLR(sp->req->vsl, SLT_HttpGarbage, -1, htc->rxbuf); + VSLbt(sp->req->vsl, SLT_HttpGarbage, htc->rxbuf); return (retval); } http_ProtoVer(hp); @@ -724,7 +724,7 @@ http_DissectResponse(struct http *hp, const struct http_conn *htc) } if (retval != 0) { - WSLR(hp->vsl, SLT_HttpGarbage, -1, htc->rxbuf); + VSLbt(hp->vsl, SLT_HttpGarbage, htc->rxbuf); assert(retval >= 100 && retval <= 999); hp->status = retval; } else { @@ -842,7 +842,7 @@ http_filterfields(struct http *to, const struct http *fm, unsigned how) to->nhd++; } else { VSC_C_main->losthdr++; - WSLR(to->vsl, SLT_LostHeader, -1, fm->hd[u]); + VSLbt(to->vsl, SLT_LostHeader, fm->hd[u]); } } } @@ -910,7 +910,7 @@ http_CopyHome(const struct http *hp) } else { /* XXX This leaves a slot empty */ VSC_C_main->losthdr++; - WSLR(hp->vsl, SLT_LostHeader, -1, hp->hd[u]); + VSLbt(hp->vsl, SLT_LostHeader, hp->hd[u]); hp->hd[u].b = NULL; hp->hd[u].e = NULL; } @@ -940,7 +940,7 @@ http_SetHeader(struct http *to, const char *hdr) CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); if (to->nhd >= to->shd) { VSC_C_main->losthdr++; - WSL(to->vsl, SLT_LostHeader, -1, "%s", hdr); + VSLb(to->vsl, SLT_LostHeader, "%s", hdr); return; } http_SetH(to, to->nhd++, hdr); @@ -958,7 +958,7 @@ http_PutField(const struct http *to, int field, const char *string) l = strlen(string); p = WS_Alloc(to->ws, l + 1); if (p == NULL) { - WSL(to->vsl, SLT_LostHeader, -1, "%s", string); + VSLb(to->vsl, SLT_LostHeader, "%s", string); to->hd[field].b = NULL; to->hd[field].e = NULL; to->hdf[field] = 0; @@ -1011,7 +1011,7 @@ http_PrintfHeader(struct http *to, const char *fmt, ...) va_end(ap); if (n + 1 >= l || to->nhd >= to->shd) { VSC_C_main->losthdr++; - WSL(to->vsl, SLT_LostHeader, -1, "%s", to->ws->f); + VSLb(to->vsl, SLT_LostHeader, "%s", to->ws->f); WS_Release(to->ws, 0); } else { to->hd[to->nhd].b = to->ws->f; diff --git a/bin/varnishd/cache/cache_httpconn.c b/bin/varnishd/cache/cache_httpconn.c index d0b7199..890dfd3 100644 --- a/bin/varnishd/cache/cache_httpconn.c +++ b/bin/varnishd/cache/cache_httpconn.c @@ -224,7 +224,7 @@ HTC_Read(struct http_conn *htc, void *d, size_t len) return (l); i = read(htc->fd, p, len); if (i < 0) { - WSL(htc->vsl, SLT_FetchError, -1, "%s", strerror(errno)); + VSLb(htc->vsl, SLT_FetchError, "%s", strerror(errno)); return (i); } return (i + l); diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 1b67d81..2c3a368 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -167,7 +167,7 @@ RFC2616_Ttl(const struct sess *sp) } /* calculated TTL, Our time, Date, Expires, max-age, age */ - WSP(sp, SLT_TTL, + VSLb(sp->req->vsl, SLT_TTL, /* XXX Bo->vsl ? */ "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u", sp->req->xid, expp->ttl, -1., -1., expp->entered, expp->age, h_date, h_expires, max_age); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 1bd74fe..15efc5f 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -375,7 +375,7 @@ SES_ReleaseReq(struct sess *sp) AN(pp->pool); CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC); MPL_AssertSane(sp->req); - WSL_Flush(sp->req->vsl, 0); + VSL_Flush(sp->req->vsl, 0); MPL_Free(pp->mpl_req, sp->req); sp->req = NULL; } diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index d4dae9d..6a54101 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -133,7 +133,7 @@ vsl_get(unsigned len, unsigned records, unsigned flushes) */ static void -VSLR(enum VSL_tag_e tag, int id, const char *b, unsigned len) +vslr(enum VSL_tag_e tag, int id, const char *b, unsigned len) { uint32_t *p; unsigned mlen; @@ -161,18 +161,18 @@ VSL(enum VSL_tag_e tag, int id, const char *fmt, ...) /* * XXX: consider formatting into a stack buffer then move into - * XXX: shmlog with VSLR(). + * XXX: shmlog with vslr(). */ AN(fmt); va_start(ap, fmt); if (strchr(fmt, '%') == NULL) { - VSLR(tag, id, fmt, strlen(fmt)); + vslr(tag, id, fmt, strlen(fmt)); } else { n = vsnprintf(buf, mlen, fmt, ap); if (n > mlen) n = mlen; - VSLR(tag, id, buf, n); + vslr(tag, id, buf, n); } va_end(ap); } @@ -180,7 +180,7 @@ VSL(enum VSL_tag_e tag, int id, const char *fmt, ...) /*--------------------------------------------------------------------*/ void -WSL_Flush(struct vsl_log *vsl, int overflow) +VSL_Flush(struct vsl_log *vsl, int overflow) { uint32_t *p; unsigned l; @@ -202,8 +202,8 @@ WSL_Flush(struct vsl_log *vsl, int overflow) /*--------------------------------------------------------------------*/ -void -WSLR(struct vsl_log *vsl, enum VSL_tag_e tag, int id, txt t) +static void +wslr(struct vsl_log *vsl, enum VSL_tag_e tag, int id, txt t) { unsigned l, mlen; @@ -223,7 +223,7 @@ WSLR(struct vsl_log *vsl, enum VSL_tag_e tag, int id, txt t) /* Wrap if necessary */ if (VSL_END(vsl->wlp, l) >= vsl->wle) - WSL_Flush(vsl, 1); + VSL_Flush(vsl, 1); assert (VSL_END(vsl->wlp, l) < vsl->wle); memcpy(VSL_DATA(vsl->wlp), t.b, l); vsl_hdr(tag, vsl->wlp, l, id); @@ -231,7 +231,7 @@ WSLR(struct vsl_log *vsl, enum VSL_tag_e tag, int id, txt t) assert(vsl->wlp < vsl->wle); vsl->wlr++; if (cache_param->diag_bitmap & 0x10000) - WSL_Flush(vsl, 0); + VSL_Flush(vsl, 0); } /*--------------------------------------------------------------------*/ @@ -253,13 +253,13 @@ wsl(struct vsl_log *vsl, enum VSL_tag_e tag, int id, const char *fmt, va_list ap if (strchr(fmt, '%') == NULL) { t.b = TRUST_ME(fmt); t.e = strchr(t.b, '\0'); - WSLR(vsl, tag, id, t); + wslr(vsl, tag, id, t); } else { assert(vsl->wlp < vsl->wle); /* Wrap if we cannot fit a full size record */ if (VSL_END(vsl->wlp, mlen) >= vsl->wle) - WSL_Flush(vsl, 1); + VSL_Flush(vsl, 1); p = VSL_DATA(vsl->wlp); n = vsnprintf(p, mlen, fmt, ap); @@ -271,7 +271,7 @@ wsl(struct vsl_log *vsl, enum VSL_tag_e tag, int id, const char *fmt, va_list ap vsl->wlr++; } if (cache_param->diag_bitmap & 0x10000) - WSL_Flush(vsl, 0); + VSL_Flush(vsl, 0); } /*--------------------------------------------------------------------*/ @@ -290,20 +290,31 @@ WSL(struct vsl_log *vsl, enum VSL_tag_e tag, int id, const char *fmt, ...) } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * VSL-buffered + */ void -VSLB(struct busyobj *bo, enum VSL_tag_e tag, const char *fmt, ...) +VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) { va_list ap; - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AN(fmt); va_start(ap, fmt); - wsl(bo->vsl, tag, bo->vsl->wid, fmt, ap); + wsl(vsl, tag, vsl->wid, fmt, ap); va_end(ap); } +/*-------------------------------------------------------------------- + * VSL-buffered-txt + */ + +void +VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t) +{ + wslr(vsl, tag, -1, t); +} + /*--------------------------------------------------------------------*/ void diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c index 9ad41d0..7ba9c60 100644 --- a/bin/varnishd/cache/cache_vary.c +++ b/bin/varnishd/cache/cache_vary.c @@ -79,7 +79,8 @@ VRY_Create(const struct sess *sp, const struct http *hp) AN(sbh); if (*v == ':') { - WSP(sp, SLT_Error, "Vary header had extra ':', fix backend"); + VSLb(sp->req->vsl, SLT_Error, + "Vary header had extra ':', fix backend"); v++; } for (p = v; *p; p++) { diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index ad5acf0..b68627b 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -335,9 +335,10 @@ VCL_##func##_method(struct sess *sp) \ \ sp->req->handling = 0; \ sp->req->cur_method = VCL_MET_ ## upper; \ - WSP(sp, SLT_VCL_call, "%s", #func); \ + VSLb(sp->req->vsl, SLT_VCL_call, "%s", #func); \ (void)sp->req->vcl->func##_func(sp); \ - WSP(sp, SLT_VCL_return, "%s", VCL_Return_Name(sp->req->handling)); \ + VSLb(sp->req->vsl, SLT_VCL_return, "%s", \ + VCL_Return_Name(sp->req->handling)); \ sp->req->cur_method = 0; \ assert((1U << sp->req->handling) & bitmap); \ assert(!((1U << sp->req->handling) & ~bitmap)); \ diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 6271e49..c9135dc 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -56,7 +56,7 @@ VRT_error(const struct sess *sp, unsigned code, const char *reason) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - WSL(sp->wrk->vsl, SLT_Debug, 0, "VCL_error(%u, %s)", code, reason ? + VSLb(sp->req->vsl, SLT_Debug, "VCL_error(%u, %s)", code, reason ? reason : "(null)"); if (code < 100 || code > 999) code = 503; @@ -75,7 +75,7 @@ VRT_count(const struct sess *sp, unsigned u) return; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); if (cache_param->vcl_trace) - WSP(sp, SLT_VCL_trace, "%u %u.%u", u, + VSLb(sp->req->vsl, SLT_VCL_trace, "%u %u.%u", u, sp->req->vcl->ref[u].line, sp->req->vcl->ref[u].pos); } @@ -85,7 +85,7 @@ void VRT_acl_log(const struct sess *sp, const char *msg) { - WSP(sp, SLT_VCL_acl, "%s", msg); + VSLb(sp->req->vsl, SLT_VCL_acl, "%s", msg); } /*--------------------------------------------------------------------*/ @@ -228,7 +228,7 @@ VRT_SetHdr(const struct sess *sp , enum gethdr_e where, const char *hdr, } else { b = VRT_String(hp->ws, hdr + 1, p, ap); if (b == NULL) { - WSP(sp, SLT_LostHeader, "%s", hdr + 1); + VSLb(sp->req->vsl, SLT_LostHeader, "%s", hdr + 1); } else { http_Unset(hp, hdr); http_SetHeader(hp, b); diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c index 867ec86..765cd27 100644 --- a/bin/varnishd/cache/cache_vrt_re.c +++ b/bin/varnishd/cache/cache_vrt_re.c @@ -75,7 +75,8 @@ VRT_re_match(const struct sess *sp, const char *s, void *re) if (i >= 0) return (1); if (i < VRE_ERROR_NOMATCH ) - WSP(sp, SLT_VCL_Error, "Regexp matching returned %d", i); + VSLb(sp->req->vsl, SLT_VCL_Error, + "Regexp matching returned %d", i); return (0); } @@ -108,7 +109,8 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re, if (i == VRE_ERROR_NOMATCH) return(str); if (i < VRE_ERROR_NOMATCH ) { - WSP(sp, SLT_VCL_Error, "Regexp matching returned %d", i); + VSLb(sp->req->vsl, SLT_VCL_Error, + "Regexp matching returned %d", i); return(str); } @@ -146,7 +148,7 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re, &cache_param->vre_limits); if (i < VRE_ERROR_NOMATCH ) { WS_Release(sp->req->http->ws, 0); - WSP(sp, SLT_VCL_Error, + VSLb(sp->req->vsl, SLT_VCL_Error, "Regexp matching returned %d", i); return(str); } diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 417fcee..50f4411 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -46,16 +46,15 @@ static char vrt_hostname[255] = ""; /*--------------------------------------------------------------------*/ static void -vrt_do_string(struct worker *w, int fd, const struct http *hp, int fld, +vrt_do_string(const struct http *hp, int fld, const char *err, const char *p, va_list ap) { char *b; - // AN(p); AN(hp); b = VRT_String(hp->ws, NULL, p, ap); if (b == NULL || *b == '\0') { - WSL(w->vsl, SLT_LostHeader, fd, "%s", err); + VSLb(hp->vsl, SLT_LostHeader, "%s", err); } else { http_SetH(hp, fld, b); } @@ -68,9 +67,9 @@ VRT_l_##obj##_##hdr(const struct sess *sp, const char *p, ...) \ { \ va_list ap; \ \ + (void)sp; \ va_start(ap, p); \ - vrt_do_string(sp->wrk, sp->fd, \ - http, fld, #obj "." #hdr, p, ap); \ + vrt_do_string(http, fld, #obj "." #hdr, p, ap); \ va_end(ap); \ } \ \ @@ -399,7 +398,7 @@ VRT_r_##which##_##fld(struct sess *sp) \ static void vrt_wsp_exp(const struct sess *sp, unsigned xid, const struct exp *e) { - WSP(sp, SLT_TTL, "%u VCL %.0f %.0f %.0f %.0f %.0f", + VSLb(sp->req->vsl, SLT_TTL, "%u VCL %.0f %.0f %.0f %.0f %.0f", xid, e->ttl - (sp->t_req - e->entered), e->grace, e->keep, sp->t_req, e->age + (sp->t_req - e->entered)); } diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index dd517be..3a9f614 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -141,7 +141,7 @@ stv_pick_stevedore(struct vsl_log *vsl, const char **hint) return (stv_transient); /* Hint was not valid, nuke it */ - WSL(vsl, SLT_Debug, -1, "Storage hint not usable"); + VSLb(vsl, SLT_Debug, "Storage hint not usable"); *hint = NULL; } /* pick a stevedore and bump the head along */ diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index e363b2d..5774cfb 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -152,7 +152,7 @@ vmod_log(struct sess *sp, const char *fmt, ...) p = VRT_StringList(buf, sizeof buf, fmt, ap); va_end(ap); if (p != NULL) - WSP(sp, SLT_VCL_Log, "%s", buf); + VSLb(sp->req->vsl, SLT_VCL_Log, "%s", buf); } void From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] a34c900 Eliminate some wrk->sp usage. Message-ID: commit a34c900c15f9b751bf18b4ac26b0991e9fbd0d5a Author: Poul-Henning Kamp Date: Sun Feb 19 09:04:30 2012 +0000 Eliminate some wrk->sp usage. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 2dd32bf..c9cef7b 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -673,7 +673,7 @@ struct sess { void VCA_Init(void); void VCA_Shutdown(void); int VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa); -void VCA_SetupSess(struct worker *w); +void VCA_SetupSess(struct worker *w, struct sess *sp); void VCA_FailSess(struct worker *w); /* cache_backend.c */ diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 4d7b771..8848682 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -233,6 +233,7 @@ VCA_FailSess(struct worker *wrk) AZ(close(wa->acceptsock)); wrk->stats.sess_drop++; vca_pace_bad(); + WS_Release(wrk->aws, 0); } /*-------------------------------------------------------------------- @@ -240,15 +241,13 @@ VCA_FailSess(struct worker *wrk) */ void -VCA_SetupSess(struct worker *wrk) +VCA_SetupSess(struct worker *wrk, struct sess *sp) { - struct sess *sp; struct wrk_accept *wa; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CAST_OBJ_NOTNULL(wa, (void*)wrk->aws->f, WRK_ACCEPT_MAGIC); - sp = wrk->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CAST_OBJ_NOTNULL(wa, (void*)wrk->aws->f, WRK_ACCEPT_MAGIC); sp->vxid = wa->vxid; sp->vseq = 0; sp->fd = wa->acceptsock; @@ -262,6 +261,7 @@ VCA_SetupSess(struct worker *wrk) sp->sockaddrlen = wa->acceptaddrlen; vca_pace_good(); wrk->stats.sess_conn++; + WS_Release(wrk->aws, 0); if (need_test) sock_test(sp->fd); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 9619a6e..bb01b52 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -285,7 +285,7 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) if (wrk->busyobj != NULL) { AN(wrk->busyobj->do_stream); VDI_CloseFd(wrk, &wrk->busyobj->vbc); - HSH_Drop(wrk); + HSH_Drop(wrk, &sp->req->obj); VBO_DerefBusyObj(wrk, &wrk->busyobj); } else { (void)HSH_Deref(&wrk->stats, NULL, &req->obj); @@ -509,7 +509,7 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) if (req->handling == VCL_RET_RESTART && req->restarts < cache_param->max_restarts) { - HSH_Drop(wrk); + HSH_Drop(wrk, &sp->req->obj); VBO_DerefBusyObj(wrk, &wrk->busyobj); req->director = NULL; req->restarts++; @@ -892,7 +892,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) AN(req->director); if (i) { - HSH_Drop(wrk); + HSH_Drop(wrk, &sp->req->obj); VBO_DerefBusyObj(wrk, &wrk->busyobj); AZ(req->obj); req->err_code = 503; diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 3d44cc4..eff6357 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -577,18 +577,17 @@ HSH_Purge(const struct sess *sp, struct objhead *oh, double ttl, double grace) */ void -HSH_Drop(struct worker *wrk) +HSH_Drop(struct worker *wrk, struct object **oo) { - struct object *o; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - o = wrk->sp->req->obj; - CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - AssertObjCorePassOrBusy(o->objcore); - o->exp.ttl = -1.; - if (o->objcore != NULL) /* Pass has no objcore */ - HSH_Unbusy(o->objcore); - (void)HSH_Deref(&wrk->stats, NULL, &wrk->sp->req->obj); + AN(oo); + CHECK_OBJ_NOTNULL(*oo, OBJECT_MAGIC); + AssertObjCorePassOrBusy((*oo)->objcore); + (*oo)->exp.ttl = -1.; + if ((*oo)->objcore != NULL) /* Pass has no objcore */ + HSH_Unbusy((*oo)->objcore); + (void)HSH_Deref(&wrk->stats, NULL, oo); } void diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 15efc5f..542ec2e 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -181,12 +181,11 @@ SES_pool_accept_task(struct worker *wrk, void *arg) wrk->sp = ses_new(pp); if (wrk->sp == NULL) { VCA_FailSess(wrk); - return; + } else { + VCA_SetupSess(wrk, wrk->sp); + wrk->sp->step = STP_FIRST; + ses_pool_task(wrk, wrk->sp); } - VCA_SetupSess(wrk); - wrk->sp->step = STP_FIRST; - WS_Release(wrk->aws, 0); - ses_pool_task(wrk, wrk->sp); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index ac8ce7e..f2b41e2 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -56,7 +56,7 @@ void HSH_Cleanup(struct worker *w); struct objcore *HSH_Lookup(struct sess *sp, struct objhead **poh); void HSH_Unbusy(struct objcore *); void HSH_Ref(struct objcore *o); -void HSH_Drop(struct worker *wrk); +void HSH_Drop(struct worker *, struct object **); void HSH_Init(const struct hash_slinger *slinger); void HSH_AddString(const struct sess *sp, const char *str); struct objcore *HSH_Insert(const struct sess *sp); From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 33a2242 Push struct worker out of STV_NewObject() Message-ID: commit 33a22425a74a3fd670f0f3e4cc20362aaa70541b Author: Poul-Henning Kamp Date: Sun Feb 19 09:39:38 2012 +0000 Push struct worker out of STV_NewObject() diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index c9cef7b..3797047 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -1005,8 +1005,8 @@ unsigned RFC2616_Req_Gzip(const struct sess *sp); int RFC2616_Do_Cond(const struct sess *sp); /* stevedore.c */ -struct object *STV_NewObject(struct worker *wrk, const char *hint, unsigned len, - uint16_t nhttp); +struct object *STV_NewObject(struct busyobj *, struct objcore **, + const char *hint, unsigned len, uint16_t nhttp); struct storage *STV_alloc(struct busyobj *, size_t size); void STV_trim(struct storage *st, size_t size); void STV_free(struct storage *st); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index bb01b52..338b3ab 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -475,9 +475,12 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) wrk->busyobj = VBO_GetBusyObj(wrk); wrk->busyobj->vsl->wid = sp->vsl_id; - req->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, - cache_param->http_resp_size, + AZ(wrk->busyobj->stats); + wrk->busyobj->stats = &wrk->stats; + req->obj = STV_NewObject(wrk->busyobj, &req->objcore, + TRANSIENT_STORAGE, cache_param->http_resp_size, (uint16_t)cache_param->http_max_hdr); + wrk->busyobj->stats = NULL; if (req->obj == NULL) { req->doclose = "Out of objects"; req->director = NULL; @@ -786,18 +789,24 @@ cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req) req->objcore == NULL) req->storage_hint = TRANSIENT_STORAGE; - req->obj = STV_NewObject(wrk, req->storage_hint, l, nhttp); + assert(bo == wrk->busyobj); + AZ(bo->stats); + bo->stats = &wrk->stats; + req->obj = STV_NewObject(bo, &req->objcore, req->storage_hint, l, + nhttp); if (req->obj == NULL) { /* * Try to salvage the transaction by allocating a * shortlived object on Transient storage. */ - req->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, l, nhttp); + req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE, + l, nhttp); if (bo->exp.ttl > cache_param->shortlived) bo->exp.ttl = cache_param->shortlived; bo->exp.grace = 0.0; bo->exp.keep = 0.0; } + bo->stats = NULL; if (req->obj == NULL) { req->err_code = 503; sp->step = STP_ERROR; diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 3a9f614..3c62d8d 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -227,13 +227,15 @@ struct stv_objsecrets { */ struct object * -STV_MkObject(struct worker *wrk, void *ptr, unsigned ltot, +STV_MkObject(struct busyobj *bo, struct objcore **ocp, void *ptr, unsigned ltot, const struct stv_objsecrets *soc) { struct object *o; unsigned l; + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC); + AN(ocp); assert(PAOK(ptr)); assert(PAOK(soc->wsl)); @@ -253,17 +255,17 @@ STV_MkObject(struct worker *wrk, void *ptr, unsigned ltot, WS_Assert(o->ws_o); assert(o->ws_o->e <= (char*)ptr + ltot); - http_Setup(o->http, o->ws_o, wrk->busyobj->vsl); + http_Setup(o->http, o->ws_o, bo->vsl); o->http->magic = HTTP_MAGIC; - o->exp = wrk->busyobj->exp; + o->exp = bo->exp; VTAILQ_INIT(&o->store); - wrk->stats.n_object++; + bo->stats->n_object++; - if (wrk->sp->req->objcore != NULL) { - CHECK_OBJ_NOTNULL(wrk->sp->req->objcore, OBJCORE_MAGIC); + if (*ocp != NULL) { + CHECK_OBJ_NOTNULL((*ocp), OBJCORE_MAGIC); - o->objcore = wrk->sp->req->objcore; - wrk->sp->req->objcore = NULL; /* refcnt follows pointer. */ + o->objcore = *ocp; + *ocp = NULL; /* refcnt follows pointer. */ BAN_NewObjCore(o->objcore); o->objcore->methods = &default_oc_methods; @@ -278,13 +280,15 @@ STV_MkObject(struct worker *wrk, void *ptr, unsigned ltot, */ struct object * -stv_default_allocobj(struct stevedore *stv, struct worker *wrk, unsigned ltot, - const struct stv_objsecrets *soc) +stv_default_allocobj(struct stevedore *stv, struct busyobj *bo, + struct objcore **ocp, unsigned ltot, const struct stv_objsecrets *soc) { struct object *o; struct storage *st; CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + AN(ocp); st = stv->alloc(stv, ltot); if (st == NULL) return (NULL); @@ -293,7 +297,7 @@ stv_default_allocobj(struct stevedore *stv, struct worker *wrk, unsigned ltot, return (NULL); } ltot = st->len = st->space; - o = STV_MkObject(wrk, st->ptr, ltot, soc); + o = STV_MkObject(bo, ocp, st->ptr, ltot, soc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); o->objstore = st; return (o); @@ -306,8 +310,8 @@ stv_default_allocobj(struct stevedore *stv, struct worker *wrk, unsigned ltot, */ struct object * -STV_NewObject(struct worker *wrk, const char *hint, unsigned wsl, - uint16_t nhttp) +STV_NewObject(struct busyobj *bo, struct objcore **ocp, const char *hint, + unsigned wsl, uint16_t nhttp) { struct object *o; struct stevedore *stv, *stv0; @@ -315,7 +319,8 @@ STV_NewObject(struct worker *wrk, const char *hint, unsigned wsl, struct stv_objsecrets soc; int i; - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + AN(ocp); assert(wsl > 0); wsl = PRNDUP(wsl); @@ -330,27 +335,23 @@ STV_NewObject(struct worker *wrk, const char *hint, unsigned wsl, ltot = sizeof *o + wsl + lhttp; - stv = stv0 = stv_pick_stevedore(wrk->vsl, &hint); + stv = stv0 = stv_pick_stevedore(bo->vsl, &hint); AN(stv->allocobj); - o = stv->allocobj(stv, wrk, ltot, &soc); + o = stv->allocobj(stv, bo, ocp, ltot, &soc); if (o == NULL && hint == NULL) { do { - stv = stv_pick_stevedore(wrk->vsl, &hint); + stv = stv_pick_stevedore(bo->vsl, &hint); AN(stv->allocobj); - o = stv->allocobj(stv, wrk, ltot, &soc); + o = stv->allocobj(stv, bo, ocp, ltot, &soc); } while (o == NULL && stv != stv0); } if (o == NULL) { - /* XXX: lend busyobj wrk's stats while we nuke */ - AZ(wrk->busyobj->stats); - wrk->busyobj->stats = &wrk->stats; /* no luck; try to free some space and keep trying */ for (i = 0; o == NULL && i < cache_param->nuke_limit; i++) { - if (EXP_NukeOne(wrk->busyobj, stv->lru) == -1) + if (EXP_NukeOne(bo, stv->lru) == -1) break; - o = stv->allocobj(stv, wrk, ltot, &soc); + o = stv->allocobj(stv, bo, ocp, ltot, &soc); } - wrk->busyobj->stats = NULL; } if (o == NULL) diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index c8c3689..80839a6 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -34,6 +34,8 @@ struct stv_objsecrets; struct stevedore; struct sess; +struct busyobj; +struct objcore; struct worker; struct lru; @@ -42,8 +44,8 @@ typedef void storage_open_f(const struct stevedore *); typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); typedef void storage_trim_f(struct storage *, size_t size); typedef void storage_free_f(struct storage *); -typedef struct object *storage_allocobj_f(struct stevedore *, - struct worker *wrk, unsigned ltot, const struct stv_objsecrets *); +typedef struct object *storage_allocobj_f(struct stevedore *, struct busyobj *, + struct objcore **, unsigned ltot, const struct stv_objsecrets *); typedef void storage_close_f(const struct stevedore *); /* Prototypes for VCL variable responders */ @@ -90,8 +92,8 @@ extern struct stevedore *stv_transient; int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx); uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity, const char *ctx); -struct object *STV_MkObject(struct worker *wrk, void *ptr, unsigned ltot, - const struct stv_objsecrets *soc); +struct object *STV_MkObject(struct busyobj *bo, struct objcore **ocp, + void *ptr, unsigned ltot, const struct stv_objsecrets *soc); struct lru *LRU_Alloc(void); void LRU_Free(struct lru *lru); diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index ff01838..c49ceac 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -460,8 +460,8 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size, */ static struct object * -smp_allocobj(struct stevedore *stv, struct worker *wrk, unsigned ltot, - const struct stv_objsecrets *soc) +smp_allocobj(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp, + unsigned ltot, const struct stv_objsecrets *soc) { struct object *o; struct storage *st; @@ -471,11 +471,11 @@ smp_allocobj(struct stevedore *stv, struct worker *wrk, unsigned ltot, struct objcore *oc; unsigned objidx; - if (wrk->sp->req->objcore == NULL) + AN(ocp); + if (*ocp == NULL) return (NULL); /* from cnt_error */ CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC); - AN(wrk->sp->req->objcore); - AN(wrk->busyobj->exp.ttl > 0.); + AN(bo->exp.ttl > 0.); ltot = IRNUP(sc, ltot); @@ -486,7 +486,7 @@ smp_allocobj(struct stevedore *stv, struct worker *wrk, unsigned ltot, assert(st->space >= ltot); ltot = st->len = st->space; - o = STV_MkObject(wrk, st->ptr, ltot, soc); + o = STV_MkObject(bo, ocp, st->ptr, ltot, soc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); o->objstore = st; From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] ce3ca92 Fix a detail I didn't catch during code review: Message-ID: commit ce3ca9285b1987203674a4445e058ce30ac750b6 Author: Poul-Henning Kamp Date: Sun Feb 19 09:59:59 2012 +0000 Fix a detail I didn't catch during code review: When WRW is used to send stuff to the backend 'req->t_resp' is not an appropriate base for the timeouts. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 3797047..7eb3372 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -889,7 +889,7 @@ int Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how); int WRW_Error(const struct worker *w); void WRW_Chunked(struct worker *w); void WRW_EndChunk(struct worker *w); -void WRW_Reserve(struct worker *w, int *fd); +void WRW_Reserve(struct worker *w, int *fd, double t0); unsigned WRW_Flush(struct worker *w); unsigned WRW_FlushRelease(struct worker *w); unsigned WRW_Write(struct worker *w, const void *ptr, int len); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 2c4d3d0..00279c2 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -113,7 +113,7 @@ ved_include(struct sess *sp, const char *src, const char *host) WS_Reset(sp->req->ws, sp_ws_wm); WS_Reset(w->aws, wrk_ws_wm); /* XXX ? */ - WRW_Reserve(sp->wrk, &sp->fd); + WRW_Reserve(sp->wrk, &sp->fd, sp->req->t_resp); if (sp->req->res_mode & RES_CHUNKED) WRW_Chunked(sp->wrk); } diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 48db272..adc5af3 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -424,7 +424,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) VDI_AddHostHeader(wrk->busyobj->bereq, vc); (void)VTCP_blocking(vc->fd); /* XXX: we should timeout instead */ - WRW_Reserve(wrk, &vc->fd); + WRW_Reserve(wrk, &vc->fd, sp->t_req); /* XXX t_resp ? */ (void)http_Write(wrk, hp, 0); /* XXX: stats ? */ /* Deal with any message-body the request might have */ diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c index 6f1fbcb..688d87d 100644 --- a/bin/varnishd/cache/cache_pipe.c +++ b/bin/varnishd/cache/cache_pipe.c @@ -78,7 +78,7 @@ PipeSession(struct sess *sp) sp->wrk->busyobj->vbc = vc; /* For panic dumping */ (void)VTCP_blocking(vc->fd); - WRW_Reserve(w, &vc->fd); + WRW_Reserve(w, &vc->fd, sp->t_req); sp->wrk->acct_tmp.hdrbytes += http_Write(w, sp->wrk->busyobj->bereq, 0); diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index bbb648c..f1e0553 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -237,7 +237,7 @@ RES_WriteObj(struct sess *sp) req = sp->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - WRW_Reserve(sp->wrk, &sp->fd); + WRW_Reserve(sp->wrk, &sp->fd, sp->req->t_resp); if (req->obj->response == 200 && req->http->conds && diff --git a/bin/varnishd/cache/cache_wrw.c b/bin/varnishd/cache/cache_wrw.c index 3685bec..a8140f5 100644 --- a/bin/varnishd/cache/cache_wrw.c +++ b/bin/varnishd/cache/cache_wrw.c @@ -56,6 +56,7 @@ struct wrw { ssize_t liov; ssize_t cliov; unsigned ciov; /* Chunked header marker */ + double t0; }; /*-------------------------------------------------------------------- @@ -69,7 +70,7 @@ WRW_Error(const struct worker *wrk) } void -WRW_Reserve(struct worker *wrk, int *fd) +WRW_Reserve(struct worker *wrk, int *fd, double t0) { struct wrw *wrw; unsigned u; @@ -93,6 +94,7 @@ WRW_Reserve(struct worker *wrk, int *fd) wrw->liov = 0; wrw->niov = 0; wrw->wfd = fd; + wrw->t0 = t0; wrk->wrw = wrw; } @@ -176,8 +178,7 @@ WRW_Flush(struct worker *wrk) * counter to prevent slowlaris attacks */ - if (VTIM_real() - wrk->sp->req->t_resp > - cache_param->send_timeout) { + if (VTIM_real() - wrw->t0 > cache_param->send_timeout) { WSL(wrk->vsl, SLT_Debug, *wrw->wfd, "Hit total send timeout, " "wrote = %zd/%zd; not retrying", From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 91d192c Eliminate wrk->sp Message-ID: commit 91d192cd2963dcd1e5ccbeaa5aee7bd9a3ab6cf2 Author: Poul-Henning Kamp Date: Sun Feb 19 10:08:32 2012 +0000 Eliminate wrk->sp diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 7eb3372..9a560c3 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -310,8 +310,6 @@ struct worker { pthread_cond_t cond; - struct sess *sp; - struct VCL_conf *vcl; struct vsl_log vsl[1]; diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 8848682..07e7f5e 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -229,7 +229,6 @@ VCA_FailSess(struct worker *wrk) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CAST_OBJ_NOTNULL(wa, (void*)wrk->aws->f, WRK_ACCEPT_MAGIC); - AZ(wrk->sp); AZ(close(wa->acceptsock)); wrk->stats.sess_drop++; vca_pace_bad(); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 338b3ab..ad0817e 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -1552,10 +1552,6 @@ CNT_Session(struct sess *sp) struct worker *wrk; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); -#if 0 - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - MPL_AssertSane(req); -#endif wrk = sp->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -1590,10 +1586,6 @@ CNT_Session(struct sess *sp) */ for (done = 0; !done; ) { assert(sp->wrk == wrk); -#if 0 - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - MPL_AssertSane(req); -#endif /* * This is a good place to be paranoid about the various * pointers still pointing to the things we expect. diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index c6253b2..29d467c 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -438,7 +438,6 @@ pool_herder(void *priv) VSC_C_main->threads--; VSC_C_main->threads_destroyed++; Lck_Unlock(&pool_mtx); - AZ(wrk->sp); wrk->task.func = NULL; wrk->task.priv = NULL; AZ(pthread_cond_signal(&wrk->cond)); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 542ec2e..10a8182 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -142,17 +142,11 @@ ses_pool_task(struct worker *wrk, void *arg) AZ(wrk->aws->r); wrk->lastused = NAN; THR_SetSession(sp); - if (wrk->sp == NULL) - wrk->sp = sp; - else - assert(wrk->sp == sp); AZ(sp->wrk); sp->wrk = wrk; CNT_Session(sp); - sp = NULL; - /* Cannot access sp now */ + sp = NULL; /* Cannot access sp any longer */ THR_SetSession(NULL); - wrk->sp = NULL; WS_Assert(wrk->aws); AZ(wrk->busyobj); AZ(wrk->wrw); @@ -171,20 +165,20 @@ void SES_pool_accept_task(struct worker *wrk, void *arg) { struct sesspool *pp; + struct sess *sp; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CAST_OBJ_NOTNULL(pp, arg, SESSPOOL_MAGIC); /* Turn accepted socket into a session */ - AZ(wrk->sp); AN(wrk->aws->r); - wrk->sp = ses_new(pp); - if (wrk->sp == NULL) { + sp = ses_new(pp); + if (sp == NULL) { VCA_FailSess(wrk); } else { - VCA_SetupSess(wrk, wrk->sp); - wrk->sp->step = STP_FIRST; - ses_pool_task(wrk, wrk->sp); + VCA_SetupSess(wrk, sp); + sp->step = STP_FIRST; + ses_pool_task(wrk, sp); } } From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 6633013 Make RFC2616_Ttl() take a busyobj Message-ID: commit 66330133c7c9bf3f454e2bea5140d8fbf29c9baa Author: Poul-Henning Kamp Date: Sun Feb 19 10:21:44 2012 +0000 Make RFC2616_Ttl() take a busyobj diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 9a560c3..a8cf350 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -997,7 +997,7 @@ char *WS_Alloc(struct ws *ws, unsigned bytes); char *WS_Snapshot(struct ws *ws); /* rfc2616.c */ -void RFC2616_Ttl(const struct sess *sp); +void RFC2616_Ttl(struct busyobj *, unsigned xid); enum body_status RFC2616_Body(const struct sess *sp); unsigned RFC2616_Req_Gzip(const struct sess *sp); int RFC2616_Do_Cond(const struct sess *sp); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index ad0817e..a58b834 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -606,7 +606,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) */ EXP_Clr(&wrk->busyobj->exp); wrk->busyobj->exp.entered = W_TIM_real(wrk); - RFC2616_Ttl(sp); + RFC2616_Ttl(wrk->busyobj, sp->req->xid); /* pass from vclrecv{} has negative TTL */ if (req->objcore == NULL) diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 2c3a368..2dafbd2 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -63,7 +63,7 @@ */ void -RFC2616_Ttl(const struct sess *sp) +RFC2616_Ttl(struct busyobj *bo, unsigned xid) { unsigned max_age, age; double h_date, h_expires; @@ -71,9 +71,10 @@ RFC2616_Ttl(const struct sess *sp) const struct http *hp; struct exp *expp; - expp = &sp->wrk->busyobj->exp; + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + expp = &bo->exp; - hp = sp->wrk->busyobj->beresp; + hp = bo->beresp; assert(expp->entered != 0.0 && !isnan(expp->entered)); /* If all else fails, cache using default ttl */ @@ -98,7 +99,7 @@ RFC2616_Ttl(const struct sess *sp) if (http_GetHdr(hp, H_Date, &p)) h_date = VTIM_parse(p); - switch (sp->req->err_code) { + switch (http_GetStatus(hp)) { default: expp->ttl = -1.; break; @@ -167,9 +168,9 @@ RFC2616_Ttl(const struct sess *sp) } /* calculated TTL, Our time, Date, Expires, max-age, age */ - VSLb(sp->req->vsl, SLT_TTL, /* XXX Bo->vsl ? */ + VSLb(bo->vsl, SLT_TTL, "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u", - sp->req->xid, expp->ttl, -1., -1., expp->entered, + xid, expp->ttl, -1., -1., expp->entered, expp->age, h_date, h_expires, max_age); } From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 3544388 More RFC2616 related argument cleanup Message-ID: commit 354438810d1eed7111647cf3ba4ec997e173b86a Author: Poul-Henning Kamp Date: Sun Feb 19 10:34:33 2012 +0000 More RFC2616 related argument cleanup diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index a8cf350..f722e7e 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -998,8 +998,8 @@ char *WS_Snapshot(struct ws *ws); /* rfc2616.c */ void RFC2616_Ttl(struct busyobj *, unsigned xid); -enum body_status RFC2616_Body(const struct sess *sp); -unsigned RFC2616_Req_Gzip(const struct sess *sp); +enum body_status RFC2616_Body(struct busyobj *, struct dstat *); +unsigned RFC2616_Req_Gzip(const struct http *); int RFC2616_Do_Cond(const struct sess *sp); /* stevedore.c */ diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index a58b834..b921fee 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -238,7 +238,7 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) } if (cache_param->http_gzip_support && req->obj->gziped && - !RFC2616_Req_Gzip(sp)) { + !RFC2616_Req_Gzip(req->http)) { /* * We don't know what it uncompresses to * XXX: we could cache that @@ -552,22 +552,24 @@ static int cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) { int i, need_host_hdr; + struct busyobj *bo; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC); + bo = wrk->busyobj; + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AN(req->director); - AZ(wrk->busyobj->vbc); - AZ(wrk->busyobj->should_close); + AZ(bo->vbc); + AZ(bo->should_close); AZ(req->storage_hint); - http_Setup(wrk->busyobj->beresp, wrk->busyobj->ws, wrk->busyobj->vsl); + http_Setup(bo->beresp, bo->ws, bo->vsl); - need_host_hdr = !http_GetHdr(wrk->busyobj->bereq, H_Host, NULL); + need_host_hdr = !http_GetHdr(bo->bereq, H_Host, NULL); i = FetchHdr(sp, need_host_hdr, req->objcore == NULL); /* @@ -589,35 +591,35 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) * and we rely on their content outside of VCL, so collect them * into one line here. */ - http_CollectHdr(wrk->busyobj->beresp, H_Cache_Control); - http_CollectHdr(wrk->busyobj->beresp, H_Vary); + http_CollectHdr(bo->beresp, H_Cache_Control); + http_CollectHdr(bo->beresp, H_Vary); /* * Figure out how the fetch is supposed to happen, before the * headers are adultered by VCL * NB: Also sets other wrk variables */ - wrk->busyobj->body_status = RFC2616_Body(sp); + bo->body_status = RFC2616_Body(bo, &wrk->stats); - req->err_code = http_GetStatus(wrk->busyobj->beresp); + req->err_code = http_GetStatus(bo->beresp); /* * What does RFC2616 think about TTL ? */ - EXP_Clr(&wrk->busyobj->exp); - wrk->busyobj->exp.entered = W_TIM_real(wrk); - RFC2616_Ttl(wrk->busyobj, sp->req->xid); + EXP_Clr(&bo->exp); + bo->exp.entered = W_TIM_real(wrk); + RFC2616_Ttl(bo, sp->req->xid); /* pass from vclrecv{} has negative TTL */ if (req->objcore == NULL) - wrk->busyobj->exp.ttl = -1.; + bo->exp.ttl = -1.; - AZ(wrk->busyobj->do_esi); - AZ(wrk->busyobj->do_pass); + AZ(bo->do_esi); + AZ(bo->do_pass); VCL_fetch_method(sp); - if (req->objcore != NULL && wrk->busyobj->do_pass) + if (req->objcore != NULL && bo->do_pass) req->objcore->flags |= OC_F_PASS; switch (req->handling) { @@ -630,11 +632,11 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) } /* We are not going to fetch the body, Close the connection */ - VDI_CloseFd(wrk, &wrk->busyobj->vbc); + VDI_CloseFd(wrk, &bo->vbc); } /* Clean up partial fetch */ - AZ(wrk->busyobj->vbc); + AZ(bo->vbc); if (req->objcore != NULL) { CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); @@ -1384,7 +1386,7 @@ cnt_recv(struct sess *sp, const struct worker *wrk, struct req *req) if (cache_param->http_gzip_support && (recv_handling != VCL_RET_PIPE) && (recv_handling != VCL_RET_PASS)) { - if (RFC2616_Req_Gzip(sp)) { + if (RFC2616_Req_Gzip(req->http)) { http_Unset(req->http, H_Accept_Encoding); http_SetHeader(req->http, "Accept-Encoding: gzip"); } else { diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 2dafbd2..60bb6a9 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -179,27 +179,27 @@ RFC2616_Ttl(struct busyobj *bo, unsigned xid) */ enum body_status -RFC2616_Body(const struct sess *sp) +RFC2616_Body(struct busyobj *bo, struct dstat *stats) { struct http *hp; char *b; - hp = sp->wrk->busyobj->beresp; + hp = bo->beresp; if (hp->protover < 11 && !http_HdrIs(hp, H_Connection, "keep-alive")) - sp->wrk->busyobj->should_close = 1; + bo->should_close = 1; else if (http_HdrIs(hp, H_Connection, "close")) - sp->wrk->busyobj->should_close = 1; + bo->should_close = 1; else - sp->wrk->busyobj->should_close = 0; + bo->should_close = 0; - if (!strcasecmp(http_GetReq(sp->wrk->busyobj->bereq), "head")) { + if (!strcasecmp(http_GetReq(bo->bereq), "head")) { /* * A HEAD request can never have a body in the reply, * no matter what the headers might say. * [RFC2516 4.3 p33] */ - sp->wrk->stats.fetch_head++; + stats->fetch_head++; return (BS_NONE); } @@ -208,7 +208,7 @@ RFC2616_Body(const struct sess *sp) * 1xx responses never have a body. * [RFC2616 4.3 p33] */ - sp->wrk->stats.fetch_1xx++; + stats->fetch_1xx++; return (BS_NONE); } @@ -217,7 +217,7 @@ RFC2616_Body(const struct sess *sp) * 204 is "No Content", obviously don't expect a body. * [RFC2616 10.2.5 p60] */ - sp->wrk->stats.fetch_204++; + stats->fetch_204++; return (BS_NONE); } @@ -226,23 +226,23 @@ RFC2616_Body(const struct sess *sp) * 304 is "Not Modified" it has no body. * [RFC2616 10.3.5 p63] */ - sp->wrk->stats.fetch_304++; + stats->fetch_304++; return (BS_NONE); } if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) { - sp->wrk->stats.fetch_chunked++; + stats->fetch_chunked++; return (BS_CHUNKED); } if (http_GetHdr(hp, H_Transfer_Encoding, &b)) { - sp->wrk->stats.fetch_bad++; + stats->fetch_bad++; return (BS_ERROR); } if (http_GetHdr(hp, H_Content_Length, - &sp->wrk->busyobj->h_content_length)) { - sp->wrk->stats.fetch_length++; + &bo->h_content_length)) { + stats->fetch_length++; return (BS_LENGTH); } @@ -251,7 +251,7 @@ RFC2616_Body(const struct sess *sp) * Keep alive with neither TE=Chunked or C-Len is impossible. * We assume a zero length body. */ - sp->wrk->stats.fetch_zero++; + stats->fetch_zero++; return (BS_ZERO); } @@ -259,7 +259,7 @@ RFC2616_Body(const struct sess *sp) /* * In this case, it is safe to just read what comes. */ - sp->wrk->stats.fetch_close++; + stats->fetch_close++; return (BS_EOF); } @@ -267,14 +267,14 @@ RFC2616_Body(const struct sess *sp) /* * With no Connection header, assume EOF. */ - sp->wrk->stats.fetch_oldhttp++; + stats->fetch_oldhttp++; return (BS_EOF); } /* * Fall back to EOF transfer. */ - sp->wrk->stats.fetch_eof++; + stats->fetch_eof++; return (BS_EOF); } @@ -283,7 +283,7 @@ RFC2616_Body(const struct sess *sp) */ unsigned -RFC2616_Req_Gzip(const struct sess *sp) +RFC2616_Req_Gzip(const struct http *hp) { @@ -292,7 +292,7 @@ RFC2616_Req_Gzip(const struct sess *sp) * p104 says to not do q values for x-gzip, so we just test * for its existence. */ - if (http_GetHdrData(sp->req->http, H_Accept_Encoding, "x-gzip", NULL)) + if (http_GetHdrData(hp, H_Accept_Encoding, "x-gzip", NULL)) return (1); /* @@ -300,7 +300,7 @@ RFC2616_Req_Gzip(const struct sess *sp) * We do not care a hoot if the client prefers some other * compression more than gzip: Varnish only does gzip. */ - if (http_GetHdrQ(sp->req->http, H_Accept_Encoding, "gzip") > 0.) + if (http_GetHdrQ(hp, H_Accept_Encoding, "gzip") > 0.) return (1); /* Bad client, no gzip. */ diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 50f4411..82aeb9c 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -355,7 +355,7 @@ VRT_r_req_can_gzip(struct sess *sp) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - return (RFC2616_Req_Gzip(sp)); + return (RFC2616_Req_Gzip(sp->req->http)); } From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 06b6c78 Detect client crashing during startup Message-ID: commit 06b6c7882458b9ffecede17f4b66f5aca1d4a3b4 Author: Poul-Henning Kamp Date: Sun Feb 19 11:05:09 2012 +0000 Detect client crashing during startup diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 880f966..c00a143 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -156,6 +156,12 @@ wait_running(const struct varnish *v) if (st != CLIS_OK) vtc_log(v->vl, 0, "CLI status command failed: %u %s", st, r); + if (!strcmp(r, "Child in state stopped")) { + vtc_log(v->vl, 0, + "Child stopped before running: %u %s", st, r); + free(r); + break; + } if (!strcmp(r, "Child in state running")) { free(r); break; From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 1bda658 Hide the hash preallocation in cache_hash.c again, by tasking persistent with allocating its own objcores. Message-ID: commit 1bda6584df023d47c6a6715089310c72b3ba2ccf Author: Poul-Henning Kamp Date: Sun Feb 19 11:41:08 2012 +0000 Hide the hash preallocation in cache_hash.c again, by tasking persistent with allocating its own objcores. diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index eff6357..ce596df 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -65,17 +65,14 @@ static const struct hash_slinger *hash; /*---------------------------------------------------------------------*/ /* Precreate an objhead and object for later use */ -void -HSH_Prealloc(const struct sess *sp) +static void +hsh_prealloc(struct worker *wrk) { - struct worker *wrk; struct objhead *oh; struct objcore *oc; struct waitinglist *wl; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); - wrk = sp->wrk; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); if (wrk->nobjcore == NULL) { ALLOC_OBJ(oc, OBJCORE_MAGIC); @@ -107,7 +104,7 @@ HSH_Prealloc(const struct sess *sp) CHECK_OBJ_NOTNULL(wrk->nwaitinglist, WAITINGLIST_MAGIC); if (hash->prep != NULL) - hash->prep(sp); + hash->prep(wrk); } void @@ -247,24 +244,26 @@ hsh_testmagic(void *result) * Return it with a reference held. */ -struct objcore * -HSH_Insert(const struct sess *sp) +void +HSH_Insert(const struct sess *sp, const void *digest, struct objcore *oc) { struct worker *wrk; struct objhead *oh; - struct objcore *oc; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); - AN(hash); + AN(digest); + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + wrk = sp->wrk; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - HSH_Prealloc(sp); + hsh_prealloc(wrk); if (cache_param->diag_bitmap & 0x80000000) - hsh_testmagic(sp->wrk->nobjhead->digest); + hsh_testmagic(wrk->nobjhead->digest); AZ(sp->req); AN(wrk->nobjhead); + memcpy(wrk->nobjhead->digest, digest, SHA256_LEN); oh = hash->lookup(sp, wrk->nobjhead); CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); if (oh == wrk->nobjhead) @@ -273,8 +272,6 @@ HSH_Insert(const struct sess *sp) assert(oh->refcnt > 0); /* Insert (precreated) objcore in objecthead */ - oc = wrk->nobjcore; - wrk->nobjcore = NULL; oc->refcnt = 1; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AZ(oc->flags & OC_F_BUSY); @@ -283,8 +280,8 @@ HSH_Insert(const struct sess *sp) /* NB: do not deref objhead the new object inherits our reference */ oc->objhead = oh; Lck_Unlock(&oh->mtx); - sp->wrk->stats.n_vampireobject++; - return (oc); + wrk->stats.n_objectcore++; + wrk->stats.n_vampireobject++; } /*--------------------------------------------------------------------- @@ -307,7 +304,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) AN(hash); wrk = sp->wrk; - HSH_Prealloc(sp); + hsh_prealloc(wrk); memcpy(sp->wrk->nobjhead->digest, sp->req->digest, sizeof sp->req->digest); if (cache_param->diag_bitmap & 0x80000000) diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 97ff04a..0c9f5d0 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -116,6 +116,7 @@ -emacro(527, NEEDLESS_RETURN) // unreachable code +-sem(EXP_Inject, custodial(1)) -sem(WS_Init, custodial(2)) -sem(http_Setup, custodial(2)) diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c index 069a1c2..fbdc906 100644 --- a/bin/varnishd/hash/hash_critbit.c +++ b/bin/varnishd/hash/hash_critbit.c @@ -469,13 +469,14 @@ hcb_lookup(const struct sess *sp, struct objhead *noh) } static void -hcb_prep(const struct sess *sp) +hcb_prep(struct worker *wrk) { struct hcb_y *y; - if (sp->wrk->nhashpriv == NULL) { + if (wrk->nhashpriv == NULL) { ALLOC_OBJ(y, HCB_Y_MAGIC); - sp->wrk->nhashpriv = y; + AN(y); + wrk->nhashpriv = y; } } diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index f2b41e2..9add938 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -34,7 +34,7 @@ struct object; typedef void hash_init_f(int ac, char * const *av); typedef void hash_start_f(void); -typedef void hash_prep_f(const struct sess *sp); +typedef void hash_prep_f(struct worker *); typedef struct objhead * hash_lookup_f(const struct sess *sp, struct objhead *nobj); typedef int hash_deref_f(struct objhead *obj); @@ -51,7 +51,6 @@ struct hash_slinger { }; /* cache_hash.c */ -void HSH_Prealloc(const struct sess *sp); void HSH_Cleanup(struct worker *w); struct objcore *HSH_Lookup(struct sess *sp, struct objhead **poh); void HSH_Unbusy(struct objcore *); @@ -59,7 +58,7 @@ void HSH_Ref(struct objcore *o); void HSH_Drop(struct worker *, struct object **); void HSH_Init(const struct hash_slinger *slinger); void HSH_AddString(const struct sess *sp, const char *str); -struct objcore *HSH_Insert(const struct sess *sp); +void HSH_Insert(const struct sess *sp, const void *hash, struct objcore *); void HSH_Purge(const struct sess *, struct objhead *, double ttl, double grace); void HSH_config(const char *h_arg); diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c index a6a02e6..4d6fd9a 100644 --- a/bin/varnishd/storage/storage_persistent_silo.c +++ b/bin/varnishd/storage/storage_persistent_silo.c @@ -150,15 +150,13 @@ smp_load_seg(const struct sess *sp, const struct smp_sc *sc, for (;no > 0; so++,no--) { if (so->ttl == 0 || so->ttl < t_now) continue; - HSH_Prealloc(sp); - oc = sp->wrk->nobjcore; + ALLOC_OBJ(oc, OBJCORE_MAGIC); + AN(oc); oc->flags |= OC_F_NEEDFIXUP | OC_F_LRUDONTMOVE; oc->flags &= ~OC_F_BUSY; smp_init_oc(oc, sg, no); oc->ban = BAN_RefBan(oc, so->ban, sc->tailban); - memcpy(sp->wrk->nobjhead->digest, so->hash, SHA256_LEN); - (void)HSH_Insert(sp); - AZ(sp->wrk->nobjcore); + HSH_Insert(sp, so->hash, oc); EXP_Inject(oc, sg->lru, so->ttl); sg->nobj++; } From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 219aa02 Eliminate struct sess from HSH_Insert() Message-ID: commit 219aa0297024653698319efa5750a19f3dccc116 Author: Poul-Henning Kamp Date: Sun Feb 19 11:58:19 2012 +0000 Eliminate struct sess from HSH_Insert() diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index a9ff554..a022cbe 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -800,7 +800,7 @@ ban_CheckLast(void) * Ban lurker thread */ -static int +static int __match_proto__(bgthread_t) ban_lurker_work(const struct sess *sp, unsigned pass) { struct ban *b, *b0, *b2; diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index e0f3d27..90702f8 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -329,7 +329,7 @@ EXP_Rearm(const struct object *o) * object expires, accounting also for graceability, it is killed. */ -static void * __match_proto__(void *start_routine(void *)) +static void * __match_proto__(bgthread_t) exp_timer(struct sess *sp, void *priv) { struct objcore *oc; diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index ce596df..15b0a2a 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -245,26 +245,21 @@ hsh_testmagic(void *result) */ void -HSH_Insert(const struct sess *sp, const void *digest, struct objcore *oc) +HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc) { - struct worker *wrk; struct objhead *oh; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AN(digest); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - wrk = sp->wrk; - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - hsh_prealloc(wrk); if (cache_param->diag_bitmap & 0x80000000) hsh_testmagic(wrk->nobjhead->digest); - AZ(sp->req); AN(wrk->nobjhead); memcpy(wrk->nobjhead->digest, digest, SHA256_LEN); - oh = hash->lookup(sp, wrk->nobjhead); + oh = hash->lookup(wrk, wrk->nobjhead); CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); if (oh == wrk->nobjhead) wrk->nobjhead = NULL; @@ -320,7 +315,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) sp->req->hash_objhead = NULL; } else { AN(wrk->nobjhead); - oh = hash->lookup(sp, wrk->nobjhead); + oh = hash->lookup(wrk, wrk->nobjhead); if (oh == wrk->nobjhead) wrk->nobjhead = NULL; } diff --git a/bin/varnishd/hash/hash_classic.c b/bin/varnishd/hash/hash_classic.c index b530f25..c84238a 100644 --- a/bin/varnishd/hash/hash_classic.c +++ b/bin/varnishd/hash/hash_classic.c @@ -54,7 +54,7 @@ static struct hcl_hd *hcl_head; * The ->init method allows the management process to pass arguments */ -static void +static void __match_proto__(hash_init_f) hcl_init(int ac, char * const *av) { int i; @@ -86,7 +86,7 @@ hcl_init(int ac, char * const *av) * initialization to happen before the first lookup. */ -static void +static void __match_proto__(hash_start_f) hcl_start(void) { unsigned u; @@ -110,15 +110,15 @@ hcl_start(void) * rare and collisions even rarer. */ -static struct objhead * -hcl_lookup(const struct sess *sp, struct objhead *noh) +static struct objhead * __match_proto__(hash_lookup_f) +hcl_lookup(struct worker *wrk, struct objhead *noh) { struct objhead *oh; struct hcl_hd *hp; unsigned u1, digest; int i; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC); assert(sizeof noh->digest > sizeof digest); @@ -153,7 +153,7 @@ hcl_lookup(const struct sess *sp, struct objhead *noh) * Dereference and if no references are left, free. */ -static int +static int __match_proto__(hash_deref_f) hcl_deref(struct objhead *oh) { struct hcl_hd *hp; diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c index fbdc906..9edef51 100644 --- a/bin/varnishd/hash/hash_critbit.c +++ b/bin/varnishd/hash/hash_critbit.c @@ -373,7 +373,7 @@ hcb_cleaner(struct sess *sp, void *priv) /*--------------------------------------------------------------------*/ -static void +static void __match_proto__(hash_start_f) hcb_start(void) { struct objhead *oh = NULL; @@ -387,7 +387,7 @@ hcb_start(void) hcb_build_bittbl(); } -static int +static int __match_proto__(hash_deref_f) hcb_deref(struct objhead *oh) { int r; @@ -412,28 +412,28 @@ hcb_deref(struct objhead *oh) return (r); } -static struct objhead * -hcb_lookup(const struct sess *sp, struct objhead *noh) +static struct objhead * __match_proto__(hash_lookup_f) +hcb_lookup(struct worker *wrk, struct objhead *noh) { struct objhead *oh; struct hcb_y *y; unsigned u; unsigned with_lock; - (void)sp; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); with_lock = 0; while (1) { if (with_lock) { - CAST_OBJ_NOTNULL(y, sp->wrk->nhashpriv, HCB_Y_MAGIC); + CAST_OBJ_NOTNULL(y, wrk->nhashpriv, HCB_Y_MAGIC); Lck_Lock(&hcb_mtx); VSC_C_main->hcb_lock++; assert(noh->refcnt == 1); - oh = hcb_insert(sp->wrk, &hcb_root, noh, 1); + oh = hcb_insert(wrk, &hcb_root, noh, 1); Lck_Unlock(&hcb_mtx); } else { VSC_C_main->hcb_nolock++; - oh = hcb_insert(sp->wrk, &hcb_root, noh, 0); + oh = hcb_insert(wrk, &hcb_root, noh, 0); } if (oh != NULL && oh == noh) { @@ -468,7 +468,7 @@ hcb_lookup(const struct sess *sp, struct objhead *noh) } } -static void +static void __match_proto__(hash_prep_f) hcb_prep(struct worker *wrk) { struct hcb_y *y; diff --git a/bin/varnishd/hash/hash_simple_list.c b/bin/varnishd/hash/hash_simple_list.c index c1cc9c8..fef2b65 100644 --- a/bin/varnishd/hash/hash_simple_list.c +++ b/bin/varnishd/hash/hash_simple_list.c @@ -45,7 +45,7 @@ static struct lock hsl_mtx; * initialization to happen before the first lookup. */ -static void +static void __match_proto__(hash_start_f) hsl_start(void) { @@ -59,13 +59,13 @@ hsl_start(void) * A reference to the returned object is held. */ -static struct objhead * -hsl_lookup(const struct sess *sp, struct objhead *noh) +static struct objhead * __match_proto__(hash_lookup_f) +hsl_lookup(struct worker *wrk, struct objhead *noh) { struct objhead *oh; int i; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC); Lck_Lock(&hsl_mtx); VTAILQ_FOREACH(oh, &hsl_head, hoh_list) { @@ -92,7 +92,7 @@ hsl_lookup(const struct sess *sp, struct objhead *noh) * Dereference and if no references are left, free. */ -static int +static int __match_proto__(hash_deref_f) hsl_deref(struct objhead *oh) { int ret; diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index 9add938..b15d9df 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -36,7 +36,7 @@ typedef void hash_init_f(int ac, char * const *av); typedef void hash_start_f(void); typedef void hash_prep_f(struct worker *); typedef struct objhead * - hash_lookup_f(const struct sess *sp, struct objhead *nobj); + hash_lookup_f(struct worker *wrk, struct objhead *nobj); typedef int hash_deref_f(struct objhead *obj); struct hash_slinger { @@ -58,7 +58,7 @@ void HSH_Ref(struct objcore *o); void HSH_Drop(struct worker *, struct object **); void HSH_Init(const struct hash_slinger *slinger); void HSH_AddString(const struct sess *sp, const char *str); -void HSH_Insert(const struct sess *sp, const void *hash, struct objcore *); +void HSH_Insert(struct worker *, const void *hash, struct objcore *); void HSH_Purge(const struct sess *, struct objhead *, double ttl, double grace); void HSH_config(const char *h_arg); diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index c49ceac..5e2db82 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -267,19 +267,19 @@ smp_open_segs(struct smp_sc *sc, struct smp_signctx *ctx) * Silo worker thread */ -static void * +static void * __match_proto__(bgthread_t) smp_thread(struct sess *sp, void *priv) { struct smp_sc *sc; struct smp_seg *sg; - (void)sp; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CAST_OBJ_NOTNULL(sc, priv, SMP_SC_MAGIC); /* First, load all the objects from all segments */ VTAILQ_FOREACH(sg, &sc->segments, list) if (sg->flags & SMP_SEG_MUSTLOAD) - smp_load_seg(sp, sc, sg); + smp_load_seg(sp->wrk, sc, sg); sc->flags |= SMP_SC_LOADED; BAN_TailDeref(&sc->tailban); diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h index 84f3d21..a755b89 100644 --- a/bin/varnishd/storage/storage_persistent.h +++ b/bin/varnishd/storage/storage_persistent.h @@ -167,8 +167,7 @@ void smp_mgt_init(struct stevedore *parent, int ac, char * const *av); /* storage_persistent_silo.c */ -void smp_load_seg(const struct sess *sp, const struct smp_sc *sc, - struct smp_seg *sg); +void smp_load_seg(struct worker *, const struct smp_sc *sc, struct smp_seg *sg); void smp_new_seg(struct smp_sc *sc); void smp_close_seg(struct smp_sc *sc, struct smp_seg *sg); void smp_init_oc(struct objcore *oc, struct smp_seg *sg, unsigned objidx); diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c index 4d6fd9a..09b2497 100644 --- a/bin/varnishd/storage/storage_persistent_silo.c +++ b/bin/varnishd/storage/storage_persistent_silo.c @@ -118,7 +118,7 @@ smp_save_segs(struct smp_sc *sc) */ void -smp_load_seg(const struct sess *sp, const struct smp_sc *sc, +smp_load_seg(struct worker *wrk, const struct smp_sc *sc, struct smp_seg *sg) { struct smp_object *so; @@ -128,7 +128,7 @@ smp_load_seg(const struct sess *sp, const struct smp_sc *sc, struct smp_signctx ctx[1]; ASSERT_SILO_THREAD(sc); - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC); CHECK_OBJ_NOTNULL(sg->lru, LRU_MAGIC); assert(sg->flags & SMP_SEG_MUSTLOAD); @@ -156,11 +156,11 @@ smp_load_seg(const struct sess *sp, const struct smp_sc *sc, oc->flags &= ~OC_F_BUSY; smp_init_oc(oc, sg, no); oc->ban = BAN_RefBan(oc, so->ban, sc->tailban); - HSH_Insert(sp, so->hash, oc); + HSH_Insert(wrk, so->hash, oc); EXP_Inject(oc, sg->lru, so->ttl); sg->nobj++; } - WRK_SumStat(sp->wrk); + WRK_SumStat(wrk); sg->flags |= SMP_SEG_LOADED; } From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] e632739 Eliminate the ban-lurkers need for struct sess Message-ID: commit e63273932d7d5945e20eb44f506ae8c005593f2c Author: Poul-Henning Kamp Date: Sun Feb 19 12:07:32 2012 +0000 Eliminate the ban-lurkers need for struct sess diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index a022cbe..e72343b 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -677,16 +677,17 @@ ban_evaluate(const uint8_t *bs, const struct http *objhttp, */ static int -ban_check_object(struct object *o, const struct sess *sp, int has_req) +ban_check_object(struct object *o, struct worker *wrk, + const struct http *req_http) { struct ban *b; struct objcore *oc; struct ban * volatile b0; unsigned tests, skipped; - struct http *http; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_ORNULL(req_http, HTTP_MAGIC); oc = o->objcore; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CHECK_OBJ_NOTNULL(oc->ban, BAN_MAGIC); @@ -697,11 +698,6 @@ ban_check_object(struct object *o, const struct sess *sp, int has_req) if (b0 == oc->ban) return (0); - if (has_req) - http = sp->req->http; - else - http = NULL; - /* * This loop is safe without locks, because we know we hold * a refcount on a ban somewhere in the list and we do not @@ -719,13 +715,13 @@ ban_check_object(struct object *o, const struct sess *sp, int has_req) /* Lurker already tested this */ continue; } - if (!has_req && (b->flags & BAN_F_REQ)) { + if (req_http == NULL && (b->flags & BAN_F_REQ)) { /* * We cannot test this one, but there might * be other bans that match, so we soldier on */ skipped++; - } else if (ban_evaluate(b->spec, o->http, http, &tests)) + } else if (ban_evaluate(b->spec, o->http, req_http, &tests)) break; } @@ -734,7 +730,7 @@ ban_check_object(struct object *o, const struct sess *sp, int has_req) VSC_C_main->bans_tests_tested += tests; if (b == oc->ban && skipped > 0) { - AZ(has_req); + AZ(req_http); Lck_Unlock(&ban_mtx); /* * Not banned, but some tests were skipped, so we cannot know @@ -762,7 +758,7 @@ ban_check_object(struct object *o, const struct sess *sp, int has_req) oc_updatemeta(oc); /* BAN also changed, but that is not important any more */ /* XXX: no req in lurker */ - VSLb(sp->wrk->vsl, SLT_ExpBan, "%u was banned", o->xid); + VSLb(wrk->vsl, SLT_ExpBan, "%u was banned", o->xid); EXP_Rearm(o); return (1); } @@ -772,7 +768,7 @@ int BAN_CheckObject(struct object *o, const struct sess *sp) { - return (ban_check_object(o, sp, 1) > 0); + return (ban_check_object(o, sp->wrk, sp->req->http) > 0); } static struct ban * @@ -800,8 +796,8 @@ ban_CheckLast(void) * Ban lurker thread */ -static int __match_proto__(bgthread_t) -ban_lurker_work(const struct sess *sp, unsigned pass) +static int +ban_lurker_work(struct worker *wrk, unsigned pass) { struct ban *b, *b0, *b2; struct objhead *oh; @@ -892,8 +888,8 @@ ban_lurker_work(const struct sess *sp, unsigned pass) /* * Get the object and check it against all relevant bans */ - o = oc_getobj(&sp->wrk->stats, oc); - i = ban_check_object(o, sp, 0); + o = oc_getobj(&wrk->stats, oc); + i = ban_check_object(o, wrk, NULL); if (cache_param->diag_bitmap & 0x80000) VSL(SLT_Debug, 0, "lurker got: %p %d", oc, i); @@ -909,7 +905,7 @@ ban_lurker_work(const struct sess *sp, unsigned pass) if (cache_param->diag_bitmap & 0x80000) VSL(SLT_Debug, 0, "lurker done: %p %u %u", oc, oc->flags & OC_F_LURK, pass); - (void)HSH_Deref(&sp->wrk->stats, NULL, &o); + (void)HSH_Deref(&wrk->stats, NULL, &o); VTIM_sleep(cache_param->ban_lurker_sleep); } Lck_AssertHeld(&ban_mtx); @@ -954,7 +950,7 @@ ban_lurker(struct sess *sp, void *priv) VTIM_sleep(1.0); } - i = ban_lurker_work(sp, pass); + i = ban_lurker_work(sp->wrk, pass); VSL_Flush(sp->wrk->vsl, 0); WRK_SumStat(sp->wrk); if (i) { From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] a1cdab1 Remove struct sess entirely from bgthreads Message-ID: commit a1cdab139ba08adcd6703aa96042036b1fa39a6a Author: Poul-Henning Kamp Date: Sun Feb 19 12:16:38 2012 +0000 Remove struct sess entirely from bgthreads diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index f722e7e..c4efc49 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -981,7 +981,7 @@ void WRK_Init(void); int WRK_TrySumStat(struct worker *w); void WRK_SumStat(struct worker *w); void *WRK_thread(void *priv); -typedef void *bgthread_t(struct sess *, void *priv); +typedef void *bgthread_t(struct worker *, void *priv); void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv); diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index e72343b..2c95f61 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -927,7 +927,7 @@ ban_lurker_work(struct worker *wrk, unsigned pass) } static void * __match_proto__(bgthread_t) -ban_lurker(struct sess *sp, void *priv) +ban_lurker(struct worker *wrk, void *priv) { struct ban *bf; unsigned pass = (1 << LURK_SHIFT); @@ -950,9 +950,9 @@ ban_lurker(struct sess *sp, void *priv) VTIM_sleep(1.0); } - i = ban_lurker_work(sp->wrk, pass); - VSL_Flush(sp->wrk->vsl, 0); - WRK_SumStat(sp->wrk); + i = ban_lurker_work(wrk, pass); + VSL_Flush(wrk->vsl, 0); + WRK_SumStat(wrk); if (i) { pass += (1 << LURK_SHIFT); pass &= BAN_F_LURK; diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index 90702f8..8eedb71 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -330,7 +330,7 @@ EXP_Rearm(const struct object *o) */ static void * __match_proto__(bgthread_t) -exp_timer(struct sess *sp, void *priv) +exp_timer(struct worker *wrk, void *priv) { struct objcore *oc; struct lru *lru; @@ -342,8 +342,8 @@ exp_timer(struct sess *sp, void *priv) oc = NULL; while (1) { if (oc == NULL) { - VSL_Flush(sp->wrk->vsl, 0); - WRK_SumStat(sp->wrk); + VSL_Flush(wrk->vsl, 0); + WRK_SumStat(wrk); VTIM_sleep(cache_param->expiry_sleep); t = VTIM_real(); } @@ -399,10 +399,10 @@ exp_timer(struct sess *sp, void *priv) VSC_C_main->n_expired++; CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); - o = oc_getobj(&sp->wrk->stats, oc); - WSL(sp->wrk->vsl, SLT_ExpKill, 0, "%u %.0f", - oc_getxid(&sp->wrk->stats, oc), EXP_Ttl(NULL, o) - t); - (void)HSH_Deref(&sp->wrk->stats, oc, NULL); + o = oc_getobj(&wrk->stats, oc); + WSL(wrk->vsl, SLT_ExpKill, 0, "%u %.0f", + oc_getxid(&wrk->stats, oc), EXP_Ttl(NULL, o) - t); + (void)HSH_Deref(&wrk->stats, oc, NULL); } NEEDLESS_RETURN(NULL); } diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 6862a1f..d28a58c 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -94,19 +94,15 @@ wrk_bgthread(void *arg) { struct bgthread *bt; struct worker wrk; - struct sess *sp; uint32_t logbuf[1024]; /* XXX: size ? */ CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC); THR_SetName(bt->name); - sp = SES_Alloc(); - XXXAN(sp); memset(&wrk, 0, sizeof wrk); - sp->wrk = &wrk; wrk.magic = WORKER_MAGIC; VSL_Setup(wrk.vsl, logbuf, sizeof logbuf); - (void)bt->func(sp, bt->priv); + (void)bt->func(&wrk, bt->priv); WRONG("BgThread terminated"); diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c index 9edef51..535661e 100644 --- a/bin/varnishd/hash/hash_critbit.c +++ b/bin/varnishd/hash/hash_critbit.c @@ -345,10 +345,9 @@ static struct cli_proto hcb_cmds[] = { /*--------------------------------------------------------------------*/ static void * __match_proto__(bgthread_t) -hcb_cleaner(struct sess *sp, void *priv) +hcb_cleaner(struct worker *wrk, void *priv) { struct hcb_y *y, *y2; - struct worker *wrk = sp->wrk; struct objhead *oh, *oh2; (void)priv; diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index 5e2db82..58c6f29 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -268,18 +268,18 @@ smp_open_segs(struct smp_sc *sc, struct smp_signctx *ctx) */ static void * __match_proto__(bgthread_t) -smp_thread(struct sess *sp, void *priv) +smp_thread(struct worker *wrk, void *priv) { struct smp_sc *sc; struct smp_seg *sg; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CAST_OBJ_NOTNULL(sc, priv, SMP_SC_MAGIC); /* First, load all the objects from all segments */ VTAILQ_FOREACH(sg, &sc->segments, list) if (sg->flags & SMP_SEG_MUSTLOAD) - smp_load_seg(sp->wrk, sc, sg); + smp_load_seg(wrk, sc, sg); sc->flags |= SMP_SC_LOADED; BAN_TailDeref(&sc->tailban); @@ -288,8 +288,7 @@ smp_thread(struct sess *sp, void *priv) while (1) { (void)sleep (1); sg = VTAILQ_FIRST(&sc->segments); - if (sg != NULL && sg -> sc->cur_seg && - sg->nobj == 0) { + if (sg != NULL && sg -> sc->cur_seg && sg->nobj == 0) { Lck_Lock(&sc->mtx); smp_save_segs(sc); Lck_Unlock(&sc->mtx); From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] ef74b85 Move the busyobj reference from worker to req Message-ID: commit ef74b856eaac7944960a0af957de842519f9d22e Author: Poul-Henning Kamp Date: Sun Feb 19 12:51:42 2012 +0000 Move the busyobj reference from worker to req diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index c4efc49..4b5fcd8 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -316,7 +316,6 @@ struct worker { struct ws aws[1]; - struct busyobj *busyobj; /* Temporary accounting */ struct acct acct_tmp; @@ -551,6 +550,7 @@ struct req { /* The busy objhead we sleep on */ struct objhead *hash_objhead; + struct busyobj *busyobj; /* Built Vary string */ uint8_t *vary_b; @@ -750,7 +750,7 @@ struct storage *FetchStorage(struct busyobj *, ssize_t sz); int FetchError(struct busyobj *, const char *error); int FetchError2(struct busyobj *, const char *error, const char *more); int FetchHdr(struct sess *sp, int need_host_hdr, int sendbody); -int FetchBody(struct worker *w, struct object *obj); +int FetchBody(struct worker *w, struct busyobj *bo, struct object *obj); int FetchReqBody(const struct sess *sp, int sendbody); void Fetch_Init(void); @@ -894,7 +894,6 @@ unsigned WRW_Write(struct worker *w, const void *ptr, int len); unsigned WRW_WriteH(struct worker *w, const txt *hh, const char *suf); /* cache_session.c [SES] */ -struct sess *SES_Alloc(void); void SES_Close(struct sess *sp, const char *reason); void SES_Delete(struct sess *sp, const char *reason, double now); void SES_Charge(struct sess *sp); diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 085c7a7..cfe86ec 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -89,8 +89,8 @@ VBE_ReleaseConn(struct vbc *vc) #define FIND_TMO(tmx, dst, sp, be) \ do { \ - CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC); \ - dst = sp->wrk->busyobj->tmx; \ + CHECK_OBJ_NOTNULL(sp->req->busyobj, BUSYOBJ_MAGIC); \ + dst = sp->req->busyobj->tmx; \ if (dst == 0.0) \ dst = be->tmx; \ if (dst == 0.0) \ diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index 2c95f61..0058c3c 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -796,7 +796,7 @@ ban_CheckLast(void) * Ban lurker thread */ -static int +static int ban_lurker_work(struct worker *wrk, unsigned pass) { struct ban *b, *b0, *b2; diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index b921fee..a56e016 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -201,29 +201,32 @@ DOT } static int cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) { + struct busyobj *bo; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + bo = req->busyobj; + CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - if (wrk->busyobj != NULL) { - CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC); - AN(wrk->busyobj->do_stream); + if (bo != NULL) { + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + AN(bo->do_stream); AssertObjCorePassOrBusy(req->obj->objcore); } req->res_mode = 0; - if (wrk->busyobj == NULL) + if (bo == NULL) req->res_mode |= RES_LEN; - if (wrk->busyobj != NULL && - (wrk->busyobj->h_content_length != NULL || - !wrk->busyobj->do_stream) && - !wrk->busyobj->do_gzip && !wrk->busyobj->do_gunzip) + if (bo != NULL && + (bo->h_content_length != NULL || + !bo->do_stream) && + !bo->do_gzip && !bo->do_gunzip) req->res_mode |= RES_LEN; if (!req->disable_esi && req->obj->esidata != NULL) { @@ -249,7 +252,7 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) if (!(req->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) { if (req->obj->len == 0 && - (wrk->busyobj == NULL || !wrk->busyobj->do_stream)) + (bo == NULL || !bo->do_stream)) /* * If the object is empty, neither ESI nor GUNZIP * can make it any different size @@ -282,11 +285,11 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) case VCL_RET_RESTART: if (req->restarts >= cache_param->max_restarts) break; - if (wrk->busyobj != NULL) { - AN(wrk->busyobj->do_stream); - VDI_CloseFd(wrk, &wrk->busyobj->vbc); + if (bo != NULL) { + AN(bo->do_stream); + VDI_CloseFd(wrk, &bo->vbc); HSH_Drop(wrk, &sp->req->obj); - VBO_DerefBusyObj(wrk, &wrk->busyobj); + VBO_DerefBusyObj(wrk, &bo); } else { (void)HSH_Deref(&wrk->stats, NULL, &req->obj); } @@ -325,7 +328,7 @@ cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - AZ(sp->wrk->busyobj); + AZ(req->busyobj); req->director = NULL; req->restarts = 0; @@ -364,12 +367,10 @@ cnt_done(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_ORNULL(req->vcl, VCL_CONF_MAGIC); AZ(req->obj); - AZ(wrk->busyobj); + AZ(req->busyobj); req->director = NULL; req->restarts = 0; - wrk->busyobj = NULL; - SES_Charge(sp); /* If we did an ESI include, don't mess up our state */ @@ -464,6 +465,7 @@ static int cnt_error(struct sess *sp, struct worker *wrk, struct req *req) { struct http *h; + struct busyobj *bo; char date[40]; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); @@ -471,21 +473,22 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AZ(req->objcore); AZ(req->obj); - AZ(wrk->busyobj); + AZ(req->busyobj); - wrk->busyobj = VBO_GetBusyObj(wrk); - wrk->busyobj->vsl->wid = sp->vsl_id; - AZ(wrk->busyobj->stats); - wrk->busyobj->stats = &wrk->stats; - req->obj = STV_NewObject(wrk->busyobj, &req->objcore, + bo = VBO_GetBusyObj(wrk); + req->busyobj = bo; + bo->vsl->wid = sp->vsl_id; + AZ(bo->stats); + bo->stats = &wrk->stats; + req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE, cache_param->http_resp_size, (uint16_t)cache_param->http_max_hdr); - wrk->busyobj->stats = NULL; + bo->stats = NULL; if (req->obj == NULL) { req->doclose = "Out of objects"; req->director = NULL; - http_Teardown(wrk->busyobj->beresp); - http_Teardown(wrk->busyobj->bereq); + http_Teardown(bo->beresp); + http_Teardown(bo->bereq); sp->step = STP_DONE; return(0); } @@ -513,7 +516,7 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) if (req->handling == VCL_RET_RESTART && req->restarts < cache_param->max_restarts) { HSH_Drop(wrk, &sp->req->obj); - VBO_DerefBusyObj(wrk, &wrk->busyobj); + VBO_DerefBusyObj(wrk, &req->busyobj); req->director = NULL; req->restarts++; sp->step = STP_RECV; @@ -529,8 +532,8 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) assert(req->handling == VCL_RET_DELIVER); req->err_code = 0; req->err_reason = NULL; - http_Teardown(wrk->busyobj->bereq); - VBO_DerefBusyObj(wrk, &wrk->busyobj); + http_Teardown(bo->bereq); + VBO_DerefBusyObj(wrk, &req->busyobj); sp->step = STP_PREPRESP; return (0); } @@ -559,7 +562,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - bo = wrk->busyobj; + bo = req->busyobj; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AN(req->director); @@ -643,7 +646,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) AZ(HSH_Deref(&wrk->stats, req->objcore, NULL)); req->objcore = NULL; } - VBO_DerefBusyObj(wrk, &wrk->busyobj); + VBO_DerefBusyObj(wrk, &req->busyobj); req->director = NULL; req->storage_hint = NULL; @@ -688,7 +691,7 @@ cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - bo = wrk->busyobj; + bo = req->busyobj; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); assert(req->handling == VCL_RET_DELIVER); @@ -791,7 +794,6 @@ cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req) req->objcore == NULL) req->storage_hint = TRANSIENT_STORAGE; - assert(bo == wrk->busyobj); AZ(bo->stats); bo->stats = &wrk->stats; req->obj = STV_NewObject(bo, &req->objcore, req->storage_hint, l, @@ -813,7 +815,7 @@ cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req) req->err_code = 503; sp->step = STP_ERROR; VDI_CloseFd(wrk, &bo->vbc); - VBO_DerefBusyObj(wrk, &wrk->busyobj); + VBO_DerefBusyObj(wrk, &req->busyobj); return (0); } CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); @@ -889,11 +891,11 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - bo = wrk->busyobj; + bo = req->busyobj; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); /* Use unmodified headers*/ - i = FetchBody(wrk, req->obj); + i = FetchBody(wrk, bo, req->obj); http_Teardown(bo->bereq); http_Teardown(bo->beresp); @@ -904,7 +906,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) if (i) { HSH_Drop(wrk, &sp->req->obj); - VBO_DerefBusyObj(wrk, &wrk->busyobj); + VBO_DerefBusyObj(wrk, &req->busyobj); AZ(req->obj); req->err_code = 503; sp->step = STP_ERROR; @@ -918,7 +920,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) AZ(req->obj->ws_o->overflow); HSH_Unbusy(req->obj->objcore); } - VBO_DerefBusyObj(wrk, &wrk->busyobj); + VBO_DerefBusyObj(wrk, &req->busyobj); wrk->acct_tmp.fetch++; sp->step = STP_PREPRESP; return (0); @@ -997,15 +999,15 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); AZ(req->objcore); - AZ(wrk->busyobj); + AZ(req->busyobj); assert(!(req->obj->objcore->flags & OC_F_PASS)); VCL_hit_method(sp); if (req->handling == VCL_RET_DELIVER) { - //AZ(wrk->busyobj->bereq->ws); - //AZ(wrk->busyobj->beresp->ws); + //AZ(req->busyobj->bereq->ws); + //AZ(req->busyobj->beresp->ws); (void)FetchReqBody(sp, 0); sp->step = STP_PREPRESP; return (0); @@ -1064,7 +1066,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) AZ(req->objcore); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - AZ(wrk->busyobj); + AZ(req->busyobj); if (req->hash_objhead == NULL) { /* Not a waiting list return */ @@ -1113,7 +1115,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) req->vary_e = NULL; req->objcore = oc; - CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(req->busyobj, BUSYOBJ_MAGIC); sp->step = STP_MISS; return (0); } @@ -1159,40 +1161,42 @@ DOT static int cnt_miss(struct sess *sp, struct worker *wrk, struct req *req) { + struct busyobj *bo; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); - CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC); + bo = req->busyobj; + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AZ(req->obj); - http_Setup(wrk->busyobj->bereq, wrk->busyobj->ws, wrk->busyobj->vsl); + http_Setup(bo->bereq, bo->ws, bo->vsl); http_FilterReq(sp, HTTPH_R_FETCH); - http_ForceGet(wrk->busyobj->bereq); + http_ForceGet(bo->bereq); if (cache_param->http_gzip_support) { /* * We always ask the backend for gzip, even if the * client doesn't grok it. We will uncompress for * the minority of clients which don't. */ - http_Unset(wrk->busyobj->bereq, H_Accept_Encoding); - http_SetHeader(wrk->busyobj->bereq, "Accept-Encoding: gzip"); + http_Unset(bo->bereq, H_Accept_Encoding); + http_SetHeader(bo->bereq, "Accept-Encoding: gzip"); } VCL_miss_method(sp); if (req->handling == VCL_RET_FETCH) { - CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); sp->step = STP_FETCH; return (0); } AZ(HSH_Deref(&wrk->stats, req->objcore, NULL)); req->objcore = NULL; - http_Teardown(wrk->busyobj->bereq); - VBO_DerefBusyObj(wrk, &wrk->busyobj); + http_Teardown(bo->bereq); + VBO_DerefBusyObj(wrk, &req->busyobj); switch(req->handling) { case VCL_RET_ERROR: @@ -1230,7 +1234,7 @@ XDOT err_pass [label="ERROR",shape=plaintext] */ static int -cnt_pass(struct sess *sp, struct worker *wrk, const struct req *req) +cnt_pass(struct sess *sp, struct worker *wrk, struct req *req) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -1238,18 +1242,18 @@ cnt_pass(struct sess *sp, struct worker *wrk, const struct req *req) CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); AZ(req->objcore); AZ(req->obj); - AZ(wrk->busyobj); + AZ(req->busyobj); - wrk->busyobj = VBO_GetBusyObj(wrk); - wrk->busyobj->vsl->wid = sp->vsl_id; - http_Setup(wrk->busyobj->bereq, wrk->busyobj->ws, wrk->busyobj->vsl); + req->busyobj = VBO_GetBusyObj(wrk); + req->busyobj->vsl->wid = sp->vsl_id; + http_Setup(req->busyobj->bereq, req->busyobj->ws, req->busyobj->vsl); http_FilterReq(sp, HTTPH_R_PASS); VCL_pass_method(sp); if (req->handling == VCL_RET_ERROR) { - http_Teardown(wrk->busyobj->bereq); - VBO_DerefBusyObj(wrk, &wrk->busyobj); + http_Teardown(req->busyobj->bereq); + VBO_DerefBusyObj(wrk, &req->busyobj); sp->step = STP_ERROR; return (0); } @@ -1285,19 +1289,19 @@ DOT err_pipe [label="ERROR",shape=plaintext] */ static int -cnt_pipe(struct sess *sp, struct worker *wrk, const struct req *req) +cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - AZ(wrk->busyobj); + AZ(req->busyobj); wrk->acct_tmp.pipe++; - wrk->busyobj = VBO_GetBusyObj(wrk); - wrk->busyobj->vsl->wid = sp->vsl_id; - http_Setup(wrk->busyobj->bereq, wrk->busyobj->ws, wrk->busyobj->vsl); + req->busyobj = VBO_GetBusyObj(wrk); + req->busyobj->vsl->wid = sp->vsl_id; + http_Setup(req->busyobj->bereq, req->busyobj->ws, req->busyobj->vsl); http_FilterReq(sp, 0); VCL_pipe_method(sp); @@ -1308,8 +1312,8 @@ cnt_pipe(struct sess *sp, struct worker *wrk, const struct req *req) PipeSession(sp); assert(WRW_IsReleased(wrk)); - http_Teardown(wrk->busyobj->bereq); - VBO_DerefBusyObj(wrk, &wrk->busyobj); + http_Teardown(req->busyobj->bereq); + VBO_DerefBusyObj(wrk, &req->busyobj); sp->step = STP_DONE; return (0); } @@ -1354,7 +1358,7 @@ cnt_recv(struct sess *sp, const struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); AZ(req->obj); - AZ(wrk->busyobj); + AZ(req->busyobj); /* By default we use the first backend */ AZ(req->director); diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c index ab74b59..3f70cb7 100644 --- a/bin/varnishd/cache/cache_dir.c +++ b/bin/varnishd/cache/cache_dir.c @@ -120,7 +120,7 @@ VDI_GetFd(const struct director *d, struct sess *sp) CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); vc = d->getfd(d, sp); if (vc != NULL) { - vc->vsl = sp->wrk->busyobj->vsl; + vc->vsl = sp->req->busyobj->vsl; vc->orig_vsl_id = vc->vsl->wid; vc->vsl->wid = vc->vsl_id; } diff --git a/bin/varnishd/cache/cache_dir_dns.c b/bin/varnishd/cache/cache_dir_dns.c index 80310b9..9cad3f6 100644 --- a/bin/varnishd/cache/cache_dir_dns.c +++ b/bin/varnishd/cache/cache_dir_dns.c @@ -347,8 +347,8 @@ vdi_dns_find_backend(const struct sess *sp, struct vdi_dns *vs) /* bereq is only present after recv et. al, otherwise use req (ie: * use req for health checks in vcl_recv and such). */ - if (sp->wrk->busyobj != NULL && sp->wrk->busyobj->bereq) - hp = sp->wrk->busyobj->bereq; + if (sp->req->busyobj != NULL && sp->req->busyobj->bereq) + hp = sp->req->busyobj->bereq; else hp = sp->req->http; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index adc5af3..24abc7a 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -46,7 +46,7 @@ static unsigned fetchfrag; * We want to issue the first error we encounter on fetching and * supress the rest. This function does that. * - * Other code is allowed to look at wrk->busyobj->fetch_failed to bail out + * Other code is allowed to look at sp->req->busyobj->fetch_failed to bail out * * For convenience, always return -1 */ @@ -385,33 +385,37 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) { struct vbc *vc; struct worker *wrk; + struct req *req; + struct busyobj *bo; struct http *hp; int retry = -1; int i; struct http_conn *htc; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); wrk = sp->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC); - htc = &wrk->busyobj->htc; + req = sp->req; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + bo = req->busyobj; + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + htc = &bo->htc; - AN(sp->req->director); - AZ(sp->req->obj); + AN(req->director); + AZ(req->obj); - if (sp->req->objcore != NULL) { /* pass has no objcore */ - CHECK_OBJ_NOTNULL(sp->req->objcore, OBJCORE_MAGIC); - AN(sp->req->objcore->flags & OC_F_BUSY); + if (req->objcore != NULL) { /* pass has no objcore */ + CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); + AN(req->objcore->flags & OC_F_BUSY); } - hp = wrk->busyobj->bereq; + hp = bo->bereq; - wrk->busyobj->vbc = VDI_GetFd(NULL, sp); - if (wrk->busyobj->vbc == NULL) { - VSLb(sp->req->vsl, SLT_FetchError, "no backend connection"); + bo->vbc = VDI_GetFd(NULL, sp); + if (bo->vbc == NULL) { + VSLb(req->vsl, SLT_FetchError, "no backend connection"); return (-1); } - vc = wrk->busyobj->vbc; + vc = bo->vbc; if (vc->recycled) retry = 1; @@ -421,7 +425,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) * because the backend may be chosen by a director. */ if (need_host_hdr) - VDI_AddHostHeader(wrk->busyobj->bereq, vc); + VDI_AddHostHeader(bo->bereq, vc); (void)VTCP_blocking(vc->fd); /* XXX: we should timeout instead */ WRW_Reserve(wrk, &vc->fd, sp->t_req); /* XXX t_resp ? */ @@ -430,10 +434,10 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) /* Deal with any message-body the request might have */ i = FetchReqBody(sp, sendbody); if (WRW_FlushRelease(wrk) || i > 0) { - VSLb(sp->req->vsl, SLT_FetchError, + VSLb(req->vsl, SLT_FetchError, "backend write error: %d (%s)", errno, strerror(errno)); - VDI_CloseFd(wrk, &wrk->busyobj->vbc); + VDI_CloseFd(wrk, &bo->vbc); /* XXX: other cleanup ? */ return (retry); } @@ -446,7 +450,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) /* Receive response */ - HTC_Init(htc, wrk->busyobj->ws, vc->fd, vc->vsl, + HTC_Init(htc, bo->ws, vc->fd, vc->vsl, cache_param->http_resp_size, cache_param->http_resp_hdr_len); @@ -455,10 +459,10 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) i = HTC_Rx(htc); if (i < 0) { - VSLb(sp->req->vsl, SLT_FetchError, + VSLb(req->vsl, SLT_FetchError, "http first read error: %d %d (%s)", i, errno, strerror(errno)); - VDI_CloseFd(wrk, &wrk->busyobj->vbc); + VDI_CloseFd(wrk, &bo->vbc); /* XXX: other cleanup ? */ /* Retryable if we never received anything */ return (i == -1 ? retry : -1); @@ -469,20 +473,20 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) while (i == 0) { i = HTC_Rx(htc); if (i < 0) { - VSLb(sp->req->vsl, SLT_FetchError, + VSLb(req->vsl, SLT_FetchError, "http first read error: %d %d (%s)", i, errno, strerror(errno)); - VDI_CloseFd(wrk, &wrk->busyobj->vbc); + VDI_CloseFd(wrk, &bo->vbc); /* XXX: other cleanup ? */ return (-1); } } - hp = wrk->busyobj->beresp; + hp = bo->beresp; if (http_DissectResponse(hp, htc)) { - VSLb(sp->req->vsl, SLT_FetchError, "http format error"); - VDI_CloseFd(wrk, &wrk->busyobj->vbc); + VSLb(req->vsl, SLT_FetchError, "http format error"); + VDI_CloseFd(wrk, &bo->vbc); /* XXX: other cleanup ? */ return (-1); } @@ -492,17 +496,15 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) /*--------------------------------------------------------------------*/ int -FetchBody(struct worker *wrk, struct object *obj) +FetchBody(struct worker *wrk, struct busyobj *bo, struct object *obj) { int cls; struct storage *st; int mklen; ssize_t cl; struct http_conn *htc; - struct busyobj *bo; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - bo = wrk->busyobj; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AZ(bo->fetch_obj); CHECK_OBJ_NOTNULL(bo->vbc, VBC_MAGIC); diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 15b0a2a..1e21214 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -443,16 +443,16 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) AN(oc->flags & OC_F_BUSY); oc->refcnt = 1; - AZ(wrk->busyobj); - wrk->busyobj = VBO_GetBusyObj(wrk); - wrk->busyobj->vsl->wid = sp->vsl_id; + AZ(sp->req->busyobj); + sp->req->busyobj = VBO_GetBusyObj(wrk); + sp->req->busyobj->vsl->wid = sp->vsl_id; VRY_Validate(sp->req->vary_b); if (sp->req->vary_l != NULL) - wrk->busyobj->vary = sp->req->vary_b; + sp->req->busyobj->vary = sp->req->vary_b; else - wrk->busyobj->vary = NULL; - oc->busyobj = wrk->busyobj; + sp->req->busyobj->vary = NULL; + oc->busyobj = sp->req->busyobj; /* * Busy objects go on the tail, so they will not trip up searches. diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index f721ea2..02ab94f 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -854,7 +854,7 @@ http_FilterReq(const struct sess *sp, unsigned how) { struct http *hp; - hp = sp->wrk->busyobj->bereq; + hp = sp->req->busyobj->bereq; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); hp->logtag = HTTP_Tx; diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 5a2a805..2689cc8 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -197,10 +197,6 @@ pan_wrk(const struct worker *wrk) VSB_printf(pan_vsp, " worker = %p {\n", wrk); pan_ws(wrk->aws, 4); - if (wrk->busyobj != NULL && wrk->busyobj->bereq->ws != NULL) - pan_http("bereq", wrk->busyobj->bereq, 4); - if (wrk->busyobj != NULL && wrk->busyobj->beresp->ws != NULL) - pan_http("beresp", wrk->busyobj->beresp, 4); VSB_printf(pan_vsp, " },\n"); } @@ -221,6 +217,10 @@ pan_busyobj(const struct busyobj *bo) VSB_printf(pan_vsp, " },\n"); if (VALID_OBJ(bo->vbc, BACKEND_MAGIC)) pan_vbc(bo->vbc); + if (bo->bereq->ws != NULL) + pan_http("bereq", bo->bereq, 4); + if (bo->beresp->ws != NULL) + pan_http("beresp", bo->beresp, 4); } @@ -261,8 +261,8 @@ pan_sess(const struct sess *sp) VSB_printf(pan_vsp, " restarts = %d, esi_level = %d\n", sp->req->restarts, sp->req->esi_level); - if (sp->wrk->busyobj != NULL) - pan_busyobj(sp->wrk->busyobj); + if (sp->req->busyobj != NULL) + pan_busyobj(sp->req->busyobj); pan_ws(sp->req->ws, 2); pan_http("req", sp->req->http, 2); diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c index 688d87d..3f7cadf 100644 --- a/bin/varnishd/cache/cache_pipe.c +++ b/bin/varnishd/cache/cache_pipe.c @@ -63,31 +63,33 @@ void PipeSession(struct sess *sp) { struct vbc *vc; - struct worker *w; + struct worker *wrk; struct pollfd fds[2]; + struct busyobj *bo; int i; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC); - w = sp->wrk; + bo = sp->req->busyobj; + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + wrk = sp->wrk; vc = VDI_GetFd(NULL, sp); if (vc == NULL) return; - sp->wrk->busyobj->vbc = vc; /* For panic dumping */ + bo->vbc = vc; /* For panic dumping */ (void)VTCP_blocking(vc->fd); - WRW_Reserve(w, &vc->fd, sp->t_req); + WRW_Reserve(wrk, &vc->fd, sp->t_req); sp->wrk->acct_tmp.hdrbytes += - http_Write(w, sp->wrk->busyobj->bereq, 0); + http_Write(wrk, bo->bereq, 0); if (sp->req->htc->pipeline.b != NULL) sp->wrk->acct_tmp.bodybytes += - WRW_Write(w, sp->req->htc->pipeline.b, + WRW_Write(wrk, sp->req->htc->pipeline.b, Tlen(sp->req->htc->pipeline)); - i = WRW_FlushRelease(w); + i = WRW_FlushRelease(wrk); if (i) { SES_Close(sp, "pipe"); @@ -132,5 +134,5 @@ PipeSession(struct sess *sp) } SES_Close(sp, "pipe"); VDI_CloseFd(sp->wrk, &vc); - sp->wrk->busyobj->vbc = NULL; + bo->vbc = NULL; } diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 10a8182..f41b7ec 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -112,22 +112,6 @@ ses_new(struct sesspool *pp) } /*-------------------------------------------------------------------- - * Allocate a session for use by background threads. - */ - -struct sess * -SES_Alloc(void) -{ - struct sess *sp; - - ALLOC_OBJ(sp, SESS_MAGIC); - AN(sp); - ses_setup(sp); - /* XXX: sp->req ? */ - return (sp); -} - -/*-------------------------------------------------------------------- * The pool-task function for sessions */ @@ -148,7 +132,6 @@ ses_pool_task(struct worker *wrk, void *arg) sp = NULL; /* Cannot access sp any longer */ THR_SetSession(NULL); WS_Assert(wrk->aws); - AZ(wrk->busyobj); AZ(wrk->wrw); assert(wrk->vsl->wlp == wrk->vsl->wlb); if (cache_param->diag_bitmap & 0x00040000) { diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index c9135dc..cbe5637 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -101,10 +101,10 @@ vrt_selecthttp(const struct sess *sp, enum gethdr_e where) hp = sp->req->http; break; case HDR_BEREQ: - hp = sp->wrk->busyobj->bereq; + hp = sp->req->busyobj->bereq; break; case HDR_BERESP: - hp = sp->wrk->busyobj->beresp; + hp = sp->req->busyobj->beresp; break; case HDR_RESP: hp = sp->req->resp; diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 82aeb9c..72fa4dc 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -84,15 +84,15 @@ VRT_r_##obj##_##hdr(const struct sess *sp) \ VRT_DO_HDR(req, request, sp->req->http, HTTP_HDR_REQ) VRT_DO_HDR(req, url, sp->req->http, HTTP_HDR_URL) VRT_DO_HDR(req, proto, sp->req->http, HTTP_HDR_PROTO) -VRT_DO_HDR(bereq, request, sp->wrk->busyobj->bereq, HTTP_HDR_REQ) -VRT_DO_HDR(bereq, url, sp->wrk->busyobj->bereq, HTTP_HDR_URL) -VRT_DO_HDR(bereq, proto, sp->wrk->busyobj->bereq, HTTP_HDR_PROTO) +VRT_DO_HDR(bereq, request, sp->req->busyobj->bereq, HTTP_HDR_REQ) +VRT_DO_HDR(bereq, url, sp->req->busyobj->bereq, HTTP_HDR_URL) +VRT_DO_HDR(bereq, proto, sp->req->busyobj->bereq, HTTP_HDR_PROTO) VRT_DO_HDR(obj, proto, sp->req->obj->http, HTTP_HDR_PROTO) VRT_DO_HDR(obj, response, sp->req->obj->http, HTTP_HDR_RESPONSE) VRT_DO_HDR(resp, proto, sp->req->resp, HTTP_HDR_PROTO) VRT_DO_HDR(resp, response, sp->req->resp, HTTP_HDR_RESPONSE) -VRT_DO_HDR(beresp, proto, sp->wrk->busyobj->beresp, HTTP_HDR_PROTO) -VRT_DO_HDR(beresp, response, sp->wrk->busyobj->beresp, HTTP_HDR_RESPONSE) +VRT_DO_HDR(beresp, proto, sp->req->busyobj->beresp, HTTP_HDR_PROTO) +VRT_DO_HDR(beresp, response, sp->req->busyobj->beresp, HTTP_HDR_RESPONSE) /*--------------------------------------------------------------------*/ @@ -114,7 +114,7 @@ VRT_r_##obj##_status(const struct sess *sp) \ } VRT_DO_STATUS(obj, sp->req->obj->http) -VRT_DO_STATUS(beresp, sp->wrk->busyobj->beresp) +VRT_DO_STATUS(beresp, sp->req->busyobj->beresp) VRT_DO_STATUS(resp, sp->req->resp) /*--------------------------------------------------------------------*/ @@ -130,14 +130,11 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a) struct trouble *new; struct trouble *tr; struct trouble *tr2; - struct worker *wrk; struct vbc *vbc; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); - wrk = sp->wrk; - CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC); - vbc = wrk->busyobj->vbc; + CHECK_OBJ_NOTNULL(sp->req->busyobj, BUSYOBJ_MAGIC); + vbc = sp->req->busyobj->vbc; if (!vbc) return; CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC); @@ -187,14 +184,14 @@ void \ VRT_l_##dir##_##onm(const struct sess *sp, type a) \ { \ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - sp->wrk->field = a; \ + sp->req->field = a; \ } \ \ type \ VRT_r_##dir##_##onm(const struct sess *sp) \ { \ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - return (sp->wrk->field); \ + return (sp->req->field); \ } VBERESP(beresp, unsigned, do_esi, busyobj->do_esi) @@ -235,8 +232,8 @@ VRT_l_bereq_##which(struct sess *sp, double num) \ { \ \ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC); \ - sp->wrk->busyobj->which = (num > 0.0 ? num : 0.0); \ + CHECK_OBJ_NOTNULL(sp->req->busyobj, BUSYOBJ_MAGIC); \ + sp->req->busyobj->which = (num > 0.0 ? num : 0.0); \ } \ \ double __match_proto__() \ @@ -244,8 +241,8 @@ VRT_r_bereq_##which(struct sess *sp) \ { \ \ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC); \ - return(sp->wrk->busyobj->which); \ + CHECK_OBJ_NOTNULL(sp->req->busyobj, BUSYOBJ_MAGIC); \ + return(sp->req->busyobj->which); \ } BEREQ_TIMEOUT(connect_timeout) @@ -259,8 +256,8 @@ VRT_r_beresp_backend_name(const struct sess *sp) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->wrk->busyobj->vbc, VBC_MAGIC); - return(sp->wrk->busyobj->vbc->backend->vcl_name); + CHECK_OBJ_NOTNULL(sp->req->busyobj->vbc, VBC_MAGIC); + return(sp->req->busyobj->vbc->backend->vcl_name); } struct sockaddr_storage * @@ -268,8 +265,8 @@ VRT_r_beresp_backend_ip(const struct sess *sp) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->wrk->busyobj->vbc, VBC_MAGIC); - return(sp->wrk->busyobj->vbc->addr); + CHECK_OBJ_NOTNULL(sp->req->busyobj->vbc, VBC_MAGIC); + return(sp->req->busyobj->vbc->addr); } int @@ -277,8 +274,8 @@ VRT_r_beresp_backend_port(const struct sess *sp) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->wrk->busyobj->vbc, VBC_MAGIC); - return (VTCP_port(sp->wrk->busyobj->vbc->addr)); + CHECK_OBJ_NOTNULL(sp->req->busyobj->vbc, VBC_MAGIC); + return (VTCP_port(sp->req->busyobj->vbc->addr)); } const char * __match_proto__() @@ -298,7 +295,7 @@ VRT_l_beresp_storage(struct sess *sp, const char *str, ...) char *b; va_start(ap, str); - b = VRT_String(sp->wrk->busyobj->ws, NULL, str, ap); + b = VRT_String(sp->req->busyobj->ws, NULL, str, ap); va_end(ap); sp->req->storage_hint = b; } @@ -417,12 +414,12 @@ VRT_DO_EXP(obj, sp->req->obj->exp, keep, 0, EXP_Rearm(sp->req->obj); vrt_wsp_exp(sp, sp->req->obj->xid, &sp->req->obj->exp);) -VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, grace, 0, - vrt_wsp_exp(sp, sp->req->xid, &sp->wrk->busyobj->exp);) -VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, ttl, 0, - vrt_wsp_exp(sp, sp->req->xid, &sp->wrk->busyobj->exp);) -VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, keep, 0, - vrt_wsp_exp(sp, sp->req->xid, &sp->wrk->busyobj->exp);) +VRT_DO_EXP(beresp, sp->req->busyobj->exp, grace, 0, + vrt_wsp_exp(sp, sp->req->xid, &sp->req->busyobj->exp);) +VRT_DO_EXP(beresp, sp->req->busyobj->exp, ttl, 0, + vrt_wsp_exp(sp, sp->req->xid, &sp->req->busyobj->exp);) +VRT_DO_EXP(beresp, sp->req->busyobj->exp, keep, 0, + vrt_wsp_exp(sp, sp->req->xid, &sp->req->busyobj->exp);) /*-------------------------------------------------------------------- * req.xid From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] e65be4b Belive it or not, I'm still not used to the guts of varnishd being exposed. Message-ID: commit e65be4b12a6bbc2bd3a15b240437bf3b1dd7d96c Author: Poul-Henning Kamp Date: Sun Feb 19 12:52:59 2012 +0000 Belive it or not, I'm still not used to the guts of varnishd being exposed. diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 5774cfb..4b9fd5f 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -192,7 +192,7 @@ vmod_collect(struct sess *sp, enum gethdr_e e, const char *h) (void)h; if (e == HDR_REQ) http_CollectHdr(sp->req->http, h); - else if (e == HDR_BERESP && sp->wrk->busyobj != NULL) - http_CollectHdr(sp->wrk->busyobj->beresp, h); + else if (e == HDR_BERESP && sp->req->busyobj != NULL) + http_CollectHdr(sp->req->busyobj->beresp, h); } From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 2608703 Give WRW a hint about which vsl to log to. Message-ID: commit 2608703ba1891b61de94074489b03bfa4a91165a Author: Poul-Henning Kamp Date: Mon Feb 20 08:11:52 2012 +0000 Give WRW a hint about which vsl to log to. Consequent Constification diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 4b5fcd8..bba390e 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -773,7 +773,7 @@ void VGZ_UpdateObj(const struct vgz*, struct object *); int VGZ_WrwInit(struct vgz *vg); int VGZ_WrwGunzip(struct worker *w, struct vgz *, const void *ibuf, ssize_t ibufl); -void VGZ_WrwFlush(struct worker *wrk, struct vgz *vg); +void VGZ_WrwFlush(const struct worker *wrk, struct vgz *vg); /* Return values */ #define VGZ_ERROR -1 @@ -789,7 +789,7 @@ const char *http_StatusMessage(unsigned); unsigned http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd); void HTTP_Init(void); void http_ClrHeader(struct http *to); -unsigned http_Write(struct worker *w, const struct http *hp, int resp); +unsigned http_Write(const struct worker *w, const struct http *hp, int resp); void http_SetResp(struct http *to, const char *proto, uint16_t status, const char *response); void http_FilterReq(const struct sess *sp, unsigned how); @@ -885,13 +885,13 @@ int Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how); #define WRW_IsReleased(w) ((w)->wrw == NULL) int WRW_Error(const struct worker *w); -void WRW_Chunked(struct worker *w); -void WRW_EndChunk(struct worker *w); -void WRW_Reserve(struct worker *w, int *fd, double t0); -unsigned WRW_Flush(struct worker *w); +void WRW_Chunked(const struct worker *w); +void WRW_EndChunk(const struct worker *w); +void WRW_Reserve(struct worker *w, int *fd, struct vsl_log *, double t0); +unsigned WRW_Flush(const struct worker *w); unsigned WRW_FlushRelease(struct worker *w); -unsigned WRW_Write(struct worker *w, const void *ptr, int len); -unsigned WRW_WriteH(struct worker *w, const txt *hh, const char *suf); +unsigned WRW_Write(const struct worker *w, const void *ptr, int len); +unsigned WRW_WriteH(const struct worker *w, const txt *hh, const char *suf); /* cache_session.c [SES] */ void SES_Close(struct sess *sp, const char *reason); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 00279c2..2d697cb 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -113,7 +113,7 @@ ved_include(struct sess *sp, const char *src, const char *host) WS_Reset(sp->req->ws, sp_ws_wm); WS_Reset(w->aws, wrk_ws_wm); /* XXX ? */ - WRW_Reserve(sp->wrk, &sp->fd, sp->req->t_resp); + WRW_Reserve(sp->wrk, &sp->fd, sp->req->vsl, sp->req->t_resp); if (sp->req->res_mode & RES_CHUNKED) WRW_Chunked(sp->wrk); } diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 24abc7a..05ac8a3 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -428,7 +428,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) VDI_AddHostHeader(bo->bereq, vc); (void)VTCP_blocking(vc->fd); /* XXX: we should timeout instead */ - WRW_Reserve(wrk, &vc->fd, sp->t_req); /* XXX t_resp ? */ + WRW_Reserve(wrk, &vc->fd, bo->vsl, sp->t_req); /* XXX t_resp ? */ (void)http_Write(wrk, hp, 0); /* XXX: stats ? */ /* Deal with any message-body the request might have */ diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index d4511bc..ec1b7cd 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -356,7 +356,7 @@ VGZ_WrwGunzip(struct worker *wrk, struct vgz *vg, const void *ibuf, /*--------------------------------------------------------------------*/ void -VGZ_WrwFlush(struct worker *wrk, struct vgz *vg) +VGZ_WrwFlush(const struct worker *wrk, struct vgz *vg) { CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC); diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 02ab94f..4d2a9fa 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -1061,7 +1061,7 @@ HTTP_Copy(struct http *to, const struct http * const fm) /*--------------------------------------------------------------------*/ unsigned -http_Write(struct worker *w, const struct http *hp, int resp) +http_Write(const struct worker *w, const struct http *hp, int resp) { unsigned u, l; diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c index 3f7cadf..8815c68 100644 --- a/bin/varnishd/cache/cache_pipe.c +++ b/bin/varnishd/cache/cache_pipe.c @@ -80,7 +80,7 @@ PipeSession(struct sess *sp) bo->vbc = vc; /* For panic dumping */ (void)VTCP_blocking(vc->fd); - WRW_Reserve(wrk, &vc->fd, sp->t_req); + WRW_Reserve(wrk, &vc->fd, bo->vsl, sp->t_req); sp->wrk->acct_tmp.hdrbytes += http_Write(wrk, bo->bereq, 0); diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index f1e0553..5a33d08 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -237,7 +237,7 @@ RES_WriteObj(struct sess *sp) req = sp->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - WRW_Reserve(sp->wrk, &sp->fd, sp->req->t_resp); + WRW_Reserve(sp->wrk, &sp->fd, sp->req->vsl, sp->req->t_resp); if (req->obj->response == 200 && req->http->conds && diff --git a/bin/varnishd/cache/cache_wrw.c b/bin/varnishd/cache/cache_wrw.c index a8140f5..3a38305 100644 --- a/bin/varnishd/cache/cache_wrw.c +++ b/bin/varnishd/cache/cache_wrw.c @@ -57,6 +57,7 @@ struct wrw { ssize_t cliov; unsigned ciov; /* Chunked header marker */ double t0; + struct vsl_log *vsl; }; /*-------------------------------------------------------------------- @@ -70,7 +71,7 @@ WRW_Error(const struct worker *wrk) } void -WRW_Reserve(struct worker *wrk, int *fd, double t0) +WRW_Reserve(struct worker *wrk, int *fd, struct vsl_log *vsl, double t0) { struct wrw *wrw; unsigned u; @@ -95,6 +96,7 @@ WRW_Reserve(struct worker *wrk, int *fd, double t0) wrw->niov = 0; wrw->wfd = fd; wrw->t0 = t0; + wrw->vsl = vsl; wrk->wrw = wrw; } @@ -136,7 +138,7 @@ wrw_prune(struct wrw *wrw, ssize_t bytes) } unsigned -WRW_Flush(struct worker *wrk) +WRW_Flush(const struct worker *wrk) { ssize_t i; struct wrw *wrw; @@ -179,7 +181,7 @@ WRW_Flush(struct worker *wrk) */ if (VTIM_real() - wrw->t0 > cache_param->send_timeout) { - WSL(wrk->vsl, SLT_Debug, *wrw->wfd, + VSLb(wrw->vsl, SLT_Debug, "Hit total send timeout, " "wrote = %zd/%zd; not retrying", i, wrw->liov); @@ -187,7 +189,7 @@ WRW_Flush(struct worker *wrk) break; } - WSL(wrk->vsl, SLT_Debug, *wrw->wfd, + VSLb(wrw->vsl, SLT_Debug, "Hit send timeout, wrote = %zd/%zd; retrying", i, wrw->liov); @@ -196,7 +198,7 @@ WRW_Flush(struct worker *wrk) } if (i <= 0) { wrw->werr++; - WSL(wrk->vsl, SLT_Debug, *wrw->wfd, + VSLb(wrw->vsl, SLT_Debug, "Write error, retval = %zd, len = %zd, errno = %s", i, wrw->liov, strerror(errno)); } @@ -222,7 +224,7 @@ WRW_FlushRelease(struct worker *wrk) } unsigned -WRW_WriteH(struct worker *wrk, const txt *hh, const char *suf) +WRW_WriteH(const struct worker *wrk, const txt *hh, const char *suf) { unsigned u; @@ -239,7 +241,7 @@ WRW_WriteH(struct worker *wrk, const txt *hh, const char *suf) } unsigned -WRW_Write(struct worker *wrk, const void *ptr, int len) +WRW_Write(const struct worker *wrk, const void *ptr, int len) { struct wrw *wrw; @@ -265,7 +267,7 @@ WRW_Write(struct worker *wrk, const void *ptr, int len) } void -WRW_Chunked(struct worker *wrk) +WRW_Chunked(const struct worker *wrk) { struct wrw *wrw; @@ -294,7 +296,7 @@ WRW_Chunked(struct worker *wrk) */ void -WRW_EndChunk(struct worker *wrk) +WRW_EndChunk(const struct worker *wrk) { struct wrw *wrw; From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 4ba2faf Eliminate WSL() usage. Message-ID: commit 4ba2fafe6fcca34b05732e464b776b1a5ccb6ea6 Author: Poul-Henning Kamp Date: Mon Feb 20 08:25:27 2012 +0000 Eliminate WSL() usage. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index cfe86ec..961cb8d 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -183,8 +183,8 @@ bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs) } else { vc->vsl_id = s | VSL_BACKENDMARKER; VTCP_myname(s, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); - WSL(sp->req->vsl, SLT_BackendOpen, vc->vsl_id, "%s %s %s ", - vs->backend->display_name, abuf1, pbuf1); + VSLb(sp->req->vsl, SLT_BackendOpen, "%d %s %s %s ", + vc->fd, vs->backend->display_name, abuf1, pbuf1); } } @@ -354,12 +354,12 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs) return (vc); } VSC_C_main->backend_toolate++; - WSL(sp->wrk->vsl, SLT_BackendClose, vc->vsl_id, "%s", - bp->display_name); + VSLb(sp->req->vsl, SLT_BackendClose, "%d %s toolate", + vc->fd, bp->display_name); /* Checkpoint log to flush all info related to this connection before the OS reuses the FD */ - VSL_Flush(sp->wrk->vsl, 0); + VSL_Flush(sp->req->vsl, 0); VTCP_close(&vc->fd); VBE_DropRefConn(bp); diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c index 3f70cb7..5275680 100644 --- a/bin/varnishd/cache/cache_dir.c +++ b/bin/varnishd/cache/cache_dir.c @@ -55,7 +55,7 @@ VDI_CloseFd(struct worker *wrk, struct vbc **vbp) bp = vc->backend; - WSL(vc->vsl, SLT_BackendClose, vc->vsl_id, "%s", bp->display_name); + VSLb(vc->vsl, SLT_BackendClose, "%s", bp->display_name); /* Checkpoint log to flush all info related to this connection before the OS reuses the FD */ @@ -89,7 +89,7 @@ VDI_RecycleFd(struct worker *wrk, struct vbc **vbp) bp = vc->backend; - WSL(vc->vsl, SLT_BackendReuse, vc->vsl_id, "%s", bp->display_name); + VSLb(vc->vsl, SLT_BackendReuse, "%s", bp->display_name); /* * Flush the shmlog, so that another session reusing this backend From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 42f4e95 Give the expiry thread its own VSL, and get rid of WSL() entirely. Message-ID: commit 42f4e9572cf14ca2b1e385e4b4e487475fb1e15d Author: Poul-Henning Kamp Date: Mon Feb 20 08:30:33 2012 +0000 Give the expiry thread its own VSL, and get rid of WSL() entirely. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index bba390e..028abe9 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -915,8 +915,6 @@ void VSM_Free(void *ptr); #ifdef VSL_ENDMARKER void VSL(enum VSL_tag_e tag, int id, const char *fmt, ...) __printflike(3, 4); -void WSL(struct vsl_log *, enum VSL_tag_e tag, int id, const char *fmt, ...) - __printflike(4, 5); void VSLb(struct vsl_log *, enum VSL_tag_e tag, const char *fmt, ...) __printflike(3, 4); void VSLbt(struct vsl_log *, enum VSL_tag_e tag, txt t); diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index 8eedb71..98b664f 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -336,13 +336,15 @@ exp_timer(struct worker *wrk, void *priv) struct lru *lru; double t; struct object *o; + struct vsl_log vsl; (void)priv; + VSL_Setup(&vsl, NULL, 0); t = VTIM_real(); oc = NULL; while (1) { if (oc == NULL) { - VSL_Flush(wrk->vsl, 0); + VSL_Flush(&vsl, 0); WRK_SumStat(wrk); VTIM_sleep(cache_param->expiry_sleep); t = VTIM_real(); @@ -400,7 +402,7 @@ exp_timer(struct worker *wrk, void *priv) CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); o = oc_getobj(&wrk->stats, oc); - WSL(wrk->vsl, SLT_ExpKill, 0, "%u %.0f", + VSLb(&vsl, SLT_ExpKill, "%u %.0f", oc_getxid(&wrk->stats, oc), EXP_Ttl(NULL, o) - t); (void)HSH_Deref(&wrk->stats, oc, NULL); } diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 6a54101..bcdfa6d 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -274,22 +274,6 @@ wsl(struct vsl_log *vsl, enum VSL_tag_e tag, int id, const char *fmt, va_list ap VSL_Flush(vsl, 0); } -/*--------------------------------------------------------------------*/ - -void -WSL(struct vsl_log *vsl, enum VSL_tag_e tag, int id, const char *fmt, ...) -{ - va_list ap; - - if (id == -1) - id = vsl->wid; - AN(fmt); - va_start(ap, fmt); - wsl(vsl, tag, id, fmt, ap); - va_end(ap); -} - - /*-------------------------------------------------------------------- * VSL-buffered */ @@ -320,6 +304,12 @@ VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t) void VSL_Setup(struct vsl_log *vsl, void *ptr, size_t len) { + + if (ptr == NULL) { + len = cache_param->vsl_buffer; + ptr = malloc(len); + AN(ptr); + } vsl->wlp = ptr; vsl->wlb = ptr; vsl->wle = ptr; From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 8140011 Renovate the ban lurkers usage of VSL to cause less synchronous VSL activity. Message-ID: commit 8140011132afcd0dfbe47ec7cd1e8ba07785c3b9 Author: Poul-Henning Kamp Date: Mon Feb 20 09:13:40 2012 +0000 Renovate the ban lurkers usage of VSL to cause less synchronous VSL activity. diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index 0058c3c..cba6fd1 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -677,7 +677,7 @@ ban_evaluate(const uint8_t *bs, const struct http *objhttp, */ static int -ban_check_object(struct object *o, struct worker *wrk, +ban_check_object(struct object *o, struct vsl_log *vsl, const struct http *req_http) { struct ban *b; @@ -686,7 +686,6 @@ ban_check_object(struct object *o, struct worker *wrk, unsigned tests, skipped; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_ORNULL(req_http, HTTP_MAGIC); oc = o->objcore; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); @@ -758,7 +757,7 @@ ban_check_object(struct object *o, struct worker *wrk, oc_updatemeta(oc); /* BAN also changed, but that is not important any more */ /* XXX: no req in lurker */ - VSLb(wrk->vsl, SLT_ExpBan, "%u was banned", o->xid); + VSLb(vsl, SLT_ExpBan, "%u was banned", o->xid); EXP_Rearm(o); return (1); } @@ -768,7 +767,7 @@ int BAN_CheckObject(struct object *o, const struct sess *sp) { - return (ban_check_object(o, sp->wrk, sp->req->http) > 0); + return (ban_check_object(o, sp->req->vsl, sp->req->http) > 0); } static struct ban * @@ -803,11 +802,14 @@ ban_lurker_work(struct worker *wrk, unsigned pass) struct objhead *oh; struct objcore *oc, *oc2; struct object *o; + struct vsl_log vsl; int i; AN(pass & BAN_F_LURK); AZ(pass & ~BAN_F_LURK); + VSL_Setup(&vsl, NULL, 0); + /* First route the last ban(s) */ do { Lck_Lock(&ban_mtx); @@ -837,13 +839,13 @@ ban_lurker_work(struct worker *wrk, unsigned pass) b->flags |= pass; } if (cache_param->diag_bitmap & 0x80000) - VSL(SLT_Debug, 0, "lurker: %d actionable bans", i); + VSLb(&vsl, SLT_Debug, "lurker: %d actionable bans", i); if (i == 0) return (0); VTAILQ_FOREACH_REVERSE(b, &ban_head, banhead_s, list) { if (cache_param->diag_bitmap & 0x80000) - VSL(SLT_Debug, 0, "lurker doing %f %d", + VSLb(&vsl, SLT_Debug, "lurker doing %f %d", ban_time(b->spec), b->refcount); while (1) { Lck_Lock(&ban_mtx); @@ -852,7 +854,7 @@ ban_lurker_work(struct worker *wrk, unsigned pass) break; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); if (cache_param->diag_bitmap & 0x80000) - VSL(SLT_Debug, 0, "test: %p %u %u", + VSLb(&vsl, SLT_Debug, "test: %p %u %u", oc, oc->flags & OC_F_LURK, pass); if ((oc->flags & OC_F_LURK) == pass) break; @@ -860,6 +862,7 @@ ban_lurker_work(struct worker *wrk, unsigned pass) CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); if (Lck_Trylock(&oh->mtx)) { Lck_Unlock(&ban_mtx); + VSL_Flush(&vsl, 0); VTIM_sleep(cache_param->ban_lurker_sleep); continue; } @@ -889,9 +892,9 @@ ban_lurker_work(struct worker *wrk, unsigned pass) * Get the object and check it against all relevant bans */ o = oc_getobj(&wrk->stats, oc); - i = ban_check_object(o, wrk, NULL); + i = ban_check_object(o, &vsl, NULL); if (cache_param->diag_bitmap & 0x80000) - VSL(SLT_Debug, 0, "lurker got: %p %d", + VSLb(&vsl, SLT_Debug, "lurker got: %p %d", oc, i); if (i == -1) { /* Not banned, not moved */ @@ -903,7 +906,7 @@ ban_lurker_work(struct worker *wrk, unsigned pass) } Lck_Unlock(&oh->mtx); if (cache_param->diag_bitmap & 0x80000) - VSL(SLT_Debug, 0, "lurker done: %p %u %u", + VSLb(&vsl, SLT_Debug, "lurker done: %p %u %u", oc, oc->flags & OC_F_LURK, pass); (void)HSH_Deref(&wrk->stats, NULL, &o); VTIM_sleep(cache_param->ban_lurker_sleep); @@ -915,7 +918,7 @@ ban_lurker_work(struct worker *wrk, unsigned pass) VSC_C_main->bans_gone++; } if (cache_param->diag_bitmap & 0x80000) - VSL(SLT_Debug, 0, "lurker BAN %f now gone", + VSLb(&vsl, SLT_Debug, "lurker BAN %f now gone", ban_time(b->spec)); } Lck_Unlock(&ban_mtx); From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 41a719c Urg, forgot the outher loop of the ban-lurker: Message-ID: commit 41a719c98079f44afbf1229be7fb02062dcef5f9 Author: Poul-Henning Kamp Date: Mon Feb 20 09:19:03 2012 +0000 Urg, forgot the outher loop of the ban-lurker: Fix previous change to not madly leak memory. diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index cba6fd1..6e6eafb 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -796,20 +796,17 @@ ban_CheckLast(void) */ static int -ban_lurker_work(struct worker *wrk, unsigned pass) +ban_lurker_work(struct worker *wrk, struct vsl_log *vsl, unsigned pass) { struct ban *b, *b0, *b2; struct objhead *oh; struct objcore *oc, *oc2; struct object *o; - struct vsl_log vsl; int i; AN(pass & BAN_F_LURK); AZ(pass & ~BAN_F_LURK); - VSL_Setup(&vsl, NULL, 0); - /* First route the last ban(s) */ do { Lck_Lock(&ban_mtx); @@ -839,13 +836,13 @@ ban_lurker_work(struct worker *wrk, unsigned pass) b->flags |= pass; } if (cache_param->diag_bitmap & 0x80000) - VSLb(&vsl, SLT_Debug, "lurker: %d actionable bans", i); + VSLb(vsl, SLT_Debug, "lurker: %d actionable bans", i); if (i == 0) return (0); VTAILQ_FOREACH_REVERSE(b, &ban_head, banhead_s, list) { if (cache_param->diag_bitmap & 0x80000) - VSLb(&vsl, SLT_Debug, "lurker doing %f %d", + VSLb(vsl, SLT_Debug, "lurker doing %f %d", ban_time(b->spec), b->refcount); while (1) { Lck_Lock(&ban_mtx); @@ -854,7 +851,7 @@ ban_lurker_work(struct worker *wrk, unsigned pass) break; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); if (cache_param->diag_bitmap & 0x80000) - VSLb(&vsl, SLT_Debug, "test: %p %u %u", + VSLb(vsl, SLT_Debug, "test: %p %u %u", oc, oc->flags & OC_F_LURK, pass); if ((oc->flags & OC_F_LURK) == pass) break; @@ -862,7 +859,7 @@ ban_lurker_work(struct worker *wrk, unsigned pass) CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); if (Lck_Trylock(&oh->mtx)) { Lck_Unlock(&ban_mtx); - VSL_Flush(&vsl, 0); + VSL_Flush(vsl, 0); VTIM_sleep(cache_param->ban_lurker_sleep); continue; } @@ -892,9 +889,9 @@ ban_lurker_work(struct worker *wrk, unsigned pass) * Get the object and check it against all relevant bans */ o = oc_getobj(&wrk->stats, oc); - i = ban_check_object(o, &vsl, NULL); + i = ban_check_object(o, vsl, NULL); if (cache_param->diag_bitmap & 0x80000) - VSLb(&vsl, SLT_Debug, "lurker got: %p %d", + VSLb(vsl, SLT_Debug, "lurker got: %p %d", oc, i); if (i == -1) { /* Not banned, not moved */ @@ -906,7 +903,7 @@ ban_lurker_work(struct worker *wrk, unsigned pass) } Lck_Unlock(&oh->mtx); if (cache_param->diag_bitmap & 0x80000) - VSLb(&vsl, SLT_Debug, "lurker done: %p %u %u", + VSLb(vsl, SLT_Debug, "lurker done: %p %u %u", oc, oc->flags & OC_F_LURK, pass); (void)HSH_Deref(&wrk->stats, NULL, &o); VTIM_sleep(cache_param->ban_lurker_sleep); @@ -918,7 +915,7 @@ ban_lurker_work(struct worker *wrk, unsigned pass) VSC_C_main->bans_gone++; } if (cache_param->diag_bitmap & 0x80000) - VSLb(&vsl, SLT_Debug, "lurker BAN %f now gone", + VSLb(vsl, SLT_Debug, "lurker BAN %f now gone", ban_time(b->spec)); } Lck_Unlock(&ban_mtx); @@ -934,8 +931,11 @@ ban_lurker(struct worker *wrk, void *priv) { struct ban *bf; unsigned pass = (1 << LURK_SHIFT); + struct vsl_log vsl; int i = 0; + VSL_Setup(&vsl, NULL, 0); + (void)priv; while (1) { @@ -953,8 +953,8 @@ ban_lurker(struct worker *wrk, void *priv) VTIM_sleep(1.0); } - i = ban_lurker_work(wrk, pass); - VSL_Flush(wrk->vsl, 0); + i = ban_lurker_work(wrk, &vsl, pass); + VSL_Flush(&vsl, 0); WRK_SumStat(wrk); if (i) { pass += (1 << LURK_SHIFT); From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] efb159e Put delivery G(un)zip VSL on req->vsl Message-ID: commit efb159e0829aab387ca2cfd2528a4ec30a08bf02 Author: Poul-Henning Kamp Date: Mon Feb 20 09:19:45 2012 +0000 Put delivery G(un)zip VSL on req->vsl diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 2d697cb..669a594 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -258,7 +258,7 @@ ESI_Deliver(struct sess *sp) } if (isgzip && !sp->req->gzip_resp) { - vgz = VGZ_NewUngzip(sp->wrk->vsl, "U D E"); + vgz = VGZ_NewUngzip(sp->req->vsl, "U D E"); AZ(VGZ_WrwInit(vgz)); /* Feed a gzip header to gunzip to make it happy */ diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index 5a33d08..4760c12 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -160,7 +160,7 @@ res_WriteGunzipObj(const struct sess *sp) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - vg = VGZ_NewUngzip(sp->wrk->vsl, "U D -"); + vg = VGZ_NewUngzip(sp->req->vsl, "U D -"); AZ(VGZ_WrwInit(vg)); VTAILQ_FOREACH(st, &sp->req->obj->store, list) { From phk at FreeBSD.org Thu Dec 18 09:27:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:38 +0100 Subject: [experimental-ims] 89558df Allocate the req early and VSL the SessionOpen there. Message-ID: commit 89558df47dd227bbf3ae145bb00274df5465da29 Author: Poul-Henning Kamp Date: Mon Feb 20 09:35:03 2012 +0000 Allocate the req early and VSL the SessionOpen there. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index a56e016..9aaa117 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -947,6 +947,13 @@ cnt_first(struct sess *sp, struct worker *wrk) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + /* Allocate a request already now, so we can VSL to it */ + SES_GetReq(sp); + CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC); + HTC_Init(sp->req->htc, sp->req->ws, sp->fd, sp->req->vsl, + cache_param->http_req_size, + cache_param->http_req_hdr_len); + VTCP_name(&sp->sockaddr, sp->sockaddrlen, sp->addr, sizeof sp->addr, sp->port, sizeof sp->port); if (cache_param->log_local_addr) { @@ -955,11 +962,11 @@ cnt_first(struct sess *sp, struct worker *wrk) VTCP_name(&sp->mysockaddr, sp->mysockaddrlen, laddr, sizeof laddr, lport, sizeof lport); /* XXX: have no req yet */ - VSLb(sp->wrk->vsl, SLT_SessionOpen, "%s %s %s %s", + VSLb(sp->req->vsl, SLT_SessionOpen, "%s %s %s %s", sp->addr, sp->port, laddr, lport); } else { /* XXX: have no req yet */ - VSLb(sp->wrk->vsl, SLT_SessionOpen, "%s %s %s", + VSLb(sp->req->vsl, SLT_SessionOpen, "%s %s %s", sp->addr, sp->port, sp->mylsock->name); } From phk at FreeBSD.org Thu Dec 18 09:27:39 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] c53ab05 Get rid of the workthread VSL buffer. Message-ID: commit c53ab05e9bc6f892d1823a3378bb47f054018906 Author: Poul-Henning Kamp Date: Mon Feb 20 09:43:41 2012 +0000 Get rid of the workthread VSL buffer. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 028abe9..45f36f2 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -312,8 +312,6 @@ struct worker { struct VCL_conf *vcl; - struct vsl_log vsl[1]; - struct ws aws[1]; diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 9aaa117..57a7e05 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -402,7 +402,7 @@ cnt_done(struct sess *sp, struct worker *wrk, struct req *req) req->xid, sp->t_req, sp->t_idle, dh, dp, da); } req->xid = 0; - VSL_Flush(wrk->vsl, 0); + VSL_Flush(sp->req->vsl, 0); sp->t_req = NAN; req->t_resp = NAN; @@ -1623,7 +1623,6 @@ CNT_Session(struct sess *sp) WS_Assert(wrk->aws); CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); } - VSL_Flush(wrk->vsl, 0); #define ACCT(foo) AZ(wrk->acct_tmp.foo); #include "tbl/acct_fields.h" #undef ACCT diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c index 5275680..6554b18 100644 --- a/bin/varnishd/cache/cache_dir.c +++ b/bin/varnishd/cache/cache_dir.c @@ -57,9 +57,10 @@ VDI_CloseFd(struct worker *wrk, struct vbc **vbp) VSLb(vc->vsl, SLT_BackendClose, "%s", bp->display_name); - /* Checkpoint log to flush all info related to this connection - before the OS reuses the FD */ - VSL_Flush(wrk->vsl, 0); + /* + * Checkpoint log to flush all info related to this connection + * before the OS reuses the FD + */ VSL_Flush(vc->vsl, 0); vc->vsl->wid = vc->orig_vsl_id; vc->vsl = NULL; @@ -91,11 +92,7 @@ VDI_RecycleFd(struct worker *wrk, struct vbc **vbp) VSLb(vc->vsl, SLT_BackendReuse, "%s", bp->display_name); - /* - * Flush the shmlog, so that another session reusing this backend - * will log chronologically later than our use of it. - */ - VSL_Flush(wrk->vsl, 0); + /* XXX: revisit this hack */ VSL_Flush(vc->vsl, 0); vc->vsl->wid = vc->orig_vsl_id; vc->vsl = NULL; diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 669a594..dd3d98a 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -45,18 +45,18 @@ static void ved_include(struct sess *sp, const char *src, const char *host) { struct object *obj; - struct worker *w; + struct worker *wrk; char *sp_ws_wm; char *wrk_ws_wm; unsigned sxid, res_mode; - w = sp->wrk; + wrk = sp->wrk; if (sp->req->esi_level >= cache_param->max_esi_depth) return; sp->req->esi_level++; - (void)WRW_FlushRelease(w); + (void)WRW_FlushRelease(wrk); obj = sp->req->obj; sp->req->obj = NULL; @@ -67,7 +67,7 @@ ved_include(struct sess *sp, const char *src, const char *host) /* Take a workspace snapshot */ sp_ws_wm = WS_Snapshot(sp->req->ws); - wrk_ws_wm = WS_Snapshot(w->aws); /* XXX ? */ + wrk_ws_wm = WS_Snapshot(wrk->aws); /* XXX ? */ http_SetH(sp->req->http, HTTP_HDR_URL, src); if (host != NULL && *host != '\0') { @@ -93,12 +93,11 @@ ved_include(struct sess *sp, const char *src, const char *host) sxid = sp->req->xid; while (1) { - sp->wrk = w; + sp->wrk = wrk; CNT_Session(sp); if (sp->step == STP_DONE) break; AZ(sp->wrk); - VSL_Flush(w->vsl, 0); DSL(0x20, SLT_Debug, sp->vsl_id, "loop waiting for ESI"); (void)usleep(10000); } @@ -111,7 +110,7 @@ ved_include(struct sess *sp, const char *src, const char *host) /* Reset the workspace */ WS_Reset(sp->req->ws, sp_ws_wm); - WS_Reset(w->aws, wrk_ws_wm); /* XXX ? */ + WS_Reset(wrk->aws, wrk_ws_wm); /* XXX ? */ WRW_Reserve(sp->wrk, &sp->fd, sp->req->vsl, sp->req->t_resp); if (sp->req->res_mode & RES_CHUNKED) diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 05ac8a3..8f76c24 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -442,9 +442,6 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) return (retry); } - /* Checkpoint the vsl.here */ - VSL_Flush(wrk->vsl, 0); - /* XXX is this the right place? */ VSC_C_main->backend_req++; diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index f41b7ec..ba52ad3 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -133,7 +133,6 @@ ses_pool_task(struct worker *wrk, void *arg) THR_SetSession(NULL); WS_Assert(wrk->aws); AZ(wrk->wrw); - assert(wrk->vsl->wlp == wrk->vsl->wlb); if (cache_param->diag_bitmap & 0x00040000) { if (wrk->vcl != NULL) VCL_Rel(&wrk->vcl); diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index d28a58c..cc3a061 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -94,13 +94,11 @@ wrk_bgthread(void *arg) { struct bgthread *bt; struct worker wrk; - uint32_t logbuf[1024]; /* XXX: size ? */ CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC); THR_SetName(bt->name); memset(&wrk, 0, sizeof wrk); wrk.magic = WORKER_MAGIC; - VSL_Setup(wrk.vsl, logbuf, sizeof logbuf); (void)bt->func(&wrk, bt->priv); @@ -126,10 +124,9 @@ WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv) /*--------------------------------------------------------------------*/ static void * -wrk_thread_real(void *priv, unsigned shm_workspace, unsigned thread_workspace) +wrk_thread_real(void *priv, unsigned thread_workspace) { struct worker *w, ww; - uint32_t wlog[shm_workspace / 4]; unsigned char ws[thread_workspace]; THR_SetName("cache-worker"); @@ -137,7 +134,6 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned thread_workspace) memset(w, 0, sizeof *w); w->magic = WORKER_MAGIC; w->lastused = NAN; - VSL_Setup(w->vsl, wlog, sizeof wlog); AZ(pthread_cond_init(&w->cond, NULL)); WS_Init(w->aws, "wrk", ws, thread_workspace); @@ -162,9 +158,7 @@ void * WRK_thread(void *priv) { - return (wrk_thread_real(priv, - cache_param->shm_workspace, - cache_param->workspace_thread)); + return (wrk_thread_real(priv, cache_param->workspace_thread)); } void diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index a5cde62..255a3ef 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -778,16 +778,6 @@ static const struct parspec input_parspec[] = { "Minimum is 1k bytes.", 0, "4k", "bytes" }, - { "shm_workspace", - tweak_bytes_u, &mgt_param.shm_workspace, 4096, UINT_MAX, - "Bytes of shmlog workspace allocated for worker threads. " - "If too big, it wastes some ram, if too small it causes " - "needless flushes of the SHM workspace.\n" - "These flushes show up in stats as " - "\"SHM flushes due to overflow\".\n" - "Minimum is 4096 bytes.", - DELAYED_EFFECT, - "8k", "bytes" }, { "shm_reclen", tweak_bytes_u, &mgt_param.shm_reclen, 16, 65535, "Maximum number of bytes in SHM log record.\n" From phk at FreeBSD.org Thu Dec 18 09:27:39 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] 27fd76a Don't use 8k stackbuffer, now that thread-stacks get smaller. Message-ID: commit 27fd76aa7d9c613bb020da8d4a6e5ea5d808d9fc Author: Poul-Henning Kamp Date: Mon Feb 20 10:23:53 2012 +0000 Don't use 8k stackbuffer, now that thread-stacks get smaller. diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 4b9fd5f..a6f348b 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -145,28 +145,39 @@ vmod_random(struct sess *sp, double lo, double hi) void __match_proto__() vmod_log(struct sess *sp, const char *fmt, ...) { - char buf[8192], *p; + char *p; + unsigned u; va_list ap; + txt t; + u = WS_Reserve(sp->req->ws, 0); + p = sp->req->ws->f; va_start(ap, fmt); - p = VRT_StringList(buf, sizeof buf, fmt, ap); + p = VRT_StringList(p, u, fmt, ap); va_end(ap); - if (p != NULL) - VSLb(sp->req->vsl, SLT_VCL_Log, "%s", buf); + if (p != NULL) { + t.b = p; + t.e = strchr(p, '\0'); + VSLbt(sp->req->vsl, SLT_VCL_Log, t); + } + WS_Release(sp->req->ws, 0); } void vmod_syslog(struct sess *sp, int fac, const char *fmt, ...) { - char buf[8192], *p; + char *p; + unsigned u; va_list ap; - (void)sp; + u = WS_Reserve(sp->req->ws, 0); + p = sp->req->ws->f; va_start(ap, fmt); - p = VRT_StringList(buf, sizeof buf, fmt, ap); + p = VRT_StringList(p, u, fmt, ap); va_end(ap); if (p != NULL) - syslog(fac, "%s", buf); + syslog(fac, "%s", p); + WS_Release(sp->req->ws, 0); } const char * __match_proto__() From phk at FreeBSD.org Thu Dec 18 09:27:39 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] a6de126 Use tweak_bytes() for wthread params. Message-ID: commit a6de1269903d07c0550ebf1cf465d0760ff849f1 Author: Poul-Henning Kamp Date: Mon Feb 20 11:00:37 2012 +0000 Use tweak_bytes() for wthread params. Reduce stacksize to 32k. diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index 916fad8..5c65f6e 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -69,7 +69,7 @@ struct params { unsigned wthread_fail_delay; unsigned wthread_purge_delay; unsigned wthread_stats_rate; - unsigned wthread_stacksize; + ssize_t wthread_stacksize; unsigned queue_max; diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index a74fd17..97ac5da 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -406,22 +406,19 @@ main(int argc, char * const *argv) * Adjust default parameters for 32 bit systems to conserve * VM space. */ - MCF_ParamSet(cli, "workspace_client", "16384"); + MCF_ParamSet(cli, "workspace_client", "16k"); cli_check(cli); - MCF_ParamSet(cli, "workspace_backend", "16384"); + MCF_ParamSet(cli, "workspace_backend", "16k"); cli_check(cli); - MCF_ParamSet(cli, "http_resp_size", "8192"); + MCF_ParamSet(cli, "http_resp_size", "8k"); cli_check(cli); - MCF_ParamSet(cli, "http_req_size", "12288"); + MCF_ParamSet(cli, "http_req_size", "12k"); cli_check(cli); - MCF_ParamSet(cli, "thread_pool_stack", "32bit"); - cli_check(cli); - - MCF_ParamSet(cli, "gzip_buffer", "4096"); + MCF_ParamSet(cli, "gzip_buffer", "4k"); cli_check(cli); } diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 255a3ef..9e42e06 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -350,7 +350,7 @@ tweak_generic_bytes(struct cli *cli, volatile ssize_t *dest, const char *arg, /*--------------------------------------------------------------------*/ -static void +void tweak_bytes(struct cli *cli, const struct parspec *par, const char *arg) { volatile ssize_t *dest; diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h index 275dfe6..a5d4d75 100644 --- a/bin/varnishd/mgt/mgt_param.h +++ b/bin/varnishd/mgt/mgt_param.h @@ -54,6 +54,7 @@ int tweak_generic_uint(struct cli *cli, void tweak_uint(struct cli *cli, const struct parspec *par, const char *arg); void tweak_timeout(struct cli *cli, const struct parspec *par, const char *arg); +void tweak_bytes(struct cli *cli, const struct parspec *par, const char *arg); /* mgt_pool.c */ extern const struct parspec WRK_parspec[]; diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index a6f020c..e5d5cd0 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -73,21 +73,13 @@ static void tweak_stack_size(struct cli *cli, const struct parspec *par, const char *arg) { - unsigned low, u; - char buf[12]; + ssize_t low; low = sysconf(_SC_THREAD_STACK_MIN); - if (arg != NULL && !strcmp(arg, "32bit")) { - u = 65536; - if (u < low) - u = low; - sprintf(buf, "%u", u); - arg = buf; - } - - (void)tweak_generic_uint(cli, &mgt_param.wthread_stacksize, arg, - low, (uint)par->max); + tweak_bytes(cli, par, arg); + if (mgt_param.wthread_stacksize < low) + mgt_param.wthread_stacksize = low; } /*--------------------------------------------------------------------*/ @@ -221,9 +213,8 @@ const struct parspec WRK_parspec[] = { tweak_stack_size, &mgt_param.wthread_stacksize, 0, UINT_MAX, "Worker thread stack size.\n" "This is likely rounded up to a multiple of 4k by the kernel.\n" - "On 32bit systems you may need to tweak this down to fit " - "many threads into the limited address space.\n", + "The kernel/OS has a lower limit which will be enforced.\n", EXPERIMENTAL, - "-1", "bytes" }, + "32k", "bytes" }, { NULL, NULL, NULL } }; From phk at FreeBSD.org Thu Dec 18 09:27:39 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] cdcaade Remove unused worker arguments to VDI/close/recycle Message-ID: commit cdcaade761df7c804c42817aa8947d3c037c0f98 Author: Poul-Henning Kamp Date: Mon Feb 20 11:10:08 2012 +0000 Remove unused worker arguments to VDI/close/recycle diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 45f36f2..bdc3659 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -677,8 +677,8 @@ void VBE_UseHealth(const struct director *vdi); struct vbc *VDI_GetFd(const struct director *, struct sess *sp); int VDI_Healthy(const struct director *, const struct sess *sp); -void VDI_CloseFd(struct worker *wrk, struct vbc **vbp); -void VDI_RecycleFd(struct worker *wrk, struct vbc **vbp); +void VDI_CloseFd(struct vbc **vbp); +void VDI_RecycleFd(struct vbc **vbp); void VDI_AddHostHeader(struct http *to, const struct vbc *vbc); void VBE_Poll(void); void VDI_Init(void); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 57a7e05..b6ee5a5 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -287,7 +287,7 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) break; if (bo != NULL) { AN(bo->do_stream); - VDI_CloseFd(wrk, &bo->vbc); + VDI_CloseFd(&bo->vbc); HSH_Drop(wrk, &sp->req->obj); VBO_DerefBusyObj(wrk, &bo); } else { @@ -635,7 +635,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) } /* We are not going to fetch the body, Close the connection */ - VDI_CloseFd(wrk, &bo->vbc); + VDI_CloseFd(&bo->vbc); } /* Clean up partial fetch */ @@ -814,7 +814,7 @@ cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req) if (req->obj == NULL) { req->err_code = 503; sp->step = STP_ERROR; - VDI_CloseFd(wrk, &bo->vbc); + VDI_CloseFd(&bo->vbc); VBO_DerefBusyObj(wrk, &req->busyobj); return (0); } diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c index 6554b18..6dda80d 100644 --- a/bin/varnishd/cache/cache_dir.c +++ b/bin/varnishd/cache/cache_dir.c @@ -40,12 +40,11 @@ /* Close a connection ------------------------------------------------*/ void -VDI_CloseFd(struct worker *wrk, struct vbc **vbp) +VDI_CloseFd(struct vbc **vbp) { struct backend *bp; struct vbc *vc; - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AN(vbp); vc = *vbp; *vbp = NULL; @@ -75,12 +74,11 @@ VDI_CloseFd(struct worker *wrk, struct vbc **vbp) /* Recycle a connection ----------------------------------------------*/ void -VDI_RecycleFd(struct worker *wrk, struct vbc **vbp) +VDI_RecycleFd(struct vbc **vbp) { struct backend *bp; struct vbc *vc; - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AN(vbp); vc = *vbp; *vbp = NULL; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 8f76c24..bb725a2 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -437,7 +437,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) VSLb(req->vsl, SLT_FetchError, "backend write error: %d (%s)", errno, strerror(errno)); - VDI_CloseFd(wrk, &bo->vbc); + VDI_CloseFd(&bo->vbc); /* XXX: other cleanup ? */ return (retry); } @@ -459,7 +459,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) VSLb(req->vsl, SLT_FetchError, "http first read error: %d %d (%s)", i, errno, strerror(errno)); - VDI_CloseFd(wrk, &bo->vbc); + VDI_CloseFd(&bo->vbc); /* XXX: other cleanup ? */ /* Retryable if we never received anything */ return (i == -1 ? retry : -1); @@ -473,7 +473,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) VSLb(req->vsl, SLT_FetchError, "http first read error: %d %d (%s)", i, errno, strerror(errno)); - VDI_CloseFd(wrk, &bo->vbc); + VDI_CloseFd(&bo->vbc); /* XXX: other cleanup ? */ return (-1); } @@ -483,7 +483,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) if (http_DissectResponse(hp, htc)) { VSLb(req->vsl, SLT_FetchError, "http format error"); - VDI_CloseFd(wrk, &bo->vbc); + VDI_CloseFd(&bo->vbc); /* XXX: other cleanup ? */ return (-1); } @@ -585,14 +585,14 @@ FetchBody(struct worker *wrk, struct busyobj *bo, struct object *obj) cls, mklen); if (bo->body_status == BS_ERROR) { - VDI_CloseFd(wrk, &bo->vbc); + VDI_CloseFd(&bo->vbc); bo->stats = NULL; return (__LINE__); } if (cls < 0) { wrk->stats.fetch_failed++; - VDI_CloseFd(wrk, &bo->vbc); + VDI_CloseFd(&bo->vbc); obj->len = 0; bo->stats = NULL; return (__LINE__); @@ -625,9 +625,9 @@ FetchBody(struct worker *wrk, struct busyobj *bo, struct object *obj) } if (cls) - VDI_CloseFd(wrk, &bo->vbc); + VDI_CloseFd(&bo->vbc); else - VDI_RecycleFd(wrk, &bo->vbc); + VDI_RecycleFd(&bo->vbc); bo->stats = NULL; return (0); diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c index 8815c68..c486a23 100644 --- a/bin/varnishd/cache/cache_pipe.c +++ b/bin/varnishd/cache/cache_pipe.c @@ -93,7 +93,7 @@ PipeSession(struct sess *sp) if (i) { SES_Close(sp, "pipe"); - VDI_CloseFd(sp->wrk, &vc); + VDI_CloseFd(&vc); return; } @@ -133,6 +133,6 @@ PipeSession(struct sess *sp) } } SES_Close(sp, "pipe"); - VDI_CloseFd(sp->wrk, &vc); + VDI_CloseFd(&vc); bo->vbc = NULL; } From phk at FreeBSD.org Thu Dec 18 09:27:39 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] ea9d288 silence flexelint Message-ID: commit ea9d288d868859c0a28258e2953913482e5583e4 Author: Poul-Henning Kamp Date: Mon Feb 20 11:11:30 2012 +0000 silence flexelint diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index a6f348b..d1f6771 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -163,7 +163,7 @@ vmod_log(struct sess *sp, const char *fmt, ...) WS_Release(sp->req->ws, 0); } -void +void __match_proto__() vmod_syslog(struct sess *sp, int fac, const char *fmt, ...) { char *p; From apj at mutt.dk Thu Dec 18 09:27:39 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] edf72b4 Documentation fixes from Federico G. Schwindt Message-ID: commit edf72b4ed1b9365ef71a017050c1a269f52cc260 Author: Andreas Plesner Jacobsen Date: Thu Feb 23 17:57:50 2012 +0100 Documentation fixes from Federico G. Schwindt - mention vmod_std(7) - HEADER is released and used by vmod_std(7) so put it in its own line. The explanation is still needed though. - for consistency replace leading tabs with spaces - *fallback* are not strings so avoid mentioning the type - use ascii characters in vmod_std(7) And a few style changes from me to vmod_std.rst diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index 57163c5..a76dee1 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -22,7 +22,7 @@ For instance:: The "std" vmod is one you get with Varnish, it will always be there and we will put "boutique" functions in it, such as the "toupper" function shown above. The full contents of the "std" module is -documented in XXX:TBW. +documented in vmod_std(7). This part of the manual is about how you go about writing your own VMOD, how the language interface between C and VCC works etc. This @@ -185,7 +185,12 @@ VOID Can only be used for return-value, which makes the function a VCL procedure. -IP, BOOL, HEADER +HEADER + C-type: ``enum gethdr_e, const char *`` + + XXX: explain me + +IP, BOOL XXX: these types are not released for use in vmods yet. diff --git a/doc/sphinx/reference/vmod_std.rst b/doc/sphinx/reference/vmod_std.rst index 1ea8193..9c28a3c 100644 --- a/doc/sphinx/reference/vmod_std.rst +++ b/doc/sphinx/reference/vmod_std.rst @@ -31,31 +31,31 @@ toupper ------- Prototype - toupper(STRING S) + toupper(STRING s) Return value - String + String Description - Converts the STRING S to upper case. + Converts the string *s* to upper case. Example set beresp.http.x-scream = std.toupper("yes!"); tolower ------- Prototype - tolower(STRING S) + tolower(STRING s) Return value - String + String Description - Converts the STRING to lower case. + Converts the string *s* to lower case. Example - set beresp.http.x-nice = std.tolower("VerY"); + set beresp.http.x-nice = std.tolower("VerY"); set_up_tos ---------- Prototype - set_ip_tos(INT I) + set_ip_tos(INT i) Return value - Void + Void Description Sets the Type-of-Service flag for the current session. Please note that the TOS flag is not removed by the end of the @@ -71,7 +71,7 @@ random Prototype random(REAL a, REAL b) Return value - Real + Real Description Returns a random REAL number between *a* and *b*. Example @@ -82,9 +82,9 @@ log Prototype log(STRING string) Return value - Void + Void Description - Logs string to the shared memory log. + Logs *string* to the shared memory log. Example std.log("Something fishy is going on with the vhost " + req.host); @@ -93,7 +93,7 @@ syslog Prototype syslog(INT priority, STRING string) Return value - Void + Void Description Logs *string* to syslog marked with *priority*. Example @@ -104,7 +104,7 @@ fileread Prototype fileread(STRING filename) Return value - String + String Description Reads a file and returns a string with the content. Please note that it is not recommended to send variables to this @@ -118,12 +118,11 @@ duration Prototype duration(STRING s, DURATION fallback) Return value - Duration + Duration Description - Converts the string s to seconds. s can be quantified with the - usual s (seconds), m (minutes), h (hours), d (days) and w - (weeks) units. If it fails to parse the string *fallback* - will be used + Converts the string *s* to seconds. *s* can be quantified with + the usual s (seconds), m (minutes), h (hours), d (days) and w + (weeks) units. If *s* fails to parse, *fallback* will be used. Example set beresp.ttl = std.duration("1w", 3600); @@ -132,19 +131,19 @@ integer Prototype integer(STRING s, INT fallback) Return value - Int + Int Description - Converts the string s to an integer. If it fails to parse the - string *fallback* will be used + Converts the string *s* to an integer. If *s* fails to parse, + *fallback* will be used Example - if (std.integer(beresp.http.x-foo, 0) > 5) { ? } + if (std.integer(beresp.http.x-foo, 0) > 5) { ... } collect ------- Prototype collect(HEADER header) Return value - Void + Void Description Collapses the header, joining the headers into one. Example From lasse at varnish-software.com Thu Dec 18 09:27:39 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] 21beea8 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit 21beea85fff9f6b645f4d35b93bafba2ace9d411 Merge: db8b1d3 edf72b4 Author: Lasse Karstensen Date: Fri Feb 24 11:04:26 2012 +0100 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From lasse at varnish-software.com Thu Dec 18 09:27:39 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] 4e529da Use american english Message-ID: commit 4e529daddf4be75948d70f41f71d2c02e84eae70 Author: Lasse Karstensen Date: Fri Feb 24 11:08:03 2012 +0100 Use american english diff --git a/doc/sphinx/tutorial/virtualized.rst b/doc/sphinx/tutorial/virtualized.rst new file mode 100644 index 0000000..317d3e2 --- /dev/null +++ b/doc/sphinx/tutorial/virtualized.rst @@ -0,0 +1,23 @@ + +Running Varnish in a virtualized environment +-------------------------------------------- + +It is possible, but not recommended for high performance, to run +Varnish on virtualized hardware. Reduced disk- and network performance +will reduce the performance a bit so make sure your system has good IO +performance. + +OpenVZ +~~~~~~ + +If you are running on 64bit OpenVZ (or Parallels VPS), you must reduce +the maximum stack size before starting Varnish. The default allocates +to much memory per thread, which will make varnish fail as soon as the +number of threads (==traffic) increases. + +Reduce the maximum stack size by running:: + + ulimit -s 256 + +in the startup script. + From lasse at varnish-software.com Thu Dec 18 09:27:39 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] a502bae Use american english Message-ID: commit a502bae63c0f3b1d4c26c8108bc964369d85458d Author: Lasse Karstensen Date: Fri Feb 24 11:08:31 2012 +0100 Use american english diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst index c3a05f5..c8eb2c2 100644 --- a/doc/sphinx/tutorial/index.rst +++ b/doc/sphinx/tutorial/index.rst @@ -28,7 +28,7 @@ separate topic. Good luck. purging compression esi - virtualised + virtualized advanced_backend_servers handling_misbehaving_servers advanced_topics From apj at mutt.dk Thu Dec 18 09:27:39 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] cd0f595 Add more info about changes from 2.1 to 3.0. Thanks to xcir.net for inspiration. Message-ID: commit cd0f595b7bd511078fd5f0d9ec536bb22f7a95a7 Author: Andreas Plesner Jacobsen Date: Fri Feb 24 12:16:25 2012 +0100 Add more info about changes from 2.1 to 3.0. Thanks to xcir.net for inspiration. diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst index 0ed4f9f..cee17b8 100644 --- a/doc/sphinx/installation/upgrade.rst +++ b/doc/sphinx/installation/upgrade.rst @@ -60,10 +60,10 @@ becomes:: } } -``beresp.cacheable`` is gone -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``beresp.cacheable`` and ``obj.cacheable``is gone +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``beresp.cacheable`` is gone, and can be replaced with ``beresp.ttl > 0s`` +``beresp.cacheable`` is gone, and can be replaced with ``beresp.ttl > 0s``. Similarly ``obj.cacheable`` can be replaced with ``obj.ttl > 0s``. returns are now done with the ``return()`` function ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -111,6 +111,56 @@ The difference in behaviour of ``pass`` in ``vcl_recv`` and different, you must now do ``return(hit_for_pass)`` when doing a pass in ``vcl_fetch``. +Changes to runtime parameters +============================= + +Deleted parameters +~~~~~~~~~~~~~~~~~~ + +``cache_vbe_conns`` and ``err_ttl`` has been removed. + +New parameters +~~~~~~~~~~~~~~ + +The following parameters have been added, see man varnishd for reference: +* ``default_keep`` +* ``expiry_sleep`` +* ``fetch_maxchunksize`` +* ``gzip_level`` +* ``gzip_memlevel`` +* ``gzip_stack_buffer`` +* ``gzip_tmp_space`` +* ``gzip_window`` +* ``http_gzip_support`` +* ``http_req_hdr_len`` +* ``http_req_size`` +* ``http_resp_hdr_len`` +* ``http_resp_size`` +* ``shortlived`` +* ``thread_pool_workspace`` +* ``vcc_err_unref`` +* ``vcl_dir`` +* ``vmod_dir`` + +Changed default values +~~~~~~~~~~~~~~~~~~~~~~ + +The following parameters have new defaults: + +* ``ban_lurker_sleep`` changed from 0 to 0.01 seconds, enabling the ban lurker by default. +* ``connect_timeout`` changed from 0.4 to 0.7 seconds. +* ``log_hashstring`` changed from off to on. +* ``send_timeout`` changed from 60 to 60 seconds. +* ``thread_pool_add_delay`` changed from 20 to 2 ms. + +Changed parameter names +~~~~~~~~~~~~~~~~~~~~~~~ + +The following parameters have new names: +* ``http_headers`` has been renamed to ``http_max_hdr``. +* ``max_esi_includes`` has been renamed to ``max_esi_depth``. +* ``overflow_max`` has been renamed to ``queue_max``. +* ``purge_dups`` has been renamed to ``ban_dups``. Changes to behaviour ==================== From apj at mutt.dk Thu Dec 18 09:27:39 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] 9f8a77d Fix syntax Message-ID: commit 9f8a77dc60d5a68b419ab97b9a2ada9aa3a397f3 Author: Andreas Plesner Jacobsen Date: Fri Feb 24 12:20:37 2012 +0100 Fix syntax diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst index cee17b8..2402af0 100644 --- a/doc/sphinx/installation/upgrade.rst +++ b/doc/sphinx/installation/upgrade.rst @@ -123,6 +123,7 @@ New parameters ~~~~~~~~~~~~~~ The following parameters have been added, see man varnishd for reference: + * ``default_keep`` * ``expiry_sleep`` * ``fetch_maxchunksize`` @@ -157,6 +158,7 @@ Changed parameter names ~~~~~~~~~~~~~~~~~~~~~~~ The following parameters have new names: + * ``http_headers`` has been renamed to ``http_max_hdr``. * ``max_esi_includes`` has been renamed to ``max_esi_depth``. * ``overflow_max`` has been renamed to ``queue_max``. From lasse at varnish-software.com Thu Dec 18 09:27:39 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] bf2da4e forgot to delete, thank you scoof Message-ID: commit bf2da4e11f7a27e057b1174eec1cab2c2b375d89 Author: Lasse Karstensen Date: Fri Feb 24 14:11:06 2012 +0100 forgot to delete, thank you scoof diff --git a/doc/sphinx/tutorial/virtualised.rst b/doc/sphinx/tutorial/virtualised.rst deleted file mode 100644 index ea8ade5..0000000 --- a/doc/sphinx/tutorial/virtualised.rst +++ /dev/null @@ -1,23 +0,0 @@ - -Running Varnish in a virtualized environment --------------------------------------------- - -It is possible, but not recommended for high performance, to run -Varnish on virtualised hardware. Reduced disk- and network performance -will reduce the performance a bit so make sure your system has good IO -performance. - -OpenVZ -~~~~~~ - -If you are running on 64bit OpenVZ (or Parallels VPS), you must reduce -the maximum stack size before starting Varnish. The default allocates -to much memory per thread, which will make varnish fail as soon as the -number of threads (==traffic) increases. - -Reduce the maximum stack size by running:: - - ulimit -s 256 - -in the startup script. - From lasse at varnish-software.com Thu Dec 18 09:27:39 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] 3a00261 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit 3a002613f7bbd0a39ac97d5d223ea299803f9133 Merge: bf2da4e 9f8a77d Author: Lasse Karstensen Date: Fri Feb 24 14:11:17 2012 +0100 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From apj at mutt.dk Thu Dec 18 09:27:39 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] db0f3b2 Grammar Message-ID: commit db0f3b2044252f2eefe4b4b0296ae5f29c3aa6af Author: Andreas Plesner Jacobsen Date: Fri Feb 24 14:52:38 2012 +0100 Grammar diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst index 2402af0..2c06a3f 100644 --- a/doc/sphinx/installation/upgrade.rst +++ b/doc/sphinx/installation/upgrade.rst @@ -60,8 +60,8 @@ becomes:: } } -``beresp.cacheable`` and ``obj.cacheable``is gone -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``beresp.cacheable`` and ``obj.cacheable`` are gone +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``beresp.cacheable`` is gone, and can be replaced with ``beresp.ttl > 0s``. Similarly ``obj.cacheable`` can be replaced with ``obj.ttl > 0s``. From lasse at varnish-software.com Thu Dec 18 09:27:39 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] e995b1a Add short example on how to get Websockets to work Message-ID: commit e995b1a518b4c6d89424eb8cdc6deb091ea04787 Author: Lasse Karstensen Date: Fri Feb 24 15:12:44 2012 +0100 Add short example on how to get Websockets to work diff --git a/doc/sphinx/tutorial/websockets.rst b/doc/sphinx/tutorial/websockets.rst new file mode 100644 index 0000000..a74353e --- /dev/null +++ b/doc/sphinx/tutorial/websockets.rst @@ -0,0 +1,20 @@ + +Using Websockets +---------------- + +Websockets is a technology for creating a bidirectional stream-based channel over HTTP. + +To run websockets through Varnish you need to pipe it, and copy the Upgrade header. Use the following +VCL config to do so:: + + sub vcl_pipe { + if (req.http.upgrade) { + set bereq.http.upgrade = req.http.upgrade; + } + } + sub vcl_recv { + if (req.http.Upgrade ~ "(?i)websocket") { + return (pipe); + } + } + From lasse at varnish-software.com Thu Dec 18 09:27:39 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] 064077f Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit 064077f0d17e28aaca5b471c3f339fa18724c30d Merge: e995b1a db0f3b2 Author: Lasse Karstensen Date: Fri Feb 24 15:12:54 2012 +0100 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From lasse at varnish-software.com Thu Dec 18 09:27:39 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] 36fca1e Add short example on how to get Websockets to work (#2) Message-ID: commit 36fca1e9b0475e141cdd82447148b5d8ab63ac24 Author: Lasse Karstensen Date: Fri Feb 24 15:14:12 2012 +0100 Add short example on how to get Websockets to work (#2) diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst index c8eb2c2..4949aa0 100644 --- a/doc/sphinx/tutorial/index.rst +++ b/doc/sphinx/tutorial/index.rst @@ -29,6 +29,7 @@ separate topic. Good luck. compression esi virtualized + websockets advanced_backend_servers handling_misbehaving_servers advanced_topics From apj at mutt.dk Thu Dec 18 09:27:39 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] 195b601 Correct value for 2.1 Message-ID: commit 195b601710ae556f3bcb6c27b73f6d60c3986bde Author: Andreas Plesner Jacobsen Date: Fri Feb 24 15:16:15 2012 +0100 Correct value for 2.1 diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst index 2c06a3f..93446ea 100644 --- a/doc/sphinx/installation/upgrade.rst +++ b/doc/sphinx/installation/upgrade.rst @@ -151,7 +151,7 @@ The following parameters have new defaults: * ``ban_lurker_sleep`` changed from 0 to 0.01 seconds, enabling the ban lurker by default. * ``connect_timeout`` changed from 0.4 to 0.7 seconds. * ``log_hashstring`` changed from off to on. -* ``send_timeout`` changed from 60 to 60 seconds. +* ``send_timeout`` changed from 600 to 60 seconds. * ``thread_pool_add_delay`` changed from 20 to 2 ms. Changed parameter names From phk at FreeBSD.org Thu Dec 18 09:27:39 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] c082835 Drop the body of hit-for-pass objects once we have delivered them to the original requester. Message-ID: commit c082835a88462548bddf78fa7c88fb948fc1f1d2 Author: Poul-Henning Kamp Date: Mon Feb 27 08:31:41 2012 +0000 Drop the body of hit-for-pass objects once we have delivered them to the original requester. Submitted by: DocWilco diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index b6ee5a5..6e6d54f 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -327,6 +327,7 @@ cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); AZ(req->busyobj); req->director = NULL; @@ -334,6 +335,13 @@ cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req) RES_WriteObj(sp); + /* No point in saving the body if it is hit-for-pass */ + if (req->obj->objcore != NULL) { + CHECK_OBJ_NOTNULL(req->obj->objcore, OBJCORE_MAGIC); + if (req->obj->objcore->flags & OC_F_PASS) + STV_Freestore(req->obj); + } + assert(WRW_IsReleased(wrk)); (void)HSH_Deref(&wrk->stats, NULL, &req->obj); http_Teardown(req->resp); From phk at FreeBSD.org Thu Dec 18 09:27:39 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] 96e2a95 Avoid taking the saintmode lock if the list empty. Message-ID: commit 96e2a95ee17b19b35ec880b08a23696cc8a16f36 Author: Poul-Henning Kamp Date: Mon Feb 27 08:47:12 2012 +0000 Avoid taking the saintmode lock if the list empty. Submitted by: DocWilco diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 961cb8d..52f1c38 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -261,16 +261,15 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp) /* VRT/VCC sets threshold to UINT_MAX to mark that it's not * specified by VCL (thus use param). */ - if (vs->vrt->saintmode_threshold == UINT_MAX) + threshold = vs->vrt->saintmode_threshold; + if (threshold == UINT_MAX) threshold = cache_param->saintmode_threshold; - else - threshold = vs->vrt->saintmode_threshold; if (backend->admin_health == ah_healthy) threshold = UINT_MAX; - /* Saintmode is disabled */ - if (threshold == 0) + /* Saintmode is disabled, or list is empty */ + if (threshold == 0 || VTAILQ_EMPTY(&backend->troublelist)) return (1); if (sp->req->objcore == NULL) From phk at FreeBSD.org Thu Dec 18 09:27:39 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] 4d047a9 Accept-filters on FreeBSD has been broken for ages, fix them. Message-ID: commit 4d047a90b19f8821262567b793e535639b1b701a Author: Poul-Henning Kamp Date: Mon Feb 27 11:14:45 2012 +0000 Accept-filters on FreeBSD has been broken for ages, fix them. Add accept-filter param which controls if we attempt kernel filteringer. Apply filters after listen() when we do. Report failuers with VSL(SLT_Error) Disable filters in pipe-lining test-case. Fixes #1101 diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 07e7f5e..3ca79c1 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -292,6 +292,7 @@ vca_acct(void *arg) #endif struct listen_sock *ls; double t0, now; + int i; THR_SetName("cache-acceptor"); (void)arg; @@ -302,6 +303,13 @@ vca_acct(void *arg) AZ(listen(ls->sock, cache_param->listen_depth)); AZ(setsockopt(ls->sock, SOL_SOCKET, SO_LINGER, &linger, sizeof linger)); + if (cache_param->accept_filter) { + i = VTCP_filter_http(ls->sock); + if (i) + VSL(SLT_Error, ls->sock, + "Kernel filtering: sock=%d, ret=%d %s\n", + ls->sock, i, strerror(errno)); + } } hack_ready = 1; diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index 5c65f6e..6e502fd 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -108,6 +108,8 @@ struct params { /* VCL traces */ unsigned vcl_trace; + unsigned accept_filter; + /* Listen address */ char *listen_address; diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 0f91d05..8661b73 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -53,7 +53,6 @@ #include "vev.h" #include "vlu.h" #include "vss.h" -#include "vtcp.h" #include "vtim.h" #include "mgt_cli.h" @@ -239,12 +238,6 @@ open_sockets(void) mgt_child_inherit(ls->sock, "sock"); - /* - * Set nonblocking mode to avoid a race where a client - * closes before we call accept(2) and nobody else are in - * the listen queue to release us. - */ - (void)VTCP_filter_http(ls->sock); good++; } if (!good) diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 9e42e06..31ef010 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -877,6 +877,10 @@ static const struct parspec input_parspec[] = { "default.", 0, "off", "bool" }, + { "accept_filter", tweak_bool, &mgt_param.accept_filter, 0, 0, + "Enable kernel accept-filters, if supported by the kernel.", + MUST_RESTART, + "on", "bool" }, { "listen_address", tweak_listen_address, NULL, 0, 0, "Whitespace separated list of network endpoints where " "Varnish will accept requests.\n" diff --git a/bin/varnishtest/tests/b00013.vtc b/bin/varnishtest/tests/b00013.vtc index 57250ad..8ae90c4 100644 --- a/bin/varnishtest/tests/b00013.vtc +++ b/bin/varnishtest/tests/b00013.vtc @@ -9,10 +9,10 @@ server s1 { txresp -body "foobar" } -start -varnish v1 -vcl+backend {} -start +varnish v1 -arg "-p accept_filter=false" -vcl+backend {} -start client c1 { - send "GET /foo HTTP/1.1\n\nGET " + send "GET /foo HTTP/1.1\r\n\r\nGET " rxresp expect resp.status == 200 expect resp.http.content-length == 3 diff --git a/configure.ac b/configure.ac index 952b71b..7e5e57e 100644 --- a/configure.ac +++ b/configure.ac @@ -328,6 +328,15 @@ fi AM_MISSING_HAS_RUN AC_CHECK_PROGS(PYTHON, [python3 python3.1 python3.2 python2.7 python2.6 python2.5 python2 python], [AC_MSG_ERROR([Python is needed to build Varnish, please install python.])]) +AC_CHECK_DECL([SO_ACCEPTFILTER], + AC_DEFINE(HAVE_ACCEPT_FILTERS,1,[Define to 1 if you have accept filters]), + , + [ +#include +#include + ] +) + # Older Solaris versions define SO_{RCV,SND}TIMEO, but do not # implement them. # diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index e361738..8bb19d4 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -130,33 +130,46 @@ VTCP_hisname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen) /*--------------------------------------------------------------------*/ +#ifdef HAVE_ACCEPT_FILTERS + int VTCP_filter_http(int sock) { -#ifdef HAVE_ACCEPT_FILTERS + int retval; struct accept_filter_arg afa; - int i; memset(&afa, 0, sizeof(afa)); strcpy(afa.af_name, "httpready"); - errno = 0; - i = setsockopt(sock, SOL_SOCKET, SO_ACCEPTFILTER, - &afa, sizeof(afa)); - /* XXX ugly */ - if (i) - printf("Acceptfilter(%d, httpready): %d %s\n", - sock, i, strerror(errno)); - return (i); + retval = setsockopt(sock, SOL_SOCKET, SO_ACCEPTFILTER, + &afa, sizeof afa ); + return (retval); +} + #elif defined(__linux) + +int +VTCP_filter_http(int sock) +{ + int retval; int defer = 1; - setsockopt(sock, SOL_TCP,TCP_DEFER_ACCEPT,(char *) &defer, sizeof(int)); - return (0); + + retval = setsockopt(sock, SOL_TCP,TCP_DEFER_ACCEPT, + &defer, sizeof defer); + return (retval); +} + #else + +int +VTCP_filter_http(int sock) +{ + errno = EOPNOTSUPP; (void)sock; - return (0); -#endif + return (-1); } +#endif + /*-------------------------------------------------------------------- * Functions for controlling NONBLOCK mode. * From phk at FreeBSD.org Thu Dec 18 09:27:39 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] 21e653c Use the hash digest as identification instead of the neutered objhead pointer, in order to not have dependency between trouble entry and objhead lifetime. Message-ID: commit 21e653c3a3e0686232646b9f933de72b6536fb50 Author: Poul-Henning Kamp Date: Mon Feb 27 12:41:33 2012 +0000 Use the hash digest as identification instead of the neutered objhead pointer, in order to not have dependency between trouble entry and objhead lifetime. Fixes #1091 diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 52f1c38..dcb87dd 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -244,7 +244,6 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp) unsigned i = 0, retval; unsigned int threshold; struct backend *backend; - uintptr_t target; double now; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); @@ -276,7 +275,6 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp) return (1); now = sp->t_req; - target = (uintptr_t)(sp->req->objcore->objhead); old = NULL; retval = 1; @@ -291,7 +289,7 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp) break; } - if (tr->target == target) { + if (!memcmp(tr->digest, sp->req->digest, sizeof tr->digest)) { retval = 0; break; } diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index b1688f9..96f0a63 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -97,7 +97,7 @@ struct director { struct trouble { unsigned magic; #define TROUBLE_MAGIC 0x4211ab21 - uintptr_t target; + unsigned char digest[DIGEST_LEN]; double timeout; VTAILQ_ENTRY(trouble) list; }; diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 72fa4dc..2e56666 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -152,7 +152,7 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a) ALLOC_OBJ(new, TROUBLE_MAGIC); AN(new); - new->target = (uintptr_t)(sp->req->objcore->objhead); + memcpy(new->digest, sp->req->digest, sizeof new->digest); new->timeout = sp->t_req + a; /* Insert the new item on the list before the first item with a diff --git a/bin/varnishtest/tests/r01091.vtc b/bin/varnishtest/tests/r01091.vtc new file mode 100644 index 0000000..9234ffc --- /dev/null +++ b/bin/varnishtest/tests/r01091.vtc @@ -0,0 +1,36 @@ +varnishtest "Test fallback director with saint mode" + +server s1 { + rxreq + txresp -hdr "Foo: 1" + accept + rxreq + txresp -hdr "Foo: 1" +} -start + +server s2 { + rxreq + txresp -hdr "Foo: 2" -bodylen 1 +} -start + +varnish v1 -vcl+backend { + director f1 fallback { + { .backend = s1; } + { .backend = s2; } + } + sub vcl_recv { + set req.backend = f1; + } + sub vcl_fetch { + if(req.restarts < 1) { + set beresp.saintmode = 1h; + return(restart); + } + } +} -start + +client c1 { + txreq + rxresp + expect resp.http.foo == "2" +} -run From perbu at varnish-software.com Thu Dec 18 09:27:39 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] 09b77e3 backend cookies Message-ID: commit 09b77e3ab021536cbfd893d602c51dfec7a8eb4a Author: Per Buer Date: Wed Feb 29 16:17:09 2012 +0100 backend cookies diff --git a/doc/sphinx/tutorial/cookies.rst b/doc/sphinx/tutorial/cookies.rst index 0278712..20fd302 100644 --- a/doc/sphinx/tutorial/cookies.rst +++ b/doc/sphinx/tutorial/cookies.rst @@ -13,6 +13,9 @@ This can be overly conservative. A lot of sites use Google Analytics cookie is used by the client side javascript and is therefore of no interest to the server. +Cookies from the client +~~~~~~~~~~~~~~~~~~~~~~~ + For a lot of web application it makes sense to completely disregard the cookies unless you are accessing a special part of the web site. This VCL snippet in vcl_recv will disregard cookies unless you are @@ -63,3 +66,12 @@ cookies named COOKIE1 and COOKIE2 and you can marvel at it:: The example is taken from the Varnish Wiki, where you can find other scary examples of what can be done in VCL. + +Cookies coming from the backend +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If your backend server sets a cookie using the Set-Cookie header +Varnish will not cache the page. A hit-for-pass object (see +:ref:`tutorial_vcl_fetch_actions`) is created. So, if the backend +server acts silly and sets unwanted cookies just unset the Set-Cookie +header and all should be fine. From apj at mutt.dk Thu Dec 18 09:27:39 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:39 +0100 Subject: [experimental-ims] 86e85bb 3.0 syntax Message-ID: commit 86e85bb6d95785709566cdfd5aab6db24945f3a9 Author: Andreas Plesner Jacobsen Date: Sun Mar 4 16:48:45 2012 +0100 3.0 syntax diff --git a/doc/sphinx/tutorial/increasing_your_hitrate.rst b/doc/sphinx/tutorial/increasing_your_hitrate.rst index 83236d6..73d983e 100644 --- a/doc/sphinx/tutorial/increasing_your_hitrate.rst +++ b/doc/sphinx/tutorial/increasing_your_hitrate.rst @@ -146,7 +146,7 @@ header. You could easily add support for this header in VCL. In vcl_fetch:: if (beresp.http.Pragma ~ "nocache") { - pass; + return(pass); } Authorization From apj at mutt.dk Thu Dec 18 09:27:40 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] 5c9d66c Really change to 3.0 syntax Message-ID: commit 5c9d66ce9fdb31a73d1371194ea97a0289db60b2 Author: Andreas Plesner Jacobsen Date: Sun Mar 4 16:51:38 2012 +0100 Really change to 3.0 syntax diff --git a/doc/sphinx/tutorial/increasing_your_hitrate.rst b/doc/sphinx/tutorial/increasing_your_hitrate.rst index 73d983e..b9fa7e6 100644 --- a/doc/sphinx/tutorial/increasing_your_hitrate.rst +++ b/doc/sphinx/tutorial/increasing_your_hitrate.rst @@ -146,7 +146,7 @@ header. You could easily add support for this header in VCL. In vcl_fetch:: if (beresp.http.Pragma ~ "nocache") { - return(pass); + return(hit_for_pass); } Authorization From phk at FreeBSD.org Thu Dec 18 09:27:40 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] 5cd69e0 Eliminate obj arg to FetchBody() Message-ID: commit 5cd69e085bf15d84ad30233a72c52817def66ca2 Author: Poul-Henning Kamp Date: Mon Mar 5 10:36:52 2012 +0000 Eliminate obj arg to FetchBody() Mostly by: martin diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index bdc3659..dceac39 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -748,7 +748,7 @@ struct storage *FetchStorage(struct busyobj *, ssize_t sz); int FetchError(struct busyobj *, const char *error); int FetchError2(struct busyobj *, const char *error, const char *more); int FetchHdr(struct sess *sp, int need_host_hdr, int sendbody); -int FetchBody(struct worker *w, struct busyobj *bo, struct object *obj); +int FetchBody(struct worker *w, struct busyobj *bo); int FetchReqBody(const struct sess *sp, int sendbody); void Fetch_Init(void); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 6e6d54f..9cbc905 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -830,8 +830,10 @@ cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req) req->storage_hint = NULL; - if (bo->do_gzip || - (bo->is_gzip && !bo->do_gunzip)) + AZ(bo->fetch_obj); + bo->fetch_obj = req->obj; + + if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) req->obj->gziped = 1; if (vary != NULL) { @@ -903,7 +905,8 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); /* Use unmodified headers*/ - i = FetchBody(wrk, bo, req->obj); + i = FetchBody(wrk, bo); + bo->fetch_obj = NULL; http_Teardown(bo->bereq); http_Teardown(bo->beresp); @@ -1115,6 +1118,8 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) /* If we inserted a new object it's a miss */ if (oc->flags & OC_F_BUSY) { + CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC); + assert(oc->busyobj == req->busyobj); wrk->stats.cache_miss++; if (req->vary_l != NULL) { @@ -1130,7 +1135,6 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) req->vary_e = NULL; req->objcore = oc; - CHECK_OBJ_NOTNULL(req->busyobj, BUSYOBJ_MAGIC); sp->step = STP_MISS; return (0); } diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index bb725a2..1dae8b9 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -493,18 +493,19 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) /*--------------------------------------------------------------------*/ int -FetchBody(struct worker *wrk, struct busyobj *bo, struct object *obj) +FetchBody(struct worker *wrk, struct busyobj *bo) { int cls; struct storage *st; int mklen; ssize_t cl; struct http_conn *htc; + struct object *obj; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - AZ(bo->fetch_obj); CHECK_OBJ_NOTNULL(bo->vbc, VBC_MAGIC); + obj = bo->fetch_obj; CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC); @@ -525,7 +526,6 @@ FetchBody(struct worker *wrk, struct busyobj *bo, struct object *obj) AZ(bo->vgz_rx); AZ(VTAILQ_FIRST(&obj->store)); - bo->fetch_obj = obj; bo->fetch_failed = 0; /* XXX: pick up estimate from objdr ? */ @@ -578,8 +578,6 @@ FetchBody(struct worker *wrk, struct busyobj *bo, struct object *obj) */ AZ(vfp_nop_end(bo)); - bo->fetch_obj = NULL; - VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s) cls %d mklen %d", bo->body_status, body_status(bo->body_status), cls, mklen); From phk at FreeBSD.org Thu Dec 18 09:27:40 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] b399a62 Add a set of VFP method functions which can do various sanity asserts for us in a single place. Message-ID: commit b399a62ff31a2ee586380b7638193a1e0d0bc236 Author: Poul-Henning Kamp Date: Mon Mar 5 11:08:27 2012 +0000 Add a set of VFP method functions which can do various sanity asserts for us in a single place. diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index 14ca31b..ffb90f8 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -346,7 +346,6 @@ vfp_esi_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes) vef = bo->vef_priv; CHECK_OBJ_NOTNULL(vef, VEF_MAGIC); - AZ(bo->fetch_failed); AN(bo->vep); assert(&bo->htc == htc); if (bo->is_gzip && bo->do_gunzip) diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 1dae8b9..1313bc1 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -73,6 +73,40 @@ FetchError(struct busyobj *bo, const char *error) } /*-------------------------------------------------------------------- + * VFP method functions + */ + +static void +VFP_Begin(struct busyobj *bo, size_t estimate) +{ + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + AN(bo->vfp); + + bo->vfp->begin(bo, estimate); +} + +static int +VFP_Bytes(struct busyobj *bo, struct http_conn *htc, ssize_t sz) +{ + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + AN(bo->vfp); + CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); + AZ(bo->fetch_failed); + + return (bo->vfp->bytes(bo, htc, sz)); +} + +static int +VFP_End(struct busyobj *bo) +{ + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + AN(bo->vfp); + + return (bo->vfp->end(bo)); +} + + +/*-------------------------------------------------------------------- * VFP_NOP * * This fetch-processor does nothing but store the object. @@ -235,7 +269,7 @@ fetch_straight(struct busyobj *bo, struct http_conn *htc, ssize_t cl) } else if (cl == 0) return (0); - i = bo->vfp->bytes(bo, htc, cl); + i = VFP_Bytes(bo, htc, cl); if (i <= 0) return (FetchError(bo, "straight insufficient bytes")); return (0); @@ -292,7 +326,7 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc) if (cl < 0) return (FetchError(bo,"chunked header number syntax")); - if (cl > 0 && bo->vfp->bytes(bo, htc, cl) <= 0) + if (cl > 0 && VFP_Bytes(bo, htc, cl) <= 0) return (-1); i = HTC_Read(htc, buf, 1); @@ -314,7 +348,7 @@ fetch_eof(struct busyobj *bo, struct http_conn *htc) int i; assert(bo->body_status == BS_EOF); - i = bo->vfp->bytes(bo, htc, SSIZE_MAX); + i = VFP_Bytes(bo, htc, SSIZE_MAX); if (i < 0) return (-1); return (0); @@ -541,24 +575,24 @@ FetchBody(struct worker *wrk, struct busyobj *bo) break; case BS_LENGTH: cl = fetch_number(bo->h_content_length, 10); - bo->vfp->begin(bo, cl > 0 ? cl : 0); + VFP_Begin(bo, cl > 0 ? cl : 0); cls = fetch_straight(bo, htc, cl); mklen = 1; - if (bo->vfp->end(bo)) + if (VFP_End(bo)) cls = -1; break; case BS_CHUNKED: - bo->vfp->begin(bo, cl); + VFP_Begin(bo, cl); cls = fetch_chunked(bo, htc); mklen = 1; - if (bo->vfp->end(bo)) + if (VFP_End(bo)) cls = -1; break; case BS_EOF: - bo->vfp->begin(bo, cl); + VFP_Begin(bo, cl); cls = fetch_eof(bo, htc); mklen = 1; - if (bo->vfp->end(bo)) + if (VFP_End(bo)) cls = -1; break; case BS_ERROR: diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index ec1b7cd..e2b4d51 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -448,7 +448,6 @@ vfp_gunzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes) const void *dp; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - AZ(bo->fetch_failed); vg = bo->vgz_rx; CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC); AZ(vg->vz.avail_in); @@ -526,7 +525,6 @@ vfp_gzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes) const void *dp; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - AZ(bo->fetch_failed); vg = bo->vgz_rx; CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC); AZ(vg->vz.avail_in); @@ -613,7 +611,6 @@ vfp_testgzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes) struct storage *st; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - AZ(bo->fetch_failed); vg = bo->vgz_rx; CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC); AZ(vg->vz.avail_in); From phk at FreeBSD.org Thu Dec 18 09:27:40 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] f9e413c Don't assert if we fail to get storage in VFP_Begin() Message-ID: commit f9e413c77249b0b6c407c087836a1cd80d72d1ab Author: Poul-Henning Kamp Date: Mon Mar 5 11:27:37 2012 +0000 Don't assert if we fail to get storage in VFP_Begin() Fixes #1100 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 1313bc1..1f146d0 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -76,13 +76,16 @@ FetchError(struct busyobj *bo, const char *error) * VFP method functions */ -static void +static int VFP_Begin(struct busyobj *bo, size_t estimate) { CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AN(bo->vfp); bo->vfp->begin(bo, estimate); + if (bo->fetch_failed) + return (-1); + return (0); } static int @@ -146,7 +149,6 @@ vfp_nop_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes) ssize_t l, wl; struct storage *st; - AZ(bo->fetch_failed); while (bytes > 0) { st = FetchStorage(bo, 0); if (st == NULL) @@ -575,22 +577,25 @@ FetchBody(struct worker *wrk, struct busyobj *bo) break; case BS_LENGTH: cl = fetch_number(bo->h_content_length, 10); - VFP_Begin(bo, cl > 0 ? cl : 0); - cls = fetch_straight(bo, htc, cl); + cls = VFP_Begin(bo, cl > 0 ? cl : 0); + if (!cls) + cls = fetch_straight(bo, htc, cl); mklen = 1; if (VFP_End(bo)) cls = -1; break; case BS_CHUNKED: - VFP_Begin(bo, cl); - cls = fetch_chunked(bo, htc); + cls = VFP_Begin(bo, cl); + if (!cls) + cls = fetch_chunked(bo, htc); mklen = 1; if (VFP_End(bo)) cls = -1; break; case BS_EOF: - VFP_Begin(bo, cl); - cls = fetch_eof(bo, htc); + cls = VFP_Begin(bo, cl); + if (!cls) + cls = fetch_eof(bo, htc); mklen = 1; if (VFP_End(bo)) cls = -1; diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 2689cc8..c5f49de 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -213,7 +213,8 @@ pan_busyobj(const struct busyobj *bo) if (bo->do_esi) VSB_printf(pan_vsp, " do_esi\n"); if (bo->do_stream) VSB_printf(pan_vsp, " do_stream\n"); if (bo->should_close) VSB_printf(pan_vsp, " should_close\n"); - VSB_printf(pan_vsp, " bodystatus = %d,\n", bo->body_status); + VSB_printf(pan_vsp, " bodystatus = %d (%s),\n", + bo->body_status, body_status(bo->body_status)); VSB_printf(pan_vsp, " },\n"); if (VALID_OBJ(bo->vbc, BACKEND_MAGIC)) pan_vbc(bo->vbc); From phk at FreeBSD.org Thu Dec 18 09:27:40 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] 501af0b Add a bo->state that tells us what's going on. Improve error reporting at the same time. Message-ID: commit 501af0ba839629de342a2c17963bb04de8599625 Author: Poul-Henning Kamp Date: Mon Mar 5 12:12:37 2012 +0000 Add a bo->state that tells us what's going on. Improve error reporting at the same time. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index dceac39..bea32a2 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -449,6 +449,13 @@ oc_getlru(const struct objcore *oc) * streaming delivery will make use of. */ +enum busyobj_state_e { + BOS_INVALID = 0, + BOS_FETCHING, + BOS_FAILED, + BOS_FINISHED +}; + struct busyobj { unsigned magic; #define BUSYOBJ_MAGIC 0x23b95567 @@ -460,7 +467,7 @@ struct busyobj { struct vfp *vfp; struct vep_state *vep; - unsigned fetch_failed; + enum busyobj_state_e state; struct vgz *vgz_rx; struct ws ws[1]; diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index ffb90f8..6699117 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -366,12 +366,13 @@ vfp_esi_end(struct busyobj *bo) struct vsb *vsb; struct vef_priv *vef; ssize_t l; - int retval; + int retval = 0; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AN(bo->vep); - retval = bo->fetch_failed; + if (bo->state == BOS_FAILED) + retval = -1; if (bo->vgz_rx != NULL && VGZ_Destroy(&bo->vgz_rx) != VGZ_END) retval = FetchError(bo, "Gunzip+ESI Failed at the very end"); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 1f146d0..fd70d07 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -56,13 +56,13 @@ FetchError2(struct busyobj *bo, const char *error, const char *more) { CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - if (!bo->fetch_failed) { + if (bo->state == BOS_FETCHING) { if (more == NULL) VSLb(bo->vsl, SLT_FetchError, "%s", error); else VSLb(bo->vsl, SLT_FetchError, "%s: %s", error, more); } - bo->fetch_failed = 1; + bo->state = BOS_FAILED; return (-1); } @@ -83,7 +83,7 @@ VFP_Begin(struct busyobj *bo, size_t estimate) AN(bo->vfp); bo->vfp->begin(bo, estimate); - if (bo->fetch_failed) + if (bo->state == BOS_FAILED) return (-1); return (0); } @@ -94,7 +94,7 @@ VFP_Bytes(struct busyobj *bo, struct http_conn *htc, ssize_t sz) CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AN(bo->vfp); CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); - AZ(bo->fetch_failed); + assert(bo->state == BOS_FETCHING); return (bo->vfp->bytes(bo, htc, sz)); } @@ -102,10 +102,16 @@ VFP_Bytes(struct busyobj *bo, struct http_conn *htc, ssize_t sz) static int VFP_End(struct busyobj *bo) { + int i; + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AN(bo->vfp); - return (bo->vfp->end(bo)); + i = bo->vfp->end(bo); + if (i) + assert(bo->state == BOS_FAILED); + return (i); + } @@ -295,8 +301,8 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc) do { /* Skip leading whitespace */ do { - if (HTC_Read(htc, buf, 1) <= 0) - return (-1); + if (HTC_Read(htc, buf, 1) <= 0) + return (FetchError(bo, "chunked read err")); } while (vct_islws(buf[0])); if (!vct_ishex(buf[0])) @@ -306,7 +312,8 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc) for (u = 1; u < sizeof buf; u++) { do { if (HTC_Read(htc, buf + u, 1) <= 0) - return (-1); + return (FetchError(bo, + "chunked read err")); } while (u == 1 && buf[0] == '0' && buf[u] == '0'); if (!vct_ishex(buf[u])) break; @@ -318,7 +325,7 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc) /* Skip trailing white space */ while(vct_islws(buf[u]) && buf[u] != '\n') if (HTC_Read(htc, buf + u, 1) <= 0) - return (-1); + return (FetchError(bo, "chunked read err")); if (buf[u] != '\n') return (FetchError(bo,"chunked header no NL")); @@ -329,13 +336,13 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc) return (FetchError(bo,"chunked header number syntax")); if (cl > 0 && VFP_Bytes(bo, htc, cl) <= 0) - return (-1); + return (FetchError(bo, "chunked read err")); i = HTC_Read(htc, buf, 1); if (i <= 0) - return (-1); + return (FetchError(bo, "chunked read err")); if (buf[0] == '\r' && HTC_Read( htc, buf, 1) <= 0) - return (-1); + return (FetchError(bo, "chunked read err")); if (buf[0] != '\n') return (FetchError(bo,"chunked tail no NL")); } while (cl > 0); @@ -352,7 +359,7 @@ fetch_eof(struct busyobj *bo, struct http_conn *htc) assert(bo->body_status == BS_EOF); i = VFP_Bytes(bo, htc, SSIZE_MAX); if (i < 0) - return (-1); + return (FetchError(bo,"eof socket fail")); return (0); } @@ -545,6 +552,9 @@ FetchBody(struct worker *wrk, struct busyobj *bo) CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC); + assert(bo->state == BOS_INVALID); + bo->state = BOS_FINISHED; + /* * XXX: The busyobj needs a dstat, but it is not obvious which one * XXX: it should be (own/borrowed). For now borrow the wrk's. @@ -562,7 +572,7 @@ FetchBody(struct worker *wrk, struct busyobj *bo) AZ(bo->vgz_rx); AZ(VTAILQ_FIRST(&obj->store)); - bo->fetch_failed = 0; + bo->state = BOS_FETCHING; /* XXX: pick up estimate from objdr ? */ cl = 0; @@ -628,13 +638,14 @@ FetchBody(struct worker *wrk, struct busyobj *bo) } if (cls < 0) { + assert(bo->state == BOS_FAILED); wrk->stats.fetch_failed++; VDI_CloseFd(&bo->vbc); obj->len = 0; bo->stats = NULL; return (__LINE__); } - AZ(bo->fetch_failed); + assert(bo->state == BOS_FETCHING); if (cls == 0 && bo->should_close) cls = 1; @@ -661,6 +672,8 @@ FetchBody(struct worker *wrk, struct busyobj *bo) http_PrintfHeader(obj->http, "Content-Length: %zd", obj->len); } + bo->state = BOS_FINISHED; + if (cls) VDI_CloseFd(&bo->vbc); else diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index e2b4d51..343bd35 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -483,7 +483,7 @@ vfp_gunzip_end(struct busyobj *bo) vg = bo->vgz_rx; bo->vgz_rx = NULL; CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC); - if (bo->fetch_failed) { + if (bo->state == BOS_FAILED) { (void)VGZ_Destroy(&vg); return(0); } @@ -560,7 +560,7 @@ vfp_gzip_end(struct busyobj *bo) vg = bo->vgz_rx; CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC); bo->vgz_rx = NULL; - if (bo->fetch_failed) { + if (bo->state == BOS_FAILED) { (void)VGZ_Destroy(&vg); return(0); } @@ -652,7 +652,7 @@ vfp_testgzip_end(struct busyobj *bo) vg = bo->vgz_rx; bo->vgz_rx = NULL; CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC); - if (bo->fetch_failed) { + if (bo->state == BOS_FAILED) { (void)VGZ_Destroy(&vg); return(0); } From phk at FreeBSD.org Thu Dec 18 09:27:40 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] 071d905 Eliminate the returnvalue from FetchBody(), rely on bo->state instead Message-ID: commit 071d9050d5285bc8caa511c7664f00051f844496 Author: Poul-Henning Kamp Date: Mon Mar 5 14:19:10 2012 +0000 Eliminate the returnvalue from FetchBody(), rely on bo->state instead diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index bea32a2..63ea682 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -755,7 +755,7 @@ struct storage *FetchStorage(struct busyobj *, ssize_t sz); int FetchError(struct busyobj *, const char *error); int FetchError2(struct busyobj *, const char *error, const char *more); int FetchHdr(struct sess *sp, int need_host_hdr, int sendbody); -int FetchBody(struct worker *w, struct busyobj *bo); +void FetchBody(struct worker *w, struct busyobj *bo); int FetchReqBody(const struct sess *sp, int sendbody); void Fetch_Init(void); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 9cbc905..d13fc46 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -895,7 +895,6 @@ DOT fetchbody:out -> prepresp [style=bold,color=blue] static int cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) { - int i; struct busyobj *bo; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); @@ -904,8 +903,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) bo = req->busyobj; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - /* Use unmodified headers*/ - i = FetchBody(wrk, bo); + FetchBody(wrk, bo); bo->fetch_obj = NULL; http_Teardown(bo->bereq); @@ -915,7 +913,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) AZ(bo->vbc); AN(req->director); - if (i) { + if (bo->state == BOS_FAILED) { HSH_Drop(wrk, &sp->req->obj); VBO_DerefBusyObj(wrk, &req->busyobj); AZ(req->obj); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index fd70d07..4fc0f8e 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -76,21 +76,20 @@ FetchError(struct busyobj *bo, const char *error) * VFP method functions */ -static int +static void VFP_Begin(struct busyobj *bo, size_t estimate) { + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AN(bo->vfp); bo->vfp->begin(bo, estimate); - if (bo->state == BOS_FAILED) - return (-1); - return (0); } static int VFP_Bytes(struct busyobj *bo, struct http_conn *htc, ssize_t sz) { + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AN(bo->vfp); CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); @@ -99,7 +98,7 @@ VFP_Bytes(struct busyobj *bo, struct http_conn *htc, ssize_t sz) return (bo->vfp->bytes(bo, htc, sz)); } -static int +static void VFP_End(struct busyobj *bo) { int i; @@ -110,8 +109,6 @@ VFP_End(struct busyobj *bo) i = bo->vfp->end(bo); if (i) assert(bo->state == BOS_FAILED); - return (i); - } @@ -351,16 +348,13 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc) /*--------------------------------------------------------------------*/ -static int +static void fetch_eof(struct busyobj *bo, struct http_conn *htc) { - int i; assert(bo->body_status == BS_EOF); - i = VFP_Bytes(bo, htc, SSIZE_MAX); - if (i < 0) - return (FetchError(bo,"eof socket fail")); - return (0); + if (VFP_Bytes(bo, htc, SSIZE_MAX) < 0) + (void)FetchError(bo,"eof socket fail"); } /*-------------------------------------------------------------------- @@ -535,7 +529,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) /*--------------------------------------------------------------------*/ -int +void FetchBody(struct worker *wrk, struct busyobj *bo) { int cls; @@ -576,42 +570,39 @@ FetchBody(struct worker *wrk, struct busyobj *bo) /* XXX: pick up estimate from objdr ? */ cl = 0; + cls = 0; switch (bo->body_status) { case BS_NONE: - cls = 0; mklen = 0; break; case BS_ZERO: - cls = 0; mklen = 1; break; case BS_LENGTH: cl = fetch_number(bo->h_content_length, 10); - cls = VFP_Begin(bo, cl > 0 ? cl : 0); - if (!cls) + VFP_Begin(bo, cl > 0 ? cl : 0); + if (bo->state == BOS_FETCHING) cls = fetch_straight(bo, htc, cl); mklen = 1; - if (VFP_End(bo)) - cls = -1; + VFP_End(bo); break; case BS_CHUNKED: - cls = VFP_Begin(bo, cl); - if (!cls) + VFP_Begin(bo, cl); + if (bo->state == BOS_FETCHING) cls = fetch_chunked(bo, htc); mklen = 1; - if (VFP_End(bo)) - cls = -1; + VFP_End(bo); break; case BS_EOF: - cls = VFP_Begin(bo, cl); - if (!cls) - cls = fetch_eof(bo, htc); + VFP_Begin(bo, cl); + if (bo->state == BOS_FETCHING) + fetch_eof(bo, htc); mklen = 1; - if (VFP_End(bo)) - cls = -1; + cls = 1; + VFP_End(bo); break; case BS_ERROR: - cls = 1; + cls = FetchError(bo, "error incompatible Transfer-Encoding"); mklen = 0; break; default: @@ -631,20 +622,14 @@ FetchBody(struct worker *wrk, struct busyobj *bo) bo->body_status, body_status(bo->body_status), cls, mklen); - if (bo->body_status == BS_ERROR) { - VDI_CloseFd(&bo->vbc); - bo->stats = NULL; - return (__LINE__); - } - - if (cls < 0) { - assert(bo->state == BOS_FAILED); + if (bo->state == BOS_FAILED) { wrk->stats.fetch_failed++; VDI_CloseFd(&bo->vbc); obj->len = 0; bo->stats = NULL; - return (__LINE__); + return; } + assert(bo->state == BOS_FETCHING); if (cls == 0 && bo->should_close) @@ -680,7 +665,6 @@ FetchBody(struct worker *wrk, struct busyobj *bo) VDI_RecycleFd(&bo->vbc); bo->stats = NULL; - return (0); } /*-------------------------------------------------------------------- From phk at FreeBSD.org Thu Dec 18 09:27:40 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] 8f2d2e7 Make FetchBody() pool-task compatible. Message-ID: commit 8f2d2e7bce9bf5a614a42a296b0b49d67ec4be25 Author: Poul-Henning Kamp Date: Mon Mar 5 14:37:54 2012 +0000 Make FetchBody() pool-task compatible. Add #ifdef 0'ed code to put another thread on body-fetching Hugely inpired by patches from: martin diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 63ea682..2bae6b4 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -479,6 +479,8 @@ struct busyobj { struct http_conn htc; enum body_status body_status; + struct pool_task task; + struct vef_priv *vef_priv; unsigned should_close; @@ -755,7 +757,7 @@ struct storage *FetchStorage(struct busyobj *, ssize_t sz); int FetchError(struct busyobj *, const char *error); int FetchError2(struct busyobj *, const char *error, const char *more); int FetchHdr(struct sess *sp, int need_host_hdr, int sendbody); -void FetchBody(struct worker *w, struct busyobj *bo); +void FetchBody(struct worker *w, void *bo); int FetchReqBody(const struct sess *sp, int sendbody); void Fetch_Init(void); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index d13fc46..ecea4b5 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -903,8 +903,19 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) bo = req->busyobj; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); +#if 1 FetchBody(wrk, bo); - bo->fetch_obj = NULL; +#else + bo->task.func = FetchBody; + bo->task.priv = bo; + if (Pool_Task(wrk->pool, &bo->task, POOL_NO_QUEUE)) { + FetchBody(wrk, bo); + } else { + while (bo->state < BOS_FAILED) + (void)usleep(10000); + } +#endif + assert(bo->state >= BOS_FAILED); http_Teardown(bo->bereq); http_Teardown(bo->beresp); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 4fc0f8e..88bca02 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -530,7 +530,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) /*--------------------------------------------------------------------*/ void -FetchBody(struct worker *wrk, struct busyobj *bo) +FetchBody(struct worker *wrk, void *priv) { int cls; struct storage *st; @@ -538,9 +538,10 @@ FetchBody(struct worker *wrk, struct busyobj *bo) ssize_t cl; struct http_conn *htc; struct object *obj; + struct busyobj *bo; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->vbc, VBC_MAGIC); obj = bo->fetch_obj; CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC); From tfheen at err.no Thu Dec 18 09:27:40 2014 From: tfheen at err.no (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] af587ef Grammar in varnishncsa man page Message-ID: commit af587effb70d2dfbffd45bb54abe7dca2bf08293 Author: Tollef Fog Heen Date: Wed Mar 7 14:18:24 2012 +0100 Grammar in varnishncsa man page diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst index 9373f5c..2937b26 100644 --- a/doc/sphinx/reference/varnishncsa.rst +++ b/doc/sphinx/reference/varnishncsa.rst @@ -64,7 +64,7 @@ The following options are available: Defaults to 127.0.0.1 for backend requests. %{X}i - The contents of request header line X. + The contents of request header X. %l Remote logname (always '-') @@ -77,7 +77,7 @@ The following options are available: empty string. %{X}o - The contents of response header line X. + The contents of response header X. %r The first line of the request. Synthesized from other From lasse at varnish-software.com Thu Dec 18 09:27:40 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] 4fe672c Add section on device detection Message-ID: commit 4fe672ca2b418043a965ab28049e1256a5011dbe Author: Lasse Karstensen Date: Wed Mar 7 14:35:41 2012 +0100 Add section on device detection diff --git a/doc/sphinx/tutorial/devicedetection.rst b/doc/sphinx/tutorial/devicedetection.rst new file mode 100644 index 0000000..a083e6e --- /dev/null +++ b/doc/sphinx/tutorial/devicedetection.rst @@ -0,0 +1,260 @@ +.. _tutorial-devicedetect: + +Device detection +~~~~~~~~~~~~~~~~ + +Device detection is figuring out what kind of content to serve to a +client based on the User-Agent string supplied in a request. + +Use cases for this are for example to send size reduced files to mobile +clients with small screens and on high latency networks, or to +provide a streaming video codec that the client understands. + +There are a couple of strategies on what to do with such clients: +1) Redirect them to another URL. +2) Use a different backend for the special clients. +3) Change the backend requests so the usual backend sends tailored content. + +To make the examples easier to understand, it is assumed in this text +that all the req.http.X-UA-Device header is present and unique per client class +that content is to be served to. + +Setting this header can be as simple as:: + + sub vcl_recv {? + if (req.http.User-Agent ~ "(?i)iphone"?{ + set req.http.X-UA-Device = "mobile-iphone"; + } + } + +There are different commercial and free offerings in doing grouping and identifiying clients +in further detail than this. + + +Serve the different content on the same URL +------------------------------------------- + +The tricks involved are: +1. Detect the client (pretty simple, just include devicedetect.vcl and call +it) +2. Figure out how to signal the backend what client class this is. This +includes for example setting a header, changing a header or even changing the +backend request URL. +3. Modify any response from the backend to add missing Vary headers, so +Varnish' internal handling of this kicks in. +4. Modify output sent to the client so any caches outside our control don't +serve the wrong content. + +All this while still making sure that we only get 1 cache object per URL per +device class. + + +Example 1: Send HTTP header to backend +'''''''''''''''''''''''''''''''''''''' + +The basic case is that Varnish add the X-UA-Device HTTP header on the backend +requests, and the backend mentions in the response Vary header that the content +is dependant on this header. + +Everything works out of the box from Varnish' perspective. + +.. 071-example1-start +VCL:: + + sub vcl_recv { + # call some detection engine that set req.http.X-UA-Device + } + + sub append_ua_device { + if (req.http.X-UA-Device) { + set bereq.http.X-UA-Device = req.http.X-UA-Device; } + } + + # This must be done in vcl_miss and vcl_pass, before any backend request is + # actually sent. vcl_fetch runs after the request to the backend has + # completed. + sub vcl_miss { call append_ua_device; } + sub vcl_pass { call append_ua_device; } + + # so, this is a bit conterintuitive. The backend creates content based on + # the normalized User-Agent, but we use Vary on X-UA-Device so Varnish will + # use the same cached object for all U-As that map to the same X-UA-Device. + # + # If the backend does not mention in Vary that it has crafted special + # content based on the User-Agent (==X-UA-Device), add it. + # If your backend does set Vary: User-Agent, you may have to remove that here. + sub vcl_fetch { + if (req.http.X-UA-Device) { + if (!beresp.http.Vary) { # no Vary at all + set beresp.http.Vary = "X-UA-Device"; + } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary + set beresp.http.Vary = beresp.http.Vary + ", X-UA-Device"; + } + } + # comment this out if you don't want the client to know your + # classification + set beresp.http.X-UA-Device = req.http.X-UA-Device; + } + + # to keep any caches in the wild from serving wrong content to client #2 + # behind them, we need to transform the Vary on the way out. + sub vcl_deliver { + if ((req.http.X-UA-Device) && (resp.http.Vary)) { + set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent"); + } + } +.. 071-example1-end + +Example 2: Normalize the User-Agent string +'''''''''''''''''''''''''''''''''''''''''' + +Another way of signaling the device type is to override or normalize the +User-Agent header sent to the backend. + +For example + + User-Agent: Mozilla/5.0 (Linux; U; Android 2.2; nb-no; HTC Desire Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1 + +becomes: + + User-Agent: mobile-android + +when seen by the backend. + +This works if you don't need the original header for anything on the backend. +A possible use for this is for CGI scripts where only a small set of predefined +headers are (by default) available for the script. + +.. 072-example2-start +VCL:: + + sub vcl_recv { + # call some detection engine that set req.http.X-UA-Device + } + + # override the header before it is sent to the backend + sub vcl_miss { if (req.http.X-UA-Device) { set bereq.http.User-Agent = req.http.X-UA-Device; } } + sub vcl_pass { if (req.http.X-UA-Device) { set bereq.http.User-Agent = req.http.X-UA-Device; } } + + # standard Vary handling code from previous examples. + sub vcl_fetch { + if (req.http.X-UA-Device) { + if (!beresp.http.Vary) { # no Vary at all + set beresp.http.Vary = "X-UA-Device"; + } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary + set beresp.http.Vary = beresp.http.Vary + ", X-UA-Device"; + } + } + set beresp.http.X-UA-Device = req.http.X-UA-Device; + } + sub vcl_deliver { + if ((req.http.X-UA-Device) && (resp.http.Vary)) { + set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent"); + } + } + +.. 072-example2-end + +Example 3: Add the device class as a GET query parameter +'''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +If everything else fails, you can add the device type as a GET argument. + + http://example.com/article/1234.html --> http://example.com/article/1234.html?devicetype=mobile-iphone + +The client itself does not see this classification, only the backend request +is changed. + +.. 073-example3-start +VCL:: + + sub vcl_recv { + # call some detection engine that set req.http.X-UA-Device + + if ((req.http.X-UA-Device) && (req.request == "GET")) { + # if there are existing GET arguments; + if (req.url ~ "\?") { + set req.http.X-get-devicetype = "&devicetype=" + req.http.X-UA-Device; + } else { + set req.http.X-get-devicetype = "?devicetype=" + req.http.X-UA-Device; + } + set req.url = req.url + req.http.X-get-devicetype; + unset req.http.X-get-devicetype; + } + } + + # Handle redirects, otherwise standard Vary handling code from previous + # examples. + sub vcl_fetch { + if (req.http.X-UA-Device) { + if (!beresp.http.Vary) { # no Vary at all + set beresp.http.Vary = "X-UA-Device"; + } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary + set beresp.http.Vary = beresp.http.Vary + ", X-UA-Device"; + } + + # if the backend returns a redirect (think missing trailing slash), + # we will potentially show the extra address to the client. we + # don't want that. if the backend reorders the get parameters, you + # may need to be smarter here. (? and & ordering) + + if (beresp.status == 301 || beresp.status == 302 || beresp.status == 303) { + set beresp.http.location = regsub(beresp.http.location, "[?&]devicetype=.*$", ""); + } + } + set beresp.http.X-UA-Device = req.http.X-UA-Device; + } + sub vcl_deliver { + if ((req.http.X-UA-Device) && (resp.http.Vary)) { + set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent"); + } + } + +.. 073-example3-end + +Different backend for mobile clients +------------------------------------ + +If you have a different backend that serves pages for mobile clients, or any +special needs in VCL, you can use the X-UA-Device header like this:: + + backend mobile { + .host = "10.0.0.1"; + .port = "80"; + } + + sub vcl_recv { + # call some detection engine + + if (req.http.X-UA-Device ~ "^mobile" || req.http.X-UA-device ~ "^tablet") { + set req.backend = mobile; + } + } + +Redirecting mobile clients +-------------------------- + +If you want to redirect mobile clients you can use the following snippet. + +.. 065-redir-mobile-start +VCL:: + + sub vcl_recv { + # call some detection engine + + if (req.http.X-UA-Device ~ "^mobile" || req.http.X-UA-device ~ "^tablet") { + error 750 "Moved Temporarily"; + } + } + + sub vcl_error { + if (obj.status == 750) { + set obj.http.Location = "http://m.example.com" + req.url; + set obj.status = 302; + return(deliver); + } + } + +.. 065-redir-mobile-end + + diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst index 4949aa0..91fdb17 100644 --- a/doc/sphinx/tutorial/index.rst +++ b/doc/sphinx/tutorial/index.rst @@ -30,6 +30,7 @@ separate topic. Good luck. esi virtualized websockets + devicedetection advanced_backend_servers handling_misbehaving_servers advanced_topics From tfheen at err.no Thu Dec 18 09:27:40 2014 From: tfheen at err.no (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] fe828b4 Fixing a typo in the documentation. Message-ID: commit fe828b4cfd698c99be3b4e09dc2acc21a88caaa3 Author: Derek Hammer Date: Thu Mar 8 03:11:26 2012 -0500 Fixing a typo in the documentation. diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst index ecf0e96..846ca82 100644 --- a/doc/sphinx/tutorial/purging.rst +++ b/doc/sphinx/tutorial/purging.rst @@ -4,7 +4,7 @@ Purging and banning ===================== -One of the most effective way of increasing your hit ratio is to +One of the most effective ways of increasing your hit ratio is to increase the time-to-live (ttl) of your objects. But, as you're aware of, in this twitterific day of age serving content that is outdated is bad for business. From lasse at varnish-software.com Thu Dec 18 09:27:40 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] 72cb9f6 typos, change example3 to allow for easier purging Message-ID: commit 72cb9f69167eff08e6498db0ce732eac0e581b20 Author: Lasse Karstensen Date: Thu Mar 8 14:26:27 2012 +0100 typos, change example3 to allow for easier purging diff --git a/doc/sphinx/tutorial/devicedetection.rst b/doc/sphinx/tutorial/devicedetection.rst index a083e6e..abf729e 100644 --- a/doc/sphinx/tutorial/devicedetection.rst +++ b/doc/sphinx/tutorial/devicedetection.rst @@ -27,8 +27,10 @@ Setting this header can be as simple as:: } } -There are different commercial and free offerings in doing grouping and identifiying clients -in further detail than this. +There are different commercial and free offerings in doing grouping and +identifiying clients in further detail than this. For a basic and community +based regular expression set, see +https://github.com/varnish/varnish-devicedetect/ . Serve the different content on the same URL @@ -45,14 +47,14 @@ Varnish' internal handling of this kicks in. 4. Modify output sent to the client so any caches outside our control don't serve the wrong content. -All this while still making sure that we only get 1 cache object per URL per +All this while still making sure that we only get 1 cached object per URL per device class. Example 1: Send HTTP header to backend '''''''''''''''''''''''''''''''''''''' -The basic case is that Varnish add the X-UA-Device HTTP header on the backend +The basic case is that Varnish adds the X-UA-Device HTTP header on the backend requests, and the backend mentions in the response Vary header that the content is dependant on this header. @@ -64,17 +66,7 @@ VCL:: sub vcl_recv { # call some detection engine that set req.http.X-UA-Device } - - sub append_ua_device { - if (req.http.X-UA-Device) { - set bereq.http.X-UA-Device = req.http.X-UA-Device; } - } - - # This must be done in vcl_miss and vcl_pass, before any backend request is - # actually sent. vcl_fetch runs after the request to the backend has - # completed. - sub vcl_miss { call append_ua_device; } - sub vcl_pass { call append_ua_device; } + # req.http.X-UA-Device is copied by Varnish into bereq.http.X-UA-Device # so, this is a bit conterintuitive. The backend creates content based on # the normalized User-Agent, but we use Vary on X-UA-Device so Varnish will @@ -170,7 +162,9 @@ VCL:: sub vcl_recv { # call some detection engine that set req.http.X-UA-Device + } + sub append_ua { if ((req.http.X-UA-Device) && (req.request == "GET")) { # if there are existing GET arguments; if (req.url ~ "\?") { @@ -183,6 +177,10 @@ VCL:: } } + # do this after vcl_hash, so all Vary-ants can be purged in one go. (avoid ban()ing) + sub vcl_miss { call append_ua; } + sub vcl_pass { call append_ua; } + # Handle redirects, otherwise standard Vary handling code from previous # examples. sub vcl_fetch { From lasse at varnish-software.com Thu Dec 18 09:27:40 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] 4b6e84e Merge branch 'master' of /home/lkarsten/work/varnish-cache-fork Message-ID: commit 4b6e84e491f2719436b4a8fffa100fe13f229668 Merge: fe828b4 72cb9f6 Author: Lasse Karstensen Date: Thu Mar 8 14:30:40 2012 +0100 Merge branch 'master' of /home/lkarsten/work/varnish-cache-fork From phk at FreeBSD.org Thu Dec 18 09:27:40 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] e725920 Remove old Date: header before adding our new one. Message-ID: commit e725920a03e5329949635ced22d02f52ab91a5b9 Author: Poul-Henning Kamp Date: Fri Mar 9 09:58:41 2012 +0000 Remove old Date: header before adding our new one. Submitted by: scoof Fixes #1104 diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index 4760c12..1e065ba 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -128,6 +128,7 @@ RES_BuildHttp(const struct sess *sp) if (req->res_mode & RES_CHUNKED) http_SetHeader(req->resp, "Transfer-Encoding: chunked"); + http_Unset(req->resp, H_Date); VTIM_format(VTIM_real(), time_str); http_PrintfHeader(req->resp, "Date: %s", time_str); From phk at FreeBSD.org Thu Dec 18 09:27:40 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] 59c6ae4 Join struct vbo and struct busyobj Message-ID: commit 59c6ae45f801a0f447d9f9647f2963950083c696 Author: Poul-Henning Kamp Date: Mon Mar 12 08:57:38 2012 +0000 Join struct vbo and struct busyobj diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 2bae6b4..48bdac0 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -108,7 +108,6 @@ struct poolparam; struct sess; struct sesspool; struct vbc; -struct vbo; struct vef_priv; struct vrt_backend; struct vsb; @@ -298,7 +297,7 @@ struct worker { struct objhead *nobjhead; struct objcore *nobjcore; struct waitinglist *nwaitinglist; - struct vbo *nvbo; + struct busyobj *nbo; void *nhashpriv; struct dstat stats; @@ -459,7 +458,14 @@ enum busyobj_state_e { struct busyobj { unsigned magic; #define BUSYOBJ_MAGIC 0x23b95567 - struct vbo *vbo; + struct lock mtx; + char *end; + + /* + * All fields from refcount and down are zeroed when the busyobj + * is recycled. + */ + unsigned refcount; uint8_t *vary; unsigned is_gzip; @@ -479,7 +485,7 @@ struct busyobj { struct http_conn htc; enum body_status body_status; - struct pool_task task; + struct pool_task fetch_task; struct vef_priv *vef_priv; @@ -721,7 +727,7 @@ void VBO_Init(void); struct busyobj *VBO_GetBusyObj(struct worker *wrk); void VBO_RefBusyObj(const struct busyobj *busyobj); void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj); -void VBO_Free(struct vbo **vbo); +void VBO_Free(struct busyobj **vbo); /* cache_center.c [CNT] */ void CNT_Session(struct sess *sp); diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 32ae7c8..601272f 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -40,15 +40,6 @@ static struct mempool *vbopool; -struct vbo { - unsigned magic; -#define VBO_MAGIC 0xde3d8223 - struct lock mtx; - unsigned refcount; - char *end; - struct busyobj bo; -}; - /*-------------------------------------------------------------------- */ @@ -56,7 +47,7 @@ void VBO_Init(void) { - vbopool = MPL_New("vbo", &cache_param->vbo_pool, + vbopool = MPL_New("busyobj", &cache_param->vbo_pool, &cache_param->workspace_backend); AN(vbopool); } @@ -65,88 +56,86 @@ VBO_Init(void) * BusyObj handling */ -static struct vbo * +static struct busyobj * vbo_New(void) { - struct vbo *vbo; + struct busyobj *bo; unsigned sz; - vbo = MPL_Get(vbopool, &sz); - AN(vbo); - vbo->magic = VBO_MAGIC; - vbo->end = (char *)vbo + sz; - Lck_New(&vbo->mtx, lck_busyobj); - return (vbo); + bo = MPL_Get(vbopool, &sz); + XXXAN(bo); + bo->magic = BUSYOBJ_MAGIC; + bo->end = (char *)bo + sz; + Lck_New(&bo->mtx, lck_busyobj); + return (bo); } void -VBO_Free(struct vbo **vbop) +VBO_Free(struct busyobj **bop) { - struct vbo *vbo; + struct busyobj *bo; - AN(vbop); - vbo = *vbop; - *vbop = NULL; - CHECK_OBJ_NOTNULL(vbo, VBO_MAGIC); - AZ(vbo->refcount); - Lck_Delete(&vbo->mtx); - MPL_Free(vbopool, vbo); + AN(bop); + bo = *bop; + *bop = NULL; + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + AZ(bo->refcount); + Lck_Delete(&bo->mtx); + MPL_Free(vbopool, bo); } struct busyobj * VBO_GetBusyObj(struct worker *wrk) { - struct vbo *vbo = NULL; + struct busyobj *bo = NULL; uint16_t nhttp; unsigned sz; char *p; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - if (wrk->nvbo != NULL) { - vbo = wrk->nvbo; - wrk->nvbo = NULL; + if (wrk->nbo != NULL) { + bo = wrk->nbo; + wrk->nbo = NULL; } - if (vbo == NULL) - vbo = vbo_New(); + if (bo == NULL) + bo = vbo_New(); - CHECK_OBJ_NOTNULL(vbo, VBO_MAGIC); - AZ(vbo->refcount); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + AZ(bo->refcount); - AZ(vbo->bo.magic); - vbo->refcount = 1; - vbo->bo.magic = BUSYOBJ_MAGIC; - vbo->bo.vbo = vbo; + bo->refcount = 1; - p = (void*)(vbo + 1); + p = (void*)(bo + 1); p = (void*)PRNDUP(p); - assert(p < vbo->end); + assert(p < bo->end); nhttp = (uint16_t)cache_param->http_max_hdr; sz = HTTP_estimate(nhttp); - vbo->bo.bereq = HTTP_create(p, nhttp); + bo->bereq = HTTP_create(p, nhttp); p += sz; p = (void*)PRNDUP(p); - assert(p < vbo->end); + assert(p < bo->end); - vbo->bo.beresp = HTTP_create(p, nhttp); + bo->beresp = HTTP_create(p, nhttp); p += sz; p = (void*)PRNDUP(p); - assert(p < vbo->end); + assert(p < bo->end); sz = cache_param->vsl_buffer; - VSL_Setup(vbo->bo.vsl, p, sz); + VSL_Setup(bo->vsl, p, sz); p += sz; p = (void*)PRNDUP(p); - assert(p < vbo->end); + assert(p < bo->end); - WS_Init(vbo->bo.ws, "bo", p, vbo->end - p); + WS_Init(bo->ws, "bo", p, bo->end - p); - return (&vbo->bo); + return (bo); } +#if 0 void VBO_RefBusyObj(const struct busyobj *busyobj) { @@ -160,12 +149,12 @@ VBO_RefBusyObj(const struct busyobj *busyobj) vbo->refcount++; Lck_Unlock(&vbo->mtx); } +#endif void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo) { struct busyobj *bo; - struct vbo *vbo; unsigned r; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -173,22 +162,21 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo) bo = *pbo; *pbo = NULL; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - vbo = bo->vbo; - CHECK_OBJ_NOTNULL(vbo, VBO_MAGIC); - Lck_Lock(&vbo->mtx); - assert(vbo->refcount > 0); - r = --vbo->refcount; - Lck_Unlock(&vbo->mtx); + Lck_Lock(&bo->mtx); + assert(bo->refcount > 0); + r = --bo->refcount; + Lck_Unlock(&bo->mtx); if (r) return; - VSL_Flush(vbo->bo.vsl, 0); - /* XXX: Sanity checks & cleanup */ - memset(&vbo->bo, 0, sizeof vbo->bo); + VSL_Flush(bo->vsl, 0); + + memset(&bo->refcount, 0, + sizeof *bo - offsetof(struct busyobj, refcount)); - if (cache_param->bo_cache && wrk->nvbo == NULL) - wrk->nvbo = vbo; + if (cache_param->bo_cache && wrk->nbo == NULL) + wrk->nbo = bo; else - VBO_Free(&vbo); + VBO_Free(&bo); } diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index ecea4b5..1ae5dd7 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -906,9 +906,9 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) #if 1 FetchBody(wrk, bo); #else - bo->task.func = FetchBody; - bo->task.priv = bo; - if (Pool_Task(wrk->pool, &bo->task, POOL_NO_QUEUE)) { + bo->fetch_task.func = FetchBody; + bo->fetch_task.priv = bo; + if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE)) { FetchBody(wrk, bo); } else { while (bo->state < BOS_FAILED) diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 1e21214..951cef9 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -131,8 +131,6 @@ HSH_Cleanup(struct worker *wrk) free(wrk->nhashpriv); wrk->nhashpriv = NULL; } - if (wrk->nvbo != NULL) - VBO_Free(&wrk->nvbo); } void diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index cc3a061..7b65479 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -147,8 +147,8 @@ wrk_thread_real(void *priv, unsigned thread_workspace) if (w->vcl != NULL) VCL_Rel(&w->vcl); AZ(pthread_cond_destroy(&w->cond)); - if (w->nvbo != NULL) - VBO_Free(&w->nvbo); + if (w->nbo != NULL) + VBO_Free(&w->nbo); HSH_Cleanup(w); WRK_SumStat(w); return (NULL); From phk at FreeBSD.org Thu Dec 18 09:27:40 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] d1d7134 Always try to set another thread on the body-fetch job. Message-ID: commit d1d71344524068ad56d170bd29feb719d9a9b902 Author: Poul-Henning Kamp Date: Mon Mar 12 09:31:40 2012 +0000 Always try to set another thread on the body-fetch job. Slight rearrangement of FetchBody() diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 1ae5dd7..7df47bb 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -903,18 +903,13 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) bo = req->busyobj; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); -#if 1 - FetchBody(wrk, bo); -#else bo->fetch_task.func = FetchBody; bo->fetch_task.priv = bo; - if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE)) { + if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE)) FetchBody(wrk, bo); - } else { - while (bo->state < BOS_FAILED) - (void)usleep(10000); - } -#endif + + while (bo->state < BOS_FAILED) + (void)usleep(10000); assert(bo->state >= BOS_FAILED); http_Teardown(bo->bereq); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 88bca02..e3bf6dc 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -627,44 +627,43 @@ FetchBody(struct worker *wrk, void *priv) wrk->stats.fetch_failed++; VDI_CloseFd(&bo->vbc); obj->len = 0; - bo->stats = NULL; - return; - } - - assert(bo->state == BOS_FETCHING); + } else { + assert(bo->state == BOS_FETCHING); - if (cls == 0 && bo->should_close) - cls = 1; + if (cls == 0 && bo->should_close) + cls = 1; - VSLb(bo->vsl, SLT_Length, "%zd", obj->len); + VSLb(bo->vsl, SLT_Length, "%zd", obj->len); - { - /* Sanity check fetch methods accounting */ - ssize_t uu; + { + /* Sanity check fetch methods accounting */ + ssize_t uu; - uu = 0; - VTAILQ_FOREACH(st, &obj->store, list) - uu += st->len; - if (bo->do_stream) - /* Streaming might have started freeing stuff */ - assert (uu <= obj->len); + uu = 0; + VTAILQ_FOREACH(st, &obj->store, list) + uu += st->len; + if (bo->do_stream) + /* Streaming might have started freeing stuff */ + assert (uu <= obj->len); - else - assert(uu == obj->len); - } + else + assert(uu == obj->len); + } - if (mklen > 0) { - http_Unset(obj->http, H_Content_Length); - http_PrintfHeader(obj->http, "Content-Length: %zd", obj->len); - } + if (mklen > 0) { + http_Unset(obj->http, H_Content_Length); + http_PrintfHeader(obj->http, + "Content-Length: %zd", obj->len); + } - bo->state = BOS_FINISHED; + bo->state = BOS_FINISHED; - if (cls) - VDI_CloseFd(&bo->vbc); - else - VDI_RecycleFd(&bo->vbc); + if (cls) + VDI_CloseFd(&bo->vbc); + else + VDI_RecycleFd(&bo->vbc); + } bo->stats = NULL; } From apj at mutt.dk Thu Dec 18 09:27:40 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] c5e141b Correct function name Message-ID: commit c5e141be4c608fc7264d36506fd79c49e04a86a5 Author: Andreas Plesner Jacobsen Date: Mon Mar 12 12:23:47 2012 +0100 Correct function name diff --git a/doc/sphinx/reference/vmod_std.rst b/doc/sphinx/reference/vmod_std.rst index 9c28a3c..a1ece16 100644 --- a/doc/sphinx/reference/vmod_std.rst +++ b/doc/sphinx/reference/vmod_std.rst @@ -50,7 +50,7 @@ Description Example set beresp.http.x-nice = std.tolower("VerY"); -set_up_tos +set_ip_tos ---------- Prototype set_ip_tos(INT i) From apj at mutt.dk Thu Dec 18 09:27:40 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] f740e80 Small doc fixes Message-ID: commit f740e80841888b03cc5685dcfeb3974afe2c376c Author: Andreas Plesner Jacobsen Date: Mon Mar 12 12:25:41 2012 +0100 Small doc fixes diff --git a/doc/sphinx/tutorial/logging.rst b/doc/sphinx/tutorial/logging.rst index 68f1d87..1f0bc18 100644 --- a/doc/sphinx/tutorial/logging.rst +++ b/doc/sphinx/tutorial/logging.rst @@ -9,7 +9,7 @@ memory segment. When the end of the segment is reached we start over, overwriting old data. This is much, much faster then logging to a file and it doesn't require disk space. -The flip side is that if you forget to have program actually write the +The flip side is that if you forget to have a program actually write the logs to disk they will disappear. varnishlog is one of the programs you can use to look at what Varnish diff --git a/doc/sphinx/tutorial/vcl.rst b/doc/sphinx/tutorial/vcl.rst index 7171ab6..54bce37 100644 --- a/doc/sphinx/tutorial/vcl.rst +++ b/doc/sphinx/tutorial/vcl.rst @@ -135,7 +135,7 @@ down for, uhm, examples. Example 1 - manipulating headers ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Lets say we want to remove the cookie for all objects in the /static +Lets say we want to remove the cookie for all objects in the /images directory of our web server:: sub vcl_recv { From phk at FreeBSD.org Thu Dec 18 09:27:40 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:40 +0100 Subject: [experimental-ims] b8d4eb8 Hand FetchBody() its own busyobj->refcount, and have it release it when done. Message-ID: commit b8d4eb8a23d1971aaef3fe087d024b7f1dd78df9 Author: Poul-Henning Kamp Date: Mon Mar 12 12:27:34 2012 +0000 Hand FetchBody() its own busyobj->refcount, and have it release it when done. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 48bdac0..99c66c3 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -725,7 +725,7 @@ double BAN_Time(const struct ban *ban); /* cache_busyobj.c */ void VBO_Init(void); struct busyobj *VBO_GetBusyObj(struct worker *wrk); -void VBO_RefBusyObj(const struct busyobj *busyobj); +void VBO_RefBusyObj(struct busyobj *busyobj); void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj); void VBO_Free(struct busyobj **vbo); diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 601272f..8de928a 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -135,21 +135,16 @@ VBO_GetBusyObj(struct worker *wrk) return (bo); } -#if 0 void -VBO_RefBusyObj(const struct busyobj *busyobj) +VBO_RefBusyObj(struct busyobj *bo) { - struct vbo *vbo; - - CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC); - vbo = busyobj->vbo; - CHECK_OBJ_NOTNULL(vbo, VBO_MAGIC); - Lck_Lock(&vbo->mtx); - assert(vbo->refcount > 0); - vbo->refcount++; - Lck_Unlock(&vbo->mtx); + + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + Lck_Lock(&bo->mtx); + assert(bo->refcount > 0); + bo->refcount++; + Lck_Unlock(&bo->mtx); } -#endif void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo) @@ -157,7 +152,7 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo) struct busyobj *bo; unsigned r; - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC); AN(pbo); bo = *pbo; *pbo = NULL; @@ -175,7 +170,7 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo) memset(&bo->refcount, 0, sizeof *bo - offsetof(struct busyobj, refcount)); - if (cache_param->bo_cache && wrk->nbo == NULL) + if (cache_param->bo_cache && wrk != NULL && wrk->nbo == NULL) wrk->nbo = bo; else VBO_Free(&bo); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 7df47bb..aa07828 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -905,6 +905,10 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) bo->fetch_task.func = FetchBody; bo->fetch_task.priv = bo; + + /* Gain a reference for FetchBody() */ + VBO_RefBusyObj(bo); + if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE)) FetchBody(wrk, bo); @@ -912,8 +916,6 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) (void)usleep(10000); assert(bo->state >= BOS_FAILED); - http_Teardown(bo->bereq); - http_Teardown(bo->beresp); bo->vfp = NULL; assert(WRW_IsReleased(wrk)); AZ(bo->vbc); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index e3bf6dc..a52201a 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -298,7 +298,7 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc) do { /* Skip leading whitespace */ do { - if (HTC_Read(htc, buf, 1) <= 0) + if (HTC_Read(htc, buf, 1) <= 0) return (FetchError(bo, "chunked read err")); } while (vct_islws(buf[0])); @@ -527,7 +527,13 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) return (0); } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * This function is either called by the requesting thread OR by a + * dedicated body-fetch work-thread. + * + * We get passed the busyobj in the priv arg, and we inherit a + * refcount on it, which we must release, when done fetching. + */ void FetchBody(struct worker *wrk, void *priv) @@ -623,6 +629,9 @@ FetchBody(struct worker *wrk, void *priv) bo->body_status, body_status(bo->body_status), cls, mklen); + http_Teardown(bo->bereq); + http_Teardown(bo->beresp); + if (bo->state == BOS_FAILED) { wrk->stats.fetch_failed++; VDI_CloseFd(&bo->vbc); @@ -665,6 +674,7 @@ FetchBody(struct worker *wrk, void *priv) } bo->stats = NULL; + VBO_DerefBusyObj(NULL, &bo); } /*-------------------------------------------------------------------- From martin at varnish-software.com Thu Dec 18 09:27:41 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] b78280f Decrement the n_waitinglist counter when freeing waitinglists. Message-ID: commit b78280f445ed69af77b42154dff4e6d3a70e0b5f Author: Martin Blix Grydeland Date: Mon Mar 12 12:39:36 2012 +0100 Decrement the n_waitinglist counter when freeing waitinglists. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index aa07828..1c9d48b 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -935,7 +935,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) AN(req->obj->objcore); AN(req->obj->objcore->ban); AZ(req->obj->ws_o->overflow); - HSH_Unbusy(req->obj->objcore); + HSH_Unbusy(&wrk->stats, req->obj->objcore); } VBO_DerefBusyObj(wrk, &req->busyobj); wrk->acct_tmp.fetch++; diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 951cef9..6e1c27f 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -125,6 +125,7 @@ HSH_Cleanup(struct worker *wrk) if (wrk->nwaitinglist != NULL) { FREE_OBJ(wrk->nwaitinglist); wrk->nwaitinglist = NULL; + wrk->stats.n_waitinglist--; } if (wrk->nhashpriv != NULL) { /* XXX: If needed, add slinger method for this */ @@ -468,7 +469,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) */ static void -hsh_rush(struct objhead *oh) +hsh_rush(struct dstat *ds, struct objhead *oh) { unsigned u; struct sess *sp; @@ -497,6 +498,7 @@ hsh_rush(struct objhead *oh) if (VTAILQ_EMPTY(&wl->list)) { oh->waitinglist = NULL; FREE_OBJ(wl); + ds->n_waitinglist--; } } @@ -576,12 +578,12 @@ HSH_Drop(struct worker *wrk, struct object **oo) AssertObjCorePassOrBusy((*oo)->objcore); (*oo)->exp.ttl = -1.; if ((*oo)->objcore != NULL) /* Pass has no objcore */ - HSH_Unbusy((*oo)->objcore); + HSH_Unbusy(&wrk->stats, (*oo)->objcore); (void)HSH_Deref(&wrk->stats, NULL, oo); } void -HSH_Unbusy(struct objcore *oc) +HSH_Unbusy(struct dstat *ds, struct objcore *oc) { struct objhead *oh; @@ -603,7 +605,7 @@ HSH_Unbusy(struct objcore *oc) oc->flags &= ~OC_F_BUSY; oc->busyobj = NULL; if (oh->waitinglist != NULL) - hsh_rush(oh); + hsh_rush(ds, oh); AN(oc->ban); Lck_Unlock(&oh->mtx); } @@ -680,7 +682,7 @@ HSH_Deref(struct dstat *ds, struct objcore *oc, struct object **oo) AN(oc->methods); } if (oh->waitinglist != NULL) - hsh_rush(oh); + hsh_rush(ds, oh); Lck_Unlock(&oh->mtx); if (r != 0) return (r); diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index b15d9df..a3577a7 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -53,7 +53,6 @@ struct hash_slinger { /* cache_hash.c */ void HSH_Cleanup(struct worker *w); struct objcore *HSH_Lookup(struct sess *sp, struct objhead **poh); -void HSH_Unbusy(struct objcore *); void HSH_Ref(struct objcore *o); void HSH_Drop(struct worker *, struct object **); void HSH_Init(const struct hash_slinger *slinger); @@ -94,6 +93,7 @@ struct objhead { #define hoh_head _u.n.u_n_hoh_head }; +void HSH_Unbusy(struct dstat *, struct objcore *); void HSH_DeleteObjHead(struct dstat *, struct objhead *oh); int HSH_Deref(struct dstat *, struct objcore *oc, struct object **o); #endif /* VARNISH_CACHE_CHILD */ From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] 40d62bd Have FetchBody reset bo->vfp once it's done with it. Message-ID: commit 40d62bd804931d5bc3a680768309529548c5d00e Author: Poul-Henning Kamp Date: Mon Mar 12 13:49:43 2012 +0000 Have FetchBody reset bo->vfp once it's done with it. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 1c9d48b..3e80d06 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -916,7 +916,6 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) (void)usleep(10000); assert(bo->state >= BOS_FAILED); - bo->vfp = NULL; assert(WRW_IsReleased(wrk)); AZ(bo->vbc); AN(req->director); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index a52201a..172f422 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -625,6 +625,8 @@ FetchBody(struct worker *wrk, void *priv) */ AZ(vfp_nop_end(bo)); + bo->vfp = NULL; + VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s) cls %d mklen %d", bo->body_status, body_status(bo->body_status), cls, mklen); @@ -674,7 +676,7 @@ FetchBody(struct worker *wrk, void *priv) } bo->stats = NULL; - VBO_DerefBusyObj(NULL, &bo); + VBO_DerefBusyObj(wrk, &bo); } /*-------------------------------------------------------------------- From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] 7b25530 These testcases depend on us not streaming, so they get a 503 for the failure to fetch a body. Make them explicitly disable streaming. Message-ID: commit 7b2553097060e4e01cbf42cbcc3951a28597bad9 Author: Poul-Henning Kamp Date: Mon Mar 12 14:57:49 2012 +0000 These testcases depend on us not streaming, so they get a 503 for the failure to fetch a body. Make them explicitly disable streaming. diff --git a/bin/varnishtest/tests/b00020.vtc b/bin/varnishtest/tests/b00020.vtc index d937ac0..365c305 100644 --- a/bin/varnishtest/tests/b00020.vtc +++ b/bin/varnishtest/tests/b00020.vtc @@ -9,7 +9,11 @@ server s1 { send "Baba\n" } -start -varnish v1 -vcl+backend {} -start +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.do_stream = false; + } +} -start varnish v1 -cliok "param.set between_bytes_timeout 1" client c1 { diff --git a/bin/varnishtest/tests/b00021.vtc b/bin/varnishtest/tests/b00021.vtc index d46caa2..5c3261a 100644 --- a/bin/varnishtest/tests/b00021.vtc +++ b/bin/varnishtest/tests/b00021.vtc @@ -13,6 +13,9 @@ varnish v1 -vcl+backend { sub vcl_miss { set bereq.between_bytes_timeout = 2s; } + sub vcl_fetch { + set beresp.do_stream = false; + } } -start client c1 { diff --git a/bin/varnishtest/tests/b00022.vtc b/bin/varnishtest/tests/b00022.vtc index aa625b6..722d9f1 100644 --- a/bin/varnishtest/tests/b00022.vtc +++ b/bin/varnishtest/tests/b00022.vtc @@ -15,6 +15,9 @@ varnish v1 -vcl { .port = "${s1_port}"; .between_bytes_timeout = 1s; } + sub vcl_fetch { + set beresp.do_stream = false; + } } -start client c1 { diff --git a/bin/varnishtest/tests/b00027.vtc b/bin/varnishtest/tests/b00027.vtc index 4076019..9cc7a7a 100644 --- a/bin/varnishtest/tests/b00027.vtc +++ b/bin/varnishtest/tests/b00027.vtc @@ -11,7 +11,11 @@ server s1 { send "\n" } -start -varnish v1 -vcl+backend {} -start +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.do_stream = false; + } +} -start client c1 { txreq -url /foo diff --git a/bin/varnishtest/tests/g00004.vtc b/bin/varnishtest/tests/g00004.vtc index 5d518e0..0ab74ad 100644 --- a/bin/varnishtest/tests/g00004.vtc +++ b/bin/varnishtest/tests/g00004.vtc @@ -21,6 +21,8 @@ varnish v1 \ -arg {-p diag_bitmap=0x00010000} \ -vcl+backend { sub vcl_fetch { + set beresp.do_stream = false; + if (req.url == "/gunzip") { set beresp.do_gunzip = true; } diff --git a/bin/varnishtest/tests/r00387.vtc b/bin/varnishtest/tests/r00387.vtc index fb73e6b..d4a32f9 100644 --- a/bin/varnishtest/tests/r00387.vtc +++ b/bin/varnishtest/tests/r00387.vtc @@ -11,7 +11,11 @@ server s1 { send "\r\n" } -start -varnish v1 -vcl+backend {} -start +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.do_stream = false; + } +} -start client c1 { txreq diff --git a/bin/varnishtest/tests/r00942.vtc b/bin/varnishtest/tests/r00942.vtc index 0df8eb2..6bb31e1 100644 --- a/bin/varnishtest/tests/r00942.vtc +++ b/bin/varnishtest/tests/r00942.vtc @@ -26,7 +26,11 @@ server s1 { varnish v1 \ -arg {-p diag_bitmap=0x00010000} \ - -vcl+backend {} + -vcl+backend { + sub vcl_fetch { + set beresp.do_stream = false; + } +} varnish v1 -start diff --git a/bin/varnishtest/tests/r01036.vtc b/bin/varnishtest/tests/r01036.vtc index 8fb7600..1c77919 100644 --- a/bin/varnishtest/tests/r01036.vtc +++ b/bin/varnishtest/tests/r01036.vtc @@ -9,6 +9,7 @@ server s1 { varnish v1 -arg "-smalloc,1M" -arg "-pgzip_level=0" -vcl+backend { sub vcl_fetch { + set beresp.do_stream = false; set beresp.do_gzip = true; } } -start From lasse at varnish-software.com Thu Dec 18 09:27:41 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] d96e763 Include device class when hashing Message-ID: commit d96e763c12f2d7229fd642e6e4900fb48fd37586 Author: Lasse Karstensen Date: Mon Mar 12 16:13:56 2012 +0100 Include device class when hashing diff --git a/doc/sphinx/tutorial/devicedetection.rst b/doc/sphinx/tutorial/devicedetection.rst index abf729e..9f02493 100644 --- a/doc/sphinx/tutorial/devicedetection.rst +++ b/doc/sphinx/tutorial/devicedetection.rst @@ -228,6 +228,11 @@ special needs in VCL, you can use the X-UA-Device header like this:: set req.backend = mobile; } } + sub vcl_hash { + if (req.http.X-UA-Device) { + hash_data(req.http.X-UA-Device); + } + } Redirecting mobile clients -------------------------- From lasse at varnish-software.com Thu Dec 18 09:27:41 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] 9831187 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit 9831187e28c682075897cd1b4295898e2f9c170d Merge: d96e763 7b25530 Author: Lasse Karstensen Date: Mon Mar 12 16:14:17 2012 +0100 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] 0326c20 Quick polish over HSH_Lookup(): Message-ID: commit 0326c20463a90e3750dbcc492244da1d0f03d9af Author: Poul-Henning Kamp Date: Mon Mar 12 15:41:07 2012 +0000 Quick polish over HSH_Lookup(): Now that we have busyobjs, we do not need to return the objhdr separately. Introduce a local req variable to reduce sp->req dereferences. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 3e80d06..4d18395 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -1104,9 +1104,8 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) req->vary_e = (void*)req->ws->r; req->vary_b[2] = '\0'; - oc = HSH_Lookup(sp, &oh); AZ(req->objcore); - + oc = HSH_Lookup(sp); if (oc == NULL) { /* * We lost the session to a busy object, disembark the @@ -1117,8 +1116,10 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) */ return (1); } + AZ(req->objcore); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + oh = oc->objhead; CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); /* If we inserted a new object it's a miss */ diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 6e1c27f..55215e6 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -282,36 +282,38 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc) */ struct objcore * -HSH_Lookup(struct sess *sp, struct objhead **poh) +HSH_Lookup(struct sess *sp) { struct worker *wrk; struct objhead *oh; struct objcore *oc; struct objcore *busy_oc, *grace_oc; struct object *o; + struct req *req; double grace_ttl; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(sp->req->http, HTTP_MAGIC); - AN(sp->req->director); - AN(hash); wrk = sp->wrk; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + req = sp->req; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->http, HTTP_MAGIC); + AN(req->director); + AN(hash); hsh_prealloc(wrk); - memcpy(sp->wrk->nobjhead->digest, sp->req->digest, - sizeof sp->req->digest); + memcpy(sp->wrk->nobjhead->digest, req->digest, sizeof req->digest); if (cache_param->diag_bitmap & 0x80000000) hsh_testmagic(sp->wrk->nobjhead->digest); - if (sp->req->hash_objhead != NULL) { + if (req->hash_objhead != NULL) { /* * This sess came off the waiting list, and brings a * oh refcnt with it. */ - CHECK_OBJ_NOTNULL(sp->req->hash_objhead, OBJHEAD_MAGIC); - oh = sp->req->hash_objhead; - sp->req->hash_objhead = NULL; + CHECK_OBJ_NOTNULL(req->hash_objhead, OBJHEAD_MAGIC); + oh = req->hash_objhead; + req->hash_objhead = NULL; } else { AN(wrk->nobjhead); oh = hash->lookup(wrk, wrk->nobjhead); @@ -333,7 +335,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) if (oc->flags & OC_F_BUSY) { CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC); - if (sp->req->hash_ignore_busy) + if (req->hash_ignore_busy) continue; if (oc->busyobj->vary != NULL && @@ -381,20 +383,20 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) * XXX: serialize fetch of all Vary's if grace is possible. */ - AZ(sp->req->objcore); - sp->req->objcore = grace_oc; /* XXX: Hack-ish */ + AZ(req->objcore); + req->objcore = grace_oc; /* XXX: Hack-ish */ if (oc == NULL /* We found no live object */ && grace_oc != NULL /* There is a grace candidate */ && (busy_oc != NULL /* Somebody else is already busy */ - || !VDI_Healthy(sp->req->director, sp))) { + || !VDI_Healthy(req->director, sp))) { /* Or it is impossible to fetch */ o = oc_getobj(&sp->wrk->stats, grace_oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); oc = grace_oc; } - sp->req->objcore = NULL; + req->objcore = NULL; - if (oc != NULL && !sp->req->hash_always_miss) { + if (oc != NULL && !req->hash_always_miss) { o = oc_getobj(&sp->wrk->stats, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); assert(oc->objhead == oh); @@ -406,13 +408,12 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) assert(oh->refcnt > 1); Lck_Unlock(&oh->mtx); assert(hash->deref(oh)); - *poh = oh; return (oc); } if (busy_oc != NULL) { /* There are one or more busy objects, wait for them */ - if (sp->req->esi_level == 0) { + if (req->esi_level == 0) { CHECK_OBJ_NOTNULL(sp->wrk->nwaitinglist, WAITINGLIST_MAGIC); if (oh->waitinglist == NULL) { @@ -422,7 +423,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) VTAILQ_INSERT_TAIL(&oh->waitinglist->list, sp, list); } if (cache_param->diag_bitmap & 0x20) - VSLb(sp->req->vsl, SLT_Debug, + VSLb(req->vsl, SLT_Debug, "on waiting list <%p>", oh); SES_Charge(sp); /* @@ -430,7 +431,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) * back when the sess comes off the waiting list and * calls us again */ - sp->req->hash_objhead = oh; + req->hash_objhead = oh; sp->wrk = NULL; Lck_Unlock(&oh->mtx); return (NULL); @@ -442,16 +443,16 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) AN(oc->flags & OC_F_BUSY); oc->refcnt = 1; - AZ(sp->req->busyobj); - sp->req->busyobj = VBO_GetBusyObj(wrk); - sp->req->busyobj->vsl->wid = sp->vsl_id; + AZ(req->busyobj); + req->busyobj = VBO_GetBusyObj(wrk); + req->busyobj->vsl->wid = sp->vsl_id; - VRY_Validate(sp->req->vary_b); - if (sp->req->vary_l != NULL) - sp->req->busyobj->vary = sp->req->vary_b; + VRY_Validate(req->vary_b); + if (req->vary_l != NULL) + req->busyobj->vary = req->vary_b; else - sp->req->busyobj->vary = NULL; - oc->busyobj = sp->req->busyobj; + req->busyobj->vary = NULL; + oc->busyobj = req->busyobj; /* * Busy objects go on the tail, so they will not trip up searches. @@ -461,7 +462,6 @@ HSH_Lookup(struct sess *sp, struct objhead **poh) oc->objhead = oh; /* NB: do not deref objhead the new object inherits our reference */ Lck_Unlock(&oh->mtx); - *poh = oh; return (oc); } diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index a3577a7..4935bfb 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -52,7 +52,7 @@ struct hash_slinger { /* cache_hash.c */ void HSH_Cleanup(struct worker *w); -struct objcore *HSH_Lookup(struct sess *sp, struct objhead **poh); +struct objcore *HSH_Lookup(struct sess *sp); void HSH_Ref(struct objcore *o); void HSH_Drop(struct worker *, struct object **); void HSH_Init(const struct hash_slinger *slinger); From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] aea8559 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit aea855912ec8e140435e7d5f6604e1c6062faf79 Merge: 0326c20 9831187 Author: Poul-Henning Kamp Date: Mon Mar 12 15:42:10 2012 +0000 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] e556146 More deref-reduction: Message-ID: commit e556146d8bca01cf714a0594e4454d725a2154c9 Author: Poul-Henning Kamp Date: Mon Mar 12 15:48:43 2012 +0000 More deref-reduction: VRY only needs req, not sess. Overlooked some lingering sp->wrk's in HSH_Lookup() diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 99c66c3..6aceab0 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -954,8 +954,8 @@ void RES_BuildHttp(const struct sess *sp); void RES_WriteObj(struct sess *sp); /* cache_vary.c */ -struct vsb *VRY_Create(const struct sess *sp, const struct http *hp); -int VRY_Match(const struct sess *sp, const uint8_t *vary); +struct vsb *VRY_Create(struct req *sp, const struct http *hp); +int VRY_Match(struct req *, const uint8_t *vary); void VRY_Validate(const uint8_t *vary); /* cache_vcl.c */ diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 4d18395..1192f81 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -784,7 +784,7 @@ cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req) /* Create Vary instructions */ if (req->objcore != NULL) { CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); - vary = VRY_Create(sp, bo->beresp); + vary = VRY_Create(req, bo->beresp); if (vary != NULL) { varyl = VSB_len(vary); assert(varyl > 0); diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 55215e6..34e86a4 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -302,9 +302,9 @@ HSH_Lookup(struct sess *sp) AN(hash); hsh_prealloc(wrk); - memcpy(sp->wrk->nobjhead->digest, req->digest, sizeof req->digest); + memcpy(wrk->nobjhead->digest, req->digest, sizeof req->digest); if (cache_param->diag_bitmap & 0x80000000) - hsh_testmagic(sp->wrk->nobjhead->digest); + hsh_testmagic(wrk->nobjhead->digest); if (req->hash_objhead != NULL) { /* @@ -339,21 +339,21 @@ HSH_Lookup(struct sess *sp) continue; if (oc->busyobj->vary != NULL && - !VRY_Match(sp, oc->busyobj->vary)) + !VRY_Match(req, oc->busyobj->vary)) continue; busy_oc = oc; continue; } - o = oc_getobj(&sp->wrk->stats, oc); + o = oc_getobj(&wrk->stats, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); if (o->exp.ttl <= 0.) continue; if (BAN_CheckObject(o, sp)) continue; - if (o->vary != NULL && !VRY_Match(sp, o->vary)) + if (o->vary != NULL && !VRY_Match(req, o->vary)) continue; /* If still valid, use it */ @@ -390,14 +390,14 @@ HSH_Lookup(struct sess *sp) && (busy_oc != NULL /* Somebody else is already busy */ || !VDI_Healthy(req->director, sp))) { /* Or it is impossible to fetch */ - o = oc_getobj(&sp->wrk->stats, grace_oc); + o = oc_getobj(&wrk->stats, grace_oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); oc = grace_oc; } req->objcore = NULL; if (oc != NULL && !req->hash_always_miss) { - o = oc_getobj(&sp->wrk->stats, oc); + o = oc_getobj(&wrk->stats, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); assert(oc->objhead == oh); @@ -414,11 +414,11 @@ HSH_Lookup(struct sess *sp) if (busy_oc != NULL) { /* There are one or more busy objects, wait for them */ if (req->esi_level == 0) { - CHECK_OBJ_NOTNULL(sp->wrk->nwaitinglist, + CHECK_OBJ_NOTNULL(wrk->nwaitinglist, WAITINGLIST_MAGIC); if (oh->waitinglist == NULL) { - oh->waitinglist = sp->wrk->nwaitinglist; - sp->wrk->nwaitinglist = NULL; + oh->waitinglist = wrk->nwaitinglist; + wrk->nwaitinglist = NULL; } VTAILQ_INSERT_TAIL(&oh->waitinglist->list, sp, list); } diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c index 7ba9c60..a7bb31c 100644 --- a/bin/varnishd/cache/cache_vary.c +++ b/bin/varnishd/cache/cache_vary.c @@ -60,7 +60,7 @@ #include "vend.h" struct vsb * -VRY_Create(const struct sess *sp, const struct http *hp) +VRY_Create(struct req *req, const struct http *hp) { char *v, *p, *q, *h, *e; struct vsb *sb, *sbh; @@ -79,7 +79,7 @@ VRY_Create(const struct sess *sp, const struct http *hp) AN(sbh); if (*v == ':') { - VSLb(sp->req->vsl, SLT_Error, + VSLb(req->vsl, SLT_Error, "Vary header had extra ':', fix backend"); v++; } @@ -97,7 +97,7 @@ VRY_Create(const struct sess *sp, const struct http *hp) (char)(1 + (q - p)), (int)(q - p), p, 0); AZ(VSB_finish(sbh)); - if (http_GetHdr(sp->req->http, VSB_data(sbh), &h)) { + if (http_GetHdr(req->http, VSB_data(sbh), &h)) { AZ(vct_issp(*h)); /* Trim trailing space */ e = strchr(h, '\0'); @@ -175,9 +175,9 @@ vry_cmp(const uint8_t *v1, const uint8_t *v2) } int -VRY_Match(const struct sess *sp, const uint8_t *vary) +VRY_Match(struct req *req, const uint8_t *vary) { - uint8_t *vsp = sp->req->vary_b; + uint8_t *vsp = req->vary_b; char *h, *e; unsigned lh, ln; int i, retval = 1, oflo = 0; @@ -188,7 +188,7 @@ VRY_Match(const struct sess *sp, const uint8_t *vary) if (i == 1) { /* Build a new entry */ - i = http_GetHdr(sp->req->http, + i = http_GetHdr(req->http, (const char*)(vary+2), &h); if (i) { /* Trim trailing space */ @@ -204,8 +204,8 @@ VRY_Match(const struct sess *sp, const uint8_t *vary) /* Length of the entire new vary entry */ ln = 2 + vary[2] + 2 + (lh == 0xffff ? 0 : lh); - if (vsp + ln >= sp->req->vary_e) { - vsp = sp->req->vary_b; + if (vsp + ln >= req->vary_e) { + vsp = req->vary_b; oflo = 1; } @@ -213,7 +213,7 @@ VRY_Match(const struct sess *sp, const uint8_t *vary) * We MUST have space for one entry and the end marker * after it, which prevents old junk from confusing us */ - assert(vsp + ln + 2 < sp->req->vary_e); + assert(vsp + ln + 2 < req->vary_e); vbe16enc(vsp, (uint16_t)lh); memcpy(vsp + 2, vary + 2, vary[2] + 2); @@ -231,20 +231,20 @@ VRY_Match(const struct sess *sp, const uint8_t *vary) vsp += vry_len(vsp); vary += vry_len(vary); } - if (vsp + 3 > sp->req->vary_e) + if (vsp + 3 > req->vary_e) oflo = 1; if (oflo) { /* XXX: Should log this */ - vsp = sp->req->vary_b; + vsp = req->vary_b; } vsp[0] = 0xff; vsp[1] = 0xff; vsp[2] = 0; if (oflo) - sp->req->vary_l = NULL; + req->vary_l = NULL; else - sp->req->vary_l = vsp + 3; + req->vary_l = vsp + 3; return (retval); } From martin at varnish-software.com Thu Dec 18 09:27:41 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] c92ad59 Utilize the oc->priv2 variable to store the pointer to the allocating stevedore for the default case (malloc and file) Message-ID: commit c92ad59cde5dfd32487eb6466b07c4817e5efad1 Author: Martin Blix Grydeland Date: Mon Mar 12 16:11:01 2012 +0100 Utilize the oc->priv2 variable to store the pointer to the allocating stevedore for the default case (malloc and file) Change the type of oc->priv2 from unsigned to uintptr_t (due to padding this does not increase the size of objcore). Change the default_oc_getlru to find the stevedore through the oc->priv2 instead of having to go through (and page-in) the object. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 6aceab0..8f2381c 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -371,7 +371,7 @@ struct objcore { int refcnt; struct objcore_methods *methods; void *priv; - unsigned priv2; + uintptr_t priv2; struct objhead *objhead; struct busyobj *busyobj; double timer_when; diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 3c62d8d..9f013d2 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -85,10 +85,10 @@ default_oc_freeobj(struct objcore *oc) static struct lru * default_oc_getlru(const struct objcore *oc) { - struct object *o; + struct stevedore *stv; - CAST_OBJ_NOTNULL(o, oc->priv, OBJECT_MAGIC); - return (o->objstore->stevedore->lru); + CAST_OBJ_NOTNULL(stv, (void *)oc->priv2, STEVEDORE_MAGIC); + return (stv->lru); } static struct objcore_methods default_oc_methods = { @@ -227,12 +227,13 @@ struct stv_objsecrets { */ struct object * -STV_MkObject(struct busyobj *bo, struct objcore **ocp, void *ptr, unsigned ltot, - const struct stv_objsecrets *soc) +STV_MkObject(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp, + void *ptr, unsigned ltot, const struct stv_objsecrets *soc) { struct object *o; unsigned l; + CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC); AN(ocp); @@ -270,6 +271,7 @@ STV_MkObject(struct busyobj *bo, struct objcore **ocp, void *ptr, unsigned ltot, o->objcore->methods = &default_oc_methods; o->objcore->priv = o; + o->objcore->priv2 = (uintptr_t)stv; } return (o); } @@ -297,7 +299,7 @@ stv_default_allocobj(struct stevedore *stv, struct busyobj *bo, return (NULL); } ltot = st->len = st->space; - o = STV_MkObject(bo, ocp, st->ptr, ltot, soc); + o = STV_MkObject(stv, bo, ocp, st->ptr, ltot, soc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); o->objstore = st; return (o); diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index 80839a6..6994314 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -92,8 +92,9 @@ extern struct stevedore *stv_transient; int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx); uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity, const char *ctx); -struct object *STV_MkObject(struct busyobj *bo, struct objcore **ocp, - void *ptr, unsigned ltot, const struct stv_objsecrets *soc); +struct object *STV_MkObject(struct stevedore *stv, struct busyobj *bo, + struct objcore **ocp, void *ptr, unsigned ltot, + const struct stv_objsecrets *soc); struct lru *LRU_Alloc(void); void LRU_Free(struct lru *lru); diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index 58c6f29..35845f0 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -485,7 +485,7 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp, assert(st->space >= ltot); ltot = st->len = st->space; - o = STV_MkObject(bo, ocp, st->ptr, ltot, soc); + o = STV_MkObject(stv, bo, ocp, st->ptr, ltot, soc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); o->objstore = st; From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] 88298f1 More sp -> req argument weakening Message-ID: commit 88298f16110c19173d0b27de4a3d8417867a192f Author: Poul-Henning Kamp Date: Mon Mar 12 19:35:00 2012 +0000 More sp -> req argument weakening diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 8f2381c..17f4f4d 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -749,8 +749,8 @@ void EXP_Set_ttl(struct exp *e, double v); void EXP_Set_grace(struct exp *e, double v); void EXP_Set_keep(struct exp *e, double v); -double EXP_Ttl(const struct sess *, const struct object*); -double EXP_Grace(const struct sess *, const struct object*); +double EXP_Ttl(const struct req *, const struct object*); +double EXP_Grace(const struct req *, const struct object*); void EXP_Insert(struct object *o); void EXP_Inject(struct objcore *oc, struct lru *lru, double when); void EXP_Init(void); diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index 98b664f..9d7136c 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -109,39 +109,39 @@ EXP_ACCESS(keep, 0.,) */ static double -EXP_Keep(const struct sess *sp, const struct object *o) +EXP_Keep(const struct req *req, const struct object *o) { double r; r = (double)cache_param->default_keep; if (o->exp.keep > 0.) r = o->exp.keep; - if (sp != NULL && sp->req->exp.keep > 0. && sp->req->exp.keep < r) - r = sp->req->exp.keep; - return (EXP_Ttl(sp, o) + r); + if (req != NULL && req->exp.keep > 0. && req->exp.keep < r) + r = req->exp.keep; + return (EXP_Ttl(req, o) + r); } double -EXP_Grace(const struct sess *sp, const struct object *o) +EXP_Grace(const struct req *req, const struct object *o) { double r; r = (double)cache_param->default_grace; if (o->exp.grace >= 0.) r = o->exp.grace; - if (sp != NULL && sp->req->exp.grace > 0. && sp->req->exp.grace < r) - r = sp->req->exp.grace; - return (EXP_Ttl(sp, o) + r); + if (req != NULL && req->exp.grace > 0. && req->exp.grace < r) + r = req->exp.grace; + return (EXP_Ttl(req, o) + r); } double -EXP_Ttl(const struct sess *sp, const struct object *o) +EXP_Ttl(const struct req *req, const struct object *o) { double r; r = o->exp.ttl; - if (sp != NULL && sp->req->exp.ttl > 0. && sp->req->exp.ttl < r) - r = sp->req->exp.ttl; + if (req != NULL && req->exp.ttl > 0. && req->exp.ttl < r) + r = req->exp.ttl; return (o->exp.entered + r); } diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 34e86a4..6a3af2f 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -357,14 +357,14 @@ HSH_Lookup(struct sess *sp) continue; /* If still valid, use it */ - if (EXP_Ttl(sp, o) >= sp->t_req) + if (EXP_Ttl(req, o) >= sp->t_req) break; /* * Remember any matching objects inside their grace period * and if there are several, use the least expired one. */ - if (EXP_Grace(sp, o) >= sp->t_req) { + if (EXP_Grace(req, o) >= sp->t_req) { if (grace_oc == NULL || grace_ttl < o->exp.entered + o->exp.ttl) { grace_oc = oc; From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] cf81441 Tag oc's with COMPLETE or FAILURE when we fetch the body. Message-ID: commit cf81441e52a10143c5383472c9d25a073b6fde46 Author: Poul-Henning Kamp Date: Mon Mar 12 22:19:48 2012 +0000 Tag oc's with COMPLETE or FAILURE when we fetch the body. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 17f4f4d..08e3405 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -381,6 +381,8 @@ struct objcore { #define OC_F_LRUDONTMOVE (1<<4) #define OC_F_PRIV (1<<5) /* Stevedore private flag */ #define OC_F_LURK (3<<6) /* Ban-lurker-color */ +#define OC_F_COMPLETE (1<<8) +#define OC_F_FAILED (1<<9) unsigned timer_idx; VTAILQ_ENTRY(objcore) list; VTAILQ_ENTRY(objcore) lru_list; diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 1192f81..9f69bfa 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -921,9 +921,9 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) AN(req->director); if (bo->state == BOS_FAILED) { - HSH_Drop(wrk, &sp->req->obj); - VBO_DerefBusyObj(wrk, &req->busyobj); + HSH_Drop(wrk, &req->obj); AZ(req->obj); + VBO_DerefBusyObj(wrk, &req->busyobj); req->err_code = 503; sp->step = STP_ERROR; return (0); @@ -1145,6 +1145,9 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) return (0); } + /* For now... */ + AN(oc->flags & (OC_F_COMPLETE|OC_F_FAILED)); + o = oc_getobj(&wrk->stats, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); req->obj = o; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 172f422..feebd9b 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -634,6 +634,13 @@ FetchBody(struct worker *wrk, void *priv) http_Teardown(bo->bereq); http_Teardown(bo->beresp); + if (obj->objcore != NULL) { + /* pass has no objcore */ + /* XXX: lock protection ?? */ + obj->objcore->flags |= + (bo->state == BOS_FAILED ? OC_F_FAILED : OC_F_COMPLETE); + } + if (bo->state == BOS_FAILED) { wrk->stats.fetch_failed++; VDI_CloseFd(&bo->vbc); diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 6a3af2f..514dbeb 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -269,6 +269,7 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc) oc->refcnt = 1; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AZ(oc->flags & OC_F_BUSY); + oc->flags |= OC_F_COMPLETE; VTAILQ_INSERT_HEAD(&oh->objcs, oc, list); /* NB: do not deref objhead the new object inherits our reference */ From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] 0041032 Let the body-fetching thread Unbusy the object. Message-ID: commit 0041032c2b01f865ed83adcf94e8789a3f31e2ce Author: Poul-Henning Kamp Date: Mon Mar 12 22:45:04 2012 +0000 Let the body-fetching thread Unbusy the object. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 9f69bfa..bc22656 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -929,13 +929,6 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) return (0); } - if (req->obj->objcore != NULL) { - EXP_Insert(req->obj); - AN(req->obj->objcore); - AN(req->obj->objcore->ban); - AZ(req->obj->ws_o->overflow); - HSH_Unbusy(&wrk->stats, req->obj->objcore); - } VBO_DerefBusyObj(wrk, &req->busyobj); wrk->acct_tmp.fetch++; sp->step = STP_PREPRESP; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index feebd9b..968db87 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -35,6 +35,8 @@ #include "cache.h" +#include "hash/hash_slinger.h" + #include "cache_backend.h" #include "vcli_priv.h" #include "vct.h" @@ -674,13 +676,22 @@ FetchBody(struct worker *wrk, void *priv) "Content-Length: %zd", obj->len); } - bo->state = BOS_FINISHED; if (cls) VDI_CloseFd(&bo->vbc); else VDI_RecycleFd(&bo->vbc); + if (obj->objcore != NULL) { + EXP_Insert(obj); + AN(obj->objcore->ban); + AZ(obj->ws_o->overflow); + HSH_Unbusy(&wrk->stats, obj->objcore); + } + + /* XXX: Atomic assignment, needs volatile/membar ? */ + bo->state = BOS_FINISHED; + } bo->stats = NULL; VBO_DerefBusyObj(wrk, &bo); From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] 2b21621 Also move dropping failed objects to FetchBody() Message-ID: commit 2b216210df2a797ff959dfdfe0231cf55c43db49 Author: Poul-Henning Kamp Date: Mon Mar 12 23:21:27 2012 +0000 Also move dropping failed objects to FetchBody() diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index bc22656..4ad05ff 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -917,12 +917,9 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) assert(bo->state >= BOS_FAILED); assert(WRW_IsReleased(wrk)); - AZ(bo->vbc); - AN(req->director); if (bo->state == BOS_FAILED) { - HSH_Drop(wrk, &req->obj); - AZ(req->obj); + req->obj = NULL; VBO_DerefBusyObj(wrk, &req->busyobj); req->err_code = 503; sp->step = STP_ERROR; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 968db87..d9b519d 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -647,6 +647,7 @@ FetchBody(struct worker *wrk, void *priv) wrk->stats.fetch_failed++; VDI_CloseFd(&bo->vbc); obj->len = 0; + HSH_Drop(wrk, &obj); } else { assert(bo->state == BOS_FETCHING); From martin at varnish-software.com Thu Dec 18 09:27:41 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] d74bc62 Add a mutex timing flag to diag_bitmap Message-ID: commit d74bc62ea6173ed270b58c2da0c97c52323eec0b Author: Martin Blix Grydeland Date: Mon Mar 12 10:14:45 2012 +0100 Add a mutex timing flag to diag_bitmap The MTX_UNLOCK debug lines will log for how long this thread held the lock. The MTX_LOCKWAIT debug lines will log how long the thread waited for the lock to be acquired. Bugs: Threads unlocking the mutex as part of pthread_cond_wait will not update the time of lock value for that mutex on pthread_cond_wait entry. diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c index 9fb34ae..3bbbfbb 100644 --- a/bin/varnishd/cache/cache_lck.c +++ b/bin/varnishd/cache/cache_lck.c @@ -37,6 +37,7 @@ #include +#include "vtim.h" #include "cache.h" /*The constability of lck depends on platform pthreads implementation */ @@ -47,6 +48,7 @@ struct ilck { pthread_mutex_t mtx; int held; pthread_t owner; + double t0; VTAILQ_ENTRY(ilck) list; const char *w; struct VSC_C_lck *stat; @@ -62,9 +64,10 @@ Lck__Lock(struct lock *lck, const char *p, const char *f, int l) { struct ilck *ilck; int r; + double t0, t; CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC); - if (!(cache_param->diag_bitmap & 0x18)) { + if (!(cache_param->diag_bitmap & 0x98)) { AZ(pthread_mutex_lock(&ilck->mtx)); AZ(ilck->held); ilck->stat->locks++; @@ -72,6 +75,8 @@ Lck__Lock(struct lock *lck, const char *p, const char *f, int l) ilck->held = 1; return; } + if (cache_param->diag_bitmap & 0x80) + t0 = VTIM_real(); r = pthread_mutex_trylock(&ilck->mtx); assert(r == 0 || r == EBUSY); if (r) { @@ -83,6 +88,12 @@ Lck__Lock(struct lock *lck, const char *p, const char *f, int l) } else if (cache_param->diag_bitmap & 0x8) { VSL(SLT_Debug, 0, "MTX_LOCK(%s,%s,%d,%s)", p, f, l, ilck->w); } + if (cache_param->diag_bitmap & 0x80) { + t = VTIM_real(); + VSL(SLT_Debug, 0, "MTX_LOCKWAIT(%s,%s,%d,%s) %.9fs", + p, f, l, ilck->w, t - t0); + ilck->t0 = t; + } AZ(ilck->held); ilck->stat->locks++; ilck->owner = pthread_self(); @@ -110,7 +121,10 @@ Lck__Unlock(struct lock *lck, const char *p, const char *f, int l) */ memset(&ilck->owner, 0, sizeof ilck->owner); AZ(pthread_mutex_unlock(&ilck->mtx)); - if (cache_param->diag_bitmap & 0x8) + if (cache_param->diag_bitmap & 0x80) + VSL(SLT_Debug, 0, "MTX_UNLOCK(%s,%s,%d,%s) %.9fs", + p, f, l, ilck->w, VTIM_real() - ilck->t0); + else if (cache_param->diag_bitmap & 0x8) VSL(SLT_Debug, 0, "MTX_UNLOCK(%s,%s,%d,%s)", p, f, l, ilck->w); } @@ -131,6 +145,8 @@ Lck__Trylock(struct lock *lck, const char *p, const char *f, int l) ilck->held = 1; ilck->stat->locks++; ilck->owner = pthread_self(); + if (cache_param->diag_bitmap & 0x80) + ilck->t0 = VTIM_real(); } return (r); } diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 31ef010..f6493ad 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1060,6 +1060,7 @@ static const struct parspec input_parspec[] = { " 0x00000010 - mutex contests.\n" " 0x00000020 - waiting list.\n" " 0x00000040 - object workspace.\n" + " 0x00000080 - mutex timing.\n" " 0x00001000 - do not core-dump child process.\n" " 0x00002000 - only short panic message.\n" " 0x00004000 - panic to stderr.\n" @@ -1069,7 +1070,11 @@ static const struct parspec input_parspec[] = { " 0x00080000 - ban-lurker debugging.\n" " 0x80000000 - do edge-detection on digest.\n" "\n" - "Use 0x notation and do the bitor in your head :-)\n", + "Use 0x notation and do the bitor in your head :-)\n" + "\n" + "Note: Mutex timing will add extra overhead and " + "synchronization between threads, consequently changing the " + "locking characteristics.\n", 0, "0", "bitmap" }, { "ban_dups", tweak_bool, &mgt_param.ban_dups, 0, 0, From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] 6393f26 Fix a misfeature in the hasher-API: Pass the digest in, so we don't have to touch the (potential) new objhead, unless we intend to insert it. Message-ID: commit 6393f26c38d4c5bd7a6391ab1a016ed79b9830d7 Author: Poul-Henning Kamp Date: Wed Mar 14 18:33:48 2012 +0000 Fix a misfeature in the hasher-API: Pass the digest in, so we don't have to touch the (potential) new objhead, unless we intend to insert it. Straighten out the colvolved logic of critbit while here. diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 514dbeb..d154717 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -253,15 +253,10 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); hsh_prealloc(wrk); - if (cache_param->diag_bitmap & 0x80000000) - hsh_testmagic(wrk->nobjhead->digest); AN(wrk->nobjhead); - memcpy(wrk->nobjhead->digest, digest, SHA256_LEN); - oh = hash->lookup(wrk, wrk->nobjhead); + oh = hash->lookup(wrk, digest, &wrk->nobjhead); CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); - if (oh == wrk->nobjhead) - wrk->nobjhead = NULL; Lck_Lock(&oh->mtx); assert(oh->refcnt > 0); @@ -303,9 +298,8 @@ HSH_Lookup(struct sess *sp) AN(hash); hsh_prealloc(wrk); - memcpy(wrk->nobjhead->digest, req->digest, sizeof req->digest); if (cache_param->diag_bitmap & 0x80000000) - hsh_testmagic(wrk->nobjhead->digest); + hsh_testmagic(req->digest); if (req->hash_objhead != NULL) { /* @@ -317,9 +311,7 @@ HSH_Lookup(struct sess *sp) req->hash_objhead = NULL; } else { AN(wrk->nobjhead); - oh = hash->lookup(wrk, wrk->nobjhead); - if (oh == wrk->nobjhead) - wrk->nobjhead = NULL; + oh = hash->lookup(wrk, req->digest, &wrk->nobjhead); } CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); diff --git a/bin/varnishd/hash/hash_classic.c b/bin/varnishd/hash/hash_classic.c index c84238a..87e0561 100644 --- a/bin/varnishd/hash/hash_classic.c +++ b/bin/varnishd/hash/hash_classic.c @@ -111,24 +111,26 @@ hcl_start(void) */ static struct objhead * __match_proto__(hash_lookup_f) -hcl_lookup(struct worker *wrk, struct objhead *noh) +hcl_lookup(struct worker *wrk, const void *digest, struct objhead **noh) { struct objhead *oh; struct hcl_hd *hp; - unsigned u1, digest; + unsigned u1, hdigest; int i; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC); + AN(digest); + AN(noh); + CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC); - assert(sizeof noh->digest > sizeof digest); - memcpy(&digest, noh->digest, sizeof digest); - u1 = digest % hcl_nhash; + assert(sizeof oh->digest >= sizeof hdigest); + memcpy(&hdigest, digest, sizeof hdigest); + u1 = hdigest % hcl_nhash; hp = &hcl_head[u1]; Lck_Lock(&hp->mtx); VTAILQ_FOREACH(oh, &hp->head, hoh_list) { - i = memcmp(oh->digest, noh->digest, sizeof oh->digest); + i = memcmp(oh->digest, digest, sizeof oh->digest); if (i < 0) continue; if (i > 0) @@ -139,14 +141,18 @@ hcl_lookup(struct worker *wrk, struct objhead *noh) } if (oh != NULL) - VTAILQ_INSERT_BEFORE(oh, noh, hoh_list); + VTAILQ_INSERT_BEFORE(oh, *noh, hoh_list); else - VTAILQ_INSERT_TAIL(&hp->head, noh, hoh_list); + VTAILQ_INSERT_TAIL(&hp->head, *noh, hoh_list); - noh->hoh_head = hp; + oh = *noh; + *noh = NULL; + memcpy(oh->digest, digest, sizeof oh->digest); + + oh->hoh_head = hp; Lck_Unlock(&hp->mtx); - return (noh); + return (oh); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c index 535661e..5bdab86 100644 --- a/bin/varnishd/hash/hash_critbit.c +++ b/bin/varnishd/hash/hash_critbit.c @@ -168,16 +168,15 @@ hcb_l_y(uintptr_t u) */ static unsigned -hcb_crit_bit(const struct objhead *oh1, const struct objhead *oh2, - struct hcb_y *y) +hcb_crit_bit(const uint8_t *digest, const struct objhead *oh2, struct hcb_y *y) { unsigned char u, r; CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC); - for (u = 0; u < DIGEST_LEN && oh1->digest[u] == oh2->digest[u]; u++) + for (u = 0; u < DIGEST_LEN && digest[u] == oh2->digest[u]; u++) ; assert(u < DIGEST_LEN); - r = hcb_bits(oh1->digest[u], oh2->digest[u]); + r = hcb_bits(digest[u], oh2->digest[u]); y->ptr = u; y->bitmask = 0x80 >> r; y->critbit = u * 8 + r; @@ -191,8 +190,8 @@ hcb_crit_bit(const struct objhead *oh1, const struct objhead *oh2, */ static struct objhead * -hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh, - int has_lock) +hcb_insert(struct worker *wrk, struct hcb_root *root, const uint8_t *digest, + struct objhead **noh) { volatile uintptr_t *p; uintptr_t pp; @@ -203,17 +202,20 @@ hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh, p = &root->origo; pp = *p; if (pp == 0) { - if (!has_lock) + if (noh == NULL) return (NULL); - *p = hcb_r_node(oh); - return (oh); + oh2 = *noh; + *noh = NULL; + memcpy(oh2->digest, digest, sizeof oh2->digest); + *p = hcb_r_node(oh2); + return (oh2); } while(hcb_is_y(pp)) { y = hcb_l_y(pp); CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC); assert(y->ptr < DIGEST_LEN); - s = (oh->digest[y->ptr] & y->bitmask) != 0; + s = (digest[y->ptr] & y->bitmask) != 0; assert(s < 2); p = &y->leaf[s]; pp = *p; @@ -221,7 +223,7 @@ hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh, if (pp == 0) { /* We raced hcb_delete and got a NULL pointer */ - assert(!has_lock); + assert(noh == NULL); return (NULL); } @@ -230,20 +232,23 @@ hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh, /* We found a node, does it match ? */ oh2 = hcb_l_node(pp); CHECK_OBJ_NOTNULL(oh2, OBJHEAD_MAGIC); - if (!memcmp(oh2->digest, oh->digest, DIGEST_LEN)) + if (!memcmp(oh2->digest, digest, DIGEST_LEN)) return (oh2); - if (!has_lock) + if (noh == NULL) return (NULL); /* Insert */ CAST_OBJ_NOTNULL(y2, wrk->nhashpriv, HCB_Y_MAGIC); wrk->nhashpriv = NULL; - (void)hcb_crit_bit(oh, oh2, y2); - s2 = (oh->digest[y2->ptr] & y2->bitmask) != 0; + (void)hcb_crit_bit(digest, oh2, y2); + s2 = (digest[y2->ptr] & y2->bitmask) != 0; assert(s2 < 2); - y2->leaf[s2] = hcb_r_node(oh); + oh2 = *noh; + *noh = NULL; + memcpy(oh2->digest, digest, sizeof oh2->digest); + y2->leaf[s2] = hcb_r_node(oh2); s2 = 1-s2; p = &root->origo; @@ -256,14 +261,14 @@ hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh, if (y->critbit > y2->critbit) break; assert(y->ptr < DIGEST_LEN); - s = (oh->digest[y->ptr] & y->bitmask) != 0; + s = (digest[y->ptr] & y->bitmask) != 0; assert(s < 2); p = &y->leaf[s]; } y2->leaf[s2] = *p; VWMB(); *p = hcb_r_y(y2); - return(oh); + return(oh2); } /*--------------------------------------------------------------------*/ @@ -412,45 +417,50 @@ hcb_deref(struct objhead *oh) } static struct objhead * __match_proto__(hash_lookup_f) -hcb_lookup(struct worker *wrk, struct objhead *noh) +hcb_lookup(struct worker *wrk, const void *digest, struct objhead **noh) { struct objhead *oh; struct hcb_y *y; unsigned u; - unsigned with_lock; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + AN(digest); + AN(noh); + CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC); + assert((*noh)->refcnt == 1); + + /* First try in read-only mode without holding a lock */ + + VSC_C_main->hcb_nolock++; + oh = hcb_insert(wrk, &hcb_root, digest, NULL); + if (oh != NULL) { + Lck_Lock(&oh->mtx); + /* + * A refcount of zero indicates that the tree changed + * under us, so fall through and try with the lock held. + */ + u = oh->refcnt; + if (u > 0) + oh->refcnt++; + Lck_Unlock(&oh->mtx); + if (u > 0) + return (oh); + } - with_lock = 0; while (1) { - if (with_lock) { - CAST_OBJ_NOTNULL(y, wrk->nhashpriv, HCB_Y_MAGIC); - Lck_Lock(&hcb_mtx); - VSC_C_main->hcb_lock++; - assert(noh->refcnt == 1); - oh = hcb_insert(wrk, &hcb_root, noh, 1); - Lck_Unlock(&hcb_mtx); - } else { - VSC_C_main->hcb_nolock++; - oh = hcb_insert(wrk, &hcb_root, noh, 0); - } + /* No luck, try with lock held, so we can modify tree */ + CAST_OBJ_NOTNULL(y, wrk->nhashpriv, HCB_Y_MAGIC); + Lck_Lock(&hcb_mtx); + VSC_C_main->hcb_lock++; + oh = hcb_insert(wrk, &hcb_root, digest, noh); + Lck_Unlock(&hcb_mtx); - if (oh != NULL && oh == noh) { - /* Assert that we only muck with the tree with a lock */ - assert(with_lock); - VSC_C_main->hcb_insert++; + CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); + if (*noh == NULL) { assert(oh->refcnt > 0); + VSC_C_main->hcb_insert++; return (oh); } - - if (oh == NULL) { - assert(!with_lock); - /* Try again, with lock */ - with_lock = 1; - continue; - } - - CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC); Lck_Lock(&oh->mtx); /* * A refcount of zero indicates that the tree changed @@ -459,8 +469,6 @@ hcb_lookup(struct worker *wrk, struct objhead *noh) u = oh->refcnt; if (u > 0) oh->refcnt++; - else - with_lock = 1; Lck_Unlock(&oh->mtx); if (u > 0) return (oh); diff --git a/bin/varnishd/hash/hash_simple_list.c b/bin/varnishd/hash/hash_simple_list.c index fef2b65..d70e76f 100644 --- a/bin/varnishd/hash/hash_simple_list.c +++ b/bin/varnishd/hash/hash_simple_list.c @@ -60,16 +60,19 @@ hsl_start(void) */ static struct objhead * __match_proto__(hash_lookup_f) -hsl_lookup(struct worker *wrk, struct objhead *noh) +hsl_lookup(struct worker *wrk, const void *digest, struct objhead **noh) { struct objhead *oh; int i; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC); + AN(digest); + AN(noh); + CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC); + Lck_Lock(&hsl_mtx); VTAILQ_FOREACH(oh, &hsl_head, hoh_list) { - i = memcmp(oh->digest, noh->digest, sizeof oh->digest); + i = memcmp(oh->digest, digest, sizeof oh->digest); if (i < 0) continue; if (i > 0) @@ -80,12 +83,15 @@ hsl_lookup(struct worker *wrk, struct objhead *noh) } if (oh != NULL) - VTAILQ_INSERT_BEFORE(oh, noh, hoh_list); + VTAILQ_INSERT_BEFORE(oh, *noh, hoh_list); else - VTAILQ_INSERT_TAIL(&hsl_head, noh, hoh_list); + VTAILQ_INSERT_TAIL(&hsl_head, *noh, hoh_list); + oh = *noh; + *noh = NULL; + memcpy(oh->digest, digest, sizeof oh->digest); Lck_Unlock(&hsl_mtx); - return (noh); + return (oh); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index 4935bfb..628d080 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -35,8 +35,8 @@ struct object; typedef void hash_init_f(int ac, char * const *av); typedef void hash_start_f(void); typedef void hash_prep_f(struct worker *); -typedef struct objhead * - hash_lookup_f(struct worker *wrk, struct objhead *nobj); +typedef struct objhead *hash_lookup_f(struct worker *wrk, const void *digest, + struct objhead **nobj); typedef int hash_deref_f(struct objhead *obj); struct hash_slinger { From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] 4bbbc69 In theory diag_bitmap could change between (non-)assignment to t0 and use of t0. Always initialize t0 to something. Message-ID: commit 4bbbc69324104deaac64d3845237f0c95ec625f4 Author: Poul-Henning Kamp Date: Wed Mar 14 21:24:42 2012 +0000 In theory diag_bitmap could change between (non-)assignment to t0 and use of t0. Always initialize t0 to something. Correctly spotted by: FlexeLint diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c index 3bbbfbb..993bb5e 100644 --- a/bin/varnishd/cache/cache_lck.c +++ b/bin/varnishd/cache/cache_lck.c @@ -64,7 +64,7 @@ Lck__Lock(struct lock *lck, const char *p, const char *f, int l) { struct ilck *ilck; int r; - double t0, t; + double t0 = 0, t; CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC); if (!(cache_param->diag_bitmap & 0x98)) { From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] dbad89e Now that saint-mode uses the digest, we can remove the temporary objcore assignment. Message-ID: commit dbad89eb134506e5c4f8c521d40d685b8388579b Author: Poul-Henning Kamp Date: Wed Mar 14 21:25:25 2012 +0000 Now that saint-mode uses the digest, we can remove the temporary objcore assignment. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index dcb87dd..92e74da 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -271,9 +271,6 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp) if (threshold == 0 || VTAILQ_EMPTY(&backend->troublelist)) return (1); - if (sp->req->objcore == NULL) - return (1); - now = sp->t_req; old = NULL; diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index d154717..749a8f3 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -377,7 +377,6 @@ HSH_Lookup(struct sess *sp) */ AZ(req->objcore); - req->objcore = grace_oc; /* XXX: Hack-ish */ if (oc == NULL /* We found no live object */ && grace_oc != NULL /* There is a grace candidate */ && (busy_oc != NULL /* Somebody else is already busy */ @@ -387,7 +386,6 @@ HSH_Lookup(struct sess *sp) CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); oc = grace_oc; } - req->objcore = NULL; if (oc != NULL && !req->hash_always_miss) { o = oc_getobj(&wrk->stats, oc); From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] ca3099c Lessen the load on oh->mtx a bit. Message-ID: commit ca3099c3b6c0e52ebf37054163300a3bdaca554d Author: Poul-Henning Kamp Date: Wed Mar 14 23:21:28 2012 +0000 Lessen the load on oh->mtx a bit. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 08e3405..daab7da 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -383,6 +383,7 @@ struct objcore { #define OC_F_LURK (3<<6) /* Ban-lurker-color */ #define OC_F_COMPLETE (1<<8) #define OC_F_FAILED (1<<9) +#define OC_F_NOTYET (1<<10) unsigned timer_idx; VTAILQ_ENTRY(objcore) list; VTAILQ_ENTRY(objcore) lru_list; diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 749a8f3..06d7e50 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -326,6 +326,9 @@ HSH_Lookup(struct sess *sp) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); assert(oc->objhead == oh); + if (oc->flags & OC_F_NOTYET) + continue; + if (oc->flags & OC_F_BUSY) { CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC); if (req->hash_ignore_busy) @@ -370,10 +373,7 @@ HSH_Lookup(struct sess *sp) * If we have seen a busy object or the backend is unhealthy, and * we have an object in grace, use it, if req.grace is also * satisified. - * XXX: Interesting footnote: The busy object might be for a - * XXX: different "Vary:" than we sought. We have no way of knowing - * XXX: this until the object is unbusy'ed, so in practice we - * XXX: serialize fetch of all Vary's if grace is possible. + * XXX: VDI_Healty() call with oh->mtx is not so cool. */ AZ(req->objcore); @@ -388,17 +388,16 @@ HSH_Lookup(struct sess *sp) } if (oc != NULL && !req->hash_always_miss) { - o = oc_getobj(&wrk->stats, oc); - CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - assert(oc->objhead == oh); - /* We found an object we like */ - oc->refcnt++; - if (o->hits < INT_MAX) - o->hits++; assert(oh->refcnt > 1); + assert(oc->objhead == oh); + oc->refcnt++; Lck_Unlock(&oh->mtx); assert(hash->deref(oh)); + o = oc_getobj(&wrk->stats, oc); + CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); + if (o->hits < INT_MAX) + o->hits++; return (oc); } @@ -428,11 +427,16 @@ HSH_Lookup(struct sess *sp) return (NULL); } - /* Insert (precreated) objcore in objecthead */ + /* Insert (precreated) objcore in objecthead and release mutex */ oc = wrk->nobjcore; wrk->nobjcore = NULL; AN(oc->flags & OC_F_BUSY); + oc->flags |= OC_F_NOTYET; oc->refcnt = 1; + oc->objhead = oh; + VTAILQ_INSERT_TAIL(&oh->objcs, oc, list); + /* NB: do not deref objhead the new object inherits our reference */ + Lck_Unlock(&oh->mtx); AZ(req->busyobj); req->busyobj = VBO_GetBusyObj(wrk); @@ -445,14 +449,7 @@ HSH_Lookup(struct sess *sp) req->busyobj->vary = NULL; oc->busyobj = req->busyobj; - /* - * Busy objects go on the tail, so they will not trip up searches. - * HSH_Unbusy() will move them to the front. - */ - VTAILQ_INSERT_TAIL(&oh->objcs, oc, list); - oc->objhead = oh; - /* NB: do not deref objhead the new object inherits our reference */ - Lck_Unlock(&oh->mtx); + oc->flags &= ~OC_F_NOTYET; return (oc); } From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] bbc0bc4 Start preparing hash-lookup for streaming objects Message-ID: commit bbc0bc445732fd451286f5bcce172fe7ffd4aa3c Author: Poul-Henning Kamp Date: Thu Mar 15 08:11:20 2012 +0000 Start preparing hash-lookup for streaming objects diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 4ad05ff..c9498b6 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -654,6 +654,8 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) AZ(HSH_Deref(&wrk->stats, req->objcore, NULL)); req->objcore = NULL; } + assert(bo->refcount == 2); + VBO_DerefBusyObj(wrk, &bo); VBO_DerefBusyObj(wrk, &req->busyobj); req->director = NULL; req->storage_hint = NULL; @@ -906,8 +908,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) bo->fetch_task.func = FetchBody; bo->fetch_task.priv = bo; - /* Gain a reference for FetchBody() */ - VBO_RefBusyObj(bo); + assert(bo->refcount == 2); /* one for each thread */ if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE)) FetchBody(wrk, bo); @@ -1264,6 +1265,7 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req) req->busyobj = VBO_GetBusyObj(wrk); req->busyobj->vsl->wid = sp->vsl_id; + req->busyobj->refcount = 2; http_Setup(req->busyobj->bereq, req->busyobj->ws, req->busyobj->vsl); http_FilterReq(sp, HTTPH_R_PASS); diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 06d7e50..60bf207 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -58,7 +58,9 @@ #include "cache.h" + #include "hash/hash_slinger.h" +#include "vmb.h" #include "vsha256.h" static const struct hash_slinger *hash; @@ -326,15 +328,17 @@ HSH_Lookup(struct sess *sp) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); assert(oc->objhead == oh); - if (oc->flags & OC_F_NOTYET) + /* We ignore failed oc's, they're no use, ever. */ + if (oc->flags & OC_F_FAILED) continue; - if (oc->flags & OC_F_BUSY) { + if (oc->flags & (OC_F_BUSY | OC_F_NOTYET)) { CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC); if (req->hash_ignore_busy) continue; - if (oc->busyobj->vary != NULL && + if (!(oc->flags & OC_F_NOTYET) && + oc->busyobj->vary != NULL && !VRY_Match(req, oc->busyobj->vary)) continue; @@ -440,15 +444,17 @@ HSH_Lookup(struct sess *sp) AZ(req->busyobj); req->busyobj = VBO_GetBusyObj(wrk); + oc->busyobj = req->busyobj; req->busyobj->vsl->wid = sp->vsl_id; + req->busyobj->refcount = 2; /* One for headers, one for body*/ VRY_Validate(req->vary_b); if (req->vary_l != NULL) req->busyobj->vary = req->vary_b; else req->busyobj->vary = NULL; - oc->busyobj = req->busyobj; + VMB(); oc->flags &= ~OC_F_NOTYET; return (oc); } From phk at FreeBSD.org Thu Dec 18 09:27:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:41 +0100 Subject: [experimental-ims] d734511 Update comment to match reality Message-ID: commit d7345115a31ddf7e80ded4b2b914ae17c0d2496a Author: Poul-Henning Kamp Date: Mon Mar 19 09:25:55 2012 +0000 Update comment to match reality diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c index a7bb31c..138efda 100644 --- a/bin/varnishd/cache/cache_vary.c +++ b/bin/varnishd/cache/cache_vary.c @@ -49,7 +49,9 @@ * '\0' / *
> Only present if length != 0xffff * } - * '\0' + * 0xff, \ Length field + * 0xff, / + * '\0' > Terminator */ #include "config.h" From phk at FreeBSD.org Thu Dec 18 09:27:42 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] 20d561d Redo the oc->NOTYET stuff after pondering. Message-ID: commit 20d561d980c291aed95c61d58b87183721760c4d Author: Poul-Henning Kamp Date: Mon Mar 19 09:26:16 2012 +0000 Redo the oc->NOTYET stuff after pondering. OC's will only have two states: BUSY or not BUSY. A BUSY OC may not yet have a busyobj, in which case we assume that it vary-matches. Replace the busy_oc pointer with flag, all we are about is the existence of a busy OC, we're not going to actually touch it. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index daab7da..17f4f4d 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -381,9 +381,6 @@ struct objcore { #define OC_F_LRUDONTMOVE (1<<4) #define OC_F_PRIV (1<<5) /* Stevedore private flag */ #define OC_F_LURK (3<<6) /* Ban-lurker-color */ -#define OC_F_COMPLETE (1<<8) -#define OC_F_FAILED (1<<9) -#define OC_F_NOTYET (1<<10) unsigned timer_idx; VTAILQ_ENTRY(objcore) list; VTAILQ_ENTRY(objcore) lru_list; diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index c9498b6..30a797b 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -1136,9 +1136,6 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) return (0); } - /* For now... */ - AN(oc->flags & (OC_F_COMPLETE|OC_F_FAILED)); - o = oc_getobj(&wrk->stats, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); req->obj = o; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index d9b519d..5406b2f 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -636,13 +636,6 @@ FetchBody(struct worker *wrk, void *priv) http_Teardown(bo->bereq); http_Teardown(bo->beresp); - if (obj->objcore != NULL) { - /* pass has no objcore */ - /* XXX: lock protection ?? */ - obj->objcore->flags |= - (bo->state == BOS_FAILED ? OC_F_FAILED : OC_F_COMPLETE); - } - if (bo->state == BOS_FAILED) { wrk->stats.fetch_failed++; VDI_CloseFd(&bo->vbc); diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 60bf207..978277a 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -266,7 +266,6 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc) oc->refcnt = 1; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AZ(oc->flags & OC_F_BUSY); - oc->flags |= OC_F_COMPLETE; VTAILQ_INSERT_HEAD(&oh->objcs, oc, list); /* NB: do not deref objhead the new object inherits our reference */ @@ -285,10 +284,11 @@ HSH_Lookup(struct sess *sp) struct worker *wrk; struct objhead *oh; struct objcore *oc; - struct objcore *busy_oc, *grace_oc; + struct objcore *grace_oc; struct object *o; struct req *req; double grace_ttl; + int busy_found; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); wrk = sp->wrk; @@ -319,7 +319,7 @@ HSH_Lookup(struct sess *sp) CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); Lck_Lock(&oh->mtx); assert(oh->refcnt > 0); - busy_oc = NULL; + busy_found = 0; grace_oc = NULL; grace_ttl = NAN; VTAILQ_FOREACH(oc, &oh->objcs, list) { @@ -328,21 +328,17 @@ HSH_Lookup(struct sess *sp) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); assert(oc->objhead == oh); - /* We ignore failed oc's, they're no use, ever. */ - if (oc->flags & OC_F_FAILED) - continue; - - if (oc->flags & (OC_F_BUSY | OC_F_NOTYET)) { - CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC); + if (oc->flags & OC_F_BUSY) { + CHECK_OBJ_ORNULL(oc->busyobj, BUSYOBJ_MAGIC); if (req->hash_ignore_busy) continue; - if (!(oc->flags & OC_F_NOTYET) && + if (oc->busyobj != NULL && oc->busyobj->vary != NULL && !VRY_Match(req, oc->busyobj->vary)) continue; - busy_oc = oc; + busy_found = 1; continue; } @@ -383,7 +379,7 @@ HSH_Lookup(struct sess *sp) AZ(req->objcore); if (oc == NULL /* We found no live object */ && grace_oc != NULL /* There is a grace candidate */ - && (busy_oc != NULL /* Somebody else is already busy */ + && (busy_found /* Somebody else is already busy */ || !VDI_Healthy(req->director, sp))) { /* Or it is impossible to fetch */ o = oc_getobj(&wrk->stats, grace_oc); @@ -405,7 +401,7 @@ HSH_Lookup(struct sess *sp) return (oc); } - if (busy_oc != NULL) { + if (busy_found) { /* There are one or more busy objects, wait for them */ if (req->esi_level == 0) { CHECK_OBJ_NOTNULL(wrk->nwaitinglist, @@ -435,7 +431,6 @@ HSH_Lookup(struct sess *sp) oc = wrk->nobjcore; wrk->nobjcore = NULL; AN(oc->flags & OC_F_BUSY); - oc->flags |= OC_F_NOTYET; oc->refcnt = 1; oc->objhead = oh; VTAILQ_INSERT_TAIL(&oh->objcs, oc, list); @@ -444,7 +439,6 @@ HSH_Lookup(struct sess *sp) AZ(req->busyobj); req->busyobj = VBO_GetBusyObj(wrk); - oc->busyobj = req->busyobj; req->busyobj->vsl->wid = sp->vsl_id; req->busyobj->refcount = 2; /* One for headers, one for body*/ @@ -455,7 +449,7 @@ HSH_Lookup(struct sess *sp) req->busyobj->vary = NULL; VMB(); - oc->flags &= ~OC_F_NOTYET; + oc->busyobj = req->busyobj; return (oc); } From phk at FreeBSD.org Thu Dec 18 09:27:42 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] bd0246a Edge closer to the new worldorder Message-ID: commit bd0246a0de438739a07bb1dca7f7badb67bc3bb5 Author: Poul-Henning Kamp Date: Mon Mar 19 10:14:41 2012 +0000 Edge closer to the new worldorder diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 17f4f4d..f337f66 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -1081,21 +1081,6 @@ Tadd(txt *t, const char *p, int l) } } -static inline void -AssertOCBusy(const struct objcore *oc) -{ - AN(oc); - AN (oc->flags & OC_F_BUSY); - AN(oc->busyobj); -} - -static inline void -AssertObjCorePassOrBusy(const struct objcore *oc) -{ - if (oc != NULL) - AN (oc->flags & OC_F_BUSY); -} - /* * We want to cache the most recent timestamp in wrk->lastused to avoid * extra timestamps in cache_pool.c. Hide this detail with a macro diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 30a797b..8b12807 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -215,7 +215,6 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) if (bo != NULL) { CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AN(bo->do_stream); - AssertObjCorePassOrBusy(req->obj->objcore); } req->res_mode = 0; @@ -635,7 +634,6 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) switch (req->handling) { case VCL_RET_DELIVER: - AssertObjCorePassOrBusy(req->objcore); sp->step = STP_PREPFETCH; return (0); default: @@ -875,8 +873,6 @@ cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req) RFC2616_Do_Cond(sp)) bo->do_stream = 0; - AssertObjCorePassOrBusy(req->obj->objcore); - sp->step = STP_FETCHBODY; return (0); } @@ -1136,6 +1132,9 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) return (0); } + /* We are not prepared to do streaming yet */ + XXXAZ(req->busyobj); + o = oc_getobj(&wrk->stats, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); req->obj = o; diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index 9d7136c..39ded77 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -224,7 +224,6 @@ EXP_Insert(struct object *o) CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); oc = o->objcore; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - AssertOCBusy(oc); HSH_Ref(oc); assert(o->exp.entered != 0 && !isnan(o->exp.entered)); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 5406b2f..cab6273 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -40,6 +40,7 @@ #include "cache_backend.h" #include "vcli_priv.h" #include "vct.h" +#include "vmb.h" #include "vtcp.h" static unsigned fetchfrag; @@ -570,8 +571,6 @@ FetchBody(struct worker *wrk, void *priv) if (bo->vfp == NULL) bo->vfp = &vfp_nop; - AssertObjCorePassOrBusy(obj->objcore); - AZ(bo->vgz_rx); AZ(VTAILQ_FIRST(&obj->store)); @@ -681,6 +680,8 @@ FetchBody(struct worker *wrk, void *priv) AN(obj->objcore->ban); AZ(obj->ws_o->overflow); HSH_Unbusy(&wrk->stats, obj->objcore); + obj->objcore->busyobj = NULL; + VMB(); } /* XXX: Atomic assignment, needs volatile/membar ? */ diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 978277a..09ee9f4 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -551,9 +551,7 @@ HSH_Purge(const struct sess *sp, struct objhead *oh, double ttl, double grace) /*--------------------------------------------------------------------- - * Kill a busy object we don't need anyway. - * There may be sessions on the waiting list, so we cannot just blow - * it out of the water. + * Kill a busy object we don't need and can't use. */ void @@ -563,13 +561,14 @@ HSH_Drop(struct worker *wrk, struct object **oo) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AN(oo); CHECK_OBJ_NOTNULL(*oo, OBJECT_MAGIC); - AssertObjCorePassOrBusy((*oo)->objcore); (*oo)->exp.ttl = -1.; - if ((*oo)->objcore != NULL) /* Pass has no objcore */ - HSH_Unbusy(&wrk->stats, (*oo)->objcore); - (void)HSH_Deref(&wrk->stats, NULL, oo); + AZ(HSH_Deref(&wrk->stats, NULL, oo)); } +/*--------------------------------------------------------------------- + * Unbusy an objcore when the object is completely fetched. + */ + void HSH_Unbusy(struct dstat *ds, struct objcore *oc) { @@ -579,9 +578,8 @@ HSH_Unbusy(struct dstat *ds, struct objcore *oc) oh = oc->objhead; CHECK_OBJ(oh, OBJHEAD_MAGIC); - AssertOCBusy(oc); + AN(oc->flags & OC_F_BUSY); AN(oc->ban); - assert(oc->refcnt > 0); assert(oh->refcnt > 0); /* XXX: pretouch neighbors on oh->objcs to prevent page-on under mtx */ @@ -591,13 +589,15 @@ HSH_Unbusy(struct dstat *ds, struct objcore *oc) VTAILQ_REMOVE(&oh->objcs, oc, list); VTAILQ_INSERT_HEAD(&oh->objcs, oc, list); oc->flags &= ~OC_F_BUSY; - oc->busyobj = NULL; if (oh->waitinglist != NULL) hsh_rush(ds, oh); - AN(oc->ban); Lck_Unlock(&oh->mtx); } +/*--------------------------------------------------------------------- + * Gain a reference on an objcore + */ + void HSH_Ref(struct objcore *oc) { From phk at FreeBSD.org Thu Dec 18 09:27:42 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] c40a7d0 Unbusy the objectcore when we start fetching the body. Message-ID: commit c40a7d0ce9ed192e1f1d482172dbe1f1c1dfe7db Author: Poul-Henning Kamp Date: Mon Mar 19 13:36:36 2012 +0000 Unbusy the objectcore when we start fetching the body. Don't deref the objcore until the busyobj dies. Apply two workarounds for now: Don't return non-busy object with a busyobj in HSH_Lookup(), and trade the busyobj ref for an objcore ref in cnt_fetchbody() Getting closer... diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 8de928a..79b55ed 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -38,6 +38,8 @@ #include "cache.h" +#include "hash/hash_slinger.h" + static struct mempool *vbopool; /*-------------------------------------------------------------------- @@ -157,6 +159,7 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo) bo = *pbo; *pbo = NULL; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + CHECK_OBJ_ORNULL(bo->fetch_obj, OBJECT_MAGIC); Lck_Lock(&bo->mtx); assert(bo->refcount > 0); r = --bo->refcount; @@ -167,6 +170,11 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo) VSL_Flush(bo->vsl, 0); + if (bo->fetch_obj != NULL && bo->fetch_obj->objcore != NULL) { + AN(wrk); + (void)HSH_Deref(&wrk->stats, NULL, &bo->fetch_obj); + } + memset(&bo->refcount, 0, sizeof *bo - offsetof(struct busyobj, refcount)); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 8b12807..a5b058b 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -906,6 +906,13 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) assert(bo->refcount == 2); /* one for each thread */ + if (req->obj->objcore != NULL) { + EXP_Insert(req->obj); + AN(req->obj->objcore->ban); + AZ(req->obj->ws_o->overflow); + HSH_Unbusy(&wrk->stats, req->obj->objcore); + } + if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE)) FetchBody(wrk, bo); @@ -923,6 +930,9 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) return (0); } + if (req->obj->objcore != NULL) + HSH_Ref(req->obj->objcore); + VBO_DerefBusyObj(wrk, &req->busyobj); wrk->acct_tmp.fetch++; sp->step = STP_PREPRESP; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index cab6273..d71a6cb 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -35,8 +35,6 @@ #include "cache.h" -#include "hash/hash_slinger.h" - #include "cache_backend.h" #include "vcli_priv.h" #include "vct.h" @@ -557,7 +555,6 @@ FetchBody(struct worker *wrk, void *priv) CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC); assert(bo->state == BOS_INVALID); - bo->state = BOS_FINISHED; /* * XXX: The busyobj needs a dstat, but it is not obvious which one @@ -638,8 +635,8 @@ FetchBody(struct worker *wrk, void *priv) if (bo->state == BOS_FAILED) { wrk->stats.fetch_failed++; VDI_CloseFd(&bo->vbc); + obj->exp.ttl = -1.; obj->len = 0; - HSH_Drop(wrk, &obj); } else { assert(bo->state == BOS_FETCHING); @@ -669,24 +666,19 @@ FetchBody(struct worker *wrk, void *priv) "Content-Length: %zd", obj->len); } - if (cls) VDI_CloseFd(&bo->vbc); else VDI_RecycleFd(&bo->vbc); - if (obj->objcore != NULL) { - EXP_Insert(obj); - AN(obj->objcore->ban); - AZ(obj->ws_o->overflow); - HSH_Unbusy(&wrk->stats, obj->objcore); - obj->objcore->busyobj = NULL; - VMB(); - } /* XXX: Atomic assignment, needs volatile/membar ? */ bo->state = BOS_FINISHED; - + } + if (obj->objcore != NULL) { + VMB(); + obj->objcore->busyobj = NULL; + VMB(); } bo->stats = NULL; VBO_DerefBusyObj(wrk, &bo); diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 09ee9f4..f9217eb 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -328,7 +328,7 @@ HSH_Lookup(struct sess *sp) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); assert(oc->objhead == oh); - if (oc->flags & OC_F_BUSY) { + if (oc->flags & OC_F_BUSY || oc->busyobj != NULL) { CHECK_OBJ_ORNULL(oc->busyobj, BUSYOBJ_MAGIC); if (req->hash_ignore_busy) continue; @@ -379,7 +379,7 @@ HSH_Lookup(struct sess *sp) AZ(req->objcore); if (oc == NULL /* We found no live object */ && grace_oc != NULL /* There is a grace candidate */ - && (busy_found /* Somebody else is already busy */ + && (busy_found /* Somebody else is already busy */ || !VDI_Healthy(req->director, sp))) { /* Or it is impossible to fetch */ o = oc_getobj(&wrk->stats, grace_oc); @@ -431,7 +431,7 @@ HSH_Lookup(struct sess *sp) oc = wrk->nobjcore; wrk->nobjcore = NULL; AN(oc->flags & OC_F_BUSY); - oc->refcnt = 1; + oc->refcnt = 1; /* Owned by busyobj */ oc->objhead = oh; VTAILQ_INSERT_TAIL(&oh->objcs, oc, list); /* NB: do not deref objhead the new object inherits our reference */ @@ -440,7 +440,7 @@ HSH_Lookup(struct sess *sp) AZ(req->busyobj); req->busyobj = VBO_GetBusyObj(wrk); req->busyobj->vsl->wid = sp->vsl_id; - req->busyobj->refcount = 2; /* One for headers, one for body*/ + req->busyobj->refcount = 2; /* One for req, one for FetchBody */ VRY_Validate(req->vary_b); if (req->vary_l != NULL) From phk at FreeBSD.org Thu Dec 18 09:27:42 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] ed645d2 Remove busyobj from objcore under oh-mtx Message-ID: commit ed645d241fd54c65bc07acab75ba88efdb4281b9 Author: Poul-Henning Kamp Date: Tue Mar 20 07:17:21 2012 +0000 Remove busyobj from objcore under oh-mtx diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index d71a6cb..eb7b855 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -35,10 +35,11 @@ #include "cache.h" +#include "hash/hash_slinger.h" + #include "cache_backend.h" #include "vcli_priv.h" #include "vct.h" -#include "vmb.h" #include "vtcp.h" static unsigned fetchfrag; @@ -675,11 +676,8 @@ FetchBody(struct worker *wrk, void *priv) /* XXX: Atomic assignment, needs volatile/membar ? */ bo->state = BOS_FINISHED; } - if (obj->objcore != NULL) { - VMB(); - obj->objcore->busyobj = NULL; - VMB(); - } + if (obj->objcore != NULL) + HSH_Complete(obj->objcore); bo->stats = NULL; VBO_DerefBusyObj(wrk, &bo); } diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index f9217eb..e746813 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -566,6 +566,24 @@ HSH_Drop(struct worker *wrk, struct object **oo) } /*--------------------------------------------------------------------- + * Remove the busyobj from an objcore + */ + +void +HSH_Complete(struct objcore *oc) +{ + struct objhead *oh; + + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + oh = oc->objhead; + CHECK_OBJ(oh, OBJHEAD_MAGIC); + + Lck_Lock(&oh->mtx); + oc->busyobj = NULL; + Lck_Unlock(&oh->mtx); +} + +/*--------------------------------------------------------------------- * Unbusy an objcore when the object is completely fetched. */ diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index 628d080..17d67f9 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -94,6 +94,7 @@ struct objhead { }; void HSH_Unbusy(struct dstat *, struct objcore *); +void HSH_Complete(struct objcore *oc); void HSH_DeleteObjHead(struct dstat *, struct objhead *oh); int HSH_Deref(struct dstat *, struct objcore *oc, struct object **o); #endif /* VARNISH_CACHE_CHILD */ From phk at FreeBSD.org Thu Dec 18 09:27:42 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] 4edc10c The busyobj refcount must be protected by the oh->mtx just like the oc->busyobj pointer. Message-ID: commit 4edc10c5d460d38c5a2cecffa572e0001b180ebb Author: Poul-Henning Kamp Date: Tue Mar 20 10:56:04 2012 +0000 The busyobj refcount must be protected by the oh->mtx just like the oc->busyobj pointer. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index f337f66..9abf2da 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -725,7 +725,6 @@ double BAN_Time(const struct ban *ban); /* cache_busyobj.c */ void VBO_Init(void); struct busyobj *VBO_GetBusyObj(struct worker *wrk); -void VBO_RefBusyObj(struct busyobj *busyobj); void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj); void VBO_Free(struct busyobj **vbo); diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 79b55ed..0489734 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -138,20 +138,10 @@ VBO_GetBusyObj(struct worker *wrk) } void -VBO_RefBusyObj(struct busyobj *bo) -{ - - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - Lck_Lock(&bo->mtx); - assert(bo->refcount > 0); - bo->refcount++; - Lck_Unlock(&bo->mtx); -} - -void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo) { struct busyobj *bo; + struct objcore *oc; unsigned r; CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC); @@ -160,17 +150,28 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo) *pbo = NULL; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_ORNULL(bo->fetch_obj, OBJECT_MAGIC); - Lck_Lock(&bo->mtx); - assert(bo->refcount > 0); - r = --bo->refcount; - Lck_Unlock(&bo->mtx); + if (bo->fetch_obj != NULL && bo->fetch_obj->objcore != NULL) { + oc = bo->fetch_obj->objcore; + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); + Lck_Lock(&oc->objhead->mtx); + assert(bo->refcount > 0); + r = --bo->refcount; + Lck_Unlock(&oc->objhead->mtx); + } else { + oc = NULL; + Lck_Lock(&bo->mtx); + assert(bo->refcount > 0); + r = --bo->refcount; + Lck_Unlock(&bo->mtx); + } if (r) return; VSL_Flush(bo->vsl, 0); - if (bo->fetch_obj != NULL && bo->fetch_obj->objcore != NULL) { + if (oc != NULL) { AN(wrk); (void)HSH_Deref(&wrk->stats, NULL, &bo->fetch_obj); } From phk at FreeBSD.org Thu Dec 18 09:27:42 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] 4d33fab Introduce a couple of local busyobj variables for clarity Message-ID: commit 4d33fabc5e7a5b2820ceaee7c60ee5bc3e8ee656 Author: Poul-Henning Kamp Date: Thu Mar 22 11:16:09 2012 +0000 Introduce a couple of local busyobj variables for clarity diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index a5b058b..6b692f8 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -1261,6 +1261,8 @@ XDOT err_pass [label="ERROR",shape=plaintext] static int cnt_pass(struct sess *sp, struct worker *wrk, struct req *req) { + struct busyobj *bo; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -1270,15 +1272,16 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req) AZ(req->busyobj); req->busyobj = VBO_GetBusyObj(wrk); - req->busyobj->vsl->wid = sp->vsl_id; - req->busyobj->refcount = 2; - http_Setup(req->busyobj->bereq, req->busyobj->ws, req->busyobj->vsl); + bo = req->busyobj; + bo->vsl->wid = sp->vsl_id; + bo->refcount = 2; + http_Setup(bo->bereq, bo->ws, bo->vsl); http_FilterReq(sp, HTTPH_R_PASS); VCL_pass_method(sp); if (req->handling == VCL_RET_ERROR) { - http_Teardown(req->busyobj->bereq); + http_Teardown(bo->bereq); VBO_DerefBusyObj(wrk, &req->busyobj); sp->step = STP_ERROR; return (0); @@ -1317,6 +1320,7 @@ DOT err_pipe [label="ERROR",shape=plaintext] static int cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req) { + struct busyobj *bo; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -1326,8 +1330,9 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req) wrk->acct_tmp.pipe++; req->busyobj = VBO_GetBusyObj(wrk); - req->busyobj->vsl->wid = sp->vsl_id; - http_Setup(req->busyobj->bereq, req->busyobj->ws, req->busyobj->vsl); + bo = req->busyobj; + bo->vsl->wid = sp->vsl_id; + http_Setup(bo->bereq, bo->ws, bo->vsl); http_FilterReq(sp, 0); VCL_pipe_method(sp); @@ -1338,7 +1343,7 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req) PipeSession(sp); assert(WRW_IsReleased(wrk)); - http_Teardown(req->busyobj->bereq); + http_Teardown(bo->bereq); VBO_DerefBusyObj(wrk, &req->busyobj); sp->step = STP_DONE; return (0); From phk at FreeBSD.org Thu Dec 18 09:27:42 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] ab58597 Add a separate table for the http VSL headers Message-ID: commit ab585970963ddf2c1bfa0882d1f4ac93da1e6d43 Author: Poul-Henning Kamp Date: Thu Mar 22 13:24:00 2012 +0000 Add a separate table for the http VSL headers diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 21ca1df..885e708 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -53,26 +53,17 @@ SLTM(Length) SLTM(FetchError) -SLTM(RxRequest) -SLTM(RxResponse) -SLTM(RxStatus) -SLTM(RxURL) -SLTM(RxProtocol) -SLTM(RxHeader) +#define SLTH(aa) SLTM(Rx##aa) +#include "tbl/vsl_tags_http.h" +#undef SLTH -SLTM(TxRequest) -SLTM(TxResponse) -SLTM(TxStatus) -SLTM(TxURL) -SLTM(TxProtocol) -SLTM(TxHeader) +#define SLTH(aa) SLTM(Tx##aa) +#include "tbl/vsl_tags_http.h" +#undef SLTH -SLTM(ObjRequest) -SLTM(ObjResponse) -SLTM(ObjStatus) -SLTM(ObjURL) -SLTM(ObjProtocol) -SLTM(ObjHeader) +#define SLTH(aa) SLTM(Obj##aa) +#include "tbl/vsl_tags_http.h" +#undef SLTH SLTM(LostHeader) diff --git a/include/tbl/vsl_tags_http.h b/include/tbl/vsl_tags_http.h new file mode 100644 index 0000000..5e8544f --- /dev/null +++ b/include/tbl/vsl_tags_http.h @@ -0,0 +1,38 @@ +/*- + * Copyright (c) 2012 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Define the VSL tags for HTTP protocol messages + * + */ + +SLTH(Request) +SLTH(Response) +SLTH(Status) +SLTH(URL) +SLTH(Protocol) +SLTH(Header) +SLTH(Lost) From phk at FreeBSD.org Thu Dec 18 09:27:42 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] e4996ef Link the HTTP VSLs to the index in the http structure Message-ID: commit e4996efa99d8a0407d6761fce47ca08968d4cb3d Author: Poul-Henning Kamp Date: Thu Mar 22 14:05:30 2012 +0000 Link the HTTP VSLs to the index in the http structure diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 885e708..ecf0a20 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -53,15 +53,15 @@ SLTM(Length) SLTM(FetchError) -#define SLTH(aa) SLTM(Rx##aa) +#define SLTH(aa, bb) SLTM(Rx##aa) #include "tbl/vsl_tags_http.h" #undef SLTH -#define SLTH(aa) SLTM(Tx##aa) +#define SLTH(aa, bb) SLTM(Tx##aa) #include "tbl/vsl_tags_http.h" #undef SLTH -#define SLTH(aa) SLTM(Obj##aa) +#define SLTH(aa, bb) SLTM(Obj##aa) #include "tbl/vsl_tags_http.h" #undef SLTH diff --git a/include/tbl/vsl_tags_http.h b/include/tbl/vsl_tags_http.h index 5e8544f..542af07 100644 --- a/include/tbl/vsl_tags_http.h +++ b/include/tbl/vsl_tags_http.h @@ -27,12 +27,15 @@ * * Define the VSL tags for HTTP protocol messages * + * The order of this table is not random, do not resort. + * In particular, the FIRST and LOST entries must be last, in that order. + * */ -SLTH(Request) -SLTH(Response) -SLTH(Status) -SLTH(URL) -SLTH(Protocol) -SLTH(Header) -SLTH(Lost) +SLTH(Request, HTTP_HDR_REQ) +SLTH(URL, HTTP_HDR_URL) +SLTH(Protocol, HTTP_HDR_PROTO) +SLTH(Status, HTTP_HDR_STATUS) +SLTH(Response, HTTP_HDR_RESPONSE) +SLTH(Header, HTTP_HDR_FIRST) +SLTH(Lost, HTTP_HDR_LOST) From phk at FreeBSD.org Thu Dec 18 09:27:42 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] 61367de Simplify the CPP magic involved in HTTP headers, now that we can rely on everybody agreeing about the order of these fields. Message-ID: commit 61367de4084f4c806944c989c021535bb5cf7e16 Author: Poul-Henning Kamp Date: Thu Mar 22 14:06:52 2012 +0000 Simplify the CPP magic involved in HTTP headers, now that we can rely on everybody agreeing about the order of these fields. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 9abf2da..ec385c5 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -82,13 +82,9 @@ body_status(enum body_status e) enum { /* Fields from the first line of HTTP proto */ - HTTP_HDR_REQ, - HTTP_HDR_URL, - HTTP_HDR_PROTO, - HTTP_HDR_STATUS, - HTTP_HDR_RESPONSE, - /* HTTP header lines */ - HTTP_HDR_FIRST, +#define SLTH(aa, bb) bb, +#include "tbl/vsl_tags_http.h" +#undef SLTH }; struct SHA256Context; diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 4d2a9fa..bcde416 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -41,24 +41,11 @@ #include "tbl/http_headers.h" #undef HTTPH -/*lint -save -e773 not () */ -#define LOGMTX2(ax, bx, cx) [bx] = SLT_##ax##cx - -#define LOGMTX1(ax) { \ - LOGMTX2(ax, HTTP_HDR_REQ, Request), \ - LOGMTX2(ax, HTTP_HDR_RESPONSE, Response), \ - LOGMTX2(ax, HTTP_HDR_STATUS, Status), \ - LOGMTX2(ax, HTTP_HDR_URL, URL), \ - LOGMTX2(ax, HTTP_HDR_PROTO, Protocol), \ - LOGMTX2(ax, HTTP_HDR_FIRST, Header), \ - } - -static const enum VSL_tag_e logmtx[][HTTP_HDR_FIRST + 1] = { - [HTTP_Rx] = LOGMTX1(Rx), - [HTTP_Tx] = LOGMTX1(Tx), - [HTTP_Obj] = LOGMTX1(Obj) +static const enum VSL_tag_e foo[] = { + [HTTP_Rx] = SLT_RxRequest, + [HTTP_Tx] = SLT_TxRequest, + [HTTP_Obj] = SLT_ObjRequest, }; -/*lint -restore */ static enum VSL_tag_e http2shmlog(const struct http *hp, int t) @@ -69,7 +56,7 @@ http2shmlog(const struct http *hp, int t) t = HTTP_HDR_FIRST; assert(hp->logtag >= HTTP_Rx && hp->logtag <= HTTP_Obj); /*lint !e685*/ assert(t >= HTTP_HDR_REQ && t <= HTTP_HDR_FIRST); - return (logmtx[hp->logtag][t]); + return ((enum VSL_tag_e)(foo[hp->logtag] + t)); } static void From phk at FreeBSD.org Thu Dec 18 09:27:42 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] f177a35 Rework the way we log HTTP headers: Rather than the Rx/Tx/Obj, tag them with Req/Resp/Bereq/Beresp/Obj instead. Message-ID: commit f177a357f792e116f9cc3ba4c64dfc5df3735ae5 Author: Poul-Henning Kamp Date: Thu Mar 22 14:33:34 2012 +0000 Rework the way we log HTTP headers: Rather than the Rx/Tx/Obj, tag them with Req/Resp/Bereq/Beresp/Obj instead. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index ec385c5..0bb75df 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -151,9 +151,11 @@ struct ws { */ enum httpwhence { - HTTP_Rx = 1, - HTTP_Tx = 2, - HTTP_Obj = 3 + HTTP_Req = 1, + HTTP_Resp, + HTTP_Bereq, + HTTP_Beresp, + HTTP_Obj }; /* NB: remember to update http_Copy() if you add fields */ @@ -810,7 +812,7 @@ void http_PrintfHeader(struct http *to, const char *fmt, ...) void http_SetHeader(struct http *to, const char *hdr); void http_SetH(const struct http *to, unsigned n, const char *fm); void http_ForceGet(const struct http *to); -void http_Setup(struct http *ht, struct ws *ws, struct vsl_log *); +void HTTP_Setup(struct http *, struct ws *, struct vsl_log *, enum httpwhence); void http_Teardown(struct http *ht); int http_GetHdr(const struct http *hp, const char *hdr, char **ptr); int http_GetHdrData(const struct http *hp, const char *hdr, diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 6b692f8..7104e0f 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -275,7 +275,7 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) req->obj->last_lru = req->t_resp; req->obj->last_use = req->t_resp; /* XXX: locking ? */ } - http_Setup(req->resp, req->ws, req->vsl); + HTTP_Setup(req->resp, req->ws, req->vsl, HTTP_Resp); RES_BuildHttp(sp); VCL_deliver_method(sp); switch (req->handling) { @@ -577,7 +577,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) AZ(bo->should_close); AZ(req->storage_hint); - http_Setup(bo->beresp, bo->ws, bo->vsl); + HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp); need_host_hdr = !http_GetHdr(bo->bereq, H_Host, NULL); @@ -1197,7 +1197,7 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); AZ(req->obj); - http_Setup(bo->bereq, bo->ws, bo->vsl); + HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); http_FilterReq(sp, HTTPH_R_FETCH); http_ForceGet(bo->bereq); if (cache_param->http_gzip_support) { @@ -1275,7 +1275,7 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req) bo = req->busyobj; bo->vsl->wid = sp->vsl_id; bo->refcount = 2; - http_Setup(bo->bereq, bo->ws, bo->vsl); + HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); http_FilterReq(sp, HTTPH_R_PASS); VCL_pass_method(sp); @@ -1332,7 +1332,7 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req) req->busyobj = VBO_GetBusyObj(wrk); bo = req->busyobj; bo->vsl->wid = sp->vsl_id; - http_Setup(bo->bereq, bo->ws, bo->vsl); + HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); http_FilterReq(sp, 0); VCL_pipe_method(sp); @@ -1508,7 +1508,7 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) EXP_Clr(&req->exp); - http_Setup(req->http, req->ws, req->vsl); + HTTP_Setup(req->http, req->ws, req->vsl, HTTP_Req); req->err_code = http_DissectRequest(sp); /* If we could not even parse the request, just close */ diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index bcde416..0380469 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -42,9 +42,11 @@ #undef HTTPH static const enum VSL_tag_e foo[] = { - [HTTP_Rx] = SLT_RxRequest, - [HTTP_Tx] = SLT_TxRequest, - [HTTP_Obj] = SLT_ObjRequest, + [HTTP_Req] = SLT_ReqRequest, + [HTTP_Resp] = SLT_RespRequest, + [HTTP_Bereq] = SLT_BereqRequest, + [HTTP_Beresp] = SLT_BerespRequest, + [HTTP_Obj] = SLT_ObjRequest, }; static enum VSL_tag_e @@ -54,7 +56,7 @@ http2shmlog(const struct http *hp, int t) CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); if (t > HTTP_HDR_FIRST) t = HTTP_HDR_FIRST; - assert(hp->logtag >= HTTP_Rx && hp->logtag <= HTTP_Obj); /*lint !e685*/ + assert(hp->logtag >= HTTP_Req && hp->logtag <= HTTP_Obj); /*lint !e685*/ assert(t >= HTTP_HDR_REQ && t <= HTTP_HDR_FIRST); return ((enum VSL_tag_e)(foo[hp->logtag] + t)); } @@ -118,9 +120,11 @@ HTTP_create(void *p, uint16_t nhttp) /*--------------------------------------------------------------------*/ void -http_Setup(struct http *hp, struct ws *ws, struct vsl_log *vsl) +HTTP_Setup(struct http *hp, struct ws *ws, struct vsl_log *vsl, + enum httpwhence whence) { http_Teardown(hp); + hp->logtag = whence; hp->ws = ws; hp->vsl = vsl; } @@ -659,8 +663,6 @@ http_DissectRequest(const struct sess *sp) hp = sp->req->http; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); - hp->logtag = HTTP_Rx; - retval = http_splitline(hp, htc, HTTP_HDR_REQ, HTTP_HDR_URL, HTTP_HDR_PROTO); if (retval != 0) { @@ -683,7 +685,6 @@ http_DissectResponse(struct http *hp, const struct http_conn *htc) CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); - hp->logtag = HTTP_Rx; if (http_splitline(hp, htc, HTTP_HDR_PROTO, HTTP_HDR_STATUS, HTTP_HDR_RESPONSE)) @@ -843,7 +844,6 @@ http_FilterReq(const struct sess *sp, unsigned how) hp = sp->req->busyobj->bereq; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); - hp->logtag = HTTP_Tx; http_linkh(hp, sp->req->http, HTTP_HDR_REQ); http_linkh(hp, sp->req->http, HTTP_HDR_URL); diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index 1e065ba..8579e37 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -115,7 +115,6 @@ RES_BuildHttp(const struct sess *sp) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); http_ClrHeader(req->resp); - req->resp->logtag = HTTP_Tx; http_FilterResp(req->obj->http, req->resp, 0); if (!(req->res_mode & RES_LEN)) { diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 9f013d2..b71401d 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -256,7 +256,7 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp, WS_Assert(o->ws_o); assert(o->ws_o->e <= (char*)ptr + ltot); - http_Setup(o->http, o->ws_o, bo->vsl); + HTTP_Setup(o->http, o->ws_o, bo->vsl, HTTP_Obj); o->http->magic = HTTP_MAGIC; o->exp = bo->exp; VTAILQ_INIT(&o->store); diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index ecf0a20..1321493 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -53,11 +53,19 @@ SLTM(Length) SLTM(FetchError) -#define SLTH(aa, bb) SLTM(Rx##aa) +#define SLTH(aa, bb) SLTM(Req##aa) #include "tbl/vsl_tags_http.h" #undef SLTH -#define SLTH(aa, bb) SLTM(Tx##aa) +#define SLTH(aa, bb) SLTM(Resp##aa) +#include "tbl/vsl_tags_http.h" +#undef SLTH + +#define SLTH(aa, bb) SLTM(Bereq##aa) +#include "tbl/vsl_tags_http.h" +#undef SLTH + +#define SLTH(aa, bb) SLTM(Beresp##aa) #include "tbl/vsl_tags_http.h" #undef SLTH From phk at FreeBSD.org Thu Dec 18 09:27:42 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] 745c828 Add two fields to the VSL tag definition, for documentation use. Message-ID: commit 745c8287c4c3c3fa1ab2f93a5cc571a622413c16 Author: Poul-Henning Kamp Date: Fri Mar 23 10:43:47 2012 +0000 Add two fields to the VSL tag definition, for documentation use. diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 1321493..a559eed 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -34,69 +34,74 @@ * * XXX: Please add new entries a the end to not break saved log-segments. * XXX: we can resort them when we have a major release. + * + * Arguments: + * Tag-Name + * Short Description (1 line, max ?? chars) + * Long Description (Multi line) */ -SLTM(Debug) -SLTM(Error) -SLTM(CLI) -SLTM(StatSess) -SLTM(ReqEnd) -SLTM(SessionOpen) -SLTM(SessionClose) -SLTM(BackendOpen) -SLTM(BackendXID) -SLTM(BackendReuse) -SLTM(BackendClose) -SLTM(HttpGarbage) -SLTM(Backend) -SLTM(Length) +SLTM(Debug, "", "") +SLTM(Error, "", "") +SLTM(CLI, "", "") +SLTM(StatSess, "", "") +SLTM(ReqEnd, "", "") +SLTM(SessionOpen, "", "") +SLTM(SessionClose, "", "") +SLTM(BackendOpen, "", "") +SLTM(BackendXID, "", "") +SLTM(BackendReuse, "", "") +SLTM(BackendClose, "", "") +SLTM(HttpGarbage, "", "") +SLTM(Backend, "", "") +SLTM(Length, "", "") -SLTM(FetchError) +SLTM(FetchError, "", "") -#define SLTH(aa, bb) SLTM(Req##aa) +#define SLTH(aa, bb) SLTM(Req##aa, "", "") #include "tbl/vsl_tags_http.h" #undef SLTH -#define SLTH(aa, bb) SLTM(Resp##aa) +#define SLTH(aa, bb) SLTM(Resp##aa, "", "") #include "tbl/vsl_tags_http.h" #undef SLTH -#define SLTH(aa, bb) SLTM(Bereq##aa) +#define SLTH(aa, bb) SLTM(Bereq##aa, "", "") #include "tbl/vsl_tags_http.h" #undef SLTH -#define SLTH(aa, bb) SLTM(Beresp##aa) +#define SLTH(aa, bb) SLTM(Beresp##aa, "", "") #include "tbl/vsl_tags_http.h" #undef SLTH -#define SLTH(aa, bb) SLTM(Obj##aa) +#define SLTH(aa, bb) SLTM(Obj##aa, "", "") #include "tbl/vsl_tags_http.h" #undef SLTH -SLTM(LostHeader) +SLTM(LostHeader, "", "") -SLTM(TTL) -SLTM(Fetch_Body) -SLTM(VCL_acl) -SLTM(VCL_call) -SLTM(VCL_trace) -SLTM(VCL_return) -SLTM(VCL_error) -SLTM(ReqStart) -SLTM(Hit) -SLTM(HitPass) -SLTM(ExpBan) -SLTM(ExpKill) -SLTM(WorkThread) +SLTM(TTL, "", "") +SLTM(Fetch_Body, "", "") +SLTM(VCL_acl, "", "") +SLTM(VCL_call, "", "") +SLTM(VCL_trace, "", "") +SLTM(VCL_return, "", "") +SLTM(VCL_error, "", "") +SLTM(ReqStart, "", "") +SLTM(Hit, "", "") +SLTM(HitPass, "", "") +SLTM(ExpBan, "", "") +SLTM(ExpKill, "", "") +SLTM(WorkThread, "", "") -SLTM(ESI_xmlerror) +SLTM(ESI_xmlerror, "", "") -SLTM(Hash) +SLTM(Hash, "", "") -SLTM(Backend_health) +SLTM(Backend_health, "", "") -SLTM(VCL_Debug) -SLTM(VCL_Log) -SLTM(VCL_Error) +SLTM(VCL_Debug, "", "") +SLTM(VCL_Log, "", "") +SLTM(VCL_Error, "", "") -SLTM(Gzip) +SLTM(Gzip, "", "") diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h index 8176cce..6f192e9 100644 --- a/include/vapi/vsl_int.h +++ b/include/vapi/vsl_int.h @@ -73,7 +73,7 @@ */ enum VSL_tag_e { SLT_Bogus = 0, -#define SLTM(foo) SLT_##foo, +#define SLTM(foo,sdesc,ldesc) SLT_##foo, #include "tbl/vsl_tags.h" #undef SLTM SLT_Reserved = 255 diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 8e7001a..cd6034d 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -55,7 +55,7 @@ /*--------------------------------------------------------------------*/ const char *VSL_tags[256] = { -# define SLTM(foo) [SLT_##foo] = #foo, +# define SLTM(foo,sdesc,ldesc) [SLT_##foo] = #foo, # include "tbl/vsl_tags.h" # undef SLTM }; From apj at mutt.dk Thu Dec 18 09:27:42 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] 9fc9e7d Add short descriptions Message-ID: commit 9fc9e7d0d498e435ee27913d7f5714000fc82ad5 Author: Andreas Plesner Jacobsen Date: Mon Mar 26 09:30:35 2012 +0200 Add short descriptions diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index a559eed..86c9cc2 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -43,20 +43,20 @@ SLTM(Debug, "", "") SLTM(Error, "", "") -SLTM(CLI, "", "") -SLTM(StatSess, "", "") -SLTM(ReqEnd, "", "") -SLTM(SessionOpen, "", "") -SLTM(SessionClose, "", "") -SLTM(BackendOpen, "", "") -SLTM(BackendXID, "", "") -SLTM(BackendReuse, "", "") -SLTM(BackendClose, "", "") +SLTM(CLI, "CLI communication", "CLI communication between master and child process.") +SLTM(StatSess, "Session statistics", "") +SLTM(ReqEnd, "Client request end", "") +SLTM(SessionOpen, "Client connection opened", "") +SLTM(SessionClose, "Client connection closed", "") +SLTM(BackendOpen, "Backend connection opened", "") +SLTM(BackendXID, "The unique ID of the backend transaction", "") +SLTM(BackendReuse, "Backend connection reused", "") +SLTM(BackendClose, "Backend connection closed", "") SLTM(HttpGarbage, "", "") -SLTM(Backend, "", "") -SLTM(Length, "", "") +SLTM(Backend, "Backend selected", "") +SLTM(Length, "Size of object body", "") -SLTM(FetchError, "", "") +SLTM(FetchError, "Error while fetching object", "") #define SLTH(aa, bb) SLTM(Req##aa, "", "") #include "tbl/vsl_tags_http.h" @@ -80,28 +80,28 @@ SLTM(FetchError, "", "") SLTM(LostHeader, "", "") -SLTM(TTL, "", "") -SLTM(Fetch_Body, "", "") +SLTM(TTL, "TTL set on object", "") +SLTM(Fetch_Body, "Body fetched from backend", "") SLTM(VCL_acl, "", "") -SLTM(VCL_call, "", "") -SLTM(VCL_trace, "", "") -SLTM(VCL_return, "", "") -SLTM(VCL_error, "", "") -SLTM(ReqStart, "", "") -SLTM(Hit, "", "") -SLTM(HitPass, "", "") -SLTM(ExpBan, "", "") -SLTM(ExpKill, "", "") +SLTM(VCL_call, "VCL method called", "") +SLTM(VCL_trace, "VCL trace data", "") +SLTM(VCL_return, "VCL method return value", "") +SLTM(VCL_error, "Unused", "") +SLTM(ReqStart, "Client request start", "") +SLTM(Hit, "Hit object in cache", "") +SLTM(HitPass, "Hit for pass object in cache", "") +SLTM(ExpBan, "Object evicted due to ban", "") +SLTM(ExpKill, "Object expired", "") SLTM(WorkThread, "", "") -SLTM(ESI_xmlerror, "", "") +SLTM(ESI_xmlerror, "Error while parsing ESI tags", "") -SLTM(Hash, "", "") +SLTM(Hash, "Value added to hash", "") -SLTM(Backend_health, "", "") +SLTM(Backend_health, "Backend health check", "") -SLTM(VCL_Debug, "", "") -SLTM(VCL_Log, "", "") +SLTM(VCL_Debug, "Unused", "") +SLTM(VCL_Log, "Log statement from VCL", "") SLTM(VCL_Error, "", "") -SLTM(Gzip, "", "") +SLTM(Gzip, "G(un)zip performed on object", "") From tfheen at varnish-software.com Thu Dec 18 09:27:42 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] 5a1b770 Adjust varnishncsa and varnishreplay for the new tag names Message-ID: commit 5a1b7707dad90e0a5ff23032c666f8502cf46e5f Author: Tollef Fog Heen Date: Fri Mar 23 15:20:51 2012 +0100 Adjust varnishncsa and varnishreplay for the new tag names diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 9abb9b6..9bdf87f 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -293,7 +293,7 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec, lp->df_h = trimfield(ptr, end); break; - case SLT_TxRequest: + case SLT_BereqRequest: if (!lp->active) break; if (lp->df_m != NULL) { @@ -303,7 +303,7 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec, lp->df_m = trimline(ptr, end); break; - case SLT_TxURL: { + case SLT_BereqURL: { char *qs; if (!lp->active) @@ -322,7 +322,7 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec, break; } - case SLT_TxProtocol: + case SLT_BereqProtocol: if (!lp->active) break; if (lp->df_H != NULL) { @@ -332,7 +332,7 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec, lp->df_H = trimline(ptr, end); break; - case SLT_RxStatus: + case SLT_BerespStatus: if (!lp->active) break; if (lp->df_s != NULL) { @@ -342,7 +342,7 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec, lp->df_s = trimline(ptr, end); break; - case SLT_RxHeader: + case SLT_BerespHeader: if (!lp->active) break; if (isprefix(ptr, "content-length:", end, &next)) @@ -353,7 +353,7 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec, } break; - case SLT_TxHeader: + case SLT_BereqHeader: if (!lp->active) break; split = strchr(ptr, ':'); @@ -412,7 +412,7 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec, lp->df_h = trimfield(ptr, end); break; - case SLT_RxRequest: + case SLT_ReqRequest: if (!lp->active) break; if (lp->df_m != NULL) { @@ -422,7 +422,7 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec, lp->df_m = trimline(ptr, end); break; - case SLT_RxURL: { + case SLT_ReqURL: { char *qs; if (!lp->active) @@ -441,7 +441,7 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec, break; } - case SLT_RxProtocol: + case SLT_ReqProtocol: if (!lp->active) break; if (lp->df_H != NULL) { @@ -451,7 +451,7 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec, lp->df_H = trimline(ptr, end); break; - case SLT_TxStatus: + case SLT_ObjStatus: if (!lp->active) break; if (lp->df_s != NULL) @@ -460,14 +460,14 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec, lp->df_s = trimline(ptr, end); break; - case SLT_TxHeader: - case SLT_RxHeader: + case SLT_ObjHeader: + case SLT_ReqHeader: if (!lp->active) break; split = strchr(ptr, ':'); if (split == NULL) break; - if (tag == SLT_RxHeader && + if (tag == SLT_ReqHeader && isprefix(ptr, "authorization:", end, &next) && isprefix(next, "basic", end, &next)) { free(lp->df_u); @@ -479,7 +479,7 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec, AN(split); h->key = trimline(ptr, split); h->value = trimline(split+1, end); - if (tag == SLT_RxHeader) + if (tag == SLT_ReqHeader) VTAILQ_INSERT_HEAD(&lp->req_headers, h, list); else VTAILQ_INSERT_HEAD(&lp->resp_headers, h, list); diff --git a/bin/varnishreplay/varnishreplay.c b/bin/varnishreplay/varnishreplay.c index 02ad83e..aec3f5f 100644 --- a/bin/varnishreplay/varnishreplay.c +++ b/bin/varnishreplay/varnishreplay.c @@ -515,28 +515,28 @@ replay_thread(void *arg) thread_log(2, 0, "%s(%s)", VSL_tags[tag], msg->ptr); switch (tag) { - case SLT_RxRequest: + case SLT_ReqRequest: if (thr->method != NULL) thr->bogus = 1; else thr->method = trimline(thr, ptr); break; - case SLT_RxURL: + case SLT_ReqURL: if (thr->url != NULL) thr->bogus = 1; else thr->url = trimline(thr, ptr); break; - case SLT_RxProtocol: + case SLT_ReqProtocol: if (thr->proto != NULL) thr->bogus = 1; else thr->proto = trimline(thr, ptr); break; - case SLT_RxHeader: + case SLT_ReqHeader: if (thr->nhdr >= sizeof thr->hdr / sizeof *thr->hdr) { thr->bogus = 1; } else { From tfheen at varnish-software.com Thu Dec 18 09:27:42 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] e37585d Make sure we distribute vsl_tags_http.h too Message-ID: commit e37585df5d519a16f3755282a1b48e41f07bc2f5 Author: Tollef Fog Heen Date: Fri Mar 23 15:18:27 2012 +0100 Make sure we distribute vsl_tags_http.h too diff --git a/include/Makefile.am b/include/Makefile.am index bb00ce6..d40169b 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -17,6 +17,7 @@ pkginclude_HEADERS = \ tbl/vsc_fields.h \ tbl/vsc_f_main.h \ tbl/vsl_tags.h \ + tbl/vsl_tags_http.h \ vapi/vsm.h \ vapi/vsm_int.h \ vapi/vsc.h \ From phk at FreeBSD.org Thu Dec 18 09:27:42 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] c5a1c75 With streaming, we cannot allow a storage chunk to move once we have started to fill it. Add a param to the STV_trim() method to indicate acceptability of move. Message-ID: commit c5a1c750982d6a758715b1ad69ea9373521ea6e7 Author: Poul-Henning Kamp Date: Tue Mar 27 08:09:39 2012 +0000 With streaming, we cannot allow a storage chunk to move once we have started to fill it. Add a param to the STV_trim() method to indicate acceptability of move. The only reason I don't remove STV_trim() entirely, is that a site with gazillions very small objects, will want to disable streaming and use malloc to squeeze as many objects into memory as possible. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 0bb75df..8f9b6d9 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -1013,7 +1013,7 @@ int RFC2616_Do_Cond(const struct sess *sp); struct object *STV_NewObject(struct busyobj *, struct objcore **, const char *hint, unsigned len, uint16_t nhttp); struct storage *STV_alloc(struct busyobj *, size_t size); -void STV_trim(struct storage *st, size_t size); +void STV_trim(struct storage *st, size_t size, int move_ok); void STV_free(struct storage *st); void STV_open(void); void STV_close(void); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index eb7b855..0246f1b 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -195,7 +195,7 @@ vfp_nop_end(struct busyobj *bo) return (0); } if (st->len < st->space) - STV_trim(st, st->len); + STV_trim(st, st->len, 1); return (0); } diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index b71401d..f1cbe94 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -391,13 +391,13 @@ STV_alloc(struct busyobj *bo, size_t size) } void -STV_trim(struct storage *st, size_t size) +STV_trim(struct storage *st, size_t size, int move_ok) { CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); AN(st->stevedore); if (st->stevedore->trim) - st->stevedore->trim(st, size); + st->stevedore->trim(st, size, move_ok); } void diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index 6994314..aae9186 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -42,7 +42,7 @@ struct lru; typedef void storage_init_f(struct stevedore *, int ac, char * const *av); typedef void storage_open_f(const struct stevedore *); typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); -typedef void storage_trim_f(struct storage *, size_t size); +typedef void storage_trim_f(struct storage *, size_t size, int move_ok); typedef void storage_free_f(struct storage *); typedef struct object *storage_allocobj_f(struct stevedore *, struct busyobj *, struct objcore **, unsigned ltot, const struct stv_objsecrets *); diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c index 680b870..3d422fe 100644 --- a/bin/varnishd/storage/storage_file.c +++ b/bin/varnishd/storage/storage_file.c @@ -488,7 +488,7 @@ smf_alloc(struct stevedore *st, size_t size) /*--------------------------------------------------------------------*/ static void -smf_trim(struct storage *s, size_t size) +smf_trim(struct storage *s, size_t size, int move_ok) { struct smf *smf; struct smf_sc *sc; @@ -499,6 +499,10 @@ smf_trim(struct storage *s, size_t size) xxxassert(size > 0); /* XXX: seen */ CAST_OBJ_NOTNULL(smf, s->priv, SMF_MAGIC); assert(size <= smf->size); + + if (!move_ok) + return; /* XXX: trim_smf needs fixed */ + sc = smf->sc; size += (sc->pagesize - 1); size &= ~(sc->pagesize - 1); diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c index 2f34c98..0470093 100644 --- a/bin/varnishd/storage/storage_malloc.c +++ b/bin/varnishd/storage/storage_malloc.c @@ -145,7 +145,7 @@ sma_free(struct storage *s) } static void -sma_trim(struct storage *s, size_t size) +sma_trim(struct storage *s, size_t size, int move_ok) { struct sma_sc *sma_sc; struct sma *sma; @@ -158,6 +158,10 @@ sma_trim(struct storage *s, size_t size) assert(sma->sz == sma->s.space); assert(size < sma->sz); + + if (!move_ok) + return; + delta = sma->sz - size; if (delta < 256) return; diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index 35845f0..d206a86 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -523,19 +523,6 @@ smp_alloc(struct stevedore *st, size_t size) } /*-------------------------------------------------------------------- - * Trim a bite - * XXX: We could trim the last allocation. - */ - -static void -smp_trim(struct storage *ss, size_t size) -{ - - (void)ss; - (void)size; -} - -/*-------------------------------------------------------------------- * We don't track frees of storage, we track the objects which own the * storage and when there are no more objects in in the first segment, * it can be reclaimed. @@ -562,7 +549,6 @@ const struct stevedore smp_stevedore = { .alloc = smp_alloc, .allocobj = smp_allocobj, .free = smp_free, - .trim = smp_trim, }; /*-------------------------------------------------------------------- From phk at FreeBSD.org Thu Dec 18 09:27:42 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] 3a4edf1 Centralize the updating of the fetch_obj->len field. Message-ID: commit 3a4edf1da8924e3bbfe3d9a1f87e34b5e57ea506 Author: Poul-Henning Kamp Date: Tue Mar 27 08:30:51 2012 +0000 Centralize the updating of the fetch_obj->len field. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 8f9b6d9..073eb75 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -216,6 +216,8 @@ struct dstat { /* Fetch processors --------------------------------------------------*/ +void VFP_update_length(const struct busyobj *, ssize_t); + typedef void vfp_begin_f(struct busyobj *, size_t ); typedef int vfp_bytes_f(struct busyobj *, struct http_conn *, ssize_t); typedef int vfp_end_f(struct busyobj *); diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index 6699117..c9cc689 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -103,7 +103,7 @@ vfp_esi_bytes_uu(struct busyobj *bo, const struct vef_priv *vef, return (wl); VEP_Parse(bo, (const char *)st->ptr + st->len, wl); st->len += wl; - bo->fetch_obj->len += wl; + VFP_update_length(bo, wl); bytes -= wl; } return (1); @@ -140,7 +140,7 @@ vfp_esi_bytes_gu(struct busyobj *bo, const struct vef_priv *vef, i = VGZ_Gunzip(vg, &dp, &dl); xxxassert(i == VGZ_OK || i == VGZ_END); VEP_Parse(bo, dp, dl); - bo->fetch_obj->len += dl; + VFP_update_length(bo, dl); } return (1); } @@ -217,7 +217,7 @@ vfp_vep_callback(struct busyobj *bo, ssize_t l, enum vgz_flag flg) } i = VGZ_Gzip(vef->vgz, &dp, &dl, flg); vef->tot += dl; - bo->fetch_obj->len += dl; + VFP_update_length(bo, dl); } while (!VGZ_IbufEmpty(vef->vgz) || (flg != VGZ_NORMAL && VGZ_ObufFull(vef->vgz))); assert(VGZ_IbufEmpty(vef->vgz)); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 0246f1b..e019702 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -113,6 +113,17 @@ VFP_End(struct busyobj *bo) assert(bo->state == BOS_FAILED); } +void +VFP_update_length(const struct busyobj *bo, ssize_t l) +{ + + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(bo->fetch_obj, OBJECT_MAGIC); + if (l == 0) + return; + assert(l > 0); + bo->fetch_obj->len += l; +} /*-------------------------------------------------------------------- * VFP_NOP @@ -165,7 +176,7 @@ vfp_nop_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes) if (wl <= 0) return (wl); st->len += wl; - bo->fetch_obj->len += wl; + VFP_update_length(bo, wl); bytes -= wl; } return (1); diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 343bd35..ca40903 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -468,7 +468,7 @@ vfp_gunzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes) i = VGZ_Gunzip(vg, &dp, &dl); if (i != VGZ_OK && i != VGZ_END) return(FetchError(bo, "Gunzip data error")); - bo->fetch_obj->len += dl; + VFP_update_length(bo, dl); } assert(i == Z_OK || i == Z_STREAM_END); return (1); @@ -543,7 +543,7 @@ vfp_gzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes) return(-1); i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL); assert(i == Z_OK); - bo->fetch_obj->len += dl; + VFP_update_length(bo, dl); } return (1); } @@ -569,7 +569,7 @@ vfp_gzip_end(struct busyobj *bo) if (VGZ_ObufStorage(bo, vg)) return(-1); i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH); - bo->fetch_obj->len += dl; + VFP_update_length(bo, dl); } while (i != Z_STREAM_END); VGZ_UpdateObj(vg, bo->fetch_obj); if (VGZ_Destroy(&vg) != VGZ_END) @@ -627,7 +627,7 @@ vfp_testgzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes) bytes -= wl; VGZ_Ibuf(vg, st->ptr + st->len, wl); st->len += wl; - bo->fetch_obj->len += wl; + VFP_update_length(bo, wl); while (!VGZ_IbufEmpty(vg)) { VGZ_Obuf(vg, vg->m_buf, vg->m_sz); From perbu at varnish-software.com Thu Dec 18 09:27:42 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] 70468df add a label for refs Message-ID: commit 70468df3126778832547f1b9085be23010c00a7b Author: Per Buer Date: Thu Mar 29 11:06:30 2012 +0200 add a label for refs diff --git a/doc/sphinx/tutorial/vcl.rst b/doc/sphinx/tutorial/vcl.rst index 54bce37..da9a340 100644 --- a/doc/sphinx/tutorial/vcl.rst +++ b/doc/sphinx/tutorial/vcl.rst @@ -49,6 +49,8 @@ In vcl_fetch you still have the request object, req, available. There is also a *backend response*, beresp. beresp will contain the HTTP headers from the backend. +.. _tutorial-vcl_fetch_actions: + actions ~~~~~~~ From perbu at varnish-software.com Thu Dec 18 09:27:42 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] 56cfc90 alternative, simpler cookie stripper by James Light Message-ID: commit 56cfc9088bd7dd888b0d13517193d9ba043219b9 Author: Per Buer Date: Thu Mar 29 11:10:36 2012 +0200 alternative, simpler cookie stripper by James Light diff --git a/doc/sphinx/tutorial/cookies.rst b/doc/sphinx/tutorial/cookies.rst index 20fd302..93289c3 100644 --- a/doc/sphinx/tutorial/cookies.rst +++ b/doc/sphinx/tutorial/cookies.rst @@ -64,8 +64,25 @@ cookies named COOKIE1 and COOKIE2 and you can marvel at it:: } } -The example is taken from the Varnish Wiki, where you can find other -scary examples of what can be done in VCL. +A somewhat simpler example that can accomplish almost the same can be +found below. Instead of filtering out the other cookies it picks out +the one cookie that is needed, copies it to another header and then +copies it back, deleting the original cookie header.:: + + sub vcl_recv { + # save the original cookie header so we can mangle it + set req.http.X-Varnish-PHP_SID = req.http.Cookie; + # using a capturing sub pattern, extract the continuous string of + # alphanumerics that immediately follows "PHPSESSID=" + set req.http.X-Varnish-PHP_SID = + regsuball(req.http.X-Varnish-PHP_SID, ";? ?PHPSESSID=([a-zA-Z0-9]+)( |;| ;).*","\1"); + set req.http.Cookie = req.X-Varnish-PHP_SID; + remove req.X-Varnish-PHP_SID; + } + +There are other scary examples of what can be done in VCL in the +Varnish Cache Wiki. + Cookies coming from the backend ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From perbu at varnish-software.com Thu Dec 18 09:27:42 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] f93c6fa kill warnings. fix a ref Message-ID: commit f93c6faae2817ef4e55f96359da2ff44d77f30f9 Author: Per Buer Date: Thu Mar 29 11:46:04 2012 +0200 kill warnings. fix a ref diff --git a/doc/sphinx/tutorial/cookies.rst b/doc/sphinx/tutorial/cookies.rst index 93289c3..88e9d07 100644 --- a/doc/sphinx/tutorial/cookies.rst +++ b/doc/sphinx/tutorial/cookies.rst @@ -89,6 +89,6 @@ Cookies coming from the backend If your backend server sets a cookie using the Set-Cookie header Varnish will not cache the page. A hit-for-pass object (see -:ref:`tutorial_vcl_fetch_actions`) is created. So, if the backend +:ref:`tutorial-vcl_fetch_actions`) is created. So, if the backend server acts silly and sets unwanted cookies just unset the Set-Cookie header and all should be fine. diff --git a/doc/sphinx/tutorial/devicedetection.rst b/doc/sphinx/tutorial/devicedetection.rst index 9f02493..6bfc4c9 100644 --- a/doc/sphinx/tutorial/devicedetection.rst +++ b/doc/sphinx/tutorial/devicedetection.rst @@ -61,6 +61,7 @@ is dependant on this header. Everything works out of the box from Varnish' perspective. .. 071-example1-start + VCL:: sub vcl_recv { @@ -95,6 +96,7 @@ VCL:: set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent"); } } + .. 071-example1-end Example 2: Normalize the User-Agent string @@ -118,6 +120,7 @@ A possible use for this is for CGI scripts where only a small set of predefined headers are (by default) available for the script. .. 072-example2-start + VCL:: sub vcl_recv { @@ -158,6 +161,7 @@ The client itself does not see this classification, only the backend request is changed. .. 073-example3-start + VCL:: sub vcl_recv { @@ -240,6 +244,7 @@ Redirecting mobile clients If you want to redirect mobile clients you can use the following snippet. .. 065-redir-mobile-start + VCL:: sub vcl_recv { diff --git a/doc/sphinx/tutorial/vcl.rst b/doc/sphinx/tutorial/vcl.rst index da9a340..0601468 100644 --- a/doc/sphinx/tutorial/vcl.rst +++ b/doc/sphinx/tutorial/vcl.rst @@ -51,7 +51,6 @@ headers from the backend. .. _tutorial-vcl_fetch_actions: - actions ~~~~~~~ From apj at mutt.dk Thu Dec 18 09:27:42 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:42 +0100 Subject: [experimental-ims] fc8678c Remove unused tag Message-ID: commit fc8678c77d60575303b13011856a7b44521b312a Author: Andreas Plesner Jacobsen Date: Mon Mar 26 10:24:47 2012 +0200 Remove unused tag diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 86c9cc2..1c404cf 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -86,7 +86,6 @@ SLTM(VCL_acl, "", "") SLTM(VCL_call, "VCL method called", "") SLTM(VCL_trace, "VCL trace data", "") SLTM(VCL_return, "VCL method return value", "") -SLTM(VCL_error, "Unused", "") SLTM(ReqStart, "Client request start", "") SLTM(Hit, "Hit object in cache", "") SLTM(HitPass, "Hit for pass object in cache", "") From perbu at varnish-software.com Thu Dec 18 09:27:43 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 9cd5b47 escape \0 Message-ID: commit 9cd5b47122ebc1219ee3f86eb88a2e027e7a5f68 Author: Per Buer Date: Wed Apr 4 10:17:01 2012 +0200 escape \0 diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 9e71fbd..5a5ca6e 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -408,7 +408,7 @@ hash_data(str) regsub(str, regex, sub) Returns a copy of str with the first occurrence of the regular - expression regex replaced with sub. Within sub, \0 (which can + expression regex replaced with sub. Within sub, \\0 (which can also be spelled &) is replaced with the entire matched string, and \n is replaced with the contents of subgroup n in the matched string. From tfheen at varnish-software.com Thu Dec 18 09:27:43 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 8b8adc6 Fix typo in HTTP header name Message-ID: commit 8b8adc6c79cfd64578ca5bdf4a8f1f4768f7a68b Author: Tollef Fog Heen Date: Tue Apr 10 10:27:33 2012 +0200 Fix typo in HTTP header name diff --git a/include/tbl/http_headers.h b/include/tbl/http_headers.h index 3f2c50a..758c7e6 100644 --- a/include/tbl/http_headers.h +++ b/include/tbl/http_headers.h @@ -56,7 +56,7 @@ HTTPH("Authorization", H_Authorization, 0 ) /* RFC2616 14.8 */ HTTPH("Cache-Control", H_Cache_Control, HTTPH_R_FETCH ) /* RFC2616 14.9 */ HTTPH("Connection", H_Connection, HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS) /* RFC2616 14.10 */ HTTPH("Content-Encoding", H_Content_Encoding, 0 ) /* RFC2616 14.11 */ -HTTPH("Content-Langugae", H_Content_Language, 0 ) /* RFC2616 14.12 */ +HTTPH("Content-Language", H_Content_Language, 0 ) /* RFC2616 14.12 */ HTTPH("Content-Length", H_Content_Length, HTTPH_R_FETCH | HTTPH_A_INS) /* RFC2616 14.13 */ HTTPH("Content-Location", H_Content_Location, 0 ) /* RFC2616 14.14 */ HTTPH("Content-MD5", H_Content_MD5, 0 ) /* RFC2616 14.15 */ From phk at FreeBSD.org Thu Dec 18 09:27:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 3cf4384 Adapt to FreeBSD 10 #include changes Message-ID: commit 3cf4384ea84b8da3a5ff3850d45c51fa06754608 Author: Poul-Henning Kamp Date: Tue Apr 10 11:42:37 2012 +0000 Adapt to FreeBSD 10 #include changes diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 0c9f5d0..4ba4dd2 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -90,7 +90,8 @@ -efunc(1791, pdiff) // return last on line ////////////// -efile(451, "sys/\*.h") // No include guard --efile(451, "machine/\*.h") // No include guard +-efile(451, "machine/*.h") // No include guard +-efile(451, "stdarg.h") // No include guard -efile(451, "tbl/*.h") // No include guard -efile(451, "vcc_types.h") // No include guard -efile(451, "symbol_kind.h") // No include guard From phk at FreeBSD.org Thu Dec 18 09:27:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 9f9429c move FreeBSD hacks to the (out of varnish-tree) freebsd configs Message-ID: commit 9f9429c35841712811d84e3e7a97a37b871445ff Author: Poul-Henning Kamp Date: Tue Apr 10 12:01:53 2012 +0000 move FreeBSD hacks to the (out of varnish-tree) freebsd configs diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 4ba4dd2..1569271 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -89,9 +89,6 @@ ////////////// -efunc(1791, pdiff) // return last on line ////////////// --efile(451, "sys/\*.h") // No include guard --efile(451, "machine/*.h") // No include guard --efile(451, "stdarg.h") // No include guard -efile(451, "tbl/*.h") // No include guard -efile(451, "vcc_types.h") // No include guard -efile(451, "symbol_kind.h") // No include guard From phk at FreeBSD.org Thu Dec 18 09:27:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] e6d7f46 Fix for #1109 Message-ID: commit e6d7f462483e518400c8a0ae151ed70438b3a56c Author: Poul-Henning Kamp Date: Tue Apr 10 12:03:16 2012 +0000 Fix for #1109 I've opted for a less intrusive fix that Martin proposed, but my fix is largely a rework of his suggested patch. Testcase by: Martin diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index dd3d98a..0e8b11a 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -421,12 +421,10 @@ ved_deliver_byterange(const struct sess *sp, ssize_t low, ssize_t high) ssize_t l, lx; u_char *p; -//printf("BR %jd %jd\n", low, high); lx = 0; VTAILQ_FOREACH(st, &sp->req->obj->store, list) { p = st->ptr; l = st->len; -//printf("[0-] %jd %jd\n", lx, lx + l); if (lx + l < low) { lx += l; continue; @@ -439,16 +437,14 @@ ved_deliver_byterange(const struct sess *sp, ssize_t low, ssize_t high) l -= (low - lx); lx = low; } -//printf("[1-] %jd %jd\n", lx, lx + l); if (lx + l >= high) l = high - lx; -//printf("[2-] %jd %jd\n", lx, lx + l); assert(lx >= low && lx + l <= high); if (l != 0) (void)WRW_Write(sp->wrk, p, l); - if (lx + st->len > high) + if (p + l < st->ptr + st->len) return(p[l]); - lx += st->len; + lx += l; } INCOMPL(); } @@ -459,10 +455,12 @@ ESI_DeliverChild(const struct sess *sp) struct storage *st; struct object *obj; ssize_t start, last, stop, lpad; - u_char *p, cc; + u_char cc; uint32_t icrc; uint32_t ilen; uint8_t *dbits; + int i, j; + uint8_t tailbuf[8]; if (!sp->req->obj->gziped) { VTAILQ_FOREACH(st, &sp->req->obj->store, list) @@ -542,12 +540,20 @@ ESI_DeliverChild(const struct sess *sp) } if (lpad > 0) (void)WRW_Write(sp->wrk, dbits + 1, lpad); + + /* We need the entire tail, but it may not be in one storage segment */ st = VTAILQ_LAST(&sp->req->obj->store, storagehead); - assert(st->len > 8); + for (i = sizeof tailbuf; i > 0; i -= j) { + j = st->len; + if (j > i) + j = i; + memcpy(tailbuf + i - j, st->ptr + st->len - j, j); + st = VTAILQ_PREV(st, storagehead, list); + assert(i == j || st != NULL); + } - p = st->ptr + st->len - 8; - icrc = vle32dec(p); - ilen = vle32dec(p + 4); + icrc = vle32dec(tailbuf); + ilen = vle32dec(tailbuf + 4); sp->req->crc = crc32_combine(sp->req->crc, icrc, ilen); sp->req->l_crc += ilen; } diff --git a/bin/varnishtest/tests/r01109.vtc b/bin/varnishtest/tests/r01109.vtc new file mode 100644 index 0000000..50f0a0e --- /dev/null +++ b/bin/varnishtest/tests/r01109.vtc @@ -0,0 +1,43 @@ +varnishtest "Test case for #1109 - Gzip+ESI broken for large included objects" + +server s1 { + rxreq + expect req.url == "/test1" + txresp -body {startend} + rxreq + expect req.url == "/include1" + # This tests ESI+gzip delivery when the ESI-included object + # has more than one storage chunk + txresp -bodylen 4082 + + rxreq + txresp -body {startend} + expect req.url == "/test2" + + rxreq + expect req.url == "/include2" + # This tests gzip trailer extraction for ESI+gzip CRC calculation + # when the trailer spans two storage chunks + txresp -bodylen 4074 +} -start + +varnish v1 -arg "-pfetch_chunksize=4k" -arg "-pgzip_level=0" -vcl+backend { + sub vcl_fetch { + if (req.url ~ "/test") { + set beresp.do_esi = true; + } + set beresp.do_gzip = true; + } +} -start + +client c1 { + txreq -url "/test1" -hdr "Accept-Encoding: gzip" + rxresp + gunzip + expect resp.bodylen == 4096 + + txreq -url "/test2" -hdr "Accept-Encoding: gzip" + rxresp + gunzip + expect resp.bodylen == 4088 +} -run From tfheen at varnish-software.com Thu Dec 18 09:27:43 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 9dc28b2 Fix libedit detection on *BSD OS's Message-ID: commit 9dc28b2ce920274432da72f78419562d9365bc2b Author: Brad Date: Mon Jan 2 14:03:46 2012 -0500 Fix libedit detection on *BSD OS's The following patch allows the autoconf script to detect the presence of libedit when there isn't a pkg-config file present and thus allowing Varnish to detect libedit on OpenBSD/FreeBSD/NetBSD/DragonFly. Fixes: #1003 diff --git a/configure.ac b/configure.ac index 7e5e57e..33b0a60 100644 --- a/configure.ac +++ b/configure.ac @@ -144,7 +144,14 @@ AC_SUBST(PCRE_LIBS) PKG_CHECK_MODULES([LIBEDIT], [libedit], [AC_DEFINE([HAVE_LIBEDIT], [1], [Define we have libedit])], - [AC_MSG_WARN([libedit not found, disabling libedit support])]) + [AC_CHECK_HEADERS([readline/readline.h]) + AC_CHECK_LIB(edit, el_init, + [ AC_DEFINE([HAVE_LIBEDIT], [1], [Define we have libedit]) + LIBEDIT_CFLAGS="" + LIBEDIT_LIBS="-ledit ${CURSES_LIBS}" + ], + [AC_MSG_WARN([libedit not found, disabling libedit support])], + [${CURSES_LIBS}])]) # Checks for header files. AC_HEADER_STDC From phk at FreeBSD.org Thu Dec 18 09:27:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 7e51d6f Stopgap fix to get FreeBSD 10-current compiling again. Message-ID: commit 7e51d6ff15713f5e4d424b179428c6fd8afe29ca Author: Poul-Henning Kamp Date: Thu Apr 12 07:24:02 2012 +0000 Stopgap fix to get FreeBSD 10-current compiling again. diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index bb5cc8e..f3b8b69 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -32,8 +32,12 @@ #include #ifdef HAVE_LIBEDIT -#include -#include +# include +# ifdef HAVE_EDIT_READLINE_READLINE_H +# include +# else +# include +# endif #endif #include diff --git a/configure.ac b/configure.ac index 33b0a60..e6ecfb9 100644 --- a/configure.ac +++ b/configure.ac @@ -145,6 +145,7 @@ AC_SUBST(PCRE_LIBS) PKG_CHECK_MODULES([LIBEDIT], [libedit], [AC_DEFINE([HAVE_LIBEDIT], [1], [Define we have libedit])], [AC_CHECK_HEADERS([readline/readline.h]) + AC_CHECK_HEADERS([edit/readline/readline.h]) AC_CHECK_LIB(edit, el_init, [ AC_DEFINE([HAVE_LIBEDIT], [1], [Define we have libedit]) LIBEDIT_CFLAGS="" From phk at FreeBSD.org Thu Dec 18 09:27:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 480c0a5 Go over the return(restart) code. Message-ID: commit 480c0a5d0b7b11e8e6b66f90dd66a2d8b22d235f Author: Poul-Henning Kamp Date: Thu Apr 12 08:06:09 2012 +0000 Go over the return(restart) code. This is vastly inspired by #1113 and scoof+martins testcase+patch. Add a new cnt* state "cnt_restart" to do the mandatory restart work, and to move the restart limit check out of vcl_recv{}, getting rid of a nasty XXX comment. Set the explicit 503 when over limit, reset in other cases. Expand scoof+martins test-case to cover more vcl methods, which were also affected, but not fixed in their patch. Fixes #1113 diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 7104e0f..5530f1b 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -293,10 +293,8 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) (void)HSH_Deref(&wrk->stats, NULL, &req->obj); } AZ(req->obj); - req->restarts++; - req->director = NULL; http_Teardown(req->resp); - sp->step = STP_RECV; + sp->step = STP_RESTART; return (0); default: WRONG("Illegal action in vcl_deliver{}"); @@ -524,9 +522,7 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) req->restarts < cache_param->max_restarts) { HSH_Drop(wrk, &sp->req->obj); VBO_DerefBusyObj(wrk, &req->busyobj); - req->director = NULL; - req->restarts++; - sp->step = STP_RECV; + sp->step = STP_RESTART; return (0); } else if (req->handling == VCL_RET_RESTART) req->handling = VCL_RET_DELIVER; @@ -660,8 +656,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) switch (req->handling) { case VCL_RET_RESTART: - req->restarts++; - sp->step = STP_RECV; + sp->step = STP_RESTART; return (0); case VCL_RET_ERROR: sp->step = STP_ERROR; @@ -1045,9 +1040,7 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req) sp->step = STP_ERROR; return (0); case VCL_RET_RESTART: - req->director = NULL; - req->restarts++; - sp->step = STP_RECV; + sp->step = STP_RESTART; return (0); default: WRONG("Illegal action in vcl_hit{}"); @@ -1231,9 +1224,7 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req) sp->step = STP_PASS; break; case VCL_RET_RESTART: - req->restarts++; - req->director = NULL; - sp->step = STP_RECV; + sp->step = STP_RESTART; break; default: WRONG("Illegal action in vcl_miss{}"); @@ -1350,6 +1341,37 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req) } /*-------------------------------------------------------------------- + * +DOT subgraph xcluster_restart { +DOT restart [ +DOT shape=record +DOT label="{cnt_restart}" +DOT ] +DOT } +DOT RESTART -> restart [color=purple] +DOT restart -> recv [color=purple] + */ + +static int +cnt_restart(struct sess *sp, const struct worker *wrk, struct req *req) +{ + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + req->director = NULL; + if (++req->restarts >= cache_param->max_restarts) { + req->err_code = 503; + sp->step = STP_ERROR; + } else { + req->err_code = 0; + sp->step = STP_RECV; + } + return (0); +} + +/*-------------------------------------------------------------------- * RECV * We have a complete request, set everything up and start it. * We can come here both with a request from the client and with @@ -1368,7 +1390,6 @@ DOT label="{cnt_recv:|{vcl_hash\{\}|req.*}}" DOT ] DOT } DOT ESI_REQ [ shape=hexagon ] -DOT RESTART -> recv [color=purple] DOT ESI_REQ -> recv DOT recv:pipe -> pipe [style=bold,color=orange] DOT recv:pass -> pass [style=bold,color=red] @@ -1406,18 +1427,6 @@ cnt_recv(struct sess *sp, const struct worker *wrk, struct req *req) VCL_recv_method(sp); recv_handling = req->handling; - if (req->restarts >= cache_param->max_restarts) { - /* - * XXX: Why not before vcl_recv{} ? We go to vcl_error{} - * XXX: without vcl_recv{} on 413 and 417 already. - * XXX tell vcl_error why we come - */ - if (req->err_code == 0) - req->err_code = 503; - sp->step = STP_ERROR; - return (0); - } - if (cache_param->http_gzip_support && (recv_handling != VCL_RET_PIPE) && (recv_handling != VCL_RET_PASS)) { diff --git a/bin/varnishtest/tests/r01113.vtc b/bin/varnishtest/tests/r01113.vtc new file mode 100644 index 0000000..abcee83 --- /dev/null +++ b/bin/varnishtest/tests/r01113.vtc @@ -0,0 +1,62 @@ +varnishtest "HTTP status code when hitting max_restarts" + +server s1 { + rxreq + txresp + accept + rxreq + txresp + accept + rxreq + txresp + accept + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_hit { + if (req.url == "/hit") { + return(restart); + } + } + sub vcl_fetch { + if (req.url == "/fetch") { + return(restart); + } + } + sub vcl_deliver { + if (req.url == "/deliver") { + return(restart); + } + } + sub vcl_miss { + if (req.url == "/miss") { + return(restart); + } + } +} -start -cliok "param.set max_restarts 1" + +client c1 { + txreq -url /hit + rxresp + expect resp.status == 200 + txreq -url /hit + rxresp + expect resp.status == 503 +} -run +client c1 { + txreq -url /miss + rxresp + expect resp.status == 503 +} -run +client c1 { + txreq -url /fetch + rxresp + expect resp.status == 503 +} -run +client c1 { + txreq -url /deliver + rxresp + expect resp.status == 503 +} -run diff --git a/include/tbl/steps.h b/include/tbl/steps.h index e6738cb..ee70874 100644 --- a/include/tbl/steps.h +++ b/include/tbl/steps.h @@ -31,6 +31,7 @@ /*lint -save -e525 -e539 */ STEP(wait, WAIT, (sp, sp->wrk, sp->req)) STEP(first, FIRST, (sp, sp->wrk)) +STEP(restart, RESTART, (sp, sp->wrk, sp->req)) STEP(recv, RECV, (sp, sp->wrk, sp->req)) STEP(start, START, (sp, sp->wrk, sp->req)) STEP(pipe, PIPE, (sp, sp->wrk, sp->req)) From phk at FreeBSD.org Thu Dec 18 09:27:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 10f1de6 Fix #1126 by properly setting the mask token to the IP number token. Message-ID: commit 10f1de6232d60c8cd2d9081a7680dad857c6d5aa Author: Poul-Henning Kamp Date: Mon Apr 16 07:04:40 2012 +0000 Fix #1126 by properly setting the mask token to the IP number token. Varnishtest doesn't see the difference between a VCC core dump and a VCC error, so a test-case is non-trivial. Fixes #1126 diff --git a/lib/libvcl/vcc_acl.c b/lib/libvcl/vcc_acl.c index e6a1065..b42f9de 100644 --- a/lib/libvcl/vcc_acl.c +++ b/lib/libvcl/vcc_acl.c @@ -263,8 +263,10 @@ vcc_acl_try_netnotation(struct vcc *tl, struct acl_e *ae) return (0); p += k + 1; } - if (ae->t_mask == NULL) + if (ae->t_mask == NULL) { ae->mask = 8 + 8 * i; + ae->t_mask = ae->t_addr; + } vcc_acl_add_entry(tl, ae, 4, b, AF_INET); return (1); } From phk at FreeBSD.org Thu Dec 18 09:27:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 35475f8 Missing errorchecks incompilation of regsub() Message-ID: commit 35475f8c5db39fa47a0d5862f00737ccd9985575 Author: Poul-Henning Kamp Date: Mon Apr 16 07:12:10 2012 +0000 Missing errorchecks incompilation of regsub() Fixes #1125 diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c index 203d76f..30e63d9 100644 --- a/lib/libvcl/vcc_expr.c +++ b/lib/libvcl/vcc_expr.c @@ -454,6 +454,8 @@ vcc_Eval_Regsub(struct vcc *tl, struct expr **e, const struct symbol *sym) SkipToken(tl, '('); vcc_expr0(tl, &e2, STRING); + if (e2 == NULL) + return; if (e2->fmt != STRING) vcc_expr_tostring(&e2, STRING); @@ -467,6 +469,8 @@ vcc_Eval_Regsub(struct vcc *tl, struct expr **e, const struct symbol *sym) SkipToken(tl, ','); vcc_expr0(tl, &e2, STRING); + if (e2 == NULL) + return; if (e2->fmt != STRING) vcc_expr_tostring(&e2, STRING); *e = vcc_expr_edit(STRING, "\v1, \v2)", *e, e2); From phk at FreeBSD.org Thu Dec 18 09:27:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 8bc9a68 Better fix for #1126 Message-ID: commit 8bc9a68068584feaab4442569f5d815ba609b925 Author: Poul-Henning Kamp Date: Mon Apr 16 07:22:57 2012 +0000 Better fix for #1126 diff --git a/lib/libvcl/vcc_acl.c b/lib/libvcl/vcc_acl.c index b42f9de..9535f59 100644 --- a/lib/libvcl/vcc_acl.c +++ b/lib/libvcl/vcc_acl.c @@ -101,7 +101,10 @@ vcc_acl_add_entry(struct vcc *tl, const struct acl_e *ae, int l, if (fam == PF_INET && ae->mask > 32) { VSB_printf(tl->sb, "Too wide mask (%u) for IPv4 address", ae->mask); - vcc_ErrWhere(tl, ae->t_mask); + if (ae->t_mask != NULL) + vcc_ErrWhere(tl, ae->t_mask); + else + vcc_ErrWhere(tl, ae->t_addr); return; } if (fam == PF_INET6 && ae->mask > 128) { @@ -263,10 +266,8 @@ vcc_acl_try_netnotation(struct vcc *tl, struct acl_e *ae) return (0); p += k + 1; } - if (ae->t_mask == NULL) { + if (ae->t_mask == NULL) ae->mask = 8 + 8 * i; - ae->t_mask = ae->t_addr; - } vcc_acl_add_entry(tl, ae, 4, b, AF_INET); return (1); } From phk at FreeBSD.org Thu Dec 18 09:27:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 7175bb3 Remove the -w argument, -p should be used instead. Message-ID: commit 7175bb38e9f8ea5df2daf3725b9d624ef81cface Author: Poul-Henning Kamp Date: Mon Apr 16 10:20:55 2012 +0000 Remove the -w argument, -p should be used instead. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 97ac5da..7047fa9 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -100,20 +100,6 @@ pick(const struct choice *cp, const char *which, const char *kind) /*--------------------------------------------------------------------*/ -static unsigned long -arg_ul(const char *p) -{ - char *q; - unsigned long ul; - - ul = strtoul(p, &q, 0); - if (*q != '\0') - ARGV_ERR("Invalid number: \"%s\"\n", p); - return (ul); -} - -/*--------------------------------------------------------------------*/ - static void usage(void) { @@ -160,52 +146,11 @@ usage(void) fprintf(stderr, FMT, "-T address:port", "Telnet listen address and port"); fprintf(stderr, FMT, "-V", "version"); - fprintf(stderr, FMT, "-w int[,int[,int]]", "Number of worker threads"); - fprintf(stderr, FMT, "", " -w "); - fprintf(stderr, FMT, "", " -w min,max"); - fprintf(stderr, FMT, "", " -w min,max,timeout [default: -w2,500,300]"); fprintf(stderr, FMT, "-u user", "Priviledge separation user id"); #undef FMT exit(1); } - -/*--------------------------------------------------------------------*/ - -static void -tackle_warg(const char *argv) -{ - char **av; - unsigned int u; - - av = VAV_Parse(argv, NULL, ARGV_COMMA); - AN(av); - - if (av[0] != NULL) - ARGV_ERR("%s\n", av[0]); - - if (av[1] == NULL) - usage(); - - u = arg_ul(av[1]); - if (u < 1) - usage(); - mgt_param.wthread_max = mgt_param.wthread_min = u; - - if (av[2] != NULL) { - u = arg_ul(av[2]); - if (u < mgt_param.wthread_min) - usage(); - mgt_param.wthread_max = u; - - if (av[3] != NULL) { - u = arg_ul(av[3]); - mgt_param.wthread_timeout = u; - } - } - VAV_Free(av); -} - /*--------------------------------------------------------------------*/ static void @@ -516,8 +461,7 @@ main(int argc, char * const *argv) usage(); break; case 'w': - tackle_warg(optarg); - break; + ARGV_ERR("-w has been removed, use -p instead\n"); default: usage(); } diff --git a/bin/varnishtest/tests/v00006.vtc b/bin/varnishtest/tests/v00006.vtc index f3faace..ca4c827 100644 --- a/bin/varnishtest/tests/v00006.vtc +++ b/bin/varnishtest/tests/v00006.vtc @@ -8,7 +8,7 @@ server s1 { } -start # Only one pool, to avoid getting more than one work thread -varnish v1 -arg "-p thread_pools=1 -w1,1,300" -vcl+backend { +varnish v1 -arg "-p thread_pools=1" -vcl+backend { } -start client c1 { From perbu at varnish-software.com Thu Dec 18 09:27:43 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 9a1b232 remove the -w option from man varnishd Message-ID: commit 9a1b232b4a9ef1fbfa68d79499b598052149f2d1 Author: Per Buer Date: Mon Apr 16 13:20:38 2012 +0200 remove the -w option from man varnishd diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 351c7b3..f0647b3 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -23,7 +23,7 @@ varnishd [-a address[:port]] [-b host[:port]] [-d] [-F] [-f config] [-g group] [-h type[,options]] [-i identity] [-l shmlogsize] [-n name] [-P file] [-p param=value] [-s type[,options]] [-T address[:port]] [-t ttl] - [-u user] [-V] [-w min[,max[,timeout]]] + [-u user] [-V] DESCRIPTION =========== @@ -120,20 +120,6 @@ OPTIONS -V Display the version number and exit. --w min[,max[,timeout]] - - Start at least min but no more than max worker threads - with the specified idle timeout. This is a shortcut for - specifying the thread_pool_min, thread_pool_max and - thread_pool_timeout run-time parameters. - - If only one number is specified, thread_pool_min and - thread_pool_max are both set to this number, and - thread_pool_timeout has no effect. - - - - Hash Algorithms --------------- From perbu at varnish-software.com Thu Dec 18 09:27:43 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 8931bc0 :: need to be on a separate line in RST. Message-ID: commit 8931bc07513bdb4cbc111ebeff7dc60e24371197 Author: Per Buer Date: Mon Apr 16 13:37:47 2012 +0200 :: need to be on a separate line in RST. diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 5a5ca6e..b8f47c3 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -96,14 +96,16 @@ file as a quoted string. Backend declarations -------------------- -A backend declaration creates and initializes a named backend object:: +A backend declaration creates and initializes a named backend object: +:: backend www { .host = "www.example.com"; .port = "http"; } -The backend object can later be used to select a backend at request time:: +The backend object can later be used to select a backend at request time: +:: if (req.http.host ~ "(?i)^(www.)?example.com$") { set req.backend = www; @@ -118,7 +120,8 @@ backend connection, .first_byte_timeout for the time to wait for the first byte from the backend and .between_bytes_timeout for time to wait between each received byte. -These can be set in the declaration like this:: +These can be set in the declaration like this: +:: backend www { .host = "www.example.com"; @@ -145,7 +148,8 @@ be used. There are several types of directors. The different director types use different algorithms to choose which backend to use. -Configuring a director may look like this:: +Configuring a director may look like this: +:: director b2 random { .retries = 5; @@ -226,7 +230,8 @@ The DNS director ~~~~~~~~~~~~~~~~ The DNS director can use backends in two different ways. Either like the -random or round-robin director or using .list:: +random or round-robin director or using .list: +:: director directorname dns { .list = { @@ -266,7 +271,8 @@ considers them in the order in which they are listed in its definition. The fallback director does not take any options. -An example of a fallback director:: +An example of a fallback director: +:: director b3 fallback { { .backend = www1; } @@ -312,7 +318,8 @@ Probes take the following parameters: Default is 2 seconds. A backend with a probe can be defined like this, together with the -backend or director:: +backend or director: +:: backend www { .host = "www.example.com"; @@ -326,7 +333,8 @@ backend or director:: } } -Or it can be defined separately and then referenced:: +Or it can be defined separately and then referenced: +:: probe healthcheck { .url = "/status.cgi"; @@ -347,7 +355,8 @@ Or it can be defined separately and then referenced:: If you have many backends this can simplify the config a lot. -It is also possible to specify the raw HTTP request:: +It is also possible to specify the raw HTTP request: +:: probe rawprobe { # NB: \r\n automatically inserted after each string! @@ -361,7 +370,8 @@ ACLs ---- An ACL declaration creates and initializes a named access control list -which can later be used to match client addresses:: +which can later be used to match client addresses: +:: acl local { "localhost"; // myself @@ -375,7 +385,8 @@ if it is preceded by a negation mark, it will reject any address it is compared to, which may not be what you intended. If the entry is enclosed in parentheses, however, it will simply be ignored. -To match an IP address against an ACL, simply use the match operator:: +To match an IP address against an ACL, simply use the match operator: +:: if (client.ip ~ local) { return (pipe); @@ -390,7 +401,8 @@ PCRE(3) man page. To send flags to the PCRE engine, such as to turn on *case insensitivity* add the flag within parens following a question mark, -like this:: +like this: +:: if (req.http.host ~ "(?i)example.com$") { ... @@ -424,7 +436,8 @@ ban_url(regex) Subroutines ~~~~~~~~~~~ -A subroutine is used to group code for legibility or reusability:: +A subroutine is used to group code for legibility or reusability: +:: sub pipe_if_local { if (client.ip ~ local) { @@ -646,7 +659,8 @@ appear in the source. The default versions distributed with Varnish will be implicitly concatenated as a last resort at the end. -Example:: +Example: +:: # in file "main.vcl" include "backends.vcl"; @@ -893,7 +907,8 @@ resp.response resp.http.header The corresponding HTTP header. -Values may be assigned to variables using the set keyword:: +Values may be assigned to variables using the set keyword: +:: sub vcl_recv { # Normalize the Host: header @@ -902,7 +917,8 @@ Values may be assigned to variables using the set keyword:: } } -HTTP headers can be removed entirely using the remove keyword:: +HTTP headers can be removed entirely using the remove keyword: +:: sub vcl_fetch { # Don't cache cookies @@ -919,7 +935,8 @@ fresh object is being generated by the backend. The following vcl code will make Varnish serve expired objects. All object will be kept up to two minutes past their expiration time or a -fresh object is generated.:: +fresh object is generated. +:: sub vcl_recv { set req.grace = 2m; @@ -944,7 +961,8 @@ EXAMPLES The following code is the equivalent of the default configuration with the backend address set to "backend.example.com" and no backend port -specified:: +specified: +:: backend default { .host = "backend.example.com"; @@ -956,7 +974,8 @@ specified:: The following example shows how to support multiple sites running on separate backends in the same Varnish instance, by selecting backends -based on the request URL:: +based on the request URL: +:: backend www { .host = "www.example.com"; @@ -982,7 +1001,8 @@ based on the request URL:: The following snippet demonstrates how to force a minimum TTL for all documents. Note that this is not the same as setting the default_ttl run-time parameter, as that only affects document for -which the backend did not specify a TTL:: +which the backend did not specify a TTL: +:: import std; # needed for std.log @@ -994,7 +1014,8 @@ which the backend did not specify a TTL:: } The following snippet demonstrates how to force Varnish to cache -documents even when cookies are present:: +documents even when cookies are present: +:: sub vcl_recv { if (req.request == "GET" && req.http.cookie) { @@ -1009,7 +1030,8 @@ documents even when cookies are present:: } The following code implements the HTTP PURGE method as used by Squid -for object invalidation:: +for object invalidation: +:: acl purge { "localhost"; From tfheen at varnish-software.com Thu Dec 18 09:27:43 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 3a82559 Fix escaping in regsub docs Message-ID: commit 3a82559d539246d3d3fedee8aa76b6e93a0b9b03 Author: Tollef Fog Heen Date: Mon Apr 16 14:11:30 2012 +0200 Fix escaping in regsub docs diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index b8f47c3..ed970e2 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -420,9 +420,9 @@ hash_data(str) regsub(str, regex, sub) Returns a copy of str with the first occurrence of the regular - expression regex replaced with sub. Within sub, \\0 (which can - also be spelled &) is replaced with the entire matched string, - and \n is replaced with the contents of subgroup n in the + expression regex replaced with sub. Within sub, \\0 (which can + also be spelled \\&) is replaced with the entire matched string, + and \\n is replaced with the contents of subgroup n in the matched string. regsuball(str, regex, sub) From phk at FreeBSD.org Thu Dec 18 09:27:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 2612267 Remove unused #include Message-ID: commit 26122673a7fc867f22e06ed4e7ff82a789463bfe Author: Poul-Henning Kamp Date: Mon Apr 16 13:54:55 2012 +0000 Remove unused #include diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 7047fa9..6ad7454 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -45,7 +45,6 @@ #include "mgt/mgt.h" #include "common/heritage.h" -#include "common/params.h" #include "hash/hash_slinger.h" #include "vav.h" From phk at FreeBSD.org Thu Dec 18 09:27:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 0679d60 Be much more cautious about how much workspace we have to build predictive vary string. Message-ID: commit 0679d603fe4dad3027206f0cb272cd9c4aedb557 Author: Poul-Henning Kamp Date: Mon Apr 16 14:06:55 2012 +0000 Be much more cautious about how much workspace we have to build predictive vary string. Fixes #1120 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 073eb75..ee17d31 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -956,6 +956,7 @@ void RES_WriteObj(struct sess *sp); struct vsb *VRY_Create(struct req *sp, const struct http *hp); int VRY_Match(struct req *, const uint8_t *vary); void VRY_Validate(const uint8_t *vary); +void VRY_Prep(struct req *); /* cache_vcl.c */ void VCL_Init(void); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 5530f1b..4a04f5b 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -1081,18 +1081,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); AZ(req->busyobj); - if (req->hash_objhead == NULL) { - /* Not a waiting list return */ - AZ(req->vary_b); - AZ(req->vary_l); - AZ(req->vary_e); - (void)WS_Reserve(req->ws, 0); - } else { - AN(req->ws->r); - } - req->vary_b = (void*)req->ws->f; - req->vary_e = (void*)req->ws->r; - req->vary_b[2] = '\0'; + VRY_Prep(req); AZ(req->objcore); oc = HSH_Lookup(sp); @@ -1359,7 +1348,7 @@ cnt_restart(struct sess *sp, const struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - + req->director = NULL; if (++req->restarts >= cache_param->max_restarts) { req->err_code = 503; diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c index 138efda..bf6537f 100644 --- a/bin/varnishd/cache/cache_vary.c +++ b/bin/varnishd/cache/cache_vary.c @@ -176,22 +176,61 @@ vry_cmp(const uint8_t *v1, const uint8_t *v2) return (retval); } +/********************************************************************** + * Prepare predictive vary string + */ + +void +VRY_Prep(struct req *req) +{ + if (req->hash_objhead == NULL) { + /* Not a waiting list return */ + AZ(req->vary_b); + AZ(req->vary_l); + AZ(req->vary_e); + (void)WS_Reserve(req->ws, 0); + } else { + AN(req->ws->r); + } + req->vary_b = (void*)req->ws->f; + req->vary_e = (void*)req->ws->r; + if (req->vary_b + 2 < req->vary_e) + req->vary_b[2] = '\0'; +} + +/********************************************************************** + * Match vary strings, and build a new cached string if possible. + * + * Return zero if there is certainly no match. + * Return non-zero if there could be a match or if we couldn't tell. + */ + int VRY_Match(struct req *req, const uint8_t *vary) { uint8_t *vsp = req->vary_b; char *h, *e; unsigned lh, ln; - int i, retval = 1, oflo = 0; + int i, oflo = 0; AN(vsp); while (vary[2]) { + if (vsp + 2 >= req->vary_e) { + /* + * Too little workspace to find out + */ + oflo = 1; + break; + } i = vry_cmp(vary, vsp); if (i == 1) { - /* Build a new entry */ + /* + * Different header, build a new entry, + * then compare again with that new entry. + */ - i = http_GetHdr(req->http, - (const char*)(vary+2), &h); + ln = 2 + vary[2] + 2; + i = http_GetHdr(req->http, (const char*)(vary+2), &h); if (i) { /* Trim trailing space */ e = strchr(h, '\0'); @@ -199,55 +238,55 @@ VRY_Match(struct req *req, const uint8_t *vary) e--; lh = e - h; assert(lh < 0xffff); + ln += lh; } else { e = h = NULL; lh = 0xffff; } - /* Length of the entire new vary entry */ - ln = 2 + vary[2] + 2 + (lh == 0xffff ? 0 : lh); - if (vsp + ln >= req->vary_e) { - vsp = req->vary_b; + if (vsp + ln + 2 >= req->vary_e) { + /* + * Not enough space to build new entry + * and put terminator behind it. + */ oflo = 1; + break; } - /* - * We MUST have space for one entry and the end marker - * after it, which prevents old junk from confusing us - */ - assert(vsp + ln + 2 < req->vary_e); - vbe16enc(vsp, (uint16_t)lh); memcpy(vsp + 2, vary + 2, vary[2] + 2); - if (h != NULL && e != NULL) { - memcpy(vsp + 2 + vsp[2] + 2, h, e - h); - vsp[2 + vary[2] + 2 + (e - h) + 2] = '\0'; - } else - vsp[2 + vary[2] + 2 + 2] = '\0'; + if (h != NULL) + memcpy(vsp + 2 + vsp[2] + 2, h, lh); + vsp[ln + 0] = 0xff; + vsp[ln + 1] = 0xff; + vsp[ln + 2] = 0; + VRY_Validate(vsp); + req->vary_l = vsp + 3; i = vry_cmp(vary, vsp); - assert(i != 1); /* hdr must be the same now */ + assert(i == 0 || i == 2); + } + if (i == 0) { + /* Same header, same contents*/ + vsp += vry_len(vsp); + vary += vry_len(vary); + } else if (i == 2) { + /* Same header, different contents, cannot match */ + return (0); } - if (i != 0) - retval = 0; - vsp += vry_len(vsp); - vary += vry_len(vary); } - if (vsp + 3 > req->vary_e) - oflo = 1; - if (oflo) { - /* XXX: Should log this */ vsp = req->vary_b; - } - vsp[0] = 0xff; - vsp[1] = 0xff; - vsp[2] = 0; - if (oflo) req->vary_l = NULL; - else - req->vary_l = vsp + 3; - return (retval); + if (vsp + 2 < req->vary_e) { + vsp[0] = 0xff; + vsp[1] = 0xff; + vsp[2] = 0; + } + return (0); + } else { + return (1); + } } void diff --git a/bin/varnishtest/tests/r01120.vtc b/bin/varnishtest/tests/r01120.vtc new file mode 100644 index 0000000..97bc2e5 --- /dev/null +++ b/bin/varnishtest/tests/r01120.vtc @@ -0,0 +1,24 @@ +varnishtest "insanely long vary string" + +server s1 { + rxreq + txresp -hdr "Vary: Foo" -body "xxxx" + rxreq + txresp -hdr "Vary: Foo" -body "yyyyy" +} -start + +varnish v1 \ + -cliok "param.set workspace_client 8k" \ + -cliok "param.set workspace_backend 200k" \ + -vcl+backend { +} -start + +client c1 { + txreq -hdr "Foo: blabla" + rxresp + expect resp.bodylen == 4 + #txreq -hdr "Foo: blablaA" + txreq -hdr "Foo: blablaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaaaaaaaaaaaaaaaaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + rxresp + expect resp.bodylen == 5 +} -run From martin at varnish-software.com Thu Dec 18 09:27:43 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] f992125 Reset output buffer on VGZ_WrwFlush() Message-ID: commit f992125b545fdd1d090941f2ee8b7c9cd6c29dc8 Author: Martin Blix Grydeland Date: Tue Mar 27 17:05:49 2012 +0200 Reset output buffer on VGZ_WrwFlush() Fixes: #1123 diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index ca40903..f7dba8e 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -366,6 +366,7 @@ VGZ_WrwFlush(const struct worker *wrk, struct vgz *vg) (void)WRW_Write(wrk, vg->m_buf, vg->m_len); (void)WRW_Flush(wrk); vg->m_len = 0; + VGZ_Obuf(vg, vg->m_buf, vg->m_sz); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishtest/tests/r01123.vtc b/bin/varnishtest/tests/r01123.vtc new file mode 100644 index 0000000..37a9254 --- /dev/null +++ b/bin/varnishtest/tests/r01123.vtc @@ -0,0 +1,25 @@ +varnishtest "Test case for #1123 - gunzip buffer reset" + +server s1 { + rxreq + txresp -body {start end} + rxreq + expect req.url == "/included" + txresp -body {INCLUDED} +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + if (req.url == "/") { + set beresp.do_esi = true; + } + set beresp.do_gzip = true; + } +} -start + +client c1 { + txreq + rxresp + expect resp.bodylen == 24 + expect resp.body == "start INCLUDED end" +} -run From martin at varnish-software.com Thu Dec 18 09:27:43 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 5625a9a Expose resp.body for inspection and testing in varnishtest Message-ID: commit 5625a9a532536f5c77ea9a1798b89127589e7fce Author: Martin Blix Grydeland Date: Tue Mar 27 14:12:33 2012 +0200 Expose resp.body for inspection and testing in varnishtest Needed to test #1123 diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 00be0be..0678efd 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -190,6 +190,8 @@ cmd_var_resolve(struct http *hp, char *spec) return(hp->chunklen); if (!strcmp(spec, "resp.bodylen")) return(hp->bodylen); + if (!strcmp(spec, "resp.body")) + return(hp->body != NULL ? hp->body : spec); if (!memcmp(spec, "req.http.", 9)) { hh = hp->req; hdr = spec + 9; From apj at mutt.dk Thu Dec 18 09:27:43 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 058fa0f stats are no longer in CLI Message-ID: commit 058fa0fac3581652600d3de317a7457bc3fcde32 Author: Andreas Plesner Jacobsen Date: Tue Apr 17 22:25:56 2012 +0200 stats are no longer in CLI diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index 668ceab..ad9cc9c 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -28,9 +28,6 @@ parameters available through the CLI. The individual parameters are documented in the varnishd(1) man page. -statistics - Statistic counters are available from the CLI. - bans Bans are filters that are applied to keep Varnish from serving stale content. When you issue a ban Varnish will not serve any @@ -149,13 +146,6 @@ quit start Start the Varnish cache process if it is not already running. -stats - Show summary statistics. - - All the numbers presented are totals since server startup; for a - better idea of the current situation, use the varnishstat(1) - utility. - status Check the status of the Varnish cache process. From phk at FreeBSD.org Thu Dec 18 09:27:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] 0f7d9a3 Cave in to the microoptimization demands and remove a pointless unlock/lock sequence when using critbit, at the expense of a slightly more ornate API for hashers. Message-ID: commit 0f7d9a361da8436bf01670be6a86f0b4d17a09ac Author: Poul-Henning Kamp Date: Thu Apr 19 07:33:41 2012 +0000 Cave in to the microoptimization demands and remove a pointless unlock/lock sequence when using critbit, at the expense of a slightly more ornate API for hashers. Also make the "noh" argument optional, as originally intended. diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index e746813..6facf0a 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -259,7 +259,7 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc) AN(wrk->nobjhead); oh = hash->lookup(wrk, digest, &wrk->nobjhead); CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); - Lck_Lock(&oh->mtx); + Lck_AssertHeld(&oh->mtx); assert(oh->refcnt > 0); /* Insert (precreated) objcore in objecthead */ @@ -310,6 +310,7 @@ HSH_Lookup(struct sess *sp) */ CHECK_OBJ_NOTNULL(req->hash_objhead, OBJHEAD_MAGIC); oh = req->hash_objhead; + Lck_Lock(&oh->mtx); req->hash_objhead = NULL; } else { AN(wrk->nobjhead); @@ -317,7 +318,8 @@ HSH_Lookup(struct sess *sp) } CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); - Lck_Lock(&oh->mtx); + Lck_AssertHeld(&oh->mtx); + assert(oh->refcnt > 0); busy_found = 0; grace_oc = NULL; diff --git a/bin/varnishd/hash/hash_classic.c b/bin/varnishd/hash/hash_classic.c index 87e0561..9121ec1 100644 --- a/bin/varnishd/hash/hash_classic.c +++ b/bin/varnishd/hash/hash_classic.c @@ -120,8 +120,8 @@ hcl_lookup(struct worker *wrk, const void *digest, struct objhead **noh) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AN(digest); - AN(noh); - CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC); + if (noh != NULL) + CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC); assert(sizeof oh->digest >= sizeof hdigest); memcpy(&hdigest, digest, sizeof hdigest); @@ -130,6 +130,7 @@ hcl_lookup(struct worker *wrk, const void *digest, struct objhead **noh) Lck_Lock(&hp->mtx); VTAILQ_FOREACH(oh, &hp->head, hoh_list) { + CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); i = memcmp(oh->digest, digest, sizeof oh->digest); if (i < 0) continue; @@ -137,9 +138,15 @@ hcl_lookup(struct worker *wrk, const void *digest, struct objhead **noh) break; oh->refcnt++; Lck_Unlock(&hp->mtx); + Lck_Lock(&oh->mtx); return (oh); } + if (noh == NULL) { + Lck_Unlock(&hp->mtx); + return (NULL); + } + if (oh != NULL) VTAILQ_INSERT_BEFORE(oh, *noh, hoh_list); else @@ -152,6 +159,7 @@ hcl_lookup(struct worker *wrk, const void *digest, struct objhead **noh) oh->hoh_head = hp; Lck_Unlock(&hp->mtx); + Lck_Lock(&oh->mtx); return (oh); } diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c index 5bdab86..08be561 100644 --- a/bin/varnishd/hash/hash_critbit.c +++ b/bin/varnishd/hash/hash_critbit.c @@ -425,9 +425,10 @@ hcb_lookup(struct worker *wrk, const void *digest, struct objhead **noh) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AN(digest); - AN(noh); - CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC); - assert((*noh)->refcnt == 1); + if (noh != NULL) { + CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC); + assert((*noh)->refcnt == 1); + } /* First try in read-only mode without holding a lock */ @@ -440,11 +441,11 @@ hcb_lookup(struct worker *wrk, const void *digest, struct objhead **noh) * under us, so fall through and try with the lock held. */ u = oh->refcnt; - if (u > 0) + if (u > 0) { oh->refcnt++; - Lck_Unlock(&oh->mtx); - if (u > 0) return (oh); + } + Lck_Unlock(&oh->mtx); } while (1) { @@ -455,23 +456,27 @@ hcb_lookup(struct worker *wrk, const void *digest, struct objhead **noh) oh = hcb_insert(wrk, &hcb_root, digest, noh); Lck_Unlock(&hcb_mtx); + if (oh == NULL) + return (NULL); + + Lck_Lock(&oh->mtx); + CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); - if (*noh == NULL) { + if (noh != NULL && *noh == NULL) { assert(oh->refcnt > 0); VSC_C_main->hcb_insert++; return (oh); } - Lck_Lock(&oh->mtx); /* * A refcount of zero indicates that the tree changed * under us, so fall through and try with the lock held. */ u = oh->refcnt; - if (u > 0) + if (u > 0) { oh->refcnt++; - Lck_Unlock(&oh->mtx); - if (u > 0) return (oh); + } + Lck_Unlock(&oh->mtx); } } diff --git a/bin/varnishd/hash/hash_simple_list.c b/bin/varnishd/hash/hash_simple_list.c index d70e76f..d99bb89 100644 --- a/bin/varnishd/hash/hash_simple_list.c +++ b/bin/varnishd/hash/hash_simple_list.c @@ -67,8 +67,8 @@ hsl_lookup(struct worker *wrk, const void *digest, struct objhead **noh) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AN(digest); - AN(noh); - CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC); + if (noh != NULL) + CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC); Lck_Lock(&hsl_mtx); VTAILQ_FOREACH(oh, &hsl_head, hoh_list) { @@ -79,9 +79,13 @@ hsl_lookup(struct worker *wrk, const void *digest, struct objhead **noh) break; oh->refcnt++; Lck_Unlock(&hsl_mtx); + Lck_Lock(&oh->mtx); return (oh); } + if (noh == NULL) + return (NULL); + if (oh != NULL) VTAILQ_INSERT_BEFORE(oh, *noh, hoh_list); else @@ -91,6 +95,7 @@ hsl_lookup(struct worker *wrk, const void *digest, struct objhead **noh) *noh = NULL; memcpy(oh->digest, digest, sizeof oh->digest); Lck_Unlock(&hsl_mtx); + Lck_Lock(&oh->mtx); return (oh); } From phk at FreeBSD.org Thu Dec 18 09:27:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:43 +0100 Subject: [experimental-ims] b8e04e6 Don't have mempool->free police the max parameter, leave that to the guard. This makes 'max' parameter mis-sizing much less problematic. Message-ID: commit b8e04e61410068ea69e5e291fabec233747072b0 Author: Poul-Henning Kamp Date: Sun Apr 22 20:05:22 2012 +0000 Don't have mempool->free police the max parameter, leave that to the guard. This makes 'max' parameter mis-sizing much less problematic. Spotted by: scoof diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c index 8906406..badb621 100644 --- a/bin/varnishd/cache/cache_mempool.c +++ b/bin/varnishd/cache/cache_mempool.c @@ -329,9 +329,6 @@ MPL_Free(struct mempool *mpl, void *item) if (mi->size < *mpl->cur_size) { mpl->vsc->toosmall++; VTAILQ_INSERT_HEAD(&mpl->surplus, mi, list); - } else if (mpl->n_pool >= mpl->param->max_pool) { - mpl->vsc->surplus++; - VTAILQ_INSERT_HEAD(&mpl->surplus, mi, list); } else { mpl->vsc->pool = ++mpl->n_pool; mi->touched = mpl->t_now; From phk at FreeBSD.org Thu Dec 18 09:27:44 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:44 +0100 Subject: [experimental-ims] 4601ddc Fix some stats-counter issues scoof pointed out yesterday: Message-ID: commit 4601ddcd8db6926bd6315c642863a123e4c15b01 Author: Poul-Henning Kamp Date: Mon Apr 23 06:45:30 2012 +0000 Fix some stats-counter issues scoof pointed out yesterday: n_objsendfile is of course no longer relevant. That makes n_objwrite pointless. n_objoverflow was unused. and threads_failed were not incremented where it should be. diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 29d467c..bf3ab8e 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -340,7 +340,7 @@ pool_breed(struct pool *qp, const pthread_attr_t *tp_attr) VSL(SLT_Debug, 0, "Create worker thread failed %d %s", errno, strerror(errno)); Lck_Lock(&pool_mtx); - VSC_C_main->threads_limited++; + VSC_C_main->threads_failed++; Lck_Unlock(&pool_mtx); VTIM_sleep(cache_param->wthread_fail_delay * 1e-3); } else { diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index 8579e37..cd45945 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -168,8 +168,6 @@ res_WriteGunzipObj(const struct sess *sp) CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); u += st->len; - VSC_C_main->n_objwrite++; - i = VGZ_WrwGunzip(sp->wrk, vg, st->ptr, st->len); /* XXX: error check */ (void)i; @@ -215,7 +213,6 @@ res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high) ptr += len; sp->wrk->acct_tmp.bodybytes += len; - VSC_C_main->n_objwrite++; (void)WRW_Write(sp->wrk, st->ptr + off, len); } assert(u == sp->req->obj->len); diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h index 36bb629..2938d7a 100644 --- a/include/tbl/vsc_f_main.h +++ b/include/tbl/vsc_f_main.h @@ -304,22 +304,6 @@ VSC_F(losthdr, uint64_t, 0, 'a', "" ) -VSC_F(n_objsendfile, uint64_t, 0, 'a', - "Objects sent with sendfile", - "The number of objects sent with the sendfile system call. If enabled " - "sendfile will be used on object larger than a certain size." -) -VSC_F(n_objwrite, uint64_t, 0, 'a', - "Objects sent with write", - "The number of objects sent with regular write calls." - "Writes are used when the objects are too small for sendfile " - "or if the sendfile call has been disabled" -) -VSC_F(n_objoverflow, uint64_t, 1, 'a', - "Objects overflowing workspace", - "" -) - VSC_F(s_sess, uint64_t, 1, 'a', "Total Sessions", "" From phk at FreeBSD.org Thu Dec 18 09:27:44 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:44 +0100 Subject: [experimental-ims] 5961b65 Fix a flexelint nit Message-ID: commit 5961b65fa57fb8eb02ac6630beec05f6a5fe28b1 Author: Poul-Henning Kamp Date: Mon Apr 23 07:10:57 2012 +0000 Fix a flexelint nit diff --git a/lib/libvcl/vcc_symb.c b/lib/libvcl/vcc_symb.c index 222680f..4066e56 100644 --- a/lib/libvcl/vcc_symb.c +++ b/lib/libvcl/vcc_symb.c @@ -70,7 +70,7 @@ vcc_AddSymbol(struct vcc *tl, const char *nb, int l, enum symkind kind) } ALLOC_OBJ(sym, SYMBOL_MAGIC); AN(sym); - sym->name = malloc(l + 1); + sym->name = malloc(l + 1L); AN(sym->name); memcpy(sym->name, nb, l); sym->name[l] = '\0'; From phk at FreeBSD.org Thu Dec 18 09:27:44 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:44 +0100 Subject: [experimental-ims] 353073c Make VSC->hcb_nolock a worker local counter for consistency, locking it would seriously miss the point. Message-ID: commit 353073cf26dc461fdc0e5f11e40fdebc34c0c763 Author: Poul-Henning Kamp Date: Mon Apr 23 08:00:01 2012 +0000 Make VSC->hcb_nolock a worker local counter for consistency, locking it would seriously miss the point. Spotted by: scoof diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c index 08be561..46f7f95 100644 --- a/bin/varnishd/hash/hash_critbit.c +++ b/bin/varnishd/hash/hash_critbit.c @@ -432,7 +432,7 @@ hcb_lookup(struct worker *wrk, const void *digest, struct objhead **noh) /* First try in read-only mode without holding a lock */ - VSC_C_main->hcb_nolock++; + wrk->stats.hcb_nolock++; oh = hcb_insert(wrk, &hcb_root, digest, NULL); if (oh != NULL) { Lck_Lock(&oh->mtx); diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h index 2938d7a..bc79d5b 100644 --- a/include/tbl/vsc_f_main.h +++ b/include/tbl/vsc_f_main.h @@ -454,7 +454,7 @@ VSC_F(bans_dups, uint64_t, 0, 'c', /**********************************************************************/ -VSC_F(hcb_nolock, uint64_t, 0, 'a', +VSC_F(hcb_nolock, uint64_t, 1, 'a', "HCB Lookups without lock", "" ) From phk at FreeBSD.org Thu Dec 18 09:27:44 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:44 +0100 Subject: [experimental-ims] 017ba98 Add an explicit macro_undef() function so we don't pass a NULL argument to a printflike function. Message-ID: commit 017ba98d6bc565756ade72736c2e6e9a8592f0d1 Author: Poul-Henning Kamp Date: Mon Apr 23 11:35:56 2012 +0000 Add an explicit macro_undef() function so we don't pass a NULL argument to a printflike function. diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index b8556ce..99cf840 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -87,6 +87,8 @@ macro_def(struct vtclog *vl, const char *instance, const char *name, struct macro *m; va_list ap; + AN(fmt); + if (instance != NULL) { bprintf(buf1, "%s_%s", instance, name); name = buf1; @@ -96,23 +98,40 @@ macro_def(struct vtclog *vl, const char *instance, const char *name, VTAILQ_FOREACH(m, ¯o_list, list) if (!strcmp(name, m->name)) break; - if (m == NULL && fmt != NULL) { + if (m == NULL) { m = calloc(sizeof *m, 1); AN(m); REPLACE(m->name, name); VTAILQ_INSERT_TAIL(¯o_list, m, list); } - if (fmt != NULL) { - AN(m); - va_start(ap, fmt); - free(m->val); - m->val = NULL; - vbprintf(buf2, fmt, ap); - va_end(ap); - m->val = strdup(buf2); - AN(m->val); - vtc_log(vl, 4, "macro def %s=%s", name, m->val); - } else if (m != NULL) { + AN(m); + va_start(ap, fmt); + free(m->val); + m->val = NULL; + vbprintf(buf2, fmt, ap); + va_end(ap); + m->val = strdup(buf2); + AN(m->val); + vtc_log(vl, 4, "macro def %s=%s", name, m->val); + AZ(pthread_mutex_unlock(¯o_mtx)); +} + +void +macro_undef(struct vtclog *vl, const char *instance, const char *name) +{ + char buf1[256]; + struct macro *m; + + if (instance != NULL) { + bprintf(buf1, "%s_%s", instance, name); + name = buf1; + } + + AZ(pthread_mutex_lock(¯o_mtx)); + VTAILQ_FOREACH(m, ¯o_list, list) + if (!strcmp(name, m->name)) + break; + if (m != NULL) { vtc_log(vl, 4, "macro undef %s", name); VTAILQ_REMOVE(¯o_list, m, list); free(m->name); diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index be20cb9..333a1df 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -87,6 +87,7 @@ void vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx, int exec_file(const char *fn, const char *script, const char *tmpdir, char *logbuf, unsigned loglen); +void macro_undef(struct vtclog *vl, const char *instance, const char *name); void macro_def(struct vtclog *vl, const char *instance, const char *name, const char *fmt, ...) __printflike(4, 5); diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 75c4f30..eaa680f 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -143,9 +143,9 @@ server_delete(struct server *s) { CHECK_OBJ_NOTNULL(s, SERVER_MAGIC); - macro_def(s->vl, s->name, "addr", NULL); - macro_def(s->vl, s->name, "port", NULL); - macro_def(s->vl, s->name, "sock", NULL); + macro_undef(s->vl, s->name, "addr"); + macro_undef(s->vl, s->name, "port"); + macro_undef(s->vl, s->name, "sock"); vtc_logclose(s->vl); free(s->name); /* XXX: MEMLEAK (?) (VSS ??) */ diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index c00a143..e493ae7 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -522,9 +522,9 @@ varnish_stop(struct varnish *v) varnish_launch(v); if (vtc_error) return; - macro_def(v->vl, v->name, "addr", NULL); - macro_def(v->vl, v->name, "port", NULL); - macro_def(v->vl, v->name, "sock", NULL); + macro_undef(v->vl, v->name, "addr"); + macro_undef(v->vl, v->name, "port"); + macro_undef(v->vl, v->name, "sock"); vtc_log(v->vl, 2, "Stop"); (void)varnish_ask_cli(v, "stop", NULL); while (1) { From phk at FreeBSD.org Thu Dec 18 09:27:44 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:44 +0100 Subject: [experimental-ims] cab6375 Extra asserts to try to catch a weird panic on tinderbox::osx Message-ID: commit cab6375cab80b2220c9eb0de8b08a304100aaac6 Author: Poul-Henning Kamp Date: Mon Apr 23 12:15:23 2012 +0000 Extra asserts to try to catch a weird panic on tinderbox::osx diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index bf3ab8e..a723a9e 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -194,6 +194,7 @@ pool_accept(struct worker *wrk, void *arg) return; } VTAILQ_REMOVE(&pp->idle_queue, &wrk2->task, list); + AZ(wrk2->task.func); Lck_Unlock(&pp->mtx); assert(sizeof *wa2 == WS_Reserve(wrk2->aws, sizeof *wa2)); wa2 = (void*)wrk2->aws->f; @@ -227,6 +228,7 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how) wrk = pool_getidleworker(pp, 0); if (wrk != NULL) { VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list); + AZ(wrk->task.func); Lck_Unlock(&pp->mtx); wrk->task.func = task->func; wrk->task.priv = task->priv; @@ -298,6 +300,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk) wrk->lastused = VTIM_real(); wrk->task.func = NULL; wrk->task.priv = wrk; + AZ(wrk->task.func); VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list); if (!stats_clean) WRK_SumStat(wrk); @@ -427,6 +430,7 @@ pool_herder(void *priv) if (wrk != NULL && (wrk->lastused < t_idle || pp->nthr > cache_param->wthread_max)) { VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list); + AZ(wrk->task.func); } else wrk = NULL; Lck_Unlock(&pp->mtx); From phk at FreeBSD.org Thu Dec 18 09:27:44 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:44 +0100 Subject: [experimental-ims] 8bf6c6d Also reflect the VCC exit code through if -C is specified. Message-ID: commit 8bf6c6d2891b50b04af463791fc380c359b1d02c Author: Poul-Henning Kamp Date: Mon Apr 23 16:17:14 2012 +0000 Also reflect the VCC exit code through if -C is specified. Fixes #1069 diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 12433ad..98dc4c4 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -394,18 +394,16 @@ mgt_vcc_default(const char *b_arg, const char *f_arg, char *vcl, int C_flag) if (VSB_len(sb) > 0) fprintf(stderr, "%s", VSB_data(sb)); VSB_delete(sb); - if (C_flag) { - if (vf != NULL) - AZ(unlink(vf)); - return (0); - } + if (C_flag && vf != NULL) + AZ(unlink(vf)); if (vf == NULL) { fprintf(stderr, "\nVCL compilation failed\n"); return (1); + } else { + vp = mgt_vcc_add(buf, vf); + vp->active = 1; + return (0); } - vp = mgt_vcc_add(buf, vf); - vp->active = 1; - return (0); } /*--------------------------------------------------------------------*/ From apj at mutt.dk Thu Dec 18 09:27:44 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:44 +0100 Subject: [experimental-ims] ad9356a Clean up docs about esi:remove and +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The and constructs can be used to present +appropriate content whether or not ESI is available, for example you can +include content when ESI is available or link to it when it is not. +ESI processors will remove the start ("") when +the page is processed, while still processing the contents. If the page +is not processed, it will remain, becoming an HTML/XML comment tag. +ESI processors will remove tags and all content contained +in them, allowing you to only render the content when the page is not +being ESI-processed. +For example:: - - www.example.com + The license - -Example: -~~~~~~~~~~~~~~~~~~~~~~~~ - - -This is a special construct to allow HTML marked up with ESI to render -without processing. ESI Processors will remove the start ("") when the page is processed, while still processing the -contents. If the page is not processed, it will remain, becoming an -HTML/XML comment tag. For example:: - - -This assures that the ESI markup will not interfere with the rendering -of the final HTML if not processed. - - +

The full text of the license:

+ + --> From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] 562c67d Make it possible for Open/Net/DragonFly-BSD to find readline.h Message-ID: commit 562c67db250ed6decea1015edb5b6170096c123a Author: Poul-Henning Kamp Date: Thu Apr 26 16:10:17 2012 +0000 Make it possible for Open/Net/DragonFly-BSD to find readline.h diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index f3b8b69..aff9336 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -35,6 +35,8 @@ # include # ifdef HAVE_EDIT_READLINE_READLINE_H # include +# elif HAVE_READLINE_READLINE_H +# include # else # include # endif From tfheen at varnish-software.com Thu Dec 18 09:27:45 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] a8ac09b Add missing + in docs Message-ID: commit a8ac09bb97bc2903c0696b0af8c659bb57d851dd Author: Tollef Fog Heen Date: Fri Apr 27 12:53:44 2012 +0200 Add missing + in docs Thanks to Jeff Williams for pointing this out. diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst index 846ca82..22056a7 100644 --- a/doc/sphinx/tutorial/purging.rst +++ b/doc/sphinx/tutorial/purging.rst @@ -146,7 +146,7 @@ You can use the following template to write ban lurker friendly bans:: if (client.ip !~ purge) { error 401 "Not allowed"; } - ban("obj.http.x-url ~ " req.url); # Assumes req.url is a regex. This might be a bit too simple + ban("obj.http.x-url ~ " + req.url); # Assumes req.url is a regex. This might be a bit too simple } } From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] 9c449d0 Make things compile on OpenBSD Message-ID: commit 9c449d0417df9e1268f81271dbfd11f700aa24bd Author: Poul-Henning Kamp Date: Fri Apr 27 13:43:51 2012 +0000 Make things compile on OpenBSD Submitted by: Brad Smith diff --git a/bin/varnishd/cache/cache_dir_dns.c b/bin/varnishd/cache/cache_dir_dns.c index 9cad3f6..ee411dc 100644 --- a/bin/varnishd/cache/cache_dir_dns.c +++ b/bin/varnishd/cache/cache_dir_dns.c @@ -29,6 +29,8 @@ #include "config.h" +#include +#include #include #include diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index cbe5637..10658cd 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -31,6 +31,8 @@ #include "config.h" +#include +#include #include #include diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c index cbea81f..3b0c1a7 100644 --- a/bin/varnishd/mgt/mgt_cli.c +++ b/bin/varnishd/mgt/mgt_cli.c @@ -31,6 +31,7 @@ #include "config.h" +#include #include #include diff --git a/bin/varnishd/storage/stevedore_utils.c b/bin/varnishd/storage/stevedore_utils.c index 89c6027..2364289 100644 --- a/bin/varnishd/storage/stevedore_utils.c +++ b/bin/varnishd/storage/stevedore_utils.c @@ -34,6 +34,7 @@ #include #include #ifdef HAVE_SYS_MOUNT_H +# include # include #endif #ifdef HAVE_SYS_STATVFS_H From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] 563b21e Missed these two in yesterdays OpenBSD compat commit. Message-ID: commit 563b21e1c6f4ea9891b527877cf36d0df8380cc4 Author: Poul-Henning Kamp Date: Sat Apr 28 07:16:13 2012 +0000 Missed these two in yesterdays OpenBSD compat commit. diff --git a/lib/libvarnish/vsha256.c b/lib/libvarnish/vsha256.c index 10b7a6a..0a521b2 100644 --- a/lib/libvarnish/vsha256.c +++ b/lib/libvarnish/vsha256.c @@ -29,6 +29,7 @@ #include "config.h" #ifdef HAVE_SYS_ENDIAN_H +#include #include #define VBYTE_ORDER _BYTE_ORDER #define VBIG_ENDIAN _BIG_ENDIAN diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index d1f6771..5b5c0aa 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -28,6 +28,8 @@ #include "config.h" +#include +#include #include #include From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] f564428 Be consistent about what environment we test the compiled VCL in. Message-ID: commit f564428bb480edf3c3d170087a227dee2797bdbb Author: Poul-Henning Kamp Date: Sat Apr 28 07:16:33 2012 +0000 Be consistent about what environment we test the compiled VCL in. diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 98dc4c4..e419261 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -186,7 +186,9 @@ run_dlopen(void *priv) of = priv; - /* Try to load the object into the management process */ + mgt_sandbox(); + + /* Try to load the object into this sub-process */ if ((dlh = dlopen(of, RTLD_NOW | RTLD_LOCAL)) == NULL) { fprintf(stderr, "Compiled VCL program failed to load:\n %s\n", From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] 6d5c733 Variable rename for easier searching. Message-ID: commit 6d5c7338eddf7f1fdeef11193db1519244319380 Author: Poul-Henning Kamp Date: Sat Apr 28 08:44:11 2012 +0000 Variable rename for easier searching. diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index f6493ad..7f63e2f 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -56,7 +56,7 @@ #define MAGIC_INIT_STRING "\001" struct params mgt_param; static int nparspec; -static struct parspec const ** parspec; +static struct parspec const ** parspecs; static int margin; /*--------------------------------------------------------------------*/ @@ -67,8 +67,8 @@ mcf_findpar(const char *name) int i; for (i = 0; i < nparspec; i++) - if (!strcmp(parspec[i]->name, name)) - return (parspec[i]); + if (!strcmp(parspecs[i]->name, name)) + return (parspecs[i]); return (NULL); } @@ -1289,7 +1289,7 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv) else lfmt = 1; for (i = 0; i < nparspec; i++) { - pp = parspec[i]; + pp = parspecs[i]; if (av[2] != NULL && !lfmt && strcmp(pp->name, av[2])) continue; VCLI_Out(cli, "%-*s ", margin, pp->name); @@ -1398,12 +1398,12 @@ MCF_AddParams(const struct parspec *ps) margin = strlen(pp->name) + 1; n++; } - parspec = realloc(parspec, (1L + nparspec + n) * sizeof *parspec); - XXXAN(parspec); + parspecs = realloc(parspecs, (1L + nparspec + n) * sizeof *parspecs); + XXXAN(parspecs); for (pp = ps; pp->name != NULL; pp++) - parspec[nparspec++] = pp; - parspec[nparspec] = NULL; - qsort (parspec, nparspec, sizeof parspec[0], parspec_cmp); + parspecs[nparspec++] = pp; + parspecs[nparspec] = NULL; + qsort (parspecs, nparspec, sizeof parspecs[0], parspec_cmp); } /*-------------------------------------------------------------------- @@ -1417,7 +1417,7 @@ MCF_SetDefaults(struct cli *cli) int i; for (i = 0; i < nparspec; i++) { - pp = parspec[i]; + pp = parspecs[i]; if (cli != NULL) VCLI_Out(cli, "Set Default for %s = %s\n", pp->name, pp->def); @@ -1453,7 +1453,7 @@ MCF_DumpRst(void) printf("\n.. The following is the autogenerated " "output from varnishd -x dumprst\n\n"); for (i = 0; i < nparspec; i++) { - pp = parspec[i]; + pp = parspecs[i]; printf("%s\n", pp->name); if (pp->units != NULL && *pp->units != '\0') printf("\t- Units: %s\n", pp->units); From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] 2361947 Another bit of the OpenBSD commit I missed. Message-ID: commit 2361947fca3b004a87c08fdefa74231520881e7b Author: Poul-Henning Kamp Date: Sat Apr 28 08:44:33 2012 +0000 Another bit of the OpenBSD commit I missed. diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index aff9336..f4c262c 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -29,6 +29,7 @@ #include "config.h" +#include #include #ifdef HAVE_LIBEDIT From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] 0b00877 Add a -r ("read-only") argument which can protect parameters from subsequent changes. Message-ID: commit 0b00877030e426e9ed3b867772074fc2e0b9df63 Author: Poul-Henning Kamp Date: Sat Apr 28 09:19:35 2012 +0000 Add a -r ("read-only") argument which can protect parameters from subsequent changes. diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index ffaef56..c319e57 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -69,6 +69,7 @@ const void *pick(const struct choice *cp, const char *which, const char *kind); /* mgt_param.c */ void MCF_ParamInit(struct cli *); void MCF_ParamSet(struct cli *, const char *param, const char *val); +void MCF_ParamProtect(struct cli *, const char *arg); void MCF_DumpRst(void); extern struct params mgt_param; diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 6ad7454..13799f9 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -127,6 +127,7 @@ usage(void) fprintf(stderr, FMT, "-n dir", "varnishd working directory"); fprintf(stderr, FMT, "-P file", "PID file"); fprintf(stderr, FMT, "-p param=value", "set parameter"); + fprintf(stderr, FMT, "-r param[,param...]", "make parameter read-only"); fprintf(stderr, FMT, "-s kind[,storageoptions]", "Backend storage specification"); fprintf(stderr, FMT, "", " -s malloc"); @@ -369,7 +370,7 @@ main(int argc, char * const *argv) cli_check(cli); while ((o = getopt(argc, argv, - "a:b:Cdf:Fg:h:i:l:L:M:n:P:p:S:s:T:t:u:Vx:w:")) != -1) + "a:b:Cdf:Fg:h:i:l:L:M:n:P:p:r:S:s:T:t:u:Vx:w:")) != -1) switch (o) { case 'a': MCF_ParamSet(cli, "listen_address", optarg); @@ -432,6 +433,10 @@ main(int argc, char * const *argv) MCF_ParamSet(cli, optarg, p); cli_check(cli); break; + case 'r': + MCF_ParamProtect(cli, optarg); + cli_check(cli); + break; case 's': s_arg_given = 1; STV_Config(optarg); diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 7f63e2f..e79b7e9 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -668,6 +668,9 @@ tweak_poolparam(struct cli *cli, const struct parspec *par, const char *arg) "\nNB: Do not change this parameter, unless a developer tell " \ "you to do so." +#define PROTECTED_TEXT \ + "\nNB: This parameter is protected and can not be changed." + #define MEMPOOL_TEXT \ "The three numbers are:\n" \ " min_pool -- minimum size of free pool.\n" \ @@ -1319,6 +1322,8 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv) mcf_wrap(cli, MUST_RESTART_TEXT); if (pp->flags & WIZARD) mcf_wrap(cli, WIZARD_TEXT); + if (pp->flags & PROTECTED) + mcf_wrap(cli, PROTECTED_TEXT); if (!lfmt) return; else @@ -1331,6 +1336,43 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv) } } +/*-------------------------------------------------------------------- + * Mark paramters as protected + */ + +void +MCF_ParamProtect(struct cli *cli, const char *args) +{ + char **av; + struct parspec *pp; + int i, j; + + av = VAV_Parse(args, NULL, ARGV_COMMA); + if (av[0] != NULL) { + VCLI_Out(cli, "Parse error: %s", av[0]); + VCLI_SetResult(cli, CLIS_PARAM); + VAV_Free(av); + return; + } + for (i = 1; av[i] != NULL; i++) { + for (j = 0; j < nparspec; j++) + if (!strcmp(parspecs[j]->name, av[i])) + break; + if (j == nparspec) { + VCLI_Out(cli, "Unknown parameter %s", av[i]); + VCLI_SetResult(cli, CLIS_PARAM); + VAV_Free(av); + return; + } + pp = calloc(sizeof *pp, 1L); + XXXAN(pp); + memcpy(pp, parspecs[j], sizeof *pp); + pp->flags |= PROTECTED; + parspecs[j] = pp; + } + VAV_Free(av); +} + /*--------------------------------------------------------------------*/ void @@ -1344,6 +1386,11 @@ MCF_ParamSet(struct cli *cli, const char *param, const char *val) VCLI_Out(cli, "Unknown parameter \"%s\".", param); return; } + if (pp->flags & PROTECTED) { + VCLI_SetResult(cli, CLIS_AUTH); + VCLI_Out(cli, "parameter \"%s\" is protected.", param); + return; + } pp->func(cli, pp, val); if (cli->result == CLIS_OK && heritage.param != NULL) diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h index a5d4d75..b4538d2 100644 --- a/bin/varnishd/mgt/mgt_param.h +++ b/bin/varnishd/mgt/mgt_param.h @@ -45,6 +45,7 @@ struct parspec { #define MUST_RESTART (1<<2) #define MUST_RELOAD (1<<3) #define WIZARD (1<<4) +#define PROTECTED (1<<5) const char *def; const char *units; }; diff --git a/bin/varnishtest/tests/c00051.vtc b/bin/varnishtest/tests/c00051.vtc new file mode 100644 index 0000000..78fdc81 --- /dev/null +++ b/bin/varnishtest/tests/c00051.vtc @@ -0,0 +1,6 @@ +varnishtest "test parameter protection" + +varnish v1 -arg "-r cli_timeout" + +varnish v1 -cliok "param.show cli_timeout" +varnish v1 -clierr 107 "param.set cli_timeout 1m" From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] 2b623ea Add a parameter to disable inline-C code in VCL Message-ID: commit 2b623eafd3de8f9dfdc814c30d264e9dc9090e33 Author: Poul-Henning Kamp Date: Sat Apr 28 15:06:14 2012 +0000 Add a parameter to disable inline-C code in VCL diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index c319e57..0ad01de 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -105,6 +105,7 @@ extern char *mgt_cc_cmd; extern const char *mgt_vcl_dir; extern const char *mgt_vmod_dir; extern unsigned mgt_vcc_err_unref; +extern unsigned mgt_vcc_allow_inline_c; #define REPORT0(pri, fmt) \ do { \ diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index e79b7e9..b72b0df 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1177,11 +1177,17 @@ static const struct parspec input_parspec[] = { ".", #endif NULL }, + { "vcc_err_unref", tweak_bool, &mgt_vcc_err_unref, 0, 0, "Unreferenced VCL objects result in error.\n", 0, "on", "bool" }, + { "vcc_allow_inline_c", tweak_bool, &mgt_vcc_allow_inline_c, 0, 0, + "Allow inline C code in VCL.\n", + 0, + "on", "bool" }, + { "pcre_match_limit", tweak_uint, &mgt_param.vre_limits.match, diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index e419261..269a453 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -63,6 +63,7 @@ char *mgt_cc_cmd; const char *mgt_vcl_dir; const char *mgt_vmod_dir; unsigned mgt_vcc_err_unref; +unsigned mgt_vcc_allow_inline_c; static struct vcc *vcc; @@ -139,6 +140,7 @@ run_vcc(void *priv) VCC_VCL_dir(vcc, mgt_vcl_dir); VCC_VMOD_dir(vcc, mgt_vmod_dir); VCC_Err_Unref(vcc, mgt_vcc_err_unref); + VCC_Allow_InlineC(vcc, mgt_vcc_allow_inline_c); csrc = VCC_Compile(vcc, sb, vp->vcl); AZ(VSB_finish(sb)); if (VSB_len(sb)) From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] fb6f3c5 Add parameter vcc_unsafe_path which allows '/' in include "..." and 'import ... from ...'. Default is on (= no change) Message-ID: commit fb6f3c54177d7aa85a59d10da3bb9b549c0b9aa8 Author: Poul-Henning Kamp Date: Sat Apr 28 16:03:17 2012 +0000 Add parameter vcc_unsafe_path which allows '/' in include "..." and 'import ... from ...'. Default is on (= no change) diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 0ad01de..65dbc02 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -106,6 +106,7 @@ extern const char *mgt_vcl_dir; extern const char *mgt_vmod_dir; extern unsigned mgt_vcc_err_unref; extern unsigned mgt_vcc_allow_inline_c; +extern unsigned mgt_vcc_unsafe_path; #define REPORT0(pri, fmt) \ do { \ diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index b72b0df..885d2ce 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1188,6 +1188,11 @@ static const struct parspec input_parspec[] = { 0, "on", "bool" }, + { "vcc_unsafe_path", tweak_bool, &mgt_vcc_unsafe_path, 0, 0, + "Allow '/' in vmod & include paths.\n" + "Allow 'import ... from ...'.\n", + 0, + "on", "bool" }, { "pcre_match_limit", tweak_uint, &mgt_param.vre_limits.match, diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 269a453..9b46da3 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -64,6 +64,7 @@ const char *mgt_vcl_dir; const char *mgt_vmod_dir; unsigned mgt_vcc_err_unref; unsigned mgt_vcc_allow_inline_c; +unsigned mgt_vcc_unsafe_path; static struct vcc *vcc; @@ -141,6 +142,7 @@ run_vcc(void *priv) VCC_VMOD_dir(vcc, mgt_vmod_dir); VCC_Err_Unref(vcc, mgt_vcc_err_unref); VCC_Allow_InlineC(vcc, mgt_vcc_allow_inline_c); + VCC_Unsafe_Path(vcc, mgt_vcc_unsafe_path); csrc = VCC_Compile(vcc, sb, vp->vcl); AZ(VSB_finish(sb)); if (VSB_len(sb)) diff --git a/bin/varnishtest/tests/c00052.vtc b/bin/varnishtest/tests/c00052.vtc new file mode 100644 index 0000000..e640230 --- /dev/null +++ b/bin/varnishtest/tests/c00052.vtc @@ -0,0 +1,51 @@ +varnishtest "Test disabling inline C code" + +server s1 { + rxreq + txresp +} -start + +varnish v1 + +varnish v1 -cliok "param.show vcc_allow_inline_c" + +varnish v1 -vcl+backend { + C{ getpid(); }C +} + +varnish v1 -cliok "param.set vcc_allow_inline_c false" + +varnish v1 -badvcl { + backend default { + .host = "${s1_sock}"; + } + C{ getpid(); }C +} + +varnish v1 -badvcl { + backend default { + .host = "${s1_sock}"; + } + sub vcl_recv { + C{ getpid(); }C + } +} + +varnish v1 -cliok "param.set vcc_allow_inline_c true" + +varnish v1 -vcl+backend { + sub vcl_recv { + C{ getpid(); }C + } +} + +varnish v1 -vcl+backend { + C{ extern int getpid(); }C +} + +varnish v1 -start + +client c1 { + txreq + rxresp +} -run diff --git a/bin/varnishtest/tests/c00053.vtc b/bin/varnishtest/tests/c00053.vtc new file mode 100644 index 0000000..c326293 --- /dev/null +++ b/bin/varnishtest/tests/c00053.vtc @@ -0,0 +1,29 @@ +varnishtest "Test inclide vs. unsafe_path" + +server s1 { + rxreq + txresp -hdr "foo: bAr" -hdr "bar: fOo" -bodylen 4 +} -start + +shell "echo > ${pwd}/_.c00053" + +varnish v1 -vcl+backend { + include "${pwd}/_.c00053"; +} + +varnish v1 -cliok "param.set vcc_unsafe_path off" + +varnish v1 -badvcl { + backend default { + .host = "${s1_sock}"; + } + include "${pwd}/_.c00053"; +} + +varnish v1 -cliok "param.set vcl_dir ${pwd}" + +varnish v1 -vcl+backend { + include "_.c00053"; +} + +shell "rm -f ${pwd}/_.c00053" diff --git a/bin/varnishtest/tests/m00008.vtc b/bin/varnishtest/tests/m00008.vtc new file mode 100644 index 0000000..3d1d665 --- /dev/null +++ b/bin/varnishtest/tests/m00008.vtc @@ -0,0 +1,23 @@ +varnishtest "Test std vmod vs. unsafe_path" + +server s1 { + rxreq + txresp -hdr "foo: bAr" -hdr "bar: fOo" -bodylen 4 +} -start + +varnish v1 -vcl+backend { + import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ; +} + +varnish v1 -cliok "param.set vcc_unsafe_path off" + +varnish v1 -badvcl { + backend default { .host = "${s1_sock}"; } + import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ; +} + +varnish v1 -cliok "param.set vmod_dir ${topbuild}/lib/libvmod_std/.libs/" + +varnish v1 -vcl+backend { + import std; +} diff --git a/include/libvcl.h b/include/libvcl.h index e046db9..5c50c34 100644 --- a/include/libvcl.h +++ b/include/libvcl.h @@ -35,5 +35,7 @@ void VCC_Default_VCL(struct vcc *, const char *str); void VCC_VCL_dir(struct vcc *, const char *str); void VCC_VMOD_dir(struct vcc *, const char *str); void VCC_Err_Unref(struct vcc *tl, unsigned u); +void VCC_Allow_InlineC(struct vcc *tl, unsigned u); +void VCC_Unsafe_Path(struct vcc *tl, unsigned u); char *VCC_Compile(const struct vcc *, struct vsb *sb, const char *b); diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c index 66d89f5..c9fe573 100644 --- a/lib/libvcl/vcc_compile.c +++ b/lib/libvcl/vcc_compile.c @@ -411,6 +411,10 @@ vcc_file_source(const struct vcc *tl, struct vsb *sb, const char *fn) char *f; struct source *sp; + if (!tl->unsafe_path && strchr(fn, '/') != NULL) { + VSB_printf(sb, "Include path is unsafe '%s'\n", fn); + return (NULL); + } f = VFIL_readfile(tl->vcl_dir, fn, NULL); if (f == NULL) { VSB_printf(sb, "Cannot read file '%s': %s\n", @@ -487,6 +491,8 @@ vcc_NewVcc(const struct vcc *tl0) REPLACE(tl->vmod_dir, tl0->vmod_dir); tl->vars = tl0->vars; tl->err_unref = tl0->err_unref; + tl->allow_inline_c = tl0->allow_inline_c; + tl->unsafe_path = tl0->unsafe_path; } else { tl->err_unref = 1; } @@ -763,7 +769,7 @@ VCC_VMOD_dir(struct vcc *tl, const char *str) } /*-------------------------------------------------------------------- - * Configure default + * Configure settings */ void @@ -773,3 +779,19 @@ VCC_Err_Unref(struct vcc *tl, unsigned u) CHECK_OBJ_NOTNULL(tl, VCC_MAGIC); tl->err_unref = u; } + +void +VCC_Allow_InlineC(struct vcc *tl, unsigned u) +{ + + CHECK_OBJ_NOTNULL(tl, VCC_MAGIC); + tl->allow_inline_c = u; +} + +void +VCC_Unsafe_Path(struct vcc *tl, unsigned u) +{ + + CHECK_OBJ_NOTNULL(tl, VCC_MAGIC); + tl->unsafe_path = u; +} diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h index b64564e..c85af5b 100644 --- a/lib/libvcl/vcc_compile.h +++ b/lib/libvcl/vcc_compile.h @@ -193,6 +193,8 @@ struct vcc { unsigned nvmodpriv; unsigned err_unref; + unsigned allow_inline_c; + unsigned unsafe_path; }; struct var { diff --git a/lib/libvcl/vcc_parse.c b/lib/libvcl/vcc_parse.c index d8e74a4..1b7ee88 100644 --- a/lib/libvcl/vcc_parse.c +++ b/lib/libvcl/vcc_parse.c @@ -153,10 +153,16 @@ vcc_Compound(struct vcc *tl) Fb(tl, 1, "}\n"); return; case CSRC: - Fb(tl, 1, "%.*s\n", - (int) (tl->t->e - (tl->t->b + 2)), - tl->t->b + 1); - vcc_NextToken(tl); + if (tl->allow_inline_c) { + Fb(tl, 1, "%.*s\n", + (int) (tl->t->e - (tl->t->b + 2)), + tl->t->b + 1); + vcc_NextToken(tl); + } else { + VSB_printf(tl->sb, + "Inline-C not allowed"); + vcc_ErrWhere(tl, tl->t); + } break; case EOI: VSB_printf(tl->sb, @@ -273,9 +279,16 @@ vcc_Parse(struct vcc *tl) ERRCHK(tl); switch (tl->t->tok) { case CSRC: - Fc(tl, 0, "%.*s\n", - (int) (tl->t->e - (tl->t->b + 4)), tl->t->b + 2); - vcc_NextToken(tl); + if (tl->allow_inline_c) { + Fc(tl, 0, "%.*s\n", + (int) (tl->t->e - (tl->t->b + 4)), + tl->t->b + 2); + vcc_NextToken(tl); + } else { + VSB_printf(tl->sb, + "Inline-C not allowed"); + vcc_ErrWhere(tl, tl->t); + } break; case EOI: break; diff --git a/lib/libvcl/vcc_vmod.c b/lib/libvcl/vcc_vmod.c index bd9f366..4ce4f95 100644 --- a/lib/libvcl/vcc_vmod.c +++ b/lib/libvcl/vcc_vmod.c @@ -56,7 +56,6 @@ vcc_ParseImport(struct vcc *tl) ExpectErr(tl, ID); mod = tl->t; - vcc_NextToken(tl); osym = VCC_FindSymbol(tl, mod, SYM_NONE); @@ -83,6 +82,14 @@ vcc_ParseImport(struct vcc *tl) sym->def_e = tl->t; if (tl->t->tok == ID) { + if (!tl->unsafe_path) { + VSB_printf(tl->sb, + "'import ... from path...'" + " not allowed.\nAt:"); + vcc_ErrToken(tl, tl->t); + vcc_ErrWhere(tl, tl->t); + return; + } if (!vcc_IdIs(tl->t, "from")) { VSB_printf(tl->sb, "Expected 'from path...' at "); vcc_ErrToken(tl, tl->t); From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] 1d588b3 Silence an annoying message when running varnishd -C Message-ID: commit 1d588b3f35772d55b4f7ea7ee471ccd3a018d48b Author: Poul-Henning Kamp Date: Sun Apr 29 18:22:08 2012 +0000 Silence an annoying message when running varnishd -C diff --git a/bin/varnishtest/tests/a00009.vtc b/bin/varnishtest/tests/a00009.vtc index 90569cc..4d35f85 100644 --- a/bin/varnishtest/tests/a00009.vtc +++ b/bin/varnishtest/tests/a00009.vtc @@ -1,3 +1,3 @@ varnishtest "See that the VCL compiler works" -shell "cd ${topbuild}/bin/varnishd && ./varnishd -b 127.0.0.1:80 -C -n ${tmpdir} > /dev/null" +shell "cd ${topbuild}/bin/varnishd && ./varnishd -b 127.0.0.1:80 -C -n ${tmpdir} > /dev/null 2>&1" From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] ee1f86d Writing an article about the utility of asserts, I noticed some inconsistent whitespace with asserts. Message-ID: commit ee1f86d208ae2c3e4e72d956419bae1b4a20dc3c Author: Poul-Henning Kamp Date: Mon Apr 30 06:30:57 2012 +0000 Writing an article about the utility of asserts, I noticed some inconsistent whitespace with asserts. diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c index 85ad69e..ab405f9 100644 --- a/bin/varnishd/cache/cache_backend_poll.c +++ b/bin/varnishd/cache/cache_backend_poll.c @@ -500,7 +500,7 @@ VBP_Insert(struct backend *b, const struct vrt_backend_probe *p, } VTAILQ_FOREACH(vcl, &vt->vcls, list) - assert (vcl->probep != p); + assert(vcl->probep != p); vcl = vbp_new_vcl(p, hosthdr); Lck_Lock(&vbp_mtx); diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index 6e6eafb..c53d945 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -525,7 +525,7 @@ BAN_Reload(const uint8_t *ban, unsigned len) VTAILQ_FOREACH(b, &ban_head, list) { t1 = ban_time(b->spec); - assert (t1 < t2); + assert(t1 < t2); t2 = t1; if (t1 == t0) { Lck_Unlock(&ban_mtx); diff --git a/bin/varnishd/cache/cache_dir_round_robin.c b/bin/varnishd/cache/cache_dir_round_robin.c index 7d75473..350f18c 100644 --- a/bin/varnishd/cache/cache_dir_round_robin.c +++ b/bin/varnishd/cache/cache_dir_round_robin.c @@ -152,7 +152,7 @@ vrt_init_dir(struct cli *cli, struct director **bp, int idx, te = t->members; for (i = 0; i < t->nmember; i++, vh++, te++) { vh->backend = bp[te->host]; - AN (vh->backend); + AN(vh->backend); } vs->nhosts = t->nmember; vs->next_host = 0; diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 0e8b11a..73aac72 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -326,7 +326,7 @@ ESI_Deliver(struct sess *sp) p = e; break; } - assert (i == VGZ_OK || i == VGZ_END); + assert(i == VGZ_OK || i == VGZ_END); } else { /* * Ungzip'ed VEC, ungzip'ed ESI response diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index 39ded77..5a41875 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -424,7 +424,7 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru) Lck_Lock(&exp_mtx); VTAILQ_FOREACH(oc, &lru->lru_head, lru_list) { CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - assert (oc->timer_idx != BINHEAP_NOIDX); + assert(oc->timer_idx != BINHEAP_NOIDX); /* * It wont release any space if we cannot release the last * reference, besides, if somebody else has a reference, diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index e019702..ee9147d 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -666,7 +666,7 @@ FetchBody(struct worker *wrk, void *priv) uu += st->len; if (bo->do_stream) /* Streaming might have started freeing stuff */ - assert (uu <= obj->len); + assert(uu <= obj->len); else assert(uu == obj->len); diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index bcdfa6d..f977400 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -224,7 +224,7 @@ wslr(struct vsl_log *vsl, enum VSL_tag_e tag, int id, txt t) /* Wrap if necessary */ if (VSL_END(vsl->wlp, l) >= vsl->wle) VSL_Flush(vsl, 1); - assert (VSL_END(vsl->wlp, l) < vsl->wle); + assert(VSL_END(vsl->wlp, l) < vsl->wle); memcpy(VSL_DATA(vsl->wlp), t.b, l); vsl_hdr(tag, vsl->wlp, l, id); vsl->wlp = VSL_END(vsl->wlp, l); diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index d206a86..6118fbf 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -216,7 +216,7 @@ smp_open_segs(struct smp_sc *sc, struct smp_signctx *ctx) } } - assert (l >= sc->free_reserve); + assert(l >= sc->free_reserve); sg1 = NULL; diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 8bb19d4..2c6dd3f 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -268,7 +268,7 @@ VTCP_close(int *s) i = close(*s); - assert (VTCP_Check(i)); + assert(VTCP_Check(i)); *s = -1; } diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c index 044d12d..b4713e6 100644 --- a/lib/libvcl/vcc_action.c +++ b/lib/libvcl/vcc_action.c @@ -352,7 +352,7 @@ vcc_ParseAction(struct vcc *tl) const struct symbol *sym; at = tl->t; - assert (at->tok == ID); + assert(at->tok == ID); for(atp = action_table; atp->name != NULL; atp++) { if (vcc_IdIs(at, atp->name)) { if (atp->bitmask != 0) diff --git a/lib/libvcl/vcc_symb.c b/lib/libvcl/vcc_symb.c index 4066e56..691f6b8 100644 --- a/lib/libvcl/vcc_symb.c +++ b/lib/libvcl/vcc_symb.c @@ -118,7 +118,7 @@ VCC_FindSymbol(struct vcc *tl, const struct token *t, enum symkind kind) if (sym->kind == SYM_WILDCARD && (t->e - t->b > sym->nlen) && !memcmp(sym->name, t->b, sym->nlen)) { - AN (sym->wildcard); + AN(sym->wildcard); return (sym->wildcard(tl, t, sym)); } if (kind != SYM_NONE && kind != sym->kind) From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] f222778 Eliminate the prepfetch state Message-ID: commit f2227781d49de76a56955e69c13dde699dbbf4e0 Author: Poul-Henning Kamp Date: Mon Apr 30 07:35:27 2012 +0000 Eliminate the prepfetch state diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 4a04f5b..f963bbd 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -550,8 +550,8 @@ DOT shape=record DOT label="{cnt_fetch:|fetch hdr\nfrom backend|(find obj.ttl)|{vcl_fetch\{\}|{req.|bereq.|beresp.}}|{error?|restart?}}" DOT ] DOT } -DOT fetch -> prepfetch [style=bold,color=red] -DOT fetch -> prepfetch [style=bold,color=blue] +DOT fetch -> fetchbody [style=bold,color=red] +DOT fetch -> fetchbody [style=bold,color=blue] */ static int @@ -630,7 +630,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) switch (req->handling) { case VCL_RET_DELIVER: - sp->step = STP_PREPFETCH; + sp->step = STP_FETCHBODY; return (0); default: break; @@ -670,18 +670,17 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) * Prepare to fetch body from backend * DOT subgraph xcluster_body { -DOT prepfetch [ +DOT fetchbody [ DOT shape=record -DOT label="{cnt_prepfetch:|error?|stream ?}" +DOT label="{cnt_fetchbody:|start fetch_thread}" DOT ] DOT } -DOT prepfetch:out -> fetchbody [style=bold,color=red] -DOT prepfetch:out -> fetchbody [style=bold,color=blue] -DOT prepfetch:out -> prepresp [label=yes,style=bold,color=cyan] +DOT fetchbody:out -> prepresp [style=bold,color=red] +DOT fetchbody:out -> prepresp [style=bold,color=blue] */ static int -cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req) +cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) { struct http *hp, *hp2; char *b; @@ -868,34 +867,9 @@ cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req) RFC2616_Do_Cond(sp)) bo->do_stream = 0; - sp->step = STP_FETCHBODY; - return (0); -} - -/*-------------------------------------------------------------------- - * Actually fetch body from backend - * -DOT subgraph xcluster_fetchbody { -DOT fetchbody [ -DOT shape=record -DOT label="{cnt_fetchbody:|error ?|success ?}" -DOT ] -DOT } -DOT fetchbody:out -> prepresp [style=bold,color=red] -DOT fetchbody:out -> prepresp [style=bold,color=blue] - */ - -static int -cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) -{ - struct busyobj *bo; - - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - bo = req->busyobj; - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - + /* + * Ready to fetch the body + */ bo->fetch_task.func = FetchBody; bo->fetch_task.priv = bo; @@ -1707,5 +1681,3 @@ CNT_Init(void) xids = random(); CLI_AddFuncs(debug_cmds); } - - diff --git a/include/tbl/steps.h b/include/tbl/steps.h index ee70874..15fd811 100644 --- a/include/tbl/steps.h +++ b/include/tbl/steps.h @@ -40,7 +40,6 @@ STEP(lookup, LOOKUP, (sp, sp->wrk, sp->req)) STEP(miss, MISS, (sp, sp->wrk, sp->req)) STEP(hit, HIT, (sp, sp->wrk, sp->req)) STEP(fetch, FETCH, (sp, sp->wrk, sp->req)) -STEP(prepfetch, PREPFETCH, (sp, sp->wrk, sp->req)) STEP(fetchbody, FETCHBODY, (sp, sp->wrk, sp->req)) STEP(prepresp, PREPRESP, (sp, sp->wrk, sp->req)) STEP(deliver, DELIVER, (sp, sp->wrk, sp->req)) From perbu at varnish-software.com Thu Dec 18 09:27:45 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] 19a9743 -r option for read only parameters Message-ID: commit 19a97432827c29a6fdc63101494ca72b109c8df2 Author: Per Buer Date: Mon Apr 30 09:48:56 2012 +0200 -r option for read only parameters diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index f0647b3..b75bbb0 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -23,7 +23,7 @@ varnishd [-a address[:port]] [-b host[:port]] [-d] [-F] [-f config] [-g group] [-h type[,options]] [-i identity] [-l shmlogsize] [-n name] [-P file] [-p param=value] [-s type[,options]] [-T address[:port]] [-t ttl] - [-u user] [-V] + [-r param[,param...]] [-u user] [-V] DESCRIPTION =========== @@ -110,6 +110,13 @@ OPTIONS documents. This is a shortcut for specifying the default_ttl run-time parameter. +-r param[,param...] + Specifies a list of parameters that are read only. In a + very secure environment you want to consider setting + parameters such as *user*, *group*, *cc_command*, + *vcc_allow_inline_c* to read only as these can potentially + be used to escalate privileges. + -u user Specifies the name of an unprivileged user to which the child process should switch before it starts accepting connections. This is a shortcut for specifying the user From perbu at varnish-software.com Thu Dec 18 09:27:45 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] 170d084 explain the reason behind -r Message-ID: commit 170d0842e5218393e9fa2e5c3025e2aedabd16e0 Author: Per Buer Date: Mon Apr 30 10:05:34 2012 +0200 explain the reason behind -r diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index b75bbb0..805e012 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -111,11 +111,13 @@ OPTIONS default_ttl run-time parameter. -r param[,param...] - Specifies a list of parameters that are read only. In a - very secure environment you want to consider setting - parameters such as *user*, *group*, *cc_command*, - *vcc_allow_inline_c* to read only as these can potentially - be used to escalate privileges. + Specifies a list of parameters that are read only. This + gives the system administrator a way to limit what someone + with access to the Varnish CLI can do. In a very secure + environment you want to consider setting parameters such + as *user*, *group*, *cc_command*, *vcc_allow_inline_c* to + read only as these can potentially be used to escalate + privileges. -u user Specifies the name of an unprivileged user to which the child process should switch before it starts accepting From tfheen at varnish-software.com Thu Dec 18 09:27:45 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] 816c1c3 Enable PCRE JIT-compiled regular expressions Message-ID: commit 816c1c34e9afacbb439c24490284573dd2fbc065 Author: Dan McGee Date: Fri Dec 30 15:14:56 2011 -0600 Enable PCRE JIT-compiled regular expressions Implemented more or less as described in the pcrejit(3) manpage, and adds some compatibility defines for use with older pre-8.20 libraries that do not have this functionality. Fixes: #1080 diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c index fb2dcda..ee462c7 100644 --- a/lib/libvarnish/vre.c +++ b/lib/libvarnish/vre.c @@ -40,9 +40,14 @@ struct vre { unsigned magic; #define VRE_MAGIC 0xe83097dc - pcre *re; + pcre *re; + pcre_extra *re_extra; }; +#ifndef PCRE_STUDY_JIT_COMPILE +#define PCRE_STUDY_JIT_COMPILE 0 +#endif + /* * We don't want to spread or even expose the majority of PCRE options * so we establish our own options and implement hard linkage to PCRE @@ -66,6 +71,20 @@ VRE_compile(const char *pattern, int options, VRE_free(&v); return (NULL); } + v->re_extra = pcre_study(v->re, PCRE_STUDY_JIT_COMPILE, errptr); + if (v->re_extra == NULL) { + if (*errptr != NULL) { + VRE_free(&v); + return (NULL); + } + /* allocate our own, pcre_study can return NULL without it + * being an error */ + v->re_extra = calloc(1, sizeof(pcre_extra)); + if (v->re_extra == NULL) { + VRE_free(&v); + return (NULL); + } + } return (v); } @@ -76,22 +95,23 @@ VRE_exec(const vre_t *code, const char *subject, int length, { CHECK_OBJ_NOTNULL(code, VRE_MAGIC); int ov[30]; - pcre_extra extra; if (ovector == NULL) { ovector = ov; ovecsize = sizeof(ov)/sizeof(ov[0]); } - memset(&extra, 0, sizeof extra); if (lim != NULL) { - extra.match_limit = lim->match; - extra.flags |= PCRE_EXTRA_MATCH_LIMIT; - extra.match_limit_recursion = lim->match_recursion; - extra.flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; + code->re_extra->match_limit = lim->match; + code->re_extra->flags |= PCRE_EXTRA_MATCH_LIMIT; + code->re_extra->match_limit_recursion = lim->match_recursion; + code->re_extra->flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; + } else { + code->re_extra->flags &= ~PCRE_EXTRA_MATCH_LIMIT; + code->re_extra->flags &= ~PCRE_EXTRA_MATCH_LIMIT_RECURSION; } - return (pcre_exec(code->re, &extra, subject, length, + return (pcre_exec(code->re, code->re_extra, subject, length, startoffset, options, ovector, ovecsize)); } @@ -102,6 +122,11 @@ VRE_free(vre_t **vv) *vv = NULL; CHECK_OBJ(v, VRE_MAGIC); +#ifdef PCRE_CONFIG_JIT + pcre_free_study(v->re_extra); +#else + free(v->re_extra); +#endif pcre_free(v->re); FREE_OBJ(v); } From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] 9190dc3 Add 16k more thread stack to cater for PCRE-JIT. Message-ID: commit 9190dc3602bb65e4a8423f2defedf62145742641 Author: Poul-Henning Kamp Date: Tue May 1 06:38:54 2012 +0000 Add 16k more thread stack to cater for PCRE-JIT. Polish vre.c a bit with respect to memory management and error messages. diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index e5d5cd0..986d405 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -215,6 +215,6 @@ const struct parspec WRK_parspec[] = { "This is likely rounded up to a multiple of 4k by the kernel.\n" "The kernel/OS has a lower limit which will be enforced.\n", EXPERIMENTAL, - "32k", "bytes" }, + "48k", "bytes" }, { NULL, NULL, NULL } }; diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c index ee462c7..2fce5c7 100644 --- a/lib/libvarnish/vre.c +++ b/lib/libvarnish/vre.c @@ -37,17 +37,22 @@ #include "vre.h" +#ifndef PCRE_STUDY_JIT_COMPILE +#define PCRE_STUDY_JIT_COMPILE 0 +#endif + +#if PCRE_MAJOR < 8 || (PCRE_MAJOR == 8 && PCRE_MINOR < 20) +# define pcre_free_study pcre_free +#endif + struct vre { unsigned magic; #define VRE_MAGIC 0xe83097dc pcre *re; pcre_extra *re_extra; + int my_extra; }; -#ifndef PCRE_STUDY_JIT_COMPILE -#define PCRE_STUDY_JIT_COMPILE 0 -#endif - /* * We don't want to spread or even expose the majority of PCRE options * so we establish our own options and implement hard linkage to PCRE @@ -58,29 +63,32 @@ const unsigned VRE_NOTEMPTY = PCRE_NOTEMPTY; vre_t * VRE_compile(const char *pattern, int options, - const char **errptr, int *erroffset) + const char **errptr, int *erroffset) { vre_t *v; *errptr = NULL; *erroffset = 0; ALLOC_OBJ(v, VRE_MAGIC); - if (v == NULL) + if (v == NULL) { + *errptr = "Out of memory for VRE"; return (NULL); + } v->re = pcre_compile(pattern, options, errptr, erroffset, NULL); if (v->re == NULL) { VRE_free(&v); return (NULL); } v->re_extra = pcre_study(v->re, PCRE_STUDY_JIT_COMPILE, errptr); + if (*errptr != NULL) { + VRE_free(&v); + return (NULL); + } if (v->re_extra == NULL) { - if (*errptr != NULL) { - VRE_free(&v); - return (NULL); - } - /* allocate our own, pcre_study can return NULL without it - * being an error */ + /* allocate our own */ v->re_extra = calloc(1, sizeof(pcre_extra)); + v->my_extra = 1; if (v->re_extra == NULL) { + *errptr = "Out of memory for pcre_extra"; VRE_free(&v); return (NULL); } @@ -122,11 +130,13 @@ VRE_free(vre_t **vv) *vv = NULL; CHECK_OBJ(v, VRE_MAGIC); -#ifdef PCRE_CONFIG_JIT - pcre_free_study(v->re_extra); -#else - free(v->re_extra); -#endif - pcre_free(v->re); + if (v->re_extra != NULL) { + if (v->my_extra) + free(v->re_extra); + else + pcre_free_study(v->re_extra); + } + if (v->re != NULL) + pcre_free(v->re); FREE_OBJ(v); } From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] 1eac7c2 Add code coverage of the RST dumping code Message-ID: commit 1eac7c2672c8afd4ceb3a701240260c2ed294c10 Author: Poul-Henning Kamp Date: Wed May 2 08:47:15 2012 +0000 Add code coverage of the RST dumping code diff --git a/bin/varnishtest/tests/a00009.vtc b/bin/varnishtest/tests/a00009.vtc index 4d35f85..f89231f 100644 --- a/bin/varnishtest/tests/a00009.vtc +++ b/bin/varnishtest/tests/a00009.vtc @@ -1,3 +1,4 @@ -varnishtest "See that the VCL compiler works" +varnishtest "Code coverage of VCL compiler and RSTdump" shell "cd ${topbuild}/bin/varnishd && ./varnishd -b 127.0.0.1:80 -C -n ${tmpdir} > /dev/null 2>&1" +shell "cd ${topbuild}/bin/varnishd && ./varnishd -x dumprst > /dev/null 2>&1" From phk at FreeBSD.org Thu Dec 18 09:27:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:45 +0100 Subject: [experimental-ims] 227a394 Add a dummy pipe to use for poll(2)'ing the death of varnishd, while we wait for it to establish CLI connection. Message-ID: commit 227a394496d089bc36d03a0e40605010c969019d Author: Poul-Henning Kamp Date: Wed May 2 09:53:09 2012 +0000 Add a dummy pipe to use for poll(2)'ing the death of varnishd, while we wait for it to establish CLI connection. diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index e493ae7..ed55353 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -59,7 +59,7 @@ struct varnish { struct vsb *storage; struct vsb *args; - int fds[4]; + int fds[6]; pid_t pid; pthread_t tp; @@ -365,6 +365,7 @@ varnish_launch(struct varnish *v) vtc_log(v->vl, 3, "CMD: %s", VSB_data(vsb)); AZ(pipe(&v->fds[0])); AZ(pipe(&v->fds[2])); + AZ(pipe(&v->fds[4])); v->pid = fork(); assert(v->pid >= 0); if (v->pid == 0) { @@ -375,8 +376,11 @@ varnish_launch(struct varnish *v) AZ(close(v->fds[1])); AZ(close(v->fds[2])); AZ(close(v->fds[3])); - for (i = 3; i fds[4])); + for (i = 3; i fds[5] != i) + (void)close(i); + } AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(vsb), NULL)); exit(1); } else { @@ -384,6 +388,7 @@ varnish_launch(struct varnish *v) } AZ(close(v->fds[0])); AZ(close(v->fds[3])); + AZ(close(v->fds[5])); v->fds[0] = v->fds[2]; v->fds[2] = v->fds[3] = -1; VSB_delete(vsb); @@ -394,8 +399,8 @@ varnish_launch(struct varnish *v) memset(fd, 0, sizeof fd); fd[0].fd = v->cli_fd; fd[0].events = POLLIN; - fd[1].fd = v->fds[0]; - fd[1].events = POLLHUP; + fd[1].fd = v->fds[4]; + fd[1].events = POLLIN|POLLHUP; #ifdef __APPLE__ /* * OSX cannot poll a pipe for POLLHUP only, poll just returns @@ -425,6 +430,7 @@ varnish_launch(struct varnish *v) return; } + AZ(close(v->fds[4])); AZ(close(v->cli_fd)); v->cli_fd = nfd; From phk at FreeBSD.org Thu Dec 18 09:27:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] 8d2ce2c Revert "Add a dummy pipe to use for poll(2)'ing the death of varnishd, " Message-ID: commit 8d2ce2c25ba26ab38193435cf21ba3d8a18a40cb Author: Poul-Henning Kamp Date: Wed May 2 10:01:26 2012 +0000 Revert "Add a dummy pipe to use for poll(2)'ing the death of varnishd," This reverts commit 227a394496d089bc36d03a0e40605010c969019d. diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index ed55353..e493ae7 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -59,7 +59,7 @@ struct varnish { struct vsb *storage; struct vsb *args; - int fds[6]; + int fds[4]; pid_t pid; pthread_t tp; @@ -365,7 +365,6 @@ varnish_launch(struct varnish *v) vtc_log(v->vl, 3, "CMD: %s", VSB_data(vsb)); AZ(pipe(&v->fds[0])); AZ(pipe(&v->fds[2])); - AZ(pipe(&v->fds[4])); v->pid = fork(); assert(v->pid >= 0); if (v->pid == 0) { @@ -376,11 +375,8 @@ varnish_launch(struct varnish *v) AZ(close(v->fds[1])); AZ(close(v->fds[2])); AZ(close(v->fds[3])); - AZ(close(v->fds[4])); - for (i = 3; i fds[5] != i) - (void)close(i); - } + for (i = 3; i fds[0])); AZ(close(v->fds[3])); - AZ(close(v->fds[5])); v->fds[0] = v->fds[2]; v->fds[2] = v->fds[3] = -1; VSB_delete(vsb); @@ -399,8 +394,8 @@ varnish_launch(struct varnish *v) memset(fd, 0, sizeof fd); fd[0].fd = v->cli_fd; fd[0].events = POLLIN; - fd[1].fd = v->fds[4]; - fd[1].events = POLLIN|POLLHUP; + fd[1].fd = v->fds[0]; + fd[1].events = POLLHUP; #ifdef __APPLE__ /* * OSX cannot poll a pipe for POLLHUP only, poll just returns @@ -430,7 +425,6 @@ varnish_launch(struct varnish *v) return; } - AZ(close(v->fds[4])); AZ(close(v->cli_fd)); v->cli_fd = nfd; From phk at FreeBSD.org Thu Dec 18 09:27:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] cb13f32 Convert a lot of testcases which don't care how the response was transferred to examine resp.bodylen instead of resp.http.content-length. Message-ID: commit cb13f327e27a11d6d928692f7c7d56ae4566f98e Author: Poul-Henning Kamp Date: Wed May 2 11:18:05 2012 +0000 Convert a lot of testcases which don't care how the response was transferred to examine resp.bodylen instead of resp.http.content-length. (This is in preparation for streaming being default) diff --git a/bin/varnishtest/tests/b00007.vtc b/bin/varnishtest/tests/b00007.vtc index 58740c6..a07ae9e 100644 --- a/bin/varnishtest/tests/b00007.vtc +++ b/bin/varnishtest/tests/b00007.vtc @@ -26,9 +26,9 @@ client c1 { txreq -url "/bar" rxresp expect resp.status == 200 - expect resp.http.content-length == "4" + expect resp.bodylen == "4" txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == "8" + expect resp.bodylen == "8" } -run diff --git a/bin/varnishtest/tests/b00011.vtc b/bin/varnishtest/tests/b00011.vtc index 9c4e2d3..1e9ee74 100644 --- a/bin/varnishtest/tests/b00011.vtc +++ b/bin/varnishtest/tests/b00011.vtc @@ -16,5 +16,5 @@ client c1 { txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.content-length == 36 + expect resp.bodylen == 36 } -run diff --git a/bin/varnishtest/tests/b00012.vtc b/bin/varnishtest/tests/b00012.vtc index 967ed85..154f6c0 100644 --- a/bin/varnishtest/tests/b00012.vtc +++ b/bin/varnishtest/tests/b00012.vtc @@ -15,15 +15,15 @@ client c1 { send "GET /foo HTTP/1.1\n\nGET /bar HTTP/1.1\n\nGET /bar HTTP/1.1\n\n" rxresp expect resp.status == 200 - expect resp.http.content-length == 3 + expect resp.bodylen == 3 expect resp.http.x-varnish == "1001" rxresp expect resp.status == 200 - expect resp.http.content-length == 6 + expect resp.bodylen == 6 expect resp.http.x-varnish == "1002" rxresp expect resp.status == 200 - expect resp.http.content-length == 6 + expect resp.bodylen == 6 expect resp.http.x-varnish == "1003 1002" } -run diff --git a/bin/varnishtest/tests/b00013.vtc b/bin/varnishtest/tests/b00013.vtc index 8ae90c4..b8e25e6 100644 --- a/bin/varnishtest/tests/b00013.vtc +++ b/bin/varnishtest/tests/b00013.vtc @@ -15,17 +15,17 @@ client c1 { send "GET /foo HTTP/1.1\r\n\r\nGET " rxresp expect resp.status == 200 - expect resp.http.content-length == 3 + expect resp.bodylen == 3 expect resp.http.x-varnish == "1001" send "/bar HTTP/1.1\n\nGET /bar " rxresp expect resp.status == 200 - expect resp.http.content-length == 6 + expect resp.bodylen == 6 expect resp.http.x-varnish == "1002" send "HTTP/1.1\n\n" rxresp expect resp.status == 200 - expect resp.http.content-length == 6 + expect resp.bodylen == 6 expect resp.http.x-varnish == "1003 1002" } -run diff --git a/bin/varnishtest/tests/c00006.vtc b/bin/varnishtest/tests/c00006.vtc index c2c8ec2..9342886 100644 --- a/bin/varnishtest/tests/c00006.vtc +++ b/bin/varnishtest/tests/c00006.vtc @@ -15,7 +15,7 @@ client c1 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 5 + expect resp.bodylen == 5 } client c1 -run @@ -26,7 +26,7 @@ client c1 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 6 + expect resp.bodylen == 6 } client c1 -run diff --git a/bin/varnishtest/tests/c00008.vtc b/bin/varnishtest/tests/c00008.vtc index 3045a5b..69c64d4 100644 --- a/bin/varnishtest/tests/c00008.vtc +++ b/bin/varnishtest/tests/c00008.vtc @@ -15,7 +15,7 @@ client c1 { rxresp expect resp.status == 200 expect resp.http.etag == "foo" - expect resp.http.content-length == 6 + expect resp.bodylen == 6 txreq -url "/foo" \ -hdr "If-Modified-Since: Thu, 26 Jun 2008 12:00:00 GMT" diff --git a/bin/varnishtest/tests/c00009.vtc b/bin/varnishtest/tests/c00009.vtc index 295b092..d0c4b71 100644 --- a/bin/varnishtest/tests/c00009.vtc +++ b/bin/varnishtest/tests/c00009.vtc @@ -32,7 +32,7 @@ client c1 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 6 + expect resp.bodylen == 6 } client c1 -run diff --git a/bin/varnishtest/tests/c00010.vtc b/bin/varnishtest/tests/c00010.vtc index 81ed5b6..6f0aca3 100644 --- a/bin/varnishtest/tests/c00010.vtc +++ b/bin/varnishtest/tests/c00010.vtc @@ -19,12 +19,12 @@ client c1 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 6 + expect resp.bodylen == 6 expect resp.http.x-varnish == "1001" txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 7 + expect resp.bodylen == 7 expect resp.http.x-varnish == "1002" } diff --git a/bin/varnishtest/tests/c00011.vtc b/bin/varnishtest/tests/c00011.vtc index 18ee757..6bea217 100644 --- a/bin/varnishtest/tests/c00011.vtc +++ b/bin/varnishtest/tests/c00011.vtc @@ -19,12 +19,12 @@ client c1 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 6 + expect resp.bodylen == 6 expect resp.http.x-varnish == "1001" txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 7 + expect resp.bodylen == 7 expect resp.http.x-varnish == "1002" } diff --git a/bin/varnishtest/tests/c00012.vtc b/bin/varnishtest/tests/c00012.vtc index 24326d8..b578625 100644 --- a/bin/varnishtest/tests/c00012.vtc +++ b/bin/varnishtest/tests/c00012.vtc @@ -19,12 +19,12 @@ client c1 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 6 + expect resp.bodylen == 6 expect resp.http.x-varnish == "1001" txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 7 + expect resp.bodylen == 7 expect resp.http.x-varnish == "1002" } diff --git a/bin/varnishtest/tests/c00013.vtc b/bin/varnishtest/tests/c00013.vtc index 94c86ca..885be31 100644 --- a/bin/varnishtest/tests/c00013.vtc +++ b/bin/varnishtest/tests/c00013.vtc @@ -16,7 +16,7 @@ client c1 { txreq -url "/foo" -hdr "client: c1" rxresp expect resp.status == 200 - expect resp.http.content-length == 12 + expect resp.bodylen == 12 expect resp.http.x-varnish == "1001" } -start @@ -26,7 +26,7 @@ client c2 { sema r1 sync 2 rxresp expect resp.status == 200 - expect resp.http.content-length == 12 + expect resp.bodylen == 12 expect resp.http.x-varnish == "1002 1001" } -run diff --git a/bin/varnishtest/tests/c00014.vtc b/bin/varnishtest/tests/c00014.vtc index 03089d8..545c469 100644 --- a/bin/varnishtest/tests/c00014.vtc +++ b/bin/varnishtest/tests/c00014.vtc @@ -23,7 +23,7 @@ client c1 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 12 + expect resp.bodylen == 12 expect resp.http.x-varnish == "1001" } -start sema r1 sync 2 @@ -31,7 +31,7 @@ client c2 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 6 + expect resp.bodylen == 6 expect resp.http.x-varnish == "1002" } -run diff --git a/bin/varnishtest/tests/c00015.vtc b/bin/varnishtest/tests/c00015.vtc index 755669b..c323b9c 100644 --- a/bin/varnishtest/tests/c00015.vtc +++ b/bin/varnishtest/tests/c00015.vtc @@ -25,7 +25,7 @@ client c1 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 6 + expect resp.bodylen == 6 expect resp.http.x-varnish == "1001" } -run @@ -35,7 +35,7 @@ client c2 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 7 + expect resp.bodylen == 7 expect resp.http.x-varnish == "1002" } -run @@ -45,7 +45,7 @@ client c3 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 6 + expect resp.bodylen == 6 expect resp.http.x-varnish == "1003 1001" } -run diff --git a/bin/varnishtest/tests/c00025.vtc b/bin/varnishtest/tests/c00025.vtc index 668fdc2..d0cd83d 100644 --- a/bin/varnishtest/tests/c00025.vtc +++ b/bin/varnishtest/tests/c00025.vtc @@ -13,7 +13,7 @@ client c1 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 6 + expect resp.bodylen == 6 txreq -url "/foo" \ -hdr "If-None-Match: 12345678" diff --git a/bin/varnishtest/tests/c00026.vtc b/bin/varnishtest/tests/c00026.vtc index 427efe4..5dcbd41 100644 --- a/bin/varnishtest/tests/c00026.vtc +++ b/bin/varnishtest/tests/c00026.vtc @@ -14,7 +14,7 @@ client c1 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.content-length == 6 + expect resp.bodylen == 6 txreq -url "/foo" \ -hdr "If-None-Match: 123456789" diff --git a/bin/varnishtest/tests/e00000.vtc b/bin/varnishtest/tests/e00000.vtc index 9aa9f55..1cc41fc 100644 --- a/bin/varnishtest/tests/e00000.vtc +++ b/bin/varnishtest/tests/e00000.vtc @@ -18,7 +18,7 @@ client c1 { txreq rxresp expect resp.status == 200 - expect resp.http.content-length == 33 + expect resp.bodylen == 33 } client c1 -run diff --git a/bin/varnishtest/tests/e00009.vtc b/bin/varnishtest/tests/e00009.vtc index 8a7927d..f3754ff 100644 --- a/bin/varnishtest/tests/e00009.vtc +++ b/bin/varnishtest/tests/e00009.vtc @@ -28,7 +28,6 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == 57 - expect resp.http.content-length == 57 } -run varnish v1 -cli "param.set esi_syntax 1" diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index 217668f..7fe2b8c 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -21,7 +21,7 @@ client c1 { txreq -url "/bar" rxresp expect resp.status == 200 - expect resp.http.content-length == "4" + expect resp.bodylen == "4" expect resp.http.foo == "BAR" expect resp.http.bar == "foo" } -run diff --git a/bin/varnishtest/tests/m00001.vtc b/bin/varnishtest/tests/m00001.vtc index b7ca47f..dd2b064 100644 --- a/bin/varnishtest/tests/m00001.vtc +++ b/bin/varnishtest/tests/m00001.vtc @@ -21,7 +21,7 @@ client c1 { txreq -url "/bar" rxresp expect resp.status == 200 - expect resp.http.content-length == "4" + expect resp.bodylen == "4" expect resp.http.foo == "BAR" expect resp.http.bar == "foo" } -run @@ -33,7 +33,7 @@ client c1 { txreq -url "/bar" rxresp expect resp.status == 200 - expect resp.http.content-length == "4" + expect resp.bodylen == "4" expect resp.http.foo == "bAr" expect resp.http.bar == "fOo" } -run diff --git a/bin/varnishtest/tests/m00003.vtc b/bin/varnishtest/tests/m00003.vtc index 30a0f3c..67d984d 100644 --- a/bin/varnishtest/tests/m00003.vtc +++ b/bin/varnishtest/tests/m00003.vtc @@ -21,7 +21,7 @@ client c1 { txreq -url "/bar" rxresp expect resp.status == 200 - expect resp.http.content-length == "4" + expect resp.bodylen == "4" expect resp.http.foo == "BAR" expect resp.http.bar == "foo" } -run diff --git a/bin/varnishtest/tests/m00004.vtc b/bin/varnishtest/tests/m00004.vtc index 0dbe2cc..232428b 100644 --- a/bin/varnishtest/tests/m00004.vtc +++ b/bin/varnishtest/tests/m00004.vtc @@ -34,21 +34,21 @@ client c1 { txreq -url "/one" rxresp expect resp.status == 200 - expect resp.http.content-length == "4" + expect resp.bodylen == "4" expect resp.http.foo == "bar" expect resp.http.one == "File One" txreq -url "/two" rxresp expect resp.status == 200 - expect resp.http.content-length == "4" + expect resp.bodylen == "4" expect resp.http.foo == "bar" expect resp.http.two == "File Two" txreq -url "/three" rxresp expect resp.status == 200 - expect resp.http.content-length == "4" + expect resp.bodylen == "4" expect resp.http.foo == "bar" expect resp.http.three == "File Three" } -run diff --git a/bin/varnishtest/tests/r00911.vtc b/bin/varnishtest/tests/r00911.vtc index f85556c..5663421 100644 --- a/bin/varnishtest/tests/r00911.vtc +++ b/bin/varnishtest/tests/r00911.vtc @@ -15,6 +15,5 @@ varnish v1 -arg "-p vcc_err_unref=false" -vcl+backend { client c1 { txreq -url /bar rxresp - expect resp.http.content-length == 6 expect resp.bodylen == 6 } -run diff --git a/bin/varnishtest/tests/r00913.vtc b/bin/varnishtest/tests/r00913.vtc index 519436f..e310360 100644 --- a/bin/varnishtest/tests/r00913.vtc +++ b/bin/varnishtest/tests/r00913.vtc @@ -17,6 +17,6 @@ varnish v1 -vcl+backend { client c1 { txreq -url /bar rxresp - expect resp.http.content-length == 6 + expect resp.bodylen == 6 expect resp.http.foo == "XXX" } -run diff --git a/bin/varnishtest/tests/r00917.vtc b/bin/varnishtest/tests/r00917.vtc index 9791034..689d952 100644 --- a/bin/varnishtest/tests/r00917.vtc +++ b/bin/varnishtest/tests/r00917.vtc @@ -11,7 +11,7 @@ varnish v1 -vcl+backend { } -start client c1 { txreq -url /bar rxresp - expect resp.http.content-length == 6 + expect resp.bodylen == 6 } -run varnish v1 -cliok {ban req.url ~ << foo diff --git a/bin/varnishtest/tests/s00000.vtc b/bin/varnishtest/tests/s00000.vtc index 717dd02..bde8934 100644 --- a/bin/varnishtest/tests/s00000.vtc +++ b/bin/varnishtest/tests/s00000.vtc @@ -15,7 +15,7 @@ varnish v1 -vcl+backend { } -start client c1 { txreq -url "/" rxresp - expect resp.http.content-length == 5 + expect resp.bodylen == 5 expect resp.http.x-varnish == "1001" expect resp.status == 200 } -run @@ -27,5 +27,5 @@ client c2 { rxresp expect resp.status == 200 expect resp.http.x-varnish == "1002" - expect resp.http.content-length == 6 + expect resp.bodylen == 6 } -run diff --git a/bin/varnishtest/tests/s00001.vtc b/bin/varnishtest/tests/s00001.vtc index 1dd34bc..68fd520 100644 --- a/bin/varnishtest/tests/s00001.vtc +++ b/bin/varnishtest/tests/s00001.vtc @@ -15,7 +15,7 @@ varnish v1 -vcl+backend { } -start client c1 { txreq -url "/" rxresp - expect resp.http.content-length == 5 + expect resp.bodylen == 5 expect resp.http.x-varnish == "1001" expect resp.status == 200 } -run @@ -28,5 +28,5 @@ client c2 { rxresp expect resp.status == 200 expect resp.http.x-varnish == "1002" - expect resp.http.content-length == 6 + expect resp.bodylen == 6 } -run diff --git a/bin/varnishtest/tests/v00009.vtc b/bin/varnishtest/tests/v00009.vtc index 04b8ccb..5a5d769 100644 --- a/bin/varnishtest/tests/v00009.vtc +++ b/bin/varnishtest/tests/v00009.vtc @@ -39,16 +39,16 @@ client c1 { timeout 3 txreq -url "/foo1" rxresp - expect resp.http.content-length == 1 + expect resp.bodylen == 1 txreq -url "/foo2" rxresp - expect resp.http.content-length == 2 + expect resp.bodylen == 2 txreq -url "/foo3" rxresp - expect resp.http.content-length == 3 + expect resp.bodylen == 3 txreq -url "/foo4" rxresp - expect resp.http.content-length == 4 + expect resp.bodylen == 4 } -run server s1 -start @@ -58,8 +58,8 @@ client c2 { timeout 3 txreq -url "/foo11" rxresp - expect resp.http.content-length == 1 + expect resp.bodylen == 1 txreq -url "/foo22" rxresp - expect resp.http.content-length == 2 + expect resp.bodylen == 2 } -run From phk at FreeBSD.org Thu Dec 18 09:27:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] b6c079b In the current scheme of things, this can never be the right thing to do. Message-ID: commit b6c079b61034c300fde3951331e2c97731fbed5d Author: Poul-Henning Kamp Date: Wed May 2 19:20:50 2012 +0000 In the current scheme of things, this can never be the right thing to do. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index f963bbd..cec3ddb 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -286,7 +286,6 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) break; if (bo != NULL) { AN(bo->do_stream); - VDI_CloseFd(&bo->vbc); HSH_Drop(wrk, &sp->req->obj); VBO_DerefBusyObj(wrk, &bo); } else { From phk at FreeBSD.org Thu Dec 18 09:27:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] 49a42aa These three tests depend on not running with streaming Message-ID: commit 49a42aa16e7cf1762df69c4709c1ae35bcac969a Author: Poul-Henning Kamp Date: Thu May 3 07:37:50 2012 +0000 These three tests depend on not running with streaming diff --git a/bin/varnishtest/tests/r00702.vtc b/bin/varnishtest/tests/r00702.vtc index a96624e..5075c16 100644 --- a/bin/varnishtest/tests/r00702.vtc +++ b/bin/varnishtest/tests/r00702.vtc @@ -6,6 +6,9 @@ server s1 { } -start varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.do_stream = false; + } } -start varnish v1 -cliok "param.set http_range_support on" diff --git a/bin/varnishtest/tests/r00704.vtc b/bin/varnishtest/tests/r00704.vtc index 5ee5702..806fc82 100644 --- a/bin/varnishtest/tests/r00704.vtc +++ b/bin/varnishtest/tests/r00704.vtc @@ -6,6 +6,9 @@ server s1 { } -start varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.do_stream = false; + } } -start varnish v1 -cliok "param.set http_range_support on" diff --git a/bin/varnishtest/tests/r00801.vtc b/bin/varnishtest/tests/r00801.vtc index f6fe5bd..3309f22 100644 --- a/bin/varnishtest/tests/r00801.vtc +++ b/bin/varnishtest/tests/r00801.vtc @@ -11,6 +11,9 @@ server s1 { varnish v1 -vcl+backend { sub vcl_recv { return (pass); } + sub vcl_fetch { + set beresp.do_stream = false; + } } -start client c1 { From phk at FreeBSD.org Thu Dec 18 09:27:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] 8a22dbf Account for all attempted fetches. Message-ID: commit 8a22dbf5d181eb125ea4652af7317cc41725969a Author: Poul-Henning Kamp Date: Thu May 3 08:10:55 2012 +0000 Account for all attempted fetches. Update some VSC descriptions while at it. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index cec3ddb..e343aa8 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -576,6 +576,8 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) need_host_hdr = !http_GetHdr(bo->bereq, H_Host, NULL); + wrk->acct_tmp.fetch++; + i = FetchHdr(sp, need_host_hdr, req->objcore == NULL); /* * If we recycle a backend connection, there is a finite chance @@ -902,7 +904,6 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) HSH_Ref(req->obj->objcore); VBO_DerefBusyObj(wrk, &req->busyobj); - wrk->acct_tmp.fetch++; sp->step = STP_PREPRESP; return (0); } diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h index bc79d5b..c3fd288 100644 --- a/include/tbl/vsc_f_main.h +++ b/include/tbl/vsc_f_main.h @@ -147,53 +147,57 @@ VSC_F(backend_retry, uint64_t, 0, 'a', "" ) -VSC_F(fetch_head, uint64_t, 1, 'a', - "Fetch head", - "" +/*--------------------------------------------------------------------- + * Backend fetch statistics + */ + +VSC_F(fetch_head, uint64_t, 1, 'c', + "Fetch no body (HEAD)", + "beresp with no body because the request is HEAD." ) -VSC_F(fetch_length, uint64_t, 1, 'a', +VSC_F(fetch_length, uint64_t, 1, 'c', "Fetch with Length", - "" + "beresp with Content-Length." ) -VSC_F(fetch_chunked, uint64_t, 1, 'a', +VSC_F(fetch_chunked, uint64_t, 1, 'c', "Fetch chunked", - "" + "beresp with Chunked." ) -VSC_F(fetch_eof, uint64_t, 1, 'a', +VSC_F(fetch_eof, uint64_t, 1, 'c', "Fetch EOF", - "" + "beresp with EOF from lack of other info." ) -VSC_F(fetch_bad, uint64_t, 1, 'a', - "Fetch had bad headers", - "" +VSC_F(fetch_bad, uint64_t, 1, 'c', + "Fetch bad T-E", + "beresp failed due to unknown Transfer-Encoding." ) -VSC_F(fetch_close, uint64_t, 1, 'a', +VSC_F(fetch_close, uint64_t, 1, 'c', "Fetch wanted close", - "" + "beresp with EOF due to Connection: Close." ) -VSC_F(fetch_oldhttp, uint64_t, 1, 'a', +VSC_F(fetch_oldhttp, uint64_t, 1, 'c', "Fetch pre HTTP/1.1 closed", - "" -) -VSC_F(fetch_zero, uint64_t, 1, 'a', - "Fetch zero len", - "" + "beresp with EOF due to HTTP < 1.1" ) -VSC_F(fetch_failed, uint64_t, 1, 'a', - "Fetch failed", - "" +VSC_F(fetch_zero, uint64_t, 1, 'c', + "Fetch zero len body", + "beresp with EOF due to keep-live but neither Chunked or Len." ) -VSC_F(fetch_1xx, uint64_t, 1, 'a', +VSC_F(fetch_1xx, uint64_t, 1, 'c', "Fetch no body (1xx)", - "" + "beresp with no body because of 1XX response." ) -VSC_F(fetch_204, uint64_t, 1, 'a', +VSC_F(fetch_204, uint64_t, 1, 'c', "Fetch no body (204)", - "" + "beresp with no body because of 204 response." ) -VSC_F(fetch_304, uint64_t, 1, 'a', +VSC_F(fetch_304, uint64_t, 1, 'c', "Fetch no body (304)", - "" + "beresp with no body because of 304 response." +) +VSC_F(fetch_failed, uint64_t, 1, 'c', + "Fetch body failed", + "beresp body fetch failed." ) /*--------------------------------------------------------------------- From phk at FreeBSD.org Thu Dec 18 09:27:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] d8094d2 Set streaming by default, to flush out problems. (We don't actually stream yet, we just pretend for some of the way) Message-ID: commit d8094d26a5d5d582f4f0aca8c28227040ebd39cb Author: Poul-Henning Kamp Date: Thu May 3 09:33:35 2012 +0000 Set streaming by default, to flush out problems. (We don't actually stream yet, we just pretend for some of the way) Fix a couple of tests to DTRT accordingly. diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 0489734..7fa0291 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -134,6 +134,8 @@ VBO_GetBusyObj(struct worker *wrk) WS_Init(bo->ws, "bo", p, bo->end - p); + bo->do_stream = 1; + return (bo); } diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index e343aa8..cce170c 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -212,23 +212,15 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - if (bo != NULL) { + if (bo != NULL) CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - AN(bo->do_stream); - } req->res_mode = 0; if (bo == NULL) req->res_mode |= RES_LEN; - if (bo != NULL && - (bo->h_content_length != NULL || - !bo->do_stream) && - !bo->do_gzip && !bo->do_gunzip) - req->res_mode |= RES_LEN; - - if (!req->disable_esi && req->obj->esidata != NULL) { + if (bo == NULL && !req->disable_esi && req->obj->esidata != NULL) { /* In ESI mode, we don't know the aggregate length */ req->res_mode &= ~RES_LEN; req->res_mode |= RES_ESI; @@ -883,7 +875,8 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) HSH_Unbusy(&wrk->stats, req->obj->objcore); } - if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE)) + if (!bo->do_stream || + Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE)) FetchBody(wrk, bo); while (bo->state < BOS_FAILED) From phk at FreeBSD.org Thu Dec 18 09:27:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] 31ffe7c Make cnt_prepresp run on incomplete busyobj Message-ID: commit 31ffe7ccab8e4b0b226dbe950f3fbf87e6d16852 Author: Poul-Henning Kamp Date: Fri May 4 08:02:56 2012 +0000 Make cnt_prepresp run on incomplete busyobj diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index cce170c..a95df05 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -278,8 +278,8 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) break; if (bo != NULL) { AN(bo->do_stream); - HSH_Drop(wrk, &sp->req->obj); - VBO_DerefBusyObj(wrk, &bo); + (void)HSH_Deref(&wrk->stats, NULL, &req->obj); + VBO_DerefBusyObj(wrk, &req->busyobj); } else { (void)HSH_Deref(&wrk->stats, NULL, &req->obj); } @@ -312,10 +312,29 @@ DOT deliver -> DONE [style=bold,color=blue] static int cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req) { + struct busyobj *bo; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); + bo = req->busyobj; + CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); + + if (bo != NULL) { + while (bo->state < BOS_FAILED) + (void)usleep(10000); + assert(bo->state >= BOS_FAILED); + + if (bo->state == BOS_FAILED) { + req->obj = NULL; + VBO_DerefBusyObj(wrk, &req->busyobj); + req->err_code = 503; + sp->step = STP_ERROR; + return (0); + } + VBO_DerefBusyObj(wrk, &req->busyobj); + } AZ(req->busyobj); req->director = NULL; @@ -879,13 +898,13 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE)) FetchBody(wrk, bo); - while (bo->state < BOS_FAILED) - (void)usleep(10000); - assert(bo->state >= BOS_FAILED); - - assert(WRW_IsReleased(wrk)); + if (req->obj->objcore != NULL) + HSH_Ref(req->obj->objcore); - if (bo->state == BOS_FAILED) { + if (bo->state > BOS_FAILED) { + VBO_DerefBusyObj(wrk, &req->busyobj); + } else if (bo->state == BOS_FAILED) { + /* handle early failures */ req->obj = NULL; VBO_DerefBusyObj(wrk, &req->busyobj); req->err_code = 503; @@ -893,10 +912,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) return (0); } - if (req->obj->objcore != NULL) - HSH_Ref(req->obj->objcore); - - VBO_DerefBusyObj(wrk, &req->busyobj); + assert(WRW_IsReleased(wrk)); sp->step = STP_PREPRESP; return (0); } From phk at FreeBSD.org Thu Dec 18 09:27:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] a0e3078 Go over prepresp and remove a suboptimal optimization for len=0 objects. Message-ID: commit a0e3078f809d3553b603d5901cce61e70c54415a Author: Poul-Henning Kamp Date: Fri May 4 10:52:28 2012 +0000 Go over prepresp and remove a suboptimal optimization for len=0 objects. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index a95df05..332211a 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -184,7 +184,8 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) } /*-------------------------------------------------------------------- - * We have a refcounted object on the session, now deliver it. + * We have a refcounted object on the session, and possibly the busyobj + * which is fetching it, prepare a response. * DOT subgraph xcluster_prepresp { DOT prepresp [ @@ -212,21 +213,22 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - if (bo != NULL) - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - req->res_mode = 0; - if (bo == NULL) - req->res_mode |= RES_LEN; - - if (bo == NULL && !req->disable_esi && req->obj->esidata != NULL) { - /* In ESI mode, we don't know the aggregate length */ - req->res_mode &= ~RES_LEN; - req->res_mode |= RES_ESI; + if (bo == NULL) { + if (!req->disable_esi && req->obj->esidata != NULL) { + /* In ESI mode, we can't know the aggregate length */ + req->res_mode &= ~RES_LEN; + req->res_mode |= RES_ESI; + } else { + req->res_mode |= RES_LEN; + } + } else { + AZ(bo->do_esi); } if (req->esi_level > 0) { + /* Included ESI object, always CHUNKED or EOF */ req->res_mode &= ~RES_LEN; req->res_mode |= RES_ESI_CHILD; } @@ -242,14 +244,8 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) } if (!(req->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) { - if (req->obj->len == 0 && - (bo == NULL || !bo->do_stream)) - /* - * If the object is empty, neither ESI nor GUNZIP - * can make it any different size - */ - req->res_mode |= RES_LEN; - else if (!req->wantbody) { + /* We havn't chosen yet, do so */ + if (!req->wantbody) { /* Nothing */ } else if (req->http->protover >= 11) { req->res_mode |= RES_CHUNKED; @@ -901,7 +897,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) if (req->obj->objcore != NULL) HSH_Ref(req->obj->objcore); - if (bo->state > BOS_FAILED) { + if (bo->state == BOS_FINISHED) { VBO_DerefBusyObj(wrk, &req->busyobj); } else if (bo->state == BOS_FAILED) { /* handle early failures */ From martin at varnish-software.com Thu Dec 18 09:27:46 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] a9e636e Make the installed headers be nobase. Message-ID: commit a9e636edfbd51d83050d6d458ab99688dbffa908 Author: Martin Blix Grydeland Date: Fri Apr 13 13:41:04 2012 +0200 Make the installed headers be nobase. After the source tree restructure, the header files installed from include/ needs the local directory hierarchy to be left intact. Mark the pkginclude_HEADERS target nobase. diff --git a/include/Makefile.am b/include/Makefile.am index d40169b..91812fa 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -1,6 +1,6 @@ # -pkginclude_HEADERS = \ +nobase_pkginclude_HEADERS = \ tbl/acct_fields.h \ tbl/backend_poll.h \ tbl/ban_vars.h \ From martin at varnish-software.com Thu Dec 18 09:27:46 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] 0351ac9 Move lib/libvmod_std/vmod.py to lib/libvcl/vmodtool.py Message-ID: commit 0351ac952b100d311d77dd77036c2976e5937661 Author: Martin Blix Grydeland Date: Tue Feb 28 14:41:49 2012 +0100 Move lib/libvmod_std/vmod.py to lib/libvcl/vmodtool.py Move it to a more natural home, as it is not really about libvmod_std, and give it a name that matches it's function (inspired from libtool) diff --git a/lib/libvcl/Makefile.am b/lib/libvcl/Makefile.am index f135e43..ad16981 100644 --- a/lib/libvcl/Makefile.am +++ b/lib/libvcl/Makefile.am @@ -31,7 +31,8 @@ libvcl_la_SOURCES = \ vcc_xref.c EXTRA_DIST = \ - generate.py + generate.py \ + vmodtool.py vcc_obj.c vcc_fixed_token.c vcc_token_defs.h: \ $(srcdir)/generate.py $(top_srcdir)/include/vrt.h diff --git a/lib/libvcl/vmodtool.py b/lib/libvcl/vmodtool.py new file mode 100755 index 0000000..041edcb --- /dev/null +++ b/lib/libvcl/vmodtool.py @@ -0,0 +1,321 @@ +#!/usr/local/bin/python +#- +# Copyright (c) 2010-2011 Varnish Software AS +# All rights reserved. +# +# Author: Poul-Henning Kamp +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# Read the vmod.spec file and produce the vmod.h and vmod.c files. +# +# vmod.h contains the prototypes for the published functions, the module +# C-code should include this file to ensure type-consistency. +# +# vmod.c contains the symbols which VCC and varnishd will use to access +# the module: A structure of properly typed function pointers, the +# size of this structure in bytes, and the definition of the structure +# as a string, suitable for inclusion in the C-source of the compile VCL +# program. + +import sys +import re + +if len(sys.argv) == 2: + specfile = sys.argv[1] +else: + specfile = "vmod.vcc" + +ctypes = { + 'IP': "struct sockaddr_storage *", + 'STRING': "const char *", + 'STRING_LIST': "const char *, ...", + 'BOOL': "unsigned", + 'BACKEND': "struct director *", + 'ENUM': "const char *", + 'TIME': "double", + 'REAL': "double", + 'DURATION': "double", + 'INT': "int", + 'HEADER': "enum gethdr_e, const char *", + 'PRIV_VCL': "struct vmod_priv *", + 'PRIV_CALL': "struct vmod_priv *", + 'VOID': "void", +} + +####################################################################### + +initname = "" +modname = "???" +pstruct = "" +pinit = "" +tdl = "" +plist = "" +slist = "" + +def do_func(fname, rval, args, vargs): + global pstruct + global pinit + global plist + global slist + global tdl + #print(fname, rval, args) + + # C argument list + cargs = "(struct sess *" + for i in args: + cargs += ", " + i + cargs += ")" + + # Prototypes for vmod implementation and interface typedef + proto = ctypes[rval] + " vmod_" + fname + cargs + sproto = ctypes[rval] + " td_" + modname + "_" + fname + cargs + + # append to lists of prototypes + plist += proto + ";\n" + tdl += "typedef " + sproto + ";\n" + + # Append to struct members + pstruct += "\ttd_" + modname + "_" + fname + "\t*" + fname + ";\n" + + # Append to struct initializer + pinit += "\tvmod_" + fname + ",\n" + + # Compose the vmod spec-string + s = modname + '.' + fname + "\\0" + s += "Vmod_Func_" + modname + "." + fname + "\\0" + s += rval + '\\0' + for i in vargs: + s += i + '\\0' + slist += '\t"' + s + '",\n' + +####################################################################### + +def partition(string, separator): + if (hasattr(string,"partition")): + return string.partition(separator) + i = string.find(separator) + if i >= 0: + return (string[:i],separator,string[i+len(separator):]) + return (string, '', '') + +####################################################################### + +def is_c_name(s): + return None != re.match("^[a-z][a-z0-9_]*$", s) + +####################################################################### + +def parse_enum(tq): + assert tq[0] == '{' + assert tq[-1] == '}' + f = tq[1:-1].split(',') + s="ENUM\\0" + b=dict() + for i in f: + i = i.strip() + if not is_c_name(i): + raise Exception("Enum value '%s' is illegal" % i) + if i in b: + raise Exception("Duplicate Enum value '%s'" % i) + b[i] = True + s = s + i.strip() + '\\0' + return s + +####################################################################### + +f = open(specfile, "r") + +def nextline(): + while True: + l0 = f.readline() + if l0 == "": + return l0 + l0 = re.sub("#.*$", "", l0) + l0 = re.sub("\s\s*", " ", l0.strip()) + if l0 != "": + return l0 + +while True: + l0 = nextline() + if l0 == "": + break; + l = partition(l0, " ") + + if l[0] == "Module": + modname = l[2].strip(); + if not is_c_name(modname): + raise Exception("Module name '%s' is illegal" % modname) + continue + + if l[0] == "Init": + initname = l[2].strip(); + if not is_c_name(initname): + raise Exception("Init name '%s' is illegal" % initname) + continue + + if l[0] != "Function": + raise Exception("Expected 'Function' line, got '%s'" % l[0]) + + # Find the return type of the function + l = partition(l[2].strip(), " ") + rt_type = l[0] + if rt_type not in ctypes: + raise Exception("Return type '%s' not a valid type" % rt_type) + + # Find the function name + l = partition(l[2].strip(), "(") + + fname = l[0].strip() + if not is_c_name(fname): + raise Exception("Function name '%s' is illegal" % fname) + + if l[1] != '(': + raise Exception("Missing '('") + + l = l[2] + + while -1 == l.find(")"): + l1 = nextline() + if l1 == "": + raise Exception("End Of Input looking for ')'") + l = l + l1 + + if -1 != l.find("("): + raise Exception("Nesting trouble with '(...)' ") + + if l[-1:] != ')': + raise Exception("Junk after ')'") + + l = l[:-1] + + args = list() + vargs = list() + + for i in re.finditer("([A-Z_]+)\s*({[^}]+})?(,|$)", l): + at = i.group(1) + tq = i.group(2) + if at not in ctypes: + raise Exception( + "Argument type '%s' not a valid type" % at) + + args.append(ctypes[at]) + + if at == "ENUM": + if tq == None: + raise Exception( + "Argument type '%s' needs qualifier {...}" + % at) + at=parse_enum(tq) + + elif tq != None: + raise Exception( + "Argument type '%s' cannot be qualified with {...}" + % at) + + vargs.append(at) + + do_func(fname, rt_type, args, vargs) + +####################################################################### +def dumps(s): + while True: + l = partition(s, "\n") + if len(l[0]) == 0: + break + fc.write('\t"' + l[0] + '\\n"\n') + s = l[2] + +####################################################################### + +if initname != "": + plist += "int " + initname + plist += "(struct vmod_priv *, const struct VCL_conf *);\n" + pstruct += "\tvmod_init_f\t*_init;\n" + pinit += "\t" + initname + ",\n" + slist += '\t"INIT\\0Vmod_Func_' + modname + '._init",\n' + +####################################################################### + +def file_header(fo): + fo.write("""/* + * NB: This file is machine generated, DO NOT EDIT! + * + * Edit vmod.vcc and run vmod.py instead + */ + +""") + +####################################################################### + +fc = open("vcc_if.c", "w") +fh = open("vcc_if.h", "w") + +file_header(fc) +file_header(fh) + +fh.write('struct sess;\n') +fh.write('struct VCL_conf;\n') +fh.write('struct vmod_priv;\n') +fh.write("\n"); + +fh.write(plist) + + +fc.write('#include "config.h"\n') +fc.write('\n') +fc.write('#include "vrt.h"\n') +fc.write('#include "vcc_if.h"\n') +fc.write('#include "vmod_abi.h"\n') +fc.write("\n"); + +fc.write("\n"); + +fc.write(tdl); +fc.write("\n"); + +fc.write('const char Vmod_Name[] = "' + modname + '";\n') + +fc.write("const struct Vmod_Func_" + modname + " {\n") +fc.write(pstruct + "} Vmod_Func = {\n" + pinit + "};\n") +fc.write("\n"); + +fc.write("const int Vmod_Len = sizeof(Vmod_Func);\n") +fc.write("\n"); + +fc.write('const char Vmod_Proto[] =\n') +dumps(tdl); +fc.write('\t"\\n"\n') +dumps("struct Vmod_Func_" + modname + " {\n") +dumps(pstruct + "} Vmod_Func_" + modname + ";\n") +fc.write('\t;\n') +fc.write("\n"); + +fc.write('const char * const Vmod_Spec[] = {\n' + slist + '\t0\n};\n') + +fc.write('const char Vmod_Varnish_ABI[] = VMOD_ABI_Version;\n') + +fh.write('extern const void * const Vmod_Id;\n') +fc.write('const void * const Vmod_Id = &Vmod_Id;\n') + +fc.write("\n") + diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am index c4128ea..a872837 100644 --- a/lib/libvmod_std/Makefile.am +++ b/lib/libvmod_std/Makefile.am @@ -9,6 +9,7 @@ dist_man_MANS = vmod_std.3 vmoddir = $(pkglibdir)/vmods vmod_srcdir = $(top_srcdir)/lib/libvmod_std +vmodtool = $(top_srcdir)/lib/libvcl/vmodtool.py vmod_LTLIBRARIES = libvmod_std.la libvmod_std_la_LDFLAGS = -module -export-dynamic -avoid-version @@ -20,10 +21,10 @@ libvmod_std_la_SOURCES = \ vmod_std_fileread.c \ vmod_std_conversions.c -vcc_if.c vcc_if.h: $(vmod_srcdir)/vmod.py $(vmod_srcdir)/vmod.vcc - @PYTHON@ $(vmod_srcdir)/vmod.py $(vmod_srcdir)/vmod.vcc +vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc + @PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc -EXTRA_DIST = vmod.py vmod.vcc +EXTRA_DIST = vmod.vcc CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h diff --git a/lib/libvmod_std/vmod.py b/lib/libvmod_std/vmod.py deleted file mode 100755 index 041edcb..0000000 --- a/lib/libvmod_std/vmod.py +++ /dev/null @@ -1,321 +0,0 @@ -#!/usr/local/bin/python -#- -# Copyright (c) 2010-2011 Varnish Software AS -# All rights reserved. -# -# Author: Poul-Henning Kamp -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -# SUCH DAMAGE. -# -# Read the vmod.spec file and produce the vmod.h and vmod.c files. -# -# vmod.h contains the prototypes for the published functions, the module -# C-code should include this file to ensure type-consistency. -# -# vmod.c contains the symbols which VCC and varnishd will use to access -# the module: A structure of properly typed function pointers, the -# size of this structure in bytes, and the definition of the structure -# as a string, suitable for inclusion in the C-source of the compile VCL -# program. - -import sys -import re - -if len(sys.argv) == 2: - specfile = sys.argv[1] -else: - specfile = "vmod.vcc" - -ctypes = { - 'IP': "struct sockaddr_storage *", - 'STRING': "const char *", - 'STRING_LIST': "const char *, ...", - 'BOOL': "unsigned", - 'BACKEND': "struct director *", - 'ENUM': "const char *", - 'TIME': "double", - 'REAL': "double", - 'DURATION': "double", - 'INT': "int", - 'HEADER': "enum gethdr_e, const char *", - 'PRIV_VCL': "struct vmod_priv *", - 'PRIV_CALL': "struct vmod_priv *", - 'VOID': "void", -} - -####################################################################### - -initname = "" -modname = "???" -pstruct = "" -pinit = "" -tdl = "" -plist = "" -slist = "" - -def do_func(fname, rval, args, vargs): - global pstruct - global pinit - global plist - global slist - global tdl - #print(fname, rval, args) - - # C argument list - cargs = "(struct sess *" - for i in args: - cargs += ", " + i - cargs += ")" - - # Prototypes for vmod implementation and interface typedef - proto = ctypes[rval] + " vmod_" + fname + cargs - sproto = ctypes[rval] + " td_" + modname + "_" + fname + cargs - - # append to lists of prototypes - plist += proto + ";\n" - tdl += "typedef " + sproto + ";\n" - - # Append to struct members - pstruct += "\ttd_" + modname + "_" + fname + "\t*" + fname + ";\n" - - # Append to struct initializer - pinit += "\tvmod_" + fname + ",\n" - - # Compose the vmod spec-string - s = modname + '.' + fname + "\\0" - s += "Vmod_Func_" + modname + "." + fname + "\\0" - s += rval + '\\0' - for i in vargs: - s += i + '\\0' - slist += '\t"' + s + '",\n' - -####################################################################### - -def partition(string, separator): - if (hasattr(string,"partition")): - return string.partition(separator) - i = string.find(separator) - if i >= 0: - return (string[:i],separator,string[i+len(separator):]) - return (string, '', '') - -####################################################################### - -def is_c_name(s): - return None != re.match("^[a-z][a-z0-9_]*$", s) - -####################################################################### - -def parse_enum(tq): - assert tq[0] == '{' - assert tq[-1] == '}' - f = tq[1:-1].split(',') - s="ENUM\\0" - b=dict() - for i in f: - i = i.strip() - if not is_c_name(i): - raise Exception("Enum value '%s' is illegal" % i) - if i in b: - raise Exception("Duplicate Enum value '%s'" % i) - b[i] = True - s = s + i.strip() + '\\0' - return s - -####################################################################### - -f = open(specfile, "r") - -def nextline(): - while True: - l0 = f.readline() - if l0 == "": - return l0 - l0 = re.sub("#.*$", "", l0) - l0 = re.sub("\s\s*", " ", l0.strip()) - if l0 != "": - return l0 - -while True: - l0 = nextline() - if l0 == "": - break; - l = partition(l0, " ") - - if l[0] == "Module": - modname = l[2].strip(); - if not is_c_name(modname): - raise Exception("Module name '%s' is illegal" % modname) - continue - - if l[0] == "Init": - initname = l[2].strip(); - if not is_c_name(initname): - raise Exception("Init name '%s' is illegal" % initname) - continue - - if l[0] != "Function": - raise Exception("Expected 'Function' line, got '%s'" % l[0]) - - # Find the return type of the function - l = partition(l[2].strip(), " ") - rt_type = l[0] - if rt_type not in ctypes: - raise Exception("Return type '%s' not a valid type" % rt_type) - - # Find the function name - l = partition(l[2].strip(), "(") - - fname = l[0].strip() - if not is_c_name(fname): - raise Exception("Function name '%s' is illegal" % fname) - - if l[1] != '(': - raise Exception("Missing '('") - - l = l[2] - - while -1 == l.find(")"): - l1 = nextline() - if l1 == "": - raise Exception("End Of Input looking for ')'") - l = l + l1 - - if -1 != l.find("("): - raise Exception("Nesting trouble with '(...)' ") - - if l[-1:] != ')': - raise Exception("Junk after ')'") - - l = l[:-1] - - args = list() - vargs = list() - - for i in re.finditer("([A-Z_]+)\s*({[^}]+})?(,|$)", l): - at = i.group(1) - tq = i.group(2) - if at not in ctypes: - raise Exception( - "Argument type '%s' not a valid type" % at) - - args.append(ctypes[at]) - - if at == "ENUM": - if tq == None: - raise Exception( - "Argument type '%s' needs qualifier {...}" - % at) - at=parse_enum(tq) - - elif tq != None: - raise Exception( - "Argument type '%s' cannot be qualified with {...}" - % at) - - vargs.append(at) - - do_func(fname, rt_type, args, vargs) - -####################################################################### -def dumps(s): - while True: - l = partition(s, "\n") - if len(l[0]) == 0: - break - fc.write('\t"' + l[0] + '\\n"\n') - s = l[2] - -####################################################################### - -if initname != "": - plist += "int " + initname - plist += "(struct vmod_priv *, const struct VCL_conf *);\n" - pstruct += "\tvmod_init_f\t*_init;\n" - pinit += "\t" + initname + ",\n" - slist += '\t"INIT\\0Vmod_Func_' + modname + '._init",\n' - -####################################################################### - -def file_header(fo): - fo.write("""/* - * NB: This file is machine generated, DO NOT EDIT! - * - * Edit vmod.vcc and run vmod.py instead - */ - -""") - -####################################################################### - -fc = open("vcc_if.c", "w") -fh = open("vcc_if.h", "w") - -file_header(fc) -file_header(fh) - -fh.write('struct sess;\n') -fh.write('struct VCL_conf;\n') -fh.write('struct vmod_priv;\n') -fh.write("\n"); - -fh.write(plist) - - -fc.write('#include "config.h"\n') -fc.write('\n') -fc.write('#include "vrt.h"\n') -fc.write('#include "vcc_if.h"\n') -fc.write('#include "vmod_abi.h"\n') -fc.write("\n"); - -fc.write("\n"); - -fc.write(tdl); -fc.write("\n"); - -fc.write('const char Vmod_Name[] = "' + modname + '";\n') - -fc.write("const struct Vmod_Func_" + modname + " {\n") -fc.write(pstruct + "} Vmod_Func = {\n" + pinit + "};\n") -fc.write("\n"); - -fc.write("const int Vmod_Len = sizeof(Vmod_Func);\n") -fc.write("\n"); - -fc.write('const char Vmod_Proto[] =\n') -dumps(tdl); -fc.write('\t"\\n"\n') -dumps("struct Vmod_Func_" + modname + " {\n") -dumps(pstruct + "} Vmod_Func_" + modname + ";\n") -fc.write('\t;\n') -fc.write("\n"); - -fc.write('const char * const Vmod_Spec[] = {\n' + slist + '\t0\n};\n') - -fc.write('const char Vmod_Varnish_ABI[] = VMOD_ABI_Version;\n') - -fh.write('extern const void * const Vmod_Id;\n') -fc.write('const void * const Vmod_Id = &Vmod_Id;\n') - -fc.write("\n") - From martin at varnish-software.com Thu Dec 18 09:27:46 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] d12b924 Install vmodtool.py in pkgdatadir Message-ID: commit d12b924b473e716b71d075ba4c81bc445e30d361 Author: Martin Blix Grydeland Date: Wed Feb 29 10:07:11 2012 +0100 Install vmodtool.py in pkgdatadir diff --git a/lib/libvcl/Makefile.am b/lib/libvcl/Makefile.am index ad16981..02a685f 100644 --- a/lib/libvcl/Makefile.am +++ b/lib/libvcl/Makefile.am @@ -31,7 +31,9 @@ libvcl_la_SOURCES = \ vcc_xref.c EXTRA_DIST = \ - generate.py \ + generate.py + +dist_pkgdata_SCRIPTS = \ vmodtool.py vcc_obj.c vcc_fixed_token.c vcc_token_defs.h: \ From martin at varnish-software.com Thu Dec 18 09:27:46 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] 4d14c2f Install headers needed for vmod compilation in datadir/varnish/include, and export these locations in the pkg_config file. Message-ID: commit 4d14c2f746732f384ff429dcb9f5caedcf247aa8 Author: Martin Blix Grydeland Date: Wed Feb 29 17:39:48 2012 +0100 Install headers needed for vmod compilation in datadir/varnish/include, and export these locations in the pkg_config file. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 8a925b9..bd91dd7 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -84,12 +84,9 @@ varnishd_SOURCES = \ waiter/cache_waiter_ports.c noinst_HEADERS = \ - cache/cache.h \ cache/cache_backend.h \ cache/cache_esi.h \ - common/common.h \ common/heritage.h \ - common/params.h \ default_vcl.h \ hash/hash_slinger.h \ mgt/mgt.h \ @@ -99,6 +96,13 @@ noinst_HEADERS = \ storage/storage_persistent.h \ waiter/waiter.h +# Headers for use with vmods +pkgdataincludedir = $(pkgdatadir)/include +nobase_pkgdatainclude_HEADERS = \ + cache/cache.h \ + common/common.h \ + common/params.h + varnishd_CFLAGS = \ @PCRE_CFLAGS@ \ -DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' \ diff --git a/include/Makefile.am b/include/Makefile.am index 91812fa..3faa540 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -1,5 +1,6 @@ # +# API headers nobase_pkginclude_HEADERS = \ tbl/acct_fields.h \ tbl/backend_poll.h \ @@ -26,6 +27,7 @@ nobase_pkginclude_HEADERS = \ vapi/vsl_int.h \ vcli.h +# Private headers nobase_noinst_HEADERS = \ binary_heap.h \ compat/daemon.h \ @@ -33,38 +35,42 @@ nobase_noinst_HEADERS = \ compat/srandomdev.h \ flopen.h \ libvcl.h \ - miniobj.h \ persistent.h \ - vas.h \ - vav.h \ - vbm.h \ - vcl.h \ vcli_common.h \ vcli_priv.h \ vcli_serve.h \ vcs_version.h \ - vcs.h \ vct.h \ - vdef.h \ vend.h \ vev.h \ vfil.h \ vin.h \ vlu.h \ vmb.h \ - vmod_abi.h \ vnum.h \ vpf.h \ + vsub.h \ + vss.h \ + vtcp.h \ + vtim.h + +# Headers for use with vmods +pkgdataincludedir = $(pkgdatadir)/include +nobase_pkgdatainclude_HEADERS = \ + miniobj.h \ + vas.h \ + vav.h \ + vbm.h \ + vcl.h \ + vcs.h \ + vmod_abi.h \ vqueue.h \ vre.h \ + vdef.h \ vrt.h \ vrt_obj.h \ vsb.h \ - vsub.h \ - vsha256.h \ - vss.h \ - vtcp.h \ - vtim.h + vsha256.h tbl/vrt_stv_var.h tbl/vcl_returns.h vcl.h vrt_obj.h: $(top_srcdir)/lib/libvcl/generate.py $(top_srcdir)/include/vrt.h mkdir -p tbl diff --git a/varnishapi.pc.in b/varnishapi.pc.in index 71cda56..7f5036a 100644 --- a/varnishapi.pc.in +++ b/varnishapi.pc.in @@ -1,11 +1,18 @@ prefix=@prefix@ exec_prefix=@exec_prefix@ +bindir=@bindir@ +sbindir=@sbindir@ libdir=@libdir@ includedir=@includedir@ -vmoddir=${libdir}/varnish/vmods +pkgincludedir=${includedir}/@PACKAGE@ +datarootdir=@datarootdir@ +datadir=@datadir@ +pkgdatadir=${datadir}/@PACKAGE@ +pkgdataincludedir=${pkgdatadir}/include +vmoddir=${libdir}/@PACKAGE@/vmods Name: VarnishAPI Description: Varnish API Version: @PACKAGE_VERSION@ -Cflags: -I${includedir}/varnish +Cflags: -I${includedir}/@PACKAGE@ Libs: -L${libdir} -lvarnishapi From martin at varnish-software.com Thu Dec 18 09:27:46 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] 7bf7038 Add a varnishapi-uninstalled.pc file, for use by out-of-tree vmod building against an uninstalled Varnish build tree. Message-ID: commit 7bf7038f4e577320b4c8e19fee89f3ef18c03725 Author: Martin Blix Grydeland Date: Fri Apr 13 13:39:01 2012 +0200 Add a varnishapi-uninstalled.pc file, for use by out-of-tree vmod building against an uninstalled Varnish build tree. diff --git a/configure.ac b/configure.ac index e6ecfb9..20134a7 100644 --- a/configure.ac +++ b/configure.ac @@ -554,5 +554,6 @@ AC_CONFIG_FILES([ man/Makefile redhat/Makefile varnishapi.pc + varnishapi-uninstalled.pc ]) AC_OUTPUT diff --git a/varnishapi-uninstalled.pc.in b/varnishapi-uninstalled.pc.in new file mode 100644 index 0000000..3a8f744 --- /dev/null +++ b/varnishapi-uninstalled.pc.in @@ -0,0 +1,20 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +bindir=@bindir@ +sbindir=@sbindir@ +libdir=@libdir@ +includedir=@includedir@ +pkgincludedir=${includedir}/@PACKAGE@ +datarootdir=@datarootdir@ +datadir=@datadir@ +pkgdatadir=${datadir}/@PACKAGE@ +pkgdataincludedir=${pkgdatadir}/include +vmoddir=${libdir}/@PACKAGE@/vmods +builddir=@abs_top_builddir@ +srcdir=@abs_top_srcdir@ + +Name: VarnishAPI +Description: Varnish API +Version: @PACKAGE_VERSION@ +Cflags: -I${includedir}/@PACKAGE@ +Libs: -L${libdir} -lvarnishapi From phk at FreeBSD.org Thu Dec 18 09:27:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] 40ee205 These are not debug commands. Message-ID: commit 40ee205a3c8c6a95db08116ff7f4d7f51f7abbc9 Author: Poul-Henning Kamp Date: Thu May 10 08:57:47 2012 +0000 These are not debug commands. diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c index c2d022d..45b85d0 100644 --- a/bin/varnishd/cache/cache_backend_cfg.c +++ b/bin/varnishd/cache/cache_backend_cfg.c @@ -487,9 +487,9 @@ cli_backend_set_health(struct cli *cli, const char * const *av, void *priv) static struct cli_proto backend_cmds[] = { { "backend.list", "backend.list", - "\tList all backends\n", 0, 1, "d", cli_backend_list }, + "\tList all backends\n", 0, 1, "", cli_backend_list }, { "backend.set_health", "backend.set_health matcher state", - "\tShow a backend\n", 2, 2, "d", cli_backend_set_health }, + "\tShow a backend\n", 2, 2, "", cli_backend_set_health }, { NULL } }; From daghf at varnish-software.com Thu Dec 18 09:27:46 2014 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] 4020b13 Req.hash_always_miss now implies req.hash_ignore_busy. Message-ID: commit 4020b13d0eab47df865d3b055404ed84b8ffd459 Author: Dag Haavi Finstad Date: Tue May 8 17:10:58 2012 +0200 Req.hash_always_miss now implies req.hash_ignore_busy. Fixes a case where we might get a cache hit even though hash_always_miss is set. Fixes: #1073 diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 6facf0a..1228126 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -332,7 +332,7 @@ HSH_Lookup(struct sess *sp) if (oc->flags & OC_F_BUSY || oc->busyobj != NULL) { CHECK_OBJ_ORNULL(oc->busyobj, BUSYOBJ_MAGIC); - if (req->hash_ignore_busy) + if (req->hash_ignore_busy || req->hash_always_miss) continue; if (oc->busyobj != NULL && diff --git a/bin/varnishtest/tests/r01073.vtc b/bin/varnishtest/tests/r01073.vtc new file mode 100644 index 0000000..936f8ce --- /dev/null +++ b/bin/varnishtest/tests/r01073.vtc @@ -0,0 +1,55 @@ +varnishtest "Test that hash_always_miss also implies hash_ignore_busy. Ticket #1073." + +server s1 { + rxreq + sema r1 sync 2 + sema r2 sync 2 + delay 1 + txresp -hdr "Server: 1" +} -start + +server s2 { + rxreq + sema r2 sync 2 + txresp -hdr "Server: 2" +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.http.x-hash-always-miss == "1") { + set req.hash_always_miss = true; + } + if (req.http.x-client == "1") { + set req.backend = s1; + } + if (req.http.x-client == "2") { + set req.backend = s2; + } + } + sub vcl_deliver { + if(obj.hits > 0) { + set resp.http.X-Cache = "HIT"; + } else { + set resp.http.X-Cache = "MISS"; + } + } +} -start + +client c1 { + txreq -url "/" -hdr "x-client: 1" + rxresp + expect resp.status == 200 + expect resp.http.Server == "1" +} -start + +client c2 { + sema r1 sync 2 + txreq -url "/" -hdr "x-client: 2" -hdr "x-hash-always-miss: 1" + txreq -url "/" -hdr "x-client: 2" + rxresp + expect resp.status == 200 + expect resp.http.Server == "2" +} -start + +client c1 -wait +client c2 -wait From phk at FreeBSD.org Thu Dec 18 09:27:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] 3814b12 Use admin health to control test-state Message-ID: commit 3814b12840885117f3b2f78de7f5bc25f99fd405 Author: Poul-Henning Kamp Date: Thu May 10 09:09:49 2012 +0000 Use admin health to control test-state diff --git a/bin/varnishtest/tests/r00306.vtc b/bin/varnishtest/tests/r00306.vtc index ebc1774..079e994 100644 --- a/bin/varnishtest/tests/r00306.vtc +++ b/bin/varnishtest/tests/r00306.vtc @@ -23,10 +23,6 @@ varnish v1 -vcl { } backend s2 { .host = "${s2_addr}"; .port = "${s2_port}"; - .probe = { - .url = "/"; - .initial = 0; - } } director foo random { { .backend = s2; .weight = 1; } @@ -38,6 +34,9 @@ varnish v1 -vcl { } } -start +varnish v1 -cliok "backend.set_health s2 sick" +varnish v1 -cliok "backend.list" + client c1 { timeout 10 From phk at FreeBSD.org Thu Dec 18 09:27:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] 0352ea2 Make the random/hash/client director use a stable hash. Message-ID: commit 0352ea2567c5b4df803635ec8ac4288ae5c0fa07 Author: Poul-Henning Kamp Date: Thu May 10 09:20:54 2012 +0000 Make the random/hash/client director use a stable hash. If the chosen backend is sick or fails to give us a connection, we rescale the keys position in the backends window onto the all the remaining backends. The upside of this is that we get a stable and predicatable hash distribution. The downside of this is that you are very unlikely to spot the healthy .weight=1 backend between two .weight=100 sick ones before your retries run out. Inspired by: patch from WaveCDN diff --git a/bin/varnishd/cache/cache_dir_random.c b/bin/varnishd/cache/cache_dir_random.c index b323343..8a0bcd2 100644 --- a/bin/varnishd/cache/cache_dir_random.c +++ b/bin/varnishd/cache/cache_dir_random.c @@ -127,32 +127,50 @@ vdi_random_init_seed(const struct vdi_random *vs, const struct sess *sp) * Find the healthy backend corresponding to the weight r [0...1[ */ static struct vbc * -vdi_random_pick_one(struct sess *sp, const struct vdi_random *vs, double r) +vdi_random_pick_one(struct sess *sp, const struct vdi_random *vs, double r, + int retries) { double w[vs->nhosts]; int i; - double s1; - - assert(r >= 0.0 && r < 1.0); + double s1, s2; + struct vbc *vbc; + /* Sum up the weights of all backends */ memset(w, 0, sizeof w); - /* Sum up the weights of healty backends */ s1 = 0.0; for (i = 0; i < vs->nhosts; i++) { - if (VDI_Healthy(vs->hosts[i].backend, sp)) - w[i] = vs->hosts[i].weight; + w[i] = vs->hosts[i].weight; s1 += w[i]; } - if (s1 == 0.0) return (NULL); - r *= s1; - s1 = 0.0; - for (i = 0; i < vs->nhosts; i++) { - s1 += w[i]; - if (r < s1) - return(VDI_GetFd(vs->hosts[i].backend, sp)); + while (retries-- > 0) { + assert(r >= 0.0 && r < 1.0); + + r *= s1; + s2 = 0.0; + for (i = 0; i < vs->nhosts; i++) { + s2 += w[i]; + if (r >= s2) + continue; + if (!VDI_Healthy(vs->hosts[i].backend, sp)) + break; + vbc = VDI_GetFd(vs->hosts[i].backend, sp); + if (vbc == NULL) + break; + return (vbc); + } + /* + * Rescale and rotate r's relative position this backends + * window onto the remaining backends and try again. + */ + r -= (s2 - w[i]); // r in [0...w[i][ + r /= w[i]; // r in [0...1[ + r *= (s1 - w[i])/s1; // r in [0...1-W[i]] + r += s2 / s1; // rotate + if (r >= 1.0) + r -= 1.0; } return (NULL); } @@ -165,10 +183,8 @@ vdi_random_pick_one(struct sess *sp, const struct vdi_random *vs, double r) static struct vbc * vdi_random_getfd(const struct director *d, struct sess *sp) { - int k; struct vdi_random *vs; double r; - struct vbc *vbe; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); @@ -176,17 +192,12 @@ vdi_random_getfd(const struct director *d, struct sess *sp) r = vdi_random_init_seed(vs, sp); - for (k = 0; k < vs->retries; k++) { - vbe = vdi_random_pick_one(sp, vs, r); - if (vbe != NULL) - return (vbe); - r = vdi_random_sha((void *)&r, sizeof(r)); - } - return (NULL); + return (vdi_random_pick_one(sp, vs, r, vs->retries)); } /* * Healthy if just a single backend is... + * XXX: we should really have a weight param/criteria here */ static unsigned vdi_random_healthy(const struct director *d, const struct sess *sp) From martin at varnish-software.com Thu Dec 18 09:27:46 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] 9d0e486 Don't consider a vcc expr to be constant after vcc_expr_tostring. Message-ID: commit 9d0e486fe21199e6c059eb829f5d666611806012 Author: Martin Blix Grydeland Date: Tue May 8 14:20:42 2012 +0200 Don't consider a vcc expr to be constant after vcc_expr_tostring. Avoids vcc errors from constructs like "std.log("Test: " + 1);" Fixes: #1134 diff --git a/bin/varnishtest/tests/r01134.vtc b/bin/varnishtest/tests/r01134.vtc new file mode 100644 index 0000000..73f4e36 --- /dev/null +++ b/bin/varnishtest/tests/r01134.vtc @@ -0,0 +1,19 @@ +varnishtest "vcc const/non-const tostring conversion - bug 1134" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + set req.http.x-test = "Test " + 1; + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run + diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c index 30e63d9..19a455c 100644 --- a/lib/libvcl/vcc_expr.c +++ b/lib/libvcl/vcc_expr.c @@ -434,8 +434,10 @@ vcc_expr_tostring(struct expr **e, enum var_type fmt) case TIME: p = "VRT_time_string(sp, \v1)"; break; default: break; } - if (p != NULL) + if (p != NULL) { *e = vcc_expr_edit(fmt, p, *e, NULL); + (*e)->constant = 0; + } } /*-------------------------------------------------------------------- From martin at varnish-software.com Thu Dec 18 09:27:46 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] 5b3e41b Fix vmod_log (VRT_StringList returns end of string, not beginning) Message-ID: commit 5b3e41b4958ea33012ef290c706e6c45d540d361 Author: Martin Blix Grydeland Date: Tue Apr 24 14:21:42 2012 +0200 Fix vmod_log (VRT_StringList returns end of string, not beginning) Fixes: #1135 diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 5b5c0aa..8ae9ec1 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -147,19 +147,18 @@ vmod_random(struct sess *sp, double lo, double hi) void __match_proto__() vmod_log(struct sess *sp, const char *fmt, ...) { - char *p; unsigned u; va_list ap; txt t; u = WS_Reserve(sp->req->ws, 0); - p = sp->req->ws->f; + t.b = sp->req->ws->f; va_start(ap, fmt); - p = VRT_StringList(p, u, fmt, ap); + t.e = VRT_StringList(t.b, u, fmt, ap); va_end(ap); - if (p != NULL) { - t.b = p; - t.e = strchr(p, '\0'); + if (t.e != NULL) { + assert(t.e > t.b); + t.e--; VSLbt(sp->req->vsl, SLT_VCL_Log, t); } WS_Release(sp->req->ws, 0); From phk at FreeBSD.org Thu Dec 18 09:27:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:46 +0100 Subject: [experimental-ims] 9efa2b0 Introduce the obj_readonly parameter which disables obj.hits and obj.lastuse in order to not dirty VM pages containing cached objects. Message-ID: commit 9efa2b0005a9e9fbfee7ba47018ee64df679594f Author: Poul-Henning Kamp Date: Thu May 10 09:56:07 2012 +0000 Introduce the obj_readonly parameter which disables obj.hits and obj.lastuse in order to not dirty VM pages containing cached objects. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 332211a..c028cb5 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -261,7 +261,8 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) cache_param->lru_timeout && EXP_Touch(req->obj->objcore)) req->obj->last_lru = req->t_resp; - req->obj->last_use = req->t_resp; /* XXX: locking ? */ + if (!cache_param->obj_readonly) + req->obj->last_use = req->t_resp; /* XXX: locking ? */ } HTTP_Setup(req->resp, req->ws, req->vsl, HTTP_Resp); RES_BuildHttp(sp); diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 1228126..6b84bab 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -398,7 +398,7 @@ HSH_Lookup(struct sess *sp) assert(hash->deref(oh)); o = oc_getobj(&wrk->stats, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - if (o->hits < INT_MAX) + if (!cache_param->obj_readonly && o->hits < INT_MAX) o->hits++; return (oc); } diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index 6e502fd..1e13657 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -188,6 +188,8 @@ struct params { unsigned gzip_window; unsigned gzip_memlevel; + unsigned obj_readonly; + double critbit_cooloff; double shortlived; diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 885d2ce..c8f62bb 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1260,6 +1260,12 @@ static const struct parspec input_parspec[] = { 0, "10,100,10", ""}, + { "obj_readonly", tweak_bool, &mgt_param.obj_readonly, 0, 0, + "If set, we do not update obj.hits and obj.lastuse to" + "avoid dirtying VM pages associated with cached objects.", + 0, + "false", ""}, + { NULL, NULL, NULL } }; From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 390fa49 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit 390fa49f574a3f8e44a6f30c508641e513c7b737 Merge: 9efa2b0 5b3e41b Author: Poul-Henning Kamp Date: Thu May 10 09:56:49 2012 +0000 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] f289def Make all objects have an objcore, even for the vcl_recv->pass case. Message-ID: commit f289def3a62bdc15aa0ef78217254edba689ef2f Author: Poul-Henning Kamp Date: Mon May 14 10:05:53 2012 +0000 Make all objects have an objcore, even for the vcl_recv->pass case. This simplifies some nasty corner cases. diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index c53d945..6986d21 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -447,6 +447,7 @@ BAN_NewObjCore(struct objcore *oc) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AZ(oc->ban); + AN(oc->objhead); Lck_Lock(&ban_mtx); oc->ban = ban_start; ban_start->refcount++; diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 7fa0291..48efe08 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -152,7 +152,7 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo) *pbo = NULL; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_ORNULL(bo->fetch_obj, OBJECT_MAGIC); - if (bo->fetch_obj != NULL && bo->fetch_obj->objcore != NULL) { + if (bo->fetch_obj != NULL && bo->fetch_obj->objcore->objhead != NULL) { oc = bo->fetch_obj->objcore; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index c028cb5..9b7a968 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -256,7 +256,7 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) } req->t_resp = W_TIM_real(wrk); - if (req->obj->objcore != NULL) { + if (req->obj->objcore->objhead != NULL) { if ((req->t_resp - req->obj->last_lru) > cache_param->lru_timeout && EXP_Touch(req->obj->objcore)) @@ -340,11 +340,8 @@ cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req) RES_WriteObj(sp); /* No point in saving the body if it is hit-for-pass */ - if (req->obj->objcore != NULL) { - CHECK_OBJ_NOTNULL(req->obj->objcore, OBJCORE_MAGIC); - if (req->obj->objcore->flags & OC_F_PASS) - STV_Freestore(req->obj); - } + if (req->obj->objcore->flags & OC_F_PASS) + STV_Freestore(req->obj); assert(WRW_IsReleased(wrk)); (void)HSH_Deref(&wrk->stats, NULL, &req->obj); @@ -492,6 +489,7 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) bo->vsl->wid = sp->vsl_id; AZ(bo->stats); bo->stats = &wrk->stats; + req->objcore = HSH_NewObjCore(wrk); req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE, cache_param->http_resp_size, (uint16_t)cache_param->http_max_hdr); @@ -586,7 +584,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) wrk->acct_tmp.fetch++; - i = FetchHdr(sp, need_host_hdr, req->objcore == NULL); + i = FetchHdr(sp, need_host_hdr, req->objcore->objhead == NULL); /* * If we recycle a backend connection, there is a finite chance * that the backend closed it before we get a request to it. @@ -594,7 +592,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) */ if (i == 1) { VSC_C_main->backend_retry++; - i = FetchHdr(sp, need_host_hdr, req->objcore == NULL); + i = FetchHdr(sp, need_host_hdr, req->objcore->objhead == NULL); } if (i) { @@ -626,7 +624,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) RFC2616_Ttl(bo, sp->req->xid); /* pass from vclrecv{} has negative TTL */ - if (req->objcore == NULL) + if (req->objcore->objhead == NULL) bo->exp.ttl = -1.; AZ(bo->do_esi); @@ -634,7 +632,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) VCL_fetch_method(sp); - if (req->objcore != NULL && bo->do_pass) + if (bo->do_pass) req->objcore->flags |= OC_F_PASS; switch (req->handling) { @@ -652,7 +650,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) /* Clean up partial fetch */ AZ(bo->vbc); - if (req->objcore != NULL) { + if (req->objcore->objhead != NULL || req->handling == VCL_RET_ERROR) { CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); AZ(HSH_Deref(&wrk->stats, req->objcore, NULL)); req->objcore = NULL; @@ -707,7 +705,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) assert(req->handling == VCL_RET_DELIVER); - if (req->objcore == NULL) { + if (req->objcore->objhead == NULL) { /* This is a pass from vcl_recv */ pass = 1; /* VCL may have fiddled this, but that doesn't help */ @@ -785,7 +783,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp); /* Create Vary instructions */ - if (req->objcore != NULL) { + if (req->objcore->objhead != NULL) { CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); vary = VRY_Create(req, bo->beresp); if (vary != NULL) { @@ -884,7 +882,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) assert(bo->refcount == 2); /* one for each thread */ - if (req->obj->objcore != NULL) { + if (req->obj->objcore->objhead != NULL) { EXP_Insert(req->obj); AN(req->obj->objcore->ban); AZ(req->obj->ws_o->overflow); @@ -895,7 +893,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE)) FetchBody(wrk, bo); - if (req->obj->objcore != NULL) + if (req->obj->objcore->objhead != NULL) HSH_Ref(req->obj->objcore); if (bo->state == BOS_FINISHED) { @@ -1249,6 +1247,9 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req) assert(req->handling == VCL_RET_PASS); wrk->acct_tmp.pass++; sp->step = STP_FETCH; + + req->objcore = HSH_NewObjCore(wrk); + req->objcore->busyobj = bo; return (0); } diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index ee9147d..7e9a25c 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -453,10 +453,8 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) AN(req->director); AZ(req->obj); - if (req->objcore != NULL) { /* pass has no objcore */ - CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); - AN(req->objcore->flags & OC_F_BUSY); - } + CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); + AN(req->objcore->flags & OC_F_BUSY); hp = bo->bereq; @@ -687,7 +685,7 @@ FetchBody(struct worker *wrk, void *priv) /* XXX: Atomic assignment, needs volatile/membar ? */ bo->state = BOS_FINISHED; } - if (obj->objcore != NULL) + if (obj->objcore->objhead != NULL) HSH_Complete(obj->objcore); bo->stats = NULL; VBO_DerefBusyObj(wrk, &bo); diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 6b84bab..bc94352 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -66,23 +66,31 @@ static const struct hash_slinger *hash; /*---------------------------------------------------------------------*/ + +struct objcore * +HSH_NewObjCore(struct worker *wrk) +{ + struct objcore *oc; + + ALLOC_OBJ(oc, OBJCORE_MAGIC); + XXXAN(oc); + wrk->stats.n_objectcore++; + oc->flags |= OC_F_BUSY; + return (oc); +} + +/*---------------------------------------------------------------------*/ /* Precreate an objhead and object for later use */ static void hsh_prealloc(struct worker *wrk) { struct objhead *oh; - struct objcore *oc; struct waitinglist *wl; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - if (wrk->nobjcore == NULL) { - ALLOC_OBJ(oc, OBJCORE_MAGIC); - XXXAN(oc); - wrk->nobjcore = oc; - wrk->stats.n_objectcore++; - oc->flags |= OC_F_BUSY; - } + if (wrk->nobjcore == NULL) + wrk->nobjcore = HSH_NewObjCore(wrk); CHECK_OBJ_NOTNULL(wrk->nobjcore, OBJCORE_MAGIC); if (wrk->nobjhead == NULL) { @@ -663,40 +671,31 @@ HSH_Deref(struct dstat *ds, struct objcore *oc, struct object **oo) oc = o->objcore; } - if (o != NULL && oc == NULL) { - /* - * A pass object with neither objcore nor objhdr reference. - * -> simply free the (Transient) storage - */ - STV_Freestore(o); - STV_free(o->objstore); - ds->n_object--; - return (0); - } - CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); oh = oc->objhead; - CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); + if (oh != NULL) { + CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); - Lck_Lock(&oh->mtx); - assert(oh->refcnt > 0); - assert(oc->refcnt > 0); - r = --oc->refcnt; - if (!r) - VTAILQ_REMOVE(&oh->objcs, oc, list); - else { - /* Must have an object */ - AN(oc->methods); - } - if (oh->waitinglist != NULL) - hsh_rush(ds, oh); - Lck_Unlock(&oh->mtx); - if (r != 0) - return (r); + Lck_Lock(&oh->mtx); + assert(oh->refcnt > 0); + assert(oc->refcnt > 0); + r = --oc->refcnt; + if (!r) + VTAILQ_REMOVE(&oh->objcs, oc, list); + else { + /* Must have an object */ + AN(oc->methods); + } + if (oh->waitinglist != NULL) + hsh_rush(ds, oh); + Lck_Unlock(&oh->mtx); + if (r != 0) + return (r); - BAN_DestroyObj(oc); - AZ(oc->ban); + BAN_DestroyObj(oc); + AZ(oc->ban); + } if (oc->methods != NULL) { oc_freeobj(oc); @@ -705,11 +704,13 @@ HSH_Deref(struct dstat *ds, struct objcore *oc, struct object **oo) FREE_OBJ(oc); ds->n_objectcore--; - /* Drop our ref on the objhead */ - assert(oh->refcnt > 0); - if (hash->deref(oh)) - return (0); - HSH_DeleteObjHead(ds, oh); + if (oh != NULL) { + /* Drop our ref on the objhead */ + assert(oh->refcnt > 0); + if (hash->deref(oh)) + return (0); + HSH_DeleteObjHead(ds, oh); + } return (0); } diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 2e56666..37fb7a0 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -141,7 +141,7 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a) if (!vbc->backend) return; CHECK_OBJ_NOTNULL(vbc->backend, BACKEND_MAGIC); - if (!sp->req->objcore) + if (!sp->req->objcore->objhead) return; CHECK_OBJ_NOTNULL(sp->req->objcore, OBJCORE_MAGIC); diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index 17d67f9..cbf62ba 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -60,6 +60,7 @@ void HSH_AddString(const struct sess *sp, const char *str); void HSH_Insert(struct worker *, const void *hash, struct objcore *); void HSH_Purge(const struct sess *, struct objhead *, double ttl, double grace); void HSH_config(const char *h_arg); +struct objcore *HSH_NewObjCore(struct worker *wrk); #ifdef VARNISH_CACHE_CHILD diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index f1cbe94..53846a0 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -237,6 +237,7 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp, CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC); AN(ocp); + CHECK_OBJ_NOTNULL((*ocp), OBJCORE_MAGIC); assert(PAOK(ptr)); assert(PAOK(soc->wsl)); @@ -262,17 +263,14 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp, VTAILQ_INIT(&o->store); bo->stats->n_object++; - if (*ocp != NULL) { - CHECK_OBJ_NOTNULL((*ocp), OBJCORE_MAGIC); - - o->objcore = *ocp; - *ocp = NULL; /* refcnt follows pointer. */ + o->objcore = *ocp; + *ocp = NULL; /* refcnt follows pointer. */ + if (o->objcore->objhead != NULL) BAN_NewObjCore(o->objcore); - o->objcore->methods = &default_oc_methods; - o->objcore->priv = o; - o->objcore->priv2 = (uintptr_t)stv; - } + o->objcore->methods = &default_oc_methods; + o->objcore->priv = o; + o->objcore->priv2 = (uintptr_t)stv; return (o); } @@ -356,8 +354,11 @@ STV_NewObject(struct busyobj *bo, struct objcore **ocp, const char *hint, } } - if (o == NULL) + if (o == NULL) { + AN(*ocp); return (NULL); + } + AZ(*ocp); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(o->objstore, STORAGE_MAGIC); return (o); From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 3f56d4e Fix ttl when backend fetches are salvaged into transient storage. Message-ID: commit 3f56d4ebb4297f383cba2d05c2eca179eb171fe6 Author: Poul-Henning Kamp Date: Mon May 21 07:23:27 2012 +0000 Fix ttl when backend fetches are salvaged into transient storage. Submitted by: Martin Fixes #1140 diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 9b7a968..b350438 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -812,12 +812,12 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) * Try to salvage the transaction by allocating a * shortlived object on Transient storage. */ - req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE, - l, nhttp); if (bo->exp.ttl > cache_param->shortlived) bo->exp.ttl = cache_param->shortlived; bo->exp.grace = 0.0; bo->exp.keep = 0.0; + req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE, + l, nhttp); } bo->stats = NULL; if (req->obj == NULL) { diff --git a/bin/varnishtest/tests/r01140.vtc b/bin/varnishtest/tests/r01140.vtc new file mode 100644 index 0000000..4bd0b3b --- /dev/null +++ b/bin/varnishtest/tests/r01140.vtc @@ -0,0 +1,39 @@ +varnishtest "Transient-salvaged objects ttl should be shortened - #1140" + +server s1 { + # This response should almost completely fill the storage + rxreq + expect req.url == /url1 + txresp -bodylen 1048050 + + # The next one should not fit in the storage, ending up in transient + # with zero ttl (=shortlived) + rxreq + expect req.url == /url2 + txresp -bodylen 1024 + + # And therefore this one should be fetched next + rxreq + expect req.url == /url2 + txresp -bodylen 1025 +} -start + +varnish v1 -arg "-p expiry_sleep=0.01 -p nuke_limit=0 -p shortlived=0" \ + -storage "-smalloc,1m" -vcl+backend { } -start + +client c1 { + txreq -url /url1 + rxresp + expect resp.status == 200 + expect resp.bodylen == 1048050 + + txreq -url /url2 + rxresp + expect resp.status == 200 + expect resp.bodylen == 1024 + + txreq -url /url2 + rxresp + expect resp.status == 200 + expect resp.bodylen == 1025 +} -run From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 304683c Adopt Martins fixed for #1138, which is mostly an artifact of me being interrupted half-way through committing a bunch of stuff. Message-ID: commit 304683cc17f4bbf39fb6eb35443e40d2755867cd Author: Poul-Henning Kamp Date: Mon May 21 08:11:04 2012 +0000 Adopt Martins fixed for #1138, which is mostly an artifact of me being interrupted half-way through committing a bunch of stuff. I'm passing on the test-case for two reasons: The code is in flux and it will soon be obsolete, and second "delay 0.2" testcases are notoriously flakey. Submitted by: Martin Fixes #1138 diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index b350438..d96bd8e 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -324,7 +324,7 @@ cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req) assert(bo->state >= BOS_FAILED); if (bo->state == BOS_FAILED) { - req->obj = NULL; + HSH_Deref(&wrk->stats, NULL, &req->obj); VBO_DerefBusyObj(wrk, &req->busyobj); req->err_code = 503; sp->step = STP_ERROR; @@ -900,7 +900,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) VBO_DerefBusyObj(wrk, &req->busyobj); } else if (bo->state == BOS_FAILED) { /* handle early failures */ - req->obj = NULL; + HSH_Deref(&wrk->stats, NULL, &req->obj); VBO_DerefBusyObj(wrk, &req->busyobj); req->err_code = 503; sp->step = STP_ERROR; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 7e9a25c..bc0e52c 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -645,8 +645,9 @@ FetchBody(struct worker *wrk, void *priv) if (bo->state == BOS_FAILED) { wrk->stats.fetch_failed++; VDI_CloseFd(&bo->vbc); - obj->exp.ttl = -1.; obj->len = 0; + EXP_Clr(&obj->exp); + EXP_Rearm(obj); } else { assert(bo->state == BOS_FETCHING); From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] e0c87d4 Move the HTTP-conditional check up before we call vcl_deliver{}, there is plenty of opportunity to prevent conditional delivery in all the other vcl_*{}'s and there is a value in exposing the actual response. Message-ID: commit e0c87d41cf1fb90442d5c2cd524ab9f88cb06dc1 Author: Poul-Henning Kamp Date: Mon May 21 12:09:02 2012 +0000 Move the HTTP-conditional check up before we call vcl_deliver{}, there is plenty of opportunity to prevent conditional delivery in all the other vcl_*{}'s and there is a value in exposing the actual response. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index d96bd8e..48fdfe4 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -266,6 +266,15 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) } HTTP_Setup(req->resp, req->ws, req->vsl, HTTP_Resp); RES_BuildHttp(sp); + + if (req->obj->response == 200 + && req->http->conds && RFC2616_Do_Cond(sp)) { + req->wantbody = 0; + http_SetResp(req->resp, "HTTP/1.1", 304, "Not Modified"); + http_Unset(req->resp, H_Content_Length); + http_Unset(req->resp, H_Transfer_Encoding); + } + VCL_deliver_method(sp); switch (req->handling) { case VCL_RET_DELIVER: diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index cd45945..f92ca81 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -234,17 +234,6 @@ RES_WriteObj(struct sess *sp) req = sp->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - WRW_Reserve(sp->wrk, &sp->fd, sp->req->vsl, sp->req->t_resp); - - if (req->obj->response == 200 && - req->http->conds && - RFC2616_Do_Cond(sp)) { - req->wantbody = 0; - http_SetResp(req->resp, "HTTP/1.1", 304, "Not Modified"); - http_Unset(req->resp, H_Content_Length); - http_Unset(req->resp, H_Transfer_Encoding); - } - /* * If nothing special planned, we can attempt Range support */ @@ -265,6 +254,8 @@ RES_WriteObj(struct sess *sp) if (req->res_mode & RES_GUNZIP) http_Unset(req->resp, H_Content_Encoding); + WRW_Reserve(sp->wrk, &sp->fd, sp->req->vsl, sp->req->t_resp); + /* * Send HTTP protocol header, unless interior ESI object */ From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 44c0c6e Collect all aspect of resp.* creation together. Message-ID: commit 44c0c6edaf84a9800382e2db6cdd8c0f62643bb7 Author: Poul-Henning Kamp Date: Mon May 21 13:17:50 2012 +0000 Collect all aspect of resp.* creation together. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 48fdfe4..81b2b3b 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -267,14 +267,6 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) HTTP_Setup(req->resp, req->ws, req->vsl, HTTP_Resp); RES_BuildHttp(sp); - if (req->obj->response == 200 - && req->http->conds && RFC2616_Do_Cond(sp)) { - req->wantbody = 0; - http_SetResp(req->resp, "HTTP/1.1", 304, "Not Modified"); - http_Unset(req->resp, H_Content_Length); - http_Unset(req->resp, H_Transfer_Encoding); - } - VCL_deliver_method(sp); switch (req->handling) { case VCL_RET_DELIVER: diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index f92ca81..82cdafc 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -124,7 +124,15 @@ RES_BuildHttp(const struct sess *sp) http_SetHeader(req->resp, "Accept-Ranges: bytes"); } - if (req->res_mode & RES_CHUNKED) + if (req->res_mode & RES_GUNZIP) + http_Unset(req->resp, H_Content_Encoding); + + if (req->obj->response == 200 + && req->http->conds && RFC2616_Do_Cond(sp)) { + req->wantbody = 0; + http_SetResp(req->resp, "HTTP/1.1", 304, "Not Modified"); + http_Unset(req->resp, H_Content_Length); + } else if (req->res_mode & RES_CHUNKED) http_SetHeader(req->resp, "Transfer-Encoding: chunked"); http_Unset(req->resp, H_Date); @@ -248,12 +256,6 @@ RES_WriteObj(struct sess *sp) http_GetHdr(req->http, H_Range, &r)) res_dorange(sp, r, &low, &high); - /* - * Always remove C-E if client don't grok it - */ - if (req->res_mode & RES_GUNZIP) - http_Unset(req->resp, H_Content_Encoding); - WRW_Reserve(sp->wrk, &sp->fd, sp->req->vsl, sp->req->t_resp); /* From apj at mutt.dk Thu Dec 18 09:27:47 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 7755ce5 Duration needs a unit Message-ID: commit 7755ce56a67e03a0f8f7e6cccbaa6992c5951619 Author: Andreas Plesner Jacobsen Date: Thu May 24 14:17:52 2012 +0200 Duration needs a unit diff --git a/doc/sphinx/reference/vmod_std.rst b/doc/sphinx/reference/vmod_std.rst index a1ece16..984331e 100644 --- a/doc/sphinx/reference/vmod_std.rst +++ b/doc/sphinx/reference/vmod_std.rst @@ -124,7 +124,7 @@ Description the usual s (seconds), m (minutes), h (hours), d (days) and w (weeks) units. If *s* fails to parse, *fallback* will be used. Example - set beresp.ttl = std.duration("1w", 3600); + set beresp.ttl = std.duration("1w", 3600s); integer -------- From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] cf811a3 Give struct req a back pointer to struct sess Message-ID: commit cf811a3e0461e4c0112ad9906a12004bfb28b88b Author: Poul-Henning Kamp Date: Tue May 29 06:38:37 2012 +0000 Give struct req a back pointer to struct sess diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index ee17d31..93e05e7 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -561,6 +561,8 @@ struct req { uint8_t hash_ignore_busy; uint8_t hash_always_miss; + struct sess *sp; + /* The busy objhead we sleep on */ struct objhead *hash_objhead; struct busyobj *busyobj; diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index ba52ad3..3a85baf 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -291,6 +291,7 @@ void SES_GetReq(struct sess *sp) { struct sesspool *pp; + struct req *req; uint16_t nhttp; unsigned sz, hl; char *p, *e; @@ -301,42 +302,44 @@ SES_GetReq(struct sess *sp) AN(pp->pool); AZ(sp->req); - sp->req = MPL_Get(pp->mpl_req, &sz); - AN(sp->req); - sp->req->magic = REQ_MAGIC; - - e = (char*)sp->req + sz; - p = (char*)(sp->req + 1); + req = MPL_Get(pp->mpl_req, &sz); + AN(req); + req->magic = REQ_MAGIC; + sp->req = req; + req->sp = sp; + + e = (char*)req + sz; + p = (char*)(req + 1); p = (void*)PRNDUP(p); assert(p < e); nhttp = (uint16_t)cache_param->http_max_hdr; hl = HTTP_estimate(nhttp); - sp->req->http = HTTP_create(p, nhttp); + req->http = HTTP_create(p, nhttp); p += hl; p = (void*)PRNDUP(p); assert(p < e); - sp->req->http0 = HTTP_create(p, nhttp); + req->http0 = HTTP_create(p, nhttp); p += hl; p = (void*)PRNDUP(p); assert(p < e); - sp->req->resp = HTTP_create(p, nhttp); + req->resp = HTTP_create(p, nhttp); p += hl; p = (void*)PRNDUP(p); assert(p < e); sz = cache_param->workspace_thread; - VSL_Setup(sp->req->vsl, p, sz); - sp->req->vsl->wid = sp->vsl_id; + VSL_Setup(req->vsl, p, sz); + req->vsl->wid = sp->vsl_id; p += sz; p = (void*)PRNDUP(p); assert(p < e); - WS_Init(sp->req->ws, "req", p, e - p); + WS_Init(req->ws, "req", p, e - p); } void @@ -352,6 +355,7 @@ SES_ReleaseReq(struct sess *sp) MPL_AssertSane(sp->req); VSL_Flush(sp->req->vsl, 0); MPL_Free(pp->mpl_req, sp->req); + sp->req->sp = NULL; sp->req = NULL; } From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 42cf6a9 Downgrade the VCL_*_method() argument from sess to req Message-ID: commit 42cf6a926d5813e97079c6ff1ec4c05aecaa4ae2 Author: Poul-Henning Kamp Date: Tue May 29 07:02:35 2012 +0000 Downgrade the VCL_*_method() argument from sess to req diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 93e05e7..d2162a6 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -967,7 +967,7 @@ void VCL_Rel(struct VCL_conf **vcc); void VCL_Poll(void); const char *VCL_Return_Name(unsigned method); -#define VCL_MET_MAC(l,u,b) void VCL_##l##_method(struct sess *); +#define VCL_MET_MAC(l,u,b) void VCL_##l##_method(struct req *); #include "tbl/vcl_returns.h" #undef VCL_MET_MAC diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 81b2b3b..21f5939 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -267,7 +267,7 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) HTTP_Setup(req->resp, req->ws, req->vsl, HTTP_Resp); RES_BuildHttp(sp); - VCL_deliver_method(sp); + VCL_deliver_method(req); switch (req->handling) { case VCL_RET_DELIVER: break; @@ -522,7 +522,7 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) http_PutResponse(h, req->err_reason); else http_PutResponse(h, http_StatusMessage(req->err_code)); - VCL_error_method(sp); + VCL_error_method(req); if (req->handling == VCL_RET_RESTART && req->restarts < cache_param->max_restarts) { @@ -631,7 +631,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) AZ(bo->do_esi); AZ(bo->do_pass); - VCL_fetch_method(sp); + VCL_fetch_method(req); if (bo->do_pass) req->objcore->flags |= OC_F_PASS; @@ -997,7 +997,7 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req) assert(!(req->obj->objcore->flags & OC_F_PASS)); - VCL_hit_method(sp); + VCL_hit_method(req); if (req->handling == VCL_RET_DELIVER) { //AZ(req->busyobj->bereq->ws); @@ -1171,7 +1171,7 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req) http_SetHeader(bo->bereq, "Accept-Encoding: gzip"); } - VCL_miss_method(sp); + VCL_miss_method(req); if (req->handling == VCL_RET_FETCH) { CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); @@ -1237,7 +1237,7 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req) HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); http_FilterReq(sp, HTTPH_R_PASS); - VCL_pass_method(sp); + VCL_pass_method(req); if (req->handling == VCL_RET_ERROR) { http_Teardown(bo->bereq); @@ -1297,7 +1297,7 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req) HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); http_FilterReq(sp, 0); - VCL_pipe_method(sp); + VCL_pipe_method(req); if (req->handling == VCL_RET_ERROR) INCOMPL(); @@ -1395,7 +1395,7 @@ cnt_recv(struct sess *sp, const struct worker *wrk, struct req *req) http_CollectHdr(req->http, H_Cache_Control); - VCL_recv_method(sp); + VCL_recv_method(req); recv_handling = req->handling; if (cache_param->http_gzip_support && @@ -1411,7 +1411,7 @@ cnt_recv(struct sess *sp, const struct worker *wrk, struct req *req) req->sha256ctx = &sha256ctx; /* so HSH_AddString() can find it */ SHA256_Init(req->sha256ctx); - VCL_hash_method(sp); + VCL_hash_method(req); assert(req->handling == VCL_RET_HASH); SHA256_Final(req->digest, req->sha256ctx); req->sha256ctx = NULL; diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index b68627b..6f28507 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -330,18 +330,18 @@ ccf_config_use(struct cli *cli, const char * const *av, void *priv) #define VCL_MET_MAC(func, upper, bitmap) \ void \ -VCL_##func##_method(struct sess *sp) \ +VCL_##func##_method(struct req *req) \ { \ \ - sp->req->handling = 0; \ - sp->req->cur_method = VCL_MET_ ## upper; \ - VSLb(sp->req->vsl, SLT_VCL_call, "%s", #func); \ - (void)sp->req->vcl->func##_func(sp); \ - VSLb(sp->req->vsl, SLT_VCL_return, "%s", \ - VCL_Return_Name(sp->req->handling)); \ - sp->req->cur_method = 0; \ - assert((1U << sp->req->handling) & bitmap); \ - assert(!((1U << sp->req->handling) & ~bitmap)); \ + req->handling = 0; \ + req->cur_method = VCL_MET_ ## upper; \ + VSLb(req->vsl, SLT_VCL_call, "%s", #func); \ + (void)req->vcl->func##_func(req->sp); \ + VSLb(req->vsl, SLT_VCL_return, "%s", \ + VCL_Return_Name(req->handling)); \ + req->cur_method = 0; \ + assert((1U << req->handling) & bitmap); \ + assert(!((1U << req->handling) & ~bitmap)); \ } #include "tbl/vcl_returns.h" From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 9243e4a Stop VCL's health-polling of backend already when we discard the VCL. Message-ID: commit 9243e4a4ba009a2626d33cd3378a73d3b8f91411 Author: Poul-Henning Kamp Date: Tue May 29 07:36:53 2012 +0000 Stop VCL's health-polling of backend already when we discard the VCL. Fixes #1141 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index d2162a6..cdd85be 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -691,6 +691,8 @@ void VCA_FailSess(struct worker *w); /* cache_backend.c */ void VBE_UseHealth(const struct director *vdi); +void VBE_DiscardHealth(const struct director *vdi); + struct vbc *VDI_GetFd(const struct director *, struct sess *sp); int VDI_Healthy(const struct director *, const struct sess *sp); diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 92e74da..6f2ce1f 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -432,6 +432,25 @@ VBE_UseHealth(const struct director *vdi) * */ +void +VBE_DiscardHealth(const struct director *vdi) +{ + struct vdi_simple *vs; + + ASSERT_CLI(); + + if (strcmp(vdi->name, "simple")) + return; + CAST_OBJ_NOTNULL(vs, vdi->priv, VDI_SIMPLE_MAGIC); + if (vs->vrt->probe == NULL) + return; + VBP_Remove(vs->backend, vs->vrt->probe); +} + +/*-------------------------------------------------------------------- + * + */ + static struct vbc * __match_proto__(vdi_getfd_f) vdi_simple_getfd(const struct director *d, struct sess *sp) { @@ -470,8 +489,6 @@ vdi_simple_fini(const struct director *d) CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC); - if (vs->vrt->probe != NULL) - VBP_Remove(vs->backend, vs->vrt->probe); VBE_DropRefVcl(vs->backend); free(vs->dir.vcl_name); vs->dir.magic = 0; diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 6f28507..43ef3cd 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -278,10 +278,10 @@ static void ccf_config_discard(struct cli *cli, const char * const *av, void *priv) { struct vcls *vcl; + int i; ASSERT_CLI(); - (void)av; - (void)priv; + AZ(priv); vcl = vcl_find(av[2]); if (vcl == NULL) { VCLI_SetResult(cli, CLIS_PARAM); @@ -299,6 +299,11 @@ ccf_config_discard(struct cli *cli, const char * const *av, void *priv) VSC_C_main->n_vcl_avail--; vcl->conf->discard = 1; Lck_Unlock(&vcl_mtx); + + /* Tickle this VCL's backends to give up health polling */ + for(i = 1; i < vcl->conf->ndirector; i++) + VBE_DiscardHealth(vcl->conf->director[i]); + if (vcl->conf->busy == 0) VCL_Nuke(vcl); } From tfheen at err.no Thu Dec 18 09:27:47 2014 From: tfheen at err.no (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 2342c8b req.url, not req.http.url Message-ID: commit 2342c8b9cd3554072f003c8b61d708ee0eb651d7 Author: Tollef Fog Heen Date: Wed May 30 11:17:17 2012 +0200 req.url, not req.http.url diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst index 22056a7..422f9f4 100644 --- a/doc/sphinx/tutorial/purging.rst +++ b/doc/sphinx/tutorial/purging.rst @@ -88,7 +88,7 @@ Support for bans is built into Varnish and available in the CLI interface. To ban every png object belonging on example.com, issue the following command:: - ban req.http.host == "example.com" && req.http.url ~ "\.png$" + ban req.http.host == "example.com" && req.url ~ "\.png$" Quite powerful, really. From tfheen at varnish-software.com Thu Dec 18 09:27:47 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 6945ed5 Add missing "}". Message-ID: commit 6945ed579e532ab8274979715c436c371a8b8e32 Author: Thomas Tourlourat Date: Fri Jun 1 11:53:16 2012 +0300 Add missing "}". diff --git a/doc/sphinx/tutorial/cookies.rst b/doc/sphinx/tutorial/cookies.rst index 88e9d07..deda74c 100644 --- a/doc/sphinx/tutorial/cookies.rst +++ b/doc/sphinx/tutorial/cookies.rst @@ -62,6 +62,7 @@ cookies named COOKIE1 and COOKIE2 and you can marvel at it:: if (req.http.Cookie == "") { remove req.http.Cookie; } + } } A somewhat simpler example that can accomplish almost the same can be From tfheen at varnish-software.com Thu Dec 18 09:27:47 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] ecb8b09 Wrong description for regsuball. Message-ID: commit ecb8b09793b8ae6ca1c0ef6b272a9da581248904 Author: Thomas Tourlourat Date: Fri Jun 1 11:55:11 2012 +0300 Wrong description for regsuball. diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index ed970e2..736c7c0 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -426,7 +426,7 @@ regsub(str, regex, sub) matched string. regsuball(str, regex, sub) - As regsuball() but this replaces all occurrences. + As regsub() but this replaces all occurrences. ban(ban expression) From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 344a709 std.fileread() should not blindly return whatever file it returned last without checking if the filename changed. Message-ID: commit 344a709ccf9559f3d8e5d7a0a9a35c6e94705f0f Author: Poul-Henning Kamp Date: Mon Jun 4 09:18:31 2012 +0000 std.fileread() should not blindly return whatever file it returned last without checking if the filename changed. Fixes #1145 Testcase by: tobixen diff --git a/bin/varnishtest/tests/r01145.vtc b/bin/varnishtest/tests/r01145.vtc new file mode 100644 index 0000000..2a2502e --- /dev/null +++ b/bin/varnishtest/tests/r01145.vtc @@ -0,0 +1,43 @@ +varnishtest "Test fileread for std VMOD" + +shell { + printf "File One" > "${tmpdir}/one" + printf "File Two" > "${tmpdir}/two" + printf "File Three" > "${tmpdir}/three" +} + +server s1 { + loop 3 { + rxreq + txresp + } +} -start + + +varnish v1 -vcl+backend { + import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ; + + sub vcl_deliver { + set resp.http.foo = std.fileread("${tmpdir}" + req.url); + } +} -start + + +client c1 { + txreq -url "/one" + rxresp + expect resp.http.foo == "File One" + + txreq -url "/two" + rxresp + expect resp.http.foo == "File Two" + + txreq -url "/three" + rxresp + expect resp.http.foo == "File Three" +} -run + +client c1 -run +client c1 -run + + diff --git a/lib/libvmod_std/vmod_std_fileread.c b/lib/libvmod_std/vmod_std_fileread.c index 154d8ca..7012b66 100644 --- a/lib/libvmod_std/vmod_std_fileread.c +++ b/lib/libvmod_std/vmod_std_fileread.c @@ -85,17 +85,21 @@ free_frfile(void *ptr) const char * vmod_fileread(struct sess *sp, struct vmod_priv *priv, const char *file_name) { - struct frfile *frf; + struct frfile *frf = NULL; char *s; (void)sp; AN(priv); + if (priv->priv != NULL) { CAST_OBJ_NOTNULL(frf, priv->priv, CACHED_FILE_MAGIC); - return (frf->contents); + if (!strcmp(file_name, frf->file_name)) + return (frf->contents); } AZ(pthread_mutex_lock(&frmtx)); + if (frf != NULL) + frf->refcount--; VTAILQ_FOREACH(frf, &frlist, list) { if (!strcmp(file_name, frf->file_name)) { frf->refcount++; From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 5011af2 Add some asserts to narrow #1147 and #1148 down Message-ID: commit 5011af25e862e25cdd86c046b7a0c9ed77df115c Author: Poul-Henning Kamp Date: Mon Jun 4 09:33:06 2012 +0000 Add some asserts to narrow #1147 and #1148 down diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 21f5939..5ec3752 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -116,6 +116,7 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) SES_GetReq(sp); req = sp->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + assert(req->sp == sp); HTC_Init(req->htc, req->ws, sp->fd, sp->req->vsl, cache_param->http_req_size, cache_param->http_req_hdr_len); @@ -928,6 +929,7 @@ DOT first -> wait [style=bold,color=green] static int cnt_first(struct sess *sp, struct worker *wrk) { + struct req *req; char laddr[ADDR_BUFSIZE]; char lport[PORT_BUFSIZE]; @@ -936,8 +938,10 @@ cnt_first(struct sess *sp, struct worker *wrk) /* Allocate a request already now, so we can VSL to it */ SES_GetReq(sp); - CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC); - HTC_Init(sp->req->htc, sp->req->ws, sp->fd, sp->req->vsl, + req = sp->req; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + assert(req->sp == sp); + HTC_Init(req->htc, req->ws, sp->fd, req->vsl, cache_param->http_req_size, cache_param->http_req_hdr_len); @@ -949,11 +953,11 @@ cnt_first(struct sess *sp, struct worker *wrk) VTCP_name(&sp->mysockaddr, sp->mysockaddrlen, laddr, sizeof laddr, lport, sizeof lport); /* XXX: have no req yet */ - VSLb(sp->req->vsl, SLT_SessionOpen, "%s %s %s %s", + VSLb(req->vsl, SLT_SessionOpen, "%s %s %s %s", sp->addr, sp->port, laddr, lport); } else { /* XXX: have no req yet */ - VSLb(sp->req->vsl, SLT_SessionOpen, "%s %s %s", + VSLb(req->vsl, SLT_SessionOpen, "%s %s %s", sp->addr, sp->port, sp->mylsock->name); } diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 43ef3cd..9cdf427 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -338,6 +338,8 @@ void \ VCL_##func##_method(struct req *req) \ { \ \ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ + CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC); \ req->handling = 0; \ req->cur_method = VCL_MET_ ## upper; \ VSLb(req->vsl, SLT_VCL_call, "%s", #func); \ From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] a9e5d35 Increase the id field in VSM to 128 bytes to make space for 64 backend VCL name + IPv4, IPv6 and portnumber. Message-ID: commit a9e5d3503f8fea332c01cdf6576a980947b39b07 Author: Poul-Henning Kamp Date: Mon Jun 4 11:38:01 2012 +0000 Increase the id field in VSM to 128 bytes to make space for 64 backend VCL name + IPv4, IPv6 and portnumber. Fixes #1144 diff --git a/bin/varnishtest/tests/r01144.vtc b/bin/varnishtest/tests/r01144.vtc new file mode 100644 index 0000000..b08060d --- /dev/null +++ b/bin/varnishtest/tests/r01144.vtc @@ -0,0 +1,38 @@ +varnishtest "very long backend names" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + backend fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210 + { + .host = "127.0.0.1"; + .port = "54321"; + } + sub vcl_recv { + if (req.url == "never") { + set req.backend = fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210; + } + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run + +varnish v1 -badvcl { + backend 0fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210 + { + .host = "127.0.0.1"; + .port = "54321"; + } + sub vcl_recv { + if (req.url == "never") { + set req.backend = fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210; + } + } +} diff --git a/include/vapi/vsm_int.h b/include/vapi/vsm_int.h index 4272aef..7e44604 100644 --- a/include/vapi/vsm_int.h +++ b/include/vapi/vsm_int.h @@ -104,7 +104,7 @@ struct VSM_chunk { ssize_t next; /* Offset in shmem */ char class[8]; char type[8]; - char ident[64]; + char ident[128]; }; struct VSM_head { diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c index fb160e7..76ef874 100644 --- a/lib/libvcl/vcc_backend.c +++ b/lib/libvcl/vcc_backend.c @@ -706,6 +706,13 @@ vcc_ParseDirector(struct vcc *tl) vcc_ExpectCid(tl); /* ID: name */ ERRCHK(tl); + if (tl->t->e - tl->t->b > 64) { + VSB_printf(tl->sb, + "Name of %.*s too long (max 64, is %zd):\n", + PF(t_first), (tl->t->e - tl->t->b)); + vcc_ErrWhere(tl, tl->t); + return; + } tl->t_dir = tl->t; vcc_NextToken(tl); From tfheen at varnish-software.com Thu Dec 18 09:27:47 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] e75cd2e Typo Message-ID: commit e75cd2e61ac9383a89a2366074f58587d8c7f7e5 Author: Tollef Fog Heen Date: Mon Jun 4 14:08:38 2012 +0200 Typo diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index c8f62bb..c7834a7 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -765,7 +765,7 @@ static const struct parspec input_parspec[] = { "Maximum number of HTTP headers we will deal with in " "client request or backend reponses. " "Note that the first line occupies five header fields.\n" - "This paramter does not influence storage consumption, " + "This parameter does not influence storage consumption, " "objects allocate exact space for the headers they store.\n", 0, "64", "header lines" }, From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 6094581 Add more asserts to nail #1147 Message-ID: commit 60945816067b3928a73e8f2257e26ab8289ecb97 Author: Poul-Henning Kamp Date: Tue Jun 12 07:58:34 2012 +0000 Add more asserts to nail #1147 diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 5ec3752..e524421 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -268,7 +268,9 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) HTTP_Setup(req->resp, req->ws, req->vsl, HTTP_Resp); RES_BuildHttp(sp); + assert(req->sp == sp); VCL_deliver_method(req); + assert(req->sp == sp); switch (req->handling) { case VCL_RET_DELIVER: break; @@ -523,7 +525,9 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) http_PutResponse(h, req->err_reason); else http_PutResponse(h, http_StatusMessage(req->err_code)); + assert(req->sp == sp); VCL_error_method(req); + assert(req->sp == sp); if (req->handling == VCL_RET_RESTART && req->restarts < cache_param->max_restarts) { @@ -632,7 +636,9 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) AZ(bo->do_esi); AZ(bo->do_pass); + assert(req->sp == sp); VCL_fetch_method(req); + assert(req->sp == sp); if (bo->do_pass) req->objcore->flags |= OC_F_PASS; @@ -1001,7 +1007,9 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req) assert(!(req->obj->objcore->flags & OC_F_PASS)); + assert(req->sp == sp); VCL_hit_method(req); + assert(req->sp == sp); if (req->handling == VCL_RET_DELIVER) { //AZ(req->busyobj->bereq->ws); @@ -1175,7 +1183,9 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req) http_SetHeader(bo->bereq, "Accept-Encoding: gzip"); } + assert(req->sp == sp); VCL_miss_method(req); + assert(req->sp == sp); if (req->handling == VCL_RET_FETCH) { CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); @@ -1241,7 +1251,9 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req) HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); http_FilterReq(sp, HTTPH_R_PASS); + assert(req->sp == sp); VCL_pass_method(req); + assert(req->sp == sp); if (req->handling == VCL_RET_ERROR) { http_Teardown(bo->bereq); @@ -1301,7 +1313,9 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req) HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); http_FilterReq(sp, 0); + assert(req->sp == sp); VCL_pipe_method(req); + assert(req->sp == sp); if (req->handling == VCL_RET_ERROR) INCOMPL(); @@ -1399,7 +1413,9 @@ cnt_recv(struct sess *sp, const struct worker *wrk, struct req *req) http_CollectHdr(req->http, H_Cache_Control); + assert(req->sp == sp); VCL_recv_method(req); + assert(req->sp == sp); recv_handling = req->handling; if (cache_param->http_gzip_support && @@ -1415,7 +1431,9 @@ cnt_recv(struct sess *sp, const struct worker *wrk, struct req *req) req->sha256ctx = &sha256ctx; /* so HSH_AddString() can find it */ SHA256_Init(req->sha256ctx); + assert(req->sp == sp); VCL_hash_method(req); + assert(req->sp == sp); assert(req->handling == VCL_RET_HASH); SHA256_Final(req->digest, req->sha256ctx); req->sha256ctx = NULL; @@ -1616,6 +1634,12 @@ CNT_Session(struct sess *sp) CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); WS_Assert(wrk->aws); + if (sp->req != NULL) { + CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC); + AN(sp->req->sp); + assert(sp->req->sp == sp); + } + switch (sp->step) { #define STEP(l,u,arg) \ case STP_##u: \ diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 3a85baf..b5226d7 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -352,6 +352,8 @@ SES_ReleaseReq(struct sess *sp) CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); AN(pp->pool); CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC); + AN(sp->req->sp); + assert(sp->req->sp == sp); MPL_AssertSane(sp->req); VSL_Flush(sp->req->vsl, 0); MPL_Free(pp->mpl_req, sp->req); diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 9cdf427..527c591 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -340,6 +340,7 @@ VCL_##func##_method(struct req *req) \ \ CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC); \ + AN(req->sp); \ req->handling = 0; \ req->cur_method = VCL_MET_ ## upper; \ VSLb(req->vsl, SLT_VCL_call, "%s", #func); \ From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 94fe4da Try to create a file in our workdir early on and give a suggestion if it fails. Message-ID: commit 94fe4daba6b51a2e97eee5f9d4719a4123179d86 Author: Poul-Henning Kamp Date: Wed Jun 13 07:58:43 2012 +0000 Try to create a file in our workdir early on and give a suggestion if it fails. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 13799f9..88f8253 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -275,7 +275,7 @@ cli_stdin_close(void *priv) int main(int argc, char * const *argv) { - int o; + int o, fd; unsigned C_flag = 0; unsigned F_flag = 0; const char *b_arg = NULL; @@ -530,7 +530,7 @@ main(int argc, char * const *argv) } if (n_arg != NULL) - openlog(n_arg, LOG_PID, LOG_LOCAL0); + openlog(n_arg, LOG_PID, LOG_LOCAL0); /* XXX: i_arg ? */ else openlog("varnishd", LOG_PID, LOG_LOCAL0); @@ -546,6 +546,17 @@ main(int argc, char * const *argv) exit(1); } + fd = open("_.testfile", O_RDWR|O_CREAT|O_EXCL, 0600); + if (fd < 0) { + fprintf(stderr, "Cannot create test-file in %s (%s)\n", + dirname, strerror(errno)); + fprintf(stderr, + "Check permissions (or delete old directory)\n"); + exit(1); + } + AZ(close(fd)); + AZ(unlink("_.testfile")); + /* XXX: should this be relative to the -n arg ? */ if (P_arg && (pfh = VPF_Open(P_arg, 0644, NULL)) == NULL) { perror(P_arg); From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] b7175b3 Priv-sep vcc and cc also. Message-ID: commit b7175b38ad96ae57888e930a12cb88e33005178e Author: Poul-Henning Kamp Date: Wed Jun 13 08:03:22 2012 +0000 Priv-sep vcc and cc also. Fixes #1153 diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 9b46da3..6f4642d 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -39,6 +39,7 @@ #include #include +#include "common/params.h" #include "mgt/mgt.h" #include "libvcl.h" @@ -136,6 +137,7 @@ run_vcc(void *priv) int fd, i, l; CAST_OBJ_NOTNULL(vp, priv, VCC_PRIV_MAGIC); + mgt_sandbox(); sb = VSB_new_auto(); XXXAN(sb); VCC_VCL_dir(vcc, mgt_vcl_dir); @@ -174,6 +176,7 @@ run_vcc(void *priv) static void run_cc(void *priv) { + mgt_sandbox(); (void)execl("/bin/sh", "/bin/sh", "-c", priv, NULL); } @@ -241,8 +244,10 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag) VSB_printf(sb, "Failed to create %s: %s", sf, strerror(errno)); return (NULL); } + (void)fchown(sfd, mgt_param.uid, mgt_param.gid); AZ(close(sfd)); + /* Run the VCC compiler in a sub-process */ memset(&vp, 0, sizeof vp); vp.magic = VCC_PRIV_MAGIC; @@ -267,6 +272,16 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag) of[sizeof sf - 1] = 'o'; of[sizeof sf] = '\0'; + i = open(of, O_WRONLY|O_CREAT|O_TRUNC, 0600); + if (i < 0) { + VSB_printf(sb, "Failed to create %s: %s", + of, strerror(errno)); + (void)unlink(sf); + return (NULL); + } + (void)fchown(i, mgt_param.uid, mgt_param.gid); + AZ(close(i)); + /* Build the C-compiler command line */ cmdsb = mgt_make_cc_cmd(sf, of); @@ -284,7 +299,7 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag) i = chmod(of, 0755); if (i) VSB_printf(sb, "Failed to set permissions on %s: %s", - of, strerror(errno)); + of, strerror(errno)); } if (i) { From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 45a408b update -r description Message-ID: commit 45a408b8185b3be93a9fd53ca543d1fbbda5592f Author: Poul-Henning Kamp Date: Wed Jun 13 08:39:27 2012 +0000 update -r description diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 805e012..48ebdc8 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -110,14 +110,13 @@ OPTIONS documents. This is a shortcut for specifying the default_ttl run-time parameter. --r param[,param...] - Specifies a list of parameters that are read only. This - gives the system administrator a way to limit what someone - with access to the Varnish CLI can do. In a very secure - environment you want to consider setting parameters such - as *user*, *group*, *cc_command*, *vcc_allow_inline_c* to - read only as these can potentially be used to escalate - privileges. +-r param[,param...] + Make the listed parameters read only. This gives the + system administrator a way to limit what the Varnish CLI can do. + Consider making parameters such as *user*, *group*, *cc_command*, + *vcc_allow_inline_c* read only as these can potentially be used + to escalate privileges from the CLI. + Protecting *listen_address* may also be a good idea. -u user Specifies the name of an unprivileged user to which the child process should switch before it starts accepting From tfheen at varnish-software.com Thu Dec 18 09:27:47 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] caac740 Move disable static libraries, we do not use it Message-ID: commit caac7404d31e19cd5e2cf7154327c4115e55a4fc Author: Tollef Fog Heen Date: Wed Jun 13 13:24:38 2012 +0200 Move disable static libraries, we do not use it diff --git a/configure.ac b/configure.ac index 20134a7..8193874 100644 --- a/configure.ac +++ b/configure.ac @@ -13,6 +13,7 @@ AC_CANONICAL_SYSTEM AC_LANG(C) AM_INIT_AUTOMAKE([foreign]) +LT_INIT([disable-static]) # Checks for programs. AC_GNU_SOURCE @@ -43,7 +44,6 @@ CFLAGS="$CFLAGS $PTHREAD_CFLAGS" CC="$PTHREAD_CC" AC_PROG_INSTALL -AC_PROG_LIBTOOL AC_PROG_MAKE_SET AC_ARG_WITH([rst2man], AS_HELP_STRING([--with-rst2man=PATH], From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] dc5b18b Improve "waiter" parameter description as discussed in param-audit. Message-ID: commit dc5b18b8e095592c107b16ff911d8e05a6872efb Author: Poul-Henning Kamp Date: Wed Jun 13 12:00:33 2012 +0000 Improve "waiter" parameter description as discussed in param-audit. diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index c7834a7..8e3e56d 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1052,8 +1052,8 @@ static const struct parspec input_parspec[] = { "off", "bool" }, { "waiter", tweak_waiter, NULL, 0, 0, "Select the waiter kernel interface.\n", - EXPERIMENTAL | MUST_RESTART, - "default", NULL }, + WIZARD | MUST_RESTART, + WAITER_DEFAULT, NULL }, { "diag_bitmap", tweak_diag_bitmap, 0, 0, 0, "Bitmap controlling diagnostics code:\n" " 0x00000001 - CNT_Session states.\n" diff --git a/bin/varnishd/waiter/mgt_waiter.c b/bin/varnishd/waiter/mgt_waiter.c index d2816a8..94d90ea 100644 --- a/bin/varnishd/waiter/mgt_waiter.c +++ b/bin/varnishd/waiter/mgt_waiter.c @@ -68,14 +68,14 @@ WAIT_tweak_waiter(struct cli *cli, const char *arg) else VCLI_Out(cli, "%s", waiter->name); - VCLI_Out(cli, " ("); + VCLI_Out(cli, " (possible values: "); for (i = 0; vca_waiters[i] != NULL; i++) VCLI_Out(cli, "%s%s", i == 0 ? "" : ", ", vca_waiters[i]->name); VCLI_Out(cli, ")"); return; } - if (!strcmp(arg, "default")) { + if (!strcmp(arg, WAITER_DEFAULT)) { waiter = vca_waiters[0]; return; } diff --git a/bin/varnishd/waiter/waiter.h b/bin/varnishd/waiter/waiter.h index 9f48884..9f9f795 100644 --- a/bin/varnishd/waiter/waiter.h +++ b/bin/varnishd/waiter/waiter.h @@ -33,6 +33,8 @@ struct sess; typedef void* waiter_init_f(void); typedef void waiter_pass_f(void *priv, const struct sess *); +#define WAITER_DEFAULT "platform dependent" + struct waiter { const char *name; waiter_init_f *init; diff --git a/bin/varnishtest/tests/b00008.vtc b/bin/varnishtest/tests/b00008.vtc index cee8d4c..5d447b2 100644 --- a/bin/varnishtest/tests/b00008.vtc +++ b/bin/varnishtest/tests/b00008.vtc @@ -30,7 +30,7 @@ varnish v1 -start varnish v1 -cliok "help" -varnish v1 -cliok "param.set waiter default" +varnish v1 -cliok "param.set waiter poll" varnish v1 -clierr 106 "param.set waiter HASH(0x8839c4c)" From phk at FreeBSD.org Thu Dec 18 09:27:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:47 +0100 Subject: [experimental-ims] 65fe9ad Don't modify req after we freed it. Message-ID: commit 65fe9ad8a6459d329f4c6b562642d4711793ad9d Author: Poul-Henning Kamp Date: Wed Jun 13 12:14:41 2012 +0000 Don't modify req after we freed it. Fixes #1147 and #1148 diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index e524421..32f67e5 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -120,6 +120,8 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) HTC_Init(req->htc, req->ws, sp->fd, sp->req->vsl, cache_param->http_req_size, cache_param->http_req_hdr_len); + } else { + assert(req->sp == sp); } AZ(req->vcl); @@ -943,6 +945,7 @@ cnt_first(struct sess *sp, struct worker *wrk) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); /* Allocate a request already now, so we can VSL to it */ + AZ(sp->req); SES_GetReq(sp); req = sp->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -1493,6 +1496,7 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) AZ(req->vcl); AZ(req->esi_level); assert(!isnan(sp->t_req)); + assert(req->sp == sp); /* Update stats of various sorts */ wrk->stats.client_req++; diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index b5226d7..e3ec28a 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -356,8 +356,8 @@ SES_ReleaseReq(struct sess *sp) assert(sp->req->sp == sp); MPL_AssertSane(sp->req); VSL_Flush(sp->req->vsl, 0); - MPL_Free(pp->mpl_req, sp->req); sp->req->sp = NULL; + MPL_Free(pp->mpl_req, sp->req); sp->req = NULL; } From phk at FreeBSD.org Thu Dec 18 09:27:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] 380a5b2 Remove the gzip_window param, it is useless. Message-ID: commit 380a5b23a66673ee6c62a6dc1a26f69c34d40747 Author: Poul-Henning Kamp Date: Wed Jun 13 12:49:45 2012 +0000 Remove the gzip_window param, it is useless. diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index f7dba8e..80ce07b 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -128,7 +128,7 @@ VGZ_NewGzip(struct vsl_log *vsl, const char *id) i = deflateInit2(&vg->vz, cache_param->gzip_level, /* Level */ Z_DEFLATED, /* Method */ - 16 + cache_param->gzip_window, /* Window bits (16=gzip + 15) */ + 16 + 15, /* Window bits (16=gzip) */ cache_param->gzip_memlevel, /* memLevel */ Z_DEFAULT_STRATEGY); assert(Z_OK == i); diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index 1e13657..2e8bf37 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -185,7 +185,6 @@ struct params { unsigned http_gzip_support; unsigned gzip_buffer; unsigned gzip_level; - unsigned gzip_window; unsigned gzip_memlevel; unsigned obj_readonly; diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 8e3e56d..c425882 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1125,11 +1125,6 @@ static const struct parspec input_parspec[] = { "Gzip compression level: 0=debug, 1=fast, 9=best", 0, "6", ""}, - { "gzip_window", tweak_uint, &mgt_param.gzip_window, 8, 15, - "Gzip window size 8=least, 15=most compression.\n" - "Memory impact is 8=1k, 9=2k, ... 15=128k.", - 0, - "15", ""}, { "gzip_memlevel", tweak_uint, &mgt_param.gzip_memlevel, 1, 9, "Gzip memory level 1=slow/least, 9=fast/most compression.\n" "Memory impact is 1=1k, 2=2k, ... 9=256k.", diff --git a/bin/varnishtest/tests/e00022.vtc b/bin/varnishtest/tests/e00022.vtc index 1f13cd3..b4b3572 100644 --- a/bin/varnishtest/tests/e00022.vtc +++ b/bin/varnishtest/tests/e00022.vtc @@ -25,7 +25,6 @@ varnish v1 -arg "-p thread_pool_stack=262144" -vcl+backend { varnish v1 -cliok "param.set esi_syntax 0xc" varnish v1 -cliok "param.set http_gzip_support true" -varnish v1 -cliok "param.set gzip_window 8" varnish v1 -cliok "param.set gzip_memlevel 1" client c1 { From phk at FreeBSD.org Thu Dec 18 09:27:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] 14b7e6e Move t_req from sp to req, and give sp a t_rx for the handover from acceptor to center. Message-ID: commit 14b7e6e94ab1acbfa1768d96fa52633b163a64e9 Author: Poul-Henning Kamp Date: Thu Jun 14 08:27:57 2012 +0000 Move t_req from sp to req, and give sp a t_rx for the handover from acceptor to center. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index cdd85be..4319b70 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -590,6 +590,7 @@ struct req { uint64_t req_bodybytes; char *ws_req; /* WS above request data */ + double t_req; double t_resp; struct http_conn htc[1]; @@ -673,7 +674,7 @@ struct sess { /* Timestamps, all on TIM_real() timescale */ double t_open; /* fd accepted */ double t_idle; /* fd accepted or resp sent */ - double t_req; + double t_rx; #if defined(HAVE_EPOLL_CTL) struct epoll_event ev; diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 6f2ce1f..fb116bf 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -271,7 +271,7 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp) if (threshold == 0 || VTAILQ_EMPTY(&backend->troublelist)) return (1); - now = sp->t_req; + now = sp->req->t_req; old = NULL; retval = 1; diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 32f67e5..d063466 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -83,11 +83,6 @@ static unsigned xids; * WAIT * Collect the request from the client. * - * We "abuse" sp->t_req a bit here: On input it means "request reception - * started at xxx" and is used to trigger timeouts. On return it means - * "we had full request headers by xxx" and is used for reporting by - * later steps. - * DOT subgraph xcluster_wait { DOT wait [ DOT shape=box @@ -112,6 +107,8 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + assert(!isnan(sp->t_rx)); + if (req == NULL) { SES_GetReq(sp); req = sp->req; @@ -128,9 +125,9 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) AZ(req->obj); AZ(req->esi_level); assert(req->xid == 0); + req->t_req = sp->t_rx; req->t_resp = NAN; - assert(!isnan(sp->t_req)); tmo = (int)(1e3 * cache_param->timeout_linger); while (1) { pfd[0].fd = sp->fd; @@ -145,7 +142,7 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) i = HTC_Complete(req->htc); if (i == 1) { /* Got it, run with it */ - sp->t_req = now; + req->t_req = now; sp->step = STP_START; return (0); } else if (i == -1) { @@ -164,7 +161,7 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) when = sp->t_idle + cache_param->timeout_linger; tmo = (int)(1e3 * (when - now)); if (when < now || tmo == 0) { - sp->t_req = NAN; + sp->t_rx = NAN; wrk->stats.sess_herd++; SES_Charge(sp); SES_ReleaseReq(sp); @@ -173,7 +170,7 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) } } else { /* Working on it */ - when = sp->t_req + cache_param->timeout_req; + when = sp->t_rx + cache_param->timeout_req; tmo = (int)(1e3 * (when - now)); if (when < now || tmo == 0) { why = "req timeout"; @@ -404,9 +401,9 @@ cnt_done(struct sess *sp, struct worker *wrk, struct req *req) if (req->xid == 0) { req->t_resp = sp->t_idle; } else { - dp = req->t_resp - sp->t_req; + dp = req->t_resp - req->t_req; da = sp->t_idle - req->t_resp; - dh = sp->t_req - sp->t_open; + dh = req->t_req - sp->t_open; /* XXX: Add StatReq == StatSess */ /* XXX: Workaround for pipe */ if (sp->fd >= 0) { @@ -414,12 +411,12 @@ cnt_done(struct sess *sp, struct worker *wrk, struct req *req) (uintmax_t)req->req_bodybytes); } VSLb(sp->req->vsl, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f", - req->xid, sp->t_req, sp->t_idle, dh, dp, da); + req->xid, req->t_req, sp->t_idle, dh, dp, da); } req->xid = 0; VSL_Flush(sp->req->vsl, 0); - sp->t_req = NAN; + req->t_req = NAN; req->t_resp = NAN; req->req_bodybytes = 0; @@ -448,12 +445,14 @@ cnt_done(struct sess *sp, struct worker *wrk, struct req *req) WS_Reset(req->ws, NULL); WS_Reset(wrk->aws, NULL); - sp->t_req = sp->t_idle; i = HTC_Reinit(req->htc); if (i == 1) { + req->t_req = sp->t_idle; wrk->stats.sess_pipeline++; sp->step = STP_START; } else { + sp->t_rx = sp->t_idle; + req->t_req = NAN; if (Tlen(req->htc->rxbuf)) wrk->stats.sess_readahead++; sp->step = STP_WAIT; @@ -510,7 +509,7 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) } CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); req->obj->xid = req->xid; - req->obj->exp.entered = sp->t_req; + req->obj->exp.entered = req->t_req; h = req->obj->http; @@ -972,7 +971,7 @@ cnt_first(struct sess *sp, struct worker *wrk) wrk->acct_tmp.sess++; - sp->t_req = sp->t_open; + sp->t_rx = sp->t_open; sp->t_idle = sp->t_open; sp->step = STP_WAIT; return (0); @@ -1495,7 +1494,7 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) AZ(req->obj); AZ(req->vcl); AZ(req->esi_level); - assert(!isnan(sp->t_req)); + assert(!isnan(req->t_req)); assert(req->sp == sp); /* Update stats of various sorts */ diff --git a/bin/varnishd/cache/cache_dir_dns.c b/bin/varnishd/cache/cache_dir_dns.c index ee411dc..068c2b9 100644 --- a/bin/varnishd/cache/cache_dir_dns.c +++ b/bin/varnishd/cache/cache_dir_dns.c @@ -203,7 +203,7 @@ vdi_dns_cache_has(const struct sess *sp, struct vdi_dns_hostgroup *hostgr2; VTAILQ_FOREACH_SAFE(hostgr, &vs->cachelist, list, hostgr2) { CHECK_OBJ_NOTNULL(hostgr, VDI_DNSDIR_MAGIC); - if (hostgr->ttl <= sp->t_req) { + if (hostgr->ttl <= sp->req->t_req) { if (rwlock) vdi_dns_pop_cache(vs, hostgr); return (0); @@ -233,7 +233,7 @@ vdi_dns_cache_list_add(const struct sess *sp, } CHECK_OBJ_NOTNULL(new, VDI_DNSDIR_MAGIC); assert(new->hostname != 0); - new->ttl = sp->t_req + vs->ttl; + new->ttl = sp->req->t_req + vs->ttl; VTAILQ_INSERT_HEAD(&vs->cachelist, new, list); vs->ncachelist++; } diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index bc0e52c..7ccba0a 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -476,7 +476,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) VDI_AddHostHeader(bo->bereq, vc); (void)VTCP_blocking(vc->fd); /* XXX: we should timeout instead */ - WRW_Reserve(wrk, &vc->fd, bo->vsl, sp->t_req); /* XXX t_resp ? */ + WRW_Reserve(wrk, &vc->fd, bo->vsl, req->t_req); /* XXX t_resp ? */ (void)http_Write(wrk, hp, 0); /* XXX: stats ? */ /* Deal with any message-body the request might have */ diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index bc94352..79efc69 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -363,14 +363,14 @@ HSH_Lookup(struct sess *sp) continue; /* If still valid, use it */ - if (EXP_Ttl(req, o) >= sp->t_req) + if (EXP_Ttl(req, o) >= req->t_req) break; /* * Remember any matching objects inside their grace period * and if there are several, use the least expired one. */ - if (EXP_Grace(req, o) >= sp->t_req) { + if (EXP_Grace(req, o) >= req->t_req) { if (grace_oc == NULL || grace_ttl < o->exp.entered + o->exp.ttl) { grace_oc = oc; diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c index c486a23..fda21f4 100644 --- a/bin/varnishd/cache/cache_pipe.c +++ b/bin/varnishd/cache/cache_pipe.c @@ -80,7 +80,7 @@ PipeSession(struct sess *sp) bo->vbc = vc; /* For panic dumping */ (void)VTCP_blocking(vc->fd); - WRW_Reserve(wrk, &vc->fd, bo->vsl, sp->t_req); + WRW_Reserve(wrk, &vc->fd, bo->vsl, sp->req->t_req); sp->wrk->acct_tmp.hdrbytes += http_Write(wrk, bo->bereq, 0); diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 60bb6a9..ece413d 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -323,7 +323,7 @@ RFC2616_Do_Cond(const struct sess *sp) if (!sp->req->obj->last_modified) return (0); ims = VTIM_parse(p); - if (ims > sp->t_req) /* [RFC2616 14.25] */ + if (ims > sp->req->t_req) /* [RFC2616 14.25] */ return (0); if (sp->req->obj->last_modified > ims) return (0); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index e3ec28a..f236389 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -90,7 +90,7 @@ ses_setup(struct sess *sp) sp->sockaddr.ss_family = sp->mysockaddr.ss_family = PF_UNSPEC; sp->t_open = NAN; sp->t_idle = NAN; - sp->t_req = NAN; + sp->t_rx = NAN; } /*-------------------------------------------------------------------- @@ -208,7 +208,7 @@ SES_Handle(struct sess *sp, double now) { sp->step = STP_WAIT; - sp->t_req = now; + sp->t_rx = now; (void)SES_Schedule(sp); } diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 37fb7a0..7a88447 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -153,7 +153,7 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a) ALLOC_OBJ(new, TROUBLE_MAGIC); AN(new); memcpy(new->digest, sp->req->digest, sizeof new->digest); - new->timeout = sp->t_req + a; + new->timeout = sp->req->t_req + a; /* Insert the new item on the list before the first item with a * timeout at a later date (ie: sort by which entry will time out @@ -396,8 +396,8 @@ static void vrt_wsp_exp(const struct sess *sp, unsigned xid, const struct exp *e) { VSLb(sp->req->vsl, SLT_TTL, "%u VCL %.0f %.0f %.0f %.0f %.0f", - xid, e->ttl - (sp->t_req - e->entered), e->grace, e->keep, - sp->t_req, e->age + (sp->t_req - e->entered)); + xid, e->ttl - (sp->req->t_req - e->entered), e->grace, e->keep, + sp->req->t_req, e->age + (sp->req->t_req - e->entered)); } VRT_DO_EXP(req, sp->req->exp, ttl, 0, ) @@ -407,7 +407,8 @@ VRT_DO_EXP(req, sp->req->exp, keep, 0, ) VRT_DO_EXP(obj, sp->req->obj->exp, grace, 0, EXP_Rearm(sp->req->obj); vrt_wsp_exp(sp, sp->req->obj->xid, &sp->req->obj->exp);) -VRT_DO_EXP(obj, sp->req->obj->exp, ttl, (sp->t_req - sp->req->obj->exp.entered), +VRT_DO_EXP(obj, sp->req->obj->exp, ttl, + (sp->req->t_req - sp->req->obj->exp.entered), EXP_Rearm(sp->req->obj); vrt_wsp_exp(sp, sp->req->obj->xid, &sp->req->obj->exp);) VRT_DO_EXP(obj, sp->req->obj->exp, keep, 0, From phk at FreeBSD.org Thu Dec 18 09:27:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] 3cea894 Reduce RFC2616_Do_Cond() arg from sp to req Message-ID: commit 3cea89452d8c9b908eaf0cc4defe2a0c7c4f39fd Author: Poul-Henning Kamp Date: Thu Jun 14 08:43:06 2012 +0000 Reduce RFC2616_Do_Cond() arg from sp to req diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 4319b70..03c3edf 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -1015,7 +1015,7 @@ char *WS_Snapshot(struct ws *ws); void RFC2616_Ttl(struct busyobj *, unsigned xid); enum body_status RFC2616_Body(struct busyobj *, struct dstat *); unsigned RFC2616_Req_Gzip(const struct http *); -int RFC2616_Do_Cond(const struct sess *sp); +int RFC2616_Do_Cond(const struct req *sp); /* stevedore.c */ struct object *STV_NewObject(struct busyobj *, struct objcore **, diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index d063466..729b91b 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -880,7 +880,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) */ if (req->obj->response == 200 && req->http->conds && - RFC2616_Do_Cond(sp)) + RFC2616_Do_Cond(req)) bo->do_stream = 0; /* diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index 82cdafc..700e791 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -128,7 +128,7 @@ RES_BuildHttp(const struct sess *sp) http_Unset(req->resp, H_Content_Encoding); if (req->obj->response == 200 - && req->http->conds && RFC2616_Do_Cond(sp)) { + && req->http->conds && RFC2616_Do_Cond(req)) { req->wantbody = 0; http_SetResp(req->resp, "HTTP/1.1", 304, "Not Modified"); http_Unset(req->resp, H_Content_Length); diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index ece413d..905b744 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -310,7 +310,7 @@ RFC2616_Req_Gzip(const struct http *hp) /*--------------------------------------------------------------------*/ int -RFC2616_Do_Cond(const struct sess *sp) +RFC2616_Do_Cond(const struct req *req) { char *p, *e; double ims; @@ -319,19 +319,19 @@ RFC2616_Do_Cond(const struct sess *sp) /* RFC 2616 13.3.4 states we need to match both ETag and If-Modified-Since if present*/ - if (http_GetHdr(sp->req->http, H_If_Modified_Since, &p) ) { - if (!sp->req->obj->last_modified) + if (http_GetHdr(req->http, H_If_Modified_Since, &p) ) { + if (!req->obj->last_modified) return (0); ims = VTIM_parse(p); - if (ims > sp->req->t_req) /* [RFC2616 14.25] */ + if (ims > req->t_req) /* [RFC2616 14.25] */ return (0); - if (sp->req->obj->last_modified > ims) + if (req->obj->last_modified > ims) return (0); do_cond = 1; } - if (http_GetHdr(sp->req->http, H_If_None_Match, &p) && - http_GetHdr(sp->req->obj->http, H_ETag, &e)) { + if (http_GetHdr(req->http, H_If_None_Match, &p) && + http_GetHdr(req->obj->http, H_ETag, &e)) { if (strcmp(p,e) != 0) return (0); do_cond = 1; From phk at FreeBSD.org Thu Dec 18 09:27:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] 967af89 Pass struct req into VCL methods Message-ID: commit 967af895463e08ffa7fb69041f6872608fba4d5c Author: Poul-Henning Kamp Date: Thu Jun 14 12:25:52 2012 +0000 Pass struct req into VCL methods diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 527c591..b909a0e 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -191,7 +191,7 @@ VCL_Load(const char *fn, const char *name, struct cli *cli) REPLACE(vcl->name, name); VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name); VTAILQ_INSERT_TAIL(&vcl_head, vcl, list); - (void)vcl->conf->init_func(NULL); + (void)vcl->conf->init_func(NULL, NULL); Lck_Lock(&vcl_mtx); if (vcl_active == NULL) vcl_active = vcl; @@ -215,7 +215,7 @@ VCL_Nuke(struct vcls *vcl) assert(vcl->conf->discard); assert(vcl->conf->busy == 0); VTAILQ_REMOVE(&vcl_head, vcl, list); - (void)vcl->conf->fini_func(NULL); + (void)vcl->conf->fini_func(NULL, NULL); vcl->conf->fini_vcl(NULL); free(vcl->name); (void)dlclose(vcl->dlh); @@ -344,7 +344,7 @@ VCL_##func##_method(struct req *req) \ req->handling = 0; \ req->cur_method = VCL_MET_ ## upper; \ VSLb(req->vsl, SLT_VCL_call, "%s", #func); \ - (void)req->vcl->func##_func(req->sp); \ + (void)req->vcl->func##_func(req->sp, req); \ VSLb(req->vsl, SLT_VCL_return, "%s", \ VCL_Return_Name(req->handling)); \ req->cur_method = 0; \ diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py index 1d397c8..0f7b360 100755 --- a/lib/libvcl/generate.py +++ b/lib/libvcl/generate.py @@ -716,11 +716,12 @@ file_header(fo) fo.write(""" struct sess; +struct req; struct cli; typedef int vcl_init_f(struct cli *); typedef void vcl_fini_f(struct cli *); -typedef int vcl_func_f(struct sess *sp); +typedef int vcl_func_f(struct sess *sp, struct req *req); """) diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c index c9fe573..db8b37a 100644 --- a/lib/libvcl/vcc_compile.c +++ b/lib/libvcl/vcc_compile.c @@ -670,7 +670,7 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp) /* Emit method functions */ for (i = 0; i < VCL_MET_MAX; i++) { Fc(tl, 1, "\nstatic int\n"); - Fc(tl, 1, "VGC_function_%s (struct sess *sp)\n", + Fc(tl, 1, "VGC_function_%s(struct sess *sp, struct req *req)\n", method_tab[i].name); AZ(VSB_finish(tl->fm[i])); Fc(tl, 1, "{\n"); diff --git a/lib/libvcl/vcc_parse.c b/lib/libvcl/vcc_parse.c index 1b7ee88..1d37dd5 100644 --- a/lib/libvcl/vcc_parse.c +++ b/lib/libvcl/vcc_parse.c @@ -224,10 +224,13 @@ vcc_Function(struct vcc *tl) return; } tl->curproc = vcc_AddProc(tl, tl->t); - Fh(tl, 0, "static int VGC_function_%.*s (struct sess *sp);\n", + Fh(tl, 0, "static int VGC_function_%.*s " + "(struct sess *, struct req *);\n", PF(tl->t)); Fc(tl, 1, "\nstatic int\n"); - Fc(tl, 1, "VGC_function_%.*s (struct sess *sp)\n", PF(tl->t)); + Fc(tl, 1, "VGC_function_%.*s" + "(struct sess *sp, struct req *req)\n", + PF(tl->t)); } vcc_NextToken(tl); tl->indent += INDENT; From phk at FreeBSD.org Thu Dec 18 09:27:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] f80c6e3 Remember to pass req also when we call user defined functions Message-ID: commit f80c6e358422d0f3534340abca48cfb7f05b398a Author: Poul-Henning Kamp Date: Thu Jun 14 12:28:13 2012 +0000 Remember to pass req also when we call user defined functions diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c index b4713e6..e725c6d 100644 --- a/lib/libvcl/vcc_action.c +++ b/lib/libvcl/vcc_action.c @@ -44,7 +44,7 @@ parse_call(struct vcc *tl) ExpectErr(tl, ID); vcc_AddCall(tl, tl->t); vcc_AddRef(tl, tl->t, SYM_SUB); - Fb(tl, 1, "if (VGC_function_%.*s(sp))\n", PF(tl->t)); + Fb(tl, 1, "if (VGC_function_%.*s(sp, req))\n", PF(tl->t)); Fb(tl, 1, "\treturn (1);\n"); vcc_NextToken(tl); return; From phk at FreeBSD.org Thu Dec 18 09:27:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] 7d90d62 Move VRT_count() from sess to req Message-ID: commit 7d90d625b2191cab73c1393230582c5b8a4b9b94 Author: Poul-Henning Kamp Date: Thu Jun 14 12:33:10 2012 +0000 Move VRT_count() from sess to req diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 10658cd..a7d68ad 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -70,15 +70,15 @@ VRT_error(const struct sess *sp, unsigned code, const char *reason) /*--------------------------------------------------------------------*/ void -VRT_count(const struct sess *sp, unsigned u) +VRT_count(struct req *req, unsigned u) { - if (sp == NULL) + if (req == NULL) return; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (cache_param->vcl_trace) - VSLb(sp->req->vsl, SLT_VCL_trace, "%u %u.%u", u, - sp->req->vcl->ref[u].line, sp->req->vcl->ref[u].pos); + VSLb(req->vsl, SLT_VCL_trace, "%u %u.%u", u, + req->vcl->ref[u].line, req->vcl->ref[u].pos); } /*--------------------------------------------------------------------*/ diff --git a/include/vrt.h b/include/vrt.h index bbf16ac..0b122ec 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -32,6 +32,7 @@ */ struct sess; +struct req; struct vsb; struct cli; struct director; @@ -154,7 +155,7 @@ void VRT_ban(struct sess *sp, char *, ...); void VRT_ban_string(struct sess *sp, const char *); void VRT_purge(const struct sess *sp, double ttl, double grace); -void VRT_count(const struct sess *, unsigned); +void VRT_count(struct req *, unsigned); int VRT_rewrite(const char *, const char *); void VRT_error(const struct sess *, unsigned, const char *); int VRT_switch_config(const char *); diff --git a/lib/libvcl/vcc_parse.c b/lib/libvcl/vcc_parse.c index 1d37dd5..369aca8 100644 --- a/lib/libvcl/vcc_parse.c +++ b/lib/libvcl/vcc_parse.c @@ -47,7 +47,7 @@ static void vcc_Compound(struct vcc *tl); } while (0) #define C(tl, sep) do { \ - Fb(tl, 1, "VRT_count(sp, %u)%s\n", ++tl->cnt, sep); \ + Fb(tl, 1, "VRT_count(req, %u)%s\n", ++tl->cnt, sep); \ tl->t->cnt = tl->cnt; \ } while (0) From phk at FreeBSD.org Thu Dec 18 09:27:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] 931ef7d Emit __match_proto__() for compiled functions Message-ID: commit 931ef7d8c4a3c24574c5411ef69c1fb76efb3ae6 Author: Poul-Henning Kamp Date: Thu Jun 14 12:49:04 2012 +0000 Emit __match_proto__() for compiled functions diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c index db8b37a..a16fbed 100644 --- a/lib/libvcl/vcc_compile.c +++ b/lib/libvcl/vcc_compile.c @@ -609,6 +609,8 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp) /* Macro for accessing directors */ Fh(tl, 0, "#define VGCDIR(n) VCL_conf.director[VGC_backend_##n]\n"); + Fh(tl, 0, "#define __match_proto__(xxx) /*lint -e{818} */\n"); + /* Register and lex the main source */ VTAILQ_INSERT_TAIL(&tl->sources, sp, list); sp->idx = tl->nsources++; @@ -669,7 +671,7 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp) /* Emit method functions */ for (i = 0; i < VCL_MET_MAX; i++) { - Fc(tl, 1, "\nstatic int\n"); + Fc(tl, 1, "\nstatic int __match_proto__(vcl_func_f)\n"); Fc(tl, 1, "VGC_function_%s(struct sess *sp, struct req *req)\n", method_tab[i].name); AZ(VSB_finish(tl->fm[i])); diff --git a/lib/libvcl/vcc_parse.c b/lib/libvcl/vcc_parse.c index 369aca8..1a6bbf3 100644 --- a/lib/libvcl/vcc_parse.c +++ b/lib/libvcl/vcc_parse.c @@ -227,7 +227,7 @@ vcc_Function(struct vcc *tl) Fh(tl, 0, "static int VGC_function_%.*s " "(struct sess *, struct req *);\n", PF(tl->t)); - Fc(tl, 1, "\nstatic int\n"); + Fc(tl, 1, "\nstatic int __match_proto__(vcl_func_t)\n"); Fc(tl, 1, "VGC_function_%.*s" "(struct sess *sp, struct req *req)\n", PF(tl->t)); From phk at FreeBSD.org Thu Dec 18 09:27:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] 30ecc41 Downgrade all the VRT-var functions from sp to req Message-ID: commit 30ecc418275ca0aacfd47b91a2b4c8eef5960df8 Author: Poul-Henning Kamp Date: Thu Jun 14 14:26:52 2012 +0000 Downgrade all the VRT-var functions from sp to req diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index a7d68ad..524a9b2 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -277,10 +277,10 @@ VRT_hashdata(const struct sess *sp, const char *str, ...) /*--------------------------------------------------------------------*/ double -VRT_r_now(const struct sess *sp) +VRT_r_now(const struct req *req) { - (void)sp; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); return (VTIM_real()); } diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 7a88447..8d325ca 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -63,59 +63,60 @@ vrt_do_string(const struct http *hp, int fld, #define VRT_DO_HDR(obj, hdr, http, fld) \ void \ -VRT_l_##obj##_##hdr(const struct sess *sp, const char *p, ...) \ +VRT_l_##obj##_##hdr(const struct req *req, const char *p, ...) \ { \ va_list ap; \ \ - (void)sp; \ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ va_start(ap, p); \ vrt_do_string(http, fld, #obj "." #hdr, p, ap); \ va_end(ap); \ } \ \ const char * \ -VRT_r_##obj##_##hdr(const struct sess *sp) \ +VRT_r_##obj##_##hdr(const struct req *req) \ { \ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ CHECK_OBJ_NOTNULL(http, HTTP_MAGIC); \ return (http->hd[fld].b); \ } -VRT_DO_HDR(req, request, sp->req->http, HTTP_HDR_REQ) -VRT_DO_HDR(req, url, sp->req->http, HTTP_HDR_URL) -VRT_DO_HDR(req, proto, sp->req->http, HTTP_HDR_PROTO) -VRT_DO_HDR(bereq, request, sp->req->busyobj->bereq, HTTP_HDR_REQ) -VRT_DO_HDR(bereq, url, sp->req->busyobj->bereq, HTTP_HDR_URL) -VRT_DO_HDR(bereq, proto, sp->req->busyobj->bereq, HTTP_HDR_PROTO) -VRT_DO_HDR(obj, proto, sp->req->obj->http, HTTP_HDR_PROTO) -VRT_DO_HDR(obj, response, sp->req->obj->http, HTTP_HDR_RESPONSE) -VRT_DO_HDR(resp, proto, sp->req->resp, HTTP_HDR_PROTO) -VRT_DO_HDR(resp, response, sp->req->resp, HTTP_HDR_RESPONSE) -VRT_DO_HDR(beresp, proto, sp->req->busyobj->beresp, HTTP_HDR_PROTO) -VRT_DO_HDR(beresp, response, sp->req->busyobj->beresp, HTTP_HDR_RESPONSE) +VRT_DO_HDR(req, request, req->http, HTTP_HDR_REQ) +VRT_DO_HDR(req, url, req->http, HTTP_HDR_URL) +VRT_DO_HDR(req, proto, req->http, HTTP_HDR_PROTO) +VRT_DO_HDR(bereq, request, req->busyobj->bereq, HTTP_HDR_REQ) +VRT_DO_HDR(bereq, url, req->busyobj->bereq, HTTP_HDR_URL) +VRT_DO_HDR(bereq, proto, req->busyobj->bereq, HTTP_HDR_PROTO) +VRT_DO_HDR(obj, proto, req->obj->http, HTTP_HDR_PROTO) +VRT_DO_HDR(obj, response, req->obj->http, HTTP_HDR_RESPONSE) +VRT_DO_HDR(resp, proto, req->resp, HTTP_HDR_PROTO) +VRT_DO_HDR(resp, response, req->resp, HTTP_HDR_RESPONSE) +VRT_DO_HDR(beresp, proto, req->busyobj->beresp, HTTP_HDR_PROTO) +VRT_DO_HDR(beresp, response, req->busyobj->beresp, HTTP_HDR_RESPONSE) /*--------------------------------------------------------------------*/ #define VRT_DO_STATUS(obj, http) \ void \ -VRT_l_##obj##_status(const struct sess *sp, int num) \ +VRT_l_##obj##_status(const struct req *req, int num) \ { \ \ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ assert(num >= 100 && num <= 999); \ http->status = (uint16_t)num; \ } \ \ int \ -VRT_r_##obj##_status(const struct sess *sp) \ +VRT_r_##obj##_status(const struct req *req) \ { \ \ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ return(http->status); \ } -VRT_DO_STATUS(obj, sp->req->obj->http) -VRT_DO_STATUS(beresp, sp->req->busyobj->beresp) -VRT_DO_STATUS(resp, sp->req->resp) +VRT_DO_STATUS(obj, req->obj->http) +VRT_DO_STATUS(beresp, req->busyobj->beresp) +VRT_DO_STATUS(resp, req->resp) /*--------------------------------------------------------------------*/ @@ -125,25 +126,25 @@ VRT_DO_STATUS(resp, sp->req->resp) * is no object. */ void -VRT_l_beresp_saintmode(const struct sess *sp, double a) +VRT_l_beresp_saintmode(const struct req *req, double a) { struct trouble *new; struct trouble *tr; struct trouble *tr2; struct vbc *vbc; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->req->busyobj, BUSYOBJ_MAGIC); - vbc = sp->req->busyobj->vbc; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->busyobj, BUSYOBJ_MAGIC); + vbc = req->busyobj->vbc; if (!vbc) return; CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC); if (!vbc->backend) return; CHECK_OBJ_NOTNULL(vbc->backend, BACKEND_MAGIC); - if (!sp->req->objcore->objhead) + if (!req->objcore->objhead) return; - CHECK_OBJ_NOTNULL(sp->req->objcore, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); /* Setting a negative holdoff period is a mistake. Detecting this * when compiling the VCL would be better. @@ -152,8 +153,8 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a) ALLOC_OBJ(new, TROUBLE_MAGIC); AN(new); - memcpy(new->digest, sp->req->digest, sizeof new->digest); - new->timeout = sp->req->t_req + a; + memcpy(new->digest, req->digest, sizeof new->digest); + new->timeout = req->t_req + a; /* Insert the new item on the list before the first item with a * timeout at a later date (ie: sort by which entry will time out @@ -181,17 +182,17 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a) #define VBERESP(dir, type, onm, field) \ void \ -VRT_l_##dir##_##onm(const struct sess *sp, type a) \ +VRT_l_##dir##_##onm(const struct req *req, type a) \ { \ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - sp->req->field = a; \ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ + req->field = a; \ } \ \ type \ -VRT_r_##dir##_##onm(const struct sess *sp) \ +VRT_r_##dir##_##onm(const struct req *req) \ { \ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - return (sp->req->field); \ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ + return (req->field); \ } VBERESP(beresp, unsigned, do_esi, busyobj->do_esi) @@ -203,46 +204,47 @@ VBERESP(beresp, unsigned, do_pass, busyobj->do_pass) /*--------------------------------------------------------------------*/ const char * -VRT_r_client_identity(const struct sess *sp) +VRT_r_client_identity(struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - if (sp->req->client_identity != NULL) - return (sp->req->client_identity); + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + if (req->client_identity != NULL) + return (req->client_identity); else - return (sp->addr); + return (req->sp->addr); } void -VRT_l_client_identity(const struct sess *sp, const char *str, ...) +VRT_l_client_identity(struct req *req, const char *str, ...) { va_list ap; char *b; va_start(ap, str); - b = VRT_String(sp->req->http->ws, NULL, str, ap); + b = VRT_String(req->http->ws, NULL, str, ap); va_end(ap); - sp->req->client_identity = b; + req->client_identity = b; } /*--------------------------------------------------------------------*/ #define BEREQ_TIMEOUT(which) \ void __match_proto__() \ -VRT_l_bereq_##which(struct sess *sp, double num) \ +VRT_l_bereq_##which(struct req *req, double num) \ { \ \ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - CHECK_OBJ_NOTNULL(sp->req->busyobj, BUSYOBJ_MAGIC); \ - sp->req->busyobj->which = (num > 0.0 ? num : 0.0); \ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ + CHECK_OBJ_NOTNULL(req->busyobj, BUSYOBJ_MAGIC); \ + req->busyobj->which = (num > 0.0 ? num : 0.0); \ } \ \ double __match_proto__() \ -VRT_r_bereq_##which(struct sess *sp) \ +VRT_r_bereq_##which(struct req *req) \ { \ \ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - CHECK_OBJ_NOTNULL(sp->req->busyobj, BUSYOBJ_MAGIC); \ - return(sp->req->busyobj->which); \ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ + CHECK_OBJ_NOTNULL(req->busyobj, BUSYOBJ_MAGIC); \ + return(req->busyobj->which); \ } BEREQ_TIMEOUT(connect_timeout) @@ -252,118 +254,121 @@ BEREQ_TIMEOUT(between_bytes_timeout) /*--------------------------------------------------------------------*/ const char * -VRT_r_beresp_backend_name(const struct sess *sp) +VRT_r_beresp_backend_name(const struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->req->busyobj->vbc, VBC_MAGIC); - return(sp->req->busyobj->vbc->backend->vcl_name); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->busyobj->vbc, VBC_MAGIC); + return(req->busyobj->vbc->backend->vcl_name); } struct sockaddr_storage * -VRT_r_beresp_backend_ip(const struct sess *sp) +VRT_r_beresp_backend_ip(const struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->req->busyobj->vbc, VBC_MAGIC); - return(sp->req->busyobj->vbc->addr); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->busyobj->vbc, VBC_MAGIC); + return(req->busyobj->vbc->addr); } int -VRT_r_beresp_backend_port(const struct sess *sp) +VRT_r_beresp_backend_port(const struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->req->busyobj->vbc, VBC_MAGIC); - return (VTCP_port(sp->req->busyobj->vbc->addr)); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->busyobj->vbc, VBC_MAGIC); + return (VTCP_port(req->busyobj->vbc->addr)); } const char * __match_proto__() -VRT_r_beresp_storage(struct sess *sp) +VRT_r_beresp_storage(struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - if (sp->req->storage_hint != NULL) - return (sp->req->storage_hint); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + if (req->storage_hint != NULL) + return (req->storage_hint); else return (NULL); } void __match_proto__() -VRT_l_beresp_storage(struct sess *sp, const char *str, ...) +VRT_l_beresp_storage(struct req *req, const char *str, ...) { va_list ap; char *b; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); va_start(ap, str); - b = VRT_String(sp->req->busyobj->ws, NULL, str, ap); + b = VRT_String(req->busyobj->ws, NULL, str, ap); va_end(ap); - sp->req->storage_hint = b; + req->storage_hint = b; } /*--------------------------------------------------------------------*/ void -VRT_l_req_backend(const struct sess *sp, struct director *be) +VRT_l_req_backend(struct req *req, struct director *be) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - sp->req->director = be; + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + req->director = be; } struct director * -VRT_r_req_backend(const struct sess *sp) +VRT_r_req_backend(struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - return (sp->req->director); + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + return (req->director); } /*--------------------------------------------------------------------*/ void -VRT_l_req_esi(const struct sess *sp, unsigned process_esi) +VRT_l_req_esi(struct req *req, unsigned process_esi) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); /* * Only allow you to turn of esi in the main request * else everything gets confused */ - if(sp->req->esi_level == 0) - sp->req->disable_esi = !process_esi; + if(req->esi_level == 0) + req->disable_esi = !process_esi; } unsigned -VRT_r_req_esi(const struct sess *sp) +VRT_r_req_esi(struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - return (!sp->req->disable_esi); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + return (!req->disable_esi); } int -VRT_r_req_esi_level(const struct sess *sp) +VRT_r_req_esi_level(const struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - return(sp->req->esi_level); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + return(req->esi_level); } /*--------------------------------------------------------------------*/ unsigned __match_proto__() -VRT_r_req_can_gzip(struct sess *sp) +VRT_r_req_can_gzip(struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - return (RFC2616_Req_Gzip(sp->req->http)); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + return (RFC2616_Req_Gzip(req->http)); } /*--------------------------------------------------------------------*/ int -VRT_r_req_restarts(const struct sess *sp) +VRT_r_req_restarts(const struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - return (sp->req->restarts); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + return (req->restarts); } /*-------------------------------------------------------------------- @@ -374,10 +379,10 @@ VRT_r_req_restarts(const struct sess *sp) #define VRT_DO_EXP(which, exp, fld, offset, extra) \ \ void __match_proto__() \ -VRT_l_##which##_##fld(struct sess *sp, double a) \ +VRT_l_##which##_##fld(struct req *req, double a) \ { \ \ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ if (a > 0.) \ a += offset; \ EXP_Set_##fld(&exp, a); \ @@ -385,57 +390,58 @@ VRT_l_##which##_##fld(struct sess *sp, double a) \ } \ \ double __match_proto__() \ -VRT_r_##which##_##fld(struct sess *sp) \ +VRT_r_##which##_##fld(struct req *req) \ { \ \ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ return(EXP_Get_##fld(&exp) - offset); \ } static void -vrt_wsp_exp(const struct sess *sp, unsigned xid, const struct exp *e) +vrt_wsp_exp(struct req *req, unsigned xid, const struct exp *e) { - VSLb(sp->req->vsl, SLT_TTL, "%u VCL %.0f %.0f %.0f %.0f %.0f", - xid, e->ttl - (sp->req->t_req - e->entered), e->grace, e->keep, - sp->req->t_req, e->age + (sp->req->t_req - e->entered)); -} - -VRT_DO_EXP(req, sp->req->exp, ttl, 0, ) -VRT_DO_EXP(req, sp->req->exp, grace, 0, ) -VRT_DO_EXP(req, sp->req->exp, keep, 0, ) - -VRT_DO_EXP(obj, sp->req->obj->exp, grace, 0, - EXP_Rearm(sp->req->obj); - vrt_wsp_exp(sp, sp->req->obj->xid, &sp->req->obj->exp);) -VRT_DO_EXP(obj, sp->req->obj->exp, ttl, - (sp->req->t_req - sp->req->obj->exp.entered), - EXP_Rearm(sp->req->obj); - vrt_wsp_exp(sp, sp->req->obj->xid, &sp->req->obj->exp);) -VRT_DO_EXP(obj, sp->req->obj->exp, keep, 0, - EXP_Rearm(sp->req->obj); - vrt_wsp_exp(sp, sp->req->obj->xid, &sp->req->obj->exp);) - -VRT_DO_EXP(beresp, sp->req->busyobj->exp, grace, 0, - vrt_wsp_exp(sp, sp->req->xid, &sp->req->busyobj->exp);) -VRT_DO_EXP(beresp, sp->req->busyobj->exp, ttl, 0, - vrt_wsp_exp(sp, sp->req->xid, &sp->req->busyobj->exp);) -VRT_DO_EXP(beresp, sp->req->busyobj->exp, keep, 0, - vrt_wsp_exp(sp, sp->req->xid, &sp->req->busyobj->exp);) + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + VSLb(req->vsl, SLT_TTL, "%u VCL %.0f %.0f %.0f %.0f %.0f", + xid, e->ttl - (req->t_req - e->entered), e->grace, e->keep, + req->t_req, e->age + (req->t_req - e->entered)); +} + +VRT_DO_EXP(req, req->exp, ttl, 0, ) +VRT_DO_EXP(req, req->exp, grace, 0, ) +VRT_DO_EXP(req, req->exp, keep, 0, ) + +VRT_DO_EXP(obj, req->obj->exp, grace, 0, + EXP_Rearm(req->obj); + vrt_wsp_exp(req, req->obj->xid, &req->obj->exp);) +VRT_DO_EXP(obj, req->obj->exp, ttl, + (req->t_req - req->obj->exp.entered), + EXP_Rearm(req->obj); + vrt_wsp_exp(req, req->obj->xid, &req->obj->exp);) +VRT_DO_EXP(obj, req->obj->exp, keep, 0, + EXP_Rearm(req->obj); + vrt_wsp_exp(req, req->obj->xid, &req->obj->exp);) + +VRT_DO_EXP(beresp, req->busyobj->exp, grace, 0, + vrt_wsp_exp(req, req->xid, &req->busyobj->exp);) +VRT_DO_EXP(beresp, req->busyobj->exp, ttl, 0, + vrt_wsp_exp(req, req->xid, &req->busyobj->exp);) +VRT_DO_EXP(beresp, req->busyobj->exp, keep, 0, + vrt_wsp_exp(req, req->xid, &req->busyobj->exp);) /*-------------------------------------------------------------------- * req.xid */ const char * __match_proto__() -VRT_r_req_xid(struct sess *sp) +VRT_r_req_xid(struct req *req) { char *p; int size; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - size = snprintf(NULL, 0, "%u", sp->req->xid) + 1; - AN(p = WS_Alloc(sp->req->http->ws, size)); - assert(snprintf(p, size, "%u", sp->req->xid) < size); + size = snprintf(NULL, 0, "%u", req->xid) + 1; + AN(p = WS_Alloc(req->http->ws, size)); + assert(snprintf(p, size, "%u", req->xid) < size); return (p); } @@ -443,19 +449,19 @@ VRT_r_req_xid(struct sess *sp) #define REQ_BOOL(hash_var) \ void __match_proto__() \ -VRT_l_req_##hash_var(struct sess *sp, unsigned val) \ +VRT_l_req_##hash_var(struct req *req, unsigned val) \ { \ \ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - sp->req->hash_var = val ? 1 : 0; \ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ + req->hash_var = val ? 1 : 0; \ } \ \ unsigned __match_proto__() \ -VRT_r_req_##hash_var(struct sess *sp) \ +VRT_r_req_##hash_var(struct req *req) \ { \ \ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ - return(sp->req->hash_var); \ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ + return(req->hash_var); \ } REQ_BOOL(hash_ignore_busy) @@ -464,46 +470,46 @@ REQ_BOOL(hash_always_miss) /*--------------------------------------------------------------------*/ struct sockaddr_storage * -VRT_r_client_ip(struct sess *sp) +VRT_r_client_ip(struct req *req) { - return (&sp->sockaddr); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + return (&req->sp->sockaddr); } struct sockaddr_storage * -VRT_r_server_ip(struct sess *sp) +VRT_r_server_ip(struct req *req) { int i; - if (sp->mysockaddr.ss_family == AF_UNSPEC) { - i = getsockname(sp->fd, - (void*)&sp->mysockaddr, &sp->mysockaddrlen); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + if (req->sp->mysockaddr.ss_family == AF_UNSPEC) { + i = getsockname(req->sp->fd, + (void*)&req->sp->mysockaddr, &req->sp->mysockaddrlen); assert(VTCP_Check(i)); } - return (&sp->mysockaddr); + return (&req->sp->mysockaddr); } const char* -VRT_r_server_identity(struct sess *sp) +VRT_r_server_identity(struct req *req) { - (void)sp; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (heritage.identity[0] != '\0') return (heritage.identity); else return (heritage.name); } - const char* -VRT_r_server_hostname(struct sess *sp) +VRT_r_server_hostname(struct req *req) { - (void)sp; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (vrt_hostname[0] == '\0') AZ(gethostname(vrt_hostname, sizeof(vrt_hostname))); - return (vrt_hostname); } @@ -512,43 +518,44 @@ VRT_r_server_hostname(struct sess *sp) */ int -VRT_r_server_port(struct sess *sp) +VRT_r_server_port(struct req *req) { int i; - if (sp->mysockaddr.ss_family == AF_UNSPEC) { - i = getsockname(sp->fd, - (void*)&sp->mysockaddr, &sp->mysockaddrlen); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + if (req->sp->mysockaddr.ss_family == AF_UNSPEC) { + i = getsockname(req->sp->fd, + (void*)&req->sp->mysockaddr, &req->sp->mysockaddrlen); assert(VTCP_Check(i)); } - return (VTCP_port(&sp->mysockaddr)); + return (VTCP_port(&req->sp->mysockaddr)); } /*--------------------------------------------------------------------*/ int -VRT_r_obj_hits(const struct sess *sp) +VRT_r_obj_hits(const struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->req->obj, OBJECT_MAGIC); /* XXX */ - return (sp->req->obj->hits); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); /* XXX */ + return (req->obj->hits); } double -VRT_r_obj_lastuse(const struct sess *sp) +VRT_r_obj_lastuse(const struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->req->obj, OBJECT_MAGIC); /* XXX */ - return (VTIM_real() - sp->req->obj->last_use); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); /* XXX */ + return (VTIM_real() - req->obj->last_use); } unsigned -VRT_r_req_backend_healthy(const struct sess *sp) +VRT_r_req_backend_healthy(struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->req->director, DIRECTOR_MAGIC); - return (VDI_Healthy(sp->req->director, sp)); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->director, DIRECTOR_MAGIC); + return (VDI_Healthy(req->director, req->sp)); } diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py index 0f7b360..0c1c02e 100755 --- a/lib/libvcl/generate.py +++ b/lib/libvcl/generate.py @@ -107,361 +107,361 @@ sp_variables = ( 'IP', ( 'proc',), ( ), - 'struct sess *' + 'struct req *' ), ('client.identity', 'STRING', ( 'proc',), ( 'proc',), - 'const struct sess *' + 'struct req *' ), ('server.ip', 'IP', ( 'proc',), ( ), - 'struct sess *' + 'struct req *' ), ('server.hostname', 'STRING', ( 'proc',), ( ), - 'struct sess *' + 'struct req *' ), ('server.identity', 'STRING', ( 'proc',), ( ), - 'struct sess *' + 'struct req *' ), ('server.port', 'INT', ( 'proc',), ( ), - 'struct sess *' + 'struct req *' ), ('req.request', 'STRING', ( 'proc',), ( 'proc',), - 'const struct sess *' + 'const struct req *' ), ('req.url', 'STRING', ( 'proc',), ( 'proc',), - 'const struct sess *' + 'const struct req *' ), ('req.proto', 'STRING', ( 'proc',), ( 'proc',), - 'const struct sess *' + 'const struct req *' ), ('req.http.', 'HDR_REQ', ( 'proc',), ( 'proc',), - 'const struct sess *' + 'const struct req *' ), ('req.backend', 'BACKEND', ( 'proc',), ( 'proc',), - 'const struct sess *' + 'struct req *' ), ('req.restarts', 'INT', ( 'proc',), ( ), - 'const struct sess *' + 'const struct req *' ), ('req.esi_level', 'INT', ( 'proc',), ( ), - 'const struct sess *' + 'const struct req *' ), ('req.ttl', 'DURATION', ( 'proc',), ( 'proc',), - 'struct sess *' + 'struct req *' ), ('req.grace', 'DURATION', ( 'proc',), ( 'proc',), - 'struct sess *' + 'struct req *' ), ('req.keep', 'DURATION', ( 'proc',), ( 'proc',), - 'struct sess *' + 'struct req *' ), ('req.xid', 'STRING', ( 'proc',), ( ), - 'struct sess *' + 'struct req *' ), ('req.esi', 'BOOL', ( 'recv', 'fetch', 'deliver', 'error',), ( 'recv', 'fetch', 'deliver', 'error',), - 'const struct sess *' + 'struct req *' ), ('req.can_gzip', 'BOOL', ( 'proc',), ( ), - 'struct sess *' + 'struct req *' ), ('req.backend.healthy', 'BOOL', ( 'proc',), ( ), - 'const struct sess *' + 'struct req *' ), ('req.hash_ignore_busy', 'BOOL', ( 'recv',), ( 'recv',), - 'struct sess *' + 'struct req *' ), ('req.hash_always_miss', 'BOOL', ( 'recv',), ( 'recv',), - 'struct sess *' + 'struct req *' ), ('bereq.request', 'STRING', ( 'pipe', 'pass', 'miss', 'fetch',), ( 'pipe', 'pass', 'miss', 'fetch',), - 'const struct sess *' + 'const struct req *' ), ('bereq.url', 'STRING', ( 'pipe', 'pass', 'miss', 'fetch',), ( 'pipe', 'pass', 'miss', 'fetch',), - 'const struct sess *' + 'const struct req *' ), ('bereq.proto', 'STRING', ( 'pipe', 'pass', 'miss', 'fetch',), ( 'pipe', 'pass', 'miss', 'fetch',), - 'const struct sess *' + 'const struct req *' ), ('bereq.http.', 'HDR_BEREQ', ( 'pipe', 'pass', 'miss', 'fetch',), ( 'pipe', 'pass', 'miss', 'fetch',), - 'const struct sess *' + 'const struct req *' ), ('bereq.connect_timeout', 'DURATION', ( 'pipe', 'pass', 'miss',), ( 'pipe', 'pass', 'miss',), - 'struct sess *' + 'struct req *' ), ('bereq.first_byte_timeout', 'DURATION', ( 'pass', 'miss',), ( 'pass', 'miss',), - 'struct sess *' + 'struct req *' ), ('bereq.between_bytes_timeout', 'DURATION', ( 'pass', 'miss',), ( 'pass', 'miss',), - 'struct sess *' + 'struct req *' ), ('beresp.proto', 'STRING', ( 'fetch',), ( 'fetch',), - 'const struct sess *' + 'const struct req *' ), ('beresp.saintmode', 'DURATION', ( ), ( 'fetch',), - 'const struct sess *' + 'const struct req *' ), ('beresp.status', 'INT', ( 'fetch',), ( 'fetch',), - 'const struct sess *' + 'const struct req *' ), ('beresp.response', 'STRING', ( 'fetch',), ( 'fetch',), - 'const struct sess *' + 'const struct req *' ), ('beresp.http.', 'HDR_BERESP', ( 'fetch',), ( 'fetch',), - 'const struct sess *' + 'const struct req *' ), ('beresp.do_esi', 'BOOL', ( 'fetch',), ( 'fetch',), - 'const struct sess *' + 'const struct req *' ), ('beresp.do_stream', 'BOOL', ( 'fetch',), ( 'fetch',), - 'const struct sess *' + 'const struct req *' ), ('beresp.do_gzip', 'BOOL', ( 'fetch',), ( 'fetch',), - 'const struct sess *' + 'const struct req *' ), ('beresp.do_gunzip', 'BOOL', ( 'fetch',), ( 'fetch',), - 'const struct sess *' + 'const struct req *' ), ('beresp.do_pass', 'BOOL', ( 'fetch',), ( 'fetch',), - 'const struct sess *' + 'const struct req *' ), ('beresp.ttl', 'DURATION', ( 'fetch',), ( 'fetch',), - 'struct sess *' + 'struct req *' ), ('beresp.grace', 'DURATION', ( 'fetch',), ( 'fetch',), - 'struct sess *' + 'struct req *' ), ('beresp.keep', 'DURATION', ( 'fetch',), ( 'fetch',), - 'struct sess *' + 'struct req *' ), ('beresp.backend.name', 'STRING', ( 'fetch',), ( ), - 'const struct sess *' + 'const struct req *' ), ('beresp.backend.ip', 'IP', ( 'fetch',), ( ), - 'const struct sess *' + 'const struct req *' ), ('beresp.backend.port', 'INT', ( 'fetch',), ( ), - 'const struct sess *' + 'const struct req *' ), ('beresp.storage', 'STRING', ( 'fetch',), ( 'fetch',), - 'struct sess *' + 'struct req *' ), ('obj.proto', 'STRING', ( 'hit', 'error',), ( 'hit', 'error',), - 'const struct sess *' + 'const struct req *' ), ('obj.status', 'INT', ( 'error',), ( 'error',), - 'const struct sess *' + 'const struct req *' ), ('obj.response', 'STRING', ( 'error',), ( 'error',), - 'const struct sess *' + 'const struct req *' ), ('obj.hits', 'INT', ( 'hit', 'deliver',), ( ), - 'const struct sess *' + 'const struct req *' ), ('obj.http.', 'HDR_OBJ', ( 'hit', 'error',), ( 'error',), # XXX ? - 'const struct sess *' + 'const struct req *' ), ('obj.ttl', 'DURATION', ( 'hit', 'error',), ( 'hit', 'error',), - 'struct sess *' + 'struct req *' ), ('obj.grace', 'DURATION', ( 'hit', 'error',), ( 'hit', 'error',), - 'struct sess *' + 'struct req *' ), ('obj.keep', 'DURATION', ( 'hit', 'error',), ( 'hit', 'error',), - 'struct sess *' + 'struct req *' ), ('obj.lastuse', 'DURATION', ( 'hit', 'deliver', 'error',), ( ), - 'const struct sess *' + 'const struct req *' ), ('resp.proto', 'STRING', ( 'deliver',), ( 'deliver',), - 'const struct sess *' + 'const struct req *' ), ('resp.status', 'INT', ( 'deliver',), ( 'deliver',), - 'const struct sess *' + 'const struct req *' ), ('resp.response', 'STRING', ( 'deliver',), ( 'deliver',), - 'const struct sess *' + 'const struct req *' ), ('resp.http.', 'HDR_RESP', ( 'deliver',), ( 'deliver',), - 'const struct sess *' + 'const struct req *' ), ('now', 'TIME', ( 'all',), ( ), - 'const struct sess *' + 'const struct req *' ), ) @@ -831,7 +831,7 @@ for i in sp_variables: fo.write("\t{ \"%s\", %s, %d,\n" % (i[0], typ, len(i[0]))) if len(i[2]) > 0: - fo.write('\t "VRT_r_%s(sp)",\n' % cnam) + fo.write('\t "VRT_r_%s(req)",\n' % cnam) if typ != "HEADER": fh.write(ctyp + " VRT_r_%s(%s);\n" % (cnam, i[4])) else: @@ -839,7 +839,7 @@ for i in sp_variables: restrict(fo, i[2]) if len(i[3]) > 0: - fo.write('\t "VRT_l_%s(sp, ",\n' % cnam) + fo.write('\t "VRT_l_%s(req, ",\n' % cnam) if typ != "HEADER": fh.write("void VRT_l_%s(%s, " % (cnam, i[4])) if typ != "STRING": From phk at FreeBSD.org Thu Dec 18 09:27:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] b4d76fe Move more of the VRT interface from 'sp' to 'req' Message-ID: commit b4d76fea32541ffa008e6187faa1ffd21919a5e5 Author: Poul-Henning Kamp Date: Mon Jun 18 08:31:14 2012 +0000 Move more of the VRT interface from 'sp' to 'req' diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 79efc69..d288e3d 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -156,20 +156,21 @@ HSH_DeleteObjHead(struct dstat *ds, struct objhead *oh) } void -HSH_AddString(const struct sess *sp, const char *str) +HSH_AddString(struct req *req, const char *str) { int l; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (str == NULL) str = ""; l = strlen(str); - AN(sp->req->sha256ctx); - SHA256_Update(sp->req->sha256ctx, str, l); - SHA256_Update(sp->req->sha256ctx, "#", 1); + AN(req->sha256ctx); + SHA256_Update(req->sha256ctx, str, l); + SHA256_Update(req->sha256ctx, "#", 1); if (cache_param->log_hash) - VSLb(sp->req->vsl, SLT_Hash, "%s", str); + VSLb(req->vsl, SLT_Hash, "%s", str); } /*--------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 524a9b2..f1b1f3c 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -93,27 +93,27 @@ VRT_acl_log(const struct sess *sp, const char *msg) /*--------------------------------------------------------------------*/ static struct http * -vrt_selecthttp(const struct sess *sp, enum gethdr_e where) +vrt_selecthttp(const struct req *req, enum gethdr_e where) { struct http *hp; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); switch (where) { case HDR_REQ: - hp = sp->req->http; + hp = req->http; break; case HDR_BEREQ: - hp = sp->req->busyobj->bereq; + hp = req->busyobj->bereq; break; case HDR_BERESP: - hp = sp->req->busyobj->beresp; + hp = req->busyobj->beresp; break; case HDR_RESP: - hp = sp->req->resp; + hp = req->resp; break; case HDR_OBJ: - CHECK_OBJ_NOTNULL(sp->req->obj, OBJECT_MAGIC); - hp = sp->req->obj->http; + CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); + hp = req->obj->http; break; default: INCOMPL(); @@ -123,13 +123,13 @@ vrt_selecthttp(const struct sess *sp, enum gethdr_e where) } char * -VRT_GetHdr(const struct sess *sp, enum gethdr_e where, const char *n) +VRT_GetHdr(const struct req *req, enum gethdr_e where, const char *n) { char *p; struct http *hp; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - hp = vrt_selecthttp(sp, where); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + hp = vrt_selecthttp(req, where); if (!http_GetHdr(hp, n, &p)) return (NULL); return (p); @@ -215,22 +215,22 @@ VRT_WrkString(const struct sess *sp, const char *p, ...) /*--------------------------------------------------------------------*/ void -VRT_SetHdr(const struct sess *sp , enum gethdr_e where, const char *hdr, +VRT_SetHdr(struct req *req , enum gethdr_e where, const char *hdr, const char *p, ...) { struct http *hp; va_list ap; char *b; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - hp = vrt_selecthttp(sp, where); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + hp = vrt_selecthttp(req, where); va_start(ap, p); if (p == NULL) { http_Unset(hp, hdr); } else { b = VRT_String(hp->ws, hdr + 1, p, ap); if (b == NULL) { - VSLb(sp->req->vsl, SLT_LostHeader, "%s", hdr + 1); + VSLb(req->vsl, SLT_LostHeader, "%s", hdr + 1); } else { http_Unset(hp, hdr); http_SetHeader(hp, b); @@ -242,16 +242,16 @@ VRT_SetHdr(const struct sess *sp , enum gethdr_e where, const char *hdr, /*--------------------------------------------------------------------*/ void -VRT_handling(const struct sess *sp, unsigned hand) +VRT_handling(struct req *req, unsigned hand) { - if (sp == NULL) { + if (req == NULL) { assert(hand == VCL_RET_OK); return; } - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); assert(hand < VCL_RET_MAX); - sp->req->handling = hand; + req->handling = hand; } /*-------------------------------------------------------------------- @@ -259,18 +259,18 @@ VRT_handling(const struct sess *sp, unsigned hand) */ void -VRT_hashdata(const struct sess *sp, const char *str, ...) +VRT_hashdata(struct req *req, const char *str, ...) { va_list ap; const char *p; - HSH_AddString(sp, str); + HSH_AddString(req, str); va_start(ap, str); while (1) { p = va_arg(ap, const char *); if (p == vrt_magic_string_end) break; - HSH_AddString(sp, p); + HSH_AddString(req, p); } } diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index cbf62ba..7eb1b14 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -29,6 +29,7 @@ */ struct sess; +struct req; struct worker; struct object; @@ -56,7 +57,7 @@ struct objcore *HSH_Lookup(struct sess *sp); void HSH_Ref(struct objcore *o); void HSH_Drop(struct worker *, struct object **); void HSH_Init(const struct hash_slinger *slinger); -void HSH_AddString(const struct sess *sp, const char *str); +void HSH_AddString(struct req *, const char *str); void HSH_Insert(struct worker *, const void *hash, struct objcore *); void HSH_Purge(const struct sess *, struct objhead *, double ttl, double grace); void HSH_config(const char *h_arg); diff --git a/include/vrt.h b/include/vrt.h index 0b122ec..dc5ba13 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -161,12 +161,12 @@ void VRT_error(const struct sess *, unsigned, const char *); int VRT_switch_config(const char *); enum gethdr_e { HDR_REQ, HDR_RESP, HDR_OBJ, HDR_BEREQ, HDR_BERESP }; -char *VRT_GetHdr(const struct sess *, enum gethdr_e where, const char *); -void VRT_SetHdr(const struct sess *, enum gethdr_e where, const char *, +char *VRT_GetHdr(const struct req *, enum gethdr_e where, const char *); +void VRT_SetHdr(struct req *, enum gethdr_e where, const char *, const char *, ...); -void VRT_handling(const struct sess *sp, unsigned hand); +void VRT_handling(struct req *, unsigned hand); -void VRT_hashdata(const struct sess *sp, const char *str, ...); +void VRT_hashdata(struct req *, const char *str, ...); /* Simple stuff */ int VRT_strcmp(const char *s1, const char *s2); @@ -217,9 +217,9 @@ char *VRT_time_string(const struct sess *sp, double); const char *VRT_bool_string(const struct sess *sp, unsigned); const char *VRT_backend_string(const struct sess *sp, const struct director *d); -#define VRT_done(sp, hand) \ +#define VRT_done(req, hand) \ do { \ - VRT_handling(sp, hand); \ + VRT_handling(req, hand); \ return (1); \ } while (0) diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c index e725c6d..31a940b 100644 --- a/lib/libvcl/vcc_action.c +++ b/lib/libvcl/vcc_action.c @@ -77,7 +77,7 @@ parse_error(struct vcc *tl) Fb(tl, 1, ", 0\n"); } Fb(tl, 1, ");\n"); - Fb(tl, 1, "VRT_done(sp, VCL_RET_ERROR);\n"); + Fb(tl, 1, "VRT_done(req, VCL_RET_ERROR);\n"); } /*--------------------------------------------------------------------*/ @@ -224,7 +224,7 @@ parse_hash_data(struct vcc *tl) vcc_NextToken(tl); SkipToken(tl, '('); - Fb(tl, 1, "VRT_hashdata(sp, "); + Fb(tl, 1, "VRT_hashdata(req, "); vcc_Expr(tl, STRING_LIST); ERRCHK(tl); Fb(tl, 0, ");\n"); @@ -259,7 +259,7 @@ parse_return(struct vcc *tl) #define VCL_RET_MAC(l, U, B) \ do { \ if (vcc_IdIs(tl->t, #l)) { \ - Fb(tl, 1, "VRT_done(sp, VCL_RET_" #U ");\n"); \ + Fb(tl, 1, "VRT_done(req, VCL_RET_" #U ");\n"); \ vcc_ProcAction(tl->curproc, VCL_RET_##U, tl->t);\ retval = 1; \ } \ diff --git a/lib/libvcl/vcc_var.c b/lib/libvcl/vcc_var.c index 1bbac15..aa728e6 100644 --- a/lib/libvcl/vcc_var.c +++ b/lib/libvcl/vcc_var.c @@ -59,10 +59,10 @@ vcc_Var_Wildcard(struct vcc *tl, const struct token *t, const struct symbol *wc) bprintf(buf, "\\%03o%s:", (unsigned)l, v->name + vh->len); v->hdr = TlDup(tl, buf); - bprintf(buf, "VRT_GetHdr(sp, %s, \"%s\")", v->http, v->hdr); + bprintf(buf, "VRT_GetHdr(req, %s, \"%s\")", v->http, v->hdr); v->rname = TlDup(tl, buf); - bprintf(buf, "VRT_SetHdr(sp, %s, \"%s\", ", v->http, v->hdr); + bprintf(buf, "VRT_SetHdr(req, %s, \"%s\", ", v->http, v->hdr); v->lname = TlDup(tl, buf); sym = VCC_AddSymbolTok(tl, t, SYM_VAR); From phk at FreeBSD.org Thu Dec 18 09:27:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] 56ce6e3 More VRT sp->req work Message-ID: commit 56ce6e353960411f281a5114993ef2e7f51cf3dd Author: Poul-Henning Kamp Date: Mon Jun 18 08:48:53 2012 +0000 More VRT sp->req work diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index f1b1f3c..5675311 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -54,17 +54,16 @@ const void * const vrt_magic_string_end = &vrt_magic_string_end; /*--------------------------------------------------------------------*/ void -VRT_error(const struct sess *sp, unsigned code, const char *reason) +VRT_error(struct req *req, unsigned code, const char *reason) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - VSLb(sp->req->vsl, SLT_Debug, "VCL_error(%u, %s)", code, reason ? - reason : "(null)"); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + VSLb(req->vsl, SLT_Debug, "VCL_error(%u, %s)", code, + reason ? reason : "(null)"); if (code < 100 || code > 999) code = 503; - sp->req->err_code = (uint16_t)code; - sp->req->err_reason = - reason ? reason : http_StatusMessage(sp->req->err_code); + req->err_code = (uint16_t)code; + req->err_reason = reason ? reason : http_StatusMessage(req->err_code); } /*--------------------------------------------------------------------*/ @@ -196,18 +195,18 @@ VRT_String(struct ws *ws, const char *h, const char *p, va_list ap) } /*-------------------------------------------------------------------- - * Build a string on the worker threads workspace + * Build a string on the request workspace */ const char * -VRT_WrkString(const struct sess *sp, const char *p, ...) +VRT_ReqString(struct req *req, const char *p, ...) { va_list ap; char *b; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); va_start(ap, p); - b = VRT_String(sp->wrk->aws, NULL, p, ap); + b = VRT_String(req->ws, NULL, p, ap); va_end(ap); return (b); } @@ -287,7 +286,7 @@ VRT_r_now(const struct req *req) /*--------------------------------------------------------------------*/ char * -VRT_IP_string(const struct sess *sp, const struct sockaddr_storage *sa) +VRT_IP_string(struct req *req, const struct sockaddr_storage *sa) { char *p; const struct sockaddr_in *si4; @@ -295,6 +294,7 @@ VRT_IP_string(const struct sess *sp, const struct sockaddr_storage *sa) const void *addr; int len; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); switch (sa->ss_family) { case AF_INET: len = INET_ADDRSTRLEN; @@ -310,61 +310,65 @@ VRT_IP_string(const struct sess *sp, const struct sockaddr_storage *sa) INCOMPL(); } XXXAN(len); - AN(p = WS_Alloc(sp->req->http->ws, len)); + AN(p = WS_Alloc(req->http->ws, len)); AN(inet_ntop(sa->ss_family, addr, p, len)); return (p); } char * -VRT_int_string(const struct sess *sp, int num) +VRT_int_string(struct req *req, int num) { char *p; int size; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); size = snprintf(NULL, 0, "%d", num) + 1; - AN(p = WS_Alloc(sp->req->http->ws, size)); + AN(p = WS_Alloc(req->http->ws, size)); assert(snprintf(p, size, "%d", num) < size); return (p); } char * -VRT_double_string(const struct sess *sp, double num) +VRT_double_string(struct req *req, double num) { char *p; int size; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); size = snprintf(NULL, 0, "%.3f", num) + 1; - AN(p = WS_Alloc(sp->req->http->ws, size)); + AN(p = WS_Alloc(req->http->ws, size)); assert(snprintf(p, size, "%.3f", num) < size); return (p); } char * -VRT_time_string(const struct sess *sp, double t) +VRT_time_string(struct req *req, double t) { char *p; - AN(p = WS_Alloc(sp->req->http->ws, VTIM_FORMAT_SIZE)); - VTIM_format(t, p); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + p = WS_Alloc(req->http->ws, VTIM_FORMAT_SIZE); + if (p != NULL) + VTIM_format(t, p); return (p); } const char * -VRT_backend_string(const struct sess *sp, const struct director *d) +VRT_backend_string(const struct req *req, const struct director *d) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (d == NULL) - d = sp->req->director; + d = req->director; if (d == NULL) return (NULL); return (d->vcl_name); } const char * -VRT_bool_string(const struct sess *sp, unsigned val) +VRT_bool_string(const struct req *req, unsigned val) { - (void)sp; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); return (val ? "true" : "false"); } @@ -395,16 +399,16 @@ VRT_panic(const struct sess *sp, const char *str, ...) /*--------------------------------------------------------------------*/ void -VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...) +VRT_synth_page(struct req *req, unsigned flags, const char *str, ...) { va_list ap; const char *p; struct vsb *vsb; (void)flags; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->req->obj, OBJECT_MAGIC); - vsb = SMS_Makesynth(sp->req->obj); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); + vsb = SMS_Makesynth(req->obj); AN(vsb); VSB_cat(vsb, str); @@ -417,10 +421,9 @@ VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...) p = va_arg(ap, const char *); } va_end(ap); - SMS_Finish(sp->req->obj); - http_Unset(sp->req->obj->http, H_Content_Length); - http_PrintfHeader(sp->req->obj->http, - "Content-Length: %zd", sp->req->obj->len); + SMS_Finish(req->obj); + http_Unset(req->obj->http, H_Content_Length); + http_PrintfHeader(req->obj->http, "Content-Length: %zd", req->obj->len); } /*--------------------------------------------------------------------*/ diff --git a/include/vrt.h b/include/vrt.h index dc5ba13..a6a2642 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -157,7 +157,7 @@ void VRT_purge(const struct sess *sp, double ttl, double grace); void VRT_count(struct req *, unsigned); int VRT_rewrite(const char *, const char *); -void VRT_error(const struct sess *, unsigned, const char *); +void VRT_error(struct req *, unsigned, const char *); int VRT_switch_config(const char *); enum gethdr_e { HDR_REQ, HDR_RESP, HDR_OBJ, HDR_BEREQ, HDR_BERESP }; @@ -176,7 +176,7 @@ void VRT_ESI(const struct sess *sp); void VRT_Rollback(const struct sess *sp); /* Synthetic pages */ -void VRT_synth_page(const struct sess *sp, unsigned flags, const char *, ...); +void VRT_synth_page(struct req *sp, unsigned flags, const char *, ...); /* Backend related */ void VRT_init_dir(struct cli *, struct director **, const char *name, @@ -210,12 +210,12 @@ int VRT_Stv(const char *nm); /* Convert things to string */ -char *VRT_IP_string(const struct sess *sp, const struct sockaddr_storage *sa); -char *VRT_int_string(const struct sess *sp, int); -char *VRT_double_string(const struct sess *sp, double); -char *VRT_time_string(const struct sess *sp, double); -const char *VRT_bool_string(const struct sess *sp, unsigned); -const char *VRT_backend_string(const struct sess *sp, const struct director *d); +char *VRT_IP_string(struct req *, const struct sockaddr_storage *sa); +char *VRT_int_string(struct req *, int); +char *VRT_double_string(struct req *, double); +char *VRT_time_string(struct req *, double); +const char *VRT_bool_string(const struct req *, unsigned); +const char *VRT_backend_string(const struct req *, const struct director *d); #define VRT_done(req, hand) \ do { \ @@ -223,4 +223,4 @@ const char *VRT_backend_string(const struct sess *sp, const struct director *d); return (1); \ } while (0) -const char *VRT_WrkString(const struct sess *sp, const char *p, ...); +const char *VRT_ReqString(struct req *, const char *p, ...); diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c index 31a940b..31b2bad 100644 --- a/lib/libvcl/vcc_action.c +++ b/lib/libvcl/vcc_action.c @@ -57,7 +57,7 @@ parse_error(struct vcc *tl) { vcc_NextToken(tl); - Fb(tl, 1, "VRT_error(sp,\n"); + Fb(tl, 1, "VRT_error(req,\n"); if (tl->t->tok == '(') { vcc_NextToken(tl); vcc_Expr(tl, INT); @@ -303,7 +303,7 @@ parse_synthetic(struct vcc *tl) { vcc_NextToken(tl); - Fb(tl, 1, "VRT_synth_page(sp, 0, "); + Fb(tl, 1, "VRT_synth_page(req, 0, "); vcc_Expr(tl, STRING_LIST); ERRCHK(tl); Fb(tl, 0, ");\n"); diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c index 19a455c..d964ff1 100644 --- a/lib/libvcl/vcc_expr.c +++ b/lib/libvcl/vcc_expr.c @@ -423,15 +423,15 @@ vcc_expr_tostring(struct expr **e, enum var_type fmt) p = NULL; switch((*e)->fmt) { - case BACKEND: p = "VRT_backend_string(sp, \v1)"; break; - case BOOL: p = "VRT_bool_string(sp, \v1)"; break; - case DURATION: p = "VRT_double_string(sp, \v1)"; break; + case BACKEND: p = "VRT_backend_string(req, \v1)"; break; + case BOOL: p = "VRT_bool_string(req, \v1)"; break; + case DURATION: p = "VRT_double_string(req, \v1)"; break; /* XXX: should DURATION insist on "s" suffix ? */ - case INT: p = "VRT_int_string(sp, \v1)"; break; - case IP: p = "VRT_IP_string(sp, \v1)"; break; - case BYTES: p = "VRT_double_string(sp, \v1)"; break; /* XXX */ - case REAL: p = "VRT_double_string(sp, \v1)"; break; - case TIME: p = "VRT_time_string(sp, \v1)"; break; + case INT: p = "VRT_int_string(req, \v1)"; break; + case IP: p = "VRT_IP_string(req, \v1)"; break; + case BYTES: p = "VRT_double_string(req, \v1)"; break; /* XXX */ + case REAL: p = "VRT_double_string(req, \v1)"; break; + case TIME: p = "VRT_time_string(req, \v1)"; break; default: break; } if (p != NULL) { @@ -819,7 +819,7 @@ vcc_expr_add(struct vcc *tl, struct expr **e, enum var_type fmt) } if (fmt != STRING_LIST && (*e)->fmt == STRING_LIST) *e = vcc_expr_edit(STRING, - "\v+VRT_WrkString(sp,\n\v1,\nvrt_magic_string_end)", + "\v+VRT_ReqString(req,\n\v1,\nvrt_magic_string_end)", *e, NULL); if (fmt == STRING_LIST && (*e)->fmt == STRING) (*e)->fmt = STRING_LIST; From tfheen at varnish-software.com Thu Dec 18 09:27:48 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] 48345b6 Use older libtool invocation to please EL5 users. Welcome to 2005. Message-ID: commit 48345b63af6a00b79f20b873280e87dd75d4253e Author: Tollef Fog Heen Date: Mon Jun 18 11:13:17 2012 +0200 Use older libtool invocation to please EL5 users. Welcome to 2005. diff --git a/configure.ac b/configure.ac index 8193874..3df1e7d 100644 --- a/configure.ac +++ b/configure.ac @@ -13,7 +13,8 @@ AC_CANONICAL_SYSTEM AC_LANG(C) AM_INIT_AUTOMAKE([foreign]) -LT_INIT([disable-static]) +AC_DISABLE_STATIC +AC_PROG_LIBTOOL # Checks for programs. AC_GNU_SOURCE From phk at FreeBSD.org Thu Dec 18 09:27:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] acab05e More VRT sp->req movements Message-ID: commit acab05e046bad0f194a699d229b981ee8d5ffecc Author: Poul-Henning Kamp Date: Mon Jun 18 09:13:27 2012 +0000 More VRT sp->req movements diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 5675311..d227949 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -83,10 +83,10 @@ VRT_count(struct req *req, unsigned u) /*--------------------------------------------------------------------*/ void -VRT_acl_log(const struct sess *sp, const char *msg) +VRT_acl_log(struct req *req, const char *msg) { - VSLb(sp->req->vsl, SLT_VCL_acl, "%s", msg); + VSLb(req->vsl, SLT_VCL_acl, "%s", msg); } /*--------------------------------------------------------------------*/ @@ -385,13 +385,14 @@ VRT_Rollback(const struct sess *sp) /*--------------------------------------------------------------------*/ void -VRT_panic(const struct sess *sp, const char *str, ...) +VRT_panic(struct req *req, const char *str, ...) { va_list ap; char *b; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); va_start(ap, str); - b = VRT_String(sp->req->http->ws, "PANIC: ", str, ap); + b = VRT_String(req->http->ws, "PANIC: ", str, ap); va_end(ap); VAS_Fail("VCL", "", 0, b, 0, 2); } diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c index 765cd27..e4b9750 100644 --- a/bin/varnishd/cache/cache_vrt_re.c +++ b/bin/varnishd/cache/cache_vrt_re.c @@ -62,11 +62,12 @@ VRT_re_fini(void *rep) } int -VRT_re_match(const struct sess *sp, const char *s, void *re) +VRT_re_match(struct req *req, const char *s, void *re) { vre_t *t; int i; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (s == NULL) s = ""; AN(re); @@ -75,13 +76,12 @@ VRT_re_match(const struct sess *sp, const char *s, void *re) if (i >= 0) return (1); if (i < VRE_ERROR_NOMATCH ) - VSLb(sp->req->vsl, SLT_VCL_Error, - "Regexp matching returned %d", i); + VSLb(req->vsl, SLT_VCL_Error, "Regexp matching returned %d", i); return (0); } const char * -VRT_regsub(const struct sess *sp, int all, const char *str, void *re, +VRT_regsub(struct req *req, int all, const char *str, void *re, const char *sub) { int ovector[30]; @@ -94,6 +94,7 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re, int options = 0; size_t len; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AN(re); if (str == NULL) str = ""; @@ -109,13 +110,12 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re, if (i == VRE_ERROR_NOMATCH) return(str); if (i < VRE_ERROR_NOMATCH ) { - VSLb(sp->req->vsl, SLT_VCL_Error, - "Regexp matching returned %d", i); + VSLb(req->vsl, SLT_VCL_Error, "Regexp matching returned %d", i); return(str); } - u = WS_Reserve(sp->req->http->ws, 0); - res.e = res.b = b0 = sp->req->http->ws->f; + u = WS_Reserve(req->http->ws, 0); + res.e = res.b = b0 = req->http->ws->f; res.e += u; do { @@ -147,8 +147,8 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re, i = VRE_exec(t, str, len, 0, options, ovector, 30, &cache_param->vre_limits); if (i < VRE_ERROR_NOMATCH ) { - WS_Release(sp->req->http->ws, 0); - VSLb(sp->req->vsl, SLT_VCL_Error, + WS_Release(req->http->ws, 0); + VSLb(req->vsl, SLT_VCL_Error, "Regexp matching returned %d", i); return(str); } @@ -157,10 +157,10 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re, /* Copy suffix to match */ Tadd(&res, str, len+1); if (res.b >= res.e) { - WS_Release(sp->req->http->ws, 0); + WS_Release(req->http->ws, 0); return (str); } Tcheck(res); - WS_ReleaseP(sp->req->http->ws, res.b); + WS_ReleaseP(req->http->ws, res.b); return (b0); } diff --git a/include/vrt.h b/include/vrt.h index a6a2642..50d4d8b 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -141,16 +141,16 @@ struct vrt_ref { /* ACL related */ #define VRT_ACL_MAXADDR 16 /* max(IPv4, IPv6) */ -void VRT_acl_log(const struct sess *, const char *msg); +void VRT_acl_log(struct req *, const char *msg); /* Regexp related */ void VRT_re_init(void **, const char *); void VRT_re_fini(void *); -int VRT_re_match(const struct sess *sp, const char *, void *re); -const char *VRT_regsub(const struct sess *sp, int all, const char *, +int VRT_re_match(struct req *, const char *, void *re); +const char *VRT_regsub(struct req *, int all, const char *, void *, const char *); -void VRT_panic(const struct sess *sp, const char *, ...); +void VRT_panic(struct req *req, const char *, ...); void VRT_ban(struct sess *sp, char *, ...); void VRT_ban_string(struct sess *sp, const char *); void VRT_purge(const struct sess *sp, double ttl, double grace); diff --git a/lib/libvcl/vcc_acl.c b/lib/libvcl/vcc_acl.c index 9535f59..3e037ec 100644 --- a/lib/libvcl/vcc_acl.c +++ b/lib/libvcl/vcc_acl.c @@ -357,7 +357,7 @@ vcc_acl_emit(const struct vcc *tl, const char *acln, int anon) const char *oc; Fh(tl, 0, "\nstatic int\n"); - Fh(tl, 0, "match_acl_%s_%s(const struct sess *sp, const void *p)\n", + Fh(tl, 0, "match_acl_%s_%s(struct req *req, const void *p)\n", anon ? "anon" : "named", acln); Fh(tl, 0, "{\n"); Fh(tl, 0, "\tconst unsigned char *a;\n"); @@ -372,7 +372,7 @@ vcc_acl_emit(const struct vcc *tl, const char *acln, int anon) Fh(tl, 0, "\telse if (fam == %d)\n", PF_INET6); Fh(tl, 0, "\t\ta += %zd;\n", offsetof(struct sockaddr_in6, sin6_addr)); Fh(tl, 0, "\telse {\n"); - Fh(tl, 0, "\t\tVRT_acl_log(sp, \"NO_FAM %s\");\n", acln); + Fh(tl, 0, "\t\tVRT_acl_log(req, \"NO_FAM %s\");\n", acln); Fh(tl, 0, "\t\treturn(0);\n"); Fh(tl, 0, "\t}\n\n"); depth = -1; @@ -424,7 +424,7 @@ vcc_acl_emit(const struct vcc *tl, const char *acln, int anon) i = (ae->mask + 7) / 8; if (!anon) { - Fh(tl, 0, "\t%*sVRT_acl_log(sp, \"%sMATCH %s \" ", + Fh(tl, 0, "\t%*sVRT_acl_log(req, \"%sMATCH %s \" ", -i, "", ae->not ? "NEG_" : "", acln); EncToken(tl->fh, ae->t_addr); if (ae->t_mask != NULL) @@ -441,7 +441,7 @@ vcc_acl_emit(const struct vcc *tl, const char *acln, int anon) /* Deny by default */ if (!anon) - Fh(tl, 0, "\tVRT_acl_log(sp, \"NO_MATCH %s\");\n", acln); + Fh(tl, 0, "\tVRT_acl_log(req, \"NO_MATCH %s\");\n", acln); Fh(tl, 0, "\treturn (0);\n}\n"); } @@ -457,7 +457,7 @@ vcc_Acl_Hack(struct vcc *tl, char *b) bprintf(acln, "%u", tl->unique++); vcc_acl_entry(tl); vcc_acl_emit(tl, acln, 1); - sprintf(b, "%smatch_acl_anon_%s(sp, \v1)", + sprintf(b, "%smatch_acl_anon_%s(req, \v1)", (tcond == T_NEQ ? "!" : ""), acln); } diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c index 31b2bad..d60b885 100644 --- a/lib/libvcl/vcc_action.c +++ b/lib/libvcl/vcc_action.c @@ -238,7 +238,7 @@ parse_panic(struct vcc *tl) { vcc_NextToken(tl); - Fb(tl, 1, "VRT_panic(sp, "); + Fb(tl, 1, "VRT_panic(req, "); vcc_Expr(tl, STRING); ERRCHK(tl); Fb(tl, 0, ", vrt_magic_string_end);\n"); diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c index d964ff1..a3ef4b6 100644 --- a/lib/libvcl/vcc_expr.c +++ b/lib/libvcl/vcc_expr.c @@ -466,7 +466,7 @@ vcc_Eval_Regsub(struct vcc *tl, struct expr **e, const struct symbol *sym) p = vcc_regexp(tl); vcc_NextToken(tl); - bprintf(buf, "VRT_regsub(sp, %d,\n\v1,\n%s\n", all, p); + bprintf(buf, "VRT_regsub(req, %d,\n\v1,\n%s\n", all, p); *e = vcc_expr_edit(STRING, buf, e2, *e); SkipToken(tl, ','); @@ -947,7 +947,7 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, enum var_type fmt) re = vcc_regexp(tl); ERRCHK(tl); vcc_NextToken(tl); - bprintf(buf, "%sVRT_re_match(sp, \v1, %s)", not, re); + bprintf(buf, "%sVRT_re_match(req, \v1, %s)", not, re); *e = vcc_expr_edit(BOOL, buf, *e, NULL); return; } @@ -957,7 +957,8 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, enum var_type fmt) vcc_NextToken(tl); ExpectErr(tl, ID); vcc_AddRef(tl, tl->t, SYM_ACL); - bprintf(buf, "%smatch_acl_named_%.*s(sp, \v1)", not, PF(tl->t)); + bprintf(buf, "%smatch_acl_named_%.*s(req, \v1)", + not, PF(tl->t)); vcc_NextToken(tl); *e = vcc_expr_edit(BOOL, buf, *e, NULL); return; From phk at FreeBSD.org Thu Dec 18 09:27:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] a15dd15 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit a15dd15d81397489f2b3ac0ad4a1ef835f07b3d2 Merge: acab05e 48345b6 Author: Poul-Henning Kamp Date: Mon Jun 18 09:13:38 2012 +0000 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From phk at FreeBSD.org Thu Dec 18 09:27:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] a20b6af Add a new VMOD.debug and move the stuff varnishtest uses to get out into the obscure corners over there, making vmod.std a better copy& paste example and removing things like "panic" from the VCC. Message-ID: commit a20b6af4f1d5353132d07d4baae6d9e9c822f896 Author: Poul-Henning Kamp Date: Mon Jun 18 09:50:18 2012 +0000 Add a new VMOD.debug and move the stuff varnishtest uses to get out into the obscure corners over there, making vmod.std a better copy& paste example and removing things like "panic" from the VCC. diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index d227949..f172769 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -385,21 +385,6 @@ VRT_Rollback(const struct sess *sp) /*--------------------------------------------------------------------*/ void -VRT_panic(struct req *req, const char *str, ...) -{ - va_list ap; - char *b; - - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - va_start(ap, str); - b = VRT_String(req->http->ws, "PANIC: ", str, ap); - va_end(ap); - VAS_Fail("VCL", "", 0, b, 0, 2); -} - -/*--------------------------------------------------------------------*/ - -void VRT_synth_page(struct req *req, unsigned flags, const char *str, ...) { va_list ap; diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh index 9067263..c32c01b 100755 --- a/bin/varnishd/flint.sh +++ b/bin/varnishd/flint.sh @@ -25,6 +25,7 @@ flexelint \ ../../lib/libvarnishcompat/execinfo.c \ ../../lib/libvcl/*.c \ ../../lib/libvmod_std/*.c \ + ../../lib/libvmod_debug/*.c \ 2>&1 | tee _.fl if [ -f _.fl.old ] ; then diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index 7fe2b8c..504601b 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -1,4 +1,4 @@ -varnishtest "Test std vmod" +varnishtest "Test std & debug vmod" server s1 { rxreq @@ -7,11 +7,14 @@ server s1 { varnish v1 -vcl+backend { import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ; + import debug from "${topbuild}/lib/libvmod_debug/.libs/libvmod_debug.so" ; sub vcl_deliver { set resp.http.foo = std.toupper(resp.http.foo); set resp.http.bar = std.tolower(resp.http.bar); - set resp.http.who = std.author(phk); + set resp.http.who = debug.author(phk); + debug.test_priv_call(); + debug.test_priv_vcl(); std.log("VCL initiated log"); std.syslog(8 + 7, "Somebody runs varnishtest"); } @@ -27,9 +30,9 @@ client c1 { } -run varnish v1 -badvcl { - import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so.1" ; + import debug from "${topbuild}/lib/libvmod_debug/.libs/libvmod_debug.so.1" ; sub vcl_deliver { - set resp.http.who = std.author(jfk); + set resp.http.who = debug.author(jfk); } } diff --git a/bin/varnishtest/tests/r00878.vtc b/bin/varnishtest/tests/r00878.vtc index 6997189..55106c9 100644 --- a/bin/varnishtest/tests/r00878.vtc +++ b/bin/varnishtest/tests/r00878.vtc @@ -6,9 +6,9 @@ server s1 { } -start varnish v1 -vcl+backend { - import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ; + import debug from "${topbuild}/lib/libvmod_debug/.libs/libvmod_debug.so" ; sub vcl_deliver { - set resp.http.who = std.author(phk); + set resp.http.who = debug.author(phk); } } -start @@ -18,9 +18,9 @@ client c1 { rxresp } -run varnish v1 -vcl+backend { - import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ; + import debug from "${topbuild}/lib/libvmod_debug/.libs/libvmod_debug.so" ; sub vcl_deliver { - set resp.http.who = std.author(des); + set resp.http.who = debug.author(des); } } @@ -30,9 +30,9 @@ client c1 { } -run varnish v1 -vcl+backend { - import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ; + import debug from "${topbuild}/lib/libvmod_debug/.libs/libvmod_debug.so" ; sub vcl_deliver { - set resp.http.who = std.author(kristian); + set resp.http.who = debug.author(kristian); } } diff --git a/bin/varnishtest/tests/v00010.vtc b/bin/varnishtest/tests/v00010.vtc index b44fc13..46d778c 100644 --- a/bin/varnishtest/tests/v00010.vtc +++ b/bin/varnishtest/tests/v00010.vtc @@ -15,9 +15,11 @@ server s1 { varnish v1 -storage "-smalloc,1m" -vcl+backend { + import debug from "${topbuild}/lib/libvmod_debug/.libs/libvmod_debug.so"; + sub vcl_deliver { if (resp.http.panic) { - panic "Had Panic header: " + resp.http.panic; + debug.panic("Had Panic header: " + resp.http.panic); } } } -start diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc index 2c129d7..d1c0d5f 100644 --- a/bin/varnishtest/tests/v00018.vtc +++ b/bin/varnishtest/tests/v00018.vtc @@ -101,11 +101,6 @@ varnish v1 -vcl { varnish v1 -badvcl { backend b { .host = "127.0.0.1"; } - sub vcl_recv { panic if; } -} - -varnish v1 -badvcl { - backend b { .host = "127.0.0.1"; } sub vcl_recv { kluf ; } } diff --git a/configure.ac b/configure.ac index 3df1e7d..2791fd9 100644 --- a/configure.ac +++ b/configure.ac @@ -550,6 +550,7 @@ AC_CONFIG_FILES([ lib/libvarnishcompat/Makefile lib/libvcl/Makefile lib/libvgz/Makefile + lib/libvmod_debug/Makefile lib/libvmod_std/Makefile lib/libjemalloc/Makefile man/Makefile diff --git a/include/vrt.h b/include/vrt.h index 50d4d8b..834f972 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -150,7 +150,6 @@ int VRT_re_match(struct req *, const char *, void *re); const char *VRT_regsub(struct req *, int all, const char *, void *, const char *); -void VRT_panic(struct req *req, const char *, ...); void VRT_ban(struct sess *sp, char *, ...); void VRT_ban_string(struct sess *sp, const char *); void VRT_purge(const struct sess *sp, double ttl, double grace); diff --git a/lib/Makefile.am b/lib/Makefile.am index 21146dc..72bc0f8 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -6,6 +6,7 @@ SUBDIRS = \ libvarnishapi \ libvcl \ libvgz \ + libvmod_debug \ libvmod_std \ @JEMALLOC_SUBDIR@ diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c index d60b885..81cf597 100644 --- a/lib/libvcl/vcc_action.c +++ b/lib/libvcl/vcc_action.c @@ -234,19 +234,6 @@ parse_hash_data(struct vcc *tl) /*--------------------------------------------------------------------*/ static void -parse_panic(struct vcc *tl) -{ - vcc_NextToken(tl); - - Fb(tl, 1, "VRT_panic(req, "); - vcc_Expr(tl, STRING); - ERRCHK(tl); - Fb(tl, 0, ", vrt_magic_string_end);\n"); -} - -/*--------------------------------------------------------------------*/ - -static void parse_return(struct vcc *tl) { int retval = 0; @@ -331,7 +318,6 @@ static struct action_table { /* Keep list sorted from here */ { "call", parse_call }, { "hash_data", parse_hash_data, VCL_MET_HASH }, - { "panic", parse_panic }, { "ban", parse_ban }, { "ban_url", parse_ban_url }, { "remove", parse_unset }, /* backward compatibility */ diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am new file mode 100644 index 0000000..3a342b5 --- /dev/null +++ b/lib/libvmod_debug/Makefile.am @@ -0,0 +1,25 @@ +# + +INCLUDES = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/bin/varnishd \ + -I$(top_builddir)/include + +vmoddir = $(pkglibdir)/vmods +vmod_srcdir = $(top_srcdir)/lib/libvmod_debug +vmodtool = $(top_srcdir)/lib/libvcl/vmodtool.py +vmod_LTLIBRARIES = libvmod_debug.la + +libvmod_debug_la_LDFLAGS = -module -export-dynamic -avoid-version + +libvmod_debug_la_SOURCES = \ + vcc_if.c \ + vcc_if.h \ + vmod_debug.c + +vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc + @PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc + +EXTRA_DIST = vmod.vcc + +CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc new file mode 100644 index 0000000..2e3c33e --- /dev/null +++ b/lib/libvmod_debug/vmod.vcc @@ -0,0 +1,33 @@ +#- +# Copyright (c) 2010-2011 Varnish Software AS +# All rights reserved. +# +# Author: Poul-Henning Kamp +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +Module debug +Init init_function +Function VOID panic(STRING_LIST) +Function STRING author(ENUM { phk, des, kristian, mithrandir }) +Function VOID test_priv_call(PRIV_CALL) +Function VOID test_priv_vcl(PRIV_VCL) diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c new file mode 100644 index 0000000..309664a --- /dev/null +++ b/lib/libvmod_debug/vmod_debug.c @@ -0,0 +1,97 @@ +/*- + * Copyright (c) 2012 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "cache/cache.h" + +#include "vrt.h" +#include "vcc_if.h" + +void +vmod_panic(struct sess *sp, const char *str, ...) +{ + va_list ap; + char *b; + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + va_start(ap, str); + b = VRT_String(sp->req->http->ws, "PANIC: ", str, ap); + va_end(ap); + VAS_Fail("VCL", "", 0, b, 0, 2); +} + +const char * __match_proto__() +vmod_author(struct sess *sp, const char *id) +{ + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + if (!strcmp(id, "phk")) + return ("Poul-Henning"); + if (!strcmp(id, "des")) + return ("Dag-Erling"); + if (!strcmp(id, "kristian")) + return ("Kristian"); + if (!strcmp(id, "mithrandir")) + return ("Tollef"); + WRONG("Illegal VMOD enum"); +} + +int +init_function(struct vmod_priv *priv, const struct VCL_conf *cfg) +{ + (void)cfg; + + priv->priv = strdup("FOO"); + priv->free = free; + return (0); +} + +void +vmod_test_priv_call(struct sess *sp, struct vmod_priv *priv) +{ + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + if (priv->priv == NULL) { + priv->priv = strdup("BAR"); + priv->free = free; + } else { + assert(!strcmp(priv->priv, "BAR")); + } +} + +void +vmod_test_priv_vcl(struct sess *sp, struct vmod_priv *priv) +{ + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + assert(!strcmp(priv->priv, "FOO")); +} + diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index 0a71e5b..3238164 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -26,15 +26,13 @@ # SUCH DAMAGE. Module std -Init init_function -Function STRING toupper(PRIV_CALL, STRING_LIST) -Function STRING tolower(PRIV_VCL, STRING_LIST) +Function STRING toupper(STRING_LIST) +Function STRING tolower(STRING_LIST) Function VOID set_ip_tos(INT) Function REAL random(REAL, REAL) Function VOID log(STRING_LIST) Function VOID syslog(INT, STRING_LIST) Function STRING fileread(PRIV_CALL, STRING) -Function STRING author(ENUM { phk, des, kristian, mithrandir }) Function DURATION duration(STRING, DURATION) Function INT integer(STRING, INT) Function VOID collect(HEADER) diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 8ae9ec1..c8937f6 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -89,18 +89,12 @@ vmod_updown(struct sess *sp, int up, const char *s, va_list ap) } const char * __match_proto__() -vmod_toupper(struct sess *sp, struct vmod_priv *priv, const char *s, ...) +vmod_toupper(struct sess *sp, const char *s, ...) { const char *p; va_list ap; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - if (priv->priv == NULL) { - priv->priv = strdup("BAR"); - priv->free = free; - } else { - assert(!strcmp(priv->priv, "BAR")); - } va_start(ap, s); p = vmod_updown(sp, 1, s, ap); va_end(ap); @@ -108,29 +102,18 @@ vmod_toupper(struct sess *sp, struct vmod_priv *priv, const char *s, ...) } const char * __match_proto__() -vmod_tolower(struct sess *sp, struct vmod_priv *priv, const char *s, ...) +vmod_tolower(struct sess *sp, const char *s, ...) { const char *p; va_list ap; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - assert(!strcmp(priv->priv, "FOO")); va_start(ap, s); p = vmod_updown(sp, 0, s, ap); va_end(ap); return (p); } -int -init_function(struct vmod_priv *priv, const struct VCL_conf *cfg) -{ - (void)cfg; - - priv->priv = strdup("FOO"); - priv->free = free; - return (0); -} - double vmod_random(struct sess *sp, double lo, double hi) { @@ -181,21 +164,6 @@ vmod_syslog(struct sess *sp, int fac, const char *fmt, ...) WS_Release(sp->req->ws, 0); } -const char * __match_proto__() -vmod_author(struct sess *sp, const char *id) -{ - (void)sp; - if (!strcmp(id, "phk")) - return ("Poul-Henning"); - if (!strcmp(id, "des")) - return ("Dag-Erling"); - if (!strcmp(id, "kristian")) - return ("Kristian"); - if (!strcmp(id, "mithrandir")) - return ("Tollef"); - WRONG("Illegal VMOD enum"); -} - void __match_proto__() vmod_collect(struct sess *sp, enum gethdr_e e, const char *h) { From daghf at varnish-software.com Thu Dec 18 09:27:48 2014 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] e9e4aa4 Fix for an off-by-one issue in do_once_cb(). Message-ID: commit e9e4aa46214c17eab5839f7305626d85551f5702 Author: Dag Haavi Finstad Date: Mon Jun 18 12:58:08 2012 +0200 Fix for an off-by-one issue in do_once_cb(). Fixes: #1133 diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c index 859c379..47cee40 100644 --- a/bin/varnishstat/varnishstat.c +++ b/bin/varnishstat/varnishstat.c @@ -168,7 +168,7 @@ do_once_cb(void *priv, const struct VSC_point * const pt) if (strcmp(pt->ident, "")) i += printf("%s.", pt->ident); i += printf("%s", pt->desc->name); - if (i > op->pad) + if (i >= op->pad) op->pad = i + 1; printf("%*.*s", op->pad - i, op->pad - i, ""); if (pt->desc->flag == 'a' || pt->desc->flag == 'c') From tfheen at varnish-software.com Thu Dec 18 09:27:48 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] b16e50d Add libvmod_debug to DIST_SUBDIRS too Message-ID: commit b16e50df0ecb9366226e2387e987e11fcacef9fd Author: Tollef Fog Heen Date: Mon Jun 18 14:12:29 2012 +0200 Add libvmod_debug to DIST_SUBDIRS too diff --git a/lib/Makefile.am b/lib/Makefile.am index 72bc0f8..4fcc0ff 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -16,5 +16,6 @@ DIST_SUBDIRS = \ libvarnishapi \ libvcl \ libvgz \ + libvmod_debug \ libvmod_std \ libjemalloc From tfheen at varnish-software.com Thu Dec 18 09:27:48 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] e113b7f Don't install the debug vmod, it's just there for test purposes Message-ID: commit e113b7fcc4d1085659ec31923ce0a9e101fe0cad Author: Tollef Fog Heen Date: Mon Jun 18 14:25:08 2012 +0200 Don't install the debug vmod, it's just there for test purposes diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am index 3a342b5..ada2a56 100644 --- a/lib/libvmod_debug/Makefile.am +++ b/lib/libvmod_debug/Makefile.am @@ -8,7 +8,7 @@ INCLUDES = \ vmoddir = $(pkglibdir)/vmods vmod_srcdir = $(top_srcdir)/lib/libvmod_debug vmodtool = $(top_srcdir)/lib/libvcl/vmodtool.py -vmod_LTLIBRARIES = libvmod_debug.la +noinst_LTLIBRARIES = libvmod_debug.la libvmod_debug_la_LDFLAGS = -module -export-dynamic -avoid-version From phk at FreeBSD.org Thu Dec 18 09:27:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:48 +0100 Subject: [experimental-ims] 14b48c8 More VRT sess->req changes Message-ID: commit 14b48c8f8a9222689cef12d5fe9f19dc87a5eaa6 Author: Poul-Henning Kamp Date: Mon Jun 18 12:27:22 2012 +0000 More VRT sess->req changes diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index d288e3d..bfb030d 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -506,15 +506,16 @@ hsh_rush(struct dstat *ds, struct objhead *oh) */ void -HSH_Purge(const struct sess *sp, struct objhead *oh, double ttl, double grace) +HSH_Purge(struct req *req, struct objhead *oh, double ttl, double grace) { struct objcore *oc, **ocp; unsigned spc, nobj, n; struct object *o; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); - spc = WS_Reserve(sp->req->ws, 0); - ocp = (void*)sp->req->ws->f; + spc = WS_Reserve(req->ws, 0); + ocp = (void*)req->ws->f; Lck_Lock(&oh->mtx); assert(oh->refcnt > 0); nobj = 0; @@ -531,7 +532,8 @@ HSH_Purge(const struct sess *sp, struct objhead *oh, double ttl, double grace) continue; } - (void)oc_getobj(&sp->wrk->stats, oc); /* XXX: still needed ? */ + (void)oc_getobj(&req->sp->wrk->stats, oc); + /* XXX: still needed ? */ xxxassert(spc >= sizeof *ocp); oc->refcnt++; @@ -548,16 +550,16 @@ HSH_Purge(const struct sess *sp, struct objhead *oh, double ttl, double grace) for (n = 0; n < nobj; n++) { oc = ocp[n]; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - o = oc_getobj(&sp->wrk->stats, oc); + o = oc_getobj(&req->sp->wrk->stats, oc); if (o == NULL) continue; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); o->exp.ttl = ttl; o->exp.grace = grace; EXP_Rearm(o); - (void)HSH_Deref(&sp->wrk->stats, NULL, &o); + (void)HSH_Deref(&req->sp->wrk->stats, NULL, &o); } - WS_Release(sp->req->ws, 0); + WS_Release(req->ws, 0); } diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index b909a0e..07c1687 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -191,7 +191,7 @@ VCL_Load(const char *fn, const char *name, struct cli *cli) REPLACE(vcl->name, name); VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name); VTAILQ_INSERT_TAIL(&vcl_head, vcl, list); - (void)vcl->conf->init_func(NULL, NULL); + (void)vcl->conf->init_func(NULL); Lck_Lock(&vcl_mtx); if (vcl_active == NULL) vcl_active = vcl; @@ -215,7 +215,7 @@ VCL_Nuke(struct vcls *vcl) assert(vcl->conf->discard); assert(vcl->conf->busy == 0); VTAILQ_REMOVE(&vcl_head, vcl, list); - (void)vcl->conf->fini_func(NULL, NULL); + (void)vcl->conf->fini_func(NULL); vcl->conf->fini_vcl(NULL); free(vcl->name); (void)dlclose(vcl->dlh); @@ -344,7 +344,7 @@ VCL_##func##_method(struct req *req) \ req->handling = 0; \ req->cur_method = VCL_MET_ ## upper; \ VSLb(req->vsl, SLT_VCL_call, "%s", #func); \ - (void)req->vcl->func##_func(req->sp, req); \ + (void)req->vcl->func##_func(req); \ VSLb(req->vsl, SLT_VCL_return, "%s", \ VCL_Return_Name(req->handling)); \ req->cur_method = 0; \ diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index f172769..8cc2920 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -286,7 +286,7 @@ VRT_r_now(const struct req *req) /*--------------------------------------------------------------------*/ char * -VRT_IP_string(struct req *req, const struct sockaddr_storage *sa) +VRT_IP_string(const struct req *req, const struct sockaddr_storage *sa) { char *p; const struct sockaddr_in *si4; @@ -316,7 +316,7 @@ VRT_IP_string(struct req *req, const struct sockaddr_storage *sa) } char * -VRT_int_string(struct req *req, int num) +VRT_int_string(const struct req *req, int num) { char *p; int size; @@ -329,7 +329,7 @@ VRT_int_string(struct req *req, int num) } char * -VRT_double_string(struct req *req, double num) +VRT_double_string(const struct req *req, double num) { char *p; int size; @@ -342,7 +342,7 @@ VRT_double_string(struct req *req, double num) } char * -VRT_time_string(struct req *req, double t) +VRT_time_string(const struct req *req, double t) { char *p; @@ -375,17 +375,18 @@ VRT_bool_string(const struct req *req, unsigned val) /*--------------------------------------------------------------------*/ void -VRT_Rollback(const struct sess *sp) +VRT_Rollback(struct req *req) { - HTTP_Copy(sp->req->http, sp->req->http0); - WS_Reset(sp->req->ws, sp->req->ws_req); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + HTTP_Copy(req->http, req->http0); + WS_Reset(req->ws, req->ws_req); } /*--------------------------------------------------------------------*/ void -VRT_synth_page(struct req *req, unsigned flags, const char *str, ...) +VRT_synth_page(const struct req *req, unsigned flags, const char *str, ...) { va_list ap; const char *p; @@ -415,14 +416,14 @@ VRT_synth_page(struct req *req, unsigned flags, const char *str, ...) /*--------------------------------------------------------------------*/ void -VRT_ban(struct sess *sp, char *cmds, ...) +VRT_ban(const struct req *req, char *cmds, ...) { char *a1, *a2, *a3; va_list ap; struct ban *b; int good; - (void)sp; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); b = BAN_New(); va_start(ap, cmds); a1 = cmds; @@ -450,7 +451,7 @@ VRT_ban(struct sess *sp, char *cmds, ...) /*--------------------------------------------------------------------*/ void -VRT_ban_string(struct sess *sp, const char *str) +VRT_ban_string(const struct req *req, const char *str) { char *a1, *a2, *a3; char **av; @@ -458,7 +459,7 @@ VRT_ban_string(struct sess *sp, const char *str) int good; int i; - (void)sp; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); av = VAV_Parse(str, NULL, ARGV_NOESC); if (av[0] != NULL) { /* XXX: report error how ? */ @@ -500,12 +501,14 @@ VRT_ban_string(struct sess *sp, const char *str) */ void -VRT_purge(const struct sess *sp, double ttl, double grace) +VRT_purge(struct req *req, double ttl, double grace) { - if (sp->req->cur_method == VCL_MET_HIT) - HSH_Purge(sp, sp->req->obj->objcore->objhead, ttl, grace); - else if (sp->req->cur_method == VCL_MET_MISS) - HSH_Purge(sp, sp->req->objcore->objhead, ttl, grace); + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + if (req->cur_method == VCL_MET_HIT) + HSH_Purge(req, req->obj->objcore->objhead, ttl, grace); + else if (req->cur_method == VCL_MET_MISS) + HSH_Purge(req, req->objcore->objhead, ttl, grace); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 1569271..5ddd6ce 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -155,6 +155,16 @@ -e441 // for clause irregularity: loop variable '___' not found in 2nd for expression +// Vmod instantiation symbols are defined in all vmods + +-esym(14, Vmod_Name) +-esym(14, Vmod_Func) +-esym(14, Vmod_Len) +-esym(14, Vmod_Proto) +-esym(14, Vmod_Spec) +-esym(14, Vmod_Varnish_ABI) +-esym(14, Vmod_Id) + // Review all below this line /////////////////////////////////////////////// -e732 // 183 Loss of sign (___) (___ to ___) diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index 7eb1b14..fe0fb9f 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -59,7 +59,7 @@ void HSH_Drop(struct worker *, struct object **); void HSH_Init(const struct hash_slinger *slinger); void HSH_AddString(struct req *, const char *str); void HSH_Insert(struct worker *, const void *hash, struct objcore *); -void HSH_Purge(const struct sess *, struct objhead *, double ttl, double grace); +void HSH_Purge(struct req *, struct objhead *, double ttl, double grace); void HSH_config(const char *h_arg); struct objcore *HSH_NewObjCore(struct worker *wrk); diff --git a/bin/varnishtest/tests/c00033.vtc b/bin/varnishtest/tests/c00033.vtc index d93a6d0..d4289e3 100644 --- a/bin/varnishtest/tests/c00033.vtc +++ b/bin/varnishtest/tests/c00033.vtc @@ -25,13 +25,13 @@ varnish v1 -vcl+backend { sub vcl_hit { if (req.request == "PURGE") { - C{ VRT_purge(sp, 0, 0); }C + C{ VRT_purge(req, 0, 0); }C error 456 "got it"; } } sub vcl_miss { if (req.request == "PURGE") { - C{ VRT_purge(sp, 0, 0); }C + C{ VRT_purge(req, 0, 0); }C error 456 "got it"; } } diff --git a/include/vrt.h b/include/vrt.h index 834f972..20545fc 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -31,7 +31,6 @@ * XXX: When this file is changed, lib/libvcl/generate.py *MUST* be rerun. */ -struct sess; struct req; struct vsb; struct cli; @@ -150,9 +149,9 @@ int VRT_re_match(struct req *, const char *, void *re); const char *VRT_regsub(struct req *, int all, const char *, void *, const char *); -void VRT_ban(struct sess *sp, char *, ...); -void VRT_ban_string(struct sess *sp, const char *); -void VRT_purge(const struct sess *sp, double ttl, double grace); +void VRT_ban(const struct req *, char *, ...); +void VRT_ban_string(const struct req *, const char *); +void VRT_purge(struct req *, double ttl, double grace); void VRT_count(struct req *, unsigned); int VRT_rewrite(const char *, const char *); @@ -171,11 +170,10 @@ void VRT_hashdata(struct req *, const char *str, ...); int VRT_strcmp(const char *s1, const char *s2); void VRT_memmove(void *dst, const void *src, unsigned len); -void VRT_ESI(const struct sess *sp); -void VRT_Rollback(const struct sess *sp); +void VRT_Rollback(struct req *); /* Synthetic pages */ -void VRT_synth_page(struct req *sp, unsigned flags, const char *, ...); +void VRT_synth_page(const struct req *, unsigned flags, const char *, ...); /* Backend related */ void VRT_init_dir(struct cli *, struct director **, const char *name, @@ -209,10 +207,10 @@ int VRT_Stv(const char *nm); /* Convert things to string */ -char *VRT_IP_string(struct req *, const struct sockaddr_storage *sa); -char *VRT_int_string(struct req *, int); -char *VRT_double_string(struct req *, double); -char *VRT_time_string(struct req *, double); +char *VRT_IP_string(const struct req *, const struct sockaddr_storage *sa); +char *VRT_int_string(const struct req *, int); +char *VRT_double_string(const struct req *, double); +char *VRT_time_string(const struct req *, double); const char *VRT_bool_string(const struct req *, unsigned); const char *VRT_backend_string(const struct req *, const struct director *d); diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py index 0c1c02e..c999186 100755 --- a/lib/libvcl/generate.py +++ b/lib/libvcl/generate.py @@ -721,7 +721,7 @@ struct cli; typedef int vcl_init_f(struct cli *); typedef void vcl_fini_f(struct cli *); -typedef int vcl_func_f(struct sess *sp, struct req *req); +typedef int vcl_func_f(struct req *req); """) diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c index 81cf597..7eecc33 100644 --- a/lib/libvcl/vcc_action.c +++ b/lib/libvcl/vcc_action.c @@ -44,7 +44,7 @@ parse_call(struct vcc *tl) ExpectErr(tl, ID); vcc_AddCall(tl, tl->t); vcc_AddRef(tl, tl->t, SYM_SUB); - Fb(tl, 1, "if (VGC_function_%.*s(sp, req))\n", PF(tl->t)); + Fb(tl, 1, "if (VGC_function_%.*s(req))\n", PF(tl->t)); Fb(tl, 1, "\treturn (1);\n"); vcc_NextToken(tl); return; @@ -178,7 +178,7 @@ parse_ban(struct vcc *tl) ExpectErr(tl, '('); vcc_NextToken(tl); - Fb(tl, 1, "VRT_ban_string(sp, "); + Fb(tl, 1, "VRT_ban_string(req, "); vcc_Expr(tl, STRING); ERRCHK(tl); Fb(tl, 0, ");\n"); @@ -197,7 +197,7 @@ parse_ban_url(struct vcc *tl) ExpectErr(tl, '('); vcc_NextToken(tl); - Fb(tl, 1, "VRT_ban(sp, \"req.url\", \"~\", "); + Fb(tl, 1, "VRT_ban(req, \"req.url\", \"~\", "); vcc_Expr(tl, STRING); ERRCHK(tl); ExpectErr(tl, ')'); @@ -270,7 +270,7 @@ parse_rollback(struct vcc *tl) { vcc_NextToken(tl); - Fb(tl, 1, "VRT_Rollback(sp);\n"); + Fb(tl, 1, "VRT_Rollback(req);\n"); } /*--------------------------------------------------------------------*/ @@ -280,7 +280,7 @@ parse_purge(struct vcc *tl) { vcc_NextToken(tl); - Fb(tl, 1, "VRT_purge(sp, 0, 0);\n"); + Fb(tl, 1, "VRT_purge(req, 0, 0);\n"); } /*--------------------------------------------------------------------*/ diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c index a16fbed..30f2a24 100644 --- a/lib/libvcl/vcc_compile.c +++ b/lib/libvcl/vcc_compile.c @@ -672,7 +672,7 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp) /* Emit method functions */ for (i = 0; i < VCL_MET_MAX; i++) { Fc(tl, 1, "\nstatic int __match_proto__(vcl_func_f)\n"); - Fc(tl, 1, "VGC_function_%s(struct sess *sp, struct req *req)\n", + Fc(tl, 1, "VGC_function_%s(struct req *req)\n", method_tab[i].name); AZ(VSB_finish(tl->fm[i])); Fc(tl, 1, "{\n"); diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c index a3ef4b6..8fbaa04 100644 --- a/lib/libvcl/vcc_expr.c +++ b/lib/libvcl/vcc_expr.c @@ -539,7 +539,7 @@ vcc_Eval_Func(struct vcc *tl, struct expr **e, const struct symbol *sym) SkipToken(tl, ID); SkipToken(tl, '('); p = sym->args; - e2 = vcc_mk_expr(vcc_arg_type(&p), "%s(sp\v+", sym->cfunc); + e2 = vcc_mk_expr(vcc_arg_type(&p), "%s(req\v+", sym->cfunc); while (*p != '\0') { e1 = NULL; fmt = vcc_arg_type(&p); diff --git a/lib/libvcl/vcc_parse.c b/lib/libvcl/vcc_parse.c index 1a6bbf3..10170b3 100644 --- a/lib/libvcl/vcc_parse.c +++ b/lib/libvcl/vcc_parse.c @@ -225,11 +225,9 @@ vcc_Function(struct vcc *tl) } tl->curproc = vcc_AddProc(tl, tl->t); Fh(tl, 0, "static int VGC_function_%.*s " - "(struct sess *, struct req *);\n", - PF(tl->t)); + "(struct req *);\n", PF(tl->t)); Fc(tl, 1, "\nstatic int __match_proto__(vcl_func_t)\n"); - Fc(tl, 1, "VGC_function_%.*s" - "(struct sess *sp, struct req *req)\n", + Fc(tl, 1, "VGC_function_%.*s(struct req *req)\n", PF(tl->t)); } vcc_NextToken(tl); diff --git a/lib/libvcl/vmodtool.py b/lib/libvcl/vmodtool.py index 041edcb..e2cf8b5 100755 --- a/lib/libvcl/vmodtool.py +++ b/lib/libvcl/vmodtool.py @@ -81,7 +81,7 @@ def do_func(fname, rval, args, vargs): #print(fname, rval, args) # C argument list - cargs = "(struct sess *" + cargs = "(struct req *" for i in args: cargs += ", " + i cargs += ")" @@ -273,7 +273,7 @@ fh = open("vcc_if.h", "w") file_header(fc) file_header(fh) -fh.write('struct sess;\n') +fh.write('struct req;\n') fh.write('struct VCL_conf;\n') fh.write('struct vmod_priv;\n') fh.write("\n"); diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 309664a..fd65fd2 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -35,24 +35,24 @@ #include "vrt.h" #include "vcc_if.h" -void -vmod_panic(struct sess *sp, const char *str, ...) +void __match_proto__(td_debug_panic) +vmod_panic(struct req *req, const char *str, ...) { va_list ap; char *b; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); va_start(ap, str); - b = VRT_String(sp->req->http->ws, "PANIC: ", str, ap); + b = VRT_String(req->http->ws, "PANIC: ", str, ap); va_end(ap); VAS_Fail("VCL", "", 0, b, 0, 2); } -const char * __match_proto__() -vmod_author(struct sess *sp, const char *id) +const char * __match_proto__(td_debug_author) +vmod_author(struct req *req, const char *id) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (!strcmp(id, "phk")) return ("Poul-Henning"); if (!strcmp(id, "des")) @@ -74,11 +74,11 @@ init_function(struct vmod_priv *priv, const struct VCL_conf *cfg) return (0); } -void -vmod_test_priv_call(struct sess *sp, struct vmod_priv *priv) +void __match_proto__(td_debug_test_priv_call) +vmod_test_priv_call(struct req *req, struct vmod_priv *priv) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (priv->priv == NULL) { priv->priv = strdup("BAR"); priv->free = free; @@ -87,11 +87,11 @@ vmod_test_priv_call(struct sess *sp, struct vmod_priv *priv) } } -void -vmod_test_priv_vcl(struct sess *sp, struct vmod_priv *priv) +void __match_proto__(td_debug_test_priv_vcl) +vmod_test_priv_vcl(struct req *req, struct vmod_priv *priv) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); assert(!strcmp(priv->priv, "FOO")); } diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index c8937f6..4265cf1 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -45,23 +45,25 @@ #include "vcc_if.h" -void __match_proto__() -vmod_set_ip_tos(struct sess *sp, int tos) +void __match_proto__(td_std_set_ip_tos) +vmod_set_ip_tos(struct req *req, int tos) { - VTCP_Assert(setsockopt(sp->fd, IPPROTO_IP, IP_TOS, &tos, sizeof(tos))); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + VTCP_Assert(setsockopt(req->sp->fd, + IPPROTO_IP, IP_TOS, &tos, sizeof(tos))); } -static const char * __match_proto__() -vmod_updown(struct sess *sp, int up, const char *s, va_list ap) +static const char * +vmod_updown(struct req *req, int up, const char *s, va_list ap) { unsigned u; char *b, *e; const char *p; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - u = WS_Reserve(sp->req->ws, 0); - e = b = sp->req->ws->f; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + u = WS_Reserve(req->ws, 0); + e = b = req->ws->f; e += u; p = s; while (p != vrt_magic_string_end && b < e) { @@ -78,101 +80,101 @@ vmod_updown(struct sess *sp, int up, const char *s, va_list ap) *b = '\0'; b++; if (b > e) { - WS_Release(sp->req->ws, 0); + WS_Release(req->ws, 0); return (NULL); } else { e = b; - b = sp->req->ws->f; - WS_Release(sp->req->ws, e - b); + b = req->ws->f; + WS_Release(req->ws, e - b); return (b); } } -const char * __match_proto__() -vmod_toupper(struct sess *sp, const char *s, ...) +const char * __match_proto__(td_std_toupper) +vmod_toupper(struct req *req, const char *s, ...) { const char *p; va_list ap; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); va_start(ap, s); - p = vmod_updown(sp, 1, s, ap); + p = vmod_updown(req, 1, s, ap); va_end(ap); return (p); } -const char * __match_proto__() -vmod_tolower(struct sess *sp, const char *s, ...) +const char * __match_proto__(td_std_tolower) +vmod_tolower(struct req *req, const char *s, ...) { const char *p; va_list ap; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); va_start(ap, s); - p = vmod_updown(sp, 0, s, ap); + p = vmod_updown(req, 0, s, ap); va_end(ap); return (p); } -double -vmod_random(struct sess *sp, double lo, double hi) +double __match_proto__(td_std_random) +vmod_random(struct req *req, double lo, double hi) { double a; - (void)sp; - + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); a = drand48(); a *= hi - lo; a += lo; return (a); } -void __match_proto__() -vmod_log(struct sess *sp, const char *fmt, ...) +void __match_proto__(td_std_log) +vmod_log(struct req *req, const char *fmt, ...) { unsigned u; va_list ap; txt t; - u = WS_Reserve(sp->req->ws, 0); - t.b = sp->req->ws->f; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + u = WS_Reserve(req->ws, 0); + t.b = req->ws->f; va_start(ap, fmt); t.e = VRT_StringList(t.b, u, fmt, ap); va_end(ap); if (t.e != NULL) { assert(t.e > t.b); t.e--; - VSLbt(sp->req->vsl, SLT_VCL_Log, t); + VSLbt(req->vsl, SLT_VCL_Log, t); } - WS_Release(sp->req->ws, 0); + WS_Release(req->ws, 0); } -void __match_proto__() -vmod_syslog(struct sess *sp, int fac, const char *fmt, ...) +void __match_proto__(td_std_syslog) +vmod_syslog(struct req *req, int fac, const char *fmt, ...) { char *p; unsigned u; va_list ap; - u = WS_Reserve(sp->req->ws, 0); - p = sp->req->ws->f; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + u = WS_Reserve(req->ws, 0); + p = req->ws->f; va_start(ap, fmt); p = VRT_StringList(p, u, fmt, ap); va_end(ap); if (p != NULL) syslog(fac, "%s", p); - WS_Release(sp->req->ws, 0); + WS_Release(req->ws, 0); } -void __match_proto__() -vmod_collect(struct sess *sp, enum gethdr_e e, const char *h) +void __match_proto__(td_std_collect) +vmod_collect(struct req *req, enum gethdr_e e, const char *h) { - (void)e; - (void)sp; - (void)h; + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (e == HDR_REQ) - http_CollectHdr(sp->req->http, h); - else if (e == HDR_BERESP && sp->req->busyobj != NULL) - http_CollectHdr(sp->req->busyobj->beresp, h); + http_CollectHdr(req->http, h); + else if (e == HDR_BERESP && req->busyobj != NULL) + http_CollectHdr(req->busyobj->beresp, h); } diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index 149be81..45cc7a1 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -39,12 +39,12 @@ #include "vcc_if.h" double __match_proto__() -vmod_duration(struct sess *sp, const char *p, double d) +vmod_duration(struct req *req, const char *p, double d) { char *e; double r; - (void)sp; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (p == NULL) return (d); @@ -89,12 +89,12 @@ vmod_duration(struct sess *sp, const char *p, double d) } int __match_proto__() -vmod_integer(struct sess *sp, const char *p, int i) +vmod_integer(struct req *req, const char *p, int i) { char *e; int r; - (void)sp; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (p == NULL) return (i); diff --git a/lib/libvmod_std/vmod_std_fileread.c b/lib/libvmod_std/vmod_std_fileread.c index 7012b66..d424d75 100644 --- a/lib/libvmod_std/vmod_std_fileread.c +++ b/lib/libvmod_std/vmod_std_fileread.c @@ -82,13 +82,13 @@ free_frfile(void *ptr) } } -const char * -vmod_fileread(struct sess *sp, struct vmod_priv *priv, const char *file_name) +const char * __match_proto__(td_std_fileread) +vmod_fileread(struct req *req, struct vmod_priv *priv, const char *file_name) { struct frfile *frf = NULL; char *s; - (void)sp; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AN(priv); if (priv->priv != NULL) { From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] b71cb6a All the _r_ object accessor functions should by definition be able to take a "const struct req *" so enforce this. Message-ID: commit b71cb6a8ea5b9f4d3788c5e2c535dca9bcd3c5f1 Author: Poul-Henning Kamp Date: Mon Jun 18 12:57:25 2012 +0000 All the _r_ object accessor functions should by definition be able to take a "const struct req *" so enforce this. diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 8d325ca..36ff4c6 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -204,7 +204,7 @@ VBERESP(beresp, unsigned, do_pass, busyobj->do_pass) /*--------------------------------------------------------------------*/ const char * -VRT_r_client_identity(struct req *req) +VRT_r_client_identity(const struct req *req) { CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -238,8 +238,8 @@ VRT_l_bereq_##which(struct req *req, double num) \ req->busyobj->which = (num > 0.0 ? num : 0.0); \ } \ \ -double __match_proto__() \ -VRT_r_bereq_##which(struct req *req) \ +double \ +VRT_r_bereq_##which(const struct req *req) \ { \ \ CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ @@ -280,8 +280,8 @@ VRT_r_beresp_backend_port(const struct req *req) return (VTCP_port(req->busyobj->vbc->addr)); } -const char * __match_proto__() -VRT_r_beresp_storage(struct req *req) +const char * +VRT_r_beresp_storage(const struct req *req) { CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (req->storage_hint != NULL) @@ -290,7 +290,7 @@ VRT_r_beresp_storage(struct req *req) return (NULL); } -void __match_proto__() +void VRT_l_beresp_storage(struct req *req, const char *str, ...) { va_list ap; @@ -314,7 +314,7 @@ VRT_l_req_backend(struct req *req, struct director *be) } struct director * -VRT_r_req_backend(struct req *req) +VRT_r_req_backend(const struct req *req) { CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -336,7 +336,7 @@ VRT_l_req_esi(struct req *req, unsigned process_esi) } unsigned -VRT_r_req_esi(struct req *req) +VRT_r_req_esi(const struct req *req) { CHECK_OBJ_NOTNULL(req, REQ_MAGIC); return (!req->disable_esi); @@ -352,8 +352,8 @@ VRT_r_req_esi_level(const struct req *req) /*--------------------------------------------------------------------*/ -unsigned __match_proto__() -VRT_r_req_can_gzip(struct req *req) +unsigned +VRT_r_req_can_gzip(const struct req *req) { CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -378,7 +378,7 @@ VRT_r_req_restarts(const struct req *req) #define VRT_DO_EXP(which, exp, fld, offset, extra) \ \ -void __match_proto__() \ +void \ VRT_l_##which##_##fld(struct req *req, double a) \ { \ \ @@ -389,8 +389,8 @@ VRT_l_##which##_##fld(struct req *req, double a) \ extra; \ } \ \ -double __match_proto__() \ -VRT_r_##which##_##fld(struct req *req) \ +double \ +VRT_r_##which##_##fld(const struct req *req) \ { \ \ CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ @@ -432,8 +432,8 @@ VRT_DO_EXP(beresp, req->busyobj->exp, keep, 0, * req.xid */ -const char * __match_proto__() -VRT_r_req_xid(struct req *req) +const char * +VRT_r_req_xid(const struct req *req) { char *p; int size; @@ -448,7 +448,7 @@ VRT_r_req_xid(struct req *req) /*--------------------------------------------------------------------*/ #define REQ_BOOL(hash_var) \ -void __match_proto__() \ +void \ VRT_l_req_##hash_var(struct req *req, unsigned val) \ { \ \ @@ -456,8 +456,8 @@ VRT_l_req_##hash_var(struct req *req, unsigned val) \ req->hash_var = val ? 1 : 0; \ } \ \ -unsigned __match_proto__() \ -VRT_r_req_##hash_var(struct req *req) \ +unsigned \ +VRT_r_req_##hash_var(const struct req *req) \ { \ \ CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ @@ -470,7 +470,7 @@ REQ_BOOL(hash_always_miss) /*--------------------------------------------------------------------*/ struct sockaddr_storage * -VRT_r_client_ip(struct req *req) +VRT_r_client_ip(const struct req *req) { CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -478,7 +478,7 @@ VRT_r_client_ip(struct req *req) } struct sockaddr_storage * -VRT_r_server_ip(struct req *req) +VRT_r_server_ip(const struct req *req) { int i; @@ -493,7 +493,7 @@ VRT_r_server_ip(struct req *req) } const char* -VRT_r_server_identity(struct req *req) +VRT_r_server_identity(const struct req *req) { CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -504,7 +504,7 @@ VRT_r_server_identity(struct req *req) } const char* -VRT_r_server_hostname(struct req *req) +VRT_r_server_hostname(const struct req *req) { CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -518,7 +518,7 @@ VRT_r_server_hostname(struct req *req) */ int -VRT_r_server_port(struct req *req) +VRT_r_server_port(const struct req *req) { int i; @@ -552,7 +552,7 @@ VRT_r_obj_lastuse(const struct req *req) } unsigned -VRT_r_req_backend_healthy(struct req *req) +VRT_r_req_backend_healthy(const struct req *req) { CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->director, DIRECTOR_MAGIC); diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py index c999186..3a1daa1 100755 --- a/lib/libvcl/generate.py +++ b/lib/libvcl/generate.py @@ -833,7 +833,7 @@ for i in sp_variables: if len(i[2]) > 0: fo.write('\t "VRT_r_%s(req)",\n' % cnam) if typ != "HEADER": - fh.write(ctyp + " VRT_r_%s(%s);\n" % (cnam, i[4])) + fh.write(ctyp + " VRT_r_%s(const %s);\n" % (cnam, i[4])) else: fo.write('\t NULL,\t/* No reads allowed */\n') restrict(fo, i[2]) From tfheen at varnish-software.com Thu Dec 18 09:27:49 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] c74b02a Add -rpath to libvmod_debug_la_LDFLAGS, which forces creation of a .so Message-ID: commit c74b02adcdb7bdcaa53b5cd45de64e2295798251 Author: Tollef Fog Heen Date: Mon Jun 18 14:58:22 2012 +0200 Add -rpath to libvmod_debug_la_LDFLAGS, which forces creation of a .so diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am index ada2a56..23a6d53 100644 --- a/lib/libvmod_debug/Makefile.am +++ b/lib/libvmod_debug/Makefile.am @@ -10,7 +10,7 @@ vmod_srcdir = $(top_srcdir)/lib/libvmod_debug vmodtool = $(top_srcdir)/lib/libvcl/vmodtool.py noinst_LTLIBRARIES = libvmod_debug.la -libvmod_debug_la_LDFLAGS = -module -export-dynamic -avoid-version +libvmod_debug_la_LDFLAGS = -module -export-dynamic -avoid-version -rpath /nowhere libvmod_debug_la_SOURCES = \ vcc_if.c \ From perbu at varnish-software.com Thu Dec 18 09:27:49 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] 4264387 reference VCL for naming storage Message-ID: commit 42643875f383db925e4b96aeb09a097f5d1c3b2f Author: Per Buer Date: Tue Jun 19 09:11:21 2012 +0200 reference VCL for naming storage diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 48ebdc8..17e6c03 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -93,7 +93,8 @@ OPTIONS Use the specified storage backend. See Storage Types for a list of supported storage types. This option can be used multiple times to specify multiple storage files. You can name the different backends. Varnish will then reference that backend with the - given name in logs, statistics, etc. + given name in logs, statistics, etc. You will also be able to ask Varnish to use + these named backends specifically through VCL by setting backend.storage in vcl_fetch. -T address[:port] Offer a management interface on the specified address and port. See Management From perbu at varnish-software.com Thu Dec 18 09:27:49 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] 820aa11 document a couple of tags Message-ID: commit 820aa1126aa841b11828af934f6f72a880f9779c Author: Per Buer Date: Tue Jun 19 09:26:30 2012 +0200 document a couple of tags diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 1c404cf..fce1429 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -45,9 +45,25 @@ SLTM(Debug, "", "") SLTM(Error, "", "") SLTM(CLI, "CLI communication", "CLI communication between master and child process.") SLTM(StatSess, "Session statistics", "") -SLTM(ReqEnd, "Client request end", "") +SLTM(ReqEnd, "Client request end", "Client request end. The first number is the XID. \n" +"The second is the time when processing of the request started.\n" +"The third is the time the request completed.\n" +"The forth is is the time elapsed between the request actually being accepted and\n" +"the start of the request processing.\n" +"The fifth number is the time elapsed from the start of the request processing \n" +"until we start delivering the object to the client.\n" +"The sixth and last number is the time from we start delivering the object\n" +"until the request completes. ") SLTM(SessionOpen, "Client connection opened", "") -SLTM(SessionClose, "Client connection closed", "") +SLTM(SessionClose, "Client connection closed", "SessionClose tells you why HTTP\n" +"client-connections are closed. These can be:\n" +"timeout - No keep-alive was received within sess_timeout\n" +"Connection: close - The client specifed that keepalive should be disabled by sending a 'Connection: close' header.\n" +"no request - No initial request was received within sess_timeout.\n" +"EOF - ???\n" +"remote closed - ???\n" +"error - Processing reached vcl_error even if the status code indicates success\n" +"blast - ???") SLTM(BackendOpen, "Backend connection opened", "") SLTM(BackendXID, "The unique ID of the backend transaction", "") SLTM(BackendReuse, "Backend connection reused", "") From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] c0cf227 Another big wash of s/sp/req/ changes Message-ID: commit c0cf227af98f8c74a26aee28f5d316cacaf11679 Author: Poul-Henning Kamp Date: Tue Jun 19 08:52:22 2012 +0000 Another big wash of s/sp/req/ changes diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 03c3edf..845fac6 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -695,8 +695,8 @@ void VBE_UseHealth(const struct director *vdi); void VBE_DiscardHealth(const struct director *vdi); -struct vbc *VDI_GetFd(const struct director *, struct sess *sp); -int VDI_Healthy(const struct director *, const struct sess *sp); +struct vbc *VDI_GetFd(const struct director *, struct req *); +int VDI_Healthy(const struct director *, const struct req *); void VDI_CloseFd(struct vbc **vbp); void VDI_RecycleFd(struct vbc **vbp); void VDI_AddHostHeader(struct http *to, const struct vbc *vbc); @@ -719,7 +719,7 @@ void BAN_Insert(struct ban *b); void BAN_Init(void); void BAN_NewObjCore(struct objcore *oc); void BAN_DestroyObj(struct objcore *oc); -int BAN_CheckObject(struct object *o, const struct sess *sp); +int BAN_CheckObject(struct object *o, struct req *sp); void BAN_Reload(const uint8_t *ban, unsigned len); struct ban *BAN_TailRef(void); void BAN_Compile(void); @@ -766,9 +766,9 @@ int EXP_NukeOne(struct busyobj *, struct lru *lru); struct storage *FetchStorage(struct busyobj *, ssize_t sz); int FetchError(struct busyobj *, const char *error); int FetchError2(struct busyobj *, const char *error, const char *more); -int FetchHdr(struct sess *sp, int need_host_hdr, int sendbody); +int FetchHdr(struct req *req, int need_host_hdr, int sendbody); void FetchBody(struct worker *w, void *bo); -int FetchReqBody(const struct sess *sp, int sendbody); +int FetchReqBody(struct req *, int sendbody); void Fetch_Init(void); /* cache_gzip.c */ @@ -809,7 +809,7 @@ void http_ClrHeader(struct http *to); unsigned http_Write(const struct worker *w, const struct http *hp, int resp); void http_SetResp(struct http *to, const char *proto, uint16_t status, const char *response); -void http_FilterReq(const struct sess *sp, unsigned how); +void http_FilterReq(const struct req *, unsigned how); void http_FilterResp(const struct http *fm, struct http *to, unsigned how); void http_PutProtocol(const struct http *to, const char *protocol); void http_PutStatus(struct http *to, uint16_t status); @@ -830,7 +830,7 @@ double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field); uint16_t http_GetStatus(const struct http *hp); const char *http_GetReq(const struct http *hp); int http_HdrIs(const struct http *hp, const char *hdr, const char *val); -uint16_t http_DissectRequest(const struct sess *sp); +uint16_t http_DissectRequest(struct req *); uint16_t http_DissectResponse(struct http *sp, const struct http_conn *htc); const char *http_DoConnection(const struct http *hp); void http_CopyHome(const struct http *hp); @@ -954,8 +954,8 @@ void VSL_Flush(struct vsl_log *, int overflow); #endif /* cache_response.c */ -void RES_BuildHttp(const struct sess *sp); -void RES_WriteObj(struct sess *sp); +void RES_BuildHttp(struct req *); +void RES_WriteObj(struct req *); /* cache_vary.c */ struct vsb *VRY_Create(struct req *sp, const struct http *hp); @@ -979,7 +979,7 @@ const char *VCL_Return_Name(unsigned method); char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap); char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap); -void ESI_Deliver(struct sess *); +void ESI_Deliver(struct req *); void ESI_DeliverChild(const struct sess *); /* cache_vrt_vmod.c */ diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index fb116bf..e640a30 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -452,32 +452,32 @@ VBE_DiscardHealth(const struct director *vdi) */ static struct vbc * __match_proto__(vdi_getfd_f) -vdi_simple_getfd(const struct director *d, struct sess *sp) +vdi_simple_getfd(const struct director *d, struct req *req) { struct vdi_simple *vs; struct vbc *vc; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC); - vc = vbe_GetVbe(sp, vs); + vc = vbe_GetVbe(req->sp, vs); if (vc != NULL) { FIND_TMO(first_byte_timeout, - vc->first_byte_timeout, sp, vs->vrt); + vc->first_byte_timeout, req->sp, vs->vrt); FIND_TMO(between_bytes_timeout, - vc->between_bytes_timeout, sp, vs->vrt); + vc->between_bytes_timeout, req->sp, vs->vrt); } return (vc); } static unsigned -vdi_simple_healthy(const struct director *d, const struct sess *sp) +vdi_simple_healthy(const struct director *d, const struct req *req) { struct vdi_simple *vs; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC); - return (vbe_Healthy(vs, sp)); + return (vbe_Healthy(vs, req->sp)); } static void diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index 96f0a63..3baa0ca 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -75,9 +75,9 @@ struct vrt_backend_probe; * backends to use. */ -typedef struct vbc *vdi_getfd_f(const struct director *, struct sess *sp); +typedef struct vbc *vdi_getfd_f(const struct director *, struct req *); typedef void vdi_fini_f(const struct director *); -typedef unsigned vdi_healthy(const struct director *, const struct sess *sp); +typedef unsigned vdi_healthy(const struct director *, const struct req *); struct director { unsigned magic; diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index 6986d21..be8af1a 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -765,10 +765,10 @@ ban_check_object(struct object *o, struct vsl_log *vsl, } int -BAN_CheckObject(struct object *o, const struct sess *sp) +BAN_CheckObject(struct object *o, struct req *req) { - return (ban_check_object(o, sp->req->vsl, sp->req->http) > 0); + return (ban_check_object(o, req->vsl, req->http) > 0); } static struct ban * diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 729b91b..240d0d0 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -265,7 +265,7 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) req->obj->last_use = req->t_resp; /* XXX: locking ? */ } HTTP_Setup(req->resp, req->ws, req->vsl, HTTP_Resp); - RES_BuildHttp(sp); + RES_BuildHttp(req); assert(req->sp == sp); VCL_deliver_method(req); @@ -340,7 +340,7 @@ cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req) req->director = NULL; req->restarts = 0; - RES_WriteObj(sp); + RES_WriteObj(req); /* No point in saving the body if it is hit-for-pass */ if (req->obj->objcore->flags & OC_F_PASS) @@ -591,7 +591,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) wrk->acct_tmp.fetch++; - i = FetchHdr(sp, need_host_hdr, req->objcore->objhead == NULL); + i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL); /* * If we recycle a backend connection, there is a finite chance * that the backend closed it before we get a request to it. @@ -599,7 +599,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) */ if (i == 1) { VSC_C_main->backend_retry++; - i = FetchHdr(sp, need_host_hdr, req->objcore->objhead == NULL); + i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL); } if (i) { @@ -1016,7 +1016,7 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req) if (req->handling == VCL_RET_DELIVER) { //AZ(req->busyobj->bereq->ws); //AZ(req->busyobj->beresp->ws); - (void)FetchReqBody(sp, 0); + (void)FetchReqBody(req, 0); sp->step = STP_PREPRESP; return (0); } @@ -1173,7 +1173,7 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req) AZ(req->obj); HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); - http_FilterReq(sp, HTTPH_R_FETCH); + http_FilterReq(req, HTTPH_R_FETCH); http_ForceGet(bo->bereq); if (cache_param->http_gzip_support) { /* @@ -1251,7 +1251,7 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req) bo->vsl->wid = sp->vsl_id; bo->refcount = 2; HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); - http_FilterReq(sp, HTTPH_R_PASS); + http_FilterReq(req, HTTPH_R_PASS); assert(req->sp == sp); VCL_pass_method(req); @@ -1313,7 +1313,7 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req) bo = req->busyobj; bo->vsl->wid = sp->vsl_id; HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); - http_FilterReq(sp, 0); + http_FilterReq(req, 0); assert(req->sp == sp); VCL_pipe_method(req); @@ -1514,7 +1514,7 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) EXP_Clr(&req->exp); HTTP_Setup(req->http, req->ws, req->vsl, HTTP_Req); - req->err_code = http_DissectRequest(sp); + req->err_code = http_DissectRequest(req); /* If we could not even parse the request, just close */ if (req->err_code == 400) { diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c index 6dda80d..528a57c 100644 --- a/bin/varnishd/cache/cache_dir.c +++ b/bin/varnishd/cache/cache_dir.c @@ -105,17 +105,17 @@ VDI_RecycleFd(struct vbc **vbp) /* Get a connection --------------------------------------------------*/ struct vbc * -VDI_GetFd(const struct director *d, struct sess *sp) +VDI_GetFd(const struct director *d, struct req *req) { struct vbc *vc; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (d == NULL) - d = sp->req->director; + d = req->director; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - vc = d->getfd(d, sp); + vc = d->getfd(d, req); if (vc != NULL) { - vc->vsl = sp->req->busyobj->vsl; + vc->vsl = req->busyobj->vsl; vc->orig_vsl_id = vc->vsl->wid; vc->vsl->wid = vc->vsl_id; } @@ -130,10 +130,10 @@ VDI_GetFd(const struct director *d, struct sess *sp) */ int -VDI_Healthy(const struct director *d, const struct sess *sp) +VDI_Healthy(const struct director *d, const struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); - return (d->healthy(d, sp)); + return (d->healthy(d, req)); } diff --git a/bin/varnishd/cache/cache_dir_dns.c b/bin/varnishd/cache/cache_dir_dns.c index 068c2b9..e947f61 100644 --- a/bin/varnishd/cache/cache_dir_dns.c +++ b/bin/varnishd/cache/cache_dir_dns.c @@ -150,7 +150,7 @@ vdi_dns_pick_host(const struct sess *sp, struct vdi_dns_hostgroup *group) { current = i + initial - nhosts; else current = i + initial; - if (VDI_Healthy(group->hosts[current], sp)) { + if (VDI_Healthy(group->hosts[current], sp->req)) { group->next_host = current+1; return (group->hosts[current]); } @@ -372,31 +372,31 @@ vdi_dns_find_backend(const struct sess *sp, struct vdi_dns *vs) } static struct vbc * -vdi_dns_getfd(const struct director *director, struct sess *sp) +vdi_dns_getfd(const struct director *director, struct req *req) { struct vdi_dns *vs; struct director *dir; struct vbc *vbe; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(director, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(vs, director->priv, VDI_DNS_MAGIC); - dir = vdi_dns_find_backend(sp, vs); - if (!dir || !VDI_Healthy(dir, sp)) + dir = vdi_dns_find_backend(req->sp, vs); + if (!dir || !VDI_Healthy(dir, req)) return (NULL); - vbe = VDI_GetFd(dir, sp); + vbe = VDI_GetFd(dir, req); return (vbe); } static unsigned -vdi_dns_healthy(const struct director *dir, const struct sess *sp) +vdi_dns_healthy(const struct director *dir, const struct req *req) { /* XXX: Fooling -Werror for a bit until it's actually implemented. */ (void)dir; - (void)sp; + (void)req; return (1); /* @@ -404,11 +404,11 @@ vdi_dns_healthy(const struct director *dir, const struct sess *sp) struct director *dir; int i; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->req->director, DIRECTOR_MAGIC); - CAST_OBJ_NOTNULL(vs, sp->req->director->priv, VDI_DNS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->director, DIRECTOR_MAGIC); + CAST_OBJ_NOTNULL(vs, req->director->priv, VDI_DNS_MAGIC); - dir = vdi_dns_find_backend(sp, vs); + dir = vdi_dns_find_backend(req->sp, vs); if (dir) return (1); diff --git a/bin/varnishd/cache/cache_dir_random.c b/bin/varnishd/cache/cache_dir_random.c index 8a0bcd2..c291fe0 100644 --- a/bin/varnishd/cache/cache_dir_random.c +++ b/bin/varnishd/cache/cache_dir_random.c @@ -98,22 +98,22 @@ vdi_random_sha(const char *input, ssize_t len) * Sets up the initial seed for picking a backend according to policy. */ static double -vdi_random_init_seed(const struct vdi_random *vs, const struct sess *sp) +vdi_random_init_seed(const struct vdi_random *vs, const struct req *req) { const char *p; double retval; switch (vs->criteria) { case c_client: - if (sp->req->client_identity != NULL) - p = sp->req->client_identity; + if (req->client_identity != NULL) + p = req->client_identity; else - p = sp->addr; + p = req->sp->addr; retval = vdi_random_sha(p, strlen(p)); break; case c_hash: - AN(sp->req->digest); - retval = scalbn(vle32dec(sp->req->digest), -32); + AN(req->digest); + retval = scalbn(vle32dec(req->digest), -32); break; case c_random: default: @@ -127,7 +127,7 @@ vdi_random_init_seed(const struct vdi_random *vs, const struct sess *sp) * Find the healthy backend corresponding to the weight r [0...1[ */ static struct vbc * -vdi_random_pick_one(struct sess *sp, const struct vdi_random *vs, double r, +vdi_random_pick_one(struct req *req, const struct vdi_random *vs, double r, int retries) { double w[vs->nhosts]; @@ -154,9 +154,9 @@ vdi_random_pick_one(struct sess *sp, const struct vdi_random *vs, double r, s2 += w[i]; if (r >= s2) continue; - if (!VDI_Healthy(vs->hosts[i].backend, sp)) + if (!VDI_Healthy(vs->hosts[i].backend, req)) break; - vbc = VDI_GetFd(vs->hosts[i].backend, sp); + vbc = VDI_GetFd(vs->hosts[i].backend, req); if (vbc == NULL) break; return (vbc); @@ -181,18 +181,18 @@ vdi_random_pick_one(struct sess *sp, const struct vdi_random *vs, double r, * random by rehashing the key. */ static struct vbc * -vdi_random_getfd(const struct director *d, struct sess *sp) +vdi_random_getfd(const struct director *d, struct req *req) { struct vdi_random *vs; double r; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(vs, d->priv, VDI_RANDOM_MAGIC); - r = vdi_random_init_seed(vs, sp); + r = vdi_random_init_seed(vs, req); - return (vdi_random_pick_one(sp, vs, r, vs->retries)); + return (vdi_random_pick_one(req, vs, r, vs->retries)); } /* @@ -200,7 +200,7 @@ vdi_random_getfd(const struct director *d, struct sess *sp) * XXX: we should really have a weight param/criteria here */ static unsigned -vdi_random_healthy(const struct director *d, const struct sess *sp) +vdi_random_healthy(const struct director *d, const struct req *req) { struct vdi_random *vs; int i; @@ -209,7 +209,7 @@ vdi_random_healthy(const struct director *d, const struct sess *sp) CAST_OBJ_NOTNULL(vs, d->priv, VDI_RANDOM_MAGIC); for (i = 0; i < vs->nhosts; i++) { - if (VDI_Healthy(vs->hosts[i].backend, sp)) + if (VDI_Healthy(vs->hosts[i].backend, req)) return (1); } return (0); diff --git a/bin/varnishd/cache/cache_dir_round_robin.c b/bin/varnishd/cache/cache_dir_round_robin.c index 350f18c..68e7071 100644 --- a/bin/varnishd/cache/cache_dir_round_robin.c +++ b/bin/varnishd/cache/cache_dir_round_robin.c @@ -55,14 +55,14 @@ struct vdi_round_robin { }; static struct vbc * -vdi_round_robin_getfd(const struct director *d, struct sess *sp) +vdi_round_robin_getfd(const struct director *d, struct req *req) { int i; struct vdi_round_robin *vs; struct director *backend; struct vbc *vbe; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(vs, d->priv, VDI_ROUND_ROBIN_MAGIC); @@ -77,9 +77,9 @@ vdi_round_robin_getfd(const struct director *d, struct sess *sp) } else /* m_fallback */ { backend = vs->hosts[i].backend; } - if (!VDI_Healthy(backend, sp)) + if (!VDI_Healthy(backend, req)) continue; - vbe = VDI_GetFd(backend, sp); + vbe = VDI_GetFd(backend, req); if (vbe != NULL) return (vbe); } @@ -88,7 +88,7 @@ vdi_round_robin_getfd(const struct director *d, struct sess *sp) } static unsigned -vdi_round_robin_healthy(const struct director *d, const struct sess *sp) +vdi_round_robin_healthy(const struct director *d, const struct req *req) { struct vdi_round_robin *vs; struct director *backend; @@ -99,7 +99,7 @@ vdi_round_robin_healthy(const struct director *d, const struct sess *sp) for (i = 0; i < vs->nhosts; i++) { backend = vs->hosts[i].backend; - if (VDI_Healthy(backend, sp)) + if (VDI_Healthy(backend, req)) return (1); } return (0); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 73aac72..9199d67 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -42,7 +42,7 @@ /*--------------------------------------------------------------------*/ static void -ved_include(struct sess *sp, const char *src, const char *host) +ved_include(struct req *req, const char *src, const char *host) { struct object *obj; struct worker *wrk; @@ -50,71 +50,71 @@ ved_include(struct sess *sp, const char *src, const char *host) char *wrk_ws_wm; unsigned sxid, res_mode; - wrk = sp->wrk; + wrk = req->sp->wrk; - if (sp->req->esi_level >= cache_param->max_esi_depth) + if (req->esi_level >= cache_param->max_esi_depth) return; - sp->req->esi_level++; + req->esi_level++; (void)WRW_FlushRelease(wrk); - obj = sp->req->obj; - sp->req->obj = NULL; - res_mode = sp->req->res_mode; + obj = req->obj; + req->obj = NULL; + res_mode = req->res_mode; /* Reset request to status before we started messing with it */ - HTTP_Copy(sp->req->http, sp->req->http0); + HTTP_Copy(req->http, req->http0); /* Take a workspace snapshot */ - sp_ws_wm = WS_Snapshot(sp->req->ws); + sp_ws_wm = WS_Snapshot(req->ws); wrk_ws_wm = WS_Snapshot(wrk->aws); /* XXX ? */ - http_SetH(sp->req->http, HTTP_HDR_URL, src); + http_SetH(req->http, HTTP_HDR_URL, src); if (host != NULL && *host != '\0') { - http_Unset(sp->req->http, H_Host); - http_Unset(sp->req->http, H_If_Modified_Since); - http_SetHeader(sp->req->http, host); + http_Unset(req->http, H_Host); + http_Unset(req->http, H_If_Modified_Since); + http_SetHeader(req->http, host); } /* * XXX: We should decide if we should cache the director * XXX: or not (for session/backend coupling). Until then * XXX: make sure we don't trip up the check in vcl_recv. */ - sp->req->director = NULL; - sp->step = STP_RECV; - http_ForceGet(sp->req->http); + req->director = NULL; + req->sp->step = STP_RECV; + http_ForceGet(req->http); /* Don't do conditionals */ - sp->req->http->conds = 0; - http_Unset(sp->req->http, H_If_Modified_Since); + req->http->conds = 0; + http_Unset(req->http, H_If_Modified_Since); /* Client content already taken care of */ - http_Unset(sp->req->http, H_Content_Length); + http_Unset(req->http, H_Content_Length); - sxid = sp->req->xid; + sxid = req->xid; while (1) { - sp->wrk = wrk; - CNT_Session(sp); - if (sp->step == STP_DONE) + req->sp->wrk = wrk; + CNT_Session(req->sp); + if (req->sp->step == STP_DONE) break; - AZ(sp->wrk); - DSL(0x20, SLT_Debug, sp->vsl_id, "loop waiting for ESI"); + AZ(req->sp->wrk); + DSL(0x20, SLT_Debug, req->sp->vsl_id, "loop waiting for ESI"); (void)usleep(10000); } - sp->req->xid = sxid; - AN(sp->wrk); - assert(sp->step == STP_DONE); - sp->req->esi_level--; - sp->req->obj = obj; - sp->req->res_mode = res_mode; + req->xid = sxid; + AN(req->sp->wrk); + assert(req->sp->step == STP_DONE); + req->esi_level--; + req->obj = obj; + req->res_mode = res_mode; /* Reset the workspace */ - WS_Reset(sp->req->ws, sp_ws_wm); + WS_Reset(req->ws, sp_ws_wm); WS_Reset(wrk->aws, wrk_ws_wm); /* XXX ? */ - WRW_Reserve(sp->wrk, &sp->fd, sp->req->vsl, sp->req->t_resp); - if (sp->req->res_mode & RES_CHUNKED) - WRW_Chunked(sp->wrk); + WRW_Reserve(req->sp->wrk, &req->sp->fd, req->vsl, req->t_resp); + if (req->res_mode & RES_CHUNKED) + WRW_Chunked(req->sp->wrk); } /*--------------------------------------------------------------------*/ @@ -171,7 +171,7 @@ ved_decode_len(uint8_t **pp) */ static void -ved_pretend_gzip(const struct sess *sp, const uint8_t *p, ssize_t l) +ved_pretend_gzip(struct req *req, const uint8_t *p, ssize_t l) { uint8_t buf1[5], buf2[5]; uint16_t lx; @@ -184,22 +184,22 @@ ved_pretend_gzip(const struct sess *sp, const uint8_t *p, ssize_t l) while (l > 0) { if (l >= 65535) { lx = 65535; - (void)WRW_Write(sp->wrk, buf1, sizeof buf1); + (void)WRW_Write(req->sp->wrk, buf1, sizeof buf1); } else { lx = (uint16_t)l; buf2[0] = 0; vle16enc(buf2 + 1, lx); vle16enc(buf2 + 3, ~lx); - (void)WRW_Write(sp->wrk, buf2, sizeof buf2); + (void)WRW_Write(req->sp->wrk, buf2, sizeof buf2); } - (void)WRW_Write(sp->wrk, p, lx); - sp->req->crc = crc32(sp->req->crc, p, lx); - sp->req->l_crc += lx; + (void)WRW_Write(req->sp->wrk, p, lx); + req->crc = crc32(req->crc, p, lx); + req->l_crc += lx; l -= lx; p += lx; } /* buf2 is local, have to flush */ - (void)WRW_Flush(sp->wrk); + (void)WRW_Flush(req->sp->wrk); } /*--------------------------------------------------------------------- @@ -213,7 +213,7 @@ static const uint8_t gzip_hdr[] = { }; void -ESI_Deliver(struct sess *sp) +ESI_Deliver(struct req *req) { struct storage *st; uint8_t *p, *e, *q, *r; @@ -227,8 +227,8 @@ ESI_Deliver(struct sess *sp) const void *dp; int i; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - st = sp->req->obj->esidata; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + st = req->obj->esidata; AN(st); p = st->ptr; @@ -241,23 +241,23 @@ ESI_Deliver(struct sess *sp) isgzip = 0; } - if (sp->req->esi_level == 0) { + if (req->esi_level == 0) { /* * Only the top level document gets to decide this. */ - sp->req->gzip_resp = 0; - if (isgzip && !(sp->req->res_mode & RES_GUNZIP)) { + req->gzip_resp = 0; + if (isgzip && !(req->res_mode & RES_GUNZIP)) { assert(sizeof gzip_hdr == 10); /* Send out the gzip header */ - (void)WRW_Write(sp->wrk, gzip_hdr, 10); - sp->req->l_crc = 0; - sp->req->gzip_resp = 1; - sp->req->crc = crc32(0L, Z_NULL, 0); + (void)WRW_Write(req->sp->wrk, gzip_hdr, 10); + req->l_crc = 0; + req->gzip_resp = 1; + req->crc = crc32(0L, Z_NULL, 0); } } - if (isgzip && !sp->req->gzip_resp) { - vgz = VGZ_NewUngzip(sp->req->vsl, "U D E"); + if (isgzip && !req->gzip_resp) { + vgz = VGZ_NewUngzip(req->vsl, "U D E"); AZ(VGZ_WrwInit(vgz)); /* Feed a gzip header to gunzip to make it happy */ @@ -268,7 +268,7 @@ ESI_Deliver(struct sess *sp) assert(dl == 0); } - st = VTAILQ_FIRST(&sp->req->obj->store); + st = VTAILQ_FIRST(&req->obj->store); off = 0; while (p < e) { @@ -283,10 +283,10 @@ ESI_Deliver(struct sess *sp) l_icrc = ved_decode_len(&p); icrc = vbe32dec(p); p += 4; - if (sp->req->gzip_resp) { - sp->req->crc = crc32_combine( - sp->req->crc, icrc, l_icrc); - sp->req->l_crc += l_icrc; + if (req->gzip_resp) { + req->crc = crc32_combine( + req->crc, icrc, l_icrc); + req->l_crc += l_icrc; } } /* @@ -300,29 +300,31 @@ ESI_Deliver(struct sess *sp) l2 = st->len - off; l -= l2; - if (sp->req->gzip_resp && isgzip) { + if (req->gzip_resp && isgzip) { /* * We have a gzip'ed VEC and delivers * a gzip'ed ESI response. */ - (void)WRW_Write(sp->wrk, + (void)WRW_Write(req->sp->wrk, st->ptr + off, l2); - } else if (sp->req->gzip_resp) { + } else if (req->gzip_resp) { /* * A gzip'ed ESI response, but the VEC * was not gzip'ed. */ - ved_pretend_gzip(sp, st->ptr + off, l2); + ved_pretend_gzip(req, + st->ptr + off, l2); } else if (isgzip) { /* * A gzip'ed VEC, but ungzip'ed ESI * response */ AN(vgz); - i = VGZ_WrwGunzip(sp->wrk, vgz, + i = VGZ_WrwGunzip(req->sp->wrk, vgz, st->ptr + off, l2); - if (WRW_Error(sp->wrk)) { - SES_Close(sp, "remote closed"); + if (WRW_Error(req->sp->wrk)) { + SES_Close(req->sp, + "remote closed"); p = e; break; } @@ -331,7 +333,7 @@ ESI_Deliver(struct sess *sp) /* * Ungzip'ed VEC, ungzip'ed ESI response */ - (void)WRW_Write(sp->wrk, + (void)WRW_Write(req->sp->wrk, st->ptr + off, l2); } off += l2; @@ -371,14 +373,14 @@ ESI_Deliver(struct sess *sp) r = (void*)strchr((const char*)q, '\0'); AN(r); if (vgz != NULL) - VGZ_WrwFlush(sp->wrk, vgz); - if (WRW_Flush(sp->wrk)) { - SES_Close(sp, "remote closed"); + VGZ_WrwFlush(req->sp->wrk, vgz); + if (WRW_Flush(req->sp->wrk)) { + SES_Close(req->sp, "remote closed"); p = e; break; } Debug("INCL [%s][%s] BEGIN\n", q, p); - ved_include(sp, (const char*)q, (const char*)p); + ved_include(req, (const char*)q, (const char*)p); Debug("INCL [%s][%s] END\n", q, p); p = r + 1; break; @@ -388,10 +390,10 @@ ESI_Deliver(struct sess *sp) } } if (vgz != NULL) { - VGZ_WrwFlush(sp->wrk, vgz); + VGZ_WrwFlush(req->sp->wrk, vgz); (void)VGZ_Destroy(&vgz); } - if (sp->req->gzip_resp && sp->req->esi_level == 0) { + if (req->gzip_resp && req->esi_level == 0) { /* Emit a gzip literal block with finish bit set */ tailbuf[0] = 0x01; tailbuf[1] = 0x00; @@ -400,14 +402,14 @@ ESI_Deliver(struct sess *sp) tailbuf[4] = 0xff; /* Emit CRC32 */ - vle32enc(tailbuf + 5, sp->req->crc); + vle32enc(tailbuf + 5, req->crc); /* MOD(2^32) length */ - vle32enc(tailbuf + 9, sp->req->l_crc); + vle32enc(tailbuf + 9, req->l_crc); - (void)WRW_Write(sp->wrk, tailbuf, 13); + (void)WRW_Write(req->sp->wrk, tailbuf, 13); } - (void)WRW_Flush(sp->wrk); + (void)WRW_Flush(req->sp->wrk); } /*--------------------------------------------------------------------- @@ -464,7 +466,7 @@ ESI_DeliverChild(const struct sess *sp) if (!sp->req->obj->gziped) { VTAILQ_FOREACH(st, &sp->req->obj->store, list) - ved_pretend_gzip(sp, st->ptr, st->len); + ved_pretend_gzip(sp->req, st->ptr, st->len); return; } /* diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 7ccba0a..129a0a7 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -378,20 +378,20 @@ fetch_eof(struct busyobj *bo, struct http_conn *htc) */ int -FetchReqBody(const struct sess *sp, int sendbody) +FetchReqBody(struct req *req, int sendbody) { unsigned long content_length; char buf[8192]; char *ptr, *endp; int rdcnt; - if (sp->req->reqbodydone) { + if (req->reqbodydone) { AZ(sendbody); return (0); } - if (http_GetHdr(sp->req->http, H_Content_Length, &ptr)) { - sp->req->reqbodydone = 1; + if (http_GetHdr(req->http, H_Content_Length, &ptr)) { + req->reqbodydone = 1; content_length = strtoul(ptr, &endp, 10); /* XXX should check result of conversion */ @@ -400,21 +400,21 @@ FetchReqBody(const struct sess *sp, int sendbody) rdcnt = sizeof buf; else rdcnt = content_length; - rdcnt = HTC_Read(sp->req->htc, buf, rdcnt); + rdcnt = HTC_Read(req->htc, buf, rdcnt); if (rdcnt <= 0) return (1); content_length -= rdcnt; if (sendbody) { /* XXX: stats ? */ - (void)WRW_Write(sp->wrk, buf, rdcnt); - if (WRW_Flush(sp->wrk)) + (void)WRW_Write(req->sp->wrk, buf, rdcnt); + if (WRW_Flush(req->sp->wrk)) return (2); } } } - if (http_GetHdr(sp->req->http, H_Transfer_Encoding, NULL)) { + if (http_GetHdr(req->http, H_Transfer_Encoding, NULL)) { /* XXX: Handle chunked encoding. */ - VSLb(sp->req->vsl, SLT_Debug, "Transfer-Encoding in request"); + VSLb(req->vsl, SLT_Debug, "Transfer-Encoding in request"); return (1); } return (0); @@ -431,20 +431,18 @@ FetchReqBody(const struct sess *sp, int sendbody) */ int -FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) +FetchHdr(struct req *req, int need_host_hdr, int sendbody) { struct vbc *vc; struct worker *wrk; - struct req *req; struct busyobj *bo; struct http *hp; int retry = -1; int i; struct http_conn *htc; - wrk = sp->wrk; + wrk = req->sp->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - req = sp->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); bo = req->busyobj; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); @@ -458,7 +456,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) hp = bo->bereq; - bo->vbc = VDI_GetFd(NULL, sp); + bo->vbc = VDI_GetFd(NULL, req); if (bo->vbc == NULL) { VSLb(req->vsl, SLT_FetchError, "no backend connection"); return (-1); @@ -480,7 +478,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody) (void)http_Write(wrk, hp, 0); /* XXX: stats ? */ /* Deal with any message-body the request might have */ - i = FetchReqBody(sp, sendbody); + i = FetchReqBody(req, sendbody); if (WRW_FlushRelease(wrk) || i > 0) { VSLb(req->vsl, SLT_FetchError, "backend write error: %d (%s)", diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index bfb030d..f2d5be3 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -358,7 +358,7 @@ HSH_Lookup(struct sess *sp) if (o->exp.ttl <= 0.) continue; - if (BAN_CheckObject(o, sp)) + if (BAN_CheckObject(o, req)) continue; if (o->vary != NULL && !VRY_Match(req, o->vary)) continue; @@ -391,7 +391,7 @@ HSH_Lookup(struct sess *sp) if (oc == NULL /* We found no live object */ && grace_oc != NULL /* There is a grace candidate */ && (busy_found /* Somebody else is already busy */ - || !VDI_Healthy(req->director, sp))) { + || !VDI_Healthy(req->director, req))) { /* Or it is impossible to fetch */ o = oc_getobj(&wrk->stats, grace_oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 0380469..411b2d3 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -651,22 +651,22 @@ http_ProtoVer(struct http *hp) /*--------------------------------------------------------------------*/ uint16_t -http_DissectRequest(const struct sess *sp) +http_DissectRequest(struct req *req) { struct http_conn *htc; struct http *hp; uint16_t retval; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - htc = sp->req->htc; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + htc = req->htc; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); - hp = sp->req->http; + hp = req->http; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); retval = http_splitline(hp, htc, HTTP_HDR_REQ, HTTP_HDR_URL, HTTP_HDR_PROTO); if (retval != 0) { - VSLbt(sp->req->vsl, SLT_HttpGarbage, htc->rxbuf); + VSLbt(req->vsl, SLT_HttpGarbage, htc->rxbuf); return (retval); } http_ProtoVer(hp); @@ -838,21 +838,21 @@ http_filterfields(struct http *to, const struct http *fm, unsigned how) /*--------------------------------------------------------------------*/ void -http_FilterReq(const struct sess *sp, unsigned how) +http_FilterReq(const struct req *req, unsigned how) { struct http *hp; - hp = sp->req->busyobj->bereq; + hp = req->busyobj->bereq; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); - http_linkh(hp, sp->req->http, HTTP_HDR_REQ); - http_linkh(hp, sp->req->http, HTTP_HDR_URL); + http_linkh(hp, req->http, HTTP_HDR_REQ); + http_linkh(hp, req->http, HTTP_HDR_URL); if (how == HTTPH_R_FETCH) http_SetH(hp, HTTP_HDR_PROTO, "HTTP/1.1"); else - http_linkh(hp, sp->req->http, HTTP_HDR_PROTO); - http_filterfields(hp, sp->req->http, how); - http_PrintfHeader(hp, "X-Varnish: %u", sp->req->xid); + http_linkh(hp, req->http, HTTP_HDR_PROTO); + http_filterfields(hp, req->http, how); + http_PrintfHeader(hp, "X-Varnish: %u", req->xid); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c index fda21f4..77ffcda 100644 --- a/bin/varnishd/cache/cache_pipe.c +++ b/bin/varnishd/cache/cache_pipe.c @@ -74,7 +74,7 @@ PipeSession(struct sess *sp) CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); wrk = sp->wrk; - vc = VDI_GetFd(NULL, sp); + vc = VDI_GetFd(NULL, sp->req); if (vc == NULL) return; bo->vbc = vc; /* For panic dumping */ diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index 700e791..59aa6cb 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -37,12 +37,10 @@ /*--------------------------------------------------------------------*/ static void -res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh) +res_dorange(const struct req *req, const char *r, ssize_t *plow, ssize_t *phigh) { ssize_t low, high, has_low; - struct req *req; - req = sp->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); assert(req->obj->response == 200); if (strncmp(r, "bytes=", 6)) @@ -105,13 +103,10 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh) /*--------------------------------------------------------------------*/ void -RES_BuildHttp(const struct sess *sp) +RES_BuildHttp(struct req *req) { char time_str[30]; - struct req *req; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - req = sp->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); http_ClrHeader(req->resp); @@ -188,17 +183,17 @@ res_WriteGunzipObj(const struct sess *sp) /*--------------------------------------------------------------------*/ static void -res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high) +res_WriteDirObj(const struct req *req, ssize_t low, ssize_t high) { ssize_t u = 0; size_t ptr, off, len; struct storage *st; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); ptr = 0; - VTAILQ_FOREACH(st, &sp->req->obj->store, list) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + VTAILQ_FOREACH(st, &req->obj->store, list) { + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); u += st->len; len = st->len; @@ -220,10 +215,10 @@ res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high) ptr += len; - sp->wrk->acct_tmp.bodybytes += len; - (void)WRW_Write(sp->wrk, st->ptr + off, len); + req->sp->wrk->acct_tmp.bodybytes += len; + (void)WRW_Write(req->sp->wrk, st->ptr + off, len); } - assert(u == sp->req->obj->len); + assert(u == req->obj->len); } /*-------------------------------------------------------------------- @@ -232,14 +227,11 @@ res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high) */ void -RES_WriteObj(struct sess *sp) +RES_WriteObj(struct req *req) { char *r; ssize_t low, high; - struct req *req; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - req = sp->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); /* @@ -254,44 +246,44 @@ RES_WriteObj(struct sess *sp) cache_param->http_range_support && req->obj->response == 200 && http_GetHdr(req->http, H_Range, &r)) - res_dorange(sp, r, &low, &high); + res_dorange(req, r, &low, &high); - WRW_Reserve(sp->wrk, &sp->fd, sp->req->vsl, sp->req->t_resp); + WRW_Reserve(req->sp->wrk, &req->sp->fd, req->vsl, req->t_resp); /* * Send HTTP protocol header, unless interior ESI object */ if (!(req->res_mode & RES_ESI_CHILD)) - sp->wrk->acct_tmp.hdrbytes += - http_Write(sp->wrk, req->resp, 1); + req->sp->wrk->acct_tmp.hdrbytes += + http_Write(req->sp->wrk, req->resp, 1); if (!req->wantbody) req->res_mode &= ~RES_CHUNKED; if (req->res_mode & RES_CHUNKED) - WRW_Chunked(sp->wrk); + WRW_Chunked(req->sp->wrk); if (!req->wantbody) { /* This was a HEAD or conditional request */ } else if (req->obj->len == 0) { /* Nothing to do here */ } else if (req->res_mode & RES_ESI) { - ESI_Deliver(sp); + ESI_Deliver(req); } else if (req->res_mode & RES_ESI_CHILD && req->gzip_resp) { - ESI_DeliverChild(sp); + ESI_DeliverChild(req->sp); } else if (req->res_mode & RES_ESI_CHILD && !req->gzip_resp && req->obj->gziped) { - res_WriteGunzipObj(sp); + res_WriteGunzipObj(req->sp); } else if (req->res_mode & RES_GUNZIP) { - res_WriteGunzipObj(sp); + res_WriteGunzipObj(req->sp); } else { - res_WriteDirObj(sp, low, high); + res_WriteDirObj(req, low, high); } if (req->res_mode & RES_CHUNKED && !(req->res_mode & RES_ESI_CHILD)) - WRW_EndChunk(sp->wrk); + WRW_EndChunk(req->sp->wrk); - if (WRW_FlushRelease(sp->wrk) && sp->fd >= 0) - SES_Close(sp, "remote closed"); + if (WRW_FlushRelease(req->sp->wrk) && req->sp->fd >= 0) + SES_Close(req->sp, "remote closed"); } diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 36ff4c6..aedbc98 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -556,6 +556,6 @@ VRT_r_req_backend_healthy(const struct req *req) { CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->director, DIRECTOR_MAGIC); - return (VDI_Healthy(req->director, req->sp)); + return (VDI_Healthy(req->director, req)); } From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] 7ac1238 More s/sp/req/ work Message-ID: commit 7ac1238ef00a50511456c6c7cfb9b7d3b71f7ec5 Author: Poul-Henning Kamp Date: Tue Jun 19 09:22:45 2012 +0000 More s/sp/req/ work diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 845fac6..5823943 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -893,7 +893,7 @@ void MPL_Free(struct mempool *mpl, void *item); void PAN_Init(void); /* cache_pipe.c */ -void PipeSession(struct sess *sp); +void PipeRequest(struct req *req); /* cache_pool.c */ void Pool_Init(void); @@ -980,7 +980,7 @@ char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap); char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap); void ESI_Deliver(struct req *); -void ESI_DeliverChild(const struct sess *); +void ESI_DeliverChild(struct req *); /* cache_vrt_vmod.c */ void VMOD_Init(void); diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index e640a30..7b3101b 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -87,10 +87,10 @@ VBE_ReleaseConn(struct vbc *vc) MPL_Free(vbcpool, vc); } -#define FIND_TMO(tmx, dst, sp, be) \ +#define FIND_TMO(tmx, dst, req, be) \ do { \ - CHECK_OBJ_NOTNULL(sp->req->busyobj, BUSYOBJ_MAGIC); \ - dst = sp->req->busyobj->tmx; \ + CHECK_OBJ_NOTNULL(req->busyobj, BUSYOBJ_MAGIC); \ + dst = req->busyobj->tmx; \ if (dst == 0.0) \ dst = be->tmx; \ if (dst == 0.0) \ @@ -107,20 +107,20 @@ VBE_ReleaseConn(struct vbc *vc) */ static int -vbe_TryConnect(const struct sess *sp, int pf, const struct sockaddr_storage *sa, +vbe_TryConnect(const struct req *req, int pf, const struct sockaddr_storage *sa, socklen_t salen, const struct vdi_simple *vs) { int s, i, tmo; double tmod; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC); s = socket(pf, SOCK_STREAM, 0); if (s < 0) return (s); - FIND_TMO(connect_timeout, tmod, sp, vs->vrt); + FIND_TMO(connect_timeout, tmod, req, vs->vrt); tmo = (int)(tmod * 1000.0); @@ -137,7 +137,7 @@ vbe_TryConnect(const struct sess *sp, int pf, const struct sockaddr_storage *sa, /*--------------------------------------------------------------------*/ static void -bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs) +bes_conn_try(struct req *req, struct vbc *vc, const struct vdi_simple *vs) { int s; struct backend *bp = vs->backend; @@ -157,17 +157,17 @@ bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs) /* release lock during stuff that can take a long time */ if (cache_param->prefer_ipv6 && bp->ipv6 != NULL) { - s = vbe_TryConnect(sp, PF_INET6, bp->ipv6, bp->ipv6len, vs); + s = vbe_TryConnect(req, PF_INET6, bp->ipv6, bp->ipv6len, vs); vc->addr = bp->ipv6; vc->addrlen = bp->ipv6len; } if (s == -1 && bp->ipv4 != NULL) { - s = vbe_TryConnect(sp, PF_INET, bp->ipv4, bp->ipv4len, vs); + s = vbe_TryConnect(req, PF_INET, bp->ipv4, bp->ipv4len, vs); vc->addr = bp->ipv4; vc->addrlen = bp->ipv4len; } if (s == -1 && !cache_param->prefer_ipv6 && bp->ipv6 != NULL) { - s = vbe_TryConnect(sp, PF_INET6, bp->ipv6, bp->ipv6len, vs); + s = vbe_TryConnect(req, PF_INET6, bp->ipv6, bp->ipv6len, vs); vc->addr = bp->ipv6; vc->addrlen = bp->ipv6len; } @@ -183,7 +183,7 @@ bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs) } else { vc->vsl_id = s | VSL_BACKENDMARKER; VTCP_myname(s, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); - VSLb(sp->req->vsl, SLT_BackendOpen, "%d %s %s %s ", + VSLb(req->vsl, SLT_BackendOpen, "%d %s %s %s ", vc->fd, vs->backend->display_name, abuf1, pbuf1); } @@ -226,7 +226,7 @@ vbe_NewConn(void) /*-------------------------------------------------------------------- * It evaluates if a backend is healthy _for_a_specific_object_. - * That means that it relies on sp->req->objcore->objhead. This is mainly for + * That means that it relies on req->objcore->objhead. This is mainly for * saint-mode, but also takes backend->healthy into account. If * cache_param->saintmode_threshold is 0, this is basically just a test of * backend->healthy. @@ -236,7 +236,7 @@ vbe_NewConn(void) */ static unsigned int -vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp) +vbe_Healthy(const struct vdi_simple *vs, const struct req *req) { struct trouble *tr; struct trouble *tr2; @@ -246,7 +246,7 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp) struct backend *backend; double now; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC); backend = vs->backend; CHECK_OBJ_NOTNULL(backend, BACKEND_MAGIC); @@ -271,7 +271,7 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp) if (threshold == 0 || VTAILQ_EMPTY(&backend->troublelist)) return (1); - now = sp->req->t_req; + now = req->t_req; old = NULL; retval = 1; @@ -286,7 +286,7 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp) break; } - if (!memcmp(tr->digest, sp->req->digest, sizeof tr->digest)) { + if (!memcmp(tr->digest, req->digest, sizeof tr->digest)) { retval = 0; break; } @@ -313,12 +313,12 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp) */ static struct vbc * -vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs) +vbe_GetVbe(struct req *req, struct vdi_simple *vs) { struct vbc *vc; struct backend *bp; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC); bp = vs->backend; CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC); @@ -340,20 +340,20 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs) if (vbe_CheckFd(vc->fd)) { /* XXX locking of stats */ VSC_C_main->backend_reuse += 1; - VSLb(sp->req->vsl, SLT_Backend, "%d %s %s", - vc->fd, sp->req->director->vcl_name, + VSLb(req->vsl, SLT_Backend, "%d %s %s", + vc->fd, req->director->vcl_name, bp->display_name); vc->vdis = vs; vc->recycled = 1; return (vc); } VSC_C_main->backend_toolate++; - VSLb(sp->req->vsl, SLT_BackendClose, "%d %s toolate", + VSLb(req->vsl, SLT_BackendClose, "%d %s toolate", vc->fd, bp->display_name); /* Checkpoint log to flush all info related to this connection before the OS reuses the FD */ - VSL_Flush(sp->req->vsl, 0); + VSL_Flush(req->vsl, 0); VTCP_close(&vc->fd); VBE_DropRefConn(bp); @@ -361,7 +361,7 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs) VBE_ReleaseConn(vc); } - if (!vbe_Healthy(vs, sp)) { + if (!vbe_Healthy(vs, req)) { VSC_C_main->backend_unhealthy++; return (NULL); } @@ -375,7 +375,7 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs) vc = vbe_NewConn(); assert(vc->fd == -1); AZ(vc->backend); - bes_conn_try(sp, vc, vs); + bes_conn_try(req, vc, vs); if (vc->fd < 0) { VBE_ReleaseConn(vc); VSC_C_main->backend_fail++; @@ -383,8 +383,8 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs) } vc->backend = bp; VSC_C_main->backend_conn++; - VSLb(sp->req->vsl, SLT_Backend, "%d %s %s", - vc->fd, sp->req->director->vcl_name, bp->display_name); + VSLb(req->vsl, SLT_Backend, "%d %s %s", + vc->fd, req->director->vcl_name, bp->display_name); vc->vdis = vs; return (vc); } @@ -460,12 +460,12 @@ vdi_simple_getfd(const struct director *d, struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC); - vc = vbe_GetVbe(req->sp, vs); + vc = vbe_GetVbe(req, vs); if (vc != NULL) { FIND_TMO(first_byte_timeout, - vc->first_byte_timeout, req->sp, vs->vrt); + vc->first_byte_timeout, req, vs->vrt); FIND_TMO(between_bytes_timeout, - vc->between_bytes_timeout, req->sp, vs->vrt); + vc->between_bytes_timeout, req, vs->vrt); } return (vc); } @@ -477,7 +477,7 @@ vdi_simple_healthy(const struct director *d, const struct req *req) CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC); - return (vbe_Healthy(vs, req->sp)); + return (vbe_Healthy(vs, req)); } static void diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 240d0d0..88c269b 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -407,14 +407,14 @@ cnt_done(struct sess *sp, struct worker *wrk, struct req *req) /* XXX: Add StatReq == StatSess */ /* XXX: Workaround for pipe */ if (sp->fd >= 0) { - VSLb(sp->req->vsl, SLT_Length, "%ju", + VSLb(req->vsl, SLT_Length, "%ju", (uintmax_t)req->req_bodybytes); } - VSLb(sp->req->vsl, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f", + VSLb(req->vsl, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f", req->xid, req->t_req, sp->t_idle, dh, dp, da); } req->xid = 0; - VSL_Flush(sp->req->vsl, 0); + VSL_Flush(req->vsl, 0); req->t_req = NAN; req->t_resp = NAN; @@ -532,7 +532,7 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) if (req->handling == VCL_RET_RESTART && req->restarts < cache_param->max_restarts) { - HSH_Drop(wrk, &sp->req->obj); + HSH_Drop(wrk, &req->obj); VBO_DerefBusyObj(wrk, &req->busyobj); sp->step = STP_RESTART; return (0); @@ -628,7 +628,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) */ EXP_Clr(&bo->exp); bo->exp.entered = W_TIM_real(wrk); - RFC2616_Ttl(bo, sp->req->xid); + RFC2616_Ttl(bo, req->xid); /* pass from vclrecv{} has negative TTL */ if (req->objcore->objhead == NULL) @@ -1077,7 +1077,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) VRY_Prep(req); AZ(req->objcore); - oc = HSH_Lookup(sp); + oc = HSH_Lookup(req); if (oc == NULL) { /* * We lost the session to a busy object, disembark the @@ -1131,7 +1131,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) if (oc->flags & OC_F_PASS) { wrk->stats.cache_hitpass++; - VSLb(sp->req->vsl, SLT_HitPass, "%u", req->obj->xid); + VSLb(req->vsl, SLT_HitPass, "%u", req->obj->xid); (void)HSH_Deref(&wrk->stats, NULL, &req->obj); AZ(req->objcore); sp->step = STP_PASS; @@ -1139,7 +1139,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) } wrk->stats.cache_hit++; - VSLb(sp->req->vsl, SLT_Hit, "%u", req->obj->xid); + VSLb(req->vsl, SLT_Hit, "%u", req->obj->xid); sp->step = STP_HIT; return (0); } @@ -1323,7 +1323,7 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req) INCOMPL(); assert(req->handling == VCL_RET_PIPE); - PipeSession(sp); + PipeRequest(req); assert(WRW_IsReleased(wrk)); http_Teardown(bo->bereq); VBO_DerefBusyObj(wrk, &req->busyobj); @@ -1503,7 +1503,7 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) /* Assign XID and log */ req->xid = ++xids; /* XXX not locked */ - VSLb(sp->req->vsl, SLT_ReqStart, "%s %s %u", + VSLb(req->vsl, SLT_ReqStart, "%s %s %u", sp->addr, sp->port, req->xid); /* Borrow VCL reference from worker thread */ diff --git a/bin/varnishd/cache/cache_dir_dns.c b/bin/varnishd/cache/cache_dir_dns.c index e947f61..d4f6772 100644 --- a/bin/varnishd/cache/cache_dir_dns.c +++ b/bin/varnishd/cache/cache_dir_dns.c @@ -135,8 +135,10 @@ vdi_dns_comp_addrinfo(const struct director *dir, * healthy ones. */ static struct director * -vdi_dns_pick_host(const struct sess *sp, struct vdi_dns_hostgroup *group) { +vdi_dns_pick_host(const struct req *req, struct vdi_dns_hostgroup *group) +{ int initial, i, nhosts, current; + if (group->nhosts == 0) return (NULL); // In case of error. if (group->next_host >= group->nhosts) @@ -150,7 +152,7 @@ vdi_dns_pick_host(const struct sess *sp, struct vdi_dns_hostgroup *group) { current = i + initial - nhosts; else current = i + initial; - if (VDI_Healthy(group->hosts[current], sp->req)) { + if (VDI_Healthy(group->hosts[current], req)) { group->next_host = current+1; return (group->hosts[current]); } @@ -192,24 +194,22 @@ vdi_dns_groupmatch(const struct vdi_dns_hostgroup *group, const char *hostname) * and freed. */ static int -vdi_dns_cache_has(const struct sess *sp, - struct vdi_dns *vs, - const char *hostname, - struct director **backend, - int rwlock) +vdi_dns_cache_has(const struct req *req, struct vdi_dns *vs, + const char *hostname, struct director **backend, int rwlock) { struct director *ret; struct vdi_dns_hostgroup *hostgr; struct vdi_dns_hostgroup *hostgr2; + VTAILQ_FOREACH_SAFE(hostgr, &vs->cachelist, list, hostgr2) { CHECK_OBJ_NOTNULL(hostgr, VDI_DNSDIR_MAGIC); - if (hostgr->ttl <= sp->req->t_req) { + if (hostgr->ttl <= req->t_req) { if (rwlock) vdi_dns_pop_cache(vs, hostgr); return (0); } if (vdi_dns_groupmatch(hostgr, hostname)) { - ret = (vdi_dns_pick_host(sp, hostgr)); + ret = (vdi_dns_pick_host(req, hostgr)); *backend = ret; if (*backend != NULL) CHECK_OBJ_NOTNULL(*backend, DIRECTOR_MAGIC); @@ -223,17 +223,17 @@ vdi_dns_cache_has(const struct sess *sp, * (Sorry for the list_add/_add confusion...) */ static void -vdi_dns_cache_list_add(const struct sess *sp, - struct vdi_dns *vs, - struct vdi_dns_hostgroup *new) +vdi_dns_cache_list_add(const struct req *req, struct vdi_dns *vs, + struct vdi_dns_hostgroup *new) { + if (vs->ncachelist >= VDI_DNS_MAX_CACHE) { VSC_C_main->dir_dns_cache_full++; vdi_dns_pop_cache(vs, NULL); } CHECK_OBJ_NOTNULL(new, VDI_DNSDIR_MAGIC); assert(new->hostname != 0); - new->ttl = sp->req->t_req + vs->ttl; + new->ttl = req->t_req + vs->ttl; VTAILQ_INSERT_HEAD(&vs->cachelist, new, list); vs->ncachelist++; } @@ -243,10 +243,8 @@ vdi_dns_cache_list_add(const struct sess *sp, * cache_has() afterwards to do multiple dns lookups in parallel... */ static int -vdi_dns_cache_add(const struct sess *sp, - struct vdi_dns *vs, - const char *hostname, - struct director **backend) +vdi_dns_cache_add(const struct req *req, struct vdi_dns *vs, + const char *hostname, struct director **backend) { int error, i, host = 0; struct addrinfo *res0, *res, hint; @@ -258,7 +256,7 @@ vdi_dns_cache_add(const struct sess *sp, * unique names or something equally troublesome). */ - if (vdi_dns_cache_has(sp, vs, hostname, backend, 1)) + if (vdi_dns_cache_has(req, vs, hostname, backend, 1)) return (1); memset(&hint, 0, sizeof hint); @@ -273,7 +271,7 @@ vdi_dns_cache_add(const struct sess *sp, error = getaddrinfo(hostname, "80", &hint, &res0); VSC_C_main->dir_dns_lookups++; if (error) { - vdi_dns_cache_list_add(sp, vs, new); + vdi_dns_cache_list_add(req, vs, new); VSC_C_main->dir_dns_failed++; return (0); } @@ -297,8 +295,8 @@ vdi_dns_cache_add(const struct sess *sp, freeaddrinfo(res0); new->nhosts = host; - vdi_dns_cache_list_add(sp, vs, new); - *backend = vdi_dns_pick_host(sp, new); + vdi_dns_cache_list_add(req, vs, new); + *backend = vdi_dns_pick_host(req, new); return (1); } @@ -308,15 +306,14 @@ vdi_dns_cache_add(const struct sess *sp, * Returns a backend or NULL. */ static struct director * -vdi_dns_walk_cache(const struct sess *sp, - struct vdi_dns *vs, - const char *hostname) +vdi_dns_walk_cache(const struct req *req, struct vdi_dns *vs, + const char *hostname) { struct director *backend = NULL; int ret; AZ(pthread_rwlock_rdlock(&vs->rwlock)); - ret = vdi_dns_cache_has(sp, vs, hostname, &backend, 0); + ret = vdi_dns_cache_has(req, vs, hostname, &backend, 0); AZ(pthread_rwlock_unlock(&vs->rwlock)); if (!ret) { /* @@ -325,7 +322,7 @@ vdi_dns_walk_cache(const struct sess *sp, * XXX: Should 'ret' be checked for that ? */ AZ(pthread_rwlock_wrlock(&vs->rwlock)); - ret = vdi_dns_cache_add(sp, vs, hostname, &backend); + ret = vdi_dns_cache_add(req, vs, hostname, &backend); AZ(pthread_rwlock_unlock(&vs->rwlock)); } else VSC_C_main->dir_dns_hit++; @@ -339,7 +336,7 @@ vdi_dns_walk_cache(const struct sess *sp, /* Parses the Host:-header and heads out to find a backend. */ static struct director * -vdi_dns_find_backend(const struct sess *sp, struct vdi_dns *vs) +vdi_dns_find_backend(const struct req *req, struct vdi_dns *vs) { struct director *ret; struct http *hp; @@ -349,10 +346,10 @@ vdi_dns_find_backend(const struct sess *sp, struct vdi_dns *vs) /* bereq is only present after recv et. al, otherwise use req (ie: * use req for health checks in vcl_recv and such). */ - if (sp->req->busyobj != NULL && sp->req->busyobj->bereq) - hp = sp->req->busyobj->bereq; + if (req->busyobj != NULL && req->busyobj->bereq) + hp = req->busyobj->bereq; else - hp = sp->req->http; + hp = req->http; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); @@ -367,7 +364,7 @@ vdi_dns_find_backend(const struct sess *sp, struct vdi_dns *vs) bprintf(hostname, "%.*s%s", (int)(q - p), p, vs->suffix ? vs->suffix : ""); - ret = vdi_dns_walk_cache(sp, vs, hostname); + ret = vdi_dns_walk_cache(req, vs, hostname); return (ret); } @@ -382,7 +379,7 @@ vdi_dns_getfd(const struct director *director, struct req *req) CHECK_OBJ_NOTNULL(director, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(vs, director->priv, VDI_DNS_MAGIC); - dir = vdi_dns_find_backend(req->sp, vs); + dir = vdi_dns_find_backend(req, vs); if (!dir || !VDI_Healthy(dir, req)) return (NULL); @@ -408,7 +405,7 @@ vdi_dns_healthy(const struct director *dir, const struct req *req) CHECK_OBJ_NOTNULL(req->director, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(vs, req->director->priv, VDI_DNS_MAGIC); - dir = vdi_dns_find_backend(req->sp, vs); + dir = vdi_dns_find_backend(req, vs); if (dir) return (1); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 9199d67..6aed2c3 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -417,14 +417,14 @@ ESI_Deliver(struct req *req) */ static uint8_t -ved_deliver_byterange(const struct sess *sp, ssize_t low, ssize_t high) +ved_deliver_byterange(const struct req *req, ssize_t low, ssize_t high) { struct storage *st; ssize_t l, lx; u_char *p; lx = 0; - VTAILQ_FOREACH(st, &sp->req->obj->store, list) { + VTAILQ_FOREACH(st, &req->obj->store, list) { p = st->ptr; l = st->len; if (lx + l < low) { @@ -443,7 +443,7 @@ ved_deliver_byterange(const struct sess *sp, ssize_t low, ssize_t high) l = high - lx; assert(lx >= low && lx + l <= high); if (l != 0) - (void)WRW_Write(sp->wrk, p, l); + (void)WRW_Write(req->sp->wrk, p, l); if (p + l < st->ptr + st->len) return(p[l]); lx += l; @@ -452,7 +452,7 @@ ved_deliver_byterange(const struct sess *sp, ssize_t low, ssize_t high) } void -ESI_DeliverChild(const struct sess *sp) +ESI_DeliverChild(struct req *req) { struct storage *st; struct object *obj; @@ -464,9 +464,10 @@ ESI_DeliverChild(const struct sess *sp) int i, j; uint8_t tailbuf[8]; - if (!sp->req->obj->gziped) { - VTAILQ_FOREACH(st, &sp->req->obj->store, list) - ved_pretend_gzip(sp->req, st->ptr, st->len); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + if (!req->obj->gziped) { + VTAILQ_FOREACH(st, &req->obj->store, list) + ved_pretend_gzip(req, st->ptr, st->len); return; } /* @@ -475,9 +476,9 @@ ESI_DeliverChild(const struct sess *sp) * padding it, as necessary, to a byte boundary. */ - dbits = (void*)WS_Alloc(sp->req->ws, 8); + dbits = (void*)WS_Alloc(req->ws, 8); AN(dbits); - obj = sp->req->obj; + obj = req->obj; CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC); start = obj->gzip_start; last = obj->gzip_last; @@ -495,10 +496,10 @@ ESI_DeliverChild(const struct sess *sp) * XXX: optimize for the case where the 'last' * XXX: bit is in a empty copy block */ - *dbits = ved_deliver_byterange(sp, start/8, last/8); + *dbits = ved_deliver_byterange(req, start/8, last/8); *dbits &= ~(1U << (last & 7)); - (void)WRW_Write(sp->wrk, dbits, 1); - cc = ved_deliver_byterange(sp, 1 + last/8, stop/8); + (void)WRW_Write(req->sp->wrk, dbits, 1); + cc = ved_deliver_byterange(req, 1 + last/8, stop/8); switch((int)(stop & 7)) { case 0: /* xxxxxxxx */ /* I think we have an off by one here, but that's OK */ @@ -541,10 +542,10 @@ ESI_DeliverChild(const struct sess *sp) INCOMPL(); } if (lpad > 0) - (void)WRW_Write(sp->wrk, dbits + 1, lpad); + (void)WRW_Write(req->sp->wrk, dbits + 1, lpad); /* We need the entire tail, but it may not be in one storage segment */ - st = VTAILQ_LAST(&sp->req->obj->store, storagehead); + st = VTAILQ_LAST(&req->obj->store, storagehead); for (i = sizeof tailbuf; i > 0; i -= j) { j = st->len; if (j > i) @@ -556,6 +557,6 @@ ESI_DeliverChild(const struct sess *sp) icrc = vle32dec(tailbuf); ilen = vle32dec(tailbuf + 4); - sp->req->crc = crc32_combine(sp->req->crc, icrc, ilen); - sp->req->l_crc += ilen; + req->crc = crc32_combine(req->crc, icrc, ilen); + req->l_crc += ilen; } diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 129a0a7..0e2fa5f 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -48,7 +48,7 @@ static unsigned fetchfrag; * We want to issue the first error we encounter on fetching and * supress the rest. This function does that. * - * Other code is allowed to look at sp->req->busyobj->fetch_failed to bail out + * Other code is allowed to look at busyobj->fetch_failed to bail out * * For convenience, always return -1 */ diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 80ce07b..e29593d 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -198,8 +198,7 @@ VGZ_ObufFull(const struct vgz *vg) } /*-------------------------------------------------------------------- - * Keep the outbuffer supplied with storage and file it under the - * sp->req->obj as it fills. + * Keep the outbuffer supplied with storage */ int diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index f2d5be3..68e4cda 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -288,22 +288,19 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc) */ struct objcore * -HSH_Lookup(struct sess *sp) +HSH_Lookup(struct req *req) { struct worker *wrk; struct objhead *oh; struct objcore *oc; struct objcore *grace_oc; struct object *o; - struct req *req; double grace_ttl; int busy_found; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - wrk = sp->wrk; - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - req = sp->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + wrk = req->sp->wrk; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req->http, HTTP_MAGIC); AN(req->director); AN(hash); @@ -421,19 +418,20 @@ HSH_Lookup(struct sess *sp) oh->waitinglist = wrk->nwaitinglist; wrk->nwaitinglist = NULL; } - VTAILQ_INSERT_TAIL(&oh->waitinglist->list, sp, list); + VTAILQ_INSERT_TAIL(&oh->waitinglist->list, + req->sp, list); } if (cache_param->diag_bitmap & 0x20) VSLb(req->vsl, SLT_Debug, "on waiting list <%p>", oh); - SES_Charge(sp); + SES_Charge(req->sp); /* * The objhead reference transfers to the sess, we get it * back when the sess comes off the waiting list and * calls us again */ req->hash_objhead = oh; - sp->wrk = NULL; + req->sp->wrk = NULL; Lck_Unlock(&oh->mtx); return (NULL); } @@ -450,7 +448,7 @@ HSH_Lookup(struct sess *sp) AZ(req->busyobj); req->busyobj = VBO_GetBusyObj(wrk); - req->busyobj->vsl->wid = sp->vsl_id; + req->busyobj->vsl->wid = req->sp->vsl_id; req->busyobj->refcount = 2; /* One for req, one for FetchBody */ VRY_Validate(req->vary_b); diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c index 77ffcda..13a8c10 100644 --- a/bin/varnishd/cache/cache_pipe.c +++ b/bin/varnishd/cache/cache_pipe.c @@ -60,7 +60,7 @@ rdf(int fd0, int fd1) } void -PipeSession(struct sess *sp) +PipeRequest(struct req *req) { struct vbc *vc; struct worker *wrk; @@ -68,36 +68,36 @@ PipeSession(struct sess *sp) struct busyobj *bo; int i; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); - bo = sp->req->busyobj; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC); + wrk = req->sp->wrk; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + bo = req->busyobj; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - wrk = sp->wrk; - vc = VDI_GetFd(NULL, sp->req); + vc = VDI_GetFd(NULL, req); if (vc == NULL) return; bo->vbc = vc; /* For panic dumping */ (void)VTCP_blocking(vc->fd); - WRW_Reserve(wrk, &vc->fd, bo->vsl, sp->req->t_req); - sp->wrk->acct_tmp.hdrbytes += - http_Write(wrk, bo->bereq, 0); + WRW_Reserve(wrk, &vc->fd, bo->vsl, req->t_req); + wrk->acct_tmp.hdrbytes += http_Write(wrk, bo->bereq, 0); - if (sp->req->htc->pipeline.b != NULL) - sp->wrk->acct_tmp.bodybytes += - WRW_Write(wrk, sp->req->htc->pipeline.b, - Tlen(sp->req->htc->pipeline)); + if (req->htc->pipeline.b != NULL) + wrk->acct_tmp.bodybytes += + WRW_Write(wrk, req->htc->pipeline.b, + Tlen(req->htc->pipeline)); i = WRW_FlushRelease(wrk); if (i) { - SES_Close(sp, "pipe"); + SES_Close(req->sp, "pipe"); VDI_CloseFd(&vc); return; } - sp->req->t_resp = VTIM_real(); + req->t_resp = VTIM_real(); memset(fds, 0, sizeof fds); @@ -105,8 +105,8 @@ PipeSession(struct sess *sp) fds[0].fd = vc->fd; fds[0].events = POLLIN | POLLERR; - // XXX: not yet (void)VTCP_linger(sp->fd, 0); - fds[1].fd = sp->fd; + // XXX: not yet (void)VTCP_linger(req->sp->fd, 0); + fds[1].fd = req->sp->fd; fds[1].events = POLLIN | POLLERR; while (fds[0].fd > -1 || fds[1].fd > -1) { @@ -115,24 +115,24 @@ PipeSession(struct sess *sp) i = poll(fds, 2, cache_param->pipe_timeout * 1000); if (i < 1) break; - if (fds[0].revents && rdf(vc->fd, sp->fd)) { + if (fds[0].revents && rdf(vc->fd, req->sp->fd)) { if (fds[1].fd == -1) break; (void)shutdown(vc->fd, SHUT_RD); - (void)shutdown(sp->fd, SHUT_WR); + (void)shutdown(req->sp->fd, SHUT_WR); fds[0].events = 0; fds[0].fd = -1; } - if (fds[1].revents && rdf(sp->fd, vc->fd)) { + if (fds[1].revents && rdf(req->sp->fd, vc->fd)) { if (fds[0].fd == -1) break; - (void)shutdown(sp->fd, SHUT_RD); + (void)shutdown(req->sp->fd, SHUT_RD); (void)shutdown(vc->fd, SHUT_WR); fds[1].events = 0; fds[1].fd = -1; } } - SES_Close(sp, "pipe"); + SES_Close(req->sp, "pipe"); VDI_CloseFd(&vc); bo->vbc = NULL; } diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index 59aa6cb..85c8233 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -154,30 +154,30 @@ RES_BuildHttp(struct req *req) */ static void -res_WriteGunzipObj(const struct sess *sp) +res_WriteGunzipObj(struct req *req) { struct storage *st; unsigned u = 0; struct vgz *vg; int i; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - vg = VGZ_NewUngzip(sp->req->vsl, "U D -"); + vg = VGZ_NewUngzip(req->vsl, "U D -"); AZ(VGZ_WrwInit(vg)); - VTAILQ_FOREACH(st, &sp->req->obj->store, list) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + VTAILQ_FOREACH(st, &req->obj->store, list) { + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); u += st->len; - i = VGZ_WrwGunzip(sp->wrk, vg, st->ptr, st->len); + i = VGZ_WrwGunzip(req->sp->wrk, vg, st->ptr, st->len); /* XXX: error check */ (void)i; } - VGZ_WrwFlush(sp->wrk, vg); + VGZ_WrwFlush(req->sp->wrk, vg); (void)VGZ_Destroy(&vg); - assert(u == sp->req->obj->len); + assert(u == req->obj->len); } /*--------------------------------------------------------------------*/ @@ -270,12 +270,12 @@ RES_WriteObj(struct req *req) } else if (req->res_mode & RES_ESI) { ESI_Deliver(req); } else if (req->res_mode & RES_ESI_CHILD && req->gzip_resp) { - ESI_DeliverChild(req->sp); + ESI_DeliverChild(req); } else if (req->res_mode & RES_ESI_CHILD && !req->gzip_resp && req->obj->gziped) { - res_WriteGunzipObj(req->sp); + res_WriteGunzipObj(req); } else if (req->res_mode & RES_GUNZIP) { - res_WriteGunzipObj(req->sp); + res_WriteGunzipObj(req); } else { res_WriteDirObj(req, low, high); } diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index fe0fb9f..5e05748 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -53,7 +53,7 @@ struct hash_slinger { /* cache_hash.c */ void HSH_Cleanup(struct worker *w); -struct objcore *HSH_Lookup(struct sess *sp); +struct objcore *HSH_Lookup(struct req *); void HSH_Ref(struct objcore *o); void HSH_Drop(struct worker *, struct object **); void HSH_Init(const struct hash_slinger *slinger); From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] 611815f Tag threads separately with request and session. Message-ID: commit 611815fad3537b2815bc6fc6028abc40f53f2477 Author: Poul-Henning Kamp Date: Tue Jun 19 09:45:09 2012 +0000 Tag threads separately with request and session. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 5823943..6ee34ec 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -854,7 +854,9 @@ uint32_t VXID_Get(struct vxid *v); extern volatile struct params * cache_param; void THR_SetName(const char *name); const char* THR_GetName(void); -void THR_SetSession(const struct sess *sp); +void THR_SetRequest(const struct req *); +const struct req * THR_GetRequest(void); +void THR_SetSession(const struct sess *); const struct sess * THR_GetSession(void); /* cache_lck.c */ diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index e29593d..9078834 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -198,7 +198,7 @@ VGZ_ObufFull(const struct vgz *vg) } /*-------------------------------------------------------------------- - * Keep the outbuffer supplied with storage + * Keep the outbuffer supplied with storage */ int diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index f038c9e..041578b 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -46,6 +46,7 @@ volatile struct params *cache_param; */ static pthread_key_t sp_key; +static pthread_key_t req_key; void THR_SetSession(const struct sess *sp) @@ -61,6 +62,20 @@ THR_GetSession(void) return (pthread_getspecific(sp_key)); } +void +THR_SetRequest(const struct req *req) +{ + + AZ(pthread_setspecific(req_key, req)); +} + +const struct req * +THR_GetRequest(void) +{ + + return (pthread_getspecific(req_key)); +} + /*-------------------------------------------------------------------- * Name threads if our pthreads implementation supports it. */ @@ -126,6 +141,7 @@ child_main(void) cache_param = heritage.param; AZ(pthread_key_create(&sp_key, NULL)); + AZ(pthread_key_create(&req_key, NULL)); AZ(pthread_key_create(&name_key, NULL)); THR_SetName("cache-main"); diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index c5f49de..af8bf6f 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -228,14 +228,52 @@ pan_busyobj(const struct busyobj *bo) /*--------------------------------------------------------------------*/ static void +pan_req(const struct req *req) +{ + const char *hand; + + VSB_printf(pan_vsp, "req = %p {\n", req); + VSB_printf(pan_vsp, " sp = %p, xid = %u,\n", req->sp, req->xid); + hand = VCL_Return_Name(req->handling); + if (hand != NULL) + VSB_printf(pan_vsp, " handling = %s,\n", hand); + else + VSB_printf(pan_vsp, " handling = 0x%x,\n", req->handling); + if (req->err_code) + VSB_printf(pan_vsp, + " err_code = %d, err_reason = %s,\n", req->err_code, + req->err_reason ? req->err_reason : "(null)"); + + VSB_printf(pan_vsp, " restarts = %d, esi_level = %d\n", + req->restarts, req->esi_level); + + if (req->busyobj != NULL) + pan_busyobj(req->busyobj); + + pan_ws(req->ws, 2); + pan_http("req", req->http, 2); + if (req->resp->ws != NULL) + pan_http("resp", req->resp, 4); + + if (VALID_OBJ(req->vcl, VCL_CONF_MAGIC)) + pan_vcl(req->vcl); + + if (VALID_OBJ(req->obj, OBJECT_MAGIC)) + pan_object(req->obj); + + VSB_printf(pan_vsp, "},\n"); +} + +/*--------------------------------------------------------------------*/ + +static void pan_sess(const struct sess *sp) { - const char *stp, *hand; + const char *stp; VSB_printf(pan_vsp, "sp = %p {\n", sp); - VSB_printf(pan_vsp, - " fd = %d, id = %u, xid = %u,\n", - sp->fd, sp->vsl_id & VSL_IDENTMASK, sp->req->xid); + VSB_printf(pan_vsp, " fd = %d, id = %u,\n", + sp->fd, sp->vsl_id & VSL_IDENTMASK); VSB_printf(pan_vsp, " client = %s %s,\n", sp->addr ? sp->addr : "?.?.?.?", sp->port ? sp->port : "?"); @@ -245,40 +283,14 @@ pan_sess(const struct sess *sp) #undef STEP default: stp = NULL; } - hand = VCL_Return_Name(sp->req->handling); if (stp != NULL) VSB_printf(pan_vsp, " step = %s,\n", stp); else VSB_printf(pan_vsp, " step = 0x%x,\n", sp->step); - if (hand != NULL) - VSB_printf(pan_vsp, " handling = %s,\n", hand); - else - VSB_printf(pan_vsp, " handling = 0x%x,\n", sp->req->handling); - if (sp->req->err_code) - VSB_printf(pan_vsp, - " err_code = %d, err_reason = %s,\n", sp->req->err_code, - sp->req->err_reason ? sp->req->err_reason : "(null)"); - - VSB_printf(pan_vsp, " restarts = %d, esi_level = %d\n", - sp->req->restarts, sp->req->esi_level); - - if (sp->req->busyobj != NULL) - pan_busyobj(sp->req->busyobj); - - pan_ws(sp->req->ws, 2); - pan_http("req", sp->req->http, 2); - if (sp->req->resp->ws != NULL) - pan_http("resp", sp->req->resp, 4); if (sp->wrk != NULL) pan_wrk(sp->wrk); - if (VALID_OBJ(sp->req->vcl, VCL_CONF_MAGIC)) - pan_vcl(sp->req->vcl); - - if (VALID_OBJ(sp->req->obj, OBJECT_MAGIC)) - pan_object(sp->req->obj); - VSB_printf(pan_vsp, "},\n"); } @@ -318,6 +330,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond, { const char *q; const struct sess *sp; + const struct req *req; AZ(pthread_mutex_lock(&panicstr_mtx)); /* Won't be released, we're going to die @@ -361,6 +374,9 @@ pan_ic(const char *func, const char *file, int line, const char *cond, sp = THR_GetSession(); if (sp != NULL) pan_sess(sp); + req = THR_GetRequest(); + if (req != NULL) + pan_req(req); } VSB_printf(pan_vsp, "\n"); VSB_bcat(pan_vsp, "", 1); /* NUL termination */ diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index f236389..df268f1 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -307,6 +307,7 @@ SES_GetReq(struct sess *sp) req->magic = REQ_MAGIC; sp->req = req; req->sp = sp; + THR_SetRequest(req); e = (char*)req + sz; p = (char*)(req + 1); @@ -359,6 +360,7 @@ SES_ReleaseReq(struct sess *sp) sp->req->sp = NULL; MPL_Free(pp->mpl_req, sp->req); sp->req = NULL; + THR_SetRequest(NULL); } /*-------------------------------------------------------------------- From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] 1c14a9d Make the waitinglist hold req instead of sp Message-ID: commit 1c14a9d49f30093fe58c6986b68456d9e300cd16 Author: Poul-Henning Kamp Date: Tue Jun 19 10:53:32 2012 +0000 Make the waitinglist hold req instead of sp diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 6ee34ec..f013084 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -562,6 +562,7 @@ struct req { uint8_t hash_always_miss; struct sess *sp; + VTAILQ_ENTRY(req) w_list; /* The busy objhead we sleep on */ struct objhead *hash_objhead; diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 68e4cda..e7c2a49 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -419,7 +419,7 @@ HSH_Lookup(struct req *req) wrk->nwaitinglist = NULL; } VTAILQ_INSERT_TAIL(&oh->waitinglist->list, - req->sp, list); + req, w_list); } if (cache_param->diag_bitmap & 0x20) VSLb(req->vsl, SLT_Debug, @@ -469,6 +469,7 @@ static void hsh_rush(struct dstat *ds, struct objhead *oh) { unsigned u; + struct req *req; struct sess *sp; struct waitinglist *wl; @@ -477,12 +478,14 @@ hsh_rush(struct dstat *ds, struct objhead *oh) wl = oh->waitinglist; CHECK_OBJ_NOTNULL(wl, WAITINGLIST_MAGIC); for (u = 0; u < cache_param->rush_exponent; u++) { - sp = VTAILQ_FIRST(&wl->list); - if (sp == NULL) + req = VTAILQ_FIRST(&wl->list); + if (req == NULL) break; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + sp = req->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); AZ(sp->wrk); - VTAILQ_REMOVE(&wl->list, sp, list); + VTAILQ_REMOVE(&wl->list, req, w_list); DSL(0x20, SLT_Debug, sp->vsl_id, "off waiting list"); if (SES_Schedule(sp)) { /* diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index 5e05748..c385ea6 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -68,7 +68,7 @@ struct objcore *HSH_NewObjCore(struct worker *wrk); struct waitinglist { unsigned magic; #define WAITINGLIST_MAGIC 0x063a477a - VTAILQ_HEAD(, sess) list; + VTAILQ_HEAD(, req) list; }; struct objhead { From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] 47c7bc1 Rename STEP to SESS_STEP Message-ID: commit 47c7bc1150dbee8048d62daef7456df60e07e923 Author: Poul-Henning Kamp Date: Tue Jun 19 11:13:51 2012 +0000 Rename STEP to SESS_STEP diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index f013084..7090654 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -123,9 +123,9 @@ typedef struct { /*--------------------------------------------------------------------*/ enum step { -#define STEP(l, u, arg) STP_##u, +#define SESS_STEP(l, u, arg) STP_##u, #include "tbl/steps.h" -#undef STEP +#undef SESS_STEP }; /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 88c269b..a925f1a 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -1644,14 +1644,14 @@ CNT_Session(struct sess *sp) } switch (sp->step) { -#define STEP(l,u,arg) \ +#define SESS_STEP(l,u,arg) \ case STP_##u: \ if (cache_param->diag_bitmap & 0x01) \ cnt_diag(sp, #u); \ done = cnt_##l arg; \ break; #include "tbl/steps.h" -#undef STEP +#undef SESS_STEP default: WRONG("State engine misfire"); } diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index af8bf6f..ce2b393 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -278,9 +278,9 @@ pan_sess(const struct sess *sp) sp->addr ? sp->addr : "?.?.?.?", sp->port ? sp->port : "?"); switch (sp->step) { -#define STEP(l, u, arg) case STP_##u: stp = "STP_" #u; break; +#define SESS_STEP(l, u, arg) case STP_##u: stp = "STP_" #u; break; #include "tbl/steps.h" -#undef STEP +#undef SESS_STEP default: stp = NULL; } if (stp != NULL) diff --git a/include/tbl/steps.h b/include/tbl/steps.h index 15fd811..ba4133c 100644 --- a/include/tbl/steps.h +++ b/include/tbl/steps.h @@ -29,20 +29,20 @@ */ /*lint -save -e525 -e539 */ -STEP(wait, WAIT, (sp, sp->wrk, sp->req)) -STEP(first, FIRST, (sp, sp->wrk)) -STEP(restart, RESTART, (sp, sp->wrk, sp->req)) -STEP(recv, RECV, (sp, sp->wrk, sp->req)) -STEP(start, START, (sp, sp->wrk, sp->req)) -STEP(pipe, PIPE, (sp, sp->wrk, sp->req)) -STEP(pass, PASS, (sp, sp->wrk, sp->req)) -STEP(lookup, LOOKUP, (sp, sp->wrk, sp->req)) -STEP(miss, MISS, (sp, sp->wrk, sp->req)) -STEP(hit, HIT, (sp, sp->wrk, sp->req)) -STEP(fetch, FETCH, (sp, sp->wrk, sp->req)) -STEP(fetchbody, FETCHBODY, (sp, sp->wrk, sp->req)) -STEP(prepresp, PREPRESP, (sp, sp->wrk, sp->req)) -STEP(deliver, DELIVER, (sp, sp->wrk, sp->req)) -STEP(error, ERROR, (sp, sp->wrk, sp->req)) -STEP(done, DONE, (sp, sp->wrk, sp->req)) +SESS_STEP(wait, WAIT, (sp, sp->wrk, sp->req)) +SESS_STEP(first, FIRST, (sp, sp->wrk)) +SESS_STEP(restart, RESTART, (sp, sp->wrk, sp->req)) +SESS_STEP(recv, RECV, (sp, sp->wrk, sp->req)) +SESS_STEP(start, START, (sp, sp->wrk, sp->req)) +SESS_STEP(pipe, PIPE, (sp, sp->wrk, sp->req)) +SESS_STEP(pass, PASS, (sp, sp->wrk, sp->req)) +SESS_STEP(lookup, LOOKUP, (sp, sp->wrk, sp->req)) +SESS_STEP(miss, MISS, (sp, sp->wrk, sp->req)) +SESS_STEP(hit, HIT, (sp, sp->wrk, sp->req)) +SESS_STEP(fetch, FETCH, (sp, sp->wrk, sp->req)) +SESS_STEP(fetchbody, FETCHBODY, (sp, sp->wrk, sp->req)) +SESS_STEP(prepresp, PREPRESP, (sp, sp->wrk, sp->req)) +SESS_STEP(deliver, DELIVER, (sp, sp->wrk, sp->req)) +SESS_STEP(error, ERROR, (sp, sp->wrk, sp->req)) +SESS_STEP(done, DONE, (sp, sp->wrk, sp->req)) /*lint -restore */ From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] 99d6751 Make HTC_Init() part of getting a request Message-ID: commit 99d6751242eb14a2f93e1c38a502fc208a2b02bb Author: Poul-Henning Kamp Date: Tue Jun 19 11:23:12 2012 +0000 Make HTC_Init() part of getting a request diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 7090654..8d45641 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -921,7 +921,7 @@ struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no); void SES_DeletePool(struct sesspool *sp); int SES_Schedule(struct sess *sp); void SES_Handle(struct sess *sp, double now); -void SES_GetReq(struct sess *sp); +struct req *SES_GetReq(struct sess *sp); void SES_ReleaseReq(struct sess *sp); pool_func_t SES_pool_accept_task; diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index a925f1a..372d0c4 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -110,16 +110,10 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) assert(!isnan(sp->t_rx)); if (req == NULL) { - SES_GetReq(sp); - req = sp->req; + req = SES_GetReq(sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - assert(req->sp == sp); - HTC_Init(req->htc, req->ws, sp->fd, sp->req->vsl, - cache_param->http_req_size, - cache_param->http_req_hdr_len); - } else { - assert(req->sp == sp); } + assert(req->sp == sp); AZ(req->vcl); AZ(req->obj); @@ -945,13 +939,9 @@ cnt_first(struct sess *sp, struct worker *wrk) /* Allocate a request already now, so we can VSL to it */ AZ(sp->req); - SES_GetReq(sp); - req = sp->req; + req = SES_GetReq(sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); assert(req->sp == sp); - HTC_Init(req->htc, req->ws, sp->fd, req->vsl, - cache_param->http_req_size, - cache_param->http_req_hdr_len); VTCP_name(&sp->sockaddr, sp->sockaddrlen, sp->addr, sizeof sp->addr, sp->port, sizeof sp->port); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index df268f1..08f77f2 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -287,7 +287,7 @@ SES_Delete(struct sess *sp, const char *reason, double now) * Alloc/Free sp->req */ -void +struct req * SES_GetReq(struct sess *sp) { struct sesspool *pp; @@ -341,6 +341,12 @@ SES_GetReq(struct sess *sp) assert(p < e); WS_Init(req->ws, "req", p, e - p); + + HTC_Init(req->htc, req->ws, sp->fd, req->vsl, + cache_param->http_req_size, + cache_param->http_req_hdr_len); + + return (req); } void From tfheen at varnish-software.com Thu Dec 18 09:27:49 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] 04fa5d7 Use automake's support for parallel tests Message-ID: commit 04fa5d7498f1b0b2bbf01ae877cac252ecdc6a5d Author: Tollef Fog Heen Date: Tue Jun 19 13:30:18 2012 +0200 Use automake's support for parallel tests In current automake, there's no way to have the list of tests be dynamic, so generate that when running configure as well as a local hook to update the list if it's changed. Also enable colourised test output. diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am index 4562dc5..ec68889 100644 --- a/bin/varnishtest/Makefile.am +++ b/bin/varnishtest/Makefile.am @@ -1,11 +1,16 @@ # -TESTS_PARALLELISM = 3 -check: varnishtest - ./varnishtest -i -j$(TESTS_PARALLELISM) $(srcdir)/tests/*.vtc - @echo "===================" - @echo "All tests succeeded" - @echo "===================" +VTC_LOG_COMPILER = ./varnishtest -v -i +TEST_EXTENSIONS = .vtc +TESTS = @VTC_TESTS@ + +# Make sure we run check-local first +check: check-local check-am +# See if list of checks have changed, recheck +check-local: + if [ "$$(cd $(srcdir) && echo tests/*.vtc)" != "@VTC_TESTS@" ]; then \ + cd $(top_builddir) && ./config.status --recheck ; \ + fi DISTCLEANFILES = _.ok diff --git a/configure.ac b/configure.ac index 2791fd9..e1311f0 100644 --- a/configure.ac +++ b/configure.ac @@ -12,7 +12,7 @@ OCFLAGS="$CFLAGS" AC_CANONICAL_SYSTEM AC_LANG(C) -AM_INIT_AUTOMAKE([foreign]) +AM_INIT_AUTOMAKE([foreign color-tests parallel-tests]) AC_DISABLE_STATIC AC_PROG_LIBTOOL @@ -526,6 +526,10 @@ fi AC_DEFINE_UNQUOTED([VCC_CC],"$VCC_CC",[C compiler command line for VCL code]) +# Stupid automake needs this +VTC_TESTS="$(cd $srcdir/bin/varnishtest && echo tests/*.vtc)" +AC_SUBST(VTC_TESTS) + # Generate output AC_CONFIG_FILES([ Makefile From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] 4f15e16 Move cnt_first{} up above cnt_wait{}, this is a more logical order and will make subsequent diffs clearer. Message-ID: commit 4f15e168abc9cf3a38fb8d096354e508a84d2e82 Author: Poul-Henning Kamp Date: Tue Jun 19 11:33:33 2012 +0000 Move cnt_first{} up above cnt_wait{}, this is a more logical order and will make subsequent diffs clearer. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 372d0c4..0d4e93b 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -80,6 +80,58 @@ DOT acceptor -> first [style=bold,color=green] static unsigned xids; /*-------------------------------------------------------------------- + * A freshly accepted socket + * +DOT subgraph xcluster_first { +DOT first [ +DOT shape=box +DOT label="cnt_first:\nrender\naddresses" +DOT ] +DOT } +DOT first -> wait [style=bold,color=green] + */ + +static int +cnt_first(struct sess *sp, struct worker *wrk) +{ + struct req *req; + char laddr[ADDR_BUFSIZE]; + char lport[PORT_BUFSIZE]; + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + + /* Allocate a request already now, so we can VSL to it */ + AZ(sp->req); + req = SES_GetReq(sp); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + assert(req->sp == sp); + + VTCP_name(&sp->sockaddr, sp->sockaddrlen, + sp->addr, sizeof sp->addr, sp->port, sizeof sp->port); + if (cache_param->log_local_addr) { + AZ(getsockname(sp->fd, (void*)&sp->mysockaddr, + &sp->mysockaddrlen)); + VTCP_name(&sp->mysockaddr, sp->mysockaddrlen, + laddr, sizeof laddr, lport, sizeof lport); + /* XXX: have no req yet */ + VSLb(req->vsl, SLT_SessionOpen, "%s %s %s %s", + sp->addr, sp->port, laddr, lport); + } else { + /* XXX: have no req yet */ + VSLb(req->vsl, SLT_SessionOpen, "%s %s %s", + sp->addr, sp->port, sp->mylsock->name); + } + + wrk->acct_tmp.sess++; + + sp->t_rx = sp->t_open; + sp->t_idle = sp->t_open; + sp->step = STP_WAIT; + return (0); +} + +/*-------------------------------------------------------------------- * WAIT * Collect the request from the client. * @@ -916,58 +968,6 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) } /*-------------------------------------------------------------------- - * A freshly accepted socket - * -DOT subgraph xcluster_first { -DOT first [ -DOT shape=box -DOT label="cnt_first:\nrender\naddresses" -DOT ] -DOT } -DOT first -> wait [style=bold,color=green] - */ - -static int -cnt_first(struct sess *sp, struct worker *wrk) -{ - struct req *req; - char laddr[ADDR_BUFSIZE]; - char lport[PORT_BUFSIZE]; - - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - - /* Allocate a request already now, so we can VSL to it */ - AZ(sp->req); - req = SES_GetReq(sp); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - assert(req->sp == sp); - - VTCP_name(&sp->sockaddr, sp->sockaddrlen, - sp->addr, sizeof sp->addr, sp->port, sizeof sp->port); - if (cache_param->log_local_addr) { - AZ(getsockname(sp->fd, (void*)&sp->mysockaddr, - &sp->mysockaddrlen)); - VTCP_name(&sp->mysockaddr, sp->mysockaddrlen, - laddr, sizeof laddr, lport, sizeof lport); - /* XXX: have no req yet */ - VSLb(req->vsl, SLT_SessionOpen, "%s %s %s %s", - sp->addr, sp->port, laddr, lport); - } else { - /* XXX: have no req yet */ - VSLb(req->vsl, SLT_SessionOpen, "%s %s %s", - sp->addr, sp->port, sp->mylsock->name); - } - - wrk->acct_tmp.sess++; - - sp->t_rx = sp->t_open; - sp->t_idle = sp->t_open; - sp->step = STP_WAIT; - return (0); -} - -/*-------------------------------------------------------------------- * HIT * We had a cache hit. Ask VCL, then march off as instructed. * From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] cbe58e7 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit cbe58e70c2dc7d8c1769dba2df5e3a16a0c4025e Merge: 4f15e16 04fa5d7 Author: Poul-Henning Kamp Date: Tue Jun 19 11:34:03 2012 +0000 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] 7d9ca7d Eliminate the "first" step, merge it with "wait" Message-ID: commit 7d9ca7de9a3b7fe44c92431c7a22038897e6acde Author: Poul-Henning Kamp Date: Tue Jun 19 11:46:24 2012 +0000 Eliminate the "first" step, merge it with "wait" diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 8d45641..5c2c923 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -665,6 +665,7 @@ struct sess { struct sockaddr_storage sockaddr; struct sockaddr_storage mysockaddr; struct listen_sock *mylsock; + int init_done; /* formatted ascii client address */ char addr[ADDR_BUFSIZE]; diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 0d4e93b..4f97c60 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -80,58 +80,6 @@ DOT acceptor -> first [style=bold,color=green] static unsigned xids; /*-------------------------------------------------------------------- - * A freshly accepted socket - * -DOT subgraph xcluster_first { -DOT first [ -DOT shape=box -DOT label="cnt_first:\nrender\naddresses" -DOT ] -DOT } -DOT first -> wait [style=bold,color=green] - */ - -static int -cnt_first(struct sess *sp, struct worker *wrk) -{ - struct req *req; - char laddr[ADDR_BUFSIZE]; - char lport[PORT_BUFSIZE]; - - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - - /* Allocate a request already now, so we can VSL to it */ - AZ(sp->req); - req = SES_GetReq(sp); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - assert(req->sp == sp); - - VTCP_name(&sp->sockaddr, sp->sockaddrlen, - sp->addr, sizeof sp->addr, sp->port, sizeof sp->port); - if (cache_param->log_local_addr) { - AZ(getsockname(sp->fd, (void*)&sp->mysockaddr, - &sp->mysockaddrlen)); - VTCP_name(&sp->mysockaddr, sp->mysockaddrlen, - laddr, sizeof laddr, lport, sizeof lport); - /* XXX: have no req yet */ - VSLb(req->vsl, SLT_SessionOpen, "%s %s %s %s", - sp->addr, sp->port, laddr, lport); - } else { - /* XXX: have no req yet */ - VSLb(req->vsl, SLT_SessionOpen, "%s %s %s", - sp->addr, sp->port, sp->mylsock->name); - } - - wrk->acct_tmp.sess++; - - sp->t_rx = sp->t_open; - sp->t_idle = sp->t_open; - sp->step = STP_WAIT; - return (0); -} - -/*-------------------------------------------------------------------- * WAIT * Collect the request from the client. * @@ -155,18 +103,44 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) struct pollfd pfd[1]; double now, when; const char *why = NULL; + char laddr[ADDR_BUFSIZE]; + char lport[PORT_BUFSIZE]; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - assert(!isnan(sp->t_rx)); - if (req == NULL) { req = SES_GetReq(sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); } assert(req->sp == sp); + if (!sp->init_done) { + VTCP_name(&sp->sockaddr, sp->sockaddrlen, + sp->addr, sizeof sp->addr, sp->port, sizeof sp->port); + if (cache_param->log_local_addr) { + AZ(getsockname(sp->fd, (void*)&sp->mysockaddr, + &sp->mysockaddrlen)); + VTCP_name(&sp->mysockaddr, sp->mysockaddrlen, + laddr, sizeof laddr, lport, sizeof lport); + /* XXX: have no req yet */ + VSLb(req->vsl, SLT_SessionOpen, "%s %s %s %s", + sp->addr, sp->port, laddr, lport); + } else { + /* XXX: have no req yet */ + VSLb(req->vsl, SLT_SessionOpen, "%s %s %s", + sp->addr, sp->port, sp->mylsock->name); + } + + wrk->acct_tmp.sess++; + + sp->t_rx = sp->t_open; + sp->t_idle = sp->t_open; + sp->init_done = 1; + } + + assert(!isnan(sp->t_rx)); + AZ(req->vcl); AZ(req->obj); AZ(req->esi_level); @@ -1591,7 +1565,6 @@ CNT_Session(struct sess *sp) * Possible entrance states */ assert( - sp->step == STP_FIRST || sp->step == STP_WAIT || sp->step == STP_LOOKUP || sp->step == STP_RECV); @@ -1604,7 +1577,7 @@ CNT_Session(struct sess *sp) * rather do the syscall in the worker thread. * On systems which return errors for ioctl, we close early */ - if ((sp->step == STP_FIRST || sp->step == STP_START) && + if ((sp->step == STP_WAIT || sp->step == STP_START) && VTCP_blocking(sp->fd)) { if (errno == ECONNRESET) SES_Close(sp, "remote closed"); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 08f77f2..34dad47 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -159,7 +159,7 @@ SES_pool_accept_task(struct worker *wrk, void *arg) VCA_FailSess(wrk); } else { VCA_SetupSess(wrk, sp); - sp->step = STP_FIRST; + sp->step = STP_WAIT; ses_pool_task(wrk, sp); } } diff --git a/include/tbl/steps.h b/include/tbl/steps.h index ba4133c..6750cd3 100644 --- a/include/tbl/steps.h +++ b/include/tbl/steps.h @@ -30,19 +30,18 @@ /*lint -save -e525 -e539 */ SESS_STEP(wait, WAIT, (sp, sp->wrk, sp->req)) -SESS_STEP(first, FIRST, (sp, sp->wrk)) -SESS_STEP(restart, RESTART, (sp, sp->wrk, sp->req)) +SESS_STEP(restart, RESTART, (sp, sp->wrk, sp->req)) SESS_STEP(recv, RECV, (sp, sp->wrk, sp->req)) -SESS_STEP(start, START, (sp, sp->wrk, sp->req)) +SESS_STEP(start, START, (sp, sp->wrk, sp->req)) SESS_STEP(pipe, PIPE, (sp, sp->wrk, sp->req)) SESS_STEP(pass, PASS, (sp, sp->wrk, sp->req)) -SESS_STEP(lookup, LOOKUP, (sp, sp->wrk, sp->req)) +SESS_STEP(lookup, LOOKUP, (sp, sp->wrk, sp->req)) SESS_STEP(miss, MISS, (sp, sp->wrk, sp->req)) SESS_STEP(hit, HIT, (sp, sp->wrk, sp->req)) -SESS_STEP(fetch, FETCH, (sp, sp->wrk, sp->req)) -SESS_STEP(fetchbody, FETCHBODY, (sp, sp->wrk, sp->req)) -SESS_STEP(prepresp, PREPRESP, (sp, sp->wrk, sp->req)) -SESS_STEP(deliver, DELIVER, (sp, sp->wrk, sp->req)) -SESS_STEP(error, ERROR, (sp, sp->wrk, sp->req)) +SESS_STEP(fetch, FETCH, (sp, sp->wrk, sp->req)) +SESS_STEP(fetchbody, FETCHBODY, (sp, sp->wrk, sp->req)) +SESS_STEP(prepresp, PREPRESP, (sp, sp->wrk, sp->req)) +SESS_STEP(deliver, DELIVER, (sp, sp->wrk, sp->req)) +SESS_STEP(error, ERROR, (sp, sp->wrk, sp->req)) SESS_STEP(done, DONE, (sp, sp->wrk, sp->req)) /*lint -restore */ From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] a1e3ed9 Apply the "alexandrian solution" to cache_center.c and split the state engine into two separate state engines, one for session and one for requests. Message-ID: commit a1e3ed99866c3a9d7954a943462900ba5f9f6316 Author: Poul-Henning Kamp Date: Wed Jun 20 08:08:38 2012 +0000 Apply the "alexandrian solution" to cache_center.c and split the state engine into two separate state engines, one for session and one for requests. Next comes the cleanup of all the bits and pieces... diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 5c2c923..299dc93 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -736,7 +736,8 @@ void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj); void VBO_Free(struct busyobj **vbo); /* cache_center.c [CNT] */ -void CNT_Session(struct sess *sp); +int CNT_Request(struct req *); +void CNT_Session(struct sess *); void CNT_Init(void); /* cache_cli.c [CLI] */ diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 4f97c60..612dbfe 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -26,12 +26,21 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * This file contains the central state machine for pushing requests. + * This file contains the two central state machine for pushing + * sessions and requests. * - * We cannot just use direct calls because it is possible to kick a - * request back to the lookup stage (usually after a rewrite). The - * state engine also allows us to break the processing up into some - * logical chunks which improves readability a little bit. + * The first part of the file, entrypoint CNT_Session() and down to + * the ==== separator, is concerned with sessions. When a session has + * a request to deal with, it calls into the second half of the file. + * This part is for all practical purposes HTTP/1.x specific. + * + * The second part of the file, entrypoint CNT_Request() and below the + * ==== separator, is intended to (over time) be(ome) protocol agnostic. + * We already use this now with ESI:includes, which are for all relevant + * purposes a different "protocol" + * + * A special complication is the fact that we can suspend processing of + * a request when hash-lookup finds a busy objhdr. * * Since the states are rather nasty in detail, I have decided to embedd * a dot(1) graph in the source code comments. So to see the big picture, @@ -204,6 +213,175 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) } /*-------------------------------------------------------------------- + * This is the final state, figure out if we should close or recycle + * the client connection + * +DOT DONE [ +DOT shape=record +DOT label="{cnt_done:|Request completed}" +DOT ] +DOT ESI_RESP [ shape=hexagon ] +DOT DONE -> start [label="full pipeline"] +DOT DONE -> wait +DOT DONE -> ESI_RESP + */ + +static int +cnt_done(struct sess *sp, struct worker *wrk, struct req *req) +{ + double dh, dp, da; + int i; + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_ORNULL(req->vcl, VCL_CONF_MAGIC); + + AZ(req->obj); + AZ(req->busyobj); + req->director = NULL; + req->restarts = 0; + + /* If we did an ESI include, don't mess up our state */ + if (req->esi_level > 0) + return (1); + + if (req->vcl != NULL) { + if (wrk->vcl != NULL) + VCL_Rel(&wrk->vcl); + wrk->vcl = req->vcl; + req->vcl = NULL; + } + + + sp->t_idle = W_TIM_real(wrk); + if (req->xid == 0) { + req->t_resp = sp->t_idle; + } else { + dp = req->t_resp - req->t_req; + da = sp->t_idle - req->t_resp; + dh = req->t_req - sp->t_open; + /* XXX: Add StatReq == StatSess */ + /* XXX: Workaround for pipe */ + if (sp->fd >= 0) { + VSLb(req->vsl, SLT_Length, "%ju", + (uintmax_t)req->req_bodybytes); + } + VSLb(req->vsl, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f", + req->xid, req->t_req, sp->t_idle, dh, dp, da); + } + req->xid = 0; + VSL_Flush(req->vsl, 0); + + req->t_req = NAN; + req->t_resp = NAN; + + req->req_bodybytes = 0; + + req->hash_always_miss = 0; + req->hash_ignore_busy = 0; + + if (sp->fd >= 0 && req->doclose != NULL) { + /* + * This is an orderly close of the connection; ditch nolinger + * before we close, to get queued data transmitted. + */ + // XXX: not yet (void)VTCP_linger(sp->fd, 0); + SES_Close(sp, req->doclose); + } + + if (sp->fd < 0) { + wrk->stats.sess_closed++; + SES_Delete(sp, NULL, NAN); + return (1); + } + + if (wrk->stats.client_req >= cache_param->wthread_stats_rate) + WRK_SumStat(wrk); + + WS_Reset(req->ws, NULL); + WS_Reset(wrk->aws, NULL); + + i = HTC_Reinit(req->htc); + if (i == 1) { + req->t_req = sp->t_idle; + wrk->stats.sess_pipeline++; + sp->step = STP_START; + } else { + sp->t_rx = sp->t_idle; + req->t_req = NAN; + if (Tlen(req->htc->rxbuf)) + wrk->stats.sess_readahead++; + sp->step = STP_WAIT; + } + return (0); +} + +/*-------------------------------------------------------------------- + */ + +void +CNT_Session(struct sess *sp) +{ + int done; + struct worker *wrk; + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + wrk = sp->wrk; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + + /* + * Whenever we come in from the acceptor or waiter, we need to set + * blocking mode, but there is no point in setting it when we come from + * ESI or when a parked sessions returns. + * It would be simpler to do this in the acceptor or waiter, but we'd + * rather do the syscall in the worker thread. + * On systems which return errors for ioctl, we close early + */ + if (sp->step == STP_WAIT && VTCP_blocking(sp->fd)) { + if (errno == ECONNRESET) + SES_Close(sp, "remote closed"); + else + SES_Close(sp, "error"); + assert(cnt_done(sp, wrk, sp->req) == 1); + return; + } + + while (1) { + /* + * Possible entrance states + */ + assert( + sp->step == STP_WAIT || + sp->step == STP_LOOKUP || + sp->step == STP_START || + sp->step == STP_RECV); + + if (sp->step != STP_WAIT) { + done = CNT_Request(sp->req); + if (done == 2) + return; + assert(done == 1); + } + + if (sp->step == STP_DONE) { + done = cnt_done(sp, wrk, sp->req); + if (done) + return; + } + + if (sp->step == STP_WAIT) { + done = cnt_wait(sp, wrk, sp->req); + if (done) + return; + } + } +} + +/*====================================================================*/ + + +/*-------------------------------------------------------------------- * We have a refcounted object on the session, and possibly the busyobj * which is fetching it, prepare a response. * @@ -370,116 +548,8 @@ cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req) (void)HSH_Deref(&wrk->stats, NULL, &req->obj); http_Teardown(req->resp); sp->step = STP_DONE; - return (0); -} - -/*-------------------------------------------------------------------- - * This is the final state, figure out if we should close or recycle - * the client connection - * -DOT DONE [ -DOT shape=record -DOT label="{cnt_done:|Request completed}" -DOT ] -DOT ESI_RESP [ shape=hexagon ] -DOT DONE -> start [label="full pipeline"] -DOT DONE -> wait -DOT DONE -> ESI_RESP - */ - -static int -cnt_done(struct sess *sp, struct worker *wrk, struct req *req) -{ - double dh, dp, da; - int i; - - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CHECK_OBJ_ORNULL(req->vcl, VCL_CONF_MAGIC); - - AZ(req->obj); - AZ(req->busyobj); - req->director = NULL; - req->restarts = 0; - - SES_Charge(sp); - - /* If we did an ESI include, don't mess up our state */ - if (req->esi_level > 0) - return (1); - - if (req->vcl != NULL) { - if (wrk->vcl != NULL) - VCL_Rel(&wrk->vcl); - wrk->vcl = req->vcl; - req->vcl = NULL; - } - - - sp->t_idle = W_TIM_real(wrk); - if (req->xid == 0) { - req->t_resp = sp->t_idle; - } else { - dp = req->t_resp - req->t_req; - da = sp->t_idle - req->t_resp; - dh = req->t_req - sp->t_open; - /* XXX: Add StatReq == StatSess */ - /* XXX: Workaround for pipe */ - if (sp->fd >= 0) { - VSLb(req->vsl, SLT_Length, "%ju", - (uintmax_t)req->req_bodybytes); - } - VSLb(req->vsl, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f", - req->xid, req->t_req, sp->t_idle, dh, dp, da); - } - req->xid = 0; - VSL_Flush(req->vsl, 0); - - req->t_req = NAN; - req->t_resp = NAN; - - req->req_bodybytes = 0; - - req->hash_always_miss = 0; - req->hash_ignore_busy = 0; - - if (sp->fd >= 0 && req->doclose != NULL) { - /* - * This is an orderly close of the connection; ditch nolinger - * before we close, to get queued data transmitted. - */ - // XXX: not yet (void)VTCP_linger(sp->fd, 0); - SES_Close(sp, req->doclose); - } - - if (sp->fd < 0) { - wrk->stats.sess_closed++; - SES_Delete(sp, NULL, NAN); - return (1); - } - - if (wrk->stats.client_req >= cache_param->wthread_stats_rate) - WRK_SumStat(wrk); - - WS_Reset(req->ws, NULL); - WS_Reset(wrk->aws, NULL); - - i = HTC_Reinit(req->htc); - if (i == 1) { - req->t_req = sp->t_idle; - wrk->stats.sess_pipeline++; - sp->step = STP_START; - } else { - sp->t_rx = sp->t_idle; - req->t_req = NAN; - if (Tlen(req->htc->rxbuf)) - wrk->stats.sess_readahead++; - sp->step = STP_WAIT; - } - return (0); + return (1); } - /*-------------------------------------------------------------------- * Emit an error * @@ -525,7 +595,7 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) http_Teardown(bo->beresp); http_Teardown(bo->bereq); sp->step = STP_DONE; - return(0); + return(1); } CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); req->obj->xid = req->xid; @@ -1050,7 +1120,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) * around to do the lookup with. * NB: Do not access sp any more ! */ - return (1); + return (2); } AZ(req->objcore); @@ -1292,7 +1362,7 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req) http_Teardown(bo->bereq); VBO_DerefBusyObj(wrk, &req->busyobj); sp->step = STP_DONE; - return (0); + return (1); } /*-------------------------------------------------------------------- @@ -1484,7 +1554,7 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) if (req->err_code == 400) { sp->step = STP_DONE; SES_Close(sp, "junk"); - return (0); + return (1); } req->ws_req = WS_Snapshot(req->ws); @@ -1501,7 +1571,7 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) } else if (strlen(r) != write(sp->fd, r, strlen(r))) { sp->step = STP_DONE; SES_Close(sp, "remote closed"); - return (0); + return (1); } } http_Unset(req->http, H_Expect); @@ -1551,12 +1621,15 @@ cnt_diag(struct sess *sp, const char *state) } } -void -CNT_Session(struct sess *sp) +int +CNT_Request(struct req *req) { int done; struct worker *wrk; + struct sess *sp; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + sp = req->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); wrk = sp->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -1565,30 +1638,10 @@ CNT_Session(struct sess *sp) * Possible entrance states */ assert( - sp->step == STP_WAIT || sp->step == STP_LOOKUP || + sp->step == STP_START || sp->step == STP_RECV); - /* - * Whenever we come in from the acceptor or waiter, we need to set - * blocking mode, but there is no point in setting it when we come from - * ESI or when a parked sessions returns. - * It would be simpler to do this in the acceptor or waiter, but we'd - * rather do the syscall in the worker thread. - * On systems which return errors for ioctl, we close early - */ - if ((sp->step == STP_WAIT || sp->step == STP_START) && - VTCP_blocking(sp->fd)) { - if (errno == ECONNRESET) - SES_Close(sp, "remote closed"); - else - SES_Close(sp, "error"); - sp->step = STP_DONE; - } - - /* - * NB: Once done is set, we can no longer touch sp! - */ for (done = 0; !done; ) { assert(sp->wrk == wrk); /* @@ -1599,12 +1652,12 @@ CNT_Session(struct sess *sp) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); WS_Assert(wrk->aws); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + AN(req->sp); + assert(req->sp == sp); - if (sp->req != NULL) { - CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC); - AN(sp->req->sp); - assert(sp->req->sp == sp); - } + assert(sp->step != STP_WAIT); + assert(sp->step != STP_DONE); switch (sp->step) { #define SESS_STEP(l,u,arg) \ @@ -1621,10 +1674,11 @@ CNT_Session(struct sess *sp) WS_Assert(wrk->aws); CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); } -#define ACCT(foo) AZ(wrk->acct_tmp.foo); -#include "tbl/acct_fields.h" -#undef ACCT + if (done == 1) + SES_Charge(sp); + assert(WRW_IsReleased(wrk)); + return (done); } /* diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 6aed2c3..6af4b70 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -49,6 +49,7 @@ ved_include(struct req *req, const char *src, const char *host) char *sp_ws_wm; char *wrk_ws_wm; unsigned sxid, res_mode; + int i; wrk = req->sp->wrk; @@ -94,9 +95,12 @@ ved_include(struct req *req, const char *src, const char *host) sxid = req->xid; while (1) { req->sp->wrk = wrk; - CNT_Session(req->sp); - if (req->sp->step == STP_DONE) + i = CNT_Request(req); + if (req->sp->step == STP_DONE) { + assert(i == 1); break; + } + assert(i == 2); AZ(req->sp->wrk); DSL(0x20, SLT_Debug, req->sp->vsl_id, "loop waiting for ESI"); (void)usleep(10000); From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] c6b0aaa Eliminate the "DONE" state, it is now a session state. Message-ID: commit c6b0aaa1a255aecd7d380759444f731cabb54b13 Author: Poul-Henning Kamp Date: Wed Jun 20 08:40:22 2012 +0000 Eliminate the "DONE" state, it is now a session state. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 612dbfe..cff480d 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -227,7 +227,7 @@ DOT DONE -> ESI_RESP */ static int -cnt_done(struct sess *sp, struct worker *wrk, struct req *req) +cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) { double dh, dp, da; int i; @@ -242,9 +242,7 @@ cnt_done(struct sess *sp, struct worker *wrk, struct req *req) req->director = NULL; req->restarts = 0; - /* If we did an ESI include, don't mess up our state */ - if (req->esi_level > 0) - return (1); + AZ(req->esi_level); if (req->vcl != NULL) { if (wrk->vcl != NULL) @@ -343,7 +341,7 @@ CNT_Session(struct sess *sp) SES_Close(sp, "remote closed"); else SES_Close(sp, "error"); - assert(cnt_done(sp, wrk, sp->req) == 1); + assert(cnt_sess_done(sp, wrk, sp->req) == 1); return; } @@ -362,10 +360,7 @@ CNT_Session(struct sess *sp) if (done == 2) return; assert(done == 1); - } - - if (sp->step == STP_DONE) { - done = cnt_done(sp, wrk, sp->req); + done = cnt_sess_done(sp, wrk, sp->req); if (done) return; } @@ -547,7 +542,6 @@ cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req) assert(WRW_IsReleased(wrk)); (void)HSH_Deref(&wrk->stats, NULL, &req->obj); http_Teardown(req->resp); - sp->step = STP_DONE; return (1); } /*-------------------------------------------------------------------- @@ -594,7 +588,6 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) req->director = NULL; http_Teardown(bo->beresp); http_Teardown(bo->bereq); - sp->step = STP_DONE; return(1); } CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); @@ -1361,7 +1354,6 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req) assert(WRW_IsReleased(wrk)); http_Teardown(bo->bereq); VBO_DerefBusyObj(wrk, &req->busyobj); - sp->step = STP_DONE; return (1); } @@ -1487,7 +1479,6 @@ cnt_recv(struct sess *sp, const struct worker *wrk, struct req *req) if (req->esi_level > 0) { /* XXX: VSL something */ INCOMPL(); - /* sp->step = STP_DONE; */ return (1); } sp->step = STP_PIPE; @@ -1552,7 +1543,6 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) /* If we could not even parse the request, just close */ if (req->err_code == 400) { - sp->step = STP_DONE; SES_Close(sp, "junk"); return (1); } @@ -1569,7 +1559,6 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) if (strcasecmp(p, "100-continue")) { req->err_code = 417; } else if (strlen(r) != write(sp->fd, r, strlen(r))) { - sp->step = STP_DONE; SES_Close(sp, "remote closed"); return (1); } @@ -1657,7 +1646,6 @@ CNT_Request(struct req *req) assert(req->sp == sp); assert(sp->step != STP_WAIT); - assert(sp->step != STP_DONE); switch (sp->step) { #define SESS_STEP(l,u,arg) \ diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 6af4b70..d346006 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -96,10 +96,8 @@ ved_include(struct req *req, const char *src, const char *host) while (1) { req->sp->wrk = wrk; i = CNT_Request(req); - if (req->sp->step == STP_DONE) { - assert(i == 1); + if (i == 1) break; - } assert(i == 2); AZ(req->sp->wrk); DSL(0x20, SLT_Debug, req->sp->vsl_id, "loop waiting for ESI"); @@ -107,7 +105,6 @@ ved_include(struct req *req, const char *src, const char *host) } req->xid = sxid; AN(req->sp->wrk); - assert(req->sp->step == STP_DONE); req->esi_level--; req->obj = obj; req->res_mode = res_mode; diff --git a/include/tbl/steps.h b/include/tbl/steps.h index 6750cd3..e429631 100644 --- a/include/tbl/steps.h +++ b/include/tbl/steps.h @@ -43,5 +43,4 @@ SESS_STEP(fetchbody, FETCHBODY, (sp, sp->wrk, sp->req)) SESS_STEP(prepresp, PREPRESP, (sp, sp->wrk, sp->req)) SESS_STEP(deliver, DELIVER, (sp, sp->wrk, sp->req)) SESS_STEP(error, ERROR, (sp, sp->wrk, sp->req)) -SESS_STEP(done, DONE, (sp, sp->wrk, sp->req)) /*lint -restore */ From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] 42780ae move the STP_ values out of the session state functions Message-ID: commit 42780ae2f6ef3ae1503256ba85f186c0db6b6ebb Author: Poul-Henning Kamp Date: Wed Jun 20 09:26:50 2012 +0000 move the STP_ values out of the session state functions diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index cff480d..eadbd3a 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -172,7 +172,6 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) if (i == 1) { /* Got it, run with it */ req->t_req = now; - sp->step = STP_START; return (0); } else if (i == -1) { why = "EOF"; @@ -226,7 +225,13 @@ DOT DONE -> wait DOT DONE -> ESI_RESP */ -static int +enum cnt_sess_done_ret { + SESS_DONE_RET_GONE, + SESS_DONE_RET_WAIT, + SESS_DONE_RET_START, +}; + +static enum cnt_sess_done_ret cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) { double dh, dp, da; @@ -291,7 +296,7 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) if (sp->fd < 0) { wrk->stats.sess_closed++; SES_Delete(sp, NULL, NAN); - return (1); + return (SESS_DONE_RET_GONE); } if (wrk->stats.client_req >= cache_param->wthread_stats_rate) @@ -304,15 +309,14 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) if (i == 1) { req->t_req = sp->t_idle; wrk->stats.sess_pipeline++; - sp->step = STP_START; + return (SESS_DONE_RET_START); } else { sp->t_rx = sp->t_idle; req->t_req = NAN; if (Tlen(req->htc->rxbuf)) wrk->stats.sess_readahead++; - sp->step = STP_WAIT; + return (SESS_DONE_RET_WAIT); } - return (0); } /*-------------------------------------------------------------------- @@ -322,6 +326,7 @@ void CNT_Session(struct sess *sp) { int done; + enum cnt_sess_done_ret sdr; struct worker *wrk; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); @@ -341,7 +346,8 @@ CNT_Session(struct sess *sp) SES_Close(sp, "remote closed"); else SES_Close(sp, "error"); - assert(cnt_sess_done(sp, wrk, sp->req) == 1); + sdr = cnt_sess_done(sp, wrk, sp->req); + assert(sdr == SESS_DONE_RET_GONE); return; } @@ -352,23 +358,33 @@ CNT_Session(struct sess *sp) assert( sp->step == STP_WAIT || sp->step == STP_LOOKUP || - sp->step == STP_START || - sp->step == STP_RECV); + sp->step == STP_START); if (sp->step != STP_WAIT) { done = CNT_Request(sp->req); if (done == 2) return; assert(done == 1); - done = cnt_sess_done(sp, wrk, sp->req); - if (done) + sdr = cnt_sess_done(sp, wrk, sp->req); + switch (sdr) { + case SESS_DONE_RET_GONE: return; + case SESS_DONE_RET_WAIT: + sp->step = STP_WAIT; + break; + case SESS_DONE_RET_START: + sp->step = STP_START; + break; + default: + WRONG("Illegal enum cnt_sess_done_ret"); + } } if (sp->step == STP_WAIT) { done = cnt_wait(sp, wrk, sp->req); if (done) return; + sp->step = STP_START; } } } From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] 3325c6c Eliminate the "sp" argument to the request state engine states. Message-ID: commit 3325c6c752c6c9b870f5aa600c113627112e0bf2 Author: Poul-Henning Kamp Date: Wed Jun 20 09:54:22 2012 +0000 Eliminate the "sp" argument to the request state engine states. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index eadbd3a..6e394e4 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * This file contains the two central state machine for pushing + * This file contains the two central state machine for pushing * sessions and requests. * * The first part of the file, entrypoint CNT_Session() and down to @@ -370,10 +370,10 @@ CNT_Session(struct sess *sp) case SESS_DONE_RET_GONE: return; case SESS_DONE_RET_WAIT: - sp->step = STP_WAIT; + sp->step = STP_WAIT; break; case SESS_DONE_RET_START: - sp->step = STP_START; + sp->step = STP_START; break; default: WRONG("Illegal enum cnt_sess_done_ret"); @@ -409,11 +409,10 @@ DOT } */ static int -cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) +cnt_prepresp(struct worker *wrk, struct req *req) { struct busyobj *bo; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); bo = req->busyobj; @@ -476,9 +475,7 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) HTTP_Setup(req->resp, req->ws, req->vsl, HTTP_Resp); RES_BuildHttp(req); - assert(req->sp == sp); VCL_deliver_method(req); - assert(req->sp == sp); switch (req->handling) { case VCL_RET_DELIVER: break; @@ -494,12 +491,12 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req) } AZ(req->obj); http_Teardown(req->resp); - sp->step = STP_RESTART; + req->sp->step = STP_RESTART; return (0); default: WRONG("Illegal action in vcl_deliver{}"); } - sp->step = STP_DELIVER; + req->sp->step = STP_DELIVER; return (0); } @@ -519,11 +516,10 @@ DOT deliver -> DONE [style=bold,color=blue] */ static int -cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req) +cnt_deliver(struct worker *wrk, struct req *req) { struct busyobj *bo; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); @@ -539,7 +535,7 @@ cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req) HSH_Deref(&wrk->stats, NULL, &req->obj); VBO_DerefBusyObj(wrk, &req->busyobj); req->err_code = 503; - sp->step = STP_ERROR; + req->sp->step = STP_ERROR; return (0); } VBO_DerefBusyObj(wrk, &req->busyobj); @@ -576,13 +572,12 @@ DOT rsterr [label="RESTART",shape=plaintext] */ static int -cnt_error(struct sess *sp, struct worker *wrk, struct req *req) +cnt_error(struct worker *wrk, struct req *req) { struct http *h; struct busyobj *bo; char date[40]; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AZ(req->objcore); @@ -591,7 +586,7 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) bo = VBO_GetBusyObj(wrk); req->busyobj = bo; - bo->vsl->wid = sp->vsl_id; + bo->vsl->wid = req->sp->vsl_id; AZ(bo->stats); bo->stats = &wrk->stats; req->objcore = HSH_NewObjCore(wrk); @@ -625,15 +620,13 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) http_PutResponse(h, req->err_reason); else http_PutResponse(h, http_StatusMessage(req->err_code)); - assert(req->sp == sp); VCL_error_method(req); - assert(req->sp == sp); if (req->handling == VCL_RET_RESTART && req->restarts < cache_param->max_restarts) { HSH_Drop(wrk, &req->obj); VBO_DerefBusyObj(wrk, &req->busyobj); - sp->step = STP_RESTART; + req->sp->step = STP_RESTART; return (0); } else if (req->handling == VCL_RET_RESTART) req->handling = VCL_RET_DELIVER; @@ -648,7 +641,7 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req) req->err_reason = NULL; http_Teardown(bo->bereq); VBO_DerefBusyObj(wrk, &req->busyobj); - sp->step = STP_PREPRESP; + req->sp->step = STP_PREPRESP; return (0); } @@ -666,12 +659,11 @@ DOT fetch -> fetchbody [style=bold,color=blue] */ static int -cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) +cnt_fetch(struct worker *wrk, struct req *req) { int i, need_host_hdr; struct busyobj *bo; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -736,16 +728,14 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) AZ(bo->do_esi); AZ(bo->do_pass); - assert(req->sp == sp); VCL_fetch_method(req); - assert(req->sp == sp); if (bo->do_pass) req->objcore->flags |= OC_F_PASS; switch (req->handling) { case VCL_RET_DELIVER: - sp->step = STP_FETCHBODY; + req->sp->step = STP_FETCHBODY; return (0); default: break; @@ -771,10 +761,10 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req) switch (req->handling) { case VCL_RET_RESTART: - sp->step = STP_RESTART; + req->sp->step = STP_RESTART; return (0); case VCL_RET_ERROR: - sp->step = STP_ERROR; + req->sp->step = STP_ERROR; return (0); default: WRONG("Illegal action in vcl_fetch{}"); @@ -795,7 +785,7 @@ DOT fetchbody:out -> prepresp [style=bold,color=blue] */ static int -cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) +cnt_fetchbody(struct worker *wrk, struct req *req) { struct http *hp, *hp2; char *b; @@ -805,7 +795,6 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) int varyl = 0, pass; struct busyobj *bo; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); bo = req->busyobj; @@ -930,7 +919,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) bo->stats = NULL; if (req->obj == NULL) { req->err_code = 503; - sp->step = STP_ERROR; + req->sp->step = STP_ERROR; VDI_CloseFd(&bo->vbc); VBO_DerefBusyObj(wrk, &req->busyobj); return (0); @@ -1011,12 +1000,12 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req) HSH_Deref(&wrk->stats, NULL, &req->obj); VBO_DerefBusyObj(wrk, &req->busyobj); req->err_code = 503; - sp->step = STP_ERROR; + req->sp->step = STP_ERROR; return (0); } assert(WRW_IsReleased(wrk)); - sp->step = STP_PREPRESP; + req->sp->step = STP_PREPRESP; return (0); } @@ -1039,9 +1028,8 @@ DOT hit:del -> prepresp [label="deliver",style=bold,color=green] */ static int -cnt_hit(struct sess *sp, struct worker *wrk, struct req *req) +cnt_hit(struct worker *wrk, struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -1052,15 +1040,13 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req) assert(!(req->obj->objcore->flags & OC_F_PASS)); - assert(req->sp == sp); VCL_hit_method(req); - assert(req->sp == sp); if (req->handling == VCL_RET_DELIVER) { //AZ(req->busyobj->bereq->ws); //AZ(req->busyobj->beresp->ws); (void)FetchReqBody(req, 0); - sp->step = STP_PREPRESP; + req->sp->step = STP_PREPRESP; return (0); } @@ -1070,13 +1056,13 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req) switch(req->handling) { case VCL_RET_PASS: - sp->step = STP_PASS; + req->sp->step = STP_PASS; return (0); case VCL_RET_ERROR: - sp->step = STP_ERROR; + req->sp->step = STP_ERROR; return (0); case VCL_RET_RESTART: - sp->step = STP_RESTART; + req->sp->step = STP_RESTART; return (0); default: WRONG("Illegal action in vcl_hit{}"); @@ -1103,13 +1089,12 @@ DOT lookup:yes -> pass [style=bold,color=red] */ static int -cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) +cnt_lookup(struct worker *wrk, struct req *req) { struct objcore *oc; struct object *o; struct objhead *oh; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AZ(req->objcore); @@ -1127,7 +1112,6 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) * worker thread. We return to STP_LOOKUP when the busy * object has been unbusied, and still have the hash digest * around to do the lookup with. - * NB: Do not access sp any more ! */ return (2); } @@ -1156,7 +1140,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) req->vary_e = NULL; req->objcore = oc; - sp->step = STP_MISS; + req->sp->step = STP_MISS; return (0); } @@ -1177,13 +1161,13 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req) VSLb(req->vsl, SLT_HitPass, "%u", req->obj->xid); (void)HSH_Deref(&wrk->stats, NULL, &req->obj); AZ(req->objcore); - sp->step = STP_PASS; + req->sp->step = STP_PASS; return (0); } wrk->stats.cache_hit++; VSLb(req->vsl, SLT_Hit, "%u", req->obj->xid); - sp->step = STP_HIT; + req->sp->step = STP_HIT; return (0); } @@ -1202,11 +1186,10 @@ DOT */ static int -cnt_miss(struct sess *sp, struct worker *wrk, struct req *req) +cnt_miss(struct worker *wrk, struct req *req) { struct busyobj *bo; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); @@ -1228,13 +1211,11 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req) http_SetHeader(bo->bereq, "Accept-Encoding: gzip"); } - assert(req->sp == sp); VCL_miss_method(req); - assert(req->sp == sp); if (req->handling == VCL_RET_FETCH) { CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - sp->step = STP_FETCH; + req->sp->step = STP_FETCH; return (0); } @@ -1245,13 +1226,13 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req) switch(req->handling) { case VCL_RET_ERROR: - sp->step = STP_ERROR; + req->sp->step = STP_ERROR; break; case VCL_RET_PASS: - sp->step = STP_PASS; + req->sp->step = STP_PASS; break; case VCL_RET_RESTART: - sp->step = STP_RESTART; + req->sp->step = STP_RESTART; break; default: WRONG("Illegal action in vcl_miss{}"); @@ -1277,11 +1258,10 @@ XDOT err_pass [label="ERROR",shape=plaintext] */ static int -cnt_pass(struct sess *sp, struct worker *wrk, struct req *req) +cnt_pass(struct worker *wrk, struct req *req) { struct busyobj *bo; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); @@ -1291,24 +1271,22 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req) req->busyobj = VBO_GetBusyObj(wrk); bo = req->busyobj; - bo->vsl->wid = sp->vsl_id; + bo->vsl->wid = req->sp->vsl_id; bo->refcount = 2; HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); http_FilterReq(req, HTTPH_R_PASS); - assert(req->sp == sp); VCL_pass_method(req); - assert(req->sp == sp); if (req->handling == VCL_RET_ERROR) { http_Teardown(bo->bereq); VBO_DerefBusyObj(wrk, &req->busyobj); - sp->step = STP_ERROR; + req->sp->step = STP_ERROR; return (0); } assert(req->handling == VCL_RET_PASS); wrk->acct_tmp.pass++; - sp->step = STP_FETCH; + req->sp->step = STP_FETCH; req->objcore = HSH_NewObjCore(wrk); req->objcore->busyobj = bo; @@ -1341,11 +1319,10 @@ DOT err_pipe [label="ERROR",shape=plaintext] */ static int -cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req) +cnt_pipe(struct worker *wrk, struct req *req) { struct busyobj *bo; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); @@ -1354,13 +1331,11 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req) wrk->acct_tmp.pipe++; req->busyobj = VBO_GetBusyObj(wrk); bo = req->busyobj; - bo->vsl->wid = sp->vsl_id; + bo->vsl->wid = req->sp->vsl_id; HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); http_FilterReq(req, 0); - assert(req->sp == sp); VCL_pipe_method(req); - assert(req->sp == sp); if (req->handling == VCL_RET_ERROR) INCOMPL(); @@ -1386,20 +1361,19 @@ DOT restart -> recv [color=purple] */ static int -cnt_restart(struct sess *sp, const struct worker *wrk, struct req *req) +cnt_restart(const struct worker *wrk, struct req *req) { - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); req->director = NULL; if (++req->restarts >= cache_param->max_restarts) { req->err_code = 503; - sp->step = STP_ERROR; + req->sp->step = STP_ERROR; } else { req->err_code = 0; - sp->step = STP_RECV; + req->sp->step = STP_RECV; } return (0); } @@ -1433,12 +1407,11 @@ DOT hash -> lookup [label="hash",style=bold,color=green] */ static int -cnt_recv(struct sess *sp, const struct worker *wrk, struct req *req) +cnt_recv(const struct worker *wrk, struct req *req) { unsigned recv_handling; struct SHA256Context sha256ctx; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); @@ -1457,9 +1430,7 @@ cnt_recv(struct sess *sp, const struct worker *wrk, struct req *req) http_CollectHdr(req->http, H_Cache_Control); - assert(req->sp == sp); VCL_recv_method(req); - assert(req->sp == sp); recv_handling = req->handling; if (cache_param->http_gzip_support && @@ -1475,9 +1446,7 @@ cnt_recv(struct sess *sp, const struct worker *wrk, struct req *req) req->sha256ctx = &sha256ctx; /* so HSH_AddString() can find it */ SHA256_Init(req->sha256ctx); - assert(req->sp == sp); VCL_hash_method(req); - assert(req->sp == sp); assert(req->handling == VCL_RET_HASH); SHA256_Final(req->digest, req->sha256ctx); req->sha256ctx = NULL; @@ -1489,7 +1458,7 @@ cnt_recv(struct sess *sp, const struct worker *wrk, struct req *req) switch(recv_handling) { case VCL_RET_LOOKUP: - sp->step = STP_LOOKUP; + req->sp->step = STP_LOOKUP; return (0); case VCL_RET_PIPE: if (req->esi_level > 0) { @@ -1497,13 +1466,13 @@ cnt_recv(struct sess *sp, const struct worker *wrk, struct req *req) INCOMPL(); return (1); } - sp->step = STP_PIPE; + req->sp->step = STP_PIPE; return (0); case VCL_RET_PASS: - sp->step = STP_PASS; + req->sp->step = STP_PASS; return (0); case VCL_RET_ERROR: - sp->step = STP_ERROR; + req->sp->step = STP_ERROR; return (0); default: WRONG("Illegal action in vcl_recv{}"); @@ -1523,12 +1492,11 @@ DOT start -> DONE [label=errors] */ static int -cnt_start(struct sess *sp, struct worker *wrk, struct req *req) +cnt_start(struct worker *wrk, struct req *req) { char *p; const char *r = "HTTP/1.1 100 Continue\r\n\r\n"; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AZ(req->restarts); @@ -1536,7 +1504,6 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) AZ(req->vcl); AZ(req->esi_level); assert(!isnan(req->t_req)); - assert(req->sp == sp); /* Update stats of various sorts */ wrk->stats.client_req++; @@ -1545,7 +1512,7 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) /* Assign XID and log */ req->xid = ++xids; /* XXX not locked */ VSLb(req->vsl, SLT_ReqStart, "%s %s %u", - sp->addr, sp->port, req->xid); + req->sp->addr, req->sp->port, req->xid); /* Borrow VCL reference from worker thread */ VCL_Refresh(&wrk->vcl); @@ -1559,7 +1526,7 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) /* If we could not even parse the request, just close */ if (req->err_code == 400) { - SES_Close(sp, "junk"); + SES_Close(req->sp, "junk"); return (1); } @@ -1574,8 +1541,8 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) if (req->err_code == 0 && http_GetHdr(req->http, H_Expect, &p)) { if (strcasecmp(p, "100-continue")) { req->err_code = 417; - } else if (strlen(r) != write(sp->fd, r, strlen(r))) { - SES_Close(sp, "remote closed"); + } else if (strlen(r) != write(req->sp->fd, r, strlen(r))) { + SES_Close(req->sp, "remote closed"); return (1); } } @@ -1587,9 +1554,9 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req) HTTP_Copy(req->http0, req->http); /* Copy for restart/ESI use */ if (req->err_code) - sp->step = STP_ERROR; + req->sp->step = STP_ERROR; else - sp->step = STP_RECV; + req->sp->step = STP_RECV; return (0); } diff --git a/include/tbl/steps.h b/include/tbl/steps.h index e429631..703c418 100644 --- a/include/tbl/steps.h +++ b/include/tbl/steps.h @@ -30,17 +30,17 @@ /*lint -save -e525 -e539 */ SESS_STEP(wait, WAIT, (sp, sp->wrk, sp->req)) -SESS_STEP(restart, RESTART, (sp, sp->wrk, sp->req)) -SESS_STEP(recv, RECV, (sp, sp->wrk, sp->req)) -SESS_STEP(start, START, (sp, sp->wrk, sp->req)) -SESS_STEP(pipe, PIPE, (sp, sp->wrk, sp->req)) -SESS_STEP(pass, PASS, (sp, sp->wrk, sp->req)) -SESS_STEP(lookup, LOOKUP, (sp, sp->wrk, sp->req)) -SESS_STEP(miss, MISS, (sp, sp->wrk, sp->req)) -SESS_STEP(hit, HIT, (sp, sp->wrk, sp->req)) -SESS_STEP(fetch, FETCH, (sp, sp->wrk, sp->req)) -SESS_STEP(fetchbody, FETCHBODY, (sp, sp->wrk, sp->req)) -SESS_STEP(prepresp, PREPRESP, (sp, sp->wrk, sp->req)) -SESS_STEP(deliver, DELIVER, (sp, sp->wrk, sp->req)) -SESS_STEP(error, ERROR, (sp, sp->wrk, sp->req)) +SESS_STEP(restart, RESTART, (wrk, req)) +SESS_STEP(recv, RECV, (wrk, req)) +SESS_STEP(start, START, (wrk, req)) +SESS_STEP(pipe, PIPE, (wrk, req)) +SESS_STEP(pass, PASS, (wrk, req)) +SESS_STEP(lookup, LOOKUP, (wrk, req)) +SESS_STEP(miss, MISS, (wrk, req)) +SESS_STEP(hit, HIT, (wrk, req)) +SESS_STEP(fetch, FETCH, (wrk, req)) +SESS_STEP(fetchbody, FETCHBODY, (wrk, req)) +SESS_STEP(prepresp, PREPRESP, (wrk, req)) +SESS_STEP(deliver, DELIVER, (wrk, req)) +SESS_STEP(error, ERROR, (wrk, req)) /*lint -restore */ From phk at FreeBSD.org Thu Dec 18 09:27:49 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:49 +0100 Subject: [experimental-ims] 8e253db More firmly split session and request states. Message-ID: commit 8e253dbe9a05cb32bea186d8927b5899a1268e16 Author: Poul-Henning Kamp Date: Wed Jun 20 10:25:25 2012 +0000 More firmly split session and request states. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 299dc93..ed8e9f9 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -122,12 +122,18 @@ typedef struct { /*--------------------------------------------------------------------*/ -enum step { -#define SESS_STEP(l, u, arg) STP_##u, +enum sess_step { +#define SESS_STEP(l, u) S_STP_##u, #include "tbl/steps.h" #undef SESS_STEP }; +enum req_step { +#define REQ_STEP(l, u, arg) R_STP_##u, +#include "tbl/steps.h" +#undef REQ_STEP +}; + /*--------------------------------------------------------------------*/ struct lock { void *priv; }; // Opaque @@ -562,6 +568,7 @@ struct req { uint8_t hash_always_miss; struct sess *sp; + enum req_step req_step; VTAILQ_ENTRY(req) w_list; /* The busy objhead we sleep on */ @@ -643,7 +650,7 @@ struct sess { unsigned magic; #define SESS_MAGIC 0x2c2f9c5a - enum step step; + enum sess_step sess_step; int fd; unsigned vsl_id; uint32_t vxid; diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 6e394e4..fff81b4 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -341,7 +341,7 @@ CNT_Session(struct sess *sp) * rather do the syscall in the worker thread. * On systems which return errors for ioctl, we close early */ - if (sp->step == STP_WAIT && VTCP_blocking(sp->fd)) { + if (sp->sess_step == S_STP_NEWREQ && VTCP_blocking(sp->fd)) { if (errno == ECONNRESET) SES_Close(sp, "remote closed"); else @@ -356,11 +356,12 @@ CNT_Session(struct sess *sp) * Possible entrance states */ assert( - sp->step == STP_WAIT || - sp->step == STP_LOOKUP || - sp->step == STP_START); + sp->sess_step == S_STP_NEWREQ || + (sp->req != NULL && + (sp->req->req_step == R_STP_LOOKUP || + sp->req->req_step == R_STP_START))); - if (sp->step != STP_WAIT) { + if (sp->sess_step == S_STP_WORKING) { done = CNT_Request(sp->req); if (done == 2) return; @@ -370,21 +371,24 @@ CNT_Session(struct sess *sp) case SESS_DONE_RET_GONE: return; case SESS_DONE_RET_WAIT: - sp->step = STP_WAIT; + sp->sess_step = S_STP_NEWREQ; break; case SESS_DONE_RET_START: - sp->step = STP_START; + sp->sess_step = S_STP_WORKING; + sp->req->req_step = R_STP_START; break; default: WRONG("Illegal enum cnt_sess_done_ret"); } } - if (sp->step == STP_WAIT) { + if (sp->sess_step == S_STP_NEWREQ) { done = cnt_wait(sp, wrk, sp->req); - if (done) + if (done) { return; - sp->step = STP_START; + } + sp->sess_step = S_STP_WORKING; + sp->req->req_step = R_STP_START; } } } @@ -491,12 +495,12 @@ cnt_prepresp(struct worker *wrk, struct req *req) } AZ(req->obj); http_Teardown(req->resp); - req->sp->step = STP_RESTART; + req->req_step = R_STP_RESTART; return (0); default: WRONG("Illegal action in vcl_deliver{}"); } - req->sp->step = STP_DELIVER; + req->req_step = R_STP_DELIVER; return (0); } @@ -535,7 +539,7 @@ cnt_deliver(struct worker *wrk, struct req *req) HSH_Deref(&wrk->stats, NULL, &req->obj); VBO_DerefBusyObj(wrk, &req->busyobj); req->err_code = 503; - req->sp->step = STP_ERROR; + req->req_step = R_STP_ERROR; return (0); } VBO_DerefBusyObj(wrk, &req->busyobj); @@ -626,7 +630,7 @@ cnt_error(struct worker *wrk, struct req *req) req->restarts < cache_param->max_restarts) { HSH_Drop(wrk, &req->obj); VBO_DerefBusyObj(wrk, &req->busyobj); - req->sp->step = STP_RESTART; + req->req_step = R_STP_RESTART; return (0); } else if (req->handling == VCL_RET_RESTART) req->handling = VCL_RET_DELIVER; @@ -641,7 +645,7 @@ cnt_error(struct worker *wrk, struct req *req) req->err_reason = NULL; http_Teardown(bo->bereq); VBO_DerefBusyObj(wrk, &req->busyobj); - req->sp->step = STP_PREPRESP; + req->req_step = R_STP_PREPRESP; return (0); } @@ -735,7 +739,7 @@ cnt_fetch(struct worker *wrk, struct req *req) switch (req->handling) { case VCL_RET_DELIVER: - req->sp->step = STP_FETCHBODY; + req->req_step = R_STP_FETCHBODY; return (0); default: break; @@ -761,10 +765,10 @@ cnt_fetch(struct worker *wrk, struct req *req) switch (req->handling) { case VCL_RET_RESTART: - req->sp->step = STP_RESTART; + req->req_step = R_STP_RESTART; return (0); case VCL_RET_ERROR: - req->sp->step = STP_ERROR; + req->req_step = R_STP_ERROR; return (0); default: WRONG("Illegal action in vcl_fetch{}"); @@ -919,7 +923,7 @@ cnt_fetchbody(struct worker *wrk, struct req *req) bo->stats = NULL; if (req->obj == NULL) { req->err_code = 503; - req->sp->step = STP_ERROR; + req->req_step = R_STP_ERROR; VDI_CloseFd(&bo->vbc); VBO_DerefBusyObj(wrk, &req->busyobj); return (0); @@ -1000,12 +1004,12 @@ cnt_fetchbody(struct worker *wrk, struct req *req) HSH_Deref(&wrk->stats, NULL, &req->obj); VBO_DerefBusyObj(wrk, &req->busyobj); req->err_code = 503; - req->sp->step = STP_ERROR; + req->req_step = R_STP_ERROR; return (0); } assert(WRW_IsReleased(wrk)); - req->sp->step = STP_PREPRESP; + req->req_step = R_STP_PREPRESP; return (0); } @@ -1046,7 +1050,7 @@ cnt_hit(struct worker *wrk, struct req *req) //AZ(req->busyobj->bereq->ws); //AZ(req->busyobj->beresp->ws); (void)FetchReqBody(req, 0); - req->sp->step = STP_PREPRESP; + req->req_step = R_STP_PREPRESP; return (0); } @@ -1056,13 +1060,13 @@ cnt_hit(struct worker *wrk, struct req *req) switch(req->handling) { case VCL_RET_PASS: - req->sp->step = STP_PASS; + req->req_step = R_STP_PASS; return (0); case VCL_RET_ERROR: - req->sp->step = STP_ERROR; + req->req_step = R_STP_ERROR; return (0); case VCL_RET_RESTART: - req->sp->step = STP_RESTART; + req->req_step = R_STP_RESTART; return (0); default: WRONG("Illegal action in vcl_hit{}"); @@ -1140,7 +1144,7 @@ cnt_lookup(struct worker *wrk, struct req *req) req->vary_e = NULL; req->objcore = oc; - req->sp->step = STP_MISS; + req->req_step = R_STP_MISS; return (0); } @@ -1161,13 +1165,13 @@ cnt_lookup(struct worker *wrk, struct req *req) VSLb(req->vsl, SLT_HitPass, "%u", req->obj->xid); (void)HSH_Deref(&wrk->stats, NULL, &req->obj); AZ(req->objcore); - req->sp->step = STP_PASS; + req->req_step = R_STP_PASS; return (0); } wrk->stats.cache_hit++; VSLb(req->vsl, SLT_Hit, "%u", req->obj->xid); - req->sp->step = STP_HIT; + req->req_step = R_STP_HIT; return (0); } @@ -1215,7 +1219,7 @@ cnt_miss(struct worker *wrk, struct req *req) if (req->handling == VCL_RET_FETCH) { CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - req->sp->step = STP_FETCH; + req->req_step = R_STP_FETCH; return (0); } @@ -1226,13 +1230,13 @@ cnt_miss(struct worker *wrk, struct req *req) switch(req->handling) { case VCL_RET_ERROR: - req->sp->step = STP_ERROR; + req->req_step = R_STP_ERROR; break; case VCL_RET_PASS: - req->sp->step = STP_PASS; + req->req_step = R_STP_PASS; break; case VCL_RET_RESTART: - req->sp->step = STP_RESTART; + req->req_step = R_STP_RESTART; break; default: WRONG("Illegal action in vcl_miss{}"); @@ -1281,12 +1285,12 @@ cnt_pass(struct worker *wrk, struct req *req) if (req->handling == VCL_RET_ERROR) { http_Teardown(bo->bereq); VBO_DerefBusyObj(wrk, &req->busyobj); - req->sp->step = STP_ERROR; + req->req_step = R_STP_ERROR; return (0); } assert(req->handling == VCL_RET_PASS); wrk->acct_tmp.pass++; - req->sp->step = STP_FETCH; + req->req_step = R_STP_FETCH; req->objcore = HSH_NewObjCore(wrk); req->objcore->busyobj = bo; @@ -1370,10 +1374,10 @@ cnt_restart(const struct worker *wrk, struct req *req) req->director = NULL; if (++req->restarts >= cache_param->max_restarts) { req->err_code = 503; - req->sp->step = STP_ERROR; + req->req_step = R_STP_ERROR; } else { req->err_code = 0; - req->sp->step = STP_RECV; + req->req_step = R_STP_RECV; } return (0); } @@ -1458,7 +1462,7 @@ cnt_recv(const struct worker *wrk, struct req *req) switch(recv_handling) { case VCL_RET_LOOKUP: - req->sp->step = STP_LOOKUP; + req->req_step = R_STP_LOOKUP; return (0); case VCL_RET_PIPE: if (req->esi_level > 0) { @@ -1466,13 +1470,13 @@ cnt_recv(const struct worker *wrk, struct req *req) INCOMPL(); return (1); } - req->sp->step = STP_PIPE; + req->req_step = R_STP_PIPE; return (0); case VCL_RET_PASS: - req->sp->step = STP_PASS; + req->req_step = R_STP_PASS; return (0); case VCL_RET_ERROR: - req->sp->step = STP_ERROR; + req->req_step = R_STP_ERROR; return (0); default: WRONG("Illegal action in vcl_recv{}"); @@ -1554,9 +1558,9 @@ cnt_start(struct worker *wrk, struct req *req) HTTP_Copy(req->http0, req->http); /* Copy for restart/ESI use */ if (req->err_code) - req->sp->step = STP_ERROR; + req->req_step = R_STP_ERROR; else - req->sp->step = STP_RECV; + req->req_step = R_STP_RECV; return (0); } @@ -1610,9 +1614,9 @@ CNT_Request(struct req *req) * Possible entrance states */ assert( - sp->step == STP_LOOKUP || - sp->step == STP_START || - sp->step == STP_RECV); + req->req_step == R_STP_LOOKUP || + req->req_step == R_STP_START || + req->req_step == R_STP_RECV); for (done = 0; !done; ) { assert(sp->wrk == wrk); @@ -1628,17 +1632,15 @@ CNT_Request(struct req *req) AN(req->sp); assert(req->sp == sp); - assert(sp->step != STP_WAIT); - - switch (sp->step) { -#define SESS_STEP(l,u,arg) \ - case STP_##u: \ + switch (req->req_step) { +#define REQ_STEP(l,u,arg) \ + case R_STP_##u: \ if (cache_param->diag_bitmap & 0x01) \ cnt_diag(sp, #u); \ done = cnt_##l arg; \ break; #include "tbl/steps.h" -#undef SESS_STEP +#undef REQ_STEP default: WRONG("State engine misfire"); } diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index d346006..9075a77 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -82,7 +82,7 @@ ved_include(struct req *req, const char *src, const char *host) * XXX: make sure we don't trip up the check in vcl_recv. */ req->director = NULL; - req->sp->step = STP_RECV; + req->req_step = R_STP_RECV; http_ForceGet(req->http); /* Don't do conditionals */ diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index ce2b393..c4511e8 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -230,10 +230,24 @@ pan_busyobj(const struct busyobj *bo) static void pan_req(const struct req *req) { - const char *hand; + const char *hand, *stp; VSB_printf(pan_vsp, "req = %p {\n", req); - VSB_printf(pan_vsp, " sp = %p, xid = %u,\n", req->sp, req->xid); + + VSB_printf(pan_vsp, " sp = %p, xid = %u,", + req->sp, req->xid); + + switch (req->req_step) { +#define REQ_STEP(l, u, arg) case R_STP_##u: stp = "R_STP_" #u; break; +#include "tbl/steps.h" +#undef REQ_STEP + default: stp = NULL; + } + if (stp != NULL) + VSB_printf(pan_vsp, " step = %s,\n", stp); + else + VSB_printf(pan_vsp, " step = 0x%x,\n", req->req_step); + hand = VCL_Return_Name(req->handling); if (hand != NULL) VSB_printf(pan_vsp, " handling = %s,\n", hand); @@ -277,8 +291,8 @@ pan_sess(const struct sess *sp) VSB_printf(pan_vsp, " client = %s %s,\n", sp->addr ? sp->addr : "?.?.?.?", sp->port ? sp->port : "?"); - switch (sp->step) { -#define SESS_STEP(l, u, arg) case STP_##u: stp = "STP_" #u; break; + switch (sp->sess_step) { +#define SESS_STEP(l, u) case S_STP_##u: stp = "S_STP_" #u; break; #include "tbl/steps.h" #undef SESS_STEP default: stp = NULL; @@ -286,7 +300,7 @@ pan_sess(const struct sess *sp) if (stp != NULL) VSB_printf(pan_vsp, " step = %s,\n", stp); else - VSB_printf(pan_vsp, " step = 0x%x,\n", sp->step); + VSB_printf(pan_vsp, " step = 0x%x,\n", sp->sess_step); if (sp->wrk != NULL) pan_wrk(sp->wrk); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 34dad47..aee58b0 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -159,7 +159,7 @@ SES_pool_accept_task(struct worker *wrk, void *arg) VCA_FailSess(wrk); } else { VCA_SetupSess(wrk, sp); - sp->step = STP_WAIT; + sp->sess_step = S_STP_NEWREQ; ses_pool_task(wrk, sp); } } @@ -207,7 +207,7 @@ void SES_Handle(struct sess *sp, double now) { - sp->step = STP_WAIT; + sp->sess_step = S_STP_NEWREQ; sp->t_rx = now; (void)SES_Schedule(sp); } diff --git a/include/tbl/steps.h b/include/tbl/steps.h index 703c418..cef939b 100644 --- a/include/tbl/steps.h +++ b/include/tbl/steps.h @@ -29,18 +29,25 @@ */ /*lint -save -e525 -e539 */ -SESS_STEP(wait, WAIT, (sp, sp->wrk, sp->req)) -SESS_STEP(restart, RESTART, (wrk, req)) -SESS_STEP(recv, RECV, (wrk, req)) -SESS_STEP(start, START, (wrk, req)) -SESS_STEP(pipe, PIPE, (wrk, req)) -SESS_STEP(pass, PASS, (wrk, req)) -SESS_STEP(lookup, LOOKUP, (wrk, req)) -SESS_STEP(miss, MISS, (wrk, req)) -SESS_STEP(hit, HIT, (wrk, req)) -SESS_STEP(fetch, FETCH, (wrk, req)) -SESS_STEP(fetchbody, FETCHBODY, (wrk, req)) -SESS_STEP(prepresp, PREPRESP, (wrk, req)) -SESS_STEP(deliver, DELIVER, (wrk, req)) -SESS_STEP(error, ERROR, (wrk, req)) + +#ifdef SESS_STEP +SESS_STEP(newreq, NEWREQ) +SESS_STEP(working, WORKING) +#endif + +#ifdef REQ_STEP +REQ_STEP(restart, RESTART, (wrk, req)) +REQ_STEP(recv, RECV, (wrk, req)) +REQ_STEP(start, START, (wrk, req)) +REQ_STEP(pipe, PIPE, (wrk, req)) +REQ_STEP(pass, PASS, (wrk, req)) +REQ_STEP(lookup, LOOKUP, (wrk, req)) +REQ_STEP(miss, MISS, (wrk, req)) +REQ_STEP(hit, HIT, (wrk, req)) +REQ_STEP(fetch, FETCH, (wrk, req)) +REQ_STEP(fetchbody, FETCHBODY, (wrk, req)) +REQ_STEP(prepresp, PREPRESP, (wrk, req)) +REQ_STEP(deliver, DELIVER, (wrk, req)) +REQ_STEP(error, ERROR, (wrk, req)) +#endif /*lint -restore */ From martin at varnish-software.com Thu Dec 18 09:27:50 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] c4d56d7 Add test case for v-c.org ticket #1157 (connection close on pass and do_stream) Message-ID: commit c4d56d7670512c41c54d3b66283f42c2230e6e20 Author: Martin Blix Grydeland Date: Wed Jun 20 14:49:44 2012 +0200 Add test case for v-c.org ticket #1157 (connection close on pass and do_stream) Bug fix itself is not applicable to trunk, but has been pushed on 3.0. diff --git a/bin/varnishtest/tests/r01157.vtc b/bin/varnishtest/tests/r01157.vtc new file mode 100644 index 0000000..9cecd9a --- /dev/null +++ b/bin/varnishtest/tests/r01157.vtc @@ -0,0 +1,26 @@ +varnishtest "#1157 - Connection close on pass and do_stream" + +server s1 { + rxreq + txresp + expect req.url == "/" + + rxreq + txresp + expect req.url == "/" +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { return (pass); } + sub vcl_fetch { set beresp.do_stream = true; } +} -start + +client c1 { + txreq -url / + rxresp + expect resp.status == 200 + + txreq -url / + rxresp + expect resp.status == 200 +} -run From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 78ad7c4 Continue the move towards multiple concurrent requests per session Message-ID: commit 78ad7c4cae8493cffa5caa414e504ee1fe871e34 Author: Poul-Henning Kamp Date: Mon Jun 25 07:14:13 2012 +0000 Continue the move towards multiple concurrent requests per session diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index ed8e9f9..3673823 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -743,7 +743,7 @@ void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj); void VBO_Free(struct busyobj **vbo); /* cache_center.c [CNT] */ -int CNT_Request(struct req *); +int CNT_Request(struct worker *, struct req *); void CNT_Session(struct sess *); void CNT_Init(void); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index fff81b4..24e6e37 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -362,7 +362,7 @@ CNT_Session(struct sess *sp) sp->req->req_step == R_STP_START))); if (sp->sess_step == S_STP_WORKING) { - done = CNT_Request(sp->req); + done = CNT_Request(sp->wrk, sp->req); if (done == 2) return; assert(done == 1); @@ -1572,43 +1572,23 @@ cnt_start(struct worker *wrk, struct req *req) */ static void -cnt_diag(struct sess *sp, const char *state) +cnt_diag(struct req *req, const char *state) { - void *vcl; - void *obj; - if (sp->req == NULL) { - vcl = NULL; - obj = NULL; - } else { - vcl = sp->req->vcl; - obj = sp->req->obj; - } + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - if (sp->req != NULL) { - VSLb(sp->req->vsl, SLT_Debug, - "vsl_id %u STP_%s sp %p obj %p vcl %p", - sp->vsl_id, state, sp, obj, vcl); - VSL_Flush(sp->req->vsl, 0); - } else { - VSL(SLT_Debug, sp->vsl_id, - "vsl_id %u STP_%s sp %p obj %p vcl %p", - sp->vsl_id, state, sp, obj, vcl); - } + VSLb(req->vsl, SLT_Debug, "vsl_id %u STP_%s sp %p obj %p vcl %p", + req->sp->vsl_id, state, req->sp, req->obj, req->vcl); + VSL_Flush(req->vsl, 0); } int -CNT_Request(struct req *req) +CNT_Request(struct worker *wrk, struct req *req) { int done; - struct worker *wrk; - struct sess *sp; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - sp = req->sp; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - wrk = sp->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); /* * Possible entrance states @@ -1619,24 +1599,20 @@ CNT_Request(struct req *req) req->req_step == R_STP_RECV); for (done = 0; !done; ) { - assert(sp->wrk == wrk); /* * This is a good place to be paranoid about the various * pointers still pointing to the things we expect. */ - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); WS_Assert(wrk->aws); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - AN(req->sp); - assert(req->sp == sp); switch (req->req_step) { #define REQ_STEP(l,u,arg) \ case R_STP_##u: \ if (cache_param->diag_bitmap & 0x01) \ - cnt_diag(sp, #u); \ + cnt_diag(req, #u); \ done = cnt_##l arg; \ break; #include "tbl/steps.h" @@ -1648,7 +1624,7 @@ CNT_Request(struct req *req) CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); } if (done == 1) - SES_Charge(sp); + SES_Charge(req->sp); assert(WRW_IsReleased(wrk)); return (done); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 9075a77..7f06dd4 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -95,7 +95,7 @@ ved_include(struct req *req, const char *src, const char *host) sxid = req->xid; while (1) { req->sp->wrk = wrk; - i = CNT_Request(req); + i = CNT_Request(wrk, req); if (i == 1) break; assert(i == 2); From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 09bf441 Mostly move sp->wrk to req->wrk Message-ID: commit 09bf441ebdd412d0b77001343273699c2391e54f Author: Poul-Henning Kamp Date: Mon Jun 25 07:59:54 2012 +0000 Mostly move sp->wrk to req->wrk diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 3673823..3a8559f 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -568,6 +568,7 @@ struct req { uint8_t hash_always_miss; struct sess *sp; + struct worker *wrk; enum req_step req_step; VTAILQ_ENTRY(req) w_list; @@ -744,7 +745,7 @@ void VBO_Free(struct busyobj **vbo); /* cache_center.c [CNT] */ int CNT_Request(struct worker *, struct req *); -void CNT_Session(struct sess *); +void CNT_Session(struct worker *, struct sess *); void CNT_Init(void); /* cache_cli.c [CLI] */ diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 24e6e37..1eb397e 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -323,15 +323,13 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) */ void -CNT_Session(struct sess *sp) +CNT_Session(struct worker *wrk, struct sess *sp) { int done; enum cnt_sess_done_ret sdr; - struct worker *wrk; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - wrk = sp->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); /* * Whenever we come in from the acceptor or waiter, we need to set @@ -363,8 +361,10 @@ CNT_Session(struct sess *sp) if (sp->sess_step == S_STP_WORKING) { done = CNT_Request(sp->wrk, sp->req); - if (done == 2) + if (done == 2) { + sp->wrk = NULL; return; + } assert(done == 1); sdr = cnt_sess_done(sp, wrk, sp->req); switch (sdr) { @@ -1598,6 +1598,8 @@ CNT_Request(struct worker *wrk, struct req *req) req->req_step == R_STP_START || req->req_step == R_STP_RECV); + req->wrk = wrk; + for (done = 0; !done; ) { /* * This is a good place to be paranoid about the various @@ -1626,6 +1628,8 @@ CNT_Request(struct worker *wrk, struct req *req) if (done == 1) SES_Charge(req->sp); + req->wrk = NULL; + assert(WRW_IsReleased(wrk)); return (done); } diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 7f06dd4..7e4305e 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -51,7 +51,7 @@ ved_include(struct req *req, const char *src, const char *host) unsigned sxid, res_mode; int i; - wrk = req->sp->wrk; + wrk = req->wrk; if (req->esi_level >= cache_param->max_esi_depth) return; @@ -94,17 +94,17 @@ ved_include(struct req *req, const char *src, const char *host) sxid = req->xid; while (1) { - req->sp->wrk = wrk; + req->wrk = wrk; i = CNT_Request(wrk, req); if (i == 1) break; assert(i == 2); - AZ(req->sp->wrk); + AZ(req->wrk); DSL(0x20, SLT_Debug, req->sp->vsl_id, "loop waiting for ESI"); (void)usleep(10000); } req->xid = sxid; - AN(req->sp->wrk); + req->wrk = wrk; req->esi_level--; req->obj = obj; req->res_mode = res_mode; @@ -113,9 +113,9 @@ ved_include(struct req *req, const char *src, const char *host) WS_Reset(req->ws, sp_ws_wm); WS_Reset(wrk->aws, wrk_ws_wm); /* XXX ? */ - WRW_Reserve(req->sp->wrk, &req->sp->fd, req->vsl, req->t_resp); + WRW_Reserve(req->wrk, &req->sp->fd, req->vsl, req->t_resp); if (req->res_mode & RES_CHUNKED) - WRW_Chunked(req->sp->wrk); + WRW_Chunked(req->wrk); } /*--------------------------------------------------------------------*/ @@ -185,22 +185,22 @@ ved_pretend_gzip(struct req *req, const uint8_t *p, ssize_t l) while (l > 0) { if (l >= 65535) { lx = 65535; - (void)WRW_Write(req->sp->wrk, buf1, sizeof buf1); + (void)WRW_Write(req->wrk, buf1, sizeof buf1); } else { lx = (uint16_t)l; buf2[0] = 0; vle16enc(buf2 + 1, lx); vle16enc(buf2 + 3, ~lx); - (void)WRW_Write(req->sp->wrk, buf2, sizeof buf2); + (void)WRW_Write(req->wrk, buf2, sizeof buf2); } - (void)WRW_Write(req->sp->wrk, p, lx); + (void)WRW_Write(req->wrk, p, lx); req->crc = crc32(req->crc, p, lx); req->l_crc += lx; l -= lx; p += lx; } /* buf2 is local, have to flush */ - (void)WRW_Flush(req->sp->wrk); + (void)WRW_Flush(req->wrk); } /*--------------------------------------------------------------------- @@ -250,7 +250,7 @@ ESI_Deliver(struct req *req) if (isgzip && !(req->res_mode & RES_GUNZIP)) { assert(sizeof gzip_hdr == 10); /* Send out the gzip header */ - (void)WRW_Write(req->sp->wrk, gzip_hdr, 10); + (void)WRW_Write(req->wrk, gzip_hdr, 10); req->l_crc = 0; req->gzip_resp = 1; req->crc = crc32(0L, Z_NULL, 0); @@ -306,7 +306,7 @@ ESI_Deliver(struct req *req) * We have a gzip'ed VEC and delivers * a gzip'ed ESI response. */ - (void)WRW_Write(req->sp->wrk, + (void)WRW_Write(req->wrk, st->ptr + off, l2); } else if (req->gzip_resp) { /* @@ -321,9 +321,9 @@ ESI_Deliver(struct req *req) * response */ AN(vgz); - i = VGZ_WrwGunzip(req->sp->wrk, vgz, + i = VGZ_WrwGunzip(req->wrk, vgz, st->ptr + off, l2); - if (WRW_Error(req->sp->wrk)) { + if (WRW_Error(req->wrk)) { SES_Close(req->sp, "remote closed"); p = e; @@ -334,7 +334,7 @@ ESI_Deliver(struct req *req) /* * Ungzip'ed VEC, ungzip'ed ESI response */ - (void)WRW_Write(req->sp->wrk, + (void)WRW_Write(req->wrk, st->ptr + off, l2); } off += l2; @@ -374,8 +374,8 @@ ESI_Deliver(struct req *req) r = (void*)strchr((const char*)q, '\0'); AN(r); if (vgz != NULL) - VGZ_WrwFlush(req->sp->wrk, vgz); - if (WRW_Flush(req->sp->wrk)) { + VGZ_WrwFlush(req->wrk, vgz); + if (WRW_Flush(req->wrk)) { SES_Close(req->sp, "remote closed"); p = e; break; @@ -391,7 +391,7 @@ ESI_Deliver(struct req *req) } } if (vgz != NULL) { - VGZ_WrwFlush(req->sp->wrk, vgz); + VGZ_WrwFlush(req->wrk, vgz); (void)VGZ_Destroy(&vgz); } if (req->gzip_resp && req->esi_level == 0) { @@ -408,9 +408,9 @@ ESI_Deliver(struct req *req) /* MOD(2^32) length */ vle32enc(tailbuf + 9, req->l_crc); - (void)WRW_Write(req->sp->wrk, tailbuf, 13); + (void)WRW_Write(req->wrk, tailbuf, 13); } - (void)WRW_Flush(req->sp->wrk); + (void)WRW_Flush(req->wrk); } /*--------------------------------------------------------------------- @@ -444,7 +444,7 @@ ved_deliver_byterange(const struct req *req, ssize_t low, ssize_t high) l = high - lx; assert(lx >= low && lx + l <= high); if (l != 0) - (void)WRW_Write(req->sp->wrk, p, l); + (void)WRW_Write(req->wrk, p, l); if (p + l < st->ptr + st->len) return(p[l]); lx += l; @@ -499,7 +499,7 @@ ESI_DeliverChild(struct req *req) */ *dbits = ved_deliver_byterange(req, start/8, last/8); *dbits &= ~(1U << (last & 7)); - (void)WRW_Write(req->sp->wrk, dbits, 1); + (void)WRW_Write(req->wrk, dbits, 1); cc = ved_deliver_byterange(req, 1 + last/8, stop/8); switch((int)(stop & 7)) { case 0: /* xxxxxxxx */ @@ -543,7 +543,7 @@ ESI_DeliverChild(struct req *req) INCOMPL(); } if (lpad > 0) - (void)WRW_Write(req->sp->wrk, dbits + 1, lpad); + (void)WRW_Write(req->wrk, dbits + 1, lpad); /* We need the entire tail, but it may not be in one storage segment */ st = VTAILQ_LAST(&req->obj->store, storagehead); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 0e2fa5f..ab2aabc 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -406,8 +406,8 @@ FetchReqBody(struct req *req, int sendbody) content_length -= rdcnt; if (sendbody) { /* XXX: stats ? */ - (void)WRW_Write(req->sp->wrk, buf, rdcnt); - if (WRW_Flush(req->sp->wrk)) + (void)WRW_Write(req->wrk, buf, rdcnt); + if (WRW_Flush(req->wrk)) return (2); } } @@ -441,7 +441,7 @@ FetchHdr(struct req *req, int need_host_hdr, int sendbody) int i; struct http_conn *htc; - wrk = req->sp->wrk; + wrk = req->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); bo = req->busyobj; diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index e7c2a49..c6f56e3 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -299,7 +299,7 @@ HSH_Lookup(struct req *req) int busy_found; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - wrk = req->sp->wrk; + wrk = req->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req->http, HTTP_MAGIC); AN(req->director); @@ -431,7 +431,6 @@ HSH_Lookup(struct req *req) * calls us again */ req->hash_objhead = oh; - req->sp->wrk = NULL; Lck_Unlock(&oh->mtx); return (NULL); } @@ -482,9 +481,9 @@ hsh_rush(struct dstat *ds, struct objhead *oh) if (req == NULL) break; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + AZ(req->wrk); sp = req->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - AZ(sp->wrk); VTAILQ_REMOVE(&wl->list, req, w_list); DSL(0x20, SLT_Debug, sp->vsl_id, "off waiting list"); if (SES_Schedule(sp)) { @@ -533,7 +532,7 @@ HSH_Purge(struct req *req, struct objhead *oh, double ttl, double grace) continue; } - (void)oc_getobj(&req->sp->wrk->stats, oc); + (void)oc_getobj(&req->wrk->stats, oc); /* XXX: still needed ? */ xxxassert(spc >= sizeof *ocp); @@ -551,14 +550,14 @@ HSH_Purge(struct req *req, struct objhead *oh, double ttl, double grace) for (n = 0; n < nobj; n++) { oc = ocp[n]; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - o = oc_getobj(&req->sp->wrk->stats, oc); + o = oc_getobj(&req->wrk->stats, oc); if (o == NULL) continue; CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); o->exp.ttl = ttl; o->exp.grace = grace; EXP_Rearm(o); - (void)HSH_Deref(&req->sp->wrk->stats, NULL, &o); + (void)HSH_Deref(&req->wrk->stats, NULL, &o); } WS_Release(req->ws, 0); } diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index c4511e8..411b8e9 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -261,8 +261,8 @@ pan_req(const struct req *req) VSB_printf(pan_vsp, " restarts = %d, esi_level = %d\n", req->restarts, req->esi_level); - if (req->busyobj != NULL) - pan_busyobj(req->busyobj); + if (req->wrk != NULL) + pan_wrk(req->wrk); pan_ws(req->ws, 2); pan_http("req", req->http, 2); @@ -272,6 +272,9 @@ pan_req(const struct req *req) if (VALID_OBJ(req->vcl, VCL_CONF_MAGIC)) pan_vcl(req->vcl); + if (req->busyobj != NULL) + pan_busyobj(req->busyobj); + if (VALID_OBJ(req->obj, OBJECT_MAGIC)) pan_object(req->obj); @@ -302,9 +305,6 @@ pan_sess(const struct sess *sp) else VSB_printf(pan_vsp, " step = 0x%x,\n", sp->sess_step); - if (sp->wrk != NULL) - pan_wrk(sp->wrk); - VSB_printf(pan_vsp, "},\n"); } diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c index 13a8c10..8fa9e3c 100644 --- a/bin/varnishd/cache/cache_pipe.c +++ b/bin/varnishd/cache/cache_pipe.c @@ -70,7 +70,7 @@ PipeRequest(struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC); - wrk = req->sp->wrk; + wrk = req->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); bo = req->busyobj; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index 85c8233..cdad653 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -171,11 +171,11 @@ res_WriteGunzipObj(struct req *req) CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); u += st->len; - i = VGZ_WrwGunzip(req->sp->wrk, vg, st->ptr, st->len); + i = VGZ_WrwGunzip(req->wrk, vg, st->ptr, st->len); /* XXX: error check */ (void)i; } - VGZ_WrwFlush(req->sp->wrk, vg); + VGZ_WrwFlush(req->wrk, vg); (void)VGZ_Destroy(&vg); assert(u == req->obj->len); } @@ -215,8 +215,8 @@ res_WriteDirObj(const struct req *req, ssize_t low, ssize_t high) ptr += len; - req->sp->wrk->acct_tmp.bodybytes += len; - (void)WRW_Write(req->sp->wrk, st->ptr + off, len); + req->wrk->acct_tmp.bodybytes += len; + (void)WRW_Write(req->wrk, st->ptr + off, len); } assert(u == req->obj->len); } @@ -248,20 +248,20 @@ RES_WriteObj(struct req *req) http_GetHdr(req->http, H_Range, &r)) res_dorange(req, r, &low, &high); - WRW_Reserve(req->sp->wrk, &req->sp->fd, req->vsl, req->t_resp); + WRW_Reserve(req->wrk, &req->sp->fd, req->vsl, req->t_resp); /* * Send HTTP protocol header, unless interior ESI object */ if (!(req->res_mode & RES_ESI_CHILD)) - req->sp->wrk->acct_tmp.hdrbytes += - http_Write(req->sp->wrk, req->resp, 1); + req->wrk->acct_tmp.hdrbytes += + http_Write(req->wrk, req->resp, 1); if (!req->wantbody) req->res_mode &= ~RES_CHUNKED; if (req->res_mode & RES_CHUNKED) - WRW_Chunked(req->sp->wrk); + WRW_Chunked(req->wrk); if (!req->wantbody) { /* This was a HEAD or conditional request */ @@ -282,8 +282,8 @@ RES_WriteObj(struct req *req) if (req->res_mode & RES_CHUNKED && !(req->res_mode & RES_ESI_CHILD)) - WRW_EndChunk(req->sp->wrk); + WRW_EndChunk(req->wrk); - if (WRW_FlushRelease(req->sp->wrk) && req->sp->fd >= 0) + if (WRW_FlushRelease(req->wrk) && req->sp->fd >= 0) SES_Close(req->sp, "remote closed"); } diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index aee58b0..01b0b18 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -128,7 +128,7 @@ ses_pool_task(struct worker *wrk, void *arg) THR_SetSession(sp); AZ(sp->wrk); sp->wrk = wrk; - CNT_Session(sp); + CNT_Session(wrk, sp); sp = NULL; /* Cannot access sp any longer */ THR_SetSession(NULL); WS_Assert(wrk->aws); From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 6dfad3d Totally eliminate sp->wrk Message-ID: commit 6dfad3d42c9c919d9a82cd5065880e48215e646d Author: Poul-Henning Kamp Date: Mon Jun 25 08:13:01 2012 +0000 Totally eliminate sp->wrk diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 3a8559f..8499a12 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -660,7 +660,6 @@ struct sess { /* Cross references ------------------------------------------*/ struct sesspool *sesspool; - struct worker *wrk; struct req *req; struct pool_task task; @@ -926,7 +925,7 @@ unsigned WRW_WriteH(const struct worker *w, const txt *hh, const char *suf); /* cache_session.c [SES] */ void SES_Close(struct sess *sp, const char *reason); void SES_Delete(struct sess *sp, const char *reason, double now); -void SES_Charge(struct sess *sp); +void SES_Charge(struct worker *, struct sess *); struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no); void SES_DeletePool(struct sesspool *sp); int SES_Schedule(struct sess *sp); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 1eb397e..bd28916 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -191,7 +191,7 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) if (when < now || tmo == 0) { sp->t_rx = NAN; wrk->stats.sess_herd++; - SES_Charge(sp); + SES_Charge(wrk, sp); SES_ReleaseReq(sp); WAIT_Enter(sp); return (1); @@ -206,7 +206,7 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) } } } - SES_Charge(sp); + SES_Charge(wrk, sp); SES_Delete(sp, why, now); return (1); } @@ -360,11 +360,9 @@ CNT_Session(struct worker *wrk, struct sess *sp) sp->req->req_step == R_STP_START))); if (sp->sess_step == S_STP_WORKING) { - done = CNT_Request(sp->wrk, sp->req); - if (done == 2) { - sp->wrk = NULL; + done = CNT_Request(wrk, sp->req); + if (done == 2) return; - } assert(done == 1); sdr = cnt_sess_done(sp, wrk, sp->req); switch (sdr) { @@ -1626,7 +1624,7 @@ CNT_Request(struct worker *wrk, struct req *req) CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); } if (done == 1) - SES_Charge(req->sp); + SES_Charge(wrk, req->sp); req->wrk = NULL; diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index c6f56e3..d3976d8 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -424,7 +424,7 @@ HSH_Lookup(struct req *req) if (cache_param->diag_bitmap & 0x20) VSLb(req->vsl, SLT_Debug, "on waiting list <%p>", oh); - SES_Charge(req->sp); + SES_Charge(req->wrk, req->sp); /* * The objhead reference transfers to the sess, we get it * back when the sess comes off the waiting list and diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 01b0b18..57ad270 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -61,15 +61,19 @@ struct sesspool { */ void -SES_Charge(struct sess *sp) +SES_Charge(struct worker *wrk, struct sess *sp) { - struct acct *a = &sp->wrk->acct_tmp; + struct acct *a; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC); + + a = &wrk->acct_tmp; sp->req->req_bodybytes += a->bodybytes; #define ACCT(foo) \ - sp->wrk->stats.s_##foo += a->foo; \ + wrk->stats.s_##foo += a->foo; \ sp->acct_ses.foo += a->foo; \ a->foo = 0; #include "tbl/acct_fields.h" @@ -126,8 +130,6 @@ ses_pool_task(struct worker *wrk, void *arg) AZ(wrk->aws->r); wrk->lastused = NAN; THR_SetSession(sp); - AZ(sp->wrk); - sp->wrk = wrk; CNT_Session(wrk, sp); sp = NULL; /* Cannot access sp any longer */ THR_SetSession(NULL); @@ -174,12 +176,10 @@ SES_Schedule(struct sess *sp) struct sesspool *pp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - AZ(sp->wrk); pp = sp->sesspool; CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); AN(pp->pool); - AZ(sp->wrk); sp->task.func = ses_pool_task; sp->task.priv = sp; @@ -243,7 +243,6 @@ void SES_Delete(struct sess *sp, const char *reason, double now) { struct acct *b; - struct worker *wrk; struct sesspool *pp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); @@ -251,9 +250,6 @@ SES_Delete(struct sess *sp, const char *reason, double now) CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); AN(pp->pool); - wrk = sp->wrk; - CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC); - if (reason != NULL) SES_Close(sp, reason); if (isnan(now)) diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c index b0f461a..3816a26 100644 --- a/bin/varnishd/waiter/cache_waiter.c +++ b/bin/varnishd/waiter/cache_waiter.c @@ -67,10 +67,8 @@ WAIT_Enter(struct sess *sp) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC); AZ(sp->req); assert(sp->fd >= 0); - sp->wrk = NULL; /* * Set nonblocking in the worker-thread, before passing to the From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 94d7a68 Make SES_Charge() and SES_ReleaseReq() take struct req* argument instead of struct sess* Message-ID: commit 94d7a686b0e553be881d378b6e5f4060123b56d4 Author: Poul-Henning Kamp Date: Mon Jun 25 08:34:37 2012 +0000 Make SES_Charge() and SES_ReleaseReq() take struct req* argument instead of struct sess* diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 8499a12..436bd20 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -925,13 +925,13 @@ unsigned WRW_WriteH(const struct worker *w, const txt *hh, const char *suf); /* cache_session.c [SES] */ void SES_Close(struct sess *sp, const char *reason); void SES_Delete(struct sess *sp, const char *reason, double now); -void SES_Charge(struct worker *, struct sess *); +void SES_Charge(struct worker *, struct req *); struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no); void SES_DeletePool(struct sesspool *sp); int SES_Schedule(struct sess *sp); void SES_Handle(struct sess *sp, double now); struct req *SES_GetReq(struct sess *sp); -void SES_ReleaseReq(struct sess *sp); +void SES_ReleaseReq(struct req *); pool_func_t SES_pool_accept_task; /* cache_shmlog.c */ diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index bd28916..5ea1f48 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -191,8 +191,8 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) if (when < now || tmo == 0) { sp->t_rx = NAN; wrk->stats.sess_herd++; - SES_Charge(wrk, sp); - SES_ReleaseReq(sp); + SES_Charge(wrk, req); + SES_ReleaseReq(req); WAIT_Enter(sp); return (1); } @@ -206,7 +206,7 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) } } } - SES_Charge(wrk, sp); + SES_Charge(wrk, req); SES_Delete(sp, why, now); return (1); } @@ -1624,7 +1624,7 @@ CNT_Request(struct worker *wrk, struct req *req) CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); } if (done == 1) - SES_Charge(wrk, req->sp); + SES_Charge(wrk, req); req->wrk = NULL; diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index d3976d8..a2710dd 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -424,7 +424,7 @@ HSH_Lookup(struct req *req) if (cache_param->diag_bitmap & 0x20) VSLb(req->vsl, SLT_Debug, "on waiting list <%p>", oh); - SES_Charge(req->wrk, req->sp); + SES_Charge(req->wrk, req); /* * The objhead reference transfers to the sess, we get it * back when the sess comes off the waiting list and diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 57ad270..49bd3bb 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -61,16 +61,18 @@ struct sesspool { */ void -SES_Charge(struct worker *wrk, struct sess *sp) +SES_Charge(struct worker *wrk, struct req *req) { + struct sess *sp; struct acct *a; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + sp = req->sp; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); a = &wrk->acct_tmp; - sp->req->req_bodybytes += a->bodybytes; + req->req_bodybytes += a->bodybytes; #define ACCT(foo) \ wrk->stats.s_##foo += a->foo; \ @@ -259,7 +261,7 @@ SES_Delete(struct sess *sp, const char *reason, double now) if (sp->req != NULL) { AZ(sp->req->vcl); - SES_ReleaseReq(sp); + SES_ReleaseReq(sp->req); } if (*sp->addr == '\0') @@ -346,21 +348,23 @@ SES_GetReq(struct sess *sp) } void -SES_ReleaseReq(struct sess *sp) +SES_ReleaseReq(struct req *req) { + struct sess *sp; struct sesspool *pp; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + sp = req->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + assert(sp->req == req); pp = sp->sesspool; CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); AN(pp->pool); - CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC); - AN(sp->req->sp); - assert(sp->req->sp == sp); - MPL_AssertSane(sp->req); - VSL_Flush(sp->req->vsl, 0); - sp->req->sp = NULL; - MPL_Free(pp->mpl_req, sp->req); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + MPL_AssertSane(req); + VSL_Flush(req->vsl, 0); + req->sp = NULL; + MPL_Free(pp->mpl_req, req); sp->req = NULL; THR_SetRequest(NULL); } From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 87f32cb Always release req before calling SES_Delete() Message-ID: commit 87f32cb739e7cb89babfae5f5943719aa3b1fed4 Author: Poul-Henning Kamp Date: Mon Jun 25 08:48:04 2012 +0000 Always release req before calling SES_Delete() Split SES_Handle() and SES_ScheduleReq(). diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 436bd20..646ee9d 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -928,7 +928,7 @@ void SES_Delete(struct sess *sp, const char *reason, double now); void SES_Charge(struct worker *, struct req *); struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no); void SES_DeletePool(struct sesspool *sp); -int SES_Schedule(struct sess *sp); +int SES_ScheduleReq(struct req *); void SES_Handle(struct sess *sp, double now); struct req *SES_GetReq(struct sess *sp); void SES_ReleaseReq(struct req *); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 5ea1f48..88eef23 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -207,6 +207,8 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) } } SES_Charge(wrk, req); + AZ(req->vcl); + SES_ReleaseReq(req); SES_Delete(sp, why, now); return (1); } @@ -295,6 +297,8 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) if (sp->fd < 0) { wrk->stats.sess_closed++; + AZ(req->vcl); + SES_ReleaseReq(req); SES_Delete(sp, NULL, NAN); return (SESS_DONE_RET_GONE); } diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index a2710dd..abcb4c0 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -469,7 +469,6 @@ hsh_rush(struct dstat *ds, struct objhead *oh) { unsigned u; struct req *req; - struct sess *sp; struct waitinglist *wl; CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); @@ -482,11 +481,9 @@ hsh_rush(struct dstat *ds, struct objhead *oh) break; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AZ(req->wrk); - sp = req->sp; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); VTAILQ_REMOVE(&wl->list, req, w_list); - DSL(0x20, SLT_Debug, sp->vsl_id, "off waiting list"); - if (SES_Schedule(sp)) { + DSL(0x20, SLT_Debug, req->sp->vsl_id, "off waiting list"); + if (SES_ScheduleReq(req)) { /* * We could not schedule the session, leave the * rest on the busy list. diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 49bd3bb..9bf3382 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -169,14 +169,17 @@ SES_pool_accept_task(struct worker *wrk, void *arg) } /*-------------------------------------------------------------------- - * Schedule a session back on a work-thread from its pool + * Schedule a request back on a work-thread from its sessions pool */ int -SES_Schedule(struct sess *sp) +SES_ScheduleReq(struct req *req) { + struct sess *sp; struct sesspool *pp; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + sp = req->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); pp = sp->sesspool; CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); @@ -188,13 +191,8 @@ SES_Schedule(struct sess *sp) if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) { VSC_C_main->client_drop_late++; sp->t_idle = VTIM_real(); - if (sp->req != NULL && sp->req->vcl != NULL) { - /* - * A session parked on a busy object can come here - * after it wakes up. Loose the VCL reference. - */ - VCL_Rel(&sp->req->vcl); - } + AN (req->vcl); + VCL_Rel(&req->vcl); SES_Delete(sp, "dropped", sp->t_idle); return (1); } @@ -208,10 +206,22 @@ SES_Schedule(struct sess *sp) void SES_Handle(struct sess *sp, double now) { + struct sesspool *pp; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + pp = sp->sesspool; + CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); + AN(pp->pool); + AZ(sp->req); + sp->task.func = ses_pool_task; + sp->task.priv = sp; sp->sess_step = S_STP_NEWREQ; sp->t_rx = now; - (void)SES_Schedule(sp); + if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) { + VSC_C_main->client_drop_late++; + sp->t_idle = VTIM_real(); + SES_Delete(sp, "dropped", sp->t_idle); + } } /*-------------------------------------------------------------------- @@ -248,6 +258,7 @@ SES_Delete(struct sess *sp, const char *reason, double now) struct sesspool *pp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + AZ(sp->req); pp = sp->sesspool; CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); AN(pp->pool); @@ -259,11 +270,6 @@ SES_Delete(struct sess *sp, const char *reason, double now) assert(!isnan(sp->t_open)); assert(sp->fd < 0); - if (sp->req != NULL) { - AZ(sp->req->vcl); - SES_ReleaseReq(sp->req); - } - if (*sp->addr == '\0') strcpy(sp->addr, "-"); if (*sp->port == '\0') From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 5894995 Always allocate a request before calling CNT_Session() Message-ID: commit 5894995f139a4cf9b131978d83508450dbe836e4 Author: Poul-Henning Kamp Date: Mon Jun 25 09:03:12 2012 +0000 Always allocate a request before calling CNT_Session() diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 646ee9d..1d83dab 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -744,7 +744,7 @@ void VBO_Free(struct busyobj **vbo); /* cache_center.c [CNT] */ int CNT_Request(struct worker *, struct req *); -void CNT_Session(struct worker *, struct sess *); +void CNT_Session(struct worker *, struct req *); void CNT_Init(void); /* cache_cli.c [CLI] */ diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 88eef23..5c76ac9 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -117,11 +117,8 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - if (req == NULL) { - req = SES_GetReq(sp); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - } assert(req->sp == sp); if (!sp->init_done) { @@ -327,13 +324,17 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) */ void -CNT_Session(struct worker *wrk, struct sess *sp) +CNT_Session(struct worker *wrk, struct req *req) { int done; + struct sess *sp; enum cnt_sess_done_ret sdr; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + sp = req->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + assert(sp->req == req); /* * Whenever we come in from the acceptor or waiter, we need to set @@ -348,7 +349,7 @@ CNT_Session(struct worker *wrk, struct sess *sp) SES_Close(sp, "remote closed"); else SES_Close(sp, "error"); - sdr = cnt_sess_done(sp, wrk, sp->req); + sdr = cnt_sess_done(sp, wrk, req); assert(sdr == SESS_DONE_RET_GONE); return; } @@ -357,18 +358,19 @@ CNT_Session(struct worker *wrk, struct sess *sp) /* * Possible entrance states */ + assert(sp->req == req); + assert( sp->sess_step == S_STP_NEWREQ || - (sp->req != NULL && - (sp->req->req_step == R_STP_LOOKUP || - sp->req->req_step == R_STP_START))); + req->req_step == R_STP_LOOKUP || + req->req_step == R_STP_START); if (sp->sess_step == S_STP_WORKING) { - done = CNT_Request(wrk, sp->req); + done = CNT_Request(wrk, req); if (done == 2) return; assert(done == 1); - sdr = cnt_sess_done(sp, wrk, sp->req); + sdr = cnt_sess_done(sp, wrk, req); switch (sdr) { case SESS_DONE_RET_GONE: return; @@ -377,7 +379,7 @@ CNT_Session(struct worker *wrk, struct sess *sp) break; case SESS_DONE_RET_START: sp->sess_step = S_STP_WORKING; - sp->req->req_step = R_STP_START; + req->req_step = R_STP_START; break; default: WRONG("Illegal enum cnt_sess_done_ret"); @@ -385,12 +387,11 @@ CNT_Session(struct worker *wrk, struct sess *sp) } if (sp->sess_step == S_STP_NEWREQ) { - done = cnt_wait(sp, wrk, sp->req); - if (done) { + done = cnt_wait(sp, wrk, req); + if (done) return; - } sp->sess_step = S_STP_WORKING; - sp->req->req_step = R_STP_START; + req->req_step = R_STP_START; } } } diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 9bf3382..2b384f6 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -124,16 +124,18 @@ ses_new(struct sesspool *pp) static void ses_pool_task(struct worker *wrk, void *arg) { + struct req *req; struct sess *sp; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CAST_OBJ_NOTNULL(sp, arg, SESS_MAGIC); + CAST_OBJ_NOTNULL(req, arg, REQ_MAGIC); + sp = req->sp; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); AZ(wrk->aws->r); wrk->lastused = NAN; THR_SetSession(sp); - CNT_Session(wrk, sp); - sp = NULL; /* Cannot access sp any longer */ + CNT_Session(wrk, req); THR_SetSession(NULL); WS_Assert(wrk->aws); AZ(wrk->wrw); @@ -151,6 +153,7 @@ void SES_pool_accept_task(struct worker *wrk, void *arg) { struct sesspool *pp; + struct req *req; struct sess *sp; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -161,11 +164,13 @@ SES_pool_accept_task(struct worker *wrk, void *arg) sp = ses_new(pp); if (sp == NULL) { VCA_FailSess(wrk); - } else { - VCA_SetupSess(wrk, sp); - sp->sess_step = S_STP_NEWREQ; - ses_pool_task(wrk, sp); - } + return; + } + VCA_SetupSess(wrk, sp); + sp->sess_step = S_STP_NEWREQ; + req = SES_GetReq(sp); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + ses_pool_task(wrk, req); } /*-------------------------------------------------------------------- @@ -186,7 +191,7 @@ SES_ScheduleReq(struct req *req) AN(pp->pool); sp->task.func = ses_pool_task; - sp->task.priv = sp; + sp->task.priv = req; if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) { VSC_C_main->client_drop_late++; @@ -206,6 +211,7 @@ SES_ScheduleReq(struct req *req) void SES_Handle(struct sess *sp, double now) { + struct req *req; struct sesspool *pp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); @@ -213,8 +219,10 @@ SES_Handle(struct sess *sp, double now) CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); AN(pp->pool); AZ(sp->req); + req = SES_GetReq(sp); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); sp->task.func = ses_pool_task; - sp->task.priv = sp; + sp->task.priv = req; sp->sess_step = S_STP_NEWREQ; sp->t_rx = now; if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) { From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 90846c4 Remove sp->req, sessions can now (almost) have more than one request at a time, paving the way for a lot of future fun... Message-ID: commit 90846c4aa1c7be8bc9abbb7b78a2f589fc5638fd Author: Poul-Henning Kamp Date: Mon Jun 25 09:13:29 2012 +0000 Remove sp->req, sessions can now (almost) have more than one request at a time, paving the way for a lot of future fun... It's painfully obvious that I set about doing this the wrong way: I should have renamed sess to req and made a new sess structure, that would have been much simpler. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 1d83dab..1d991f1 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -660,7 +660,6 @@ struct sess { /* Cross references ------------------------------------------*/ struct sesspool *sesspool; - struct req *req; struct pool_task task; VTAILQ_ENTRY(sess) list; @@ -866,8 +865,6 @@ void THR_SetName(const char *name); const char* THR_GetName(void); void THR_SetRequest(const struct req *); const struct req * THR_GetRequest(void); -void THR_SetSession(const struct sess *); -const struct sess * THR_GetSession(void); /* cache_lck.c */ @@ -930,7 +927,6 @@ struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no); void SES_DeletePool(struct sesspool *sp); int SES_ScheduleReq(struct req *); void SES_Handle(struct sess *sp, double now); -struct req *SES_GetReq(struct sess *sp); void SES_ReleaseReq(struct req *); pool_func_t SES_pool_accept_task; diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 5c76ac9..004f815 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -334,7 +334,6 @@ CNT_Session(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); sp = req->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - assert(sp->req == req); /* * Whenever we come in from the acceptor or waiter, we need to set @@ -358,7 +357,6 @@ CNT_Session(struct worker *wrk, struct req *req) /* * Possible entrance states */ - assert(sp->req == req); assert( sp->sess_step == S_STP_NEWREQ || diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index 041578b..20d12bc 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -49,20 +49,6 @@ static pthread_key_t sp_key; static pthread_key_t req_key; void -THR_SetSession(const struct sess *sp) -{ - - AZ(pthread_setspecific(sp_key, sp)); -} - -const struct sess * -THR_GetSession(void) -{ - - return (pthread_getspecific(sp_key)); -} - -void THR_SetRequest(const struct req *req) { diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 411b8e9..94b2aee 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -57,6 +57,8 @@ static struct vsb pan_vsp_storage, *pan_vsp; static pthread_mutex_t panicstr_mtx = PTHREAD_MUTEX_INITIALIZER; +static void pan_sess(const struct sess *sp); + /*--------------------------------------------------------------------*/ static void @@ -261,6 +263,9 @@ pan_req(const struct req *req) VSB_printf(pan_vsp, " restarts = %d, esi_level = %d\n", req->restarts, req->esi_level); + if (req->sp != NULL) + pan_sess(req->sp); + if (req->wrk != NULL) pan_wrk(req->wrk); @@ -343,7 +348,6 @@ pan_ic(const char *func, const char *file, int line, const char *cond, int err, int xxx) { const char *q; - const struct sess *sp; const struct req *req; AZ(pthread_mutex_lock(&panicstr_mtx)); /* Won't be released, @@ -385,9 +389,6 @@ pan_ic(const char *func, const char *file, int line, const char *cond, pan_backtrace(); if (!(cache_param->diag_bitmap & 0x2000)) { - sp = THR_GetSession(); - if (sp != NULL) - pan_sess(sp); req = THR_GetRequest(); if (req != NULL) pan_req(req); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 2b384f6..15b2d9a 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -46,6 +46,8 @@ static unsigned ses_size = sizeof (struct sess); +static struct req * ses_GetReq(struct sess *sp); + /*--------------------------------------------------------------------*/ struct sesspool { @@ -134,9 +136,7 @@ ses_pool_task(struct worker *wrk, void *arg) AZ(wrk->aws->r); wrk->lastused = NAN; - THR_SetSession(sp); CNT_Session(wrk, req); - THR_SetSession(NULL); WS_Assert(wrk->aws); AZ(wrk->wrw); if (cache_param->diag_bitmap & 0x00040000) { @@ -168,7 +168,7 @@ SES_pool_accept_task(struct worker *wrk, void *arg) } VCA_SetupSess(wrk, sp); sp->sess_step = S_STP_NEWREQ; - req = SES_GetReq(sp); + req = ses_GetReq(sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); ses_pool_task(wrk, req); } @@ -218,8 +218,7 @@ SES_Handle(struct sess *sp, double now) pp = sp->sesspool; CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); AN(pp->pool); - AZ(sp->req); - req = SES_GetReq(sp); + req = ses_GetReq(sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); sp->task.func = ses_pool_task; sp->task.priv = req; @@ -266,7 +265,6 @@ SES_Delete(struct sess *sp, const char *reason, double now) struct sesspool *pp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - AZ(sp->req); pp = sp->sesspool; CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); AN(pp->pool); @@ -296,11 +294,11 @@ SES_Delete(struct sess *sp, const char *reason, double now) } /*-------------------------------------------------------------------- - * Alloc/Free sp->req + * Alloc/Free a request */ -struct req * -SES_GetReq(struct sess *sp) +static struct req * +ses_GetReq(struct sess *sp) { struct sesspool *pp; struct req *req; @@ -313,11 +311,9 @@ SES_GetReq(struct sess *sp) CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); AN(pp->pool); - AZ(sp->req); req = MPL_Get(pp->mpl_req, &sz); AN(req); req->magic = REQ_MAGIC; - sp->req = req; req->sp = sp; THR_SetRequest(req); @@ -370,7 +366,6 @@ SES_ReleaseReq(struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); sp = req->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - assert(sp->req == req); pp = sp->sesspool; CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); AN(pp->pool); @@ -379,7 +374,6 @@ SES_ReleaseReq(struct req *req) VSL_Flush(req->vsl, 0); req->sp = NULL; MPL_Free(pp->mpl_req, req); - sp->req = NULL; THR_SetRequest(NULL); } diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c index 3816a26..e89d550 100644 --- a/bin/varnishd/waiter/cache_waiter.c +++ b/bin/varnishd/waiter/cache_waiter.c @@ -67,7 +67,6 @@ WAIT_Enter(struct sess *sp) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - AZ(sp->req); assert(sp->fd >= 0); /* From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 1c82be0 Fix a regression in panic output. Message-ID: commit 1c82be0c4664a0311a5c5207764bae9d25fd1023 Author: Poul-Henning Kamp Date: Mon Jun 25 10:11:03 2012 +0000 Fix a regression in panic output. Update test-case to exercise more of the panic formatter. diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index 20d12bc..dcb3cb1 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -45,7 +45,6 @@ volatile struct params *cache_param; * the thread. This is used for panic messages. */ -static pthread_key_t sp_key; static pthread_key_t req_key; void @@ -126,7 +125,6 @@ child_main(void) cache_param = heritage.param; - AZ(pthread_key_create(&sp_key, NULL)); AZ(pthread_key_create(&req_key, NULL)); AZ(pthread_key_create(&name_key, NULL)); diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 94b2aee..ff9a65b 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -182,12 +182,12 @@ pan_vcl(const struct VCL_conf *vcl) { int i; - VSB_printf(pan_vsp, " vcl = {\n"); - VSB_printf(pan_vsp, " srcname = {\n"); + VSB_printf(pan_vsp, " vcl = {\n"); + VSB_printf(pan_vsp, " srcname = {\n"); for (i = 0; i < vcl->nsrc; ++i) - VSB_printf(pan_vsp, " \"%s\",\n", vcl->srcname[i]); - VSB_printf(pan_vsp, " },\n"); + VSB_printf(pan_vsp, " \"%s\",\n", vcl->srcname[i]); VSB_printf(pan_vsp, " },\n"); + VSB_printf(pan_vsp, " },\n"); } @@ -224,7 +224,7 @@ pan_busyobj(const struct busyobj *bo) pan_http("bereq", bo->bereq, 4); if (bo->beresp->ws != NULL) pan_http("beresp", bo->beresp, 4); - + VSB_printf(pan_vsp, " }\n"); } /*--------------------------------------------------------------------*/ @@ -272,7 +272,7 @@ pan_req(const struct req *req) pan_ws(req->ws, 2); pan_http("req", req->http, 2); if (req->resp->ws != NULL) - pan_http("resp", req->resp, 4); + pan_http("resp", req->resp, 2); if (VALID_OBJ(req->vcl, VCL_CONF_MAGIC)) pan_vcl(req->vcl); @@ -293,10 +293,10 @@ pan_sess(const struct sess *sp) { const char *stp; - VSB_printf(pan_vsp, "sp = %p {\n", sp); - VSB_printf(pan_vsp, " fd = %d, id = %u,\n", + VSB_printf(pan_vsp, " sp = %p {\n", sp); + VSB_printf(pan_vsp, " fd = %d, id = %u,\n", sp->fd, sp->vsl_id & VSL_IDENTMASK); - VSB_printf(pan_vsp, " client = %s %s,\n", + VSB_printf(pan_vsp, " client = %s %s,\n", sp->addr ? sp->addr : "?.?.?.?", sp->port ? sp->port : "?"); switch (sp->sess_step) { @@ -306,11 +306,11 @@ pan_sess(const struct sess *sp) default: stp = NULL; } if (stp != NULL) - VSB_printf(pan_vsp, " step = %s,\n", stp); + VSB_printf(pan_vsp, " step = %s,\n", stp); else - VSB_printf(pan_vsp, " step = 0x%x,\n", sp->sess_step); + VSB_printf(pan_vsp, " step = 0x%x,\n", sp->sess_step); - VSB_printf(pan_vsp, "},\n"); + VSB_printf(pan_vsp, " },\n"); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 15b2d9a..e8a1ac1 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -134,6 +134,7 @@ ses_pool_task(struct worker *wrk, void *arg) sp = req->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + THR_SetRequest(req); AZ(wrk->aws->r); wrk->lastused = NAN; CNT_Session(wrk, req); @@ -143,6 +144,7 @@ ses_pool_task(struct worker *wrk, void *arg) if (wrk->vcl != NULL) VCL_Rel(&wrk->vcl); } + THR_SetRequest(NULL); } /*-------------------------------------------------------------------- @@ -315,7 +317,6 @@ ses_GetReq(struct sess *sp) AN(req); req->magic = REQ_MAGIC; req->sp = sp; - THR_SetRequest(req); e = (char*)req + sz; p = (char*)(req + 1); @@ -374,7 +375,6 @@ SES_ReleaseReq(struct req *req) VSL_Flush(req->vsl, 0); req->sp = NULL; MPL_Free(pp->mpl_req, req); - THR_SetRequest(NULL); } /*-------------------------------------------------------------------- diff --git a/bin/varnishtest/tests/v00010.vtc b/bin/varnishtest/tests/v00010.vtc index 46d778c..2e9a385 100644 --- a/bin/varnishtest/tests/v00010.vtc +++ b/bin/varnishtest/tests/v00010.vtc @@ -4,7 +4,15 @@ server s1 { rxreq txresp -hdr "Foo: bar" -body "abcdef\n" rxreq - txresp -hdr "Panic: please" -body "012345\n" + txresp -hdr "Panic: fetch" -body "012345\n" + close + + sema r1 sync 2 + accept + rxreq + txresp -hdr "Foo: bar" -body "abcdef\n" + rxreq + txresp -hdr "Panic: deliver" -body "012345\n" close sema r1 sync 2 @@ -17,8 +25,13 @@ varnish v1 -storage "-smalloc,1m" -vcl+backend { import debug from "${topbuild}/lib/libvmod_debug/.libs/libvmod_debug.so"; + sub vcl_fetch { + if (beresp.http.panic == "fetch") { + debug.panic("Had Panic header: " + beresp.http.panic); + } + } sub vcl_deliver { - if (resp.http.panic) { + if (resp.http.panic == "deliver") { debug.panic("Had Panic header: " + resp.http.panic); } } @@ -49,5 +62,22 @@ delay 0.5 client c1 { txreq -url "/" rxresp + txreq -url "/foo" + # Don't expect answer, the server crashed. +} -run + +varnish v1 -wait-stopped +varnish v1 -cliok "panic.show" +varnish v1 -cliok "panic.clear" +varnish v1 -clierr 300 "panic.clear" +varnish v1 -cliok "start" +varnish v1 -wait-running +sema r1 sync 2 + +delay 0.5 + +client c1 { + txreq -url "/" + rxresp expect resp.http.foo == "foo" } -run From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] ea1df10 Compress this eyesore using CPP Message-ID: commit ea1df10930ec15629c386dc692cb6c08d3d82392 Author: Poul-Henning Kamp Date: Tue Jun 26 09:24:11 2012 +0000 Compress this eyesore using CPP diff --git a/lib/libvarnishcompat/execinfo.c b/lib/libvarnishcompat/execinfo.c index b1a17cf..113ae32 100644 --- a/lib/libvarnishcompat/execinfo.c +++ b/lib/libvarnishcompat/execinfo.c @@ -126,278 +126,37 @@ backtrace_symbols(void *const *buffer, int size) return (rval); } +/* Binary expansion */ +#define L1(x) L0(x); L0(x + 0x01); +#define L2(x) L1(x); L1(x + 0x02); +#define L3(x) L2(x); L2(x + 0x04); +#define L4(x) L3(x); L3(x + 0x08); +#define L5(x) L4(x); L4(x + 0x10); +#define L6(x) L5(x); L5(x + 0x20); +#define L7(x) L6(x); L6(x + 0x40); + static void * getreturnaddr(int level) { - switch(level) { - case 0: return __builtin_return_address(1); - case 1: return __builtin_return_address(2); - case 2: return __builtin_return_address(3); - case 3: return __builtin_return_address(4); - case 4: return __builtin_return_address(5); - case 5: return __builtin_return_address(6); - case 6: return __builtin_return_address(7); - case 7: return __builtin_return_address(8); - case 8: return __builtin_return_address(9); - case 9: return __builtin_return_address(10); - case 10: return __builtin_return_address(11); - case 11: return __builtin_return_address(12); - case 12: return __builtin_return_address(13); - case 13: return __builtin_return_address(14); - case 14: return __builtin_return_address(15); - case 15: return __builtin_return_address(16); - case 16: return __builtin_return_address(17); - case 17: return __builtin_return_address(18); - case 18: return __builtin_return_address(19); - case 19: return __builtin_return_address(20); - case 20: return __builtin_return_address(21); - case 21: return __builtin_return_address(22); - case 22: return __builtin_return_address(23); - case 23: return __builtin_return_address(24); - case 24: return __builtin_return_address(25); - case 25: return __builtin_return_address(26); - case 26: return __builtin_return_address(27); - case 27: return __builtin_return_address(28); - case 28: return __builtin_return_address(29); - case 29: return __builtin_return_address(30); - case 30: return __builtin_return_address(31); - case 31: return __builtin_return_address(32); - case 32: return __builtin_return_address(33); - case 33: return __builtin_return_address(34); - case 34: return __builtin_return_address(35); - case 35: return __builtin_return_address(36); - case 36: return __builtin_return_address(37); - case 37: return __builtin_return_address(38); - case 38: return __builtin_return_address(39); - case 39: return __builtin_return_address(40); - case 40: return __builtin_return_address(41); - case 41: return __builtin_return_address(42); - case 42: return __builtin_return_address(43); - case 43: return __builtin_return_address(44); - case 44: return __builtin_return_address(45); - case 45: return __builtin_return_address(46); - case 46: return __builtin_return_address(47); - case 47: return __builtin_return_address(48); - case 48: return __builtin_return_address(49); - case 49: return __builtin_return_address(50); - case 50: return __builtin_return_address(51); - case 51: return __builtin_return_address(52); - case 52: return __builtin_return_address(53); - case 53: return __builtin_return_address(54); - case 54: return __builtin_return_address(55); - case 55: return __builtin_return_address(56); - case 56: return __builtin_return_address(57); - case 57: return __builtin_return_address(58); - case 58: return __builtin_return_address(59); - case 59: return __builtin_return_address(60); - case 60: return __builtin_return_address(61); - case 61: return __builtin_return_address(62); - case 62: return __builtin_return_address(63); - case 63: return __builtin_return_address(64); - case 64: return __builtin_return_address(65); - case 65: return __builtin_return_address(66); - case 66: return __builtin_return_address(67); - case 67: return __builtin_return_address(68); - case 68: return __builtin_return_address(69); - case 69: return __builtin_return_address(70); - case 70: return __builtin_return_address(71); - case 71: return __builtin_return_address(72); - case 72: return __builtin_return_address(73); - case 73: return __builtin_return_address(74); - case 74: return __builtin_return_address(75); - case 75: return __builtin_return_address(76); - case 76: return __builtin_return_address(77); - case 77: return __builtin_return_address(78); - case 78: return __builtin_return_address(79); - case 79: return __builtin_return_address(80); - case 80: return __builtin_return_address(81); - case 81: return __builtin_return_address(82); - case 82: return __builtin_return_address(83); - case 83: return __builtin_return_address(84); - case 84: return __builtin_return_address(85); - case 85: return __builtin_return_address(86); - case 86: return __builtin_return_address(87); - case 87: return __builtin_return_address(88); - case 88: return __builtin_return_address(89); - case 89: return __builtin_return_address(90); - case 90: return __builtin_return_address(91); - case 91: return __builtin_return_address(92); - case 92: return __builtin_return_address(93); - case 93: return __builtin_return_address(94); - case 94: return __builtin_return_address(95); - case 95: return __builtin_return_address(96); - case 96: return __builtin_return_address(97); - case 97: return __builtin_return_address(98); - case 98: return __builtin_return_address(99); - case 99: return __builtin_return_address(100); - case 100: return __builtin_return_address(101); - case 101: return __builtin_return_address(102); - case 102: return __builtin_return_address(103); - case 103: return __builtin_return_address(104); - case 104: return __builtin_return_address(105); - case 105: return __builtin_return_address(106); - case 106: return __builtin_return_address(107); - case 107: return __builtin_return_address(108); - case 108: return __builtin_return_address(109); - case 109: return __builtin_return_address(110); - case 110: return __builtin_return_address(111); - case 111: return __builtin_return_address(112); - case 112: return __builtin_return_address(113); - case 113: return __builtin_return_address(114); - case 114: return __builtin_return_address(115); - case 115: return __builtin_return_address(116); - case 116: return __builtin_return_address(117); - case 117: return __builtin_return_address(118); - case 118: return __builtin_return_address(119); - case 119: return __builtin_return_address(120); - case 120: return __builtin_return_address(121); - case 121: return __builtin_return_address(122); - case 122: return __builtin_return_address(123); - case 123: return __builtin_return_address(124); - case 124: return __builtin_return_address(125); - case 125: return __builtin_return_address(126); - case 126: return __builtin_return_address(127); - case 127: return __builtin_return_address(128); - default: return NULL; - } + switch(level) { +#define L0(x) case x: return __builtin_return_address(x + 1); + L7(0) +#undef L0 + default: return NULL; + } } static void * getframeaddr(int level) { - switch(level) { - case 0: return __builtin_frame_address(1); - case 1: return __builtin_frame_address(2); - case 2: return __builtin_frame_address(3); - case 3: return __builtin_frame_address(4); - case 4: return __builtin_frame_address(5); - case 5: return __builtin_frame_address(6); - case 6: return __builtin_frame_address(7); - case 7: return __builtin_frame_address(8); - case 8: return __builtin_frame_address(9); - case 9: return __builtin_frame_address(10); - case 10: return __builtin_frame_address(11); - case 11: return __builtin_frame_address(12); - case 12: return __builtin_frame_address(13); - case 13: return __builtin_frame_address(14); - case 14: return __builtin_frame_address(15); - case 15: return __builtin_frame_address(16); - case 16: return __builtin_frame_address(17); - case 17: return __builtin_frame_address(18); - case 18: return __builtin_frame_address(19); - case 19: return __builtin_frame_address(20); - case 20: return __builtin_frame_address(21); - case 21: return __builtin_frame_address(22); - case 22: return __builtin_frame_address(23); - case 23: return __builtin_frame_address(24); - case 24: return __builtin_frame_address(25); - case 25: return __builtin_frame_address(26); - case 26: return __builtin_frame_address(27); - case 27: return __builtin_frame_address(28); - case 28: return __builtin_frame_address(29); - case 29: return __builtin_frame_address(30); - case 30: return __builtin_frame_address(31); - case 31: return __builtin_frame_address(32); - case 32: return __builtin_frame_address(33); - case 33: return __builtin_frame_address(34); - case 34: return __builtin_frame_address(35); - case 35: return __builtin_frame_address(36); - case 36: return __builtin_frame_address(37); - case 37: return __builtin_frame_address(38); - case 38: return __builtin_frame_address(39); - case 39: return __builtin_frame_address(40); - case 40: return __builtin_frame_address(41); - case 41: return __builtin_frame_address(42); - case 42: return __builtin_frame_address(43); - case 43: return __builtin_frame_address(44); - case 44: return __builtin_frame_address(45); - case 45: return __builtin_frame_address(46); - case 46: return __builtin_frame_address(47); - case 47: return __builtin_frame_address(48); - case 48: return __builtin_frame_address(49); - case 49: return __builtin_frame_address(50); - case 50: return __builtin_frame_address(51); - case 51: return __builtin_frame_address(52); - case 52: return __builtin_frame_address(53); - case 53: return __builtin_frame_address(54); - case 54: return __builtin_frame_address(55); - case 55: return __builtin_frame_address(56); - case 56: return __builtin_frame_address(57); - case 57: return __builtin_frame_address(58); - case 58: return __builtin_frame_address(59); - case 59: return __builtin_frame_address(60); - case 60: return __builtin_frame_address(61); - case 61: return __builtin_frame_address(62); - case 62: return __builtin_frame_address(63); - case 63: return __builtin_frame_address(64); - case 64: return __builtin_frame_address(65); - case 65: return __builtin_frame_address(66); - case 66: return __builtin_frame_address(67); - case 67: return __builtin_frame_address(68); - case 68: return __builtin_frame_address(69); - case 69: return __builtin_frame_address(70); - case 70: return __builtin_frame_address(71); - case 71: return __builtin_frame_address(72); - case 72: return __builtin_frame_address(73); - case 73: return __builtin_frame_address(74); - case 74: return __builtin_frame_address(75); - case 75: return __builtin_frame_address(76); - case 76: return __builtin_frame_address(77); - case 77: return __builtin_frame_address(78); - case 78: return __builtin_frame_address(79); - case 79: return __builtin_frame_address(80); - case 80: return __builtin_frame_address(81); - case 81: return __builtin_frame_address(82); - case 82: return __builtin_frame_address(83); - case 83: return __builtin_frame_address(84); - case 84: return __builtin_frame_address(85); - case 85: return __builtin_frame_address(86); - case 86: return __builtin_frame_address(87); - case 87: return __builtin_frame_address(88); - case 88: return __builtin_frame_address(89); - case 89: return __builtin_frame_address(90); - case 90: return __builtin_frame_address(91); - case 91: return __builtin_frame_address(92); - case 92: return __builtin_frame_address(93); - case 93: return __builtin_frame_address(94); - case 94: return __builtin_frame_address(95); - case 95: return __builtin_frame_address(96); - case 96: return __builtin_frame_address(97); - case 97: return __builtin_frame_address(98); - case 98: return __builtin_frame_address(99); - case 99: return __builtin_frame_address(100); - case 100: return __builtin_frame_address(101); - case 101: return __builtin_frame_address(102); - case 102: return __builtin_frame_address(103); - case 103: return __builtin_frame_address(104); - case 104: return __builtin_frame_address(105); - case 105: return __builtin_frame_address(106); - case 106: return __builtin_frame_address(107); - case 107: return __builtin_frame_address(108); - case 108: return __builtin_frame_address(109); - case 109: return __builtin_frame_address(110); - case 110: return __builtin_frame_address(111); - case 111: return __builtin_frame_address(112); - case 112: return __builtin_frame_address(113); - case 113: return __builtin_frame_address(114); - case 114: return __builtin_frame_address(115); - case 115: return __builtin_frame_address(116); - case 116: return __builtin_frame_address(117); - case 117: return __builtin_frame_address(118); - case 118: return __builtin_frame_address(119); - case 119: return __builtin_frame_address(120); - case 120: return __builtin_frame_address(121); - case 121: return __builtin_frame_address(122); - case 122: return __builtin_frame_address(123); - case 123: return __builtin_frame_address(124); - case 124: return __builtin_frame_address(125); - case 125: return __builtin_frame_address(126); - case 126: return __builtin_frame_address(127); - case 127: return __builtin_frame_address(128); - default: return NULL; - } + switch(level) { +#define L0(x) case x: return __builtin_frame_address(x + 1); + L7(0) +#undef L0 + default: return NULL; + } } #else From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] c477891 White-space cleanup Message-ID: commit c4778915db1fa39e96e8a77ad47608a7285951da Author: Poul-Henning Kamp Date: Tue Jun 26 10:06:53 2012 +0000 White-space cleanup diff --git a/bin/varnishd/cache/cache_dir_round_robin.c b/bin/varnishd/cache/cache_dir_round_robin.c index 68e7071..b3a57f2 100644 --- a/bin/varnishd/cache/cache_dir_round_robin.c +++ b/bin/varnishd/cache/cache_dir_round_robin.c @@ -173,4 +173,3 @@ VRT_init_dir_fallback(struct cli *cli, struct director **bp, int idx, { vrt_init_dir(cli, bp, idx, priv, m_fallback); } - diff --git a/bin/varnishd/cache/cache_esi.h b/bin/varnishd/cache/cache_esi.h index c6c2376..9903bd8 100644 --- a/bin/varnishd/cache/cache_esi.h +++ b/bin/varnishd/cache/cache_esi.h @@ -44,5 +44,3 @@ typedef ssize_t vep_callback_t(struct busyobj *, ssize_t l, enum vgz_flag flg); void VEP_Init(struct busyobj *, vep_callback_t *cb); void VEP_Parse(const struct busyobj *, const char *p, size_t l); struct vsb *VEP_Finish(struct busyobj *); - - diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index e8a1ac1..25dc687 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -167,7 +167,7 @@ SES_pool_accept_task(struct worker *wrk, void *arg) if (sp == NULL) { VCA_FailSess(wrk); return; - } + } VCA_SetupSess(wrk, sp); sp->sess_step = S_STP_NEWREQ; req = ses_GetReq(sp); diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index f977400..85a603f 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -241,7 +241,8 @@ wsl(struct vsl_log *, enum VSL_tag_e tag, int id, const char *fmt, va_list ap) __printflike(4, 0); static void -wsl(struct vsl_log *vsl, enum VSL_tag_e tag, int id, const char *fmt, va_list ap) +wsl(struct vsl_log *vsl, enum VSL_tag_e tag, int id, const char *fmt, + va_list ap) { char *p; unsigned n, mlen; diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index aedbc98..7563184 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -558,4 +558,3 @@ VRT_r_req_backend_healthy(const struct req *req) CHECK_OBJ_NOTNULL(req->director, DIRECTOR_MAGIC); return (VDI_Healthy(req->director, req)); } - diff --git a/bin/varnishd/cache/cache_wrw.c b/bin/varnishd/cache/cache_wrw.c index 3a38305..2324318 100644 --- a/bin/varnishd/cache/cache_wrw.c +++ b/bin/varnishd/cache/cache_wrw.c @@ -311,6 +311,3 @@ WRW_EndChunk(const struct worker *wrk) wrw->cliov = 0; (void)WRW_Write(wrk, "0\r\n\r\n", -1); } - - - diff --git a/bin/varnishd/hash/hash_mgt.c b/bin/varnishd/hash/hash_mgt.c index ffbba29..30460b3 100644 --- a/bin/varnishd/hash/hash_mgt.c +++ b/bin/varnishd/hash/hash_mgt.c @@ -81,4 +81,3 @@ HSH_config(const char *h_arg) ARGV_ERR("Hash method \"%s\" takes no arguments\n", hp->name); } - diff --git a/include/vcli_serve.h b/include/vcli_serve.h index 7e4e6e4..fe240c1 100644 --- a/include/vcli_serve.h +++ b/include/vcli_serve.h @@ -43,4 +43,3 @@ void VCLS_Destroy(struct VCLS **); cli_func_t VCLS_func_close; cli_func_t VCLS_func_help; cli_func_t VCLS_func_ping; - diff --git a/lib/libvarnish/cli_serve.c b/lib/libvarnish/cli_serve.c index 0de26b1..43c4ecf 100644 --- a/lib/libvarnish/cli_serve.c +++ b/lib/libvarnish/cli_serve.c @@ -577,4 +577,3 @@ VCLS_Destroy(struct VCLS **csp) } FREE_OBJ(cs); } - diff --git a/lib/libvarnish/vsha256.c b/lib/libvarnish/vsha256.c index 0a521b2..a2208cf 100644 --- a/lib/libvarnish/vsha256.c +++ b/lib/libvarnish/vsha256.c @@ -332,4 +332,3 @@ SHA256_Test(void) assert(!memcmp(o, p->output, 32)); } } - diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 5f992cf..67f0690 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -409,5 +409,3 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv) #undef VSC_F #undef VSC_DO #undef VSC_DONE - - diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h index 0b38539..b203ae8 100644 --- a/lib/libvarnishapi/vsl_api.h +++ b/lib/libvarnishapi/vsl_api.h @@ -89,4 +89,3 @@ struct vsl { }; struct vsl *vsl_Setup(struct VSM_data *vd); - diff --git a/lib/libvcl/vmodtool.py b/lib/libvcl/vmodtool.py index e2cf8b5..d32eb4a 100755 --- a/lib/libvcl/vmodtool.py +++ b/lib/libvcl/vmodtool.py @@ -318,4 +318,3 @@ fh.write('extern const void * const Vmod_Id;\n') fc.write('const void * const Vmod_Id = &Vmod_Id;\n') fc.write("\n") - diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index fd65fd2..cd23473 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -94,4 +94,3 @@ vmod_test_priv_vcl(struct req *req, struct vmod_priv *priv) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); assert(!strcmp(priv->priv, "FOO")); } - diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 4265cf1..037ffdd 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -177,4 +177,3 @@ vmod_collect(struct req *req, enum gethdr_e e, const char *h) else if (e == HDR_BERESP && req->busyobj != NULL) http_CollectHdr(req->busyobj->beresp, h); } - From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 590299e Make Flexelint hate me less Message-ID: commit 590299ee7c4b6cc4a25375db0d37fe600d6ff95a Author: Poul-Henning Kamp Date: Wed Jun 27 08:38:51 2012 +0000 Make Flexelint hate me less diff --git a/lib/libvarnishcompat/execinfo.c b/lib/libvarnishcompat/execinfo.c index 113ae32..7babdd4 100644 --- a/lib/libvarnishcompat/execinfo.c +++ b/lib/libvarnishcompat/execinfo.c @@ -127,22 +127,22 @@ backtrace_symbols(void *const *buffer, int size) } /* Binary expansion */ -#define L1(x) L0(x); L0(x + 0x01); -#define L2(x) L1(x); L1(x + 0x02); -#define L3(x) L2(x); L2(x + 0x04); -#define L4(x) L3(x); L3(x + 0x08); -#define L5(x) L4(x); L4(x + 0x10); -#define L6(x) L5(x); L5(x + 0x20); -#define L7(x) L6(x); L6(x + 0x40); +#define DO_P2_TIMES_1(x) DO_P2_TIMES_0(x); DO_P2_TIMES_0((x) + (1<<0)) +#define DO_P2_TIMES_2(x) DO_P2_TIMES_1(x); DO_P2_TIMES_1((x) + (1<<1)) +#define DO_P2_TIMES_3(x) DO_P2_TIMES_2(x); DO_P2_TIMES_2((x) + (1<<2)) +#define DO_P2_TIMES_4(x) DO_P2_TIMES_3(x); DO_P2_TIMES_3((x) + (1<<3)) +#define DO_P2_TIMES_5(x) DO_P2_TIMES_4(x); DO_P2_TIMES_4((x) + (1<<4)) +#define DO_P2_TIMES_6(x) DO_P2_TIMES_5(x); DO_P2_TIMES_5((x) + (1<<5)) +#define DO_P2_TIMES_7(x) DO_P2_TIMES_6(x); DO_P2_TIMES_6((x) + (1<<6)) static void * getreturnaddr(int level) { switch(level) { -#define L0(x) case x: return __builtin_return_address(x + 1); - L7(0) -#undef L0 +#define DO_P2_TIMES_0(x) case (x): return __builtin_return_address((x) + 1) + DO_P2_TIMES_7(0); +#undef DO_P2_TIMES_0 default: return NULL; } } @@ -152,9 +152,9 @@ getframeaddr(int level) { switch(level) { -#define L0(x) case x: return __builtin_frame_address(x + 1); - L7(0) -#undef L0 +#define DO_P2_TIMES_0(x) case (x): return __builtin_frame_address((x) + 1) + DO_P2_TIMES_7(0); +#undef DO_P2_TIMES_0 default: return NULL; } } From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 7e25234 Overhaul the thread-pool herding code. Message-ID: commit 7e25234d6f25cf1dd622e4d17e70902c99e63b8b Author: Poul-Henning Kamp Date: Wed Jun 27 09:51:12 2012 +0000 Overhaul the thread-pool herding code. NB: Changes to parameter defaults & units included in this commit! I tried to apply some queuing theory to this problem and as much as I admire Agner Krarup Erlang, his math isn't much help when you don't know any of the relevant metrics for your queue. Instead I took a much simpler approach: "If we fail to get a thread, we probably need more threads", and have rewritten the herder to to react faster and more reliably to such events. I went over the parameters for thread-pools and normalized timeouts to seconds rather than milliseconds (beware!) and polished descriptions etc. diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index a723a9e..fa1f401 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -31,13 +31,6 @@ * Pools can be added on the fly, as a means to mitigate lock contention, * but can only be removed again by a restart. (XXX: we could fix that) * - * Two threads herd the pools, one eliminates idle threads and aggregates - * statistics for all the pools, the other thread creates new threads - * on demand, subject to various numerical constraints. - * - * The algorithm for when to create threads needs to be reactive enough - * to handle startup spikes, but sufficiently attenuated to not cause - * thread pileups. This remains subject for improvement. */ #include "config.h" @@ -50,36 +43,6 @@ #include "vtim.h" -/*-------------------------------------------------------------------- - * MAC OS/X is incredibly moronic when it comes to time and such... - */ - -#ifndef CLOCK_MONOTONIC -#define CLOCK_MONOTONIC 0 - -#include - -static int -clock_gettime(int foo, struct timespec *ts) -{ - struct timeval tv; - - (void)foo; - gettimeofday(&tv, NULL); - ts->tv_sec = tv.tv_sec; - ts->tv_nsec = tv.tv_usec * 1000; - return (0); -} - -static int -pthread_condattr_setclock(pthread_condattr_t *attr, int foo) -{ - (void)attr; - (void)foo; - return (0); -} -#endif /* !CLOCK_MONOTONIC */ - VTAILQ_HEAD(taskhead, pool_task); struct poolsock { @@ -97,7 +60,6 @@ struct pool { VTAILQ_ENTRY(pool) list; pthread_cond_t herder_cond; - struct lock herder_mtx; pthread_t herder_thr; struct vxid vxid; @@ -107,8 +69,8 @@ struct pool { struct taskhead front_queue; struct taskhead back_queue; unsigned nthr; + unsigned dry; unsigned lqueue; - unsigned last_lqueue; uintmax_t ndropped; uintmax_t nqueued; struct sesspool *sesspool; @@ -121,19 +83,21 @@ static pthread_t thr_pool_herder; */ static struct worker * -pool_getidleworker(const struct pool *pp, int back) +pool_getidleworker(struct pool *pp) { struct pool_task *pt; struct worker *wrk; CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); Lck_AssertHeld(&pp->mtx); - if (back) - pt = VTAILQ_LAST(&pp->idle_queue, taskhead); - else - pt = VTAILQ_FIRST(&pp->idle_queue); - if (pt == NULL) + pt = VTAILQ_FIRST(&pp->idle_queue); + if (pt == NULL) { + if (pp->nthr < cache_param->wthread_max) { + pp->dry++; + AZ(pthread_cond_signal(&pp->herder_cond)); + } return (NULL); + } AZ(pt->func); CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC); return (wrk); @@ -185,7 +149,7 @@ pool_accept(struct worker *wrk, void *arg) Lck_Lock(&pp->mtx); wa->vxid = VXID_Get(&pp->vxid); - wrk2 = pool_getidleworker(pp, 0); + wrk2 = pool_getidleworker(pp); if (wrk2 == NULL) { /* No idle threads, do it ourselves */ Lck_Unlock(&pp->mtx); @@ -225,7 +189,7 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how) * The common case first: Take an idle thread, do it. */ - wrk = pool_getidleworker(pp, 0); + wrk = pool_getidleworker(pp); if (wrk != NULL) { VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list); AZ(wrk->task.func); @@ -242,7 +206,7 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how) break; case POOL_QUEUE_FRONT: /* If we have too much in the queue already, refuse. */ - if (pp->lqueue > (cache_param->queue_max * pp->nthr) / 100) { + if (pp->lqueue > cache_param->wthread_queue_limit) { pp->ndropped++; retval = -1; } else { @@ -258,8 +222,6 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how) WRONG("Unknown enum pool_how"); } Lck_Unlock(&pp->mtx); - if (retval) - AZ(pthread_cond_signal(&pp->herder_cond)); return (retval); } @@ -320,7 +282,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk) } /*-------------------------------------------------------------------- - * Create another thread, if necessary & possible + * Create another thread. */ static void @@ -328,35 +290,23 @@ pool_breed(struct pool *qp, const pthread_attr_t *tp_attr) { pthread_t tp; - /* - * If we need more threads, and have space, create - * one more thread. - */ - if (qp->nthr < cache_param->wthread_min || /* Not enough threads yet */ - (qp->lqueue > cache_param->wthread_add_threshold && /* need more */ - qp->lqueue > qp->last_lqueue)) { /* not getting better since last */ - if (qp->nthr > cache_param->wthread_max) { - Lck_Lock(&pool_mtx); - VSC_C_main->threads_limited++; - Lck_Unlock(&pool_mtx); - } else if (pthread_create(&tp, tp_attr, WRK_thread, qp)) { - VSL(SLT_Debug, 0, "Create worker thread failed %d %s", - errno, strerror(errno)); - Lck_Lock(&pool_mtx); - VSC_C_main->threads_failed++; - Lck_Unlock(&pool_mtx); - VTIM_sleep(cache_param->wthread_fail_delay * 1e-3); - } else { - AZ(pthread_detach(tp)); - VTIM_sleep(cache_param->wthread_add_delay * 1e-3); - qp->nthr++; - Lck_Lock(&pool_mtx); - VSC_C_main->threads++; - VSC_C_main->threads_created++; - Lck_Unlock(&pool_mtx); - } + if (pthread_create(&tp, tp_attr, WRK_thread, qp)) { + VSL(SLT_Debug, 0, "Create worker thread failed %d %s", + errno, strerror(errno)); + Lck_Lock(&pool_mtx); + VSC_C_main->threads_failed++; + Lck_Unlock(&pool_mtx); + VTIM_sleep(cache_param->wthread_fail_delay); + } else { + AZ(pthread_detach(tp)); + qp->dry = 0; + qp->nthr++; + Lck_Lock(&pool_mtx); + VSC_C_main->threads++; + VSC_C_main->threads_created++; + Lck_Unlock(&pool_mtx); + VTIM_sleep(cache_param->wthread_add_delay); } - qp->last_lqueue = qp->lqueue; } /*-------------------------------------------------------------------- @@ -378,11 +328,10 @@ static void* pool_herder(void *priv) { struct pool *pp; + struct pool_task *pt; pthread_attr_t tp_attr; - struct timespec ts; double t_idle; struct worker *wrk; - int i; CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC); AZ(pthread_attr_init(&tp_attr)); @@ -397,55 +346,56 @@ pool_herder(void *priv) AZ(pthread_attr_init(&tp_attr)); } - pool_breed(pp, &tp_attr); - - if (pp->nthr < cache_param->wthread_min) + /* Make more threads if needed and allowed */ + if (pp->nthr < cache_param->wthread_min || + (pp->dry && pp->nthr < cache_param->wthread_max)) { + pool_breed(pp, &tp_attr); continue; - - AZ(clock_gettime(CLOCK_MONOTONIC, &ts)); - ts.tv_sec += cache_param->wthread_purge_delay / 1000; - ts.tv_nsec += - (cache_param->wthread_purge_delay % 1000) * 1000000; - if (ts.tv_nsec >= 1000000000) { - ts.tv_sec++; - ts.tv_nsec -= 1000000000; } - Lck_Lock(&pp->herder_mtx); - i = Lck_CondWait(&pp->herder_cond, &pp->herder_mtx, &ts); - Lck_Unlock(&pp->herder_mtx); - if (!i) - continue; + if (pp->nthr > cache_param->wthread_min) { - if (pp->nthr <= cache_param->wthread_min) - continue; + t_idle = VTIM_real() - cache_param->wthread_timeout; - t_idle = VTIM_real() - cache_param->wthread_timeout; + Lck_Lock(&pp->mtx); + /* XXX: unsafe counters */ + VSC_C_main->sess_queued += pp->nqueued; + VSC_C_main->sess_dropped += pp->ndropped; + pp->nqueued = pp->ndropped = 0; - Lck_Lock(&pp->mtx); - VSC_C_main->sess_queued += pp->nqueued; - VSC_C_main->sess_dropped += pp->ndropped; - pp->nqueued = pp->ndropped = 0; - wrk = pool_getidleworker(pp, 1); - if (wrk != NULL && (wrk->lastused < t_idle || - pp->nthr > cache_param->wthread_max)) { - VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list); - AZ(wrk->task.func); - } else wrk = NULL; - Lck_Unlock(&pp->mtx); + pt = VTAILQ_LAST(&pp->idle_queue, taskhead); + if (pt != NULL) { + AZ(pt->func); + CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC); + + if (wrk->lastused < t_idle || + pp->nthr > cache_param->wthread_max) + VTAILQ_REMOVE(&pp->idle_queue, + &wrk->task, list); + else + wrk = NULL; + } + Lck_Unlock(&pp->mtx); - /* And give it a kiss on the cheek... */ - if (wrk != NULL) { - pp->nthr--; - Lck_Lock(&pool_mtx); - VSC_C_main->threads--; - VSC_C_main->threads_destroyed++; - Lck_Unlock(&pool_mtx); - wrk->task.func = NULL; - wrk->task.priv = NULL; - AZ(pthread_cond_signal(&wrk->cond)); + /* And give it a kiss on the cheek... */ + if (wrk != NULL) { + pp->nthr--; + Lck_Lock(&pool_mtx); + VSC_C_main->threads--; + VSC_C_main->threads_destroyed++; + Lck_Unlock(&pool_mtx); + wrk->task.func = NULL; + wrk->task.priv = NULL; + VTIM_sleep(cache_param->wthread_destroy_delay); + continue; + } } + + Lck_Lock(&pp->mtx); + if (!pp->dry) + (void)Lck_CondWait(&pp->herder_cond, &pp->mtx, NULL); + Lck_Unlock(&pp->mtx); } NEEDLESS_RETURN(NULL); } @@ -460,10 +410,10 @@ pool_mkpool(unsigned pool_no) struct pool *pp; struct listen_sock *ls; struct poolsock *ps; - pthread_condattr_t cv_attr; ALLOC_OBJ(pp, POOL_MAGIC); - XXXAN(pp); + if (pp == NULL) + return (NULL); Lck_New(&pp->mtx, lck_wq); VTAILQ_INIT(&pp->idle_queue); @@ -483,11 +433,7 @@ pool_mkpool(unsigned pool_no) AZ(Pool_Task(pp, &ps->task, POOL_QUEUE_BACK)); } - AZ(pthread_condattr_init(&cv_attr)); - AZ(pthread_condattr_setclock(&cv_attr, CLOCK_MONOTONIC)); - AZ(pthread_cond_init(&pp->herder_cond, &cv_attr)); - AZ(pthread_condattr_destroy(&cv_attr)); - Lck_New(&pp->herder_mtx, lck_herder); + AZ(pthread_cond_init(&pp->herder_cond, NULL)); AZ(pthread_create(&pp->herder_thr, NULL, pool_herder, pp)); return (pp); diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index 2e8bf37..a66bcf8 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -62,16 +62,15 @@ struct params { /* Worker threads and pool */ unsigned wthread_min; unsigned wthread_max; - unsigned wthread_timeout; + double wthread_timeout; unsigned wthread_pools; unsigned wthread_add_threshold; - unsigned wthread_add_delay; - unsigned wthread_fail_delay; - unsigned wthread_purge_delay; - unsigned wthread_stats_rate; + double wthread_add_delay; + double wthread_fail_delay; + double wthread_destroy_delay; + double wthread_stats_rate; ssize_t wthread_stacksize; - - unsigned queue_max; + unsigned wthread_queue_limit; /* Memory allocation hints */ unsigned workspace_client; diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index c425882..859ef14 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -93,7 +93,7 @@ tweak_generic_timeout(struct cli *cli, volatile unsigned *dst, const char *arg) /*--------------------------------------------------------------------*/ -void +static void tweak_timeout(struct cli *cli, const struct parspec *par, const char *arg) { volatile unsigned *dest; @@ -137,7 +137,7 @@ tweak_generic_timeout_double(struct cli *cli, volatile double *dest, return (0); } -static void +void tweak_timeout_double(struct cli *cli, const struct parspec *par, const char *arg) { diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h index b4538d2..ee1ad52 100644 --- a/bin/varnishd/mgt/mgt_param.h +++ b/bin/varnishd/mgt/mgt_param.h @@ -53,7 +53,7 @@ struct parspec { int tweak_generic_uint(struct cli *cli, volatile unsigned *dest, const char *arg, unsigned min, unsigned max); void tweak_uint(struct cli *cli, const struct parspec *par, const char *arg); -void tweak_timeout(struct cli *cli, +void tweak_timeout_double(struct cli *cli, const struct parspec *par, const char *arg); void tweak_bytes(struct cli *cli, const struct parspec *par, const char *arg); diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index 986d405..385a500 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -97,7 +97,8 @@ tweak_thread_pool_max(struct cli *cli, const struct parspec *par, /*--------------------------------------------------------------------*/ const struct parspec WRK_parspec[] = { - { "thread_pools", tweak_uint, &mgt_param.wthread_pools, 1, UINT_MAX, + { "thread_pools", tweak_uint, &mgt_param.wthread_pools, + 1, UINT_MAX, "Number of worker thread pools.\n" "\n" "Increasing number of worker pools decreases lock " @@ -110,71 +111,69 @@ const struct parspec WRK_parspec[] = { "restart to take effect.", EXPERIMENTAL | DELAYED_EFFECT, "2", "pools" }, - { "thread_pool_max", tweak_thread_pool_max, NULL, 1, 0, + { "thread_pool_max", tweak_thread_pool_max, NULL, 10, 0, "The maximum number of worker threads in each pool.\n" "\n" "Do not set this higher than you have to, since excess " "worker threads soak up RAM and CPU and generally just get " - "in the way of getting work done.\n", - EXPERIMENTAL | DELAYED_EFFECT, - "500", "threads" }, - { "thread_pool_min", tweak_thread_pool_min, NULL, 2, 0, + "in the way of getting work done.\n" + "\n" + "Minimum is 10 threads.", + DELAYED_EFFECT, + "5000", "threads" }, + { "thread_pool_min", tweak_thread_pool_min, NULL, 10, 0, "The minimum number of worker threads in each pool.\n" "\n" "Increasing this may help ramp up faster from low load " - "situations where threads have expired.\n" - "\n" - "Minimum is 2 threads.", - EXPERIMENTAL | DELAYED_EFFECT, - "5", "threads" }, - { "thread_pool_timeout", tweak_timeout, &mgt_param.wthread_timeout, - 1, 0, + "situations or when threads have expired.\n" + "\n" + "Minimum is 10 threads.", + DELAYED_EFFECT, + "100", "threads" }, + { "thread_pool_timeout", + tweak_timeout_double, &mgt_param.wthread_timeout, + 10, UINT_MAX, "Thread idle threshold.\n" "\n" "Threads in excess of thread_pool_min, which have been idle " - "for at least this long are candidates for purging.\n" + "for at least this long, will be destroyed.\n" "\n" - "Minimum is 1 second.", + "Minimum is 10 seconds.", EXPERIMENTAL | DELAYED_EFFECT, "300", "seconds" }, - { "thread_pool_purge_delay", - tweak_timeout, &mgt_param.wthread_purge_delay, 100, 0, - "Wait this long between purging threads.\n" + { "thread_pool_destroy_delay", + tweak_timeout_double, &mgt_param.wthread_destroy_delay, + 0.01, UINT_MAX, + "Wait this long after destroying a thread.\n" "\n" "This controls the decay of thread pools when idle(-ish).\n" "\n" - "Minimum is 100 milliseconds.", + "Minimum is 0.01 second.", EXPERIMENTAL | DELAYED_EFFECT, - "1000", "milliseconds" }, - { "thread_pool_add_threshold", - tweak_uint, &mgt_param.wthread_add_threshold, 0, UINT_MAX, - "Overflow threshold for worker thread creation.\n" - "\n" - "Setting this too low, will result in excess worker threads, " - "which is generally a bad idea.\n" - "\n" - "Setting it too high results in insuffient worker threads.\n", - EXPERIMENTAL, - "2", "requests" }, + "1", "seconds" }, { "thread_pool_add_delay", - tweak_timeout, &mgt_param.wthread_add_delay, 0, UINT_MAX, - "Wait at least this long between creating threads.\n" + tweak_timeout_double, &mgt_param.wthread_add_delay, + 0, UINT_MAX, + "Wait at least this long after creating a thread.\n" "\n" - "Setting this too long results in insuffient worker threads.\n" + "Some (buggy) systems may need a short (sub-second) " + "delay between creating threads.\n" + "Set this to a few milliseconds if you see the " + "'threads_failed' counter grow too much.\n" "\n" - "Setting this too short increases the risk of worker " - "thread pile-up.\n", - 0, - "2", "milliseconds" }, + "Setting this too high results in insuffient worker threads.\n", + EXPERIMENTAL, + "0", "seconds" }, { "thread_pool_fail_delay", - tweak_timeout, &mgt_param.wthread_fail_delay, 100, UINT_MAX, + tweak_timeout_double, &mgt_param.wthread_fail_delay, + 10e-3, UINT_MAX, "Wait at least this long after a failed thread creation " "before trying to create another thread.\n" "\n" "Failure to create a worker thread is often a sign that " " the end is near, because the process is running out of " - "RAM resources for thread stacks.\n" - "This delay tries to not rush it on needlessly.\n" + "some resource. " + "This delay tries to not rush the end on needlessly.\n" "\n" "If thread creation failures are a problem, check that " "thread_pool_max is not too high.\n" @@ -183,7 +182,7 @@ const struct parspec WRK_parspec[] = { "thread_pool_min, to reduce the rate at which treads are " "destroyed and later recreated.\n", EXPERIMENTAL, - "200", "milliseconds" }, + "0.2", "seconds" }, { "thread_stats_rate", tweak_uint, &mgt_param.wthread_stats_rate, 0, UINT_MAX, "Worker threads accumulate statistics, and dump these into " @@ -194,13 +193,15 @@ const struct parspec WRK_parspec[] = { "its accumulated stats into the global counters.\n", EXPERIMENTAL, "10", "requests" }, - { "queue_max", tweak_uint, &mgt_param.queue_max, 0, UINT_MAX, - "Percentage permitted queue length.\n" + { "thread_queue_limit", tweak_uint, &mgt_param.wthread_queue_limit, + 0, UINT_MAX, + "Permitted queue length per thread-pool.\n" "\n" - "This sets the ratio of queued requests to worker threads, " - "above which sessions will be dropped instead of queued.\n", + "This sets the number of requests we will queue, waiting " + "for an available thread. Above this limit sessions will " + "be dropped instead of queued.\n", EXPERIMENTAL, - "100", "%" }, + "20", "" }, { "rush_exponent", tweak_uint, &mgt_param.rush_exponent, 2, UINT_MAX, "How many parked request we start for each completed " "request on the object.\n" diff --git a/bin/varnishtest/tests/c00002.vtc b/bin/varnishtest/tests/c00002.vtc index 6914736..5351fec 100644 --- a/bin/varnishtest/tests/c00002.vtc +++ b/bin/varnishtest/tests/c00002.vtc @@ -5,7 +5,7 @@ server s1 { txresp -hdr "Connection: close" -body "012345\n" } -start -varnish v1 -arg "-p thread_pool_min=2 -p thread_pool_max=8 -p thread_pools=4 -p thread_pool_purge_delay=100 -p thread_pool_timeout=1 -p thread_pool_add_delay=100" +varnish v1 -arg "-p thread_pool_min=10 -p thread_pool_max=10 -p thread_pools=2" varnish v1 -vcl+backend {} -start @@ -16,4 +16,4 @@ client c1 { expect resp.status == 200 } -run -varnish v1 -expect threads == 8 +varnish v1 -expect threads == 20 From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 27d63c2 Eliminate the "init_done" and listen socket fields from struct sess by moving the VSL'ing of socket endpoints earlier in the processing. Message-ID: commit 27d63c26da9b445bf6994228668fbe9fdb6bcd2b Author: Poul-Henning Kamp Date: Mon Jul 2 08:21:23 2012 +0000 Eliminate the "init_done" and listen socket fields from struct sess by moving the VSL'ing of socket endpoints earlier in the processing. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 1d991f1..d19dc32 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -670,8 +670,6 @@ struct sess { socklen_t mysockaddrlen; struct sockaddr_storage sockaddr; struct sockaddr_storage mysockaddr; - struct listen_sock *mylsock; - int init_done; /* formatted ascii client address */ char addr[ADDR_BUFSIZE]; @@ -695,7 +693,7 @@ struct sess { void VCA_Init(void); void VCA_Shutdown(void); int VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa); -void VCA_SetupSess(struct worker *w, struct sess *sp); +const char *VCA_SetupSess(struct worker *w, struct sess *sp); void VCA_FailSess(struct worker *w); /* cache_backend.c */ diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 3ca79c1..5e71d00 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -239,10 +239,11 @@ VCA_FailSess(struct worker *wrk) * We have allocated a session, move our info into it. */ -void +const char * VCA_SetupSess(struct worker *wrk, struct sess *sp) { struct wrk_accept *wa; + const char *retval; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); @@ -252,9 +253,7 @@ VCA_SetupSess(struct worker *wrk, struct sess *sp) sp->fd = wa->acceptsock; sp->vsl_id = wa->acceptsock | VSL_CLIENTMARKER ; wa->acceptsock = -1; - sp->t_open = VTIM_real(); - sp->mylsock = wa->acceptlsock; - CHECK_OBJ_NOTNULL(sp->mylsock, LISTEN_SOCK_MAGIC); + retval = wa->acceptlsock->name; assert(wa->acceptaddrlen <= sp->sockaddrlen); memcpy(&sp->sockaddr, &wa->acceptaddr, wa->acceptaddrlen); sp->sockaddrlen = wa->acceptaddrlen; @@ -277,6 +276,7 @@ VCA_SetupSess(struct worker *wrk, struct sess *sp) VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_RCVTIMEO, &tv_rcvtimeo, sizeof tv_rcvtimeo)); #endif + return (retval); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 004f815..84fc3ea 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -73,8 +73,6 @@ DOT acceptor -> first [style=bold,color=green] #include "cache.h" -#include "common/heritage.h" - #include "hash/hash_slinger.h" #include "vcl.h" #include "vcli_priv.h" @@ -112,8 +110,6 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) struct pollfd pfd[1]; double now, when; const char *why = NULL; - char laddr[ADDR_BUFSIZE]; - char lport[PORT_BUFSIZE]; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -121,30 +117,6 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) assert(req->sp == sp); - if (!sp->init_done) { - VTCP_name(&sp->sockaddr, sp->sockaddrlen, - sp->addr, sizeof sp->addr, sp->port, sizeof sp->port); - if (cache_param->log_local_addr) { - AZ(getsockname(sp->fd, (void*)&sp->mysockaddr, - &sp->mysockaddrlen)); - VTCP_name(&sp->mysockaddr, sp->mysockaddrlen, - laddr, sizeof laddr, lport, sizeof lport); - /* XXX: have no req yet */ - VSLb(req->vsl, SLT_SessionOpen, "%s %s %s %s", - sp->addr, sp->port, laddr, lport); - } else { - /* XXX: have no req yet */ - VSLb(req->vsl, SLT_SessionOpen, "%s %s %s", - sp->addr, sp->port, sp->mylsock->name); - } - - wrk->acct_tmp.sess++; - - sp->t_rx = sp->t_open; - sp->t_idle = sp->t_open; - sp->init_done = 1; - } - assert(!isnan(sp->t_rx)); AZ(req->vcl); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 25dc687..3ed251d 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -42,6 +42,7 @@ #include "cache.h" #include "waiter/waiter.h" +#include "vtcp.h" #include "vtim.h" static unsigned ses_size = sizeof (struct sess); @@ -148,7 +149,40 @@ ses_pool_task(struct worker *wrk, void *arg) } /*-------------------------------------------------------------------- + */ +static void +ses_vsl_socket(struct req *req, const char *lsockname) +{ + char laddr[ADDR_BUFSIZE]; + char lport[PORT_BUFSIZE]; + struct sess *sp; + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + AN(lsockname); + sp = req->sp; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + + VTCP_name(&sp->sockaddr, sp->sockaddrlen, + sp->addr, sizeof sp->addr, sp->port, sizeof sp->port); + if (cache_param->log_local_addr) { + AZ(getsockname(sp->fd, (void*)&sp->mysockaddr, + &sp->mysockaddrlen)); + VTCP_name(&sp->mysockaddr, sp->mysockaddrlen, + laddr, sizeof laddr, lport, sizeof lport); + /* XXX: have no req yet */ + VSLb(req->vsl, SLT_SessionOpen, "%s %s %s %s %s", + sp->addr, sp->port, lsockname, laddr, lport); + } else { + /* XXX: have no req yet */ + VSLb(req->vsl, SLT_SessionOpen, "%s %s %s - -", + sp->addr, sp->port, lsockname); + } +} + +/*-------------------------------------------------------------------- * The pool-task for a newly accepted session + * + * Called from assigned worker thread */ void @@ -157,6 +191,7 @@ SES_pool_accept_task(struct worker *wrk, void *arg) struct sesspool *pp; struct req *req; struct sess *sp; + const char *lsockname; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CAST_OBJ_NOTNULL(pp, arg, SESSPOOL_MAGIC); @@ -168,10 +203,21 @@ SES_pool_accept_task(struct worker *wrk, void *arg) VCA_FailSess(wrk); return; } - VCA_SetupSess(wrk, sp); - sp->sess_step = S_STP_NEWREQ; + + sp->t_open = VTIM_real(); + sp->t_rx = sp->t_open; + sp->t_idle = sp->t_open; + + lsockname = VCA_SetupSess(wrk, sp); + req = ses_GetReq(sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + ses_vsl_socket(req, lsockname); + + wrk->acct_tmp.sess++; + + sp->sess_step = S_STP_NEWREQ; ses_pool_task(wrk, req); } From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 4c4d6f6 Make sure that all sessions, requests and busyobj's have a unique transaction id 'vxid' Message-ID: commit 4c4d6f652d0b96aba584575e692a913172146168 Author: Poul-Henning Kamp Date: Mon Jul 2 10:37:36 2012 +0000 Make sure that all sessions, requests and busyobj's have a unique transaction id 'vxid' diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index d19dc32..ae42dd0 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -259,7 +259,7 @@ struct vsl_log { /*--------------------------------------------------------------------*/ -struct vxid { +struct vxid_pool { uint32_t next; uint32_t count; }; @@ -275,7 +275,6 @@ struct wrk_accept { socklen_t acceptaddrlen; int acceptsock; struct listen_sock *acceptlsock; - uint32_t vxid; }; /* Worker pool stuff -------------------------------------------------*/ @@ -319,6 +318,7 @@ struct worker { struct ws aws[1]; + struct vxid_pool vxid_pool; /* Temporary accounting */ struct acct acct_tmp; @@ -472,6 +472,7 @@ struct busyobj { * is recycled. */ unsigned refcount; + uint32_t vxid; uint8_t *vary; unsigned is_gzip; @@ -560,6 +561,7 @@ struct req { unsigned magic; #define REQ_MAGIC 0x2751aaa1 + uint32_t vxid; unsigned xid; int restarts; int esi_level; @@ -655,7 +657,6 @@ struct sess { int fd; unsigned vsl_id; uint32_t vxid; - uint32_t vseq; /* Cross references ------------------------------------------*/ @@ -857,7 +858,7 @@ int HTC_Complete(struct http_conn *htc); #undef HTTPH /* cache_main.c */ -uint32_t VXID_Get(struct vxid *v); +uint32_t VXID_Get(struct vxid_pool *v); extern volatile struct params * cache_param; void THR_SetName(const char *name); const char* THR_GetName(void); diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 5e71d00..bbec54f 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -248,8 +248,6 @@ VCA_SetupSess(struct worker *wrk, struct sess *sp) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CAST_OBJ_NOTNULL(wa, (void*)wrk->aws->f, WRK_ACCEPT_MAGIC); - sp->vxid = wa->vxid; - sp->vseq = 0; sp->fd = wa->acceptsock; sp->vsl_id = wa->acceptsock | VSL_CLIENTMARKER ; wa->acceptsock = -1; diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 48efe08..8f0cc50 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -108,6 +108,7 @@ VBO_GetBusyObj(struct worker *wrk) AZ(bo->refcount); bo->refcount = 1; + bo->vxid = VXID_Get(&wrk->vxid_pool); p = (void*)(bo + 1); p = (void*)PRNDUP(p); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 84fc3ea..c50d503 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -277,6 +277,7 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) WS_Reset(req->ws, NULL); WS_Reset(wrk->aws, NULL); + req->vxid = VXID_Get(&wrk->vxid_pool); i = HTC_Reinit(req->htc); if (i == 1) { diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index dcb3cb1..94f2c8f 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -85,13 +85,15 @@ THR_GetName(void) } /*-------------------------------------------------------------------- + * VXID's are unique transaction numbers allocated with a minimum of + * locking overhead via pools in the worker threads. */ static uint32_t vxid_base; static struct lock vxid_lock; static void -vxid_More(struct vxid *v) +vxid_More(struct vxid_pool *v) { Lck_Lock(&vxid_lock); @@ -102,7 +104,7 @@ vxid_More(struct vxid *v) } uint32_t -VXID_Get(struct vxid *v) +VXID_Get(struct vxid_pool *v) { if (v->count == 0) vxid_More(v); diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index fa1f401..21a8a31 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -62,8 +62,6 @@ struct pool { pthread_cond_t herder_cond; pthread_t herder_thr; - struct vxid vxid; - struct lock mtx; struct taskhead idle_queue; struct taskhead front_queue; @@ -148,7 +146,6 @@ pool_accept(struct worker *wrk, void *arg) } Lck_Lock(&pp->mtx); - wa->vxid = VXID_Get(&pp->vxid); wrk2 = pool_getidleworker(pp); if (wrk2 == NULL) { /* No idle threads, do it ourselves */ diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 3ed251d..64ee6f5 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -207,12 +207,15 @@ SES_pool_accept_task(struct worker *wrk, void *arg) sp->t_open = VTIM_real(); sp->t_rx = sp->t_open; sp->t_idle = sp->t_open; + sp->vxid = VXID_Get(&wrk->vxid_pool); lsockname = VCA_SetupSess(wrk, sp); req = ses_GetReq(sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + req->vxid = VXID_Get(&wrk->vxid_pool); + ses_vsl_socket(req, lsockname); wrk->acct_tmp.sess++; @@ -262,12 +265,16 @@ SES_Handle(struct sess *sp, double now) struct req *req; struct sesspool *pp; + /* NB: This only works with single-threaded waiters */ + static struct vxid_pool vxid_pool; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); pp = sp->sesspool; CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); AN(pp->pool); req = ses_GetReq(sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + req->vxid = VXID_Get(&vxid_pool); sp->task.func = ses_pool_task; sp->task.priv = req; sp->sess_step = S_STP_NEWREQ; From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] ef1b8cb Don't hand out zero VXIDs we need that as a magic value. Message-ID: commit ef1b8cb1b27d1e2092c147a966fc15f0e203dd68 Author: Poul-Henning Kamp Date: Mon Jul 2 13:31:42 2012 +0000 Don't hand out zero VXIDs we need that as a magic value. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index ae42dd0..2162413 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -937,7 +937,7 @@ void *VSM_Alloc(unsigned size, const char *class, const char *type, void VSL_Setup(struct vsl_log *vsl, void *ptr, size_t len); void VSM_Free(void *ptr); #ifdef VSL_ENDMARKER -void VSL(enum VSL_tag_e tag, int id, const char *fmt, ...) +void VSL(enum VSL_tag_e tag, uint32_t vxid, const char *fmt, ...) __printflike(3, 4); void VSLb(struct vsl_log *, enum VSL_tag_e tag, const char *fmt, ...) __printflike(3, 4); diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index 94f2c8f..c48a3d9 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -87,30 +87,29 @@ THR_GetName(void) /*-------------------------------------------------------------------- * VXID's are unique transaction numbers allocated with a minimum of * locking overhead via pools in the worker threads. + * + * VXID's are mostly for use in VSL and for that reason we never return + * zero vxid, in order to reserve that for "unassociated" VSL records. */ static uint32_t vxid_base; static struct lock vxid_lock; -static void -vxid_More(struct vxid_pool *v) -{ - - Lck_Lock(&vxid_lock); - v->next = vxid_base; - v->count = 32768; - vxid_base = v->count; - Lck_Unlock(&vxid_lock); -} - uint32_t VXID_Get(struct vxid_pool *v) { - if (v->count == 0) - vxid_More(v); - AN(v->count); - v->count--; - return (v->next++); + do { + if (v->count == 0) { + Lck_Lock(&vxid_lock); + v->next = vxid_base; + v->count = 32768; + vxid_base = v->count; + Lck_Unlock(&vxid_lock); + } + v->count--; + v->next++; + } while (v->next == 0); + return (v->next); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 85a603f..fb63e75 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -61,12 +61,12 @@ vsl_w0(uint32_t type, uint32_t length) /*--------------------------------------------------------------------*/ static inline void -vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, unsigned id) +vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, uint32_t vxid) { assert(((uintptr_t)p & 0x3) == 0); - p[1] = id; + p[1] = vxid; VMB(); p[0] = vsl_w0(tag, len); } @@ -133,7 +133,7 @@ vsl_get(unsigned len, unsigned records, unsigned flushes) */ static void -vslr(enum VSL_tag_e tag, int id, const char *b, unsigned len) +vslr(enum VSL_tag_e tag, uint32_t vxid, const char *b, unsigned len) { uint32_t *p; unsigned mlen; @@ -147,13 +147,13 @@ vslr(enum VSL_tag_e tag, int id, const char *b, unsigned len) p = vsl_get(len, 1, 0); memcpy(p + 2, b, len); - vsl_hdr(tag, p, len, id); + vsl_hdr(tag, p, len, vxid); } /*--------------------------------------------------------------------*/ void -VSL(enum VSL_tag_e tag, int id, const char *fmt, ...) +VSL(enum VSL_tag_e tag, uint32_t vxid, const char *fmt, ...) { va_list ap; unsigned n, mlen = cache_param->shm_reclen; @@ -167,12 +167,12 @@ VSL(enum VSL_tag_e tag, int id, const char *fmt, ...) va_start(ap, fmt); if (strchr(fmt, '%') == NULL) { - vslr(tag, id, fmt, strlen(fmt)); + vslr(tag, vxid, fmt, strlen(fmt)); } else { n = vsnprintf(buf, mlen, fmt, ap); if (n > mlen) n = mlen; - vslr(tag, id, buf, n); + vslr(tag, vxid, buf, n); } va_end(ap); } From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] f41fbd1 Rename '-x dumprst' to '-x dumprstparam', we may have more RST coming in the future. Message-ID: commit f41fbd19bdb300f4ec571b09e88202e6a81b72cd Author: Poul-Henning Kamp Date: Mon Jul 2 20:49:56 2012 +0000 Rename '-x dumprst' to '-x dumprstparam', we may have more RST coming in the future. diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 65dbc02..512c6dc 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -70,7 +70,7 @@ const void *pick(const struct choice *cp, const char *which, const char *kind); void MCF_ParamInit(struct cli *); void MCF_ParamSet(struct cli *, const char *param, const char *val); void MCF_ParamProtect(struct cli *, const char *arg); -void MCF_DumpRst(void); +void MCF_DumpRstParam(void); extern struct params mgt_param; /* mgt_sandbox.c */ diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 88f8253..18f1bb3 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -458,8 +458,8 @@ main(int argc, char * const *argv) VCS_Message("varnishd"); exit(0); case 'x': - if (!strcmp(optarg, "dumprst")) { - MCF_DumpRst(); + if (!strcmp(optarg, "dumprstparam")) { + MCF_DumpRstParam(); exit (0); } usage(); diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 859ef14..d17f9e2 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1503,14 +1503,14 @@ MCF_ParamInit(struct cli *cli) /*--------------------------------------------------------------------*/ void -MCF_DumpRst(void) +MCF_DumpRstParam(void) { const struct parspec *pp; const char *p, *q; int i; printf("\n.. The following is the autogenerated " - "output from varnishd -x dumprst\n\n"); + "output from varnishd -x dumprstparam\n\n"); for (i = 0; i < nparspec; i++) { pp = parspecs[i]; printf("%s\n", pp->name); diff --git a/bin/varnishtest/tests/a00009.vtc b/bin/varnishtest/tests/a00009.vtc index f89231f..e488b3f 100644 --- a/bin/varnishtest/tests/a00009.vtc +++ b/bin/varnishtest/tests/a00009.vtc @@ -1,4 +1,4 @@ varnishtest "Code coverage of VCL compiler and RSTdump" shell "cd ${topbuild}/bin/varnishd && ./varnishd -b 127.0.0.1:80 -C -n ${tmpdir} > /dev/null 2>&1" -shell "cd ${topbuild}/bin/varnishd && ./varnishd -x dumprst > /dev/null 2>&1" +shell "cd ${topbuild}/bin/varnishd && ./varnishd -x dumprstparam > /dev/null 2>&1" From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 24d426d Add code to dump VSL descriptions in RST format Message-ID: commit 24d426d569038afc81d0652fcb10ad4014f8ec2b Author: Poul-Henning Kamp Date: Mon Jul 2 21:08:15 2012 +0000 Add code to dump VSL descriptions in RST format diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 18f1bb3..fd7c97f 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -70,6 +70,40 @@ struct vev_base *mgt_evb; int exit_status = 0; struct vsb *vident; +/*--------------------------------------------------------------------*/ + +static void +mgt_sltm(const char *tag, const char *sdesc, const char *ldesc) +{ + int i; + + printf("\n%s\n", tag); + i = strlen(tag); + printf("%*.*s\n\n", i, i, "------------------------------------"); + if (*ldesc != '\0') + printf("%s\n", ldesc); + else if (*sdesc != '\0') + printf("%s\n", sdesc); + else + printf("%s\n", "(description not yet written)"); + +} + +static void +mgt_DumpRstVsl(void) +{ + + printf( + "\n.. The following is autogenerated output from " + "varnishd -x dumprstvsl\n\n"); + +#define SLTM(tag, sdesc, ldesc) mgt_sltm(#tag, sdesc, ldesc); +#include "tbl/vsl_tags.h" +#undef SLTM +} + +/*--------------------------------------------------------------------*/ + static void build_vident(void) { @@ -462,6 +496,10 @@ main(int argc, char * const *argv) MCF_DumpRstParam(); exit (0); } + if (!strcmp(optarg, "dumprstvsl")) { + mgt_DumpRstVsl(); + exit (0); + } usage(); break; case 'w': diff --git a/bin/varnishtest/tests/a00009.vtc b/bin/varnishtest/tests/a00009.vtc index e488b3f..7a2cee2 100644 --- a/bin/varnishtest/tests/a00009.vtc +++ b/bin/varnishtest/tests/a00009.vtc @@ -2,3 +2,4 @@ varnishtest "Code coverage of VCL compiler and RSTdump" shell "cd ${topbuild}/bin/varnishd && ./varnishd -b 127.0.0.1:80 -C -n ${tmpdir} > /dev/null 2>&1" shell "cd ${topbuild}/bin/varnishd && ./varnishd -x dumprstparam > /dev/null 2>&1" +shell "cd ${topbuild}/bin/varnishd && ./varnishd -x dumprstvsl > /dev/null 2>&1" From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 7aa0548 Turn some descriptions into intended RST format. Message-ID: commit 7aa0548615bd2f49c0e7816e127aea0b4bc36414 Author: Poul-Henning Kamp Date: Mon Jul 2 21:19:13 2012 +0000 Turn some descriptions into intended RST format. diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index fce1429..5a6c862 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -38,32 +38,55 @@ * Arguments: * Tag-Name * Short Description (1 line, max ?? chars) - * Long Description (Multi line) + * Long Description (in RST "definition list" format) */ -SLTM(Debug, "", "") -SLTM(Error, "", "") -SLTM(CLI, "CLI communication", "CLI communication between master and child process.") +SLTM(Debug, "Debug messages", + "Debug messages can normally be ignored, but are sometimes\n" + "helpful during trouble-shooting. Most debug messages must\n" + "be explicitly enabled with parameters." +) +SLTM(Error, "Error messages", + "Error messages are stuff you probably want to know." +) +SLTM(CLI, "CLI communication", + "CLI communication between master and child process." +) SLTM(StatSess, "Session statistics", "") -SLTM(ReqEnd, "Client request end", "Client request end. The first number is the XID. \n" -"The second is the time when processing of the request started.\n" -"The third is the time the request completed.\n" -"The forth is is the time elapsed between the request actually being accepted and\n" -"the start of the request processing.\n" -"The fifth number is the time elapsed from the start of the request processing \n" -"until we start delivering the object to the client.\n" -"The sixth and last number is the time from we start delivering the object\n" -"until the request completes. ") -SLTM(SessionOpen, "Client connection opened", "") -SLTM(SessionClose, "Client connection closed", "SessionClose tells you why HTTP\n" -"client-connections are closed. These can be:\n" -"timeout - No keep-alive was received within sess_timeout\n" -"Connection: close - The client specifed that keepalive should be disabled by sending a 'Connection: close' header.\n" -"no request - No initial request was received within sess_timeout.\n" -"EOF - ???\n" -"remote closed - ???\n" -"error - Processing reached vcl_error even if the status code indicates success\n" -"blast - ???") + +SLTM(ReqEnd, "Client request end", + "Marks the end of client request.\n\n" + "xid\n Transaction id.\n\n" + "Trxd\n Timestamp when the request started.\n\n" + "Tidle\n Timestamp when the request ended.\n\n" + "dTrx\n Time to receive request\n\n" + "dTproc\n Time to process request\n\n" + "dTtx\n Time to transmit response\n\n" +) + +SLTM(SessionOpen, "Client connection opened", + "The first record for a client connection, with the\n" + "socket-endpoints of the connection.\n\n" + "caddr\n Client IPv4/6 address\n\n" + "cport\n Client TCP port\n\n" + "lsock\n Listen socket name\n\n" + "laddr\n Local IPv4/6 address ('-' if $log_local_addr not set)\n\n" + "lport\n Local TCP port ('-' if $log_local_addr not set)\n\n" +) + +SLTM(SessionClose, "Client connection closed", + "SessionClose tells you why HTTP client-connections are closed. " + "These can be: " + "'timeout' - No keep-alive was received within sess_timeout. " + "'Connection: close' - The client specifed that keepalive should " + "be disabled by sending a 'Connection: close' header. " + "'no request' - No initial request was received within sess_timeout. " + "'EOF' - ??? " + "'remote closed' - ??? " + "'error' - Processing reached vcl_error even if the status code " + "indicates success. " + "' blast' - ???" +) SLTM(BackendOpen, "Backend connection opened", "") SLTM(BackendXID, "The unique ID of the backend transaction", "") SLTM(BackendReuse, "Backend connection reused", "") From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] e027388 Remove XXX comment overtaken by implementation. Message-ID: commit e02738851aeaebc4688f53b100087c8a30419098 Author: Poul-Henning Kamp Date: Mon Jul 2 21:22:16 2012 +0000 Remove XXX comment overtaken by implementation. diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index fb63e75..f5cc879 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -159,10 +159,6 @@ VSL(enum VSL_tag_e tag, uint32_t vxid, const char *fmt, ...) unsigned n, mlen = cache_param->shm_reclen; char buf[mlen]; - /* - * XXX: consider formatting into a stack buffer then move into - * XXX: shmlog with vslr(). - */ AN(fmt); va_start(ap, fmt); From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 72d3574 Log SessionOpen with VSL() so we can be sure it is always the first VSL record for a session. Message-ID: commit 72d35747a9ed1f2d5a5d146df4b5eceedb201981 Author: Poul-Henning Kamp Date: Tue Jul 3 07:53:42 2012 +0000 Log SessionOpen with VSL() so we can be sure it is always the first VSL record for a session. Make $log_local_address default to on. diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 64ee6f5..7f88212 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -149,18 +149,20 @@ ses_pool_task(struct worker *wrk, void *arg) } /*-------------------------------------------------------------------- + * VSL log the endpoints of the TCP connection. + * + * We use VSL() to get the sessions vxid and to make sure tha this + * VSL comes before anything else for this session. */ + static void -ses_vsl_socket(struct req *req, const char *lsockname) +ses_vsl_socket(struct sess *sp, const char *lsockname) { char laddr[ADDR_BUFSIZE]; char lport[PORT_BUFSIZE]; - struct sess *sp; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - AN(lsockname); - sp = req->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + AN(lsockname); VTCP_name(&sp->sockaddr, sp->sockaddrlen, sp->addr, sizeof sp->addr, sp->port, sizeof sp->port); @@ -169,14 +171,12 @@ ses_vsl_socket(struct req *req, const char *lsockname) &sp->mysockaddrlen)); VTCP_name(&sp->mysockaddr, sp->mysockaddrlen, laddr, sizeof laddr, lport, sizeof lport); - /* XXX: have no req yet */ - VSLb(req->vsl, SLT_SessionOpen, "%s %s %s %s %s", - sp->addr, sp->port, lsockname, laddr, lport); } else { - /* XXX: have no req yet */ - VSLb(req->vsl, SLT_SessionOpen, "%s %s %s - -", - sp->addr, sp->port, lsockname); + strcpy(laddr, "-"); + strcpy(lport, "-"); } + VSL(SLT_SessionOpen, sp->vxid, "%s %s %s %s %s", + sp->addr, sp->port, lsockname, laddr, lport); } /*-------------------------------------------------------------------- @@ -210,13 +210,13 @@ SES_pool_accept_task(struct worker *wrk, void *arg) sp->vxid = VXID_Get(&wrk->vxid_pool); lsockname = VCA_SetupSess(wrk, sp); + ses_vsl_socket(sp, lsockname); req = ses_GetReq(sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); req->vxid = VXID_Get(&wrk->vxid_pool); - ses_vsl_socket(req, lsockname); wrk->acct_tmp.sess++; diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index d17f9e2..b030caa 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1047,9 +1047,11 @@ static const struct parspec input_parspec[] = { "on", "bool" }, { "log_local_address", tweak_bool, &mgt_param.log_local_addr, 0, 0, "Log the local address on the TCP connection in the " - "SessionOpen shared memory record.\n", + "SessionOpen VSL record.\n" + "Disabling this saves a getsockname(2) system call " + "per TCP connection.\n", 0, - "off", "bool" }, + "on", "bool" }, { "waiter", tweak_waiter, NULL, 0, 0, "Select the waiter kernel interface.\n", WIZARD | MUST_RESTART, diff --git a/bin/varnishtest/tests/c00003.vtc b/bin/varnishtest/tests/c00003.vtc index 598eae9..e0c5afa 100644 --- a/bin/varnishtest/tests/c00003.vtc +++ b/bin/varnishtest/tests/c00003.vtc @@ -8,6 +8,7 @@ server s1 { # This requires non-local binds to be disabled. If you see this fail # and are on Linux, ensure /proc/net/ipv4/ip_nonlocal_bind is set to 0. varnish v1 -cliok "param.set listen_address ${bad_ip}:0" +varnish v1 -cliok "param.set log_local_address off" varnish v1 -vcl+backend {} -clierr 300 start varnish v1 -cliok "param.set listen_address 127.0.0.1:0,${bad_ip}:9082" diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 5a6c862..c79f378 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -37,7 +37,7 @@ * * Arguments: * Tag-Name - * Short Description (1 line, max ?? chars) + * Short Description (1 line, max ? chars) * Long Description (in RST "definition list" format) */ @@ -69,9 +69,9 @@ SLTM(SessionOpen, "Client connection opened", "socket-endpoints of the connection.\n\n" "caddr\n Client IPv4/6 address\n\n" "cport\n Client TCP port\n\n" - "lsock\n Listen socket name\n\n" - "laddr\n Local IPv4/6 address ('-' if $log_local_addr not set)\n\n" - "lport\n Local TCP port ('-' if $log_local_addr not set)\n\n" + "lsock\n Listen socket\n\n" + "laddr\n Local IPv4/6 address ('-' if !$log_local_addr)\n\n" + "lport\n Local TCP port ('-' if !$log_local_addr)\n\n" ) SLTM(SessionClose, "Client connection closed", @@ -81,11 +81,11 @@ SLTM(SessionClose, "Client connection closed", "'Connection: close' - The client specifed that keepalive should " "be disabled by sending a 'Connection: close' header. " "'no request' - No initial request was received within sess_timeout. " - "'EOF' - ??? " - "'remote closed' - ??? " + "'EOF' - ? " + "'remote closed' - ? " "'error' - Processing reached vcl_error even if the status code " "indicates success. " - "' blast' - ???" + "' blast' - ?" ) SLTM(BackendOpen, "Backend connection opened", "") SLTM(BackendXID, "The unique ID of the backend transaction", "") From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 0008c89 Sanitize and document the Sess{Open|Close} VSL messages. Message-ID: commit 0008c89ed6dd5c6010a43cfb4b6762d0d9bbebf4 Author: Poul-Henning Kamp Date: Tue Jul 3 10:06:24 2012 +0000 Sanitize and document the Sess{Open|Close} VSL messages. Some breakage on the far side of varnishapi expected, mode to come. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 2162413..1a42bd0 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -75,6 +75,30 @@ body_status(enum body_status e) } } +/*--------------------------------------------------------------------*/ + +enum sess_close { + SC_NULL = 0, +#define SESS_CLOSE(nm, desc) SC_##nm, +#include "tbl/sess_close.h" +#undef SESS_CLOSE +}; + +static inline const char * +sess_close_str(enum sess_close sc, int want_desc) +{ + switch (sc) { + case SC_NULL: return(want_desc ? "(null)": "NULL"); +#define SESS_CLOSE(nm, desc) case SC_##nm: return(want_desc ? desc : #nm); +#include "tbl/sess_close.h" +#undef SESS_CLOSE + + default: return(want_desc ? "(invalid)" : "INVALID"); + } +} + +/*--------------------------------------------------------------------*/ + /* * NB: HDR_STATUS is only used in cache_http.c, everybody else uses the * http->status integer field. @@ -585,7 +609,7 @@ struct req { unsigned char digest[DIGEST_LEN]; - const char *doclose; + enum sess_close doclose; struct exp exp; unsigned cur_method; unsigned handling; @@ -655,6 +679,7 @@ struct sess { enum sess_step sess_step; int fd; + enum sess_close reason; unsigned vsl_id; uint32_t vxid; @@ -840,7 +865,7 @@ const char *http_GetReq(const struct http *hp); int http_HdrIs(const struct http *hp, const char *hdr, const char *val); uint16_t http_DissectRequest(struct req *); uint16_t http_DissectResponse(struct http *sp, const struct http_conn *htc); -const char *http_DoConnection(const struct http *hp); +enum sess_close http_DoConnection(const struct http *); void http_CopyHome(const struct http *hp); void http_Unset(struct http *hp, const char *hdr); void http_CollectHdr(struct http *hp, const char *hdr); @@ -919,8 +944,8 @@ unsigned WRW_Write(const struct worker *w, const void *ptr, int len); unsigned WRW_WriteH(const struct worker *w, const txt *hh, const char *suf); /* cache_session.c [SES] */ -void SES_Close(struct sess *sp, const char *reason); -void SES_Delete(struct sess *sp, const char *reason, double now); +void SES_Close(struct sess *sp, enum sess_close reason); +void SES_Delete(struct sess *sp, enum sess_close reason, double now); void SES_Charge(struct worker *, struct req *); struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no); void SES_DeletePool(struct sesspool *sp); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index c50d503..e71ad87 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -109,7 +109,7 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) int i, j, tmo; struct pollfd pfd[1]; double now, when; - const char *why = NULL; + enum sess_close why = SC_NULL; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -143,16 +143,16 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) req->t_req = now; return (0); } else if (i == -1) { - why = "EOF"; + why = SC_REM_CLOSE; break; } else if (i == -2) { - why = "overflow"; + why = SC_RX_OVERFLOW; break; } else if (i == -3) { /* Nothing but whitespace */ when = sp->t_idle + cache_param->timeout_idle; if (when < now) { - why = "timeout"; + why = SC_RX_TIMEOUT; break; } when = sp->t_idle + cache_param->timeout_linger; @@ -170,7 +170,7 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) when = sp->t_rx + cache_param->timeout_req; tmo = (int)(1e3 * (when - now)); if (when < now || tmo == 0) { - why = "req timeout"; + why = SC_RX_TIMEOUT; break; } } @@ -255,7 +255,7 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) req->hash_always_miss = 0; req->hash_ignore_busy = 0; - if (sp->fd >= 0 && req->doclose != NULL) { + if (sp->fd >= 0 && req->doclose != SC_NULL) { /* * This is an orderly close of the connection; ditch nolinger * before we close, to get queued data transmitted. @@ -268,7 +268,7 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) wrk->stats.sess_closed++; AZ(req->vcl); SES_ReleaseReq(req); - SES_Delete(sp, NULL, NAN); + SES_Delete(sp, SC_NULL, NAN); return (SESS_DONE_RET_GONE); } @@ -318,9 +318,9 @@ CNT_Session(struct worker *wrk, struct req *req) */ if (sp->sess_step == S_STP_NEWREQ && VTCP_blocking(sp->fd)) { if (errno == ECONNRESET) - SES_Close(sp, "remote closed"); + SES_Close(sp, SC_REM_CLOSE); else - SES_Close(sp, "error"); + SES_Close(sp, SC_TX_ERROR); sdr = cnt_sess_done(sp, wrk, req); assert(sdr == SESS_DONE_RET_GONE); return; @@ -437,7 +437,7 @@ cnt_prepresp(struct worker *wrk, struct req *req) req->res_mode |= RES_CHUNKED; } else { req->res_mode |= RES_EOF; - req->doclose = "EOF mode"; + req->doclose = SC_TX_EOF; } } @@ -573,7 +573,7 @@ cnt_error(struct worker *wrk, struct req *req) (uint16_t)cache_param->http_max_hdr); bo->stats = NULL; if (req->obj == NULL) { - req->doclose = "Out of objects"; + req->doclose = SC_OVERLOAD; req->director = NULL; http_Teardown(bo->beresp); http_Teardown(bo->bereq); @@ -611,7 +611,7 @@ cnt_error(struct worker *wrk, struct req *req) /* We always close when we take this path */ - req->doclose = "error"; + req->doclose = SC_TX_ERROR; req->wantbody = 1; assert(req->handling == VCL_RET_DELIVER); @@ -1504,7 +1504,7 @@ cnt_start(struct worker *wrk, struct req *req) /* If we could not even parse the request, just close */ if (req->err_code == 400) { - SES_Close(req->sp, "junk"); + SES_Close(req->sp, SC_RX_JUNK); return (1); } @@ -1520,7 +1520,7 @@ cnt_start(struct worker *wrk, struct req *req) if (strcasecmp(p, "100-continue")) { req->err_code = 417; } else if (strlen(r) != write(req->sp->fd, r, strlen(r))) { - SES_Close(req->sp, "remote closed"); + SES_Close(req->sp, SC_REM_CLOSE); return (1); } } diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 7e4305e..cc722ce 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -325,7 +325,7 @@ ESI_Deliver(struct req *req) st->ptr + off, l2); if (WRW_Error(req->wrk)) { SES_Close(req->sp, - "remote closed"); + SC_REM_CLOSE); p = e; break; } @@ -376,7 +376,7 @@ ESI_Deliver(struct req *req) if (vgz != NULL) VGZ_WrwFlush(req->wrk, vgz); if (WRW_Flush(req->wrk)) { - SES_Close(req->sp, "remote closed"); + SES_Close(req->sp, SC_REM_CLOSE); p = e; break; } diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 411b2d3..eb5998a 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -410,19 +410,19 @@ http_GetHdrField(const struct http *hp, const char *hdr, * XXX: redo with http_GetHdrField() ? */ -const char * +enum sess_close http_DoConnection(const struct http *hp) { char *p, *q; - const char *ret; + enum sess_close ret; unsigned u; if (!http_GetHdr(hp, H_Connection, &p)) { if (hp->protover < 11) - return ("not HTTP/1.1"); - return (NULL); + return (SC_REQ_HTTP10); + return (SC_NULL); } - ret = NULL; + ret = SC_NULL; AN(p); for (; *p; p++) { if (vct_issp(*p)) @@ -434,7 +434,7 @@ http_DoConnection(const struct http *hp) break; u = pdiff(p, q); if (u == 5 && !strncasecmp(p, "close", u)) - ret = "Connection: close"; + ret = SC_REQ_CLOSE; u = http_findhdr(hp, u, p); if (u != 0) hp->hdf[u] |= HDF_FILTER; diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c index 8fa9e3c..238d0a3 100644 --- a/bin/varnishd/cache/cache_pipe.c +++ b/bin/varnishd/cache/cache_pipe.c @@ -92,7 +92,7 @@ PipeRequest(struct req *req) i = WRW_FlushRelease(wrk); if (i) { - SES_Close(req->sp, "pipe"); + SES_Close(req->sp, SC_TX_PIPE); VDI_CloseFd(&vc); return; } @@ -132,7 +132,7 @@ PipeRequest(struct req *req) fds[1].fd = -1; } } - SES_Close(req->sp, "pipe"); + SES_Close(req->sp, SC_TX_PIPE); VDI_CloseFd(&vc); bo->vbc = NULL; } diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index cdad653..e82ab67 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -285,5 +285,5 @@ RES_WriteObj(struct req *req) WRW_EndChunk(req->wrk); if (WRW_FlushRelease(req->wrk) && req->sp->fd >= 0) - SES_Close(req->sp, "remote closed"); + SES_Close(req->sp, SC_REM_CLOSE); } diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 7f88212..59dccf6 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -175,7 +175,7 @@ ses_vsl_socket(struct sess *sp, const char *lsockname) strcpy(laddr, "-"); strcpy(lport, "-"); } - VSL(SLT_SessionOpen, sp->vxid, "%s %s %s %s %s", + VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s", sp->addr, sp->port, lsockname, laddr, lport); } @@ -203,6 +203,7 @@ SES_pool_accept_task(struct worker *wrk, void *arg) VCA_FailSess(wrk); return; } + wrk->acct_tmp.sess++; sp->t_open = VTIM_real(); sp->t_rx = sp->t_open; @@ -217,9 +218,6 @@ SES_pool_accept_task(struct worker *wrk, void *arg) req->vxid = VXID_Get(&wrk->vxid_pool); - - wrk->acct_tmp.sess++; - sp->sess_step = S_STP_NEWREQ; ses_pool_task(wrk, req); } @@ -249,7 +247,7 @@ SES_ScheduleReq(struct req *req) sp->t_idle = VTIM_real(); AN (req->vcl); VCL_Rel(&req->vcl); - SES_Delete(sp, "dropped", sp->t_idle); + SES_Delete(sp, SC_OVERLOAD, sp->t_idle); return (1); } return (0); @@ -282,7 +280,7 @@ SES_Handle(struct sess *sp, double now) if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) { VSC_C_main->client_drop_late++; sp->t_idle = VTIM_real(); - SES_Delete(sp, "dropped", sp->t_idle); + SES_Delete(sp, SC_OVERLOAD, sp->t_idle); } } @@ -293,12 +291,12 @@ SES_Handle(struct sess *sp, double now) */ void -SES_Close(struct sess *sp, const char *reason) +SES_Close(struct sess *sp, enum sess_close reason) { int i; assert(sp->fd >= 0); - VSL(SLT_SessionClose, sp->vsl_id, "%s", reason); + sp->reason = reason; i = close(sp->fd); assert(i == 0 || errno != EBADF); /* XXX EINVAL seen */ sp->fd = -1; @@ -314,7 +312,7 @@ SES_Close(struct sess *sp, const char *reason) */ void -SES_Delete(struct sess *sp, const char *reason, double now) +SES_Delete(struct sess *sp, enum sess_close reason, double now) { struct acct *b; struct sesspool *pp; @@ -324,7 +322,7 @@ SES_Delete(struct sess *sp, const char *reason, double now) CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); AN(pp->pool); - if (reason != NULL) + if (reason != SC_NULL) SES_Close(sp, reason); if (isnan(now)) now = VTIM_real(); @@ -338,10 +336,11 @@ SES_Delete(struct sess *sp, const char *reason, double now) b = &sp->acct_ses; - VSL(SLT_StatSess, sp->vsl_id, "%s %s %.0f %ju %ju %ju %ju %ju %ju %ju", - sp->addr, sp->port, + VSL(SLT_SessClose, sp->vxid, + "%s %.3f %ju %ju %ju %ju %ju %ju", + sess_close_str(sp->reason, 0), now - sp->t_open, - b->sess, b->req, b->pipe, b->pass, + b->req, b->pipe, b->pass, b->fetch, b->hdrbytes, b->bodybytes); MPL_Free(pp->mpl_sess, sp); diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 5ddd6ce..2a15a34 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -109,6 +109,10 @@ -esym(458, name_key) ////////////// +// 436 = Apparent preprocessor directive in invocation of macro '___' +-emacro(436, SLTM) + +////////////// +libh mgt_event.h diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index fd7c97f..7b42437 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -86,7 +86,7 @@ mgt_sltm(const char *tag, const char *sdesc, const char *ldesc) printf("%s\n", sdesc); else printf("%s\n", "(description not yet written)"); - + } static void diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c index e89d550..53d7f0f 100644 --- a/bin/varnishd/waiter/cache_waiter.c +++ b/bin/varnishd/waiter/cache_waiter.c @@ -74,6 +74,6 @@ WAIT_Enter(struct sess *sp) * acceptor thread, to reduce syscall density of the latter. */ if (VTCP_nonblocking(sp->fd)) - SES_Close(sp, "remote closed"); + SES_Close(sp, SC_REM_CLOSE); waiter->pass(waiter_priv, sp); } diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c index 423bf26..a7403d6 100644 --- a/bin/varnishd/waiter/cache_waiter_kqueue.c +++ b/bin/varnishd/waiter/cache_waiter_kqueue.c @@ -133,7 +133,7 @@ vwk_sess_ev(struct vwk *vwk, const struct kevent *kp, double now) return; } else if (kp->flags & EV_EOF) { VTAILQ_REMOVE(&vwk->sesshead, sp, list); - SES_Delete(sp, "EOF", now); + SES_Delete(sp, SC_REM_CLOSE, now); return; } else { VSL(SLT_Debug, sp->vsl_id, @@ -204,7 +204,7 @@ vwk_thread(void *priv) break; VTAILQ_REMOVE(&vwk->sesshead, sp, list); // XXX: not yet (void)VTCP_linger(sp->fd, 0); - SES_Delete(sp, "timeout", now); + SES_Delete(sp, SC_RX_TIMEOUT, now); } } } diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c index a31bec1..4339617 100644 --- a/bin/varnishd/waiter/cache_waiter_poll.c +++ b/bin/varnishd/waiter/cache_waiter_poll.c @@ -162,7 +162,7 @@ vwp_main(void *priv) VTAILQ_REMOVE(&vwp->sesshead, sp, list); vwp_unpoll(vwp, fd); // XXX: not yet (void)VTCP_linger(sp->fd, 0); - SES_Delete(sp, "timeout", now); + SES_Delete(sp, SC_RX_TIMEOUT, now); } } if (v2 && vwp->pollfd[vwp->pipes[0]].revents) { diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c index a606861..bf7d32b 100644 --- a/bin/varnishlog/varnishlog.c +++ b/bin/varnishlog/varnishlog.c @@ -115,9 +115,9 @@ h_order(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len, ob[fd] = VSB_new_auto(); assert(ob[fd] != NULL); } - if ((tag == SLT_BackendOpen || tag == SLT_SessionOpen || + if ((tag == SLT_BackendOpen || tag == SLT_SessOpen || (tag == SLT_ReqStart && - last[fd] != SLT_SessionOpen && + last[fd] != SLT_SessOpen && last[fd] != SLT_VCL_acl) || (tag == SLT_BackendXID && last[fd] != SLT_BackendOpen)) && @@ -127,7 +127,7 @@ h_order(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len, * the end of the previous one. Spit it out anyway before * starting on the new one. */ - if (last[fd] != SLT_SessionClose) + if (last[fd] != SLT_SessClose) VSB_printf(ob[fd], "%5d %-12s %c %s\n", fd, "Interrupted", type, VSL_tags[tag]); h_order_finish(fd, vd); @@ -165,7 +165,6 @@ h_order(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len, case SLT_ReqEnd: case SLT_BackendClose: case SLT_BackendReuse: - case SLT_StatSess: h_order_finish(fd, vd); break; default: @@ -180,8 +179,8 @@ do_order(struct VSM_data *vd) int i; if (!b_flag) { - VSL_Select(vd, SLT_SessionOpen); - VSL_Select(vd, SLT_SessionClose); + VSL_Select(vd, SLT_SessOpen); + VSL_Select(vd, SLT_SessClose); VSL_Select(vd, SLT_ReqEnd); } if (!c_flag) { diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 9bdf87f..9c9267e 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -535,11 +535,11 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec, lp->df_b = trimline(ptr, end); break; - case SLT_SessionClose: + case SLT_SessClose: if (!lp->active) break; - if (strncmp(ptr, "pipe", len) == 0 || - strncmp(ptr, "error", len) == 0) { + if (strncmp(ptr, "TX_PIPE", len) == 0 || + strncmp(ptr, "TX_ERROR", len) == 0) { clean_logline(lp); break; } diff --git a/include/tbl/sess_close.h b/include/tbl/sess_close.h new file mode 100644 index 0000000..c9d28ba --- /dev/null +++ b/include/tbl/sess_close.h @@ -0,0 +1,43 @@ +/*- + * Copyright (c) 2012 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +/*lint -save -e525 -e539 */ + +SESS_CLOSE(REM_CLOSE, "Client Closed") +SESS_CLOSE(REQ_CLOSE, "Client requested close") +SESS_CLOSE(REQ_HTTP10, "proto < HTTP.1.1") +SESS_CLOSE(RX_JUNK, "Received junk data") +SESS_CLOSE(RX_OVERFLOW, "Received buffer overflow") +SESS_CLOSE(RX_TIMEOUT, "Receive timeout") +SESS_CLOSE(TX_PIPE, "Piped transaction") +SESS_CLOSE(TX_ERROR, "Error transaction") +SESS_CLOSE(TX_EOF, "EOF transmission") +SESS_CLOSE(OVERLOAD, "Out of some resource") + +/*lint -restore */ diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index c79f378..27fed4d 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -52,7 +52,6 @@ SLTM(Error, "Error messages", SLTM(CLI, "CLI communication", "CLI communication between master and child process." ) -SLTM(StatSess, "Session statistics", "") SLTM(ReqEnd, "Client request end", "Marks the end of client request.\n\n" @@ -64,7 +63,9 @@ SLTM(ReqEnd, "Client request end", "dTtx\n Time to transmit response\n\n" ) -SLTM(SessionOpen, "Client connection opened", +/*---------------------------------------------------------------------*/ + +SLTM(SessOpen, "Client connection opened", "The first record for a client connection, with the\n" "socket-endpoints of the connection.\n\n" "caddr\n Client IPv4/6 address\n\n" @@ -74,19 +75,24 @@ SLTM(SessionOpen, "Client connection opened", "lport\n Local TCP port ('-' if !$log_local_addr)\n\n" ) -SLTM(SessionClose, "Client connection closed", - "SessionClose tells you why HTTP client-connections are closed. " - "These can be: " - "'timeout' - No keep-alive was received within sess_timeout. " - "'Connection: close' - The client specifed that keepalive should " - "be disabled by sending a 'Connection: close' header. " - "'no request' - No initial request was received within sess_timeout. " - "'EOF' - ? " - "'remote closed' - ? " - "'error' - Processing reached vcl_error even if the status code " - "indicates success. " - "' blast' - ?" +#define SESS_CLOSE(nm, desc) " " #nm "\n\t" desc "\n\n" + +SLTM(SessClose, "Client connection closed", + "SessionClose is the last record for any client connection.\n\n" + "reason\n Why the connection closed.\n\n" +#include + "duration\n How long the session were open.\n\n" + "Nreq\n How many requests on session.\n\n" + "Npipe\n If 'pipe' were used on session.\n\n" + "Npass\n Requests handled with pass.\n\n" + "Nfetch\n Backend fetches by session.\n\n" + "Bhdr\n Header bytes sent on session.\n\n" + "Bbody\n Body bytes sent on session.\n\n" ) +#undef SESS_CLOSE + +/*---------------------------------------------------------------------*/ + SLTM(BackendOpen, "Backend connection opened", "") SLTM(BackendXID, "The unique ID of the backend transaction", "") SLTM(BackendReuse, "Backend connection reused", "") From martin at varnish-software.com Thu Dec 18 09:27:50 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 673c50d Fix parsing of -T and -S arguments from shared memory in varnishadm when using the -n option. Message-ID: commit 673c50deaa447eb80c7830f2adef0749347f44bc Author: Martin Blix Grydeland Date: Tue Jul 3 13:30:40 2012 +0200 Fix parsing of -T and -S arguments from shared memory in varnishadm when using the -n option. Code had reversed the no -T argument error check, and also required the optional -S argument to be present. Fixes: #1160 diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index f4c262c..3dc7c79 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -298,20 +298,19 @@ n_arg_sock(const char *n_arg) fprintf(stderr, "%s\n", VSM_Error(vsd)); return (-1); } - if (T_arg == NULL) { - if (VSM_Get(vsd, &vt, "Arg", "-T", "")) { - fprintf(stderr, "No -T arg in shared memory\n"); - return (-1); - } - T_start = T_arg = strdup(vt.b); + + if (!VSM_Get(vsd, &vt, "Arg", "-T", "")) { + fprintf(stderr, "No -T arg in shared memory\n"); + return (-1); } - if (S_arg == NULL) { - if (VSM_Get(vsd, &vt, "Arg", "-S", "")) { - fprintf(stderr, "No -S arg in shared memory\n"); - return (-1); - } + AN(vt.b); + T_start = T_arg = strdup(vt.b); + + if (VSM_Get(vsd, &vt, "Arg", "-S", "")) { + AN(vt.b); S_arg = strdup(vt.b); } + sock = -1; while (*T_arg) { p = strchr(T_arg, '\n'); From martin at varnish-software.com Thu Dec 18 09:27:50 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] ad65eb5 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit ad65eb54a7b57d116e7517385b0c4ab2f2fb6217 Merge: 673c50d 0008c89 Author: Martin Blix Grydeland Date: Tue Jul 3 14:11:51 2012 +0200 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From phk at FreeBSD.org Thu Dec 18 09:27:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 74e2d3f Seems I have either hit a GCC bug or a lack of imagination in the C-standardisation. (Even money if you ask me...) Message-ID: commit 74e2d3f3dc50967a64fe818f7fcb78af06f76e1a Author: Poul-Henning Kamp Date: Tue Jul 3 12:33:09 2012 +0000 Seems I have either hit a GCC bug or a lack of imagination in the C-standardisation. (Even money if you ask me...) GCC users will just have to live with marginally works docs. diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 27fed4d..37c2e6b 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -80,7 +80,13 @@ SLTM(SessOpen, "Client connection opened", SLTM(SessClose, "Client connection closed", "SessionClose is the last record for any client connection.\n\n" "reason\n Why the connection closed.\n\n" +#ifdef __clang__ +/* + * GCC barfs at this construct, but we have no way to ask compiler + * if it is _not_ a GCC compiler since CLANG also defines __GNUC__ + */ #include +#endif "duration\n How long the session were open.\n\n" "Nreq\n How many requests on session.\n\n" "Npipe\n If 'pipe' were used on session.\n\n" From martin at varnish-software.com Thu Dec 18 09:27:50 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:50 +0100 Subject: [experimental-ims] 47dd5d2 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit 47dd5d223032610b0dc01d54277eb18d8bb75c5d Merge: ad65eb5 74e2d3f Author: Martin Blix Grydeland Date: Tue Jul 3 14:36:02 2012 +0200 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From phk at varnish-cache.org Thu Dec 18 09:27:51 2014 From: phk at varnish-cache.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] 88c7737 Also update waiter_epoll to new SES_* api. Message-ID: commit 88c77374bab8f4c63235005200a452ae29b669b0 Author: Poul-Henning Kamp Date: Tue Jul 3 14:46:06 2012 +0200 Also update waiter_epoll to new SES_* api. diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c index 6ad5be5..a15bbe5 100644 --- a/bin/varnishd/waiter/cache_waiter_epoll.c +++ b/bin/varnishd/waiter/cache_waiter_epoll.c @@ -133,13 +133,13 @@ vwe_eev(struct vwe *vwe, const struct epoll_event *ep, double now) SES_Handle(sp, now); } else if (ep->events & EPOLLERR) { VTAILQ_REMOVE(&vwe->sesshead, sp, list); - SES_Delete(sp, "ERR", now); + SES_Delete(sp, SC_REM_CLOSE, now); } else if (ep->events & EPOLLHUP) { VTAILQ_REMOVE(&vwe->sesshead, sp, list); - SES_Delete(sp, "HUP", now); + SES_Delete(sp, SC_REM_CLOSE, now); } else if (ep->events & EPOLLRDHUP) { VTAILQ_REMOVE(&vwe->sesshead, sp, list); - SES_Delete(sp, "RHUP", now); + SES_Delete(sp, SC_REM_CLOSE, now); } } } @@ -192,7 +192,7 @@ vwe_thread(void *priv) break; VTAILQ_REMOVE(&vwe->sesshead, sp, list); // XXX: not yet VTCP_linger(sp->fd, 0); - SES_Delete(sp, "timeout", now); + SES_Delete(sp, SC_RX_TIMEOUT, now); } } return (NULL); From tfheen at varnish-software.com Thu Dec 18 09:27:51 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] 0c82d95 Distribute sess_close.h too Message-ID: commit 0c82d957d368863d18a32175da247ff753ed33d2 Author: Tollef Fog Heen Date: Wed Jul 4 15:14:51 2012 +0200 Distribute sess_close.h too diff --git a/include/Makefile.am b/include/Makefile.am index 3faa540..b70a825 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -9,6 +9,7 @@ nobase_pkginclude_HEADERS = \ tbl/http_headers.h \ tbl/http_response.h \ tbl/locks.h \ + tbl/sess_close.h \ tbl/steps.h \ tbl/symbol_kind.h \ tbl/vcc_types.h \ From martin at varnish-software.com Thu Dec 18 09:27:51 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] aa9e09c Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache Message-ID: commit aa9e09c8df6e32742fec5eaa700ae4626dc1cb35 Merge: 47dd5d2 88c7737 Author: Martin Blix Grydeland Date: Tue Jul 3 15:00:42 2012 +0200 Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache From daghf at varnish-software.com Thu Dec 18 09:27:51 2014 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] dec9455 Fix for an issue where varnishlog reports an incorrect "Length" entry on gunzipped delivery Message-ID: commit dec9455a9f0bb1b1e3c5b65d3d626d531db25fbb Author: Dag Haavi Finstad Date: Mon Jul 2 09:50:30 2012 +0200 Fix for an issue where varnishlog reports an incorrect "Length" entry on gunzipped delivery diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 1a42bd0..6c46831 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -823,7 +823,7 @@ void VGZ_UpdateObj(const struct vgz*, struct object *); int VGZ_WrwInit(struct vgz *vg); int VGZ_WrwGunzip(struct worker *w, struct vgz *, const void *ibuf, ssize_t ibufl); -void VGZ_WrwFlush(const struct worker *wrk, struct vgz *vg); +void VGZ_WrwFlush(struct worker *wrk, struct vgz *vg); /* Return values */ #define VGZ_ERROR -1 diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 9078834..1ebabb0 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -355,13 +355,15 @@ VGZ_WrwGunzip(struct worker *wrk, struct vgz *vg, const void *ibuf, /*--------------------------------------------------------------------*/ void -VGZ_WrwFlush(const struct worker *wrk, struct vgz *vg) +VGZ_WrwFlush(struct worker *wrk, struct vgz *vg) { CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC); if (vg->m_len == 0) return; + + wrk->acct_tmp.bodybytes += vg->m_len; (void)WRW_Write(wrk, vg->m_buf, vg->m_len); (void)WRW_Flush(wrk); vg->m_len = 0; From apj at mutt.dk Thu Dec 18 09:27:51 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] e855267 Make sure the user understands that this is about the default configuration Message-ID: commit e855267d8c4029a0056434eaf373b1fe9e9ffb96 Author: Andreas Plesner Jacobsen Date: Fri Jul 6 15:25:48 2012 +0200 Make sure the user understands that this is about the default configuration diff --git a/doc/sphinx/tutorial/cookies.rst b/doc/sphinx/tutorial/cookies.rst index deda74c..c75171f 100644 --- a/doc/sphinx/tutorial/cookies.rst +++ b/doc/sphinx/tutorial/cookies.rst @@ -89,7 +89,7 @@ Cookies coming from the backend ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If your backend server sets a cookie using the Set-Cookie header -Varnish will not cache the page. A hit-for-pass object (see -:ref:`tutorial-vcl_fetch_actions`) is created. So, if the backend -server acts silly and sets unwanted cookies just unset the Set-Cookie -header and all should be fine. +Varnish will not cache the page in the default configuration. A +hit-for-pass object (see :ref:`tutorial-vcl_fetch_actions`) is created. +So, if the backend server acts silly and sets unwanted cookies just unset +the Set-Cookie header and all should be fine. From phk at FreeBSD.org Thu Dec 18 09:27:51 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] c31f908 Add my official "statement of interest" in HTTP/2.0 Message-ID: commit c31f908382f51b0dba5dce10fc6127574afb40a9 Author: Poul-Henning Kamp Date: Fri Jul 13 00:03:33 2012 +0000 Add my official "statement of interest" in HTTP/2.0 diff --git a/doc/sphinx/phk/http20.rst b/doc/sphinx/phk/http20.rst new file mode 100644 index 0000000..e2a6ed0 --- /dev/null +++ b/doc/sphinx/phk/http20.rst @@ -0,0 +1,316 @@ +.. _phk_http20_lack_of_interest: + +====================================== +Why HTTP/2.0 does not seem interesting +====================================== + +This is the email I sent to the IETF HTTP Working Group:: + + + From: Poul-Henning Kamp + Subject: HTTP/2 Expression of luke-warm interest: Varnish + To: HTTP Working Group + Message-Id: <41677.1342136900 at critter.freebsd.dk> + Date: Thu, 12 Jul 2012 23:48:20 GMT + + +This is Varnish' response to the call for expression of interest +in HTTP/2[1]. + +Varnish +------- + +Presently Varnish[2] only implements a subset of HTTP/1.1 consistent +with its hybrid/dual "http-server" / "http-proxy" role. + +I cannot at this point say much about what Varnish will or will +not implement protocol wise in the future. + +Our general policy is to only add protocols if we can do a better +job than the alternative, which is why we have not implemented HTTPS +for instance. + +Should the outcome of the HTTP/2.0 effort result in a protocol which +gains traction, Varnish will probably implement it, but we are +unlikely to become an early implementation, given the current +proposals at the table. + + +Why I'm not impressed +--------------------- + +I have read all, and participated in one, of the three proposals +presently on the table. + +Overall, I find all three proposals are focused on solving yesteryears +problems, rather than on creating a protocol that stands a chance +to last us the next 20 years. + +Each proposal comes out of a particular "camp" and therefore +all seem to suffer a certain amount from tunnel-vision. + +It is my considered opinion that none of the proposals have what +it will take to replace HTTP/1.1 in practice. + + +What if they made a new protocol, and nobody used it ? +------------------------------------------------------ + +We have learned, painfully, that an IPv6 which is only marginally +better than IPv4 and which offers no tangible benefit for the people +who have the cost/trouble of the upgrade, does not penetrate the +network on its own, and barely even on goverments mandate. + +We have also learned that a protocol which delivers the goods can +replace all competition in virtually no time. + +See for instance how SSH replaced TELNET, REXEC, RSH, SUPDUP, and +to a large extent KERBEROS, in a matter of a few years. + +Or I might add, how HTTP replaced GOPHER[3]. + +HTTP/1.1 is arguably in the top-five most used protocols, after +IP, TCP, UDP and, sadly, ICMP, and therefore coming up with a +replacement should be approached humbly. + + +Beating HTTP/1.1 +---------------- + +Fortunately, there are many ways to improve over HTTP/1.1, which +lacks support for several widely used features, and sports many +trouble-causing weeds, both of which are ripe for HTTP/2.0 to pounce +on. + +Most notably HTTP/1.1 lacks a working session/endpoint-identity +facility, a shortcoming which people have pasted over with the +ill-conceived Cookie hack. + +Cookies are, as the EU commision correctly noted, fundamentally +flawed, because they store potentially sensitive information on +whatever computer the user happens to use, and as a result of various +abuses and incompetences, EU felt compelled to legislate a "notice +and announce" policy for HTTP-cookies. + +But it doesn't stop there: The information stored in cookies have +potentialiiy very high value for the HTTP server, and because the +server has no control over the integrity of the storage, we are now +seing cookies being crypto-signed, to prevent forgeries. + +The term "bass ackwards" comes to mind. + +Cookies are also one of the main wasters of bandwidth, disabling +caching by default, sending lots of cookies were they are are not +needed, which made many sites register separate domains for image +content, to "save" bandwidth by avoiding cookies. + +The term "not really helping" also comes to mind. + +In my view, HTTP/2.0 should kill Cookies as a concept, and replace +it with a session/identity facility, which makes it easier to +do things right with HTTP/2.0 than with HTTP/1.1. + +Being able to be "automatically in compliance" by using HTTP/2.0 +no matter how big dick-heads your advertisers are or how incompetent +your web-developers are, would be a big selling point for HTTP/2.0 +over HTTP/1.1. + +However, as I read them, none of the three proposals try to address, +much less remedy, this situation, nor for that matter any of the +many other issues or troubles with HTTP/1.x. + +What's even worse, they are all additive proposals, which add a +new layer of complexity without removing any of the old complexity +from the protocol. + +My conclusion is that HTTP/2.0 is really just a grandiose name for +HTTP/1.2: An attempt to smoothe out some sharp corners, to save a +bit of bandwidth, but not get anywhere near all the architectural +problems of HTTP/1.1 and to preserve faithfully its heritage of +badly thought out sedimentary hacks. + +And therefore, I don't see much chance that the current crop of +HTTP/2.0 proposals will fare significantly better than IPv6 with +respect to adoption. + + +HTTP Routers +------------ + +One particular hot-spot in the HTTP world these days is the +"load-balancer" or as I prefer to call it, the "HTTP router". + +These boxes sit at the DNS resolved IP numbers and distributes +client requests to a farm of HTTP servers, based on simple criteria +such as "Host:", URI patterns and/or server availability, sometimes +with an added twist of geo-location[4]. + +HTTP routers see very high traffic densities, the highest traffic +densities, because they are the focal point of DoS mitigation, flash +mobs and special event traffic spikes. + +In the time frame where HTTP/2.0 will become standardized, HTTP +routers will routinely deal with 40Gbit/s traffic and people will +start to arcitect for 1Tbit/s traffic. + +HTTP routers are usually only interested in a small part of the +HTTP request and barely in the response at all, usually only the +status code. + +The demands for bandwidth efficiency has made makers of these devices +take many unwarranted shortcuts, for instance assuming that requests +always start on a packet boundary, "nulling out" HTTP headers by +changing the first character and so on. + +Whatever HTTP/2.0 becomes, I strongly urge IETF and the WG to +formally recognize the role of HTTP routers, and to actively design +the protocol to make life easier for HTTP routers, so that they can +fulfill their job, while being standards compliant. + +The need for HTTP routers does not disappear just because HTTPS is +employed, and serious thought should be turned to the question of +mixing HTTP and HTTPS traffic on the same TCP connection, while +allowing a HTTP router on the server side to correctly distribute +requests to different servers. + +One simple way to gain a lot of benefit for little cost in this +area, would be to assign "flow-labels" which each are restricted +to one particular Host: header, allowing HTTP routers to only examine +the first request on each flow. + + +SPDY +---- + +SPDY has come a long way, and has served as a very worthwhile proof +of concept prototype, to document that there are gains to be had. + +But as Frederick P. Brooks admonishes us: Always throw the prototype +away and start over, because you will throw it away eventually, and +doing so early saves time and effort. + +Overall, I find the design approach taken in SPDY deeply flawed. + +For instance identifying the standardized HTTP headers, by a 4-byte +length and textual name, and then applying a deflate compressor to +save bandwidth is totally at odds with the job of HTTP routers which +need to quickly extract the Host: header in order to route the +traffic, preferably without committing extensive resources to each +request. + +It is also not at all clear if the built-in dictionary is well +researched or just happens to work well for some subset of present +day websites, and at the very least some kind of versioning of this +dictionary should be incorporated. + +It is still unclear for me if or how SPDY can be used on TCP port +80 or if it will need a WKS allocation of its own, which would open +a ton of issues with firewalling, filtering and proxying during +deployment. + +(This is one of the things which makes it hard to avoid the feeling +that SPDY really wants to do away with all the "middle-men") + +With my security-analyst hat on, I see a lot of DoS potential in +the SPDY protocol, many ways in which the client can make the server +expend resources, and foresee a lot of complexity in implementing +the server side to mitigate and deflect malicious traffic. + +Server Push breaks the HTTP transaction model, and opens a pile of +cans of security and privacy issues, which whould not be sneaked +in during the design of a transport-encoding for HTTP/1+ traffic, +but rather be standardized as an independent and well analysed +extension to HTTP in general. + + +HTTP Speed+Mobility +------------------- + +Is really just SPDY with WebSockets underneath. + +I'm really not sure I see any benefit to that, execept that the +encoding chosen is marginally more efficient to implement in +hardware than SPDY. + +I have not understood why it has "mobility" in the name, a word +which only makes an appearance in the ID as part of the name. + +If the use of the word "mobility" only refers only to bandwidth +usage, I would call its use borderline-deceptive. + +If it covers session stability across IP# changes for mobile +devices, I have missed it in my reading. + + +draft-tarreau-httpbis-network-friendly-00 +----------------------------------------- + +I have participated a little bit in this draft initially, but it +uses a number of concepts which I think are very problematic for +high performance (as in 1Tbit/s) implementations, for instance +variant-size length fields etc. + +I do think the proposal is much better than the other two, taking +a much more fundamental view of the task, and if for no other reason, +because it takes an approach to bandwidth-saving based on enumeration +and repeat markers, rather than throwing everything after deflate +and hope for a miracle. + +I think this protocol is the best basis to start from, but like +the other two, it has a long way to go, before it can truly +earn the name HTTP/2.0. + + +Conclusion +---------- + +Overall, I don't see any of the three proposals offer anything that +will make the majority of web-sites go "Ohh we've been waiting for +that!" + +Bigger sites will be entised by small bandwidth savings, but the +majority of the HTTP users will see scant or no net positive benefit +if one or more of these three proposals were to become HTTP/2.0 + +Considering how sketchy the HTTP/1.1 interop is described it is hard +to estimate how much trouble (as in: "Why doesn't this website work ?") +their deployment will cause, nor is it entirely clear to what extent +the experience with SPDY is representative of a wider deployment or +only of 'flying under the radar' with respect to people with an +interest in intercepting HTTP traffic. + +Given the role of HTTP/1.1 in the net, I fear that the current rush +to push out a HTTP/2.0 by purely additive means is badly misguided, +and approaching a critical mass which will delay or prevent adoption +on its own. + +At the end of the day, a HTTP request or a HTTP response is just +some metadata and an optional chunk of bytes as body, and if it +already takes 700 pages to standardize that, and HTTP/2.0 will add +another 100 pages to it, we're clearly doing something wrong. + +I think it would be far better to start from scratch, look at what +HTTP/2.0 should actually do, and then design a simple, efficient +and future proof protocol to do just that, and leave behind all +the aggregations of badly thought out hacks of HTTP/1.1. + +But to the extent that the WG produces a HTTP/2.0 protocol which +people will start to use, the Varnish project will be interested. + + +Poul-Henning Kamp + +Author of Varnish + + +[1] http://trac.tools.ietf.org/wg/httpbis/trac/wiki/Http2CfI + +[2] http://varnish-cache.org/ + +[3] Yes, I'm that old. + +[4] Which is really a transport level job, but it was left out of IPv6 + along with other useful features, to not delay adoption[5]. + +[5] No, I'm not kidding. + diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index 58935c7..d4a2513 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -7,7 +7,9 @@ Poul-Hennings random outbursts You may or may not want to know what Poul-Henning think. .. toctree:: + :titlesonly: + http20.rst varnish_does_not_hash.rst thetoolsweworkwith.rst three-zero.rst From phk at FreeBSD.org Thu Dec 18 09:27:51 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] 9f1fd24 Hmm, that hid too much, try this instead... Message-ID: commit 9f1fd247f1d12de5409e7291de326cc554d825e5 Author: Poul-Henning Kamp Date: Fri Jul 13 00:07:10 2012 +0000 Hmm, that hid too much, try this instead... diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index d4a2513..f4f880d 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -7,7 +7,7 @@ Poul-Hennings random outbursts You may or may not want to know what Poul-Henning think. .. toctree:: - :titlesonly: + :maxdepth: 1 http20.rst varnish_does_not_hash.rst From martin at varnish-software.com Thu Dec 18 09:27:51 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] d105f98 Document the version requirements of the varnishreplay input file. Message-ID: commit d105f989a0586c0ffefcf46458d4891adf0bce49 Author: Martin Blix Grydeland Date: Fri Jul 13 09:56:24 2012 +0200 Document the version requirements of the varnishreplay input file. diff --git a/doc/sphinx/reference/varnishreplay.rst b/doc/sphinx/reference/varnishreplay.rst index 4be103f..da3df36 100644 --- a/doc/sphinx/reference/varnishreplay.rst +++ b/doc/sphinx/reference/varnishreplay.rst @@ -32,8 +32,9 @@ The following options are available: -D Turn on debugging mode. --r file Parse logs from this file. This option is mandatory. - +-r file Parse logs from this file. The input file has to be from + a varnishlog of the same version as the varnishreplay + binary. This option is mandatory. SEE ALSO ======== From lasse at varnish-software.com Thu Dec 18 09:27:51 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] 83ab485 Front is slightly easier to understand Message-ID: commit 83ab485743c413ce50693769aaf4a7bc268b1dc4 Author: Lasse Karstensen Date: Mon Jul 16 10:17:43 2012 +0200 Front is slightly easier to understand diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 6c46831..7328779 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -171,7 +171,7 @@ struct ws { unsigned overflow; /* workspace overflowed */ const char *id; /* identity */ char *s; /* (S)tart of buffer */ - char *f; /* (F)ree pointer */ + char *f; /* (F)ree/front pointer */ char *r; /* (R)eserved length */ char *e; /* (E)nd of buffer */ }; From martin at varnish-software.com Thu Dec 18 09:27:51 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] c288891 Fix the parallell make check Message-ID: commit c2888919964c0498c511db5e7158244e15771eab Author: Martin Blix Grydeland Date: Mon Jul 16 12:37:17 2012 +0200 Fix the parallell make check diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am index ec68889..9287b3c 100644 --- a/bin/varnishtest/Makefile.am +++ b/bin/varnishtest/Makefile.am @@ -5,7 +5,8 @@ TEST_EXTENSIONS = .vtc TESTS = @VTC_TESTS@ # Make sure we run check-local first -check: check-local check-am +check: check-am check-local +check-am: check-local # See if list of checks have changed, recheck check-local: if [ "$$(cd $(srcdir) && echo tests/*.vtc)" != "@VTC_TESTS@" ]; then \ From daghf at varnish-software.com Thu Dec 18 09:27:51 2014 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] b33efe4 Allow 'storage' in backend identifiers. Message-ID: commit b33efe4cbe2ee8b69222d7f53bd3833938542390 Author: Dag Haavi Finstad Date: Tue Jul 17 14:58:05 2012 +0200 Allow 'storage' in backend identifiers. Fixes: #1164 diff --git a/bin/varnishtest/tests/r01164.vtc b/bin/varnishtest/tests/r01164.vtc new file mode 100644 index 0000000..43c0996 --- /dev/null +++ b/bin/varnishtest/tests/r01164.vtc @@ -0,0 +1,23 @@ +varnishtest "Regression test for #1163: allow backends to be named storage*" + +varnish v1 -vcl { + backend storage_foo { + .host = "127.0.0.1"; + } + + sub vcl_recv { + set req.backend = storage_foo; + } +} + +varnish v1 -vcl { + backend storagefoo { + .host = "127.0.0.1"; + } + + sub vcl_recv { + set req.backend = storagefoo; + } +} + + diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c index 30f2a24..318c1b9 100644 --- a/lib/libvcl/vcc_compile.c +++ b/lib/libvcl/vcc_compile.c @@ -599,7 +599,7 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp) sym->r_methods = v->r_methods; } - sym = VCC_AddSymbolStr(tl, "storage", SYM_WILDCARD); + sym = VCC_AddSymbolStr(tl, "storage.", SYM_WILDCARD); sym->wildcard = vcc_Stv_Wildcard; vcl_output_lang_h(tl->fh); From phk at FreeBSD.org Thu Dec 18 09:27:51 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] ae3e5a3 Push allocation of request into the assigned worker thread when we come back from the waiter, to lessen amount of work waiter does. Message-ID: commit ae3e5a3a9fcf0214c0d0cfc75f7ae909cf439f8a Author: Poul-Henning Kamp Date: Tue Jul 17 20:10:09 2012 +0000 Push allocation of request into the assigned worker thread when we come back from the waiter, to lessen amount of work waiter does. diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index bbec54f..da2cdf8 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -172,6 +172,8 @@ vca_pace_good(void) /*-------------------------------------------------------------------- * Accept on a listen socket, and handle error returns. + * + * Called from a worker thread from a pool */ int diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 59dccf6..d2cca4b 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -28,9 +28,14 @@ * * Session management * - * This is a little bit of a mixed back, containing both memory management + * This is a little bit of a mixed bag, containing both memory management * and various state-change functions. * + * The overall picture is complicated by the fact that requests can + * disembark their worker-threads if they hit a busy object, then come + * back later in a different worker thread to continue. + * XXX: I wonder if that complexity pays of any more ? + * */ #include "config.h" @@ -121,11 +126,11 @@ ses_new(struct sesspool *pp) } /*-------------------------------------------------------------------- - * The pool-task function for sessions + * Process new/existing request on this session. */ static void -ses_pool_task(struct worker *wrk, void *arg) +ses_req_pool_task(struct worker *wrk, void *arg) { struct req *req; struct sess *sp; @@ -149,6 +154,28 @@ ses_pool_task(struct worker *wrk, void *arg) } /*-------------------------------------------------------------------- + * Allocate a request + vxid, call ses_req_pool_task() + */ + +static void +ses_sess_pool_task(struct worker *wrk, void *arg) +{ + struct req *req; + struct sess *sp; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CAST_OBJ_NOTNULL(sp, arg, SESS_MAGIC); + + req = ses_GetReq(sp); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + req->vxid = VXID_Get(&wrk->vxid_pool); + + sp->sess_step = S_STP_NEWREQ; + ses_req_pool_task(wrk, req); +} + +/*-------------------------------------------------------------------- * VSL log the endpoints of the TCP connection. * * We use VSL() to get the sessions vxid and to make sure tha this @@ -189,7 +216,6 @@ void SES_pool_accept_task(struct worker *wrk, void *arg) { struct sesspool *pp; - struct req *req; struct sess *sp; const char *lsockname; @@ -213,17 +239,13 @@ SES_pool_accept_task(struct worker *wrk, void *arg) lsockname = VCA_SetupSess(wrk, sp); ses_vsl_socket(sp, lsockname); - req = ses_GetReq(sp); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - - req->vxid = VXID_Get(&wrk->vxid_pool); - - sp->sess_step = S_STP_NEWREQ; - ses_pool_task(wrk, req); + ses_sess_pool_task(wrk, sp); } /*-------------------------------------------------------------------- * Schedule a request back on a work-thread from its sessions pool + * + * This is used to reschedule requests waiting on busy objects */ int @@ -239,7 +261,7 @@ SES_ScheduleReq(struct req *req) CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); AN(pp->pool); - sp->task.func = ses_pool_task; + sp->task.func = ses_req_pool_task; sp->task.priv = req; if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) { @@ -260,22 +282,14 @@ SES_ScheduleReq(struct req *req) void SES_Handle(struct sess *sp, double now) { - struct req *req; struct sesspool *pp; - /* NB: This only works with single-threaded waiters */ - static struct vxid_pool vxid_pool; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); pp = sp->sesspool; CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); AN(pp->pool); - req = ses_GetReq(sp); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - req->vxid = VXID_Get(&vxid_pool); - sp->task.func = ses_pool_task; - sp->task.priv = req; - sp->sess_step = S_STP_NEWREQ; + sp->task.func = ses_sess_pool_task; + sp->task.priv = sp; sp->t_rx = now; if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) { VSC_C_main->client_drop_late++; From phk at FreeBSD.org Thu Dec 18 09:27:51 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] 3cc4a72 Introduce an enum to make the states of HTTP reception more readable. Message-ID: commit 3cc4a726bf5d4ff8044883daacffc9bd46f607ec Author: Poul-Henning Kamp Date: Wed Jul 18 20:09:21 2012 +0000 Introduce an enum to make the states of HTTP reception more readable. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 7328779..63a7c3d 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -871,12 +871,20 @@ void http_Unset(struct http *hp, const char *hdr); void http_CollectHdr(struct http *hp, const char *hdr); /* cache_httpconn.c */ +enum htc_status_e { + HTC_ALL_WHITESPACE = -3, + HTC_OVERFLOW = -2, + HTC_ERROR_EOF = -1, + HTC_NEED_MORE = 0, + HTC_COMPLETE = 1 +}; + void HTC_Init(struct http_conn *htc, struct ws *ws, int fd, struct vsl_log *, unsigned maxbytes, unsigned maxhdr); int HTC_Reinit(struct http_conn *htc); -int HTC_Rx(struct http_conn *htc); +enum htc_status_e HTC_Rx(struct http_conn *htc); ssize_t HTC_Read(struct http_conn *htc, void *d, size_t len); -int HTC_Complete(struct http_conn *htc); +enum htc_status_e HTC_Complete(struct http_conn *htc); #define HTTPH(a, b, c) extern char b[]; #include "tbl/http_headers.h" diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index e71ad87..d98d490 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -106,10 +106,11 @@ DOT } static int cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) { - int i, j, tmo; + int j, tmo; struct pollfd pfd[1]; double now, when; enum sess_close why = SC_NULL; + enum htc_status_e hs; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -135,20 +136,20 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) assert(j >= 0); now = VTIM_real(); if (j != 0) - i = HTC_Rx(req->htc); + hs = HTC_Rx(req->htc); else - i = HTC_Complete(req->htc); - if (i == 1) { + hs = HTC_Complete(req->htc); + if (hs == HTC_COMPLETE) { /* Got it, run with it */ req->t_req = now; return (0); - } else if (i == -1) { + } else if (hs == HTC_ERROR_EOF) { why = SC_REM_CLOSE; break; - } else if (i == -2) { + } else if (hs == HTC_OVERFLOW) { why = SC_RX_OVERFLOW; break; - } else if (i == -3) { + } else if (hs == HTC_ALL_WHITESPACE) { /* Nothing but whitespace */ when = sp->t_idle + cache_param->timeout_idle; if (when < now) { diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index ab2aabc..3b9b08d 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -437,8 +437,9 @@ FetchHdr(struct req *req, int need_host_hdr, int sendbody) struct worker *wrk; struct busyobj *bo; struct http *hp; + enum htc_status_e hs; int retry = -1; - int i; + int i, first; struct http_conn *htc; wrk = req->wrk; @@ -499,31 +500,31 @@ FetchHdr(struct req *req, int need_host_hdr, int sendbody) VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout); - i = HTC_Rx(htc); - - if (i < 0) { - VSLb(req->vsl, SLT_FetchError, - "http first read error: %d %d (%s)", - i, errno, strerror(errno)); - VDI_CloseFd(&bo->vbc); - /* XXX: other cleanup ? */ - /* Retryable if we never received anything */ - return (i == -1 ? retry : -1); - } - - VTCP_set_read_timeout(vc->fd, vc->between_bytes_timeout); - - while (i == 0) { - i = HTC_Rx(htc); - if (i < 0) { + first = 1; + do { + hs = HTC_Rx(htc); + if (hs == HTC_OVERFLOW) { VSLb(req->vsl, SLT_FetchError, - "http first read error: %d %d (%s)", - i, errno, strerror(errno)); + "http %sread error: overflow", + first ? "first " : ""); VDI_CloseFd(&bo->vbc); /* XXX: other cleanup ? */ return (-1); } - } + if (hs == HTC_ERROR_EOF) { + VSLb(req->vsl, SLT_FetchError, + "http %sread error: EOF", + first ? "first " : ""); + VDI_CloseFd(&bo->vbc); + /* XXX: other cleanup ? */ + return (retry); + } + if (first) { + first = 0; + VTCP_set_read_timeout(vc->fd, + vc->between_bytes_timeout); + } + } while (hs != HTC_COMPLETE); hp = bo->beresp; diff --git a/bin/varnishd/cache/cache_httpconn.c b/bin/varnishd/cache/cache_httpconn.c index 890dfd3..211e8a9 100644 --- a/bin/varnishd/cache/cache_httpconn.c +++ b/bin/varnishd/cache/cache_httpconn.c @@ -46,44 +46,6 @@ #include "vct.h" -/*-------------------------------------------------------------------- - * Check if we have a complete HTTP request or response yet - * - * Return values: - * -3 All whitespace so far - * 0 No, keep trying - * >0 Yes, it is this many bytes long. - */ - -static int -htc_header_complete(txt *t) -{ - const char *p; - - Tcheck(*t); - assert(*t->e == '\0'); - /* Skip any leading white space */ - for (p = t->b ; vct_islws(*p); p++) - continue; - if (p == t->e) { - /* All white space */ - t->e = t->b; - *t->e = '\0'; - return (-3); - } - while (1) { - p = strchr(p, '\n'); - if (p == NULL) - return (0); - p++; - if (*p == '\r') - p++; - if (*p == '\n') - break; - } - p++; - return (p - t->b); -} /*--------------------------------------------------------------------*/ @@ -139,15 +101,45 @@ HTC_Reinit(struct http_conn *htc) * Return 1 if we have a complete HTTP procol header */ -int +/*-------------------------------------------------------------------- + * Check if we have a complete HTTP request or response yet + * + */ + +enum htc_status_e HTC_Complete(struct http_conn *htc) { int i; + const char *p; + txt *t; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); - i = htc_header_complete(&htc->rxbuf); - if (i <= 0) - return (i); + + t = &htc->rxbuf; + Tcheck(*t); + assert(*t->e == '\0'); + + /* Skip any leading white space */ + for (p = t->b ; vct_islws(*p); p++) + continue; + if (p == t->e) { + /* All white space */ + t->e = t->b; + *t->e = '\0'; + return (HTC_ALL_WHITESPACE); + } + while (1) { + p = strchr(p, '\n'); + if (p == NULL) + return (HTC_NEED_MORE); + p++; + if (*p == '\r') + p++; + if (*p == '\n') + break; + } + p++; + i = p - t->b; WS_ReleaseP(htc->ws, htc->rxbuf.e); AZ(htc->pipeline.b); AZ(htc->pipeline.e); @@ -156,20 +148,14 @@ HTC_Complete(struct http_conn *htc) htc->pipeline.e = htc->rxbuf.e; htc->rxbuf.e = htc->pipeline.b; } - return (1); + return (HTC_COMPLETE); } /*-------------------------------------------------------------------- * Receive more HTTP protocol bytes - * Returns: - * -3 all whitespace so far - * -2 overflow - * -1 error/EOF - * 0 more needed - * 1 got complete HTTP header */ -int +enum htc_status_e HTC_Rx(struct http_conn *htc) { int i; @@ -179,7 +165,7 @@ HTC_Rx(struct http_conn *htc) i = (htc->ws->r - htc->rxbuf.e) - 1; /* space for NUL */ if (i <= 0) { WS_ReleaseP(htc->ws, htc->rxbuf.b); - return (-2); + return (HTC_OVERFLOW); } i = read(htc->fd, htc->rxbuf.e, i); if (i <= 0) { @@ -188,7 +174,7 @@ HTC_Rx(struct http_conn *htc) * so consequently an EOF can not be OK */ WS_ReleaseP(htc->ws, htc->rxbuf.b); - return (-1); + return (HTC_ERROR_EOF); } htc->rxbuf.e += i; *htc->rxbuf.e = '\0'; From phk at FreeBSD.org Thu Dec 18 09:27:51 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] c34f10e Move the temporary accounting structure from wrk to req and eliminate two SES_Charge() calls that have nothing to charge. Message-ID: commit c34f10e1213ae4df0db74d6f0477140cfa047aea Author: Poul-Henning Kamp Date: Wed Jul 18 21:26:16 2012 +0000 Move the temporary accounting structure from wrk to req and eliminate two SES_Charge() calls that have nothing to charge. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 63a7c3d..dfc8f69 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -344,8 +344,6 @@ struct worker { struct vxid_pool vxid_pool; - /* Temporary accounting */ - struct acct acct_tmp; }; /* LRU ---------------------------------------------------------------*/ @@ -661,6 +659,8 @@ struct req { /* Transaction VSL buffer */ struct vsl_log vsl[1]; + /* Temporary accounting */ + struct acct acct_req; }; /*-------------------------------------------------------------------- @@ -821,9 +821,9 @@ int VGZ_Destroy(struct vgz **); void VGZ_UpdateObj(const struct vgz*, struct object *); int VGZ_WrwInit(struct vgz *vg); -int VGZ_WrwGunzip(struct worker *w, struct vgz *, const void *ibuf, +int VGZ_WrwGunzip(struct req *, struct vgz *, const void *ibuf, ssize_t ibufl); -void VGZ_WrwFlush(struct worker *wrk, struct vgz *vg); +void VGZ_WrwFlush(struct req *, struct vgz *vg); /* Return values */ #define VGZ_ERROR -1 diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index d98d490..1fcaff4 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -161,7 +161,6 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) if (when < now || tmo == 0) { sp->t_rx = NAN; wrk->stats.sess_herd++; - SES_Charge(wrk, req); SES_ReleaseReq(req); WAIT_Enter(sp); return (1); @@ -176,7 +175,6 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) } } } - SES_Charge(wrk, req); AZ(req->vcl); SES_ReleaseReq(req); SES_Delete(sp, why, now); @@ -659,7 +657,7 @@ cnt_fetch(struct worker *wrk, struct req *req) need_host_hdr = !http_GetHdr(bo->bereq, H_Host, NULL); - wrk->acct_tmp.fetch++; + req->acct_req.fetch++; i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL); /* @@ -1264,7 +1262,7 @@ cnt_pass(struct worker *wrk, struct req *req) return (0); } assert(req->handling == VCL_RET_PASS); - wrk->acct_tmp.pass++; + req->acct_req.pass++; req->req_step = R_STP_FETCH; req->objcore = HSH_NewObjCore(wrk); @@ -1307,7 +1305,7 @@ cnt_pipe(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); AZ(req->busyobj); - wrk->acct_tmp.pipe++; + req->acct_req.pipe++; req->busyobj = VBO_GetBusyObj(wrk); bo = req->busyobj; bo->vsl->wid = req->sp->vsl_id; @@ -1486,7 +1484,7 @@ cnt_start(struct worker *wrk, struct req *req) /* Update stats of various sorts */ wrk->stats.client_req++; - wrk->acct_tmp.req++; + req->acct_req.req++; /* Assign XID and log */ req->xid = ++xids; /* XXX not locked */ @@ -1600,8 +1598,10 @@ CNT_Request(struct worker *wrk, struct req *req) WS_Assert(wrk->aws); CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); } - if (done == 1) + if (done == 1) { + /* done == 2 was charged by cache_hash.c */ SES_Charge(wrk, req); + } req->wrk = NULL; diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index cc722ce..a6e5f11 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -321,7 +321,7 @@ ESI_Deliver(struct req *req) * response */ AN(vgz); - i = VGZ_WrwGunzip(req->wrk, vgz, + i = VGZ_WrwGunzip(req, vgz, st->ptr + off, l2); if (WRW_Error(req->wrk)) { SES_Close(req->sp, @@ -374,7 +374,7 @@ ESI_Deliver(struct req *req) r = (void*)strchr((const char*)q, '\0'); AN(r); if (vgz != NULL) - VGZ_WrwFlush(req->wrk, vgz); + VGZ_WrwFlush(req, vgz); if (WRW_Flush(req->wrk)) { SES_Close(req->sp, SC_REM_CLOSE); p = e; @@ -391,7 +391,7 @@ ESI_Deliver(struct req *req) } } if (vgz != NULL) { - VGZ_WrwFlush(req->wrk, vgz); + VGZ_WrwFlush(req, vgz); (void)VGZ_Destroy(&vgz); } if (req->gzip_resp && req->esi_level == 0) { diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 1ebabb0..aed84d9 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -315,13 +315,16 @@ VGZ_WrwInit(struct vgz *vg) */ int -VGZ_WrwGunzip(struct worker *wrk, struct vgz *vg, const void *ibuf, +VGZ_WrwGunzip(struct req *req, struct vgz *vg, const void *ibuf, ssize_t ibufl) { int i; size_t dl; const void *dp; + struct worker *wrk; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + wrk = req->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC); AN(vg->m_buf); @@ -340,7 +343,7 @@ VGZ_WrwGunzip(struct worker *wrk, struct vgz *vg, const void *ibuf, return (-1); } if (vg->m_len == vg->m_sz || i == VGZ_STUCK) { - wrk->acct_tmp.bodybytes += vg->m_len; + req->acct_req.bodybytes += vg->m_len; (void)WRW_Write(wrk, vg->m_buf, vg->m_len); (void)WRW_Flush(wrk); vg->m_len = 0; @@ -355,15 +358,19 @@ VGZ_WrwGunzip(struct worker *wrk, struct vgz *vg, const void *ibuf, /*--------------------------------------------------------------------*/ void -VGZ_WrwFlush(struct worker *wrk, struct vgz *vg) +VGZ_WrwFlush(struct req *req, struct vgz *vg) { + struct worker *wrk; + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + wrk = req->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC); if (vg->m_len == 0) return; - wrk->acct_tmp.bodybytes += vg->m_len; + req->acct_req.bodybytes += vg->m_len; (void)WRW_Write(wrk, vg->m_buf, vg->m_len); (void)WRW_Flush(wrk); vg->m_len = 0; diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c index 238d0a3..f13bbcc 100644 --- a/bin/varnishd/cache/cache_pipe.c +++ b/bin/varnishd/cache/cache_pipe.c @@ -82,10 +82,10 @@ PipeRequest(struct req *req) (void)VTCP_blocking(vc->fd); WRW_Reserve(wrk, &vc->fd, bo->vsl, req->t_req); - wrk->acct_tmp.hdrbytes += http_Write(wrk, bo->bereq, 0); + req->acct_req.hdrbytes += http_Write(wrk, bo->bereq, 0); if (req->htc->pipeline.b != NULL) - wrk->acct_tmp.bodybytes += + req->acct_req.bodybytes += WRW_Write(wrk, req->htc->pipeline.b, Tlen(req->htc->pipeline)); diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index e82ab67..fda117d 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -171,11 +171,11 @@ res_WriteGunzipObj(struct req *req) CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); u += st->len; - i = VGZ_WrwGunzip(req->wrk, vg, st->ptr, st->len); + i = VGZ_WrwGunzip(req, vg, st->ptr, st->len); /* XXX: error check */ (void)i; } - VGZ_WrwFlush(req->wrk, vg); + VGZ_WrwFlush(req, vg); (void)VGZ_Destroy(&vg); assert(u == req->obj->len); } @@ -183,7 +183,7 @@ res_WriteGunzipObj(struct req *req) /*--------------------------------------------------------------------*/ static void -res_WriteDirObj(const struct req *req, ssize_t low, ssize_t high) +res_WriteDirObj(struct req *req, ssize_t low, ssize_t high) { ssize_t u = 0; size_t ptr, off, len; @@ -215,7 +215,7 @@ res_WriteDirObj(const struct req *req, ssize_t low, ssize_t high) ptr += len; - req->wrk->acct_tmp.bodybytes += len; + req->acct_req.bodybytes += len; (void)WRW_Write(req->wrk, st->ptr + off, len); } assert(u == req->obj->len); @@ -254,7 +254,7 @@ RES_WriteObj(struct req *req) * Send HTTP protocol header, unless interior ESI object */ if (!(req->res_mode & RES_ESI_CHILD)) - req->wrk->acct_tmp.hdrbytes += + req->acct_req.hdrbytes += http_Write(req->wrk, req->resp, 1); if (!req->wantbody) diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index d2cca4b..5e994f7 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -79,7 +79,7 @@ SES_Charge(struct worker *wrk, struct req *req) sp = req->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - a = &wrk->acct_tmp; + a = &req->acct_req; req->req_bodybytes += a->bodybytes; #define ACCT(foo) \ @@ -229,7 +229,7 @@ SES_pool_accept_task(struct worker *wrk, void *arg) VCA_FailSess(wrk); return; } - wrk->acct_tmp.sess++; + wrk->stats.s_sess++; sp->t_open = VTIM_real(); sp->t_rx = sp->t_open; @@ -431,6 +431,9 @@ SES_ReleaseReq(struct req *req) struct sesspool *pp; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); +#define ACCT(foo) AZ(req->acct_req.foo); +#include "tbl/acct_fields.h" +#undef ACCT sp = req->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); pp = sp->sesspool; diff --git a/include/tbl/acct_fields.h b/include/tbl/acct_fields.h index 154f106..3dc1ff6 100644 --- a/include/tbl/acct_fields.h +++ b/include/tbl/acct_fields.h @@ -26,12 +26,11 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * These are the stats we keep track of per session. They will be summed, - * via the sp->wrk->stats into the s_ fields in the SHM file. + * These are the stats we keep track of per session. + * SES_Charge() sums them into wrk->stats * NB: Remember to mark those in vsc_fields.h to be included in struct dstat. */ -ACCT(sess) ACCT(req) ACCT(pipe) ACCT(pass) From phk at FreeBSD.org Thu Dec 18 09:27:51 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] 1172fc9 Various minor cleanups for code clarity. Message-ID: commit 1172fc93e4d5d3827fcc2dc9abceaef96b1f5cb1 Author: Poul-Henning Kamp Date: Thu Jul 19 08:29:54 2012 +0000 Various minor cleanups for code clarity. diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 1fcaff4..c200bd3 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -175,8 +175,8 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) } } } - AZ(req->vcl); SES_ReleaseReq(req); + assert(why != SC_NULL); SES_Delete(sp, why, now); return (1); } @@ -254,14 +254,8 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) req->hash_always_miss = 0; req->hash_ignore_busy = 0; - if (sp->fd >= 0 && req->doclose != SC_NULL) { - /* - * This is an orderly close of the connection; ditch nolinger - * before we close, to get queued data transmitted. - */ - // XXX: not yet (void)VTCP_linger(sp->fd, 0); + if (sp->fd >= 0 && req->doclose != SC_NULL) SES_Close(sp, req->doclose); - } if (sp->fd < 0) { wrk->stats.sess_closed++; diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 5e994f7..6478a00 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -133,12 +133,9 @@ static void ses_req_pool_task(struct worker *wrk, void *arg) { struct req *req; - struct sess *sp; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CAST_OBJ_NOTNULL(req, arg, REQ_MAGIC); - sp = req->sp; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); THR_SetRequest(req); AZ(wrk->aws->r); @@ -180,6 +177,9 @@ ses_sess_pool_task(struct worker *wrk, void *arg) * * We use VSL() to get the sessions vxid and to make sure tha this * VSL comes before anything else for this session. + * + * This is a separate procedure only to isolate the two stack buffers. + * */ static void @@ -317,12 +317,7 @@ SES_Close(struct sess *sp, enum sess_close reason) } /*-------------------------------------------------------------------- - * (Close &) Free or Recycle a session. - * - * If the workspace has changed, deleted it, otherwise wash it, and put - * it up for adoption. - * - * XXX: We should also check nhttp + * Report and dismantle a session. */ void @@ -338,27 +333,18 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now) if (reason != SC_NULL) SES_Close(sp, reason); + assert(sp->fd < 0); + if (isnan(now)) now = VTIM_real(); assert(!isnan(sp->t_open)); - assert(sp->fd < 0); - - if (*sp->addr == '\0') - strcpy(sp->addr, "-"); - if (*sp->port == '\0') - strcpy(sp->addr, "-"); b = &sp->acct_ses; - - VSL(SLT_SessClose, sp->vxid, - "%s %.3f %ju %ju %ju %ju %ju %ju", - sess_close_str(sp->reason, 0), - now - sp->t_open, - b->req, b->pipe, b->pass, - b->fetch, b->hdrbytes, b->bodybytes); + VSL(SLT_SessClose, sp->vxid, "%s %.3f %ju %ju %ju %ju %ju %ju", + sess_close_str(sp->reason, 0), now - sp->t_open, b->req, + b->pipe, b->pass, b->fetch, b->hdrbytes, b->bodybytes); MPL_Free(pp->mpl_sess, sp); - } /*-------------------------------------------------------------------- From martin at varnish-software.com Thu Dec 18 09:27:51 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] c22afc8 Do not consider busy objects as candidates in EXP_NukeOne. Message-ID: commit c22afc8a73d3e55831e06293c4b2f42a7e15b9e8 Author: Martin Blix Grydeland Date: Wed Jul 4 13:25:08 2012 +0200 Do not consider busy objects as candidates in EXP_NukeOne. An attempt to nuke a busy object will end in assertion. diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index 5a41875..8f072b6 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -428,9 +428,10 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru) /* * It wont release any space if we cannot release the last * reference, besides, if somebody else has a reference, - * it's a bad idea to nuke this object anyway. + * it's a bad idea to nuke this object anyway. Also do not + * touch busy objects. */ - if (oc->refcnt == 1) + if (oc->refcnt == 1 && !(oc->flags & OC_F_BUSY)) break; } if (oc != NULL) { From martin at varnish-software.com Thu Dec 18 09:27:51 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] 116393f Do not try to expire objects that are still busy. Message-ID: commit 116393f289258b2bb9b49d518b0206cec6550a75 Author: Martin Blix Grydeland Date: Wed Jul 4 13:07:15 2012 +0200 Do not try to expire objects that are still busy. There is a short window of opportunity between HSH_Insert() and HSH_Unbusy(), where the locks are released and the expiry thread can try to expire an object that is still busy (for objects with very short TTL). This patch makes the expiry thread check for busy objects and wait if the head of the binheap is busy. No test case as the window is too small. Fixes: #1150 diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index 8f072b6..3d18d78 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -369,6 +369,13 @@ exp_timer(struct worker *wrk, void *priv) continue; } + /* If the object is busy, we have to wait for it */ + if (oc->flags & OC_F_BUSY) { + Lck_Unlock(&exp_mtx); + oc = NULL; + continue; + } + /* * It's time... * Technically we should drop the exp_mtx, get the lru->mtx From martin at varnish-software.com Thu Dec 18 09:27:51 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] 0c9975c Make the ban lurker skip busy objects Message-ID: commit 0c9975c1be926799c368d904dd65204eca25ed50 Author: Martin Blix Grydeland Date: Wed Jul 4 13:02:55 2012 +0200 Make the ban lurker skip busy objects If the ban lurker finds a busy object, skip the object and defer the checking to a later pass. No test case because it is very timing sensitive (but see the trac bug report for a VTC-file that mostly triggers it on the 3.0 branch only) Fixes: #1162 diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index be8af1a..4164e4b 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -879,6 +879,18 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl, unsigned pass) continue; } /* + * If the object is busy, we can't touch + * it. Defer it to a later run. + */ + if (oc->flags & OC_F_BUSY) { + oc->flags |= pass; + VTAILQ_REMOVE(&b->objcore, oc, ban_list); + VTAILQ_INSERT_TAIL(&b->objcore, oc, ban_list); + Lck_Unlock(&oh->mtx); + Lck_Unlock(&ban_mtx); + continue; + } + /* * Grab a reference to the OC and we can let go of * the BAN mutex */ From tfheen at varnish-software.com Thu Dec 18 09:27:51 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] 8d1a7b8 Test commit to check trac hooks Message-ID: commit 8d1a7b8461db6508d6e3e4f4138cf9059e0de784 Author: Tollef Fog Heen Date: Thu Jul 19 14:27:13 2012 +0200 Test commit to check trac hooks diff --git a/Makefile.am b/Makefile.am index dad3621..44b2481 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,5 +1,3 @@ -# - ACLOCAL_AMFLAGS = -I m4 SUBDIRS = include lib bin man etc doc From tfheen at varnish-software.com Thu Dec 18 09:27:51 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:51 +0100 Subject: [experimental-ims] 5d976a5 INSTALL: minor update Message-ID: commit 5d976a5d500df0b59226606db05a17269bba87ea Author: Tollef Fog Heen Date: Thu Jul 19 14:30:10 2012 +0200 INSTALL: minor update diff --git a/INSTALL b/INSTALL index ca9f152..97b2502 100644 --- a/INSTALL +++ b/INSTALL @@ -5,8 +5,8 @@ run the 'configure' script in the top-level directory, then run 'make' and 'make install'. On Linux, you need to run 'ldconfig' as root afterwards in order to update the shared library cache. -If you obtained the sources directly from the Subversion repository, -you will need to run autogen.sh first to create the configure script. +If you obtained the sources directly from the Git repository, you will +need to run autogen.sh first to create the configure script. Varnish will store run-time state in $localstatedir/varnish; you may want to tune this using configure's --localstatedir parameter. From martin at varnish-software.com Thu Dec 18 09:27:52 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] f9c42ca Extend cli_limit to 48k to make room for the full output of 'param.show -l'. Message-ID: commit f9c42ca8306792ceb1d52cb78c4af23b7ee474af Author: Martin Blix Grydeland Date: Fri Jul 20 12:35:56 2012 +0200 Extend cli_limit to 48k to make room for the full output of 'param.show -l'. Fixes: #1169 diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index b030caa..2f090dc 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -908,7 +908,7 @@ static const struct parspec input_parspec[] = { " this limit, the reponse code will be 201 instead of" " 200 and the last line will indicate the truncation.", 0, - "4k", "bytes" }, + "48k", "bytes" }, { "cli_timeout", tweak_timeout, &mgt_param.cli_timeout, 0, 0, "Timeout for the childs replies to CLI requests from " "the mgt_param.", diff --git a/bin/varnishtest/tests/r01169.vtc b/bin/varnishtest/tests/r01169.vtc new file mode 100644 index 0000000..73e741d --- /dev/null +++ b/bin/varnishtest/tests/r01169.vtc @@ -0,0 +1,9 @@ +varnishtest "cli_limit truncating full parameter listing - #1169" + +server s1 { + rxreq +} -start + +varnish v1 -vcl+backend { } + +varnish v1 -cliok "param.show -l" From lasse at varnish-software.com Thu Dec 18 09:27:52 2014 From: lasse at varnish-software.com (Lasse Karstensen) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] acc8909 Regex set for doing device detection Message-ID: commit acc8909f7439d55a085d90b879b57db46bc4da13 Author: Lasse Karstensen Date: Fri Jul 20 12:47:06 2012 +0200 Regex set for doing device detection diff --git a/etc/devicedetect.vcl b/etc/devicedetect.vcl new file mode 100644 index 0000000..ed5b39d --- /dev/null +++ b/etc/devicedetect.vcl @@ -0,0 +1,69 @@ +# +# detectdevice.vcl - regex based device detection for Varnish +# http://github.com/varnish/varnish-devicedetect/ +# +# Author: Lasse Karstensen + +sub devicedetect { + unset req.http.X-UA-Device; + set req.http.X-UA-Device = "pc"; + + if (req.http.User-Agent ~ "(?i)(ads|google|bing|msn|yandex|baidu|ro|career|)bot" || + req.http.User-Agent ~ "(?i)(baidu|symantec)spider" || + req.http.User-Agent ~ "(?i)scanner" || + req.http.User-Agent ~ "(?i)(web)crawler") { + set req.http.X-UA-Device = "bot"; } + elsif (req.http.User-Agent ~ "(?i)ip(hone|od)") { set req.http.X-UA-Device = "mobile-iphone"; } + elsif (req.http.User-Agent ~ "(?i)ipad") { set req.http.X-UA-Device = "tablet-ipad"; } + # how do we differ between an android phone and an android tablet? + # http://stackoverflow.com/questions/5341637/how-do-detect-android-tablets-in-general-useragent + elsif (req.http.User-Agent ~ "(?i)android.*(mobile|mini)") { set req.http.X-UA-Device = "mobile-android"; } + # android 3/honeycomb was just about tablet-only, and any phones will probably handle a bigger page layout. + elsif (req.http.User-Agent ~ "(?i)android 3") { set req.http.X-UA-Device = "tablet-android"; } + # may very well give false positives towards android tablets. Suggestions welcome. + elsif (req.http.User-Agent ~ "(?i)android") { set req.http.X-UA-Device = "tablet-android"; } + + elsif (req.http.User-Agent ~ "^HTC" || + req.http.User-Agent ~ "Fennec" || + req.http.User-Agent ~ "IEMobile" || + req.http.User-Agent ~ "BlackBerry" || + req.http.User-Agent ~ "SymbianOS.*AppleWebKit" || + req.http.User-Agent ~ "Opera Mobi") { + set req.http.X-UA-Device = "mobile-smartphone"; + } + elsif (req.http.User-Agent ~ "(?i)symbian" || + req.http.User-Agent ~ "(?i)^sonyericsson" || + req.http.User-Agent ~ "(?i)^nokia" || + req.http.User-Agent ~ "(?i)^samsung" || + req.http.User-Agent ~ "(?i)^lg" || + req.http.User-Agent ~ "(?i)bada" || + req.http.User-Agent ~ "(?i)blazer" || + req.http.User-Agent ~ "(?i)cellphone" || + req.http.User-Agent ~ "(?i)iemobile" || + req.http.User-Agent ~ "(?i)midp-2.0" || + req.http.User-Agent ~ "(?i)u990" || + req.http.User-Agent ~ "(?i)netfront" || + req.http.User-Agent ~ "(?i)opera mini" || + req.http.User-Agent ~ "(?i)palm" || + req.http.User-Agent ~ "(?i)nintendo wii" || + req.http.User-Agent ~ "(?i)playstation portable" || + req.http.User-Agent ~ "(?i)portalmmm" || + req.http.User-Agent ~ "(?i)proxinet" || + req.http.User-Agent ~ "(?i)sonyericsson" || + req.http.User-Agent ~ "(?i)symbian" || + req.http.User-Agent ~ "(?i)windows\ ?ce" || + req.http.User-Agent ~ "(?i)winwap" || + req.http.User-Agent ~ "(?i)eudoraweb" || + req.http.User-Agent ~ "(?i)htc" || + req.http.User-Agent ~ "(?i)240x320" || + req.http.User-Agent ~ "(?i)avantgo") { + set req.http.X-UA-Device = "mobile-generic"; + } + # handle overrides + if (req.http.Cookie ~ "(i?)X-UA-Device-force") { + # ;?? means zero or one ;, non-greedy to match the first. + set req.http.X-UA-Device = regsub(req.http.Cookie, "(?i).*X-UA-Device-force=([^;]+);??.*", "\1"); + } +} + +# vim: sw=4:tw=120 # meh From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] 73d989b Patch from Geoff: Message-ID: commit 73d989b5ccf1056399c3fe3e05ccf036f175c5f6 Author: Poul-Henning Kamp Date: Mon Jul 30 09:15:06 2012 +0000 Patch from Geoff: Fixes a compile error (incorrect signature for SES_Delete() in cache_waiter_ports.c) diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c index 87eaeac..af5d965 100644 --- a/bin/varnishd/waiter/cache_waiter_ports.c +++ b/bin/varnishd/waiter/cache_waiter_ports.c @@ -80,14 +80,13 @@ vws_port_ev(struct vws *vws, port_event_t *ev, double now) { VTAILQ_INSERT_TAIL(&vws->sesshead, sp, list); vws_add(vws, sp->fd, sp); } else { - int i; assert(ev->portev_source == PORT_SOURCE_FD); CAST_OBJ_NOTNULL(sp, ev->portev_user, SESS_MAGIC); assert(sp->fd >= 0); if(ev->portev_events & POLLERR) { vws_del(vws, sp->fd); VTAILQ_REMOVE(&vws->sesshead, sp, list); - SES_Delete(sp, "EOF", now); + SES_Delete(sp, SC_REM_CLOSE, now); return; } @@ -211,7 +210,7 @@ vws_thread(void *priv) if(sp->fd != -1) { vws_del(vws, sp->fd); } - SES_Delete(sp, "timeout", now); + SES_Delete(sp, SC_RX_TIMEOUT, now); } /* From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] 15330c9 Make the sandbox interface more modular and tell the implementation which subprocess we are locking down. Message-ID: commit 15330c9023befc6653bec269eabb0576b61732cf Author: Poul-Henning Kamp Date: Mon Jul 30 15:05:25 2012 +0000 Make the sandbox interface more modular and tell the implementation which subprocess we are locking down. Inspired by Geoff's patch for Solaris, but does not solve the Solaris problem, only makes it easier to fix. diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 6478a00..a521f6f 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -179,7 +179,7 @@ ses_sess_pool_task(struct worker *wrk, void *arg) * VSL comes before anything else for this session. * * This is a separate procedure only to isolate the two stack buffers. - * + * */ static void diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 512c6dc..905fbcc 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -74,13 +74,19 @@ void MCF_DumpRstParam(void); extern struct params mgt_param; /* mgt_sandbox.c */ -void mgt_sandbox(void); +enum sandbox_e { + SANDBOX_VCC = 1, + SANDBOX_CC = 2, + SANDBOX_VCLLOAD = 3, + SANDBOX_WORKER = 4, +}; + +typedef void mgt_sandbox_f(enum sandbox_e); +extern mgt_sandbox_f *mgt_sandbox; /* mgt_sandbox_solaris.c */ #ifdef HAVE_SETPPRIV -void mgt_sandbox_solaris_init(void); -void mgt_sandbox_solaris_fini(void); -void mgt_sandbox_solaris_privsep(void); +mgt_sandbox_f mgt_sandbox_solaris; #endif /* mgt_shmem.c */ diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 8661b73..c869897 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -338,7 +338,7 @@ start_child(struct cli *cli) (void)signal(SIGINT, SIG_DFL); (void)signal(SIGTERM, SIG_DFL); - mgt_sandbox(); + mgt_sandbox(SANDBOX_WORKER); child_main(); diff --git a/bin/varnishd/mgt/mgt_sandbox.c b/bin/varnishd/mgt/mgt_sandbox.c index 3cd1d98..bf9802f 100644 --- a/bin/varnishd/mgt/mgt_sandbox.c +++ b/bin/varnishd/mgt/mgt_sandbox.c @@ -59,32 +59,41 @@ /* Waive all privileges in the child, it does not need any */ -void -mgt_sandbox(void) +static void __match_proto__(mgt_sandbox_f) +mgt_sandbox_unix(enum sandbox_e who) { -#ifdef HAVE_SETPPRIV - mgt_sandbox_solaris_init(); - mgt_sandbox_solaris_privsep(); -#else + (void)who; if (geteuid() == 0) { XXXAZ(setgid(mgt_param.gid)); XXXAZ(setuid(mgt_param.uid)); } else { REPORT0(LOG_INFO, "Not running as root, no priv-sep"); } -#endif +} - /* On Linux >= 2.4, you need to set the dumpable flag - to get core dumps after you have done a setuid. */ +/*--------------------------------------------------------------------*/ #ifdef __linux__ - if (prctl(PR_SET_DUMPABLE, 1) != 0) +static void __match_proto__(mgt_sandbox_f) +mgt_sandbox_linux(enum sandbox_e who) +{ + mgt_sandbox_unix(who); + + if (prctl(PR_SET_DUMPABLE, 1) != 0) { REPORT0(LOG_INFO, "Could not set dumpable bit. Core dumps turned off\n"); + } +} #endif -#ifdef HAVE_SETPPRIV - mgt_sandbox_solaris_fini(); -#endif -} +/*--------------------------------------------------------------------*/ + +mgt_sandbox_f *mgt_sandbox = +#ifdef HAVE_SETPRIV + mgt_sandbox_solaris; +#elif defined (__linux__) + mgt_sandbox_linux; +#else + mgt_sandbox_unix; +#endif diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 6f4642d..f7f199a 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -137,7 +137,7 @@ run_vcc(void *priv) int fd, i, l; CAST_OBJ_NOTNULL(vp, priv, VCC_PRIV_MAGIC); - mgt_sandbox(); + mgt_sandbox(SANDBOX_VCC); sb = VSB_new_auto(); XXXAN(sb); VCC_VCL_dir(vcc, mgt_vcl_dir); @@ -176,7 +176,7 @@ run_vcc(void *priv) static void run_cc(void *priv) { - mgt_sandbox(); + mgt_sandbox(SANDBOX_CC); (void)execl("/bin/sh", "/bin/sh", "-c", priv, NULL); } @@ -193,7 +193,7 @@ run_dlopen(void *priv) of = priv; - mgt_sandbox(); + mgt_sandbox(SANDBOX_VCLLOAD); /* Try to load the object into this sub-process */ if ((dlh = dlopen(of, RTLD_NOW | RTLD_LOCAL)) == NULL) { From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] d5c5d5b Sigh... Message-ID: commit d5c5d5b68cd62b633c25e44ff3bd9d93985c78d6 Author: Poul-Henning Kamp Date: Mon Jul 30 15:17:12 2012 +0000 Sigh... It was a nice idea, but obviously compiler-writers and/or language laywers are pussies who hate table-based programming :-) diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 37c2e6b..ae83bf7 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -75,18 +75,19 @@ SLTM(SessOpen, "Client connection opened", "lport\n Local TCP port ('-' if !$log_local_addr)\n\n" ) +/* + * XXX: compilers are _so_ picky, and won't let us do an #include + * XXX: in the middle of a macro invocation :-( + * XXX: If we could, these three lines would have described the + * XXX: 'reason' field below. #define SESS_CLOSE(nm, desc) " " #nm "\n\t" desc "\n\n" +#include +#undef SESS_CLOSE +*/ SLTM(SessClose, "Client connection closed", "SessionClose is the last record for any client connection.\n\n" "reason\n Why the connection closed.\n\n" -#ifdef __clang__ -/* - * GCC barfs at this construct, but we have no way to ask compiler - * if it is _not_ a GCC compiler since CLANG also defines __GNUC__ - */ -#include -#endif "duration\n How long the session were open.\n\n" "Nreq\n How many requests on session.\n\n" "Npipe\n If 'pipe' were used on session.\n\n" @@ -95,7 +96,6 @@ SLTM(SessClose, "Client connection closed", "Bhdr\n Header bytes sent on session.\n\n" "Bbody\n Body bytes sent on session.\n\n" ) -#undef SESS_CLOSE /*---------------------------------------------------------------------*/ From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] 610d3cb I have no idea how I ended up incrementting the fail counter by the size of the request, probably sleep copy&paste ? Message-ID: commit 610d3cbee4fd6b1e2ca6c84ef385beae07f56144 Author: Poul-Henning Kamp Date: Mon Jul 30 15:52:55 2012 +0000 I have no idea how I ended up incrementting the fail counter by the size of the request, probably sleep copy&paste ? Fixed #1175 Submitted by: martin diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c index 0470093..88cb948 100644 --- a/bin/varnishd/storage/storage_malloc.c +++ b/bin/varnishd/storage/storage_malloc.c @@ -67,7 +67,7 @@ sma_alloc(struct stevedore *st, size_t size) Lck_Lock(&sma_sc->sma_mtx); sma_sc->stats->c_req++; if (sma_sc->sma_alloc + size > sma_sc->sma_max) { - sma_sc->stats->c_fail += size; + sma_sc->stats->c_fail++; size = 0; } else { sma_sc->sma_alloc += size; diff --git a/bin/varnishtest/tests/r01175.vtc b/bin/varnishtest/tests/r01175.vtc new file mode 100644 index 0000000..739c56a --- /dev/null +++ b/bin/varnishtest/tests/r01175.vtc @@ -0,0 +1,20 @@ +varnishtest "#1175 - -smalloc c_fail incremented by bytes" + +server s1 { + rxreq + txresp -nolen -hdr "Content-Length: 1048576" +} -start + +varnish v1 -storage "-s test=malloc,1M" -vcl+backend { + sub vcl_fetch { + set beresp.storage = "test"; + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 503 +} -run + +varnish v1 -expect SMA.test.c_fail < 5 From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] ffa30f0 Change our chosen "bad_ip" to 192.0.2.255 which is a broadcast address from RFC5737's TEST-NET-1 and put a comment there to explain. Message-ID: commit ffa30f0490d6e2b59204757ddbdc7f032d64ce3a Author: Poul-Henning Kamp Date: Mon Jul 30 16:33:48 2012 +0000 Change our chosen "bad_ip" to 192.0.2.255 which is a broadcast address from RFC5737's TEST-NET-1 and put a comment there to explain. I fear this is a loosing battle, but at least this is more "official" than the previous "10.255.255.255" choice. Fixes #1159 diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 99cf840..a2c0571 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -563,7 +563,15 @@ exec_file(const char *fn, const char *script, const char *tmpdir, cwd = getcwd(NULL, PATH_MAX); macro_def(vltop, NULL, "pwd", "%s", cwd); macro_def(vltop, NULL, "topbuild", "%s/%s", cwd, TOP_BUILDDIR); - macro_def(vltop, NULL, "bad_ip", "10.255.255.255"); + + /* + * We need an IP number which will not repond, ever, and that is a + * lot harder than it sounds. This IP# is from RFC5737 and a + * C-class broadcast at that. + * If tests involving ${bad_ip} fails and you run linux, you should + * check your /proc/sys/net/ipv4/ip_nonlocal_bind setting. + */ + macro_def(vltop, NULL, "bad_ip", "192.0.2.255"); /* Move into our tmpdir */ AZ(chdir(tmpdir)); From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] feac371 Don't do automatic backend retry if we have received anything from the backend at all. The retry is only (pseudo-)valid when we have not received anything at all, indicating that the backend closed before it got our request. Message-ID: commit feac37150e5820cb36b0181811a47b957cd9b51c Author: Poul-Henning Kamp Date: Tue Jul 31 08:04:15 2012 +0000 Don't do automatic backend retry if we have received anything from the backend at all. The retry is only (pseudo-)valid when we have not received anything at all, indicating that the backend closed before it got our request. Also don't retry if we sent the request body (not part of test-case). Fixes #1156 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 3b9b08d..1d97c51 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -480,6 +480,8 @@ FetchHdr(struct req *req, int need_host_hdr, int sendbody) /* Deal with any message-body the request might have */ i = FetchReqBody(req, sendbody); + if (sendbody && req->reqbodydone) + retry = -1; if (WRW_FlushRelease(wrk) || i > 0) { VSLb(req->vsl, SLT_FetchError, "backend write error: %d (%s)", @@ -520,6 +522,7 @@ FetchHdr(struct req *req, int need_host_hdr, int sendbody) return (retry); } if (first) { + retry = -1; first = 0; VTCP_set_read_timeout(vc->fd, vc->between_bytes_timeout); diff --git a/bin/varnishtest/tests/r01156.vtc b/bin/varnishtest/tests/r01156.vtc new file mode 100644 index 0000000..2624ddd --- /dev/null +++ b/bin/varnishtest/tests/r01156.vtc @@ -0,0 +1,53 @@ +varnishtest "Don't retry if backend ever sent anything" + +server s1 { + rxreq + expect req.url == /1 + txresp -bodylen 1 + + rxreq + expect req.url == /2 + close + accept + + rxreq + expect req.url == /2 + txresp -bodylen 2 + + rxreq + expect req.url == /3 + send "200 " + delay .1 + close + accept + + rxreq + expect req.url == /4 + txresp -bodylen 4 +} -start + +varnish v1 -vcl+backend {} -start + +client c1 { + txreq -url /1 + rxresp + expect resp.status == 200 + expect resp.bodylen == 1 + + # This one should get retried + txreq -url /2 + rxresp + expect resp.status == 200 + expect resp.bodylen == 2 + + # This one should not get retried + txreq -url /3 + rxresp + expect resp.status == 503 +} -run +client c1 { + txreq -url /4 + rxresp + expect resp.status == 200 + expect resp.bodylen == 4 +} -run From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] 5536bb3 Further untangle req/session state engines Message-ID: commit 5536bb3168169e1af04b4d82b1e64370e238b552 Author: Poul-Henning Kamp Date: Tue Jul 31 09:32:49 2012 +0000 Further untangle req/session state engines diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index c200bd3..64efb5d 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -93,7 +93,7 @@ static unsigned xids; DOT subgraph xcluster_wait { DOT wait [ DOT shape=box -DOT label="cnt_wait:\nwait for\ncomplete\nrequest" +DOT label="cnt_sess_wait:\nwait for\ncomplete\nrequest" DOT ] DOT herding [shape=hexagon] DOT wait -> start [label="got req",style=bold,color=green] @@ -104,7 +104,7 @@ DOT } */ static int -cnt_wait(struct sess *sp, struct worker *wrk, struct req *req) +cnt_sess_wait(struct sess *sp, struct worker *wrk, struct req *req) { int j, tmo; struct pollfd pfd[1]; @@ -204,7 +204,6 @@ enum cnt_sess_done_ret { static enum cnt_sess_done_ret cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) { - double dh, dp, da; int i; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); @@ -228,21 +227,8 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) sp->t_idle = W_TIM_real(wrk); - if (req->xid == 0) { + if (req->xid == 0) req->t_resp = sp->t_idle; - } else { - dp = req->t_resp - req->t_req; - da = sp->t_idle - req->t_resp; - dh = req->t_req - sp->t_open; - /* XXX: Add StatReq == StatSess */ - /* XXX: Workaround for pipe */ - if (sp->fd >= 0) { - VSLb(req->vsl, SLT_Length, "%ju", - (uintmax_t)req->req_bodybytes); - } - VSLb(req->vsl, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f", - req->xid, req->t_req, sp->t_idle, dh, dp, da); - } req->xid = 0; VSL_Flush(req->vsl, 0); @@ -351,7 +337,7 @@ CNT_Session(struct worker *wrk, struct req *req) } if (sp->sess_step == S_STP_NEWREQ) { - done = cnt_wait(sp, wrk, req); + done = cnt_sess_wait(sp, wrk, req); if (done) return; sp->sess_step = S_STP_WORKING; @@ -1593,6 +1579,19 @@ CNT_Request(struct worker *wrk, struct req *req) CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); } if (done == 1) { + /* XXX: Workaround for pipe */ + if (req->sp->fd >= 0) { + VSLb(req->vsl, SLT_Length, "%ju", + (uintmax_t)req->req_bodybytes); + } + VSLb(req->vsl, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f", + req->xid, + req->t_req, + req->sp->t_idle, + req->sp->t_idle - req->t_resp, + req->t_resp - req->t_req, + req->sp->t_idle - req->t_resp); + /* done == 2 was charged by cache_hash.c */ SES_Charge(wrk, req); } From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] a6419c8 Fix HTC_Reinit() return type Message-ID: commit a6419c8abeceb1b7759865c23eb57dbcc46dcded Author: Poul-Henning Kamp Date: Tue Jul 31 10:09:41 2012 +0000 Fix HTC_Reinit() return type diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index dfc8f69..56c4e6b 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -881,7 +881,7 @@ enum htc_status_e { void HTC_Init(struct http_conn *htc, struct ws *ws, int fd, struct vsl_log *, unsigned maxbytes, unsigned maxhdr); -int HTC_Reinit(struct http_conn *htc); +enum htc_status_e HTC_Reinit(struct http_conn *htc); enum htc_status_e HTC_Rx(struct http_conn *htc); ssize_t HTC_Read(struct http_conn *htc, void *d, size_t len); enum htc_status_e HTC_Complete(struct http_conn *htc); diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 64efb5d..4132de9 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -204,7 +204,6 @@ enum cnt_sess_done_ret { static enum cnt_sess_done_ret cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) { - int i; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -258,8 +257,7 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) WS_Reset(wrk->aws, NULL); req->vxid = VXID_Get(&wrk->vxid_pool); - i = HTC_Reinit(req->htc); - if (i == 1) { + if (HTC_Reinit(req->htc) == HTC_COMPLETE) { req->t_req = sp->t_idle; wrk->stats.sess_pipeline++; return (SESS_DONE_RET_START); diff --git a/bin/varnishd/cache/cache_httpconn.c b/bin/varnishd/cache/cache_httpconn.c index 211e8a9..6ebb86c 100644 --- a/bin/varnishd/cache/cache_httpconn.c +++ b/bin/varnishd/cache/cache_httpconn.c @@ -75,7 +75,7 @@ HTC_Init(struct http_conn *htc, struct ws *ws, int fd, struct vsl_log *vsl, * the ws somewhere, because WS_Reset only fiddles pointers. */ -int +enum htc_status_e HTC_Reinit(struct http_conn *htc) { unsigned l; From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] f81aedd More untangling of req vs. session state. Message-ID: commit f81aeddd81c12fc09c9b257963d17cca7548dda6 Author: Poul-Henning Kamp Date: Tue Jul 31 10:41:49 2012 +0000 More untangling of req vs. session state. Eliminate the t_rx timestamp, I doubt it made anything clearer for anybody. Add timestamp to SessOpen VSL record. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 56c4e6b..831c641 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -706,7 +706,6 @@ struct sess { /* Timestamps, all on TIM_real() timescale */ double t_open; /* fd accepted */ double t_idle; /* fd accepted or resp sent */ - double t_rx; #if defined(HAVE_EPOLL_CTL) struct epoll_event ev; diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c index 4132de9..dc7b4cc 100644 --- a/bin/varnishd/cache/cache_center.c +++ b/bin/varnishd/cache/cache_center.c @@ -118,14 +118,13 @@ cnt_sess_wait(struct sess *sp, struct worker *wrk, struct req *req) assert(req->sp == sp); - assert(!isnan(sp->t_rx)); AZ(req->vcl); AZ(req->obj); AZ(req->esi_level); assert(req->xid == 0); - req->t_req = sp->t_rx; - req->t_resp = NAN; + assert(isnan(req->t_req)); + assert(isnan(req->t_resp)); tmo = (int)(1e3 * cache_param->timeout_linger); while (1) { @@ -159,7 +158,7 @@ cnt_sess_wait(struct sess *sp, struct worker *wrk, struct req *req) when = sp->t_idle + cache_param->timeout_linger; tmo = (int)(1e3 * (when - now)); if (when < now || tmo == 0) { - sp->t_rx = NAN; + req->t_req = NAN; wrk->stats.sess_herd++; SES_ReleaseReq(req); WAIT_Enter(sp); @@ -167,7 +166,9 @@ cnt_sess_wait(struct sess *sp, struct worker *wrk, struct req *req) } } else { /* Working on it */ - when = sp->t_rx + cache_param->timeout_req; + if (isnan(req->t_req)) + req->t_req = now; + when = req->t_req + cache_param->timeout_req; tmo = (int)(1e3 * (when - now)); if (when < now || tmo == 0) { why = SC_RX_TIMEOUT; @@ -224,7 +225,6 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) req->vcl = NULL; } - sp->t_idle = W_TIM_real(wrk); if (req->xid == 0) req->t_resp = sp->t_idle; @@ -262,8 +262,6 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) wrk->stats.sess_pipeline++; return (SESS_DONE_RET_START); } else { - sp->t_rx = sp->t_idle; - req->t_req = NAN; if (Tlen(req->htc->rxbuf)) wrk->stats.sess_readahead++; return (SESS_DONE_RET_WAIT); @@ -303,6 +301,12 @@ CNT_Session(struct worker *wrk, struct req *req) return; } + if (sp->sess_step == S_STP_NEWREQ) { + HTC_Init(req->htc, req->ws, sp->fd, req->vsl, + cache_param->http_req_size, + cache_param->http_req_hdr_len); + } + while (1) { /* * Possible entrance states diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index a521f6f..d67d5ff 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -104,7 +104,6 @@ ses_setup(struct sess *sp) sp->sockaddr.ss_family = sp->mysockaddr.ss_family = PF_UNSPEC; sp->t_open = NAN; sp->t_idle = NAN; - sp->t_rx = NAN; } /*-------------------------------------------------------------------- @@ -202,8 +201,8 @@ ses_vsl_socket(struct sess *sp, const char *lsockname) strcpy(laddr, "-"); strcpy(lport, "-"); } - VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s", - sp->addr, sp->port, lsockname, laddr, lport); + VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s %.6f", + sp->addr, sp->port, lsockname, laddr, lport, sp->t_open); } /*-------------------------------------------------------------------- @@ -232,7 +231,6 @@ SES_pool_accept_task(struct worker *wrk, void *arg) wrk->stats.s_sess++; sp->t_open = VTIM_real(); - sp->t_rx = sp->t_open; sp->t_idle = sp->t_open; sp->vxid = VXID_Get(&wrk->vxid_pool); @@ -266,10 +264,9 @@ SES_ScheduleReq(struct req *req) if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) { VSC_C_main->client_drop_late++; - sp->t_idle = VTIM_real(); AN (req->vcl); VCL_Rel(&req->vcl); - SES_Delete(sp, SC_OVERLOAD, sp->t_idle); + SES_Delete(sp, SC_OVERLOAD, NAN); return (1); } return (0); @@ -290,11 +287,9 @@ SES_Handle(struct sess *sp, double now) AN(pp->pool); sp->task.func = ses_sess_pool_task; sp->task.priv = sp; - sp->t_rx = now; if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) { VSC_C_main->client_drop_late++; - sp->t_idle = VTIM_real(); - SES_Delete(sp, SC_OVERLOAD, sp->t_idle); + SES_Delete(sp, SC_OVERLOAD, now); } } @@ -403,9 +398,8 @@ ses_GetReq(struct sess *sp) WS_Init(req->ws, "req", p, e - p); - HTC_Init(req->htc, req->ws, sp->fd, req->vsl, - cache_param->http_req_size, - cache_param->http_req_hdr_len); + req->t_req = NAN; + req->t_resp = NAN; return (req); } @@ -417,6 +411,7 @@ SES_ReleaseReq(struct req *req) struct sesspool *pp; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + AZ(req->vcl); #define ACCT(foo) AZ(req->acct_req.foo); #include "tbl/acct_fields.h" #undef ACCT From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] d47db77 Add the version identifier to the CLI banner. Message-ID: commit d47db772a308970b8e0b61cbc985e51c1a2e2947 Author: Poul-Henning Kamp Date: Mon Aug 6 08:12:13 2012 +0000 Add the version identifier to the CLI banner. Inspired by: patch from geoff diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c index 3b0c1a7..6f319f6 100644 --- a/bin/varnishd/mgt/mgt_cli.c +++ b/bin/varnishd/mgt/mgt_cli.c @@ -80,6 +80,7 @@ mcf_banner(struct cli *cli, const char *const *av, void *priv) VCLI_Out(cli, "Varnish Cache CLI 1.0\n"); VCLI_Out(cli, "-----------------------------\n"); VCLI_Out(cli, "%s\n", VSB_data(vident) + 1); + VCLI_Out(cli, "%s\n", VCS_version); VCLI_Out(cli, "\n"); VCLI_Out(cli, "Type 'help' for command list.\n"); VCLI_Out(cli, "Type 'quit' to close CLI session.\n"); diff --git a/include/vcs.h b/include/vcs.h index f764100..d3a1585 100644 --- a/include/vcs.h +++ b/include/vcs.h @@ -29,4 +29,5 @@ */ /* from libvarnish/version.c */ +extern const char *VCS_version; void VCS_Message(const char *); diff --git a/lib/libvarnish/version.c b/lib/libvarnish/version.c index 2220a69..b7b2edc 100644 --- a/lib/libvarnish/version.c +++ b/lib/libvarnish/version.c @@ -36,11 +36,13 @@ #include "vcs.h" #include "vcs_version.h" +const char *VCS_version = + PACKAGE_TARNAME "-" PACKAGE_VERSION " revision " VCS_Version; + void VCS_Message(const char *progname) { - fprintf(stderr, "%s (%s-%s revision %s)\n", progname, - PACKAGE_TARNAME, PACKAGE_VERSION, VCS_Version); + fprintf(stderr, "%s (%s)\n", progname, VCS_version); fprintf(stderr, "Copyright (c) 2006 Verdens Gang AS\n"); fprintf(stderr, "Copyright (c) 2006-2011 Varnish Software AS\n"); } From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] 38a7199 Add updated Solaris Sandbox from Nils Goroll Message-ID: commit 38a7199215f5f76ef771f55e44d25ae11d3b3f06 Author: Poul-Henning Kamp Date: Mon Aug 6 08:28:52 2012 +0000 Add updated Solaris Sandbox from Nils Goroll (untested by me) diff --git a/bin/varnishd/mgt/mgt_sandbox.c b/bin/varnishd/mgt/mgt_sandbox.c index bf9802f..069d660 100644 --- a/bin/varnishd/mgt/mgt_sandbox.c +++ b/bin/varnishd/mgt/mgt_sandbox.c @@ -57,8 +57,7 @@ /*--------------------------------------------------------------------*/ -/* Waive all privileges in the child, it does not need any */ - +#ifndef HAVE_SETPPRIV static void __match_proto__(mgt_sandbox_f) mgt_sandbox_unix(enum sandbox_e who) { @@ -70,6 +69,7 @@ mgt_sandbox_unix(enum sandbox_e who) REPORT0(LOG_INFO, "Not running as root, no priv-sep"); } } +#endif /*--------------------------------------------------------------------*/ @@ -90,7 +90,7 @@ mgt_sandbox_linux(enum sandbox_e who) /*--------------------------------------------------------------------*/ mgt_sandbox_f *mgt_sandbox = -#ifdef HAVE_SETPRIV +#ifdef HAVE_SETPPRIV mgt_sandbox_solaris; #elif defined (__linux__) mgt_sandbox_linux; diff --git a/bin/varnishd/mgt/mgt_sandbox_solaris.c b/bin/varnishd/mgt/mgt_sandbox_solaris.c index e043fef..d443cc0 100644 --- a/bin/varnishd/mgt/mgt_sandbox_solaris.c +++ b/bin/varnishd/mgt/mgt_sandbox_solaris.c @@ -1,5 +1,6 @@ /*- * Copyright (c) 2006-2011 Varnish Software AS + * Copyright (c) 2011-2012 UPLEX - Nils Goroll Systemoptimierung * All rights reserved. * * Author: Poul-Henning Kamp @@ -38,6 +39,7 @@ #include #endif #include +#include #include #include #include @@ -95,49 +97,113 @@ * */ -/* effective during runtime of the child */ -static inline void -mgt_sandbox_solaris_add_effective(priv_set_t *pset) +static void +mgt_sandbox_solaris_add_inheritable(priv_set_t *pset, enum sandbox_e who) { - /* PSARC/2009/685 - 8eca52188202 - onnv_132 */ - priv_addset(pset, "net_access"); + switch (who) { + case SANDBOX_VCC: + break; + case SANDBOX_CC: + priv_addset(pset, "proc_exec"); + priv_addset(pset, "proc_fork"); + /* PSARC/2009/378 - 63678502e95e - onnv_140 */ + priv_addset(pset, "file_read"); + priv_addset(pset, "file_write"); + break; + case SANDBOX_VCLLOAD: + break; + case SANDBOX_WORKER: + break; + default: + REPORT(LOG_ERR, "INCOMPLETE AT: %s(%d)\n", __func__, __LINE__); + exit(1); + } +} + +/* + * effective is initialized from inheritable (see mgt_sandbox_solaris_waive) + * so only additionally required privileges need to be added here + */ - /* PSARC/2009/378 - 63678502e95e - onnv_140 */ - priv_addset(pset, "file_read"); - priv_addset(pset, "file_write"); +static void +mgt_sandbox_solaris_add_effective(priv_set_t *pset, enum sandbox_e who) +{ + switch (who) { + case SANDBOX_VCC: + /* PSARC/2009/378 - 63678502e95e - onnv_140 */ + priv_addset(pset, "file_write"); + break; + case SANDBOX_CC: + break; + case SANDBOX_VCLLOAD: + /* PSARC/2009/378 - 63678502e95e - onnv_140 */ + priv_addset(pset, "file_read"); + case SANDBOX_WORKER: + /* PSARC/2009/685 - 8eca52188202 - onnv_132 */ + priv_addset(pset, "net_access"); + /* PSARC/2009/378 - 63678502e95e - onnv_140 */ + priv_addset(pset, "file_read"); + priv_addset(pset, "file_write"); + break; + default: + REPORT(LOG_ERR, "INCOMPLETE AT: %s(%d)\n", __func__, __LINE__); + exit(1); + } } -/* permitted during runtime of the child - for privilege bracketing */ -static inline void -mgt_sandbox_solaris_add_permitted(priv_set_t *pset) +/* + * permitted is initialized from effective (see mgt_sandbox_solaris_waive) + * so only additionally required privileges need to be added here + */ + +static void +mgt_sandbox_solaris_add_permitted(priv_set_t *pset, enum sandbox_e who) { - /* for raising limits in cache_waiter_ports.c */ - priv_addset(pset, PRIV_SYS_RESOURCE); + switch (who) { + case SANDBOX_VCC: + case SANDBOX_CC: + case SANDBOX_VCLLOAD: + break; + case SANDBOX_WORKER: + /* for raising limits in cache_waiter_ports.c */ + priv_addset(pset, PRIV_SYS_RESOURCE); + break; + default: + REPORT(LOG_ERR, "INCOMPLETE AT: %s(%d)\n", __func__, __LINE__); + exit(1); + } } -/* effective during mgt_sandbox */ -static inline void -mgt_sandbox_solaris_add_initial(priv_set_t *pset) +/* + * additional privileges needed by mgt_sandbox_solaris_privsep - + * will get waived in mgt_sandbox_solaris_waive + */ +static void +mgt_sandbox_solaris_add_initial(priv_set_t *pset, enum sandbox_e who) { + (void)who; + /* for setgid/setuid */ priv_addset(pset, PRIV_PROC_SETID); } /* * if we are not yet privilege-aware already (ie we have been started - * not-privilege aware wird euid 0), we need to grab any additional privileges - * needed during mgt_standbox, until we reduce to least privileges in - * mgt_sandbox_waive, otherwise we would loose them with setuid() + * not-privilege aware with euid 0), we try to grab any privileges we + * will need later. + * We will reduce to least privileges in mgt_sandbox_solaris_waive + * + * We need to become privilege-aware to avoid setuid resetting them. */ -void -mgt_sandbox_solaris_init(void) +static void +mgt_sandbox_solaris_init(enum sandbox_e who) { priv_set_t *priv_all; if (! (priv_all = priv_allocset())) { REPORT(LOG_ERR, - "Child start warning: " + "Sandbox warning: " " mgt_sandbox_init - priv_allocset failed: errno=%d (%s)", errno, strerror(errno)); return; @@ -145,9 +211,10 @@ mgt_sandbox_solaris_init(void) priv_emptyset(priv_all); - mgt_sandbox_solaris_add_effective(priv_all); - mgt_sandbox_solaris_add_permitted(priv_all); - mgt_sandbox_solaris_add_initial(priv_all); + mgt_sandbox_solaris_add_inheritable(priv_all, who); + mgt_sandbox_solaris_add_effective(priv_all, who); + mgt_sandbox_solaris_add_permitted(priv_all, who); + mgt_sandbox_solaris_add_initial(priv_all, who); setppriv(PRIV_ON, PRIV_PERMITTED, priv_all); setppriv(PRIV_ON, PRIV_EFFECTIVE, priv_all); @@ -156,9 +223,11 @@ mgt_sandbox_solaris_init(void) priv_freeset(priv_all); } -void -mgt_sandbox_solaris_privsep(void) +static void +mgt_sandbox_solaris_privsep(enum sandbox_e who) { + (void)who; + if (priv_ineffect(PRIV_PROC_SETID)) { if (getgid() != mgt_param.gid) XXXAZ(setgid(mgt_param.gid)); @@ -187,8 +256,8 @@ mgt_sandbox_solaris_privsep(void) * We should keep sys_resource in P in order to adjust our limits if we need to */ -void -mgt_sandbox_solaris_fini(void) +static void +mgt_sandbox_solaris_waive(enum sandbox_e who) { priv_set_t *effective, *inheritable, *permitted; @@ -196,19 +265,22 @@ mgt_sandbox_solaris_fini(void) !(inheritable = priv_allocset()) || !(permitted = priv_allocset())) { REPORT(LOG_ERR, - "Child start warning: " + "Sandbox warning: " " mgt_sandbox_waive - priv_allocset failed: errno=%d (%s)", errno, strerror(errno)); return; } + /* simple scheme: (inheritable subset-of effective) subset-of permitted */ + priv_emptyset(inheritable); + mgt_sandbox_solaris_add_inheritable(inheritable, who); - priv_emptyset(effective); - mgt_sandbox_solaris_add_effective(effective); + priv_copyset(inheritable, effective); + mgt_sandbox_solaris_add_effective(effective, who); priv_copyset(effective, permitted); - mgt_sandbox_solaris_add_permitted(permitted); + mgt_sandbox_solaris_add_permitted(permitted, who); /* * invert the sets and clear privileges such that setppriv will always @@ -221,7 +293,7 @@ mgt_sandbox_solaris_fini(void) #define SETPPRIV(which, set) \ if (setppriv(PRIV_OFF, which, set)) \ REPORT(LOG_ERR, \ - "Child start warning: " \ + "Sandbox warning: " \ " Waiving privileges failed on %s: errno=%d (%s)", \ #which, errno, strerror(errno)); @@ -233,6 +305,14 @@ mgt_sandbox_solaris_fini(void) priv_freeset(inheritable); priv_freeset(effective); + priv_freeset(permitted); } +void __match_proto__(mgt_sandbox_f) +mgt_sandbox_solaris(enum sandbox_e who) +{ + mgt_sandbox_solaris_init(who); + mgt_sandbox_solaris_privsep(who); + mgt_sandbox_solaris_waive(who); +} #endif /* HAVE_SETPPRIV */ From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] 57412eb Disable CLANG for now, until some auto* magic happens to remove the -g option from linker-commands. Message-ID: commit 57412ebfc088ec242e93722337f110c0f40f79b1 Author: Poul-Henning Kamp Date: Mon Aug 6 09:58:21 2012 +0000 Disable CLANG for now, until some auto* magic happens to remove the -g option from linker-commands. diff --git a/autogen.des b/autogen.des index 58f3c2b..3a706f3 100755 --- a/autogen.des +++ b/autogen.des @@ -9,6 +9,10 @@ make -k distclean > /dev/null 2>&1 || true # Prefer CLANG if we have it, and have not given preferences if [ -f /usr/bin/clang -a "x${CC}" = "x" ] ; then CC=clang + + # XXX: clang complains about -g in linker invocations + # XXX: so fall back to gcc for now... + CC=gcc export CC fi From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] fe2ad14 Finally split cache_center.c into a HTTP/1 and a Request Finite State Machine. Message-ID: commit fe2ad14cdc379db7c5de9454ac1d8135849e1af3 Author: Poul-Henning Kamp Date: Mon Aug 6 09:58:49 2012 +0000 Finally split cache_center.c into a HTTP/1 and a Request Finite State Machine. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index bd91dd7..967d3bb 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -17,7 +17,6 @@ varnishd_SOURCES = \ cache/cache_backend_poll.c \ cache/cache_ban.c \ cache/cache_busyobj.c \ - cache/cache_center.c \ cache/cache_cli.c \ cache/cache_dir.c \ cache/cache_dir_dns.c \ @@ -31,6 +30,7 @@ varnishd_SOURCES = \ cache/cache_gzip.c \ cache/cache_hash.c \ cache/cache_http.c \ + cache/cache_http1_fsm.c \ cache/cache_httpconn.c \ cache/cache_lck.c \ cache/cache_main.c \ @@ -38,6 +38,7 @@ varnishd_SOURCES = \ cache/cache_panic.c \ cache/cache_pipe.c \ cache/cache_pool.c \ + cache/cache_req_fsm.c \ cache/cache_response.c \ cache/cache_rfc2616.c \ cache/cache_session.c \ diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c deleted file mode 100644 index dc7b4cc..0000000 --- a/bin/varnishd/cache/cache_center.c +++ /dev/null @@ -1,1662 +0,0 @@ -/*- - * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2011 Varnish Software AS - * All rights reserved. - * - * Author: Poul-Henning Kamp - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * This file contains the two central state machine for pushing - * sessions and requests. - * - * The first part of the file, entrypoint CNT_Session() and down to - * the ==== separator, is concerned with sessions. When a session has - * a request to deal with, it calls into the second half of the file. - * This part is for all practical purposes HTTP/1.x specific. - * - * The second part of the file, entrypoint CNT_Request() and below the - * ==== separator, is intended to (over time) be(ome) protocol agnostic. - * We already use this now with ESI:includes, which are for all relevant - * purposes a different "protocol" - * - * A special complication is the fact that we can suspend processing of - * a request when hash-lookup finds a busy objhdr. - * - * Since the states are rather nasty in detail, I have decided to embedd - * a dot(1) graph in the source code comments. So to see the big picture, - * extract the DOT lines and run though dot(1), for instance with the - * command: - * sed -n '/^DOT/s///p' cache/cache_center.c | dot -Tps > /tmp/_.ps - */ - -/* -DOT digraph vcl_center { -xDOT page="8.2,11.5" -DOT size="7.2,10.5" -DOT margin="0.5" -DOT center="1" -DOT acceptor [ -DOT shape=hexagon -DOT label="Request received" -DOT ] -DOT ERROR [shape=plaintext] -DOT RESTART [shape=plaintext] -DOT acceptor -> first [style=bold,color=green] - */ - -#include "config.h" - -#include -#include -#include -#include - -#include "cache.h" - -#include "hash/hash_slinger.h" -#include "vcl.h" -#include "vcli_priv.h" -#include "vsha256.h" -#include "vtcp.h" -#include "vtim.h" - -#ifndef HAVE_SRANDOMDEV -#include "compat/srandomdev.h" -#endif - -static unsigned xids; - -/*-------------------------------------------------------------------- - * WAIT - * Collect the request from the client. - * -DOT subgraph xcluster_wait { -DOT wait [ -DOT shape=box -DOT label="cnt_sess_wait:\nwait for\ncomplete\nrequest" -DOT ] -DOT herding [shape=hexagon] -DOT wait -> start [label="got req",style=bold,color=green] -DOT wait -> "SES_Delete()" [label="errors"] -DOT wait -> herding [label="timeout_linger"] -DOT herding -> wait [label="fd read_ready"] -DOT } - */ - -static int -cnt_sess_wait(struct sess *sp, struct worker *wrk, struct req *req) -{ - int j, tmo; - struct pollfd pfd[1]; - double now, when; - enum sess_close why = SC_NULL; - enum htc_status_e hs; - - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - - assert(req->sp == sp); - - - AZ(req->vcl); - AZ(req->obj); - AZ(req->esi_level); - assert(req->xid == 0); - assert(isnan(req->t_req)); - assert(isnan(req->t_resp)); - - tmo = (int)(1e3 * cache_param->timeout_linger); - while (1) { - pfd[0].fd = sp->fd; - pfd[0].events = POLLIN; - pfd[0].revents = 0; - j = poll(pfd, 1, tmo); - assert(j >= 0); - now = VTIM_real(); - if (j != 0) - hs = HTC_Rx(req->htc); - else - hs = HTC_Complete(req->htc); - if (hs == HTC_COMPLETE) { - /* Got it, run with it */ - req->t_req = now; - return (0); - } else if (hs == HTC_ERROR_EOF) { - why = SC_REM_CLOSE; - break; - } else if (hs == HTC_OVERFLOW) { - why = SC_RX_OVERFLOW; - break; - } else if (hs == HTC_ALL_WHITESPACE) { - /* Nothing but whitespace */ - when = sp->t_idle + cache_param->timeout_idle; - if (when < now) { - why = SC_RX_TIMEOUT; - break; - } - when = sp->t_idle + cache_param->timeout_linger; - tmo = (int)(1e3 * (when - now)); - if (when < now || tmo == 0) { - req->t_req = NAN; - wrk->stats.sess_herd++; - SES_ReleaseReq(req); - WAIT_Enter(sp); - return (1); - } - } else { - /* Working on it */ - if (isnan(req->t_req)) - req->t_req = now; - when = req->t_req + cache_param->timeout_req; - tmo = (int)(1e3 * (when - now)); - if (when < now || tmo == 0) { - why = SC_RX_TIMEOUT; - break; - } - } - } - SES_ReleaseReq(req); - assert(why != SC_NULL); - SES_Delete(sp, why, now); - return (1); -} - -/*-------------------------------------------------------------------- - * This is the final state, figure out if we should close or recycle - * the client connection - * -DOT DONE [ -DOT shape=record -DOT label="{cnt_done:|Request completed}" -DOT ] -DOT ESI_RESP [ shape=hexagon ] -DOT DONE -> start [label="full pipeline"] -DOT DONE -> wait -DOT DONE -> ESI_RESP - */ - -enum cnt_sess_done_ret { - SESS_DONE_RET_GONE, - SESS_DONE_RET_WAIT, - SESS_DONE_RET_START, -}; - -static enum cnt_sess_done_ret -cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) -{ - - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CHECK_OBJ_ORNULL(req->vcl, VCL_CONF_MAGIC); - - AZ(req->obj); - AZ(req->busyobj); - req->director = NULL; - req->restarts = 0; - - AZ(req->esi_level); - - if (req->vcl != NULL) { - if (wrk->vcl != NULL) - VCL_Rel(&wrk->vcl); - wrk->vcl = req->vcl; - req->vcl = NULL; - } - - sp->t_idle = W_TIM_real(wrk); - if (req->xid == 0) - req->t_resp = sp->t_idle; - req->xid = 0; - VSL_Flush(req->vsl, 0); - - req->t_req = NAN; - req->t_resp = NAN; - - req->req_bodybytes = 0; - - req->hash_always_miss = 0; - req->hash_ignore_busy = 0; - - if (sp->fd >= 0 && req->doclose != SC_NULL) - SES_Close(sp, req->doclose); - - if (sp->fd < 0) { - wrk->stats.sess_closed++; - AZ(req->vcl); - SES_ReleaseReq(req); - SES_Delete(sp, SC_NULL, NAN); - return (SESS_DONE_RET_GONE); - } - - if (wrk->stats.client_req >= cache_param->wthread_stats_rate) - WRK_SumStat(wrk); - - WS_Reset(req->ws, NULL); - WS_Reset(wrk->aws, NULL); - req->vxid = VXID_Get(&wrk->vxid_pool); - - if (HTC_Reinit(req->htc) == HTC_COMPLETE) { - req->t_req = sp->t_idle; - wrk->stats.sess_pipeline++; - return (SESS_DONE_RET_START); - } else { - if (Tlen(req->htc->rxbuf)) - wrk->stats.sess_readahead++; - return (SESS_DONE_RET_WAIT); - } -} - -/*-------------------------------------------------------------------- - */ - -void -CNT_Session(struct worker *wrk, struct req *req) -{ - int done; - struct sess *sp; - enum cnt_sess_done_ret sdr; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - sp = req->sp; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - - /* - * Whenever we come in from the acceptor or waiter, we need to set - * blocking mode, but there is no point in setting it when we come from - * ESI or when a parked sessions returns. - * It would be simpler to do this in the acceptor or waiter, but we'd - * rather do the syscall in the worker thread. - * On systems which return errors for ioctl, we close early - */ - if (sp->sess_step == S_STP_NEWREQ && VTCP_blocking(sp->fd)) { - if (errno == ECONNRESET) - SES_Close(sp, SC_REM_CLOSE); - else - SES_Close(sp, SC_TX_ERROR); - sdr = cnt_sess_done(sp, wrk, req); - assert(sdr == SESS_DONE_RET_GONE); - return; - } - - if (sp->sess_step == S_STP_NEWREQ) { - HTC_Init(req->htc, req->ws, sp->fd, req->vsl, - cache_param->http_req_size, - cache_param->http_req_hdr_len); - } - - while (1) { - /* - * Possible entrance states - */ - - assert( - sp->sess_step == S_STP_NEWREQ || - req->req_step == R_STP_LOOKUP || - req->req_step == R_STP_START); - - if (sp->sess_step == S_STP_WORKING) { - done = CNT_Request(wrk, req); - if (done == 2) - return; - assert(done == 1); - sdr = cnt_sess_done(sp, wrk, req); - switch (sdr) { - case SESS_DONE_RET_GONE: - return; - case SESS_DONE_RET_WAIT: - sp->sess_step = S_STP_NEWREQ; - break; - case SESS_DONE_RET_START: - sp->sess_step = S_STP_WORKING; - req->req_step = R_STP_START; - break; - default: - WRONG("Illegal enum cnt_sess_done_ret"); - } - } - - if (sp->sess_step == S_STP_NEWREQ) { - done = cnt_sess_wait(sp, wrk, req); - if (done) - return; - sp->sess_step = S_STP_WORKING; - req->req_step = R_STP_START; - } - } -} - -/*====================================================================*/ - - -/*-------------------------------------------------------------------- - * We have a refcounted object on the session, and possibly the busyobj - * which is fetching it, prepare a response. - * -DOT subgraph xcluster_prepresp { -DOT prepresp [ -DOT shape=record -DOT label="{cnt_prepresp:|Filter obj.-\>resp.|{vcl_deliver\{\}|{req.|resp.}}|{error?|restart?}|stream ?}" -DOT ] -DOT prepresp -> deliver [style=bold,color=green,label=deliver] -DOT prepresp -> deliver [style=bold,color=red] -DOT prepresp -> deliver [style=bold,color=blue] -DOT } - * - */ - -static int -cnt_prepresp(struct worker *wrk, struct req *req) -{ - struct busyobj *bo; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - bo = req->busyobj; - CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); - - CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); - CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - - req->res_mode = 0; - - if (bo == NULL) { - if (!req->disable_esi && req->obj->esidata != NULL) { - /* In ESI mode, we can't know the aggregate length */ - req->res_mode &= ~RES_LEN; - req->res_mode |= RES_ESI; - } else { - req->res_mode |= RES_LEN; - } - } else { - AZ(bo->do_esi); - } - - if (req->esi_level > 0) { - /* Included ESI object, always CHUNKED or EOF */ - req->res_mode &= ~RES_LEN; - req->res_mode |= RES_ESI_CHILD; - } - - if (cache_param->http_gzip_support && req->obj->gziped && - !RFC2616_Req_Gzip(req->http)) { - /* - * We don't know what it uncompresses to - * XXX: we could cache that - */ - req->res_mode &= ~RES_LEN; - req->res_mode |= RES_GUNZIP; - } - - if (!(req->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) { - /* We havn't chosen yet, do so */ - if (!req->wantbody) { - /* Nothing */ - } else if (req->http->protover >= 11) { - req->res_mode |= RES_CHUNKED; - } else { - req->res_mode |= RES_EOF; - req->doclose = SC_TX_EOF; - } - } - - req->t_resp = W_TIM_real(wrk); - if (req->obj->objcore->objhead != NULL) { - if ((req->t_resp - req->obj->last_lru) > - cache_param->lru_timeout && - EXP_Touch(req->obj->objcore)) - req->obj->last_lru = req->t_resp; - if (!cache_param->obj_readonly) - req->obj->last_use = req->t_resp; /* XXX: locking ? */ - } - HTTP_Setup(req->resp, req->ws, req->vsl, HTTP_Resp); - RES_BuildHttp(req); - - VCL_deliver_method(req); - switch (req->handling) { - case VCL_RET_DELIVER: - break; - case VCL_RET_RESTART: - if (req->restarts >= cache_param->max_restarts) - break; - if (bo != NULL) { - AN(bo->do_stream); - (void)HSH_Deref(&wrk->stats, NULL, &req->obj); - VBO_DerefBusyObj(wrk, &req->busyobj); - } else { - (void)HSH_Deref(&wrk->stats, NULL, &req->obj); - } - AZ(req->obj); - http_Teardown(req->resp); - req->req_step = R_STP_RESTART; - return (0); - default: - WRONG("Illegal action in vcl_deliver{}"); - } - req->req_step = R_STP_DELIVER; - return (0); -} - -/*-------------------------------------------------------------------- - * Deliver an already stored object - * -DOT subgraph xcluster_deliver { -DOT deliver [ -DOT shape=record -DOT label="{cnt_deliver:|Send body}" -DOT ] -DOT } -DOT deliver -> DONE [style=bold,color=green] -DOT deliver -> DONE [style=bold,color=red] -DOT deliver -> DONE [style=bold,color=blue] - * - */ - -static int -cnt_deliver(struct worker *wrk, struct req *req) -{ - struct busyobj *bo; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); - bo = req->busyobj; - CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); - - if (bo != NULL) { - while (bo->state < BOS_FAILED) - (void)usleep(10000); - assert(bo->state >= BOS_FAILED); - - if (bo->state == BOS_FAILED) { - HSH_Deref(&wrk->stats, NULL, &req->obj); - VBO_DerefBusyObj(wrk, &req->busyobj); - req->err_code = 503; - req->req_step = R_STP_ERROR; - return (0); - } - VBO_DerefBusyObj(wrk, &req->busyobj); - } - - AZ(req->busyobj); - req->director = NULL; - req->restarts = 0; - - RES_WriteObj(req); - - /* No point in saving the body if it is hit-for-pass */ - if (req->obj->objcore->flags & OC_F_PASS) - STV_Freestore(req->obj); - - assert(WRW_IsReleased(wrk)); - (void)HSH_Deref(&wrk->stats, NULL, &req->obj); - http_Teardown(req->resp); - return (1); -} -/*-------------------------------------------------------------------- - * Emit an error - * -DOT subgraph xcluster_error { -DOT vcl_error [ -DOT shape=record -DOT label="vcl_error()|resp." -DOT ] -DOT ERROR -> vcl_error -DOT vcl_error-> prepresp [label=deliver] -DOT } -DOT vcl_error-> rsterr [label="restart",color=purple] -DOT rsterr [label="RESTART",shape=plaintext] - */ - -static int -cnt_error(struct worker *wrk, struct req *req) -{ - struct http *h; - struct busyobj *bo; - char date[40]; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - AZ(req->objcore); - AZ(req->obj); - AZ(req->busyobj); - - bo = VBO_GetBusyObj(wrk); - req->busyobj = bo; - bo->vsl->wid = req->sp->vsl_id; - AZ(bo->stats); - bo->stats = &wrk->stats; - req->objcore = HSH_NewObjCore(wrk); - req->obj = STV_NewObject(bo, &req->objcore, - TRANSIENT_STORAGE, cache_param->http_resp_size, - (uint16_t)cache_param->http_max_hdr); - bo->stats = NULL; - if (req->obj == NULL) { - req->doclose = SC_OVERLOAD; - req->director = NULL; - http_Teardown(bo->beresp); - http_Teardown(bo->bereq); - return(1); - } - CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); - req->obj->xid = req->xid; - req->obj->exp.entered = req->t_req; - - h = req->obj->http; - - if (req->err_code < 100 || req->err_code > 999) - req->err_code = 501; - - http_PutProtocol(h, "HTTP/1.1"); - http_PutStatus(h, req->err_code); - VTIM_format(W_TIM_real(wrk), date); - http_PrintfHeader(h, "Date: %s", date); - http_SetHeader(h, "Server: Varnish"); - - if (req->err_reason != NULL) - http_PutResponse(h, req->err_reason); - else - http_PutResponse(h, http_StatusMessage(req->err_code)); - VCL_error_method(req); - - if (req->handling == VCL_RET_RESTART && - req->restarts < cache_param->max_restarts) { - HSH_Drop(wrk, &req->obj); - VBO_DerefBusyObj(wrk, &req->busyobj); - req->req_step = R_STP_RESTART; - return (0); - } else if (req->handling == VCL_RET_RESTART) - req->handling = VCL_RET_DELIVER; - - - /* We always close when we take this path */ - req->doclose = SC_TX_ERROR; - req->wantbody = 1; - - assert(req->handling == VCL_RET_DELIVER); - req->err_code = 0; - req->err_reason = NULL; - http_Teardown(bo->bereq); - VBO_DerefBusyObj(wrk, &req->busyobj); - req->req_step = R_STP_PREPRESP; - return (0); -} - -/*-------------------------------------------------------------------- - * Fetch response headers from the backend - * -DOT subgraph xcluster_fetch { -DOT fetch [ -DOT shape=record -DOT label="{cnt_fetch:|fetch hdr\nfrom backend|(find obj.ttl)|{vcl_fetch\{\}|{req.|bereq.|beresp.}}|{error?|restart?}}" -DOT ] -DOT } -DOT fetch -> fetchbody [style=bold,color=red] -DOT fetch -> fetchbody [style=bold,color=blue] - */ - -static int -cnt_fetch(struct worker *wrk, struct req *req) -{ - int i, need_host_hdr; - struct busyobj *bo; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - - CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - bo = req->busyobj; - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - - AN(req->director); - AZ(bo->vbc); - AZ(bo->should_close); - AZ(req->storage_hint); - - HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp); - - need_host_hdr = !http_GetHdr(bo->bereq, H_Host, NULL); - - req->acct_req.fetch++; - - i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL); - /* - * If we recycle a backend connection, there is a finite chance - * that the backend closed it before we get a request to it. - * Do a single retry in that case. - */ - if (i == 1) { - VSC_C_main->backend_retry++; - i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL); - } - - if (i) { - req->handling = VCL_RET_ERROR; - req->err_code = 503; - } else { - /* - * These two headers can be spread over multiple actual headers - * and we rely on their content outside of VCL, so collect them - * into one line here. - */ - http_CollectHdr(bo->beresp, H_Cache_Control); - http_CollectHdr(bo->beresp, H_Vary); - - /* - * Figure out how the fetch is supposed to happen, before the - * headers are adultered by VCL - * NB: Also sets other wrk variables - */ - bo->body_status = RFC2616_Body(bo, &wrk->stats); - - req->err_code = http_GetStatus(bo->beresp); - - /* - * What does RFC2616 think about TTL ? - */ - EXP_Clr(&bo->exp); - bo->exp.entered = W_TIM_real(wrk); - RFC2616_Ttl(bo, req->xid); - - /* pass from vclrecv{} has negative TTL */ - if (req->objcore->objhead == NULL) - bo->exp.ttl = -1.; - - AZ(bo->do_esi); - AZ(bo->do_pass); - - VCL_fetch_method(req); - - if (bo->do_pass) - req->objcore->flags |= OC_F_PASS; - - switch (req->handling) { - case VCL_RET_DELIVER: - req->req_step = R_STP_FETCHBODY; - return (0); - default: - break; - } - - /* We are not going to fetch the body, Close the connection */ - VDI_CloseFd(&bo->vbc); - } - - /* Clean up partial fetch */ - AZ(bo->vbc); - - if (req->objcore->objhead != NULL || req->handling == VCL_RET_ERROR) { - CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); - AZ(HSH_Deref(&wrk->stats, req->objcore, NULL)); - req->objcore = NULL; - } - assert(bo->refcount == 2); - VBO_DerefBusyObj(wrk, &bo); - VBO_DerefBusyObj(wrk, &req->busyobj); - req->director = NULL; - req->storage_hint = NULL; - - switch (req->handling) { - case VCL_RET_RESTART: - req->req_step = R_STP_RESTART; - return (0); - case VCL_RET_ERROR: - req->req_step = R_STP_ERROR; - return (0); - default: - WRONG("Illegal action in vcl_fetch{}"); - } -} - -/*-------------------------------------------------------------------- - * Prepare to fetch body from backend - * -DOT subgraph xcluster_body { -DOT fetchbody [ -DOT shape=record -DOT label="{cnt_fetchbody:|start fetch_thread}" -DOT ] -DOT } -DOT fetchbody:out -> prepresp [style=bold,color=red] -DOT fetchbody:out -> prepresp [style=bold,color=blue] - */ - -static int -cnt_fetchbody(struct worker *wrk, struct req *req) -{ - struct http *hp, *hp2; - char *b; - uint16_t nhttp; - unsigned l; - struct vsb *vary = NULL; - int varyl = 0, pass; - struct busyobj *bo; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - bo = req->busyobj; - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - - assert(req->handling == VCL_RET_DELIVER); - - if (req->objcore->objhead == NULL) { - /* This is a pass from vcl_recv */ - pass = 1; - /* VCL may have fiddled this, but that doesn't help */ - bo->exp.ttl = -1.; - } else if (bo->do_pass) { - pass = 1; - } else { - /* regular object */ - pass = 0; - } - - /* - * The VCL variables beresp.do_g[un]zip tells us how we want the - * object processed before it is stored. - * - * The backend Content-Encoding header tells us what we are going - * to receive, which we classify in the following three classes: - * - * "Content-Encoding: gzip" --> object is gzip'ed. - * no Content-Encoding --> object is not gzip'ed. - * anything else --> do nothing wrt gzip - * - */ - - /* We do nothing unless the param is set */ - if (!cache_param->http_gzip_support) - bo->do_gzip = bo->do_gunzip = 0; - - bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); - - bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); - - /* It can't be both */ - assert(bo->is_gzip == 0 || bo->is_gunzip == 0); - - /* We won't gunzip unless it is gzip'ed */ - if (bo->do_gunzip && !bo->is_gzip) - bo->do_gunzip = 0; - - /* If we do gunzip, remove the C-E header */ - if (bo->do_gunzip) - http_Unset(bo->beresp, H_Content_Encoding); - - /* We wont gzip unless it is ungziped */ - if (bo->do_gzip && !bo->is_gunzip) - bo->do_gzip = 0; - - /* If we do gzip, add the C-E header */ - if (bo->do_gzip) - http_SetHeader(bo->beresp, "Content-Encoding: gzip"); - - /* But we can't do both at the same time */ - assert(bo->do_gzip == 0 || bo->do_gunzip == 0); - - /* ESI takes precedence and handles gzip/gunzip itself */ - if (bo->do_esi) - bo->vfp = &vfp_esi; - else if (bo->do_gunzip) - bo->vfp = &vfp_gunzip; - else if (bo->do_gzip) - bo->vfp = &vfp_gzip; - else if (bo->is_gzip) - bo->vfp = &vfp_testgzip; - - if (bo->do_esi || req->esi_level > 0) - bo->do_stream = 0; - if (!req->wantbody) - bo->do_stream = 0; - - /* No reason to try streaming a non-existing body */ - if (bo->body_status == BS_NONE) - bo->do_stream = 0; - - l = http_EstimateWS(bo->beresp, - pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp); - - /* Create Vary instructions */ - if (req->objcore->objhead != NULL) { - CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); - vary = VRY_Create(req, bo->beresp); - if (vary != NULL) { - varyl = VSB_len(vary); - assert(varyl > 0); - l += varyl; - } - } - - /* - * Space for producing a Content-Length: header including padding - * A billion gigabytes is enough for anybody. - */ - l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *); - - if (bo->exp.ttl < cache_param->shortlived || - req->objcore == NULL) - req->storage_hint = TRANSIENT_STORAGE; - - AZ(bo->stats); - bo->stats = &wrk->stats; - req->obj = STV_NewObject(bo, &req->objcore, req->storage_hint, l, - nhttp); - if (req->obj == NULL) { - /* - * Try to salvage the transaction by allocating a - * shortlived object on Transient storage. - */ - if (bo->exp.ttl > cache_param->shortlived) - bo->exp.ttl = cache_param->shortlived; - bo->exp.grace = 0.0; - bo->exp.keep = 0.0; - req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE, - l, nhttp); - } - bo->stats = NULL; - if (req->obj == NULL) { - req->err_code = 503; - req->req_step = R_STP_ERROR; - VDI_CloseFd(&bo->vbc); - VBO_DerefBusyObj(wrk, &req->busyobj); - return (0); - } - CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); - - req->storage_hint = NULL; - - AZ(bo->fetch_obj); - bo->fetch_obj = req->obj; - - if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) - req->obj->gziped = 1; - - if (vary != NULL) { - req->obj->vary = (void *)WS_Alloc(req->obj->http->ws, varyl); - AN(req->obj->vary); - memcpy(req->obj->vary, VSB_data(vary), varyl); - VRY_Validate(req->obj->vary); - VSB_delete(vary); - } - - req->obj->xid = req->xid; - req->obj->response = req->err_code; - WS_Assert(req->obj->ws_o); - - /* Filter into object */ - hp = bo->beresp; - hp2 = req->obj->http; - - hp2->logtag = HTTP_Obj; - http_FilterResp(hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS); - http_CopyHome(hp2); - - if (http_GetHdr(hp, H_Last_Modified, &b)) - req->obj->last_modified = VTIM_parse(b); - else - req->obj->last_modified = floor(bo->exp.entered); - - assert(WRW_IsReleased(wrk)); - - /* - * If we can deliver a 304 reply, we don't bother streaming. - * Notice that vcl_deliver{} could still nuke the headers - * that allow the 304, in which case we return 200 non-stream. - */ - if (req->obj->response == 200 && - req->http->conds && - RFC2616_Do_Cond(req)) - bo->do_stream = 0; - - /* - * Ready to fetch the body - */ - bo->fetch_task.func = FetchBody; - bo->fetch_task.priv = bo; - - assert(bo->refcount == 2); /* one for each thread */ - - if (req->obj->objcore->objhead != NULL) { - EXP_Insert(req->obj); - AN(req->obj->objcore->ban); - AZ(req->obj->ws_o->overflow); - HSH_Unbusy(&wrk->stats, req->obj->objcore); - } - - if (!bo->do_stream || - Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE)) - FetchBody(wrk, bo); - - if (req->obj->objcore->objhead != NULL) - HSH_Ref(req->obj->objcore); - - if (bo->state == BOS_FINISHED) { - VBO_DerefBusyObj(wrk, &req->busyobj); - } else if (bo->state == BOS_FAILED) { - /* handle early failures */ - HSH_Deref(&wrk->stats, NULL, &req->obj); - VBO_DerefBusyObj(wrk, &req->busyobj); - req->err_code = 503; - req->req_step = R_STP_ERROR; - return (0); - } - - assert(WRW_IsReleased(wrk)); - req->req_step = R_STP_PREPRESP; - return (0); -} - -/*-------------------------------------------------------------------- - * HIT - * We had a cache hit. Ask VCL, then march off as instructed. - * -DOT subgraph xcluster_hit { -DOT hit [ -DOT shape=record -DOT label="{cnt_hit:|{vcl_hit()|{req.|obj.}}|{error?|restart?}|{deliver?|pass?}}" -DOT ] -DOT } -XDOT hit:err -> err_hit [label="error"] -XDOT err_hit [label="ERROR",shape=plaintext] -XDOT hit:rst -> rst_hit [label="restart",color=purple] -XDOT rst_hit [label="RESTART",shape=plaintext] -DOT hit:pass -> pass [label=pass,style=bold,color=red] -DOT hit:del -> prepresp [label="deliver",style=bold,color=green] - */ - -static int -cnt_hit(struct worker *wrk, struct req *req) -{ - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - - CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); - CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - AZ(req->objcore); - AZ(req->busyobj); - - assert(!(req->obj->objcore->flags & OC_F_PASS)); - - VCL_hit_method(req); - - if (req->handling == VCL_RET_DELIVER) { - //AZ(req->busyobj->bereq->ws); - //AZ(req->busyobj->beresp->ws); - (void)FetchReqBody(req, 0); - req->req_step = R_STP_PREPRESP; - return (0); - } - - /* Drop our object, we won't need it */ - (void)HSH_Deref(&wrk->stats, NULL, &req->obj); - req->objcore = NULL; - - switch(req->handling) { - case VCL_RET_PASS: - req->req_step = R_STP_PASS; - return (0); - case VCL_RET_ERROR: - req->req_step = R_STP_ERROR; - return (0); - case VCL_RET_RESTART: - req->req_step = R_STP_RESTART; - return (0); - default: - WRONG("Illegal action in vcl_hit{}"); - } -} - -/*-------------------------------------------------------------------- - * LOOKUP - * Hash things together and look object up in hash-table. - * - * LOOKUP consists of two substates so that we can reenter if we - * encounter a busy object. - * -DOT subgraph xcluster_lookup { -DOT lookup [ -DOT shape=record -DOT label="{cnt_lookup:|hash lookup|{busy ?|miss ?}|{no|obj.f.pass?|yes}}" -DOT ] -DOT } -DOT lookup:busy -> lookup:top [label="(waitinglist)"] -DOT lookup:miss -> miss [style=bold,color=blue] -DOT lookup:no -> hit [style=bold,color=green] -DOT lookup:yes -> pass [style=bold,color=red] - */ - -static int -cnt_lookup(struct worker *wrk, struct req *req) -{ - struct objcore *oc; - struct object *o; - struct objhead *oh; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - AZ(req->objcore); - - CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - AZ(req->busyobj); - - VRY_Prep(req); - - AZ(req->objcore); - oc = HSH_Lookup(req); - if (oc == NULL) { - /* - * We lost the session to a busy object, disembark the - * worker thread. We return to STP_LOOKUP when the busy - * object has been unbusied, and still have the hash digest - * around to do the lookup with. - */ - return (2); - } - AZ(req->objcore); - - CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - oh = oc->objhead; - CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); - - /* If we inserted a new object it's a miss */ - if (oc->flags & OC_F_BUSY) { - CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC); - assert(oc->busyobj == req->busyobj); - wrk->stats.cache_miss++; - - if (req->vary_l != NULL) { - assert(oc->busyobj->vary == req->vary_b); - VRY_Validate(oc->busyobj->vary); - WS_ReleaseP(req->ws, (void*)req->vary_l); - } else { - AZ(oc->busyobj->vary); - WS_Release(req->ws, 0); - } - req->vary_b = NULL; - req->vary_l = NULL; - req->vary_e = NULL; - - req->objcore = oc; - req->req_step = R_STP_MISS; - return (0); - } - - /* We are not prepared to do streaming yet */ - XXXAZ(req->busyobj); - - o = oc_getobj(&wrk->stats, oc); - CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - req->obj = o; - - WS_Release(req->ws, 0); - req->vary_b = NULL; - req->vary_l = NULL; - req->vary_e = NULL; - - if (oc->flags & OC_F_PASS) { - wrk->stats.cache_hitpass++; - VSLb(req->vsl, SLT_HitPass, "%u", req->obj->xid); - (void)HSH_Deref(&wrk->stats, NULL, &req->obj); - AZ(req->objcore); - req->req_step = R_STP_PASS; - return (0); - } - - wrk->stats.cache_hit++; - VSLb(req->vsl, SLT_Hit, "%u", req->obj->xid); - req->req_step = R_STP_HIT; - return (0); -} - -/*-------------------------------------------------------------------- - * We had a miss, ask VCL, proceed as instructed - * -DOT subgraph xcluster_miss { -DOT miss [ -DOT shape=record -DOT label="{cnt_miss:|filter req.-\>bereq.|{vcl_miss\{\}|{req.*|bereq.*}}|{error?|restart?}|{pass?|fetch?}}" -DOT ] -DOT } -DOT miss:fetch -> fetch [label="fetch",style=bold,color=blue] -DOT miss:pass -> pass [label="pass",style=bold,color=red] -DOT - */ - -static int -cnt_miss(struct worker *wrk, struct req *req) -{ - struct busyobj *bo; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); - bo = req->busyobj; - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - AZ(req->obj); - - HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); - http_FilterReq(req, HTTPH_R_FETCH); - http_ForceGet(bo->bereq); - if (cache_param->http_gzip_support) { - /* - * We always ask the backend for gzip, even if the - * client doesn't grok it. We will uncompress for - * the minority of clients which don't. - */ - http_Unset(bo->bereq, H_Accept_Encoding); - http_SetHeader(bo->bereq, "Accept-Encoding: gzip"); - } - - VCL_miss_method(req); - - if (req->handling == VCL_RET_FETCH) { - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - req->req_step = R_STP_FETCH; - return (0); - } - - AZ(HSH_Deref(&wrk->stats, req->objcore, NULL)); - req->objcore = NULL; - http_Teardown(bo->bereq); - VBO_DerefBusyObj(wrk, &req->busyobj); - - switch(req->handling) { - case VCL_RET_ERROR: - req->req_step = R_STP_ERROR; - break; - case VCL_RET_PASS: - req->req_step = R_STP_PASS; - break; - case VCL_RET_RESTART: - req->req_step = R_STP_RESTART; - break; - default: - WRONG("Illegal action in vcl_miss{}"); - } - return (0); -} - -/*-------------------------------------------------------------------- - * Start pass processing by getting headers from backend, then - * continue in passbody. - * -DOT subgraph xcluster_pass { -DOT pass [ -DOT shape=record -DOT label="{cnt_pass:|(XXX: deref obj.)|filter req.*-\>bereq.|{vcl_pass\{\}|{req.*|bereq.*}}|{error?|restart?}|create anon obj}" -DOT ] -DOT } -DOT pass:pass -> fetch [style=bold, color=red] -XDOT pass:rst -> rst_pass [label="restart",color=purple] -XDOT rst_pass [label="RESTART",shape=plaintext] -XDOT pass:err -> err_pass [label="error"] -XDOT err_pass [label="ERROR",shape=plaintext] - */ - -static int -cnt_pass(struct worker *wrk, struct req *req) -{ - struct busyobj *bo; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - AZ(req->objcore); - AZ(req->obj); - AZ(req->busyobj); - - req->busyobj = VBO_GetBusyObj(wrk); - bo = req->busyobj; - bo->vsl->wid = req->sp->vsl_id; - bo->refcount = 2; - HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); - http_FilterReq(req, HTTPH_R_PASS); - - VCL_pass_method(req); - - if (req->handling == VCL_RET_ERROR) { - http_Teardown(bo->bereq); - VBO_DerefBusyObj(wrk, &req->busyobj); - req->req_step = R_STP_ERROR; - return (0); - } - assert(req->handling == VCL_RET_PASS); - req->acct_req.pass++; - req->req_step = R_STP_FETCH; - - req->objcore = HSH_NewObjCore(wrk); - req->objcore->busyobj = bo; - return (0); -} - -/*-------------------------------------------------------------------- - * Ship the request header to the backend unchanged, then pipe - * until one of the ends close the connection. - * -DOT subgraph xcluster_pipe { -DOT pipe [ -DOT shape=ellipse -DOT label="Filter req.->bereq." -DOT ] -DOT vcl_pipe [ -DOT shape=record -DOT label="vcl_pipe()|req.\nbereq\." -DOT ] -DOT pipe_do [ -DOT shape=ellipse -DOT label="send bereq.\npipe until close" -DOT ] -DOT vcl_pipe -> pipe_do [label="pipe",style=bold,color=orange] -DOT pipe -> vcl_pipe [style=bold,color=orange] -DOT } -DOT pipe_do -> DONE [style=bold,color=orange] -DOT vcl_pipe -> err_pipe [label="error"] -DOT err_pipe [label="ERROR",shape=plaintext] - */ - -static int -cnt_pipe(struct worker *wrk, struct req *req) -{ - struct busyobj *bo; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - AZ(req->busyobj); - - req->acct_req.pipe++; - req->busyobj = VBO_GetBusyObj(wrk); - bo = req->busyobj; - bo->vsl->wid = req->sp->vsl_id; - HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); - http_FilterReq(req, 0); - - VCL_pipe_method(req); - - if (req->handling == VCL_RET_ERROR) - INCOMPL(); - assert(req->handling == VCL_RET_PIPE); - - PipeRequest(req); - assert(WRW_IsReleased(wrk)); - http_Teardown(bo->bereq); - VBO_DerefBusyObj(wrk, &req->busyobj); - return (1); -} - -/*-------------------------------------------------------------------- - * -DOT subgraph xcluster_restart { -DOT restart [ -DOT shape=record -DOT label="{cnt_restart}" -DOT ] -DOT } -DOT RESTART -> restart [color=purple] -DOT restart -> recv [color=purple] - */ - -static int -cnt_restart(const struct worker *wrk, struct req *req) -{ - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - - req->director = NULL; - if (++req->restarts >= cache_param->max_restarts) { - req->err_code = 503; - req->req_step = R_STP_ERROR; - } else { - req->err_code = 0; - req->req_step = R_STP_RECV; - } - return (0); -} - -/*-------------------------------------------------------------------- - * RECV - * We have a complete request, set everything up and start it. - * We can come here both with a request from the client and with - * a interior request during ESI delivery. - * -DOT subgraph xcluster_recv { -DOT recv [ -DOT shape=record -DOT label="{cnt_recv:|{vcl_recv\{\}|req.*}|{pipe?|pass?|error?|lookup?}}" -DOT ] -DOT } -DOT subgraph xcluster_hash { -DOT hash [ -DOT shape=record -DOT label="{cnt_recv:|{vcl_hash\{\}|req.*}}" -DOT ] -DOT } -DOT ESI_REQ [ shape=hexagon ] -DOT ESI_REQ -> recv -DOT recv:pipe -> pipe [style=bold,color=orange] -DOT recv:pass -> pass [style=bold,color=red] -#DOT recv:error -> err_recv -#DOT err_recv [label="ERROR",shape=plaintext] -DOT recv:lookup -> hash [style=bold,color=green] -DOT hash -> lookup [label="hash",style=bold,color=green] - */ - -static int -cnt_recv(const struct worker *wrk, struct req *req) -{ - unsigned recv_handling; - struct SHA256Context sha256ctx; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - AZ(req->obj); - AZ(req->busyobj); - - /* By default we use the first backend */ - AZ(req->director); - req->director = req->vcl->director[0]; - AN(req->director); - - req->disable_esi = 0; - req->hash_always_miss = 0; - req->hash_ignore_busy = 0; - req->client_identity = NULL; - - http_CollectHdr(req->http, H_Cache_Control); - - VCL_recv_method(req); - recv_handling = req->handling; - - if (cache_param->http_gzip_support && - (recv_handling != VCL_RET_PIPE) && - (recv_handling != VCL_RET_PASS)) { - if (RFC2616_Req_Gzip(req->http)) { - http_Unset(req->http, H_Accept_Encoding); - http_SetHeader(req->http, "Accept-Encoding: gzip"); - } else { - http_Unset(req->http, H_Accept_Encoding); - } - } - - req->sha256ctx = &sha256ctx; /* so HSH_AddString() can find it */ - SHA256_Init(req->sha256ctx); - VCL_hash_method(req); - assert(req->handling == VCL_RET_HASH); - SHA256_Final(req->digest, req->sha256ctx); - req->sha256ctx = NULL; - - if (!strcmp(req->http->hd[HTTP_HDR_REQ].b, "HEAD")) - req->wantbody = 0; - else - req->wantbody = 1; - - switch(recv_handling) { - case VCL_RET_LOOKUP: - req->req_step = R_STP_LOOKUP; - return (0); - case VCL_RET_PIPE: - if (req->esi_level > 0) { - /* XXX: VSL something */ - INCOMPL(); - return (1); - } - req->req_step = R_STP_PIPE; - return (0); - case VCL_RET_PASS: - req->req_step = R_STP_PASS; - return (0); - case VCL_RET_ERROR: - req->req_step = R_STP_ERROR; - return (0); - default: - WRONG("Illegal action in vcl_recv{}"); - } -} - -/*-------------------------------------------------------------------- - * START - * First time we see a request - * -DOT start [ -DOT shape=box -DOT label="cnt_start:\nDissect request\nHandle expect" -DOT ] -DOT start -> recv [style=bold,color=green] -DOT start -> DONE [label=errors] - */ - -static int -cnt_start(struct worker *wrk, struct req *req) -{ - char *p; - const char *r = "HTTP/1.1 100 Continue\r\n\r\n"; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - AZ(req->restarts); - AZ(req->obj); - AZ(req->vcl); - AZ(req->esi_level); - assert(!isnan(req->t_req)); - - /* Update stats of various sorts */ - wrk->stats.client_req++; - req->acct_req.req++; - - /* Assign XID and log */ - req->xid = ++xids; /* XXX not locked */ - VSLb(req->vsl, SLT_ReqStart, "%s %s %u", - req->sp->addr, req->sp->port, req->xid); - - /* Borrow VCL reference from worker thread */ - VCL_Refresh(&wrk->vcl); - req->vcl = wrk->vcl; - wrk->vcl = NULL; - - EXP_Clr(&req->exp); - - HTTP_Setup(req->http, req->ws, req->vsl, HTTP_Req); - req->err_code = http_DissectRequest(req); - - /* If we could not even parse the request, just close */ - if (req->err_code == 400) { - SES_Close(req->sp, SC_RX_JUNK); - return (1); - } - - req->ws_req = WS_Snapshot(req->ws); - - req->doclose = http_DoConnection(req->http); - - /* - * We want to deal with Expect: headers the first time we - * attempt the request, and remove them before we move on. - */ - if (req->err_code == 0 && http_GetHdr(req->http, H_Expect, &p)) { - if (strcasecmp(p, "100-continue")) { - req->err_code = 417; - } else if (strlen(r) != write(req->sp->fd, r, strlen(r))) { - SES_Close(req->sp, SC_REM_CLOSE); - return (1); - } - } - http_Unset(req->http, H_Expect); - - /* XXX: pull in req-body and make it available instead. */ - req->reqbodydone = 0; - - HTTP_Copy(req->http0, req->http); /* Copy for restart/ESI use */ - - if (req->err_code) - req->req_step = R_STP_ERROR; - else - req->req_step = R_STP_RECV; - return (0); -} - -/*-------------------------------------------------------------------- - * Central state engine dispatcher. - * - * Kick the session around until it has had enough. - * - */ - -static void -cnt_diag(struct req *req, const char *state) -{ - - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - - VSLb(req->vsl, SLT_Debug, "vsl_id %u STP_%s sp %p obj %p vcl %p", - req->sp->vsl_id, state, req->sp, req->obj, req->vcl); - VSL_Flush(req->vsl, 0); -} - -int -CNT_Request(struct worker *wrk, struct req *req) -{ - int done; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - - /* - * Possible entrance states - */ - assert( - req->req_step == R_STP_LOOKUP || - req->req_step == R_STP_START || - req->req_step == R_STP_RECV); - - req->wrk = wrk; - - for (done = 0; !done; ) { - /* - * This is a good place to be paranoid about the various - * pointers still pointing to the things we expect. - */ - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); - WS_Assert(wrk->aws); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - - switch (req->req_step) { -#define REQ_STEP(l,u,arg) \ - case R_STP_##u: \ - if (cache_param->diag_bitmap & 0x01) \ - cnt_diag(req, #u); \ - done = cnt_##l arg; \ - break; -#include "tbl/steps.h" -#undef REQ_STEP - default: - WRONG("State engine misfire"); - } - WS_Assert(wrk->aws); - CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); - } - if (done == 1) { - /* XXX: Workaround for pipe */ - if (req->sp->fd >= 0) { - VSLb(req->vsl, SLT_Length, "%ju", - (uintmax_t)req->req_bodybytes); - } - VSLb(req->vsl, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f", - req->xid, - req->t_req, - req->sp->t_idle, - req->sp->t_idle - req->t_resp, - req->t_resp - req->t_req, - req->sp->t_idle - req->t_resp); - - /* done == 2 was charged by cache_hash.c */ - SES_Charge(wrk, req); - } - - req->wrk = NULL; - - assert(WRW_IsReleased(wrk)); - return (done); -} - -/* -DOT } -*/ - -/*-------------------------------------------------------------------- - * Debugging aids - */ - -static void -cli_debug_xid(struct cli *cli, const char * const *av, void *priv) -{ - (void)priv; - if (av[2] != NULL) - xids = strtoul(av[2], NULL, 0); - VCLI_Out(cli, "XID is %u", xids); -} - -/* - * Default to seed=1, this is the only seed value POSIXl guarantees will - * result in a reproducible random number sequence. - */ -static void -cli_debug_srandom(struct cli *cli, const char * const *av, void *priv) -{ - (void)priv; - unsigned seed = 1; - - if (av[2] != NULL) - seed = strtoul(av[2], NULL, 0); - srandom(seed); - srand48(random()); - VCLI_Out(cli, "Random(3) seeded with %u", seed); -} - -static struct cli_proto debug_cmds[] = { - { "debug.xid", "debug.xid", - "\tExamine or set XID\n", 0, 1, "d", cli_debug_xid }, - { "debug.srandom", "debug.srandom", - "\tSeed the random(3) function\n", 0, 1, "d", - cli_debug_srandom }, - { NULL } -}; - -/*-------------------------------------------------------------------- - * - */ - -void -CNT_Init(void) -{ - - srandomdev(); - srand48(random()); - xids = random(); - CLI_AddFuncs(debug_cmds); -} diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c new file mode 100644 index 0000000..586fd50 --- /dev/null +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -0,0 +1,348 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2011 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * This file contains the two central state machine for pushing + * sessions and requests. + * + * The first part of the file, entrypoint CNT_Session() and down to + * the ==== separator, is concerned with sessions. When a session has + * a request to deal with, it calls into the second half of the file. + * This part is for all practical purposes HTTP/1.x specific. + * + * The second part of the file, entrypoint CNT_Request() and below the + * ==== separator, is intended to (over time) be(ome) protocol agnostic. + * We already use this now with ESI:includes, which are for all relevant + * purposes a different "protocol" + * + * A special complication is the fact that we can suspend processing of + * a request when hash-lookup finds a busy objhdr. + * + * Since the states are rather nasty in detail, I have decided to embedd + * a dot(1) graph in the source code comments. So to see the big picture, + * extract the DOT lines and run though dot(1), for instance with the + * command: + * sed -n '/^DOT/s///p' cache/cache_center.c | dot -Tps > /tmp/_.ps + */ + +/* +DOT digraph vcl_center { +xDOT page="8.2,11.5" +DOT size="7.2,10.5" +DOT margin="0.5" +DOT center="1" +DOT acceptor [ +DOT shape=hexagon +DOT label="Request received" +DOT ] +DOT ERROR [shape=plaintext] +DOT RESTART [shape=plaintext] +DOT acceptor -> first [style=bold,color=green] + */ + +#include "config.h" + +#include +#include +#include +#include + +#include "cache.h" + +#include "hash/hash_slinger.h" +#include "vcl.h" +#include "vcli_priv.h" +#include "vsha256.h" +#include "vtcp.h" +#include "vtim.h" + +#ifndef HAVE_SRANDOMDEV +#include "compat/srandomdev.h" +#endif + + +/*-------------------------------------------------------------------- + * WAIT + * Collect the request from the client. + * +DOT subgraph xcluster_wait { +DOT wait [ +DOT shape=box +DOT label="cnt_sess_wait:\nwait for\ncomplete\nrequest" +DOT ] +DOT herding [shape=hexagon] +DOT wait -> start [label="got req",style=bold,color=green] +DOT wait -> "SES_Delete()" [label="errors"] +DOT wait -> herding [label="timeout_linger"] +DOT herding -> wait [label="fd read_ready"] +DOT } + */ + +static int +cnt_sess_wait(struct sess *sp, struct worker *wrk, struct req *req) +{ + int j, tmo; + struct pollfd pfd[1]; + double now, when; + enum sess_close why = SC_NULL; + enum htc_status_e hs; + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + assert(req->sp == sp); + + + AZ(req->vcl); + AZ(req->obj); + AZ(req->esi_level); + assert(req->xid == 0); + assert(isnan(req->t_req)); + assert(isnan(req->t_resp)); + + tmo = (int)(1e3 * cache_param->timeout_linger); + while (1) { + pfd[0].fd = sp->fd; + pfd[0].events = POLLIN; + pfd[0].revents = 0; + j = poll(pfd, 1, tmo); + assert(j >= 0); + now = VTIM_real(); + if (j != 0) + hs = HTC_Rx(req->htc); + else + hs = HTC_Complete(req->htc); + if (hs == HTC_COMPLETE) { + /* Got it, run with it */ + req->t_req = now; + return (0); + } else if (hs == HTC_ERROR_EOF) { + why = SC_REM_CLOSE; + break; + } else if (hs == HTC_OVERFLOW) { + why = SC_RX_OVERFLOW; + break; + } else if (hs == HTC_ALL_WHITESPACE) { + /* Nothing but whitespace */ + when = sp->t_idle + cache_param->timeout_idle; + if (when < now) { + why = SC_RX_TIMEOUT; + break; + } + when = sp->t_idle + cache_param->timeout_linger; + tmo = (int)(1e3 * (when - now)); + if (when < now || tmo == 0) { + req->t_req = NAN; + wrk->stats.sess_herd++; + SES_ReleaseReq(req); + WAIT_Enter(sp); + return (1); + } + } else { + /* Working on it */ + if (isnan(req->t_req)) + req->t_req = now; + when = req->t_req + cache_param->timeout_req; + tmo = (int)(1e3 * (when - now)); + if (when < now || tmo == 0) { + why = SC_RX_TIMEOUT; + break; + } + } + } + SES_ReleaseReq(req); + assert(why != SC_NULL); + SES_Delete(sp, why, now); + return (1); +} + +/*-------------------------------------------------------------------- + * This is the final state, figure out if we should close or recycle + * the client connection + * +DOT DONE [ +DOT shape=record +DOT label="{cnt_done:|Request completed}" +DOT ] +DOT ESI_RESP [ shape=hexagon ] +DOT DONE -> start [label="full pipeline"] +DOT DONE -> wait +DOT DONE -> ESI_RESP + */ + +enum cnt_sess_done_ret { + SESS_DONE_RET_GONE, + SESS_DONE_RET_WAIT, + SESS_DONE_RET_START, +}; + +static enum cnt_sess_done_ret +cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) +{ + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_ORNULL(req->vcl, VCL_CONF_MAGIC); + + AZ(req->obj); + AZ(req->busyobj); + req->director = NULL; + req->restarts = 0; + + AZ(req->esi_level); + + if (req->vcl != NULL) { + if (wrk->vcl != NULL) + VCL_Rel(&wrk->vcl); + wrk->vcl = req->vcl; + req->vcl = NULL; + } + + sp->t_idle = W_TIM_real(wrk); + if (req->xid == 0) + req->t_resp = sp->t_idle; + req->xid = 0; + VSL_Flush(req->vsl, 0); + + req->t_req = NAN; + req->t_resp = NAN; + + req->req_bodybytes = 0; + + req->hash_always_miss = 0; + req->hash_ignore_busy = 0; + + if (sp->fd >= 0 && req->doclose != SC_NULL) + SES_Close(sp, req->doclose); + + if (sp->fd < 0) { + wrk->stats.sess_closed++; + AZ(req->vcl); + SES_ReleaseReq(req); + SES_Delete(sp, SC_NULL, NAN); + return (SESS_DONE_RET_GONE); + } + + if (wrk->stats.client_req >= cache_param->wthread_stats_rate) + WRK_SumStat(wrk); + + WS_Reset(req->ws, NULL); + WS_Reset(wrk->aws, NULL); + req->vxid = VXID_Get(&wrk->vxid_pool); + + if (HTC_Reinit(req->htc) == HTC_COMPLETE) { + req->t_req = sp->t_idle; + wrk->stats.sess_pipeline++; + return (SESS_DONE_RET_START); + } else { + if (Tlen(req->htc->rxbuf)) + wrk->stats.sess_readahead++; + return (SESS_DONE_RET_WAIT); + } +} + +/*-------------------------------------------------------------------- + */ + +void +CNT_Session(struct worker *wrk, struct req *req) +{ + int done; + struct sess *sp; + enum cnt_sess_done_ret sdr; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + sp = req->sp; + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + + /* + * Whenever we come in from the acceptor or waiter, we need to set + * blocking mode, but there is no point in setting it when we come from + * ESI or when a parked sessions returns. + * It would be simpler to do this in the acceptor or waiter, but we'd + * rather do the syscall in the worker thread. + * On systems which return errors for ioctl, we close early + */ + if (sp->sess_step == S_STP_NEWREQ && VTCP_blocking(sp->fd)) { + if (errno == ECONNRESET) + SES_Close(sp, SC_REM_CLOSE); + else + SES_Close(sp, SC_TX_ERROR); + sdr = cnt_sess_done(sp, wrk, req); + assert(sdr == SESS_DONE_RET_GONE); + return; + } + + if (sp->sess_step == S_STP_NEWREQ) { + HTC_Init(req->htc, req->ws, sp->fd, req->vsl, + cache_param->http_req_size, + cache_param->http_req_hdr_len); + } + + while (1) { + /* + * Possible entrance states + */ + + assert( + sp->sess_step == S_STP_NEWREQ || + req->req_step == R_STP_LOOKUP || + req->req_step == R_STP_START); + + if (sp->sess_step == S_STP_WORKING) { + done = CNT_Request(wrk, req); + if (done == 2) + return; + assert(done == 1); + sdr = cnt_sess_done(sp, wrk, req); + switch (sdr) { + case SESS_DONE_RET_GONE: + return; + case SESS_DONE_RET_WAIT: + sp->sess_step = S_STP_NEWREQ; + break; + case SESS_DONE_RET_START: + sp->sess_step = S_STP_WORKING; + req->req_step = R_STP_START; + break; + default: + WRONG("Illegal enum cnt_sess_done_ret"); + } + } + + if (sp->sess_step == S_STP_NEWREQ) { + done = cnt_sess_wait(sp, wrk, req); + if (done) + return; + sp->sess_step = S_STP_WORKING; + req->req_step = R_STP_START; + } + } +} diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c new file mode 100644 index 0000000..34cfbfa --- /dev/null +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -0,0 +1,1395 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2011 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * This file contains the two central state machine for pushing + * sessions and requests. + * + * The first part of the file, entrypoint CNT_Session() and down to + * the ==== separator, is concerned with sessions. When a session has + * a request to deal with, it calls into the second half of the file. + * This part is for all practical purposes HTTP/1.x specific. + * + * The second part of the file, entrypoint CNT_Request() and below the + * ==== separator, is intended to (over time) be(ome) protocol agnostic. + * We already use this now with ESI:includes, which are for all relevant + * purposes a different "protocol" + * + * A special complication is the fact that we can suspend processing of + * a request when hash-lookup finds a busy objhdr. + * + * Since the states are rather nasty in detail, I have decided to embedd + * a dot(1) graph in the source code comments. So to see the big picture, + * extract the DOT lines and run though dot(1), for instance with the + * command: + * sed -n '/^DOT/s///p' cache/cache_center.c | dot -Tps > /tmp/_.ps + */ + +/* +DOT digraph vcl_center { +xDOT page="8.2,11.5" +DOT size="7.2,10.5" +DOT margin="0.5" +DOT center="1" +DOT acceptor [ +DOT shape=hexagon +DOT label="Request received" +DOT ] +DOT ERROR [shape=plaintext] +DOT RESTART [shape=plaintext] +DOT acceptor -> first [style=bold,color=green] + */ + +#include "config.h" + +#include +#include +#include +#include + +#include "cache.h" + +#include "hash/hash_slinger.h" +#include "vcl.h" +#include "vcli_priv.h" +#include "vsha256.h" +#include "vtim.h" + +#ifndef HAVE_SRANDOMDEV +#include "compat/srandomdev.h" +#endif + +static unsigned xids; +/*-------------------------------------------------------------------- + * We have a refcounted object on the session, and possibly the busyobj + * which is fetching it, prepare a response. + * +DOT subgraph xcluster_prepresp { +DOT prepresp [ +DOT shape=record +DOT label="{cnt_prepresp:|Filter obj.-\>resp.|{vcl_deliver\{\}|{req.|resp.}}|{error?|restart?}|stream ?}" +DOT ] +DOT prepresp -> deliver [style=bold,color=green,label=deliver] +DOT prepresp -> deliver [style=bold,color=red] +DOT prepresp -> deliver [style=bold,color=blue] +DOT } + * + */ + +static int +cnt_prepresp(struct worker *wrk, struct req *req) +{ + struct busyobj *bo; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + bo = req->busyobj; + CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); + + CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); + CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); + + req->res_mode = 0; + + if (bo == NULL) { + if (!req->disable_esi && req->obj->esidata != NULL) { + /* In ESI mode, we can't know the aggregate length */ + req->res_mode &= ~RES_LEN; + req->res_mode |= RES_ESI; + } else { + req->res_mode |= RES_LEN; + } + } else { + AZ(bo->do_esi); + } + + if (req->esi_level > 0) { + /* Included ESI object, always CHUNKED or EOF */ + req->res_mode &= ~RES_LEN; + req->res_mode |= RES_ESI_CHILD; + } + + if (cache_param->http_gzip_support && req->obj->gziped && + !RFC2616_Req_Gzip(req->http)) { + /* + * We don't know what it uncompresses to + * XXX: we could cache that + */ + req->res_mode &= ~RES_LEN; + req->res_mode |= RES_GUNZIP; + } + + if (!(req->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) { + /* We havn't chosen yet, do so */ + if (!req->wantbody) { + /* Nothing */ + } else if (req->http->protover >= 11) { + req->res_mode |= RES_CHUNKED; + } else { + req->res_mode |= RES_EOF; + req->doclose = SC_TX_EOF; + } + } + + req->t_resp = W_TIM_real(wrk); + if (req->obj->objcore->objhead != NULL) { + if ((req->t_resp - req->obj->last_lru) > + cache_param->lru_timeout && + EXP_Touch(req->obj->objcore)) + req->obj->last_lru = req->t_resp; + if (!cache_param->obj_readonly) + req->obj->last_use = req->t_resp; /* XXX: locking ? */ + } + HTTP_Setup(req->resp, req->ws, req->vsl, HTTP_Resp); + RES_BuildHttp(req); + + VCL_deliver_method(req); + switch (req->handling) { + case VCL_RET_DELIVER: + break; + case VCL_RET_RESTART: + if (req->restarts >= cache_param->max_restarts) + break; + if (bo != NULL) { + AN(bo->do_stream); + (void)HSH_Deref(&wrk->stats, NULL, &req->obj); + VBO_DerefBusyObj(wrk, &req->busyobj); + } else { + (void)HSH_Deref(&wrk->stats, NULL, &req->obj); + } + AZ(req->obj); + http_Teardown(req->resp); + req->req_step = R_STP_RESTART; + return (0); + default: + WRONG("Illegal action in vcl_deliver{}"); + } + req->req_step = R_STP_DELIVER; + return (0); +} + +/*-------------------------------------------------------------------- + * Deliver an already stored object + * +DOT subgraph xcluster_deliver { +DOT deliver [ +DOT shape=record +DOT label="{cnt_deliver:|Send body}" +DOT ] +DOT } +DOT deliver -> DONE [style=bold,color=green] +DOT deliver -> DONE [style=bold,color=red] +DOT deliver -> DONE [style=bold,color=blue] + * + */ + +static int +cnt_deliver(struct worker *wrk, struct req *req) +{ + struct busyobj *bo; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); + bo = req->busyobj; + CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); + + if (bo != NULL) { + while (bo->state < BOS_FAILED) + (void)usleep(10000); + assert(bo->state >= BOS_FAILED); + + if (bo->state == BOS_FAILED) { + HSH_Deref(&wrk->stats, NULL, &req->obj); + VBO_DerefBusyObj(wrk, &req->busyobj); + req->err_code = 503; + req->req_step = R_STP_ERROR; + return (0); + } + VBO_DerefBusyObj(wrk, &req->busyobj); + } + + AZ(req->busyobj); + req->director = NULL; + req->restarts = 0; + + RES_WriteObj(req); + + /* No point in saving the body if it is hit-for-pass */ + if (req->obj->objcore->flags & OC_F_PASS) + STV_Freestore(req->obj); + + assert(WRW_IsReleased(wrk)); + (void)HSH_Deref(&wrk->stats, NULL, &req->obj); + http_Teardown(req->resp); + return (1); +} +/*-------------------------------------------------------------------- + * Emit an error + * +DOT subgraph xcluster_error { +DOT vcl_error [ +DOT shape=record +DOT label="vcl_error()|resp." +DOT ] +DOT ERROR -> vcl_error +DOT vcl_error-> prepresp [label=deliver] +DOT } +DOT vcl_error-> rsterr [label="restart",color=purple] +DOT rsterr [label="RESTART",shape=plaintext] + */ + +static int +cnt_error(struct worker *wrk, struct req *req) +{ + struct http *h; + struct busyobj *bo; + char date[40]; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + AZ(req->objcore); + AZ(req->obj); + AZ(req->busyobj); + + bo = VBO_GetBusyObj(wrk); + req->busyobj = bo; + bo->vsl->wid = req->sp->vsl_id; + AZ(bo->stats); + bo->stats = &wrk->stats; + req->objcore = HSH_NewObjCore(wrk); + req->obj = STV_NewObject(bo, &req->objcore, + TRANSIENT_STORAGE, cache_param->http_resp_size, + (uint16_t)cache_param->http_max_hdr); + bo->stats = NULL; + if (req->obj == NULL) { + req->doclose = SC_OVERLOAD; + req->director = NULL; + http_Teardown(bo->beresp); + http_Teardown(bo->bereq); + return(1); + } + CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); + req->obj->xid = req->xid; + req->obj->exp.entered = req->t_req; + + h = req->obj->http; + + if (req->err_code < 100 || req->err_code > 999) + req->err_code = 501; + + http_PutProtocol(h, "HTTP/1.1"); + http_PutStatus(h, req->err_code); + VTIM_format(W_TIM_real(wrk), date); + http_PrintfHeader(h, "Date: %s", date); + http_SetHeader(h, "Server: Varnish"); + + if (req->err_reason != NULL) + http_PutResponse(h, req->err_reason); + else + http_PutResponse(h, http_StatusMessage(req->err_code)); + VCL_error_method(req); + + if (req->handling == VCL_RET_RESTART && + req->restarts < cache_param->max_restarts) { + HSH_Drop(wrk, &req->obj); + VBO_DerefBusyObj(wrk, &req->busyobj); + req->req_step = R_STP_RESTART; + return (0); + } else if (req->handling == VCL_RET_RESTART) + req->handling = VCL_RET_DELIVER; + + + /* We always close when we take this path */ + req->doclose = SC_TX_ERROR; + req->wantbody = 1; + + assert(req->handling == VCL_RET_DELIVER); + req->err_code = 0; + req->err_reason = NULL; + http_Teardown(bo->bereq); + VBO_DerefBusyObj(wrk, &req->busyobj); + req->req_step = R_STP_PREPRESP; + return (0); +} + +/*-------------------------------------------------------------------- + * Fetch response headers from the backend + * +DOT subgraph xcluster_fetch { +DOT fetch [ +DOT shape=record +DOT label="{cnt_fetch:|fetch hdr\nfrom backend|(find obj.ttl)|{vcl_fetch\{\}|{req.|bereq.|beresp.}}|{error?|restart?}}" +DOT ] +DOT } +DOT fetch -> fetchbody [style=bold,color=red] +DOT fetch -> fetchbody [style=bold,color=blue] + */ + +static int +cnt_fetch(struct worker *wrk, struct req *req) +{ + int i, need_host_hdr; + struct busyobj *bo; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); + bo = req->busyobj; + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + + AN(req->director); + AZ(bo->vbc); + AZ(bo->should_close); + AZ(req->storage_hint); + + HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp); + + need_host_hdr = !http_GetHdr(bo->bereq, H_Host, NULL); + + req->acct_req.fetch++; + + i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL); + /* + * If we recycle a backend connection, there is a finite chance + * that the backend closed it before we get a request to it. + * Do a single retry in that case. + */ + if (i == 1) { + VSC_C_main->backend_retry++; + i = FetchHdr(req, need_host_hdr, req->objcore->objhead == NULL); + } + + if (i) { + req->handling = VCL_RET_ERROR; + req->err_code = 503; + } else { + /* + * These two headers can be spread over multiple actual headers + * and we rely on their content outside of VCL, so collect them + * into one line here. + */ + http_CollectHdr(bo->beresp, H_Cache_Control); + http_CollectHdr(bo->beresp, H_Vary); + + /* + * Figure out how the fetch is supposed to happen, before the + * headers are adultered by VCL + * NB: Also sets other wrk variables + */ + bo->body_status = RFC2616_Body(bo, &wrk->stats); + + req->err_code = http_GetStatus(bo->beresp); + + /* + * What does RFC2616 think about TTL ? + */ + EXP_Clr(&bo->exp); + bo->exp.entered = W_TIM_real(wrk); + RFC2616_Ttl(bo, req->xid); + + /* pass from vclrecv{} has negative TTL */ + if (req->objcore->objhead == NULL) + bo->exp.ttl = -1.; + + AZ(bo->do_esi); + AZ(bo->do_pass); + + VCL_fetch_method(req); + + if (bo->do_pass) + req->objcore->flags |= OC_F_PASS; + + switch (req->handling) { + case VCL_RET_DELIVER: + req->req_step = R_STP_FETCHBODY; + return (0); + default: + break; + } + + /* We are not going to fetch the body, Close the connection */ + VDI_CloseFd(&bo->vbc); + } + + /* Clean up partial fetch */ + AZ(bo->vbc); + + if (req->objcore->objhead != NULL || req->handling == VCL_RET_ERROR) { + CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); + AZ(HSH_Deref(&wrk->stats, req->objcore, NULL)); + req->objcore = NULL; + } + assert(bo->refcount == 2); + VBO_DerefBusyObj(wrk, &bo); + VBO_DerefBusyObj(wrk, &req->busyobj); + req->director = NULL; + req->storage_hint = NULL; + + switch (req->handling) { + case VCL_RET_RESTART: + req->req_step = R_STP_RESTART; + return (0); + case VCL_RET_ERROR: + req->req_step = R_STP_ERROR; + return (0); + default: + WRONG("Illegal action in vcl_fetch{}"); + } +} + +/*-------------------------------------------------------------------- + * Prepare to fetch body from backend + * +DOT subgraph xcluster_body { +DOT fetchbody [ +DOT shape=record +DOT label="{cnt_fetchbody:|start fetch_thread}" +DOT ] +DOT } +DOT fetchbody:out -> prepresp [style=bold,color=red] +DOT fetchbody:out -> prepresp [style=bold,color=blue] + */ + +static int +cnt_fetchbody(struct worker *wrk, struct req *req) +{ + struct http *hp, *hp2; + char *b; + uint16_t nhttp; + unsigned l; + struct vsb *vary = NULL; + int varyl = 0, pass; + struct busyobj *bo; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + bo = req->busyobj; + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + + assert(req->handling == VCL_RET_DELIVER); + + if (req->objcore->objhead == NULL) { + /* This is a pass from vcl_recv */ + pass = 1; + /* VCL may have fiddled this, but that doesn't help */ + bo->exp.ttl = -1.; + } else if (bo->do_pass) { + pass = 1; + } else { + /* regular object */ + pass = 0; + } + + /* + * The VCL variables beresp.do_g[un]zip tells us how we want the + * object processed before it is stored. + * + * The backend Content-Encoding header tells us what we are going + * to receive, which we classify in the following three classes: + * + * "Content-Encoding: gzip" --> object is gzip'ed. + * no Content-Encoding --> object is not gzip'ed. + * anything else --> do nothing wrt gzip + * + */ + + /* We do nothing unless the param is set */ + if (!cache_param->http_gzip_support) + bo->do_gzip = bo->do_gunzip = 0; + + bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip"); + + bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL); + + /* It can't be both */ + assert(bo->is_gzip == 0 || bo->is_gunzip == 0); + + /* We won't gunzip unless it is gzip'ed */ + if (bo->do_gunzip && !bo->is_gzip) + bo->do_gunzip = 0; + + /* If we do gunzip, remove the C-E header */ + if (bo->do_gunzip) + http_Unset(bo->beresp, H_Content_Encoding); + + /* We wont gzip unless it is ungziped */ + if (bo->do_gzip && !bo->is_gunzip) + bo->do_gzip = 0; + + /* If we do gzip, add the C-E header */ + if (bo->do_gzip) + http_SetHeader(bo->beresp, "Content-Encoding: gzip"); + + /* But we can't do both at the same time */ + assert(bo->do_gzip == 0 || bo->do_gunzip == 0); + + /* ESI takes precedence and handles gzip/gunzip itself */ + if (bo->do_esi) + bo->vfp = &vfp_esi; + else if (bo->do_gunzip) + bo->vfp = &vfp_gunzip; + else if (bo->do_gzip) + bo->vfp = &vfp_gzip; + else if (bo->is_gzip) + bo->vfp = &vfp_testgzip; + + if (bo->do_esi || req->esi_level > 0) + bo->do_stream = 0; + if (!req->wantbody) + bo->do_stream = 0; + + /* No reason to try streaming a non-existing body */ + if (bo->body_status == BS_NONE) + bo->do_stream = 0; + + l = http_EstimateWS(bo->beresp, + pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp); + + /* Create Vary instructions */ + if (req->objcore->objhead != NULL) { + CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); + vary = VRY_Create(req, bo->beresp); + if (vary != NULL) { + varyl = VSB_len(vary); + assert(varyl > 0); + l += varyl; + } + } + + /* + * Space for producing a Content-Length: header including padding + * A billion gigabytes is enough for anybody. + */ + l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *); + + if (bo->exp.ttl < cache_param->shortlived || + req->objcore == NULL) + req->storage_hint = TRANSIENT_STORAGE; + + AZ(bo->stats); + bo->stats = &wrk->stats; + req->obj = STV_NewObject(bo, &req->objcore, req->storage_hint, l, + nhttp); + if (req->obj == NULL) { + /* + * Try to salvage the transaction by allocating a + * shortlived object on Transient storage. + */ + if (bo->exp.ttl > cache_param->shortlived) + bo->exp.ttl = cache_param->shortlived; + bo->exp.grace = 0.0; + bo->exp.keep = 0.0; + req->obj = STV_NewObject(bo, &req->objcore, TRANSIENT_STORAGE, + l, nhttp); + } + bo->stats = NULL; + if (req->obj == NULL) { + req->err_code = 503; + req->req_step = R_STP_ERROR; + VDI_CloseFd(&bo->vbc); + VBO_DerefBusyObj(wrk, &req->busyobj); + return (0); + } + CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); + + req->storage_hint = NULL; + + AZ(bo->fetch_obj); + bo->fetch_obj = req->obj; + + if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) + req->obj->gziped = 1; + + if (vary != NULL) { + req->obj->vary = (void *)WS_Alloc(req->obj->http->ws, varyl); + AN(req->obj->vary); + memcpy(req->obj->vary, VSB_data(vary), varyl); + VRY_Validate(req->obj->vary); + VSB_delete(vary); + } + + req->obj->xid = req->xid; + req->obj->response = req->err_code; + WS_Assert(req->obj->ws_o); + + /* Filter into object */ + hp = bo->beresp; + hp2 = req->obj->http; + + hp2->logtag = HTTP_Obj; + http_FilterResp(hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS); + http_CopyHome(hp2); + + if (http_GetHdr(hp, H_Last_Modified, &b)) + req->obj->last_modified = VTIM_parse(b); + else + req->obj->last_modified = floor(bo->exp.entered); + + assert(WRW_IsReleased(wrk)); + + /* + * If we can deliver a 304 reply, we don't bother streaming. + * Notice that vcl_deliver{} could still nuke the headers + * that allow the 304, in which case we return 200 non-stream. + */ + if (req->obj->response == 200 && + req->http->conds && + RFC2616_Do_Cond(req)) + bo->do_stream = 0; + + /* + * Ready to fetch the body + */ + bo->fetch_task.func = FetchBody; + bo->fetch_task.priv = bo; + + assert(bo->refcount == 2); /* one for each thread */ + + if (req->obj->objcore->objhead != NULL) { + EXP_Insert(req->obj); + AN(req->obj->objcore->ban); + AZ(req->obj->ws_o->overflow); + HSH_Unbusy(&wrk->stats, req->obj->objcore); + } + + if (!bo->do_stream || + Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE)) + FetchBody(wrk, bo); + + if (req->obj->objcore->objhead != NULL) + HSH_Ref(req->obj->objcore); + + if (bo->state == BOS_FINISHED) { + VBO_DerefBusyObj(wrk, &req->busyobj); + } else if (bo->state == BOS_FAILED) { + /* handle early failures */ + HSH_Deref(&wrk->stats, NULL, &req->obj); + VBO_DerefBusyObj(wrk, &req->busyobj); + req->err_code = 503; + req->req_step = R_STP_ERROR; + return (0); + } + + assert(WRW_IsReleased(wrk)); + req->req_step = R_STP_PREPRESP; + return (0); +} + +/*-------------------------------------------------------------------- + * HIT + * We had a cache hit. Ask VCL, then march off as instructed. + * +DOT subgraph xcluster_hit { +DOT hit [ +DOT shape=record +DOT label="{cnt_hit:|{vcl_hit()|{req.|obj.}}|{error?|restart?}|{deliver?|pass?}}" +DOT ] +DOT } +XDOT hit:err -> err_hit [label="error"] +XDOT err_hit [label="ERROR",shape=plaintext] +XDOT hit:rst -> rst_hit [label="restart",color=purple] +XDOT rst_hit [label="RESTART",shape=plaintext] +DOT hit:pass -> pass [label=pass,style=bold,color=red] +DOT hit:del -> prepresp [label="deliver",style=bold,color=green] + */ + +static int +cnt_hit(struct worker *wrk, struct req *req) +{ + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); + CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); + AZ(req->objcore); + AZ(req->busyobj); + + assert(!(req->obj->objcore->flags & OC_F_PASS)); + + VCL_hit_method(req); + + if (req->handling == VCL_RET_DELIVER) { + //AZ(req->busyobj->bereq->ws); + //AZ(req->busyobj->beresp->ws); + (void)FetchReqBody(req, 0); + req->req_step = R_STP_PREPRESP; + return (0); + } + + /* Drop our object, we won't need it */ + (void)HSH_Deref(&wrk->stats, NULL, &req->obj); + req->objcore = NULL; + + switch(req->handling) { + case VCL_RET_PASS: + req->req_step = R_STP_PASS; + return (0); + case VCL_RET_ERROR: + req->req_step = R_STP_ERROR; + return (0); + case VCL_RET_RESTART: + req->req_step = R_STP_RESTART; + return (0); + default: + WRONG("Illegal action in vcl_hit{}"); + } +} + +/*-------------------------------------------------------------------- + * LOOKUP + * Hash things together and look object up in hash-table. + * + * LOOKUP consists of two substates so that we can reenter if we + * encounter a busy object. + * +DOT subgraph xcluster_lookup { +DOT lookup [ +DOT shape=record +DOT label="{cnt_lookup:|hash lookup|{busy ?|miss ?}|{no|obj.f.pass?|yes}}" +DOT ] +DOT } +DOT lookup:busy -> lookup:top [label="(waitinglist)"] +DOT lookup:miss -> miss [style=bold,color=blue] +DOT lookup:no -> hit [style=bold,color=green] +DOT lookup:yes -> pass [style=bold,color=red] + */ + +static int +cnt_lookup(struct worker *wrk, struct req *req) +{ + struct objcore *oc; + struct object *o; + struct objhead *oh; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + AZ(req->objcore); + + CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); + AZ(req->busyobj); + + VRY_Prep(req); + + AZ(req->objcore); + oc = HSH_Lookup(req); + if (oc == NULL) { + /* + * We lost the session to a busy object, disembark the + * worker thread. We return to STP_LOOKUP when the busy + * object has been unbusied, and still have the hash digest + * around to do the lookup with. + */ + return (2); + } + AZ(req->objcore); + + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + oh = oc->objhead; + CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); + + /* If we inserted a new object it's a miss */ + if (oc->flags & OC_F_BUSY) { + CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC); + assert(oc->busyobj == req->busyobj); + wrk->stats.cache_miss++; + + if (req->vary_l != NULL) { + assert(oc->busyobj->vary == req->vary_b); + VRY_Validate(oc->busyobj->vary); + WS_ReleaseP(req->ws, (void*)req->vary_l); + } else { + AZ(oc->busyobj->vary); + WS_Release(req->ws, 0); + } + req->vary_b = NULL; + req->vary_l = NULL; + req->vary_e = NULL; + + req->objcore = oc; + req->req_step = R_STP_MISS; + return (0); + } + + /* We are not prepared to do streaming yet */ + XXXAZ(req->busyobj); + + o = oc_getobj(&wrk->stats, oc); + CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); + req->obj = o; + + WS_Release(req->ws, 0); + req->vary_b = NULL; + req->vary_l = NULL; + req->vary_e = NULL; + + if (oc->flags & OC_F_PASS) { + wrk->stats.cache_hitpass++; + VSLb(req->vsl, SLT_HitPass, "%u", req->obj->xid); + (void)HSH_Deref(&wrk->stats, NULL, &req->obj); + AZ(req->objcore); + req->req_step = R_STP_PASS; + return (0); + } + + wrk->stats.cache_hit++; + VSLb(req->vsl, SLT_Hit, "%u", req->obj->xid); + req->req_step = R_STP_HIT; + return (0); +} + +/*-------------------------------------------------------------------- + * We had a miss, ask VCL, proceed as instructed + * +DOT subgraph xcluster_miss { +DOT miss [ +DOT shape=record +DOT label="{cnt_miss:|filter req.-\>bereq.|{vcl_miss\{\}|{req.*|bereq.*}}|{error?|restart?}|{pass?|fetch?}}" +DOT ] +DOT } +DOT miss:fetch -> fetch [label="fetch",style=bold,color=blue] +DOT miss:pass -> pass [label="pass",style=bold,color=red] +DOT + */ + +static int +cnt_miss(struct worker *wrk, struct req *req) +{ + struct busyobj *bo; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); + CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); + bo = req->busyobj; + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + AZ(req->obj); + + HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); + http_FilterReq(req, HTTPH_R_FETCH); + http_ForceGet(bo->bereq); + if (cache_param->http_gzip_support) { + /* + * We always ask the backend for gzip, even if the + * client doesn't grok it. We will uncompress for + * the minority of clients which don't. + */ + http_Unset(bo->bereq, H_Accept_Encoding); + http_SetHeader(bo->bereq, "Accept-Encoding: gzip"); + } + + VCL_miss_method(req); + + if (req->handling == VCL_RET_FETCH) { + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + req->req_step = R_STP_FETCH; + return (0); + } + + AZ(HSH_Deref(&wrk->stats, req->objcore, NULL)); + req->objcore = NULL; + http_Teardown(bo->bereq); + VBO_DerefBusyObj(wrk, &req->busyobj); + + switch(req->handling) { + case VCL_RET_ERROR: + req->req_step = R_STP_ERROR; + break; + case VCL_RET_PASS: + req->req_step = R_STP_PASS; + break; + case VCL_RET_RESTART: + req->req_step = R_STP_RESTART; + break; + default: + WRONG("Illegal action in vcl_miss{}"); + } + return (0); +} + +/*-------------------------------------------------------------------- + * Start pass processing by getting headers from backend, then + * continue in passbody. + * +DOT subgraph xcluster_pass { +DOT pass [ +DOT shape=record +DOT label="{cnt_pass:|(XXX: deref obj.)|filter req.*-\>bereq.|{vcl_pass\{\}|{req.*|bereq.*}}|{error?|restart?}|create anon obj}" +DOT ] +DOT } +DOT pass:pass -> fetch [style=bold, color=red] +XDOT pass:rst -> rst_pass [label="restart",color=purple] +XDOT rst_pass [label="RESTART",shape=plaintext] +XDOT pass:err -> err_pass [label="error"] +XDOT err_pass [label="ERROR",shape=plaintext] + */ + +static int +cnt_pass(struct worker *wrk, struct req *req) +{ + struct busyobj *bo; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); + AZ(req->objcore); + AZ(req->obj); + AZ(req->busyobj); + + req->busyobj = VBO_GetBusyObj(wrk); + bo = req->busyobj; + bo->vsl->wid = req->sp->vsl_id; + bo->refcount = 2; + HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); + http_FilterReq(req, HTTPH_R_PASS); + + VCL_pass_method(req); + + if (req->handling == VCL_RET_ERROR) { + http_Teardown(bo->bereq); + VBO_DerefBusyObj(wrk, &req->busyobj); + req->req_step = R_STP_ERROR; + return (0); + } + assert(req->handling == VCL_RET_PASS); + req->acct_req.pass++; + req->req_step = R_STP_FETCH; + + req->objcore = HSH_NewObjCore(wrk); + req->objcore->busyobj = bo; + return (0); +} + +/*-------------------------------------------------------------------- + * Ship the request header to the backend unchanged, then pipe + * until one of the ends close the connection. + * +DOT subgraph xcluster_pipe { +DOT pipe [ +DOT shape=ellipse +DOT label="Filter req.->bereq." +DOT ] +DOT vcl_pipe [ +DOT shape=record +DOT label="vcl_pipe()|req.\nbereq\." +DOT ] +DOT pipe_do [ +DOT shape=ellipse +DOT label="send bereq.\npipe until close" +DOT ] +DOT vcl_pipe -> pipe_do [label="pipe",style=bold,color=orange] +DOT pipe -> vcl_pipe [style=bold,color=orange] +DOT } +DOT pipe_do -> DONE [style=bold,color=orange] +DOT vcl_pipe -> err_pipe [label="error"] +DOT err_pipe [label="ERROR",shape=plaintext] + */ + +static int +cnt_pipe(struct worker *wrk, struct req *req) +{ + struct busyobj *bo; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); + AZ(req->busyobj); + + req->acct_req.pipe++; + req->busyobj = VBO_GetBusyObj(wrk); + bo = req->busyobj; + bo->vsl->wid = req->sp->vsl_id; + HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); + http_FilterReq(req, 0); + + VCL_pipe_method(req); + + if (req->handling == VCL_RET_ERROR) + INCOMPL(); + assert(req->handling == VCL_RET_PIPE); + + PipeRequest(req); + assert(WRW_IsReleased(wrk)); + http_Teardown(bo->bereq); + VBO_DerefBusyObj(wrk, &req->busyobj); + return (1); +} + +/*-------------------------------------------------------------------- + * +DOT subgraph xcluster_restart { +DOT restart [ +DOT shape=record +DOT label="{cnt_restart}" +DOT ] +DOT } +DOT RESTART -> restart [color=purple] +DOT restart -> recv [color=purple] + */ + +static int +cnt_restart(const struct worker *wrk, struct req *req) +{ + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + req->director = NULL; + if (++req->restarts >= cache_param->max_restarts) { + req->err_code = 503; + req->req_step = R_STP_ERROR; + } else { + req->err_code = 0; + req->req_step = R_STP_RECV; + } + return (0); +} + +/*-------------------------------------------------------------------- + * RECV + * We have a complete request, set everything up and start it. + * We can come here both with a request from the client and with + * a interior request during ESI delivery. + * +DOT subgraph xcluster_recv { +DOT recv [ +DOT shape=record +DOT label="{cnt_recv:|{vcl_recv\{\}|req.*}|{pipe?|pass?|error?|lookup?}}" +DOT ] +DOT } +DOT subgraph xcluster_hash { +DOT hash [ +DOT shape=record +DOT label="{cnt_recv:|{vcl_hash\{\}|req.*}}" +DOT ] +DOT } +DOT ESI_REQ [ shape=hexagon ] +DOT ESI_REQ -> recv +DOT recv:pipe -> pipe [style=bold,color=orange] +DOT recv:pass -> pass [style=bold,color=red] +#DOT recv:error -> err_recv +#DOT err_recv [label="ERROR",shape=plaintext] +DOT recv:lookup -> hash [style=bold,color=green] +DOT hash -> lookup [label="hash",style=bold,color=green] + */ + +static int +cnt_recv(const struct worker *wrk, struct req *req) +{ + unsigned recv_handling; + struct SHA256Context sha256ctx; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); + AZ(req->obj); + AZ(req->busyobj); + + /* By default we use the first backend */ + AZ(req->director); + req->director = req->vcl->director[0]; + AN(req->director); + + req->disable_esi = 0; + req->hash_always_miss = 0; + req->hash_ignore_busy = 0; + req->client_identity = NULL; + + http_CollectHdr(req->http, H_Cache_Control); + + VCL_recv_method(req); + recv_handling = req->handling; + + if (cache_param->http_gzip_support && + (recv_handling != VCL_RET_PIPE) && + (recv_handling != VCL_RET_PASS)) { + if (RFC2616_Req_Gzip(req->http)) { + http_Unset(req->http, H_Accept_Encoding); + http_SetHeader(req->http, "Accept-Encoding: gzip"); + } else { + http_Unset(req->http, H_Accept_Encoding); + } + } + + req->sha256ctx = &sha256ctx; /* so HSH_AddString() can find it */ + SHA256_Init(req->sha256ctx); + VCL_hash_method(req); + assert(req->handling == VCL_RET_HASH); + SHA256_Final(req->digest, req->sha256ctx); + req->sha256ctx = NULL; + + if (!strcmp(req->http->hd[HTTP_HDR_REQ].b, "HEAD")) + req->wantbody = 0; + else + req->wantbody = 1; + + switch(recv_handling) { + case VCL_RET_LOOKUP: + req->req_step = R_STP_LOOKUP; + return (0); + case VCL_RET_PIPE: + if (req->esi_level > 0) { + /* XXX: VSL something */ + INCOMPL(); + return (1); + } + req->req_step = R_STP_PIPE; + return (0); + case VCL_RET_PASS: + req->req_step = R_STP_PASS; + return (0); + case VCL_RET_ERROR: + req->req_step = R_STP_ERROR; + return (0); + default: + WRONG("Illegal action in vcl_recv{}"); + } +} + +/*-------------------------------------------------------------------- + * START + * First time we see a request + * +DOT start [ +DOT shape=box +DOT label="cnt_start:\nDissect request\nHandle expect" +DOT ] +DOT start -> recv [style=bold,color=green] +DOT start -> DONE [label=errors] + */ + +static int +cnt_start(struct worker *wrk, struct req *req) +{ + char *p; + const char *r = "HTTP/1.1 100 Continue\r\n\r\n"; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + AZ(req->restarts); + AZ(req->obj); + AZ(req->vcl); + AZ(req->esi_level); + assert(!isnan(req->t_req)); + + /* Update stats of various sorts */ + wrk->stats.client_req++; + req->acct_req.req++; + + /* Assign XID and log */ + req->xid = ++xids; /* XXX not locked */ + VSLb(req->vsl, SLT_ReqStart, "%s %s %u", + req->sp->addr, req->sp->port, req->xid); + + /* Borrow VCL reference from worker thread */ + VCL_Refresh(&wrk->vcl); + req->vcl = wrk->vcl; + wrk->vcl = NULL; + + EXP_Clr(&req->exp); + + HTTP_Setup(req->http, req->ws, req->vsl, HTTP_Req); + req->err_code = http_DissectRequest(req); + + /* If we could not even parse the request, just close */ + if (req->err_code == 400) { + SES_Close(req->sp, SC_RX_JUNK); + return (1); + } + + req->ws_req = WS_Snapshot(req->ws); + + req->doclose = http_DoConnection(req->http); + + /* + * We want to deal with Expect: headers the first time we + * attempt the request, and remove them before we move on. + */ + if (req->err_code == 0 && http_GetHdr(req->http, H_Expect, &p)) { + if (strcasecmp(p, "100-continue")) { + req->err_code = 417; + } else if (strlen(r) != write(req->sp->fd, r, strlen(r))) { + SES_Close(req->sp, SC_REM_CLOSE); + return (1); + } + } + http_Unset(req->http, H_Expect); + + /* XXX: pull in req-body and make it available instead. */ + req->reqbodydone = 0; + + HTTP_Copy(req->http0, req->http); /* Copy for restart/ESI use */ + + if (req->err_code) + req->req_step = R_STP_ERROR; + else + req->req_step = R_STP_RECV; + return (0); +} + +/*-------------------------------------------------------------------- + * Central state engine dispatcher. + * + * Kick the session around until it has had enough. + * + */ + +static void +cnt_diag(struct req *req, const char *state) +{ + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + VSLb(req->vsl, SLT_Debug, "vsl_id %u STP_%s sp %p obj %p vcl %p", + req->sp->vsl_id, state, req->sp, req->obj, req->vcl); + VSL_Flush(req->vsl, 0); +} + +int +CNT_Request(struct worker *wrk, struct req *req) +{ + int done; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + /* + * Possible entrance states + */ + assert( + req->req_step == R_STP_LOOKUP || + req->req_step == R_STP_START || + req->req_step == R_STP_RECV); + + req->wrk = wrk; + + for (done = 0; !done; ) { + /* + * This is a good place to be paranoid about the various + * pointers still pointing to the things we expect. + */ + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); + WS_Assert(wrk->aws); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + switch (req->req_step) { +#define REQ_STEP(l,u,arg) \ + case R_STP_##u: \ + if (cache_param->diag_bitmap & 0x01) \ + cnt_diag(req, #u); \ + done = cnt_##l arg; \ + break; +#include "tbl/steps.h" +#undef REQ_STEP + default: + WRONG("State engine misfire"); + } + WS_Assert(wrk->aws); + CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); + } + if (done == 1) { + /* XXX: Workaround for pipe */ + if (req->sp->fd >= 0) { + VSLb(req->vsl, SLT_Length, "%ju", + (uintmax_t)req->req_bodybytes); + } + VSLb(req->vsl, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f", + req->xid, + req->t_req, + req->sp->t_idle, + req->sp->t_idle - req->t_resp, + req->t_resp - req->t_req, + req->sp->t_idle - req->t_resp); + + /* done == 2 was charged by cache_hash.c */ + SES_Charge(wrk, req); + } + + req->wrk = NULL; + + assert(WRW_IsReleased(wrk)); + return (done); +} + +/* +DOT } +*/ + +/*-------------------------------------------------------------------- + * Debugging aids + */ + +static void +cli_debug_xid(struct cli *cli, const char * const *av, void *priv) +{ + (void)priv; + if (av[2] != NULL) + xids = strtoul(av[2], NULL, 0); + VCLI_Out(cli, "XID is %u", xids); +} + +/* + * Default to seed=1, this is the only seed value POSIXl guarantees will + * result in a reproducible random number sequence. + */ +static void +cli_debug_srandom(struct cli *cli, const char * const *av, void *priv) +{ + (void)priv; + unsigned seed = 1; + + if (av[2] != NULL) + seed = strtoul(av[2], NULL, 0); + srandom(seed); + srand48(random()); + VCLI_Out(cli, "Random(3) seeded with %u", seed); +} + +static struct cli_proto debug_cmds[] = { + { "debug.xid", "debug.xid", + "\tExamine or set XID\n", 0, 1, "d", cli_debug_xid }, + { "debug.srandom", "debug.srandom", + "\tSeed the random(3) function\n", 0, 1, "d", + cli_debug_srandom }, + { NULL } +}; + +/*-------------------------------------------------------------------- + * + */ + +void +CNT_Init(void) +{ + + srandomdev(); + srand48(random()); + xids = random(); + CLI_AddFuncs(debug_cmds); +} From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] 1226479 Clean up the HTTP1 fsm and document it with a small dot-graph. Message-ID: commit 1226479c26eb10373d74d98bc0628706b64c3f4d Author: Poul-Henning Kamp Date: Tue Aug 7 08:05:34 2012 +0000 Clean up the HTTP1 fsm and document it with a small dot-graph. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 831c641..920d067 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -764,9 +764,11 @@ struct busyobj *VBO_GetBusyObj(struct worker *wrk); void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj); void VBO_Free(struct busyobj **vbo); -/* cache_center.c [CNT] */ +/* cache_http1_fsm.c [HTTP1] */ +void HTTP1_Session(struct worker *, struct req *); + +/* cache_req_fsm.c [FSM] */ int CNT_Request(struct worker *, struct req *); -void CNT_Session(struct worker *, struct req *); void CNT_Init(void); /* cache_cli.c [CLI] */ diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 586fd50..ae01a00 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -26,42 +26,44 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * This file contains the two central state machine for pushing - * sessions and requests. + * This file contains the two central state machine for pushing HTTP1 + * sessions through their states. * - * The first part of the file, entrypoint CNT_Session() and down to - * the ==== separator, is concerned with sessions. When a session has - * a request to deal with, it calls into the second half of the file. - * This part is for all practical purposes HTTP/1.x specific. + * The following dot-graph shows the big picture, and the two major + * complicating features: * - * The second part of the file, entrypoint CNT_Request() and below the - * ==== separator, is intended to (over time) be(ome) protocol agnostic. - * We already use this now with ESI:includes, which are for all relevant - * purposes a different "protocol" + * - The blue path is where a request disembarks its worker thread while + * waiting for a busy object to become available: * - * A special complication is the fact that we can suspend processing of - * a request when hash-lookup finds a busy objhdr. + * - The green path is where we time out waiting for the next request to + * arrive, release the worker thread and hand the session to the waiter. + * + * Render the graph with: + * sed -n '/^..DOT/s///p' % | dot -Tps > /tmp/_.ps + * + *DOT digraph vcl_center { + *DOT size="7.2,10.5" + *DOT margin="0.5" + *DOT center="1" + *DOT + *DOT acceptor -> http1_wait [label=S_STP_NEWREQ, align=center] + *DOT hash -> CNT_Request [label="Busy object\nS_STP_WORKING\nR_STP_LOOKUP" + *DOT color=blue] + *DOT disembark -> hash [style=dotted, color=blue] + *DOT http1_wait -> CNT_Request [label="S_STP_WORKING\nR_STP_START"] + *DOT http1_wait -> disembark [label="Session close"] + *DOT http1_wait -> disembark [label="Timeout" color=green] + *DOT disembark -> waiter [style=dotted, color=green] + *DOT waiter -> http1_wait [color=green] + *DOT CNT_Request -> disembark + *DOT [label="Busy object\nS_STP_WORKING\nR_STP_LOOKUP" color=blue] + *DOT CNT_Request -> http1_cleanup + *DOT http1_cleanup -> disembark [label="Session close"] + *DOT http1_cleanup -> CNT_Request [label="S_STP_WORKING\nR_STP_START"] + *DOT http1_cleanup -> http1_wait [label="S_STP_NEWREQ"] + *DOT + *DOT } * - * Since the states are rather nasty in detail, I have decided to embedd - * a dot(1) graph in the source code comments. So to see the big picture, - * extract the DOT lines and run though dot(1), for instance with the - * command: - * sed -n '/^DOT/s///p' cache/cache_center.c | dot -Tps > /tmp/_.ps - */ - -/* -DOT digraph vcl_center { -xDOT page="8.2,11.5" -DOT size="7.2,10.5" -DOT margin="0.5" -DOT center="1" -DOT acceptor [ -DOT shape=hexagon -DOT label="Request received" -DOT ] -DOT ERROR [shape=plaintext] -DOT RESTART [shape=plaintext] -DOT acceptor -> first [style=bold,color=green] */ #include "config.h" @@ -73,37 +75,16 @@ DOT acceptor -> first [style=bold,color=green] #include "cache.h" -#include "hash/hash_slinger.h" #include "vcl.h" -#include "vcli_priv.h" -#include "vsha256.h" #include "vtcp.h" #include "vtim.h" -#ifndef HAVE_SRANDOMDEV -#include "compat/srandomdev.h" -#endif - - /*-------------------------------------------------------------------- - * WAIT - * Collect the request from the client. - * -DOT subgraph xcluster_wait { -DOT wait [ -DOT shape=box -DOT label="cnt_sess_wait:\nwait for\ncomplete\nrequest" -DOT ] -DOT herding [shape=hexagon] -DOT wait -> start [label="got req",style=bold,color=green] -DOT wait -> "SES_Delete()" [label="errors"] -DOT wait -> herding [label="timeout_linger"] -DOT herding -> wait [label="fd read_ready"] -DOT } + * Collect a request from the client. */ static int -cnt_sess_wait(struct sess *sp, struct worker *wrk, struct req *req) +http1_wait(struct sess *sp, struct worker *wrk, struct req *req) { int j, tmo; struct pollfd pfd[1]; @@ -117,7 +98,6 @@ cnt_sess_wait(struct sess *sp, struct worker *wrk, struct req *req) assert(req->sp == sp); - AZ(req->vcl); AZ(req->obj); AZ(req->esi_level); @@ -184,25 +164,16 @@ cnt_sess_wait(struct sess *sp, struct worker *wrk, struct req *req) /*-------------------------------------------------------------------- * This is the final state, figure out if we should close or recycle * the client connection - * -DOT DONE [ -DOT shape=record -DOT label="{cnt_done:|Request completed}" -DOT ] -DOT ESI_RESP [ shape=hexagon ] -DOT DONE -> start [label="full pipeline"] -DOT DONE -> wait -DOT DONE -> ESI_RESP */ -enum cnt_sess_done_ret { +enum http1_cleanup_ret { SESS_DONE_RET_GONE, SESS_DONE_RET_WAIT, SESS_DONE_RET_START, }; -static enum cnt_sess_done_ret -cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) +static enum http1_cleanup_ret +http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); @@ -225,7 +196,7 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) } sp->t_idle = W_TIM_real(wrk); - if (req->xid == 0) + if (req->xid == 0) req->t_resp = sp->t_idle; req->xid = 0; VSL_Flush(req->vsl, 0); @@ -271,11 +242,11 @@ cnt_sess_done(struct sess *sp, struct worker *wrk, struct req *req) */ void -CNT_Session(struct worker *wrk, struct req *req) +HTTP1_Session(struct worker *wrk, struct req *req) { int done; struct sess *sp; - enum cnt_sess_done_ret sdr; + enum http1_cleanup_ret sdr; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -295,7 +266,7 @@ CNT_Session(struct worker *wrk, struct req *req) SES_Close(sp, SC_REM_CLOSE); else SES_Close(sp, SC_TX_ERROR); - sdr = cnt_sess_done(sp, wrk, req); + sdr = http1_cleanup(sp, wrk, req); assert(sdr == SESS_DONE_RET_GONE); return; } @@ -307,10 +278,6 @@ CNT_Session(struct worker *wrk, struct req *req) } while (1) { - /* - * Possible entrance states - */ - assert( sp->sess_step == S_STP_NEWREQ || req->req_step == R_STP_LOOKUP || @@ -321,7 +288,7 @@ CNT_Session(struct worker *wrk, struct req *req) if (done == 2) return; assert(done == 1); - sdr = cnt_sess_done(sp, wrk, req); + sdr = http1_cleanup(sp, wrk, req); switch (sdr) { case SESS_DONE_RET_GONE: return; @@ -333,12 +300,12 @@ CNT_Session(struct worker *wrk, struct req *req) req->req_step = R_STP_START; break; default: - WRONG("Illegal enum cnt_sess_done_ret"); + WRONG("Illegal enum http1_cleanup_ret"); } } if (sp->sess_step == S_STP_NEWREQ) { - done = cnt_sess_wait(sp, wrk, req); + done = http1_wait(sp, wrk, req); if (done) return; sp->sess_step = S_STP_WORKING; diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index d67d5ff..41dca5c 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -139,7 +139,7 @@ ses_req_pool_task(struct worker *wrk, void *arg) THR_SetRequest(req); AZ(wrk->aws->r); wrk->lastused = NAN; - CNT_Session(wrk, req); + HTTP1_Session(wrk, req); WS_Assert(wrk->aws); AZ(wrk->wrw); if (cache_param->diag_bitmap & 0x00040000) { From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] 27f24b5 Fix the dot graph of the request fsm Message-ID: commit 27f24b57f958f02854d63e48aca1ce0baf29928c Author: Poul-Henning Kamp Date: Tue Aug 7 08:19:11 2012 +0000 Fix the dot graph of the request fsm diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 920d067..89931aa 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -767,7 +767,7 @@ void VBO_Free(struct busyobj **vbo); /* cache_http1_fsm.c [HTTP1] */ void HTTP1_Session(struct worker *, struct req *); -/* cache_req_fsm.c [FSM] */ +/* cache_req_fsm.c [CNT] */ int CNT_Request(struct worker *, struct req *); void CNT_Init(void); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 34cfbfa..588b4ff 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -26,13 +26,8 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * This file contains the two central state machine for pushing - * sessions and requests. - * - * The first part of the file, entrypoint CNT_Session() and down to - * the ==== separator, is concerned with sessions. When a session has - * a request to deal with, it calls into the second half of the file. - * This part is for all practical purposes HTTP/1.x specific. + * This file contains the two central state machine for pushing HTTP + * requests through their paces. * * The second part of the file, entrypoint CNT_Request() and below the * ==== separator, is intended to (over time) be(ome) protocol agnostic. @@ -46,7 +41,7 @@ * a dot(1) graph in the source code comments. So to see the big picture, * extract the DOT lines and run though dot(1), for instance with the * command: - * sed -n '/^DOT/s///p' cache/cache_center.c | dot -Tps > /tmp/_.ps + * sed -n '/^DOT/s///p' cache/cache_req_fsm.c | dot -Tps > /tmp/_.ps */ /* @@ -61,7 +56,7 @@ DOT label="Request received" DOT ] DOT ERROR [shape=plaintext] DOT RESTART [shape=plaintext] -DOT acceptor -> first [style=bold,color=green] +DOT acceptor -> start [style=bold,color=green] */ #include "config.h" @@ -1284,7 +1279,7 @@ CNT_Request(struct worker *wrk, struct req *req) assert( req->req_step == R_STP_LOOKUP || req->req_step == R_STP_START || - req->req_step == R_STP_RECV); + req->req_step == R_STP_RECV); // from ESI req->wrk = wrk; From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] 279bc2e Move most of CNT::start into HTTP1_fsm where it belongs. Message-ID: commit 279bc2eae8f73555e392909a936c63e720ba2b7b Author: Poul-Henning Kamp Date: Tue Aug 7 08:53:47 2012 +0000 Move most of CNT::start into HTTP1_fsm where it belongs. diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index ae01a00..e8c26ca 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -241,10 +241,57 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req) /*-------------------------------------------------------------------- */ +static int +http1_dissect(struct worker *wrk, struct req *req) +{ + const char *r = "HTTP/1.1 100 Continue\r\n\r\n"; + char *p; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + /* Borrow VCL reference from worker thread */ + VCL_Refresh(&wrk->vcl); + req->vcl = wrk->vcl; + wrk->vcl = NULL; + + HTTP_Setup(req->http, req->ws, req->vsl, HTTP_Req); + req->err_code = http_DissectRequest(req); + + /* If we could not even parse the request, just close */ + if (req->err_code == 400) { + SES_Close(req->sp, SC_RX_JUNK); + return (1); + } + + req->ws_req = WS_Snapshot(req->ws); + req->doclose = http_DoConnection(req->http); + + /* XXX: Expect headers are a mess */ + if (req->err_code == 0 && http_GetHdr(req->http, H_Expect, &p)) { + if (strcasecmp(p, "100-continue")) { + req->err_code = 417; + } else if (strlen(r) != write(req->sp->fd, r, strlen(r))) { + SES_Close(req->sp, SC_REM_CLOSE); + return (1); + } + } + http_Unset(req->http, H_Expect); + /* XXX: pull in req-body and make it available instead. */ + req->reqbodydone = 0; + + HTTP_Copy(req->http0, req->http); // For ESI & restart + + return (0); +} + +/*-------------------------------------------------------------------- + */ + void HTTP1_Session(struct worker *wrk, struct req *req) { - int done; + int done = 0; struct sess *sp; enum http1_cleanup_ret sdr; @@ -284,7 +331,10 @@ HTTP1_Session(struct worker *wrk, struct req *req) req->req_step == R_STP_START); if (sp->sess_step == S_STP_WORKING) { - done = CNT_Request(wrk, req); + if (req->req_step == R_STP_START) + done = http1_dissect(wrk, req); + if (done == 0) + done = CNT_Request(wrk, req); if (done == 2) return; assert(done == 1); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 588b4ff..63ba22c 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -1181,14 +1181,12 @@ DOT start -> DONE [label=errors] static int cnt_start(struct worker *wrk, struct req *req) { - char *p; - const char *r = "HTTP/1.1 100 Continue\r\n\r\n"; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AZ(req->restarts); AZ(req->obj); - AZ(req->vcl); + AN(req->vcl); AZ(req->esi_level); assert(!isnan(req->t_req)); @@ -1201,45 +1199,8 @@ cnt_start(struct worker *wrk, struct req *req) VSLb(req->vsl, SLT_ReqStart, "%s %s %u", req->sp->addr, req->sp->port, req->xid); - /* Borrow VCL reference from worker thread */ - VCL_Refresh(&wrk->vcl); - req->vcl = wrk->vcl; - wrk->vcl = NULL; - EXP_Clr(&req->exp); - HTTP_Setup(req->http, req->ws, req->vsl, HTTP_Req); - req->err_code = http_DissectRequest(req); - - /* If we could not even parse the request, just close */ - if (req->err_code == 400) { - SES_Close(req->sp, SC_RX_JUNK); - return (1); - } - - req->ws_req = WS_Snapshot(req->ws); - - req->doclose = http_DoConnection(req->http); - - /* - * We want to deal with Expect: headers the first time we - * attempt the request, and remove them before we move on. - */ - if (req->err_code == 0 && http_GetHdr(req->http, H_Expect, &p)) { - if (strcasecmp(p, "100-continue")) { - req->err_code = 417; - } else if (strlen(r) != write(req->sp->fd, r, strlen(r))) { - SES_Close(req->sp, SC_REM_CLOSE); - return (1); - } - } - http_Unset(req->http, H_Expect); - - /* XXX: pull in req-body and make it available instead. */ - req->reqbodydone = 0; - - HTTP_Copy(req->http0, req->http); /* Copy for restart/ESI use */ - if (req->err_code) req->req_step = R_STP_ERROR; else From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] d11b853 Shave another couple of lines out of cnt_start{} Message-ID: commit d11b853126d1be95a8bf7412975753c6bd4bde49 Author: Poul-Henning Kamp Date: Tue Aug 7 09:38:57 2012 +0000 Shave another couple of lines out of cnt_start{} diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index e8c26ca..0cee630 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -264,6 +264,9 @@ http1_dissect(struct worker *wrk, struct req *req) return (1); } + wrk->stats.client_req++; + req->acct_req.req++; + req->ws_req = WS_Snapshot(req->ws); req->doclose = http_DoConnection(req->http); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 63ba22c..b809c6d 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -1110,6 +1110,8 @@ cnt_recv(const struct worker *wrk, struct req *req) req->director = req->vcl->director[0]; AN(req->director); + EXP_Clr(&req->exp); + req->disable_esi = 0; req->hash_always_miss = 0; req->hash_ignore_busy = 0; @@ -1190,17 +1192,11 @@ cnt_start(struct worker *wrk, struct req *req) AZ(req->esi_level); assert(!isnan(req->t_req)); - /* Update stats of various sorts */ - wrk->stats.client_req++; - req->acct_req.req++; - /* Assign XID and log */ req->xid = ++xids; /* XXX not locked */ VSLb(req->vsl, SLT_ReqStart, "%s %s %u", req->sp->addr, req->sp->port, req->xid); - EXP_Clr(&req->exp); - if (req->err_code) req->req_step = R_STP_ERROR; else From phk at FreeBSD.org Thu Dec 18 09:27:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:52 +0100 Subject: [experimental-ims] 5ce8786 Expose SES_GetReq() Message-ID: commit 5ce878666db6db0b8d5d0ff61c5f28442ab8ea31 Author: Poul-Henning Kamp Date: Wed Aug 8 07:57:36 2012 +0000 Expose SES_GetReq() diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 89931aa..71bba01 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -959,6 +959,7 @@ void SES_Charge(struct worker *, struct req *); struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no); void SES_DeletePool(struct sesspool *sp); int SES_ScheduleReq(struct req *); +struct req *SES_GetReq(struct sess *sp); void SES_Handle(struct sess *sp, double now); void SES_ReleaseReq(struct req *); pool_func_t SES_pool_accept_task; diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 41dca5c..9d3d3cb 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -52,8 +52,6 @@ static unsigned ses_size = sizeof (struct sess); -static struct req * ses_GetReq(struct sess *sp); - /*--------------------------------------------------------------------*/ struct sesspool { @@ -162,7 +160,7 @@ ses_sess_pool_task(struct worker *wrk, void *arg) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CAST_OBJ_NOTNULL(sp, arg, SESS_MAGIC); - req = ses_GetReq(sp); + req = SES_GetReq(sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); req->vxid = VXID_Get(&wrk->vxid_pool); @@ -346,8 +344,8 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now) * Alloc/Free a request */ -static struct req * -ses_GetReq(struct sess *sp) +struct req * +SES_GetReq(struct sess *sp) { struct sesspool *pp; struct req *req; From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] d9a9ecd Doing rollback in a esi:include request would roll back to the original ESI processed request, rather than to the included requests. Message-ID: commit d9a9ecd999f78123ac6ef5dfa8fd4aac38130c26 Author: Poul-Henning Kamp Date: Wed Aug 8 10:04:57 2012 +0000 Doing rollback in a esi:include request would roll back to the original ESI processed request, rather than to the included requests. Fix by allocating/cloning a new request for the esi:include transactions. Testcase by: scoof Fixes #1168 diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index a6e5f11..daef6a9 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -42,80 +42,93 @@ /*--------------------------------------------------------------------*/ static void -ved_include(struct req *req, const char *src, const char *host) +ved_include(struct req *preq, const char *src, const char *host) { - struct object *obj; struct worker *wrk; - char *sp_ws_wm; + struct req *req; char *wrk_ws_wm; - unsigned sxid, res_mode; int i; - wrk = req->wrk; + wrk = preq->wrk; - if (req->esi_level >= cache_param->max_esi_depth) + if (preq->esi_level >= cache_param->max_esi_depth) return; - req->esi_level++; (void)WRW_FlushRelease(wrk); - obj = req->obj; - req->obj = NULL; - res_mode = req->res_mode; - - /* Reset request to status before we started messing with it */ - HTTP_Copy(req->http, req->http0); - /* Take a workspace snapshot */ - sp_ws_wm = WS_Snapshot(req->ws); wrk_ws_wm = WS_Snapshot(wrk->aws); /* XXX ? */ - http_SetH(req->http, HTTP_HDR_URL, src); + req = SES_GetReq(preq->sp); + req->esi_level = preq->esi_level + 1; + + HTTP_Copy(req->http0, preq->http0); + + req->http0->conds = 0; + + HTTP_Setup(req->http, req->ws, req->vsl, HTTP_Req); + + http_SetH(req->http0, HTTP_HDR_URL, src); if (host != NULL && *host != '\0') { - http_Unset(req->http, H_Host); - http_Unset(req->http, H_If_Modified_Since); - http_SetHeader(req->http, host); + http_Unset(req->http0, H_Host); + http_SetHeader(req->http0, host); } + + http_ForceGet(req->http0); + http_Unset(req->http0, H_If_Modified_Since); + + /* Client content already taken care of */ + http_Unset(req->http0, H_Content_Length); + + /* Reset request to status before we started messing with it */ + HTTP_Copy(req->http, req->http0); + + req->vcl = preq->vcl; + preq->vcl = NULL; + req->wrk = preq->wrk; + /* * XXX: We should decide if we should cache the director * XXX: or not (for session/backend coupling). Until then * XXX: make sure we don't trip up the check in vcl_recv. */ - req->director = NULL; req->req_step = R_STP_RECV; - http_ForceGet(req->http); + req->t_req = preq->t_req; + req->gzip_resp = preq->gzip_resp; + req->crc = preq->crc; + req->l_crc = preq->l_crc; - /* Don't do conditionals */ - req->http->conds = 0; - http_Unset(req->http, H_If_Modified_Since); + THR_SetRequest(req); - /* Client content already taken care of */ - http_Unset(req->http, H_Content_Length); - - sxid = req->xid; while (1) { req->wrk = wrk; i = CNT_Request(wrk, req); if (i == 1) break; + DSL(0x20, SLT_Debug, req->sp->vsl_id, + "loop waiting for ESI (%d)", i); assert(i == 2); AZ(req->wrk); - DSL(0x20, SLT_Debug, req->sp->vsl_id, "loop waiting for ESI"); (void)usleep(10000); } - req->xid = sxid; - req->wrk = wrk; - req->esi_level--; - req->obj = obj; - req->res_mode = res_mode; /* Reset the workspace */ - WS_Reset(req->ws, sp_ws_wm); WS_Reset(wrk->aws, wrk_ws_wm); /* XXX ? */ - WRW_Reserve(req->wrk, &req->sp->fd, req->vsl, req->t_resp); - if (req->res_mode & RES_CHUNKED) - WRW_Chunked(req->wrk); + WRW_Reserve(preq->wrk, &preq->sp->fd, preq->vsl, preq->t_resp); + if (preq->res_mode & RES_CHUNKED) + WRW_Chunked(preq->wrk); + + preq->vcl = req->vcl; + req->vcl = NULL; + + preq->crc = req->crc; + preq->l_crc = req->l_crc; + + req->wrk = NULL; + + THR_SetRequest(preq); + SES_ReleaseReq(req); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishtest/tests/r01168.vtc b/bin/varnishtest/tests/r01168.vtc new file mode 100644 index 0000000..fb0a0c5 --- /dev/null +++ b/bin/varnishtest/tests/r01168.vtc @@ -0,0 +1,26 @@ +varnishtest "Test ESI rollback interaction" + +server s1 { + rxreq + expect req.url == "/" + txresp -body {} + + rxreq + expect req.url == "/esi" + txresp -body "1" +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + rollback; + } + sub vcl_fetch { + set beresp.do_esi = true; + } +} -start + +client c1 { + txreq + rxresp + expect resp.bodylen == 1 +} -run From apj at mutt.dk Thu Dec 18 09:27:53 2014 From: apj at mutt.dk (Andreas Plesner Jacobsen) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] e9a47e8 Spelling Message-ID: commit e9a47e802bc95d4685b5c0a1eee50a9ce9d497c5 Author: Andreas Plesner Jacobsen Date: Wed Aug 8 21:40:52 2012 +0200 Spelling diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index f4f880d..3b49a85 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -4,7 +4,7 @@ Poul-Hennings random outbursts %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -You may or may not want to know what Poul-Henning think. +You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] cc710d5 Explicitly sync the header of the VSM so other processes can read(2) it on kernels without coherent VM/buf (OpenBSD, still ?, really ?) Message-ID: commit cc710d56021a4d1927ceedf426cebfff48ef1860 Author: Poul-Henning Kamp Date: Thu Aug 9 08:22:24 2012 +0000 Explicitly sync the header of the VSM so other processes can read(2) it on kernels without coherent VM/buf (OpenBSD, still ?, really ?) Submitted by: Federico G. Schwindt diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c index 79f8c41..bc59194 100644 --- a/bin/varnishd/mgt/mgt_shmem.c +++ b/bin/varnishd/mgt/mgt_shmem.c @@ -187,8 +187,7 @@ mgt_shm_size(void) size = mgt_param.vsl_space + mgt_param.vsm_space; ps = getpagesize(); - size += ps - 1; - size &= ~(ps - 1U); + size = RUP2(size, ps); return (size); } @@ -247,6 +246,9 @@ mgt_SHM_Create(void) (void)unlink(fnbuf); exit (-1); } + + /* Commit changes, for OS's without coherent VM/buf */ + AZ(msync(p, getpagesize(), MS_SYNC)); } /*-------------------------------------------------------------------- From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] efaba47 Recognize OpenBSD as platform. Federico says it almost works. Message-ID: commit efaba477e6235e283252d3bd63c570e02166ef64 Author: Poul-Henning Kamp Date: Thu Aug 9 08:27:03 2012 +0000 Recognize OpenBSD as platform. Federico says it almost works. Submitted by: Federico G. Schwindt diff --git a/autogen.sh b/autogen.sh index 07df626..6d28093 100755 --- a/autogen.sh +++ b/autogen.sh @@ -12,6 +12,9 @@ Darwin) FreeBSD) LIBTOOLIZE=libtoolize ;; +OpenBSD) + LIBTOOLIZE=libtoolize + ;; Linux) LIBTOOLIZE=libtoolize ;; From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] 1b3b48e Remove long gone cli.debug cli calls Message-ID: commit 1b3b48e3e9b3ea1d8659da13fb78983d8299bed2 Author: Poul-Henning Kamp Date: Thu Aug 9 08:28:56 2012 +0000 Remove long gone cli.debug cli calls diff --git a/bin/varnishtest/tests/v00006.vtc b/bin/varnishtest/tests/v00006.vtc index ca4c827..494007f 100644 --- a/bin/varnishtest/tests/v00006.vtc +++ b/bin/varnishtest/tests/v00006.vtc @@ -41,7 +41,7 @@ varnish v1 -expect n_backend == 2 varnish v1 -expect n_vcl_avail == 2 varnish v1 -expect n_vcl_discard == 0 -varnish v1 -cli "debug.backend" -cli "vcl.list" +varnish v1 -cli "vcl.list" # Discard the first VCL @@ -64,7 +64,7 @@ client c1 { # The workthread should have released its VCL reference now # but we need to tickle the CLI to notice -varnish v1 -cli "debug.backend" -cli "vcl.list" +varnish v1 -cli "vcl.list" varnish v1 -expect n_backend == 1 varnish v1 -expect n_vcl_avail == 1 From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] a47e651 Remove long since gone debug.backend cli. Message-ID: commit a47e651f8632c4da0b6be5e4b5d8db386a6bd8f4 Author: Poul-Henning Kamp Date: Thu Aug 9 08:29:39 2012 +0000 Remove long since gone debug.backend cli. Submitted by: Federico G. Schwindt diff --git a/bin/varnishtest/tests/v00012.vtc b/bin/varnishtest/tests/v00012.vtc index 285f2c1..e7c8d44 100644 --- a/bin/varnishtest/tests/v00012.vtc +++ b/bin/varnishtest/tests/v00012.vtc @@ -33,8 +33,6 @@ client c2 { expect resp.status == 503 } -run -varnish v1 -cli "debug.backend" - sema r2 sync 2 client c1 -wait From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] 5d82c73 Remove comment OBE. Message-ID: commit 5d82c73b6686d66bfd0279fd3b7be6bd1767015f Author: Poul-Henning Kamp Date: Thu Aug 9 09:49:06 2012 +0000 Remove comment OBE. diff --git a/bin/varnishd/cache/cache_httpconn.c b/bin/varnishd/cache/cache_httpconn.c index 6ebb86c..8b331af 100644 --- a/bin/varnishd/cache/cache_httpconn.c +++ b/bin/varnishd/cache/cache_httpconn.c @@ -96,12 +96,6 @@ HTC_Reinit(struct http_conn *htc) } /*-------------------------------------------------------------------- - * Return -3 if it's all whitespace so far - * Return 0 if we need more text - * Return 1 if we have a complete HTTP procol header - */ - -/*-------------------------------------------------------------------- * Check if we have a complete HTTP request or response yet * */ From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] 09c459c Eliminate the cnt_start state by folding it into cnt_recv Message-ID: commit 09c459cb263f2551213d52fb4c56b27e7757cf71 Author: Poul-Henning Kamp Date: Mon Aug 13 06:48:12 2012 +0000 Eliminate the cnt_start state by folding it into cnt_recv diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 0cee630..31e0a46 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -50,7 +50,7 @@ *DOT hash -> CNT_Request [label="Busy object\nS_STP_WORKING\nR_STP_LOOKUP" *DOT color=blue] *DOT disembark -> hash [style=dotted, color=blue] - *DOT http1_wait -> CNT_Request [label="S_STP_WORKING\nR_STP_START"] + *DOT http1_wait -> CNT_Request [label="S_STP_WORKING\nR_STP_RECV"] *DOT http1_wait -> disembark [label="Session close"] *DOT http1_wait -> disembark [label="Timeout" color=green] *DOT disembark -> waiter [style=dotted, color=green] @@ -59,7 +59,7 @@ *DOT [label="Busy object\nS_STP_WORKING\nR_STP_LOOKUP" color=blue] *DOT CNT_Request -> http1_cleanup *DOT http1_cleanup -> disembark [label="Session close"] - *DOT http1_cleanup -> CNT_Request [label="S_STP_WORKING\nR_STP_START"] + *DOT http1_cleanup -> CNT_Request [label="S_STP_WORKING\nR_STP_RECV"] *DOT http1_cleanup -> http1_wait [label="S_STP_NEWREQ"] *DOT *DOT } @@ -331,10 +331,10 @@ HTTP1_Session(struct worker *wrk, struct req *req) assert( sp->sess_step == S_STP_NEWREQ || req->req_step == R_STP_LOOKUP || - req->req_step == R_STP_START); + req->req_step == R_STP_RECV); if (sp->sess_step == S_STP_WORKING) { - if (req->req_step == R_STP_START) + if (req->req_step == R_STP_RECV) done = http1_dissect(wrk, req); if (done == 0) done = CNT_Request(wrk, req); @@ -350,7 +350,7 @@ HTTP1_Session(struct worker *wrk, struct req *req) break; case SESS_DONE_RET_START: sp->sess_step = S_STP_WORKING; - req->req_step = R_STP_START; + req->req_step = R_STP_RECV; break; default: WRONG("Illegal enum http1_cleanup_ret"); @@ -362,7 +362,7 @@ HTTP1_Session(struct worker *wrk, struct req *req) if (done) return; sp->sess_step = S_STP_WORKING; - req->req_step = R_STP_START; + req->req_step = R_STP_RECV; } } } diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index b809c6d..0fbca29 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -1105,6 +1105,16 @@ cnt_recv(const struct worker *wrk, struct req *req) AZ(req->obj); AZ(req->busyobj); + /* Assign XID and log */ + req->xid = ++xids; /* XXX not locked */ + VSLb(req->vsl, SLT_ReqStart, "%s %s %u", + req->sp->addr, req->sp->port, req->xid); + + if (req->err_code) { + req->req_step = R_STP_ERROR; + return (0); + } + /* By default we use the first backend */ AZ(req->director); req->director = req->vcl->director[0]; @@ -1169,42 +1179,6 @@ cnt_recv(const struct worker *wrk, struct req *req) } /*-------------------------------------------------------------------- - * START - * First time we see a request - * -DOT start [ -DOT shape=box -DOT label="cnt_start:\nDissect request\nHandle expect" -DOT ] -DOT start -> recv [style=bold,color=green] -DOT start -> DONE [label=errors] - */ - -static int -cnt_start(struct worker *wrk, struct req *req) -{ - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - AZ(req->restarts); - AZ(req->obj); - AN(req->vcl); - AZ(req->esi_level); - assert(!isnan(req->t_req)); - - /* Assign XID and log */ - req->xid = ++xids; /* XXX not locked */ - VSLb(req->vsl, SLT_ReqStart, "%s %s %u", - req->sp->addr, req->sp->port, req->xid); - - if (req->err_code) - req->req_step = R_STP_ERROR; - else - req->req_step = R_STP_RECV; - return (0); -} - -/*-------------------------------------------------------------------- * Central state engine dispatcher. * * Kick the session around until it has had enough. @@ -1235,8 +1209,7 @@ CNT_Request(struct worker *wrk, struct req *req) */ assert( req->req_step == R_STP_LOOKUP || - req->req_step == R_STP_START || - req->req_step == R_STP_RECV); // from ESI + req->req_step == R_STP_RECV); req->wrk = wrk; diff --git a/include/tbl/steps.h b/include/tbl/steps.h index cef939b..3ceaa5e 100644 --- a/include/tbl/steps.h +++ b/include/tbl/steps.h @@ -38,7 +38,6 @@ SESS_STEP(working, WORKING) #ifdef REQ_STEP REQ_STEP(restart, RESTART, (wrk, req)) REQ_STEP(recv, RECV, (wrk, req)) -REQ_STEP(start, START, (wrk, req)) REQ_STEP(pipe, PIPE, (wrk, req)) REQ_STEP(pass, PASS, (wrk, req)) REQ_STEP(lookup, LOOKUP, (wrk, req)) From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] e7d5afe Allocate a unique "vxid" to all sessions, requests and backend requests (busyobj). Message-ID: commit e7d5afec68f4a62ca8037d928c6298fca7691287 Author: Poul-Henning Kamp Date: Mon Aug 13 09:27:41 2012 +0000 Allocate a unique "vxid" to all sessions, requests and backend requests (busyobj). This replaces the current "VID" as visible in the X-Varnish header. The new "stolen numbers" causes a lot of testcases to need updates, most trivial, but a few needed more extensive work to be predictable under the new circumstances. Next step is to carry this change through in VSL, and link all the VXID's to each other. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 71bba01..35c2b52 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -543,7 +543,7 @@ VTAILQ_HEAD(storagehead, storage); struct object { unsigned magic; #define OBJECT_MAGIC 0x32851d42 - unsigned xid; + uint32_t vxid; struct storage *objstore; struct objcore *objcore; @@ -584,7 +584,6 @@ struct req { #define REQ_MAGIC 0x2751aaa1 uint32_t vxid; - unsigned xid; int restarts; int esi_level; int disable_esi; @@ -769,7 +768,6 @@ void HTTP1_Session(struct worker *, struct req *); /* cache_req_fsm.c [CNT] */ int CNT_Request(struct worker *, struct req *); -void CNT_Init(void); /* cache_cli.c [CLI] */ void CLI_Init(void); @@ -1054,7 +1052,7 @@ char *WS_Alloc(struct ws *ws, unsigned bytes); char *WS_Snapshot(struct ws *ws); /* rfc2616.c */ -void RFC2616_Ttl(struct busyobj *, unsigned xid); +void RFC2616_Ttl(struct busyobj *); enum body_status RFC2616_Body(struct busyobj *, struct dstat *); unsigned RFC2616_Req_Gzip(const struct http *); int RFC2616_Do_Cond(const struct req *sp); diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index 4164e4b..c949acc 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -758,7 +758,7 @@ ban_check_object(struct object *o, struct vsl_log *vsl, oc_updatemeta(oc); /* BAN also changed, but that is not important any more */ /* XXX: no req in lurker */ - VSLb(vsl, SLT_ExpBan, "%u was banned", o->xid); + VSLb(vsl, SLT_ExpBan, "%u was banned", o->vxid); EXP_Rearm(o); return (1); } diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index eb5998a..3b9ae2e 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -852,7 +852,7 @@ http_FilterReq(const struct req *req, unsigned how) else http_linkh(hp, req->http, HTTP_HDR_PROTO); http_filterfields(hp, req->http, how); - http_PrintfHeader(hp, "X-Varnish: %u", req->xid); + http_PrintfHeader(hp, "X-Varnish: %u", req->vxid); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 31e0a46..d181d11 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -101,7 +101,6 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req) AZ(req->vcl); AZ(req->obj); AZ(req->esi_level); - assert(req->xid == 0); assert(isnan(req->t_req)); assert(isnan(req->t_resp)); @@ -196,9 +195,6 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req) } sp->t_idle = W_TIM_real(wrk); - if (req->xid == 0) - req->t_resp = sp->t_idle; - req->xid = 0; VSL_Flush(req->vsl, 0); req->t_req = NAN; @@ -225,7 +221,6 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req) WS_Reset(req->ws, NULL); WS_Reset(wrk->aws, NULL); - req->vxid = VXID_Get(&wrk->vxid_pool); if (HTC_Reinit(req->htc) == HTC_COMPLETE) { req->t_req = sp->t_idle; @@ -341,6 +336,7 @@ HTTP1_Session(struct worker *wrk, struct req *req) if (done == 2) return; assert(done == 1); + assert(req->vxid == 0); sdr = http1_cleanup(sp, wrk, req); switch (sdr) { case SESS_DONE_RET_GONE: diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index c48a3d9..37aea3a 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -35,6 +35,8 @@ #include "cache.h" #include "common/heritage.h" +#include "vcli_priv.h" + #include "waiter/waiter.h" #include "hash/hash_slinger.h" @@ -93,6 +95,7 @@ THR_GetName(void) */ static uint32_t vxid_base; +static uint32_t vxid_chunk = 32768; static struct lock vxid_lock; uint32_t @@ -102,8 +105,8 @@ VXID_Get(struct vxid_pool *v) if (v->count == 0) { Lck_Lock(&vxid_lock); v->next = vxid_base; - v->count = 32768; - vxid_base = v->count; + v->count = vxid_chunk; + vxid_base += v->count; Lck_Unlock(&vxid_lock); } v->count--; @@ -113,6 +116,52 @@ VXID_Get(struct vxid_pool *v) } /*-------------------------------------------------------------------- + * Debugging aids + */ + +/* + * Dumb down the VXID allocation to make it predictable for + * varnishtest cases + */ +static void +cli_debug_xid(struct cli *cli, const char * const *av, void *priv) +{ + (void)priv; + if (av[2] != NULL) { + vxid_base = strtoul(av[2], NULL, 0); + vxid_chunk = 1; + } + VCLI_Out(cli, "XID is %u", vxid_base); +} + +/* + * Default to seed=1, this is the only seed value POSIXl guarantees will + * result in a reproducible random number sequence. + */ +static void +cli_debug_srandom(struct cli *cli, const char * const *av, void *priv) +{ + (void)priv; + unsigned seed = 1; + + if (av[2] != NULL) + seed = strtoul(av[2], NULL, 0); + srandom(seed); + srand48(random()); + VCLI_Out(cli, "Random(3) seeded with %u", seed); +} + +static struct cli_proto debug_cmds[] = { + { "debug.xid", "debug.xid", + "\tExamine or set XID\n", 0, 1, "d", cli_debug_xid }, + { "debug.srandom", "debug.srandom", + "\tSeed the random(3) function\n", 0, 1, "d", + cli_debug_srandom }, + { NULL } +}; + + +/*-------------------------------------------------------------------- * XXX: Think more about which order we start things */ @@ -142,7 +191,6 @@ child_main(void) CLI_Init(); Fetch_Init(); - CNT_Init(); VCL_Init(); HTTP_Init(); @@ -168,6 +216,10 @@ child_main(void) BAN_Compile(); + srandomdev(); + srand48(random()); + CLI_AddFuncs(debug_cmds); + /* Wait for persistent storage to load if asked to */ if (cache_param->diag_bitmap & 0x00020000) SMP_Ready(); diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index ff9a65b..c38da38 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -164,7 +164,7 @@ pan_object(const struct object *o) const struct storage *st; VSB_printf(pan_vsp, " obj = %p {\n", o); - VSB_printf(pan_vsp, " xid = %u,\n", o->xid); + VSB_printf(pan_vsp, " vxid = %u,\n", o->vxid); pan_ws(o->ws_o, 4); pan_http("obj", o->http, 4); VSB_printf(pan_vsp, " len = %jd,\n", (intmax_t)o->len); @@ -236,8 +236,8 @@ pan_req(const struct req *req) VSB_printf(pan_vsp, "req = %p {\n", req); - VSB_printf(pan_vsp, " sp = %p, xid = %u,", - req->sp, req->xid); + VSB_printf(pan_vsp, " sp = %p, vxid = %u,", + req->sp, req->vxid); switch (req->req_step) { #define REQ_STEP(l, u, arg) case R_STP_##u: stp = "R_STP_" #u; break; diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 0fbca29..6e03e4c 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -62,7 +62,6 @@ DOT acceptor -> start [style=bold,color=green] #include "config.h" #include -#include #include #include @@ -70,7 +69,6 @@ DOT acceptor -> start [style=bold,color=green] #include "hash/hash_slinger.h" #include "vcl.h" -#include "vcli_priv.h" #include "vsha256.h" #include "vtim.h" @@ -78,7 +76,6 @@ DOT acceptor -> start [style=bold,color=green] #include "compat/srandomdev.h" #endif -static unsigned xids; /*-------------------------------------------------------------------- * We have a refcounted object on the session, and possibly the busyobj * which is fetching it, prepare a response. @@ -289,7 +286,7 @@ cnt_error(struct worker *wrk, struct req *req) return(1); } CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); - req->obj->xid = req->xid; + req->obj->vxid = req->vxid; req->obj->exp.entered = req->t_req; h = req->obj->http; @@ -406,7 +403,7 @@ cnt_fetch(struct worker *wrk, struct req *req) */ EXP_Clr(&bo->exp); bo->exp.entered = W_TIM_real(wrk); - RFC2616_Ttl(bo, req->xid); + RFC2616_Ttl(bo); /* pass from vclrecv{} has negative TTL */ if (req->objcore->objhead == NULL) @@ -629,7 +626,7 @@ cnt_fetchbody(struct worker *wrk, struct req *req) VSB_delete(vary); } - req->obj->xid = req->xid; + req->obj->vxid = req->vxid; req->obj->response = req->err_code; WS_Assert(req->obj->ws_o); @@ -845,7 +842,7 @@ cnt_lookup(struct worker *wrk, struct req *req) if (oc->flags & OC_F_PASS) { wrk->stats.cache_hitpass++; - VSLb(req->vsl, SLT_HitPass, "%u", req->obj->xid); + VSLb(req->vsl, SLT_HitPass, "%u", req->obj->vxid); (void)HSH_Deref(&wrk->stats, NULL, &req->obj); AZ(req->objcore); req->req_step = R_STP_PASS; @@ -853,7 +850,7 @@ cnt_lookup(struct worker *wrk, struct req *req) } wrk->stats.cache_hit++; - VSLb(req->vsl, SLT_Hit, "%u", req->obj->xid); + VSLb(req->vsl, SLT_Hit, "%u", req->obj->vxid); req->req_step = R_STP_HIT; return (0); } @@ -1106,9 +1103,8 @@ cnt_recv(const struct worker *wrk, struct req *req) AZ(req->busyobj); /* Assign XID and log */ - req->xid = ++xids; /* XXX not locked */ VSLb(req->vsl, SLT_ReqStart, "%s %s %u", - req->sp->addr, req->sp->port, req->xid); + req->sp->addr, req->sp->port, req->vxid); if (req->err_code) { req->req_step = R_STP_ERROR; @@ -1211,6 +1207,9 @@ CNT_Request(struct worker *wrk, struct req *req) req->req_step == R_STP_LOOKUP || req->req_step == R_STP_RECV); + if (req->req_step == R_STP_RECV) + req->vxid = VXID_Get(&wrk->vxid_pool); + req->wrk = wrk; for (done = 0; !done; ) { @@ -1245,7 +1244,7 @@ CNT_Request(struct worker *wrk, struct req *req) (uintmax_t)req->req_bodybytes); } VSLb(req->vsl, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f", - req->xid, + req->vxid, req->t_req, req->sp->t_idle, req->sp->t_idle - req->t_resp, @@ -1254,6 +1253,7 @@ CNT_Request(struct worker *wrk, struct req *req) /* done == 2 was charged by cache_hash.c */ SES_Charge(wrk, req); + req->vxid = 0; } req->wrk = NULL; @@ -1265,56 +1265,3 @@ CNT_Request(struct worker *wrk, struct req *req) /* DOT } */ - -/*-------------------------------------------------------------------- - * Debugging aids - */ - -static void -cli_debug_xid(struct cli *cli, const char * const *av, void *priv) -{ - (void)priv; - if (av[2] != NULL) - xids = strtoul(av[2], NULL, 0); - VCLI_Out(cli, "XID is %u", xids); -} - -/* - * Default to seed=1, this is the only seed value POSIXl guarantees will - * result in a reproducible random number sequence. - */ -static void -cli_debug_srandom(struct cli *cli, const char * const *av, void *priv) -{ - (void)priv; - unsigned seed = 1; - - if (av[2] != NULL) - seed = strtoul(av[2], NULL, 0); - srandom(seed); - srand48(random()); - VCLI_Out(cli, "Random(3) seeded with %u", seed); -} - -static struct cli_proto debug_cmds[] = { - { "debug.xid", "debug.xid", - "\tExamine or set XID\n", 0, 1, "d", cli_debug_xid }, - { "debug.srandom", "debug.srandom", - "\tSeed the random(3) function\n", 0, 1, "d", - cli_debug_srandom }, - { NULL } -}; - -/*-------------------------------------------------------------------- - * - */ - -void -CNT_Init(void) -{ - - srandomdev(); - srand48(random()); - xids = random(); - CLI_AddFuncs(debug_cmds); -} diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index fda117d..0a554f4 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -134,11 +134,11 @@ RES_BuildHttp(struct req *req) VTIM_format(VTIM_real(), time_str); http_PrintfHeader(req->resp, "Date: %s", time_str); - if (req->xid != req->obj->xid) + if (req->vxid != req->obj->vxid) http_PrintfHeader(req->resp, - "X-Varnish: %u %u", req->xid, req->obj->xid); + "X-Varnish: %u %u", req->vxid, req->obj->vxid); else - http_PrintfHeader(req->resp, "X-Varnish: %u", req->xid); + http_PrintfHeader(req->resp, "X-Varnish: %u", req->vxid); http_PrintfHeader(req->resp, "Age: %.0f", req->obj->exp.age + req->t_resp - req->obj->exp.entered); diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 905b744..7f5fd35 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -63,7 +63,7 @@ */ void -RFC2616_Ttl(struct busyobj *bo, unsigned xid) +RFC2616_Ttl(struct busyobj *bo) { unsigned max_age, age; double h_date, h_expires; @@ -170,7 +170,7 @@ RFC2616_Ttl(struct busyobj *bo, unsigned xid) /* calculated TTL, Our time, Date, Expires, max-age, age */ VSLb(bo->vsl, SLT_TTL, "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u", - xid, expp->ttl, -1., -1., expp->entered, + bo->vxid, expp->ttl, -1., -1., expp->entered, expp->age, h_date, h_expires, max_age); } diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 9d3d3cb..c7b9a4a 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -163,8 +163,6 @@ ses_sess_pool_task(struct worker *wrk, void *arg) req = SES_GetReq(sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - req->vxid = VXID_Get(&wrk->vxid_pool); - sp->sess_step = S_STP_NEWREQ; ses_req_pool_task(wrk, req); } @@ -199,8 +197,8 @@ ses_vsl_socket(struct sess *sp, const char *lsockname) strcpy(laddr, "-"); strcpy(lport, "-"); } - VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s %.6f", - sp->addr, sp->port, lsockname, laddr, lport, sp->t_open); + VSL(SLT_SessOpen, sp->vxid, "%u %s %s %s %s %s %.6f", + sp->vxid, sp->addr, sp->port, lsockname, laddr, lport, sp->t_open); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 7563184..215e7b6 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -412,21 +412,21 @@ VRT_DO_EXP(req, req->exp, keep, 0, ) VRT_DO_EXP(obj, req->obj->exp, grace, 0, EXP_Rearm(req->obj); - vrt_wsp_exp(req, req->obj->xid, &req->obj->exp);) + vrt_wsp_exp(req, req->obj->vxid, &req->obj->exp);) VRT_DO_EXP(obj, req->obj->exp, ttl, (req->t_req - req->obj->exp.entered), EXP_Rearm(req->obj); - vrt_wsp_exp(req, req->obj->xid, &req->obj->exp);) + vrt_wsp_exp(req, req->obj->vxid, &req->obj->exp);) VRT_DO_EXP(obj, req->obj->exp, keep, 0, EXP_Rearm(req->obj); - vrt_wsp_exp(req, req->obj->xid, &req->obj->exp);) + vrt_wsp_exp(req, req->obj->vxid, &req->obj->exp);) VRT_DO_EXP(beresp, req->busyobj->exp, grace, 0, - vrt_wsp_exp(req, req->xid, &req->busyobj->exp);) + vrt_wsp_exp(req, req->vxid, &req->busyobj->exp);) VRT_DO_EXP(beresp, req->busyobj->exp, ttl, 0, - vrt_wsp_exp(req, req->xid, &req->busyobj->exp);) + vrt_wsp_exp(req, req->vxid, &req->busyobj->exp);) VRT_DO_EXP(beresp, req->busyobj->exp, keep, 0, - vrt_wsp_exp(req, req->xid, &req->busyobj->exp);) + vrt_wsp_exp(req, req->vxid, &req->busyobj->exp);) /*-------------------------------------------------------------------- * req.xid @@ -439,9 +439,9 @@ VRT_r_req_xid(const struct req *req) int size; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - size = snprintf(NULL, 0, "%u", req->xid) + 1; + size = snprintf(NULL, 0, "%u", req->vxid) + 1; AN(p = WS_Alloc(req->http->ws, size)); - assert(snprintf(p, size, "%u", req->xid) < size); + assert(snprintf(p, size, "%u", req->vxid) < size); return (p); } diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 53846a0..910808f 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -54,7 +54,7 @@ default_oc_getxid(struct dstat *ds, struct objcore *oc) struct object *o; o = oc_getobj(ds, oc); - return (o->xid); + return (o->vxid); } static struct object * __match_proto__(getobj_f) diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c index 09b2497..ff53af8 100644 --- a/bin/varnishd/storage/storage_persistent_silo.c +++ b/bin/varnishd/storage/storage_persistent_silo.c @@ -389,7 +389,7 @@ smp_oc_getxid(struct dstat *ds, struct objcore *oc) */ ASSERT_PTR_IN_SILO(sg->sc, o); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - return (o->xid); + return (o->vxid); } /*--------------------------------------------------------------------- diff --git a/bin/varnishtest/tests/b00003.vtc b/bin/varnishtest/tests/b00003.vtc index 8c2bf00..177643c 100644 --- a/bin/varnishtest/tests/b00003.vtc +++ b/bin/varnishtest/tests/b00003.vtc @@ -18,7 +18,7 @@ client c2 { txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1002 1001" + expect resp.http.X-Varnish == "1004 1001" } -run # Give varnish a chance to update stats diff --git a/bin/varnishtest/tests/b00010.vtc b/bin/varnishtest/tests/b00010.vtc index b27dd42..cfa3266 100644 --- a/bin/varnishtest/tests/b00010.vtc +++ b/bin/varnishtest/tests/b00010.vtc @@ -14,5 +14,5 @@ client c1 { txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.x-varnish == "1002 1001" + expect resp.http.x-varnish == "1003 1001" } -run diff --git a/bin/varnishtest/tests/b00012.vtc b/bin/varnishtest/tests/b00012.vtc index 154f6c0..302767b 100644 --- a/bin/varnishtest/tests/b00012.vtc +++ b/bin/varnishtest/tests/b00012.vtc @@ -20,11 +20,11 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == 6 - expect resp.http.x-varnish == "1002" + expect resp.http.x-varnish == "1003" rxresp expect resp.status == 200 expect resp.bodylen == 6 - expect resp.http.x-varnish == "1003 1002" + expect resp.http.x-varnish == "1005 1003" } -run varnish v1 -expect sess_pipeline == 2 diff --git a/bin/varnishtest/tests/b00013.vtc b/bin/varnishtest/tests/b00013.vtc index b8e25e6..f0be1d9 100644 --- a/bin/varnishtest/tests/b00013.vtc +++ b/bin/varnishtest/tests/b00013.vtc @@ -21,12 +21,12 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == 6 - expect resp.http.x-varnish == "1002" + expect resp.http.x-varnish == "1003" send "HTTP/1.1\n\n" rxresp expect resp.status == 200 expect resp.bodylen == 6 - expect resp.http.x-varnish == "1003 1002" + expect resp.http.x-varnish == "1005 1003" } -run varnish v1 -expect sess_readahead == 2 diff --git a/bin/varnishtest/tests/b00015.vtc b/bin/varnishtest/tests/b00015.vtc index 6e9d693..2e94add 100644 --- a/bin/varnishtest/tests/b00015.vtc +++ b/bin/varnishtest/tests/b00015.vtc @@ -19,7 +19,7 @@ client c1 { txreq -url "/" rxresp expect resp.status == 503 - expect resp.http.X-varnish == "1002" + expect resp.http.X-varnish == "1005" } -run # Then check that an cacheable error from the backend is @@ -35,14 +35,14 @@ client c1 { txreq -url "/" rxresp expect resp.status == 302 - expect resp.http.X-varnish == "1003" + expect resp.http.X-varnish == "1009" } -run client c1 { txreq -url "/" rxresp expect resp.status == 302 - expect resp.http.X-varnish == "1004 1003" + expect resp.http.X-varnish == "1012 1009" } -run # Then check that a non-cacheable error from the backend can be @@ -64,12 +64,12 @@ client c1 { txreq -url "/2" rxresp expect resp.status == 502 - expect resp.http.X-varnish == "1005" + expect resp.http.X-varnish == "1014" } -run client c1 { txreq -url "/2" rxresp expect resp.status == 502 - expect resp.http.X-varnish == "1006 1005" + expect resp.http.X-varnish == "1017 1014" } -run diff --git a/bin/varnishtest/tests/c00004.vtc b/bin/varnishtest/tests/c00004.vtc index 6b47ad5..cd79441 100644 --- a/bin/varnishtest/tests/c00004.vtc +++ b/bin/varnishtest/tests/c00004.vtc @@ -26,25 +26,25 @@ client c1 { txreq -hdr "Foobar: 2" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1002" + expect resp.http.X-Varnish == "1003" expect resp.http.snafu == "2" txreq -hdr "Foobar: 3" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1003" + expect resp.http.X-Varnish == "1005" expect resp.http.snafu == "3" txreq rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1004" + expect resp.http.X-Varnish == "1007" expect resp.http.snafu == "4" txreq -hdr "Foobar: 1 " rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1005 1001" + expect resp.http.X-Varnish == "1009 1001" expect resp.http.snafu == "1" } -run diff --git a/bin/varnishtest/tests/c00010.vtc b/bin/varnishtest/tests/c00010.vtc index 6f0aca3..1b6a5ec 100644 --- a/bin/varnishtest/tests/c00010.vtc +++ b/bin/varnishtest/tests/c00010.vtc @@ -25,7 +25,7 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == 7 - expect resp.http.x-varnish == "1002" + expect resp.http.x-varnish == "1003" } client c1 -run diff --git a/bin/varnishtest/tests/c00011.vtc b/bin/varnishtest/tests/c00011.vtc index 6bea217..952966b 100644 --- a/bin/varnishtest/tests/c00011.vtc +++ b/bin/varnishtest/tests/c00011.vtc @@ -25,7 +25,7 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == 7 - expect resp.http.x-varnish == "1002" + expect resp.http.x-varnish == "1003" } client c1 -run diff --git a/bin/varnishtest/tests/c00012.vtc b/bin/varnishtest/tests/c00012.vtc index b578625..79185e1 100644 --- a/bin/varnishtest/tests/c00012.vtc +++ b/bin/varnishtest/tests/c00012.vtc @@ -25,7 +25,7 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == 7 - expect resp.http.x-varnish == "1002" + expect resp.http.x-varnish == "1004" } client c1 -run diff --git a/bin/varnishtest/tests/c00013.vtc b/bin/varnishtest/tests/c00013.vtc index 885be31..fee1271 100644 --- a/bin/varnishtest/tests/c00013.vtc +++ b/bin/varnishtest/tests/c00013.vtc @@ -20,14 +20,16 @@ client c1 { expect resp.http.x-varnish == "1001" } -start +sema r1 sync 2 + client c2 { - sema r1 sync 2 txreq -url "/foo" -hdr "client: c2" + delay .1 sema r1 sync 2 rxresp expect resp.status == 200 expect resp.bodylen == 12 - expect resp.http.x-varnish == "1002 1001" + expect resp.http.x-varnish == "1004 1001" } -run client c1 -wait diff --git a/bin/varnishtest/tests/c00014.vtc b/bin/varnishtest/tests/c00014.vtc index 545c469..7e5dab1 100644 --- a/bin/varnishtest/tests/c00014.vtc +++ b/bin/varnishtest/tests/c00014.vtc @@ -32,7 +32,7 @@ client c2 { rxresp expect resp.status == 200 expect resp.bodylen == 6 - expect resp.http.x-varnish == "1002" + expect resp.http.x-varnish == "1004" } -run client c1 -wait diff --git a/bin/varnishtest/tests/c00015.vtc b/bin/varnishtest/tests/c00015.vtc index c323b9c..8490bf3 100644 --- a/bin/varnishtest/tests/c00015.vtc +++ b/bin/varnishtest/tests/c00015.vtc @@ -36,7 +36,7 @@ client c2 { rxresp expect resp.status == 200 expect resp.bodylen == 7 - expect resp.http.x-varnish == "1002" + expect resp.http.x-varnish == "1004" } -run varnish v1 -cli "vcl.use vcl1" @@ -46,7 +46,7 @@ client c3 { rxresp expect resp.status == 200 expect resp.bodylen == 6 - expect resp.http.x-varnish == "1003 1001" + expect resp.http.x-varnish == "1007 1001" } -run varnish v1 -cli "vcl.show vcl2" diff --git a/bin/varnishtest/tests/c00020.vtc b/bin/varnishtest/tests/c00020.vtc index 7fc707c..8e14f12 100644 --- a/bin/varnishtest/tests/c00020.vtc +++ b/bin/varnishtest/tests/c00020.vtc @@ -19,7 +19,7 @@ client c2 { txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1002 1001" + expect resp.http.X-Varnish == "1004 1001" } -run server s1 { @@ -35,19 +35,19 @@ client c2 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1003" + expect resp.http.X-Varnish == "1006" txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1004 1001" + expect resp.http.X-Varnish == "1008 1001" txreq -url "/bar" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1005" + expect resp.http.X-Varnish == "1009" txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1006 1003" + expect resp.http.X-Varnish == "1011 1006" } -run varnish v1 -expect sess_conn == 3 diff --git a/bin/varnishtest/tests/c00023.vtc b/bin/varnishtest/tests/c00023.vtc index 8c65eaa..a29908d 100644 --- a/bin/varnishtest/tests/c00023.vtc +++ b/bin/varnishtest/tests/c00023.vtc @@ -44,49 +44,49 @@ client c1 { rxresp expect resp.bodylen == 2 expect resp.status == 200 - expect resp.http.X-Varnish == "1002" + expect resp.http.X-Varnish == "1003" txreq -url "/3" rxresp expect resp.bodylen == 3 expect resp.status == 200 - expect resp.http.X-Varnish == "1003" + expect resp.http.X-Varnish == "1005" txreq -url "/4" rxresp expect resp.bodylen == 4 expect resp.status == 200 - expect resp.http.X-Varnish == "1004" + expect resp.http.X-Varnish == "1007" txreq -url "/5" rxresp expect resp.bodylen == 5 expect resp.status == 200 - expect resp.http.X-Varnish == "1005" + expect resp.http.X-Varnish == "1009" txreq -url "/6" rxresp expect resp.bodylen == 6 expect resp.status == 200 - expect resp.http.X-Varnish == "1006" + expect resp.http.X-Varnish == "1011" txreq -url "/7" rxresp expect resp.bodylen == 7 expect resp.status == 200 - expect resp.http.X-Varnish == "1007" + expect resp.http.X-Varnish == "1013" txreq -url "/8" rxresp expect resp.bodylen == 8 expect resp.status == 200 - expect resp.http.X-Varnish == "1008" + expect resp.http.X-Varnish == "1015" txreq -url "/9" rxresp expect resp.bodylen == 9 expect resp.status == 200 - expect resp.http.X-Varnish == "1009" + expect resp.http.X-Varnish == "1017" } -run @@ -95,55 +95,55 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == 1 - expect resp.http.X-Varnish == "1010 1001" + expect resp.http.X-Varnish == "1020 1001" txreq -url "/2" rxresp expect resp.bodylen == 2 expect resp.status == 200 - expect resp.http.X-Varnish == "1011 1002" + expect resp.http.X-Varnish == "1021 1003" txreq -url "/3" rxresp expect resp.bodylen == 3 expect resp.status == 200 - expect resp.http.X-Varnish == "1012 1003" + expect resp.http.X-Varnish == "1022 1005" txreq -url "/4" rxresp expect resp.bodylen == 4 expect resp.status == 200 - expect resp.http.X-Varnish == "1013 1004" + expect resp.http.X-Varnish == "1023 1007" txreq -url "/5" rxresp expect resp.bodylen == 5 expect resp.status == 200 - expect resp.http.X-Varnish == "1014 1005" + expect resp.http.X-Varnish == "1024 1009" txreq -url "/6" rxresp expect resp.bodylen == 6 expect resp.status == 200 - expect resp.http.X-Varnish == "1015 1006" + expect resp.http.X-Varnish == "1025 1011" txreq -url "/7" rxresp expect resp.bodylen == 7 expect resp.status == 200 - expect resp.http.X-Varnish == "1016 1007" + expect resp.http.X-Varnish == "1026 1013" txreq -url "/8" rxresp expect resp.bodylen == 8 expect resp.status == 200 - expect resp.http.X-Varnish == "1017 1008" + expect resp.http.X-Varnish == "1027 1015" txreq -url "/9" rxresp expect resp.bodylen == 9 expect resp.status == 200 - expect resp.http.X-Varnish == "1018 1009" + expect resp.http.X-Varnish == "1028 1017" } -run varnish v1 -cliok "hcb.dump" diff --git a/bin/varnishtest/tests/p00004.vtc b/bin/varnishtest/tests/p00004.vtc index e9b08f5..dab8ea4 100644 --- a/bin/varnishtest/tests/p00004.vtc +++ b/bin/varnishtest/tests/p00004.vtc @@ -29,7 +29,7 @@ client c1 { txreq -url "/bar" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1002" + expect resp.http.X-Varnish == "1004" expect resp.http.bar == "bar" } -run @@ -37,7 +37,7 @@ varnish v1 -expect n_object == 2 varnish v1 -stop varnish v1 -start -varnish v1 -cliok "debug.xid 2000" +varnish v1 -cliok "debug.xid 1999" varnish v1 -expect n_vampireobject == 2 varnish v1 -expect n_object == 0 @@ -57,7 +57,7 @@ client c1 { txreq -url "/bar" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "2002 1002" + expect resp.http.X-Varnish == "2003 1004" expect resp.http.bar == "bar" } -run diff --git a/bin/varnishtest/tests/p00006.vtc b/bin/varnishtest/tests/p00006.vtc index d30c6cb..3334f36 100644 --- a/bin/varnishtest/tests/p00006.vtc +++ b/bin/varnishtest/tests/p00006.vtc @@ -25,7 +25,7 @@ client c1 { txreq -url "/foo" -hdr "foo: 2" -hdr "bar: 1" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1002" + expect resp.http.X-Varnish == "1003" expect resp.http.foo == "foo2" } -run @@ -35,6 +35,7 @@ server s1 -wait varnish v1 -stop varnish v1 -start +varnish v1 -cliok "debug.xid 1999" varnish v1 -expect n_vampireobject == 2 @@ -42,13 +43,13 @@ client c1 { txreq -url "/foo" -hdr "foo: 1" -hdr "bar: 2" rxresp expect resp.status == 200 - #expect resp.http.X-Varnish == "1001" + expect resp.http.X-Varnish == "2001 1001" expect resp.http.foo == "foo1" txreq -url "/foo" -hdr "foo: 2" -hdr "bar: 1" rxresp expect resp.status == 200 - #expect resp.http.X-Varnish == "1002" + expect resp.http.X-Varnish == "2002 1003" expect resp.http.foo == "foo2" } -run diff --git a/bin/varnishtest/tests/r00102.vtc b/bin/varnishtest/tests/r00102.vtc index 6d2d8aa..d2a3c13 100644 --- a/bin/varnishtest/tests/r00102.vtc +++ b/bin/varnishtest/tests/r00102.vtc @@ -28,7 +28,7 @@ client c1 { -body "123456789\n" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1002 1001" + expect resp.http.X-Varnish == "1003 1001" } client c1 -run diff --git a/bin/varnishtest/tests/r00262.vtc b/bin/varnishtest/tests/r00262.vtc index b3c1cf9..1deb9ea 100644 --- a/bin/varnishtest/tests/r00262.vtc +++ b/bin/varnishtest/tests/r00262.vtc @@ -19,7 +19,7 @@ client c1 { send "GET / HTTP/1.1\r\n\r\n" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1002 1001" + expect resp.http.X-Varnish == "1003 1001" } client c1 -run diff --git a/bin/varnishtest/tests/r00466.vtc b/bin/varnishtest/tests/r00466.vtc index 8d753ba..3d64b22 100644 --- a/bin/varnishtest/tests/r00466.vtc +++ b/bin/varnishtest/tests/r00466.vtc @@ -34,5 +34,5 @@ client c1 { txreq -url "/bar" -hdr "Range: 200-300" rxresp expect resp.status == 206 - expect resp.http.X-Varnish == "1002" + expect resp.http.X-Varnish == "1003" } -run diff --git a/bin/varnishtest/tests/s00000.vtc b/bin/varnishtest/tests/s00000.vtc index bde8934..b5090ff 100644 --- a/bin/varnishtest/tests/s00000.vtc +++ b/bin/varnishtest/tests/s00000.vtc @@ -26,6 +26,6 @@ client c2 { txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.x-varnish == "1002" + expect resp.http.x-varnish == "1004" expect resp.bodylen == 6 } -run diff --git a/bin/varnishtest/tests/s00001.vtc b/bin/varnishtest/tests/s00001.vtc index 68fd520..44ea211 100644 --- a/bin/varnishtest/tests/s00001.vtc +++ b/bin/varnishtest/tests/s00001.vtc @@ -27,6 +27,6 @@ client c2 { txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.x-varnish == "1002" + expect resp.http.x-varnish == "1004" expect resp.bodylen == 6 } -run diff --git a/bin/varnishtest/tests/s00002.vtc b/bin/varnishtest/tests/s00002.vtc index 3cb56cf..31fb400 100644 --- a/bin/varnishtest/tests/s00002.vtc +++ b/bin/varnishtest/tests/s00002.vtc @@ -69,5 +69,5 @@ client c2 { rxresp expect resp.http.foo == "bar" expect resp.status == 200 - expect resp.http.x-varnish == "1002 1001" + expect resp.http.x-varnish == "1004 1001" } -run diff --git a/bin/varnishtest/tests/v00011.vtc b/bin/varnishtest/tests/v00011.vtc index 6cce4de..43ea8c2 100644 --- a/bin/varnishtest/tests/v00011.vtc +++ b/bin/varnishtest/tests/v00011.vtc @@ -27,13 +27,13 @@ client c1 { client c1 { txreq -req "PURGE" rxresp - expect resp.http.X-Varnish == "1002" + expect resp.http.X-Varnish == "1004" expect resp.status == 209 } -run client c1 { txreq rxresp - expect resp.http.X-Varnish == "1003" + expect resp.http.X-Varnish == "1007" } -run diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index e493ae7..0ddf1d0 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -483,7 +483,7 @@ varnish_start(struct varnish *v) vtc_log(v->vl, 0, "CLI start command failed: %u %s", u, resp); wait_running(v); free(resp); - u = varnish_ask_cli(v, "debug.xid 1000", &resp); + u = varnish_ask_cli(v, "debug.xid 999", &resp); if (vtc_error) return; if (u != CLIS_OK) From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] 4670e05 Add a couple of counters to keep track of how many requests are on the busy-obj waitinglists Message-ID: commit 4670e0541e9559239c6f9aa360385ef45dc2928a Author: Poul-Henning Kamp Date: Mon Aug 13 10:43:38 2012 +0000 Add a couple of counters to keep track of how many requests are on the busy-obj waitinglists diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index abcb4c0..fd0bbe1 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -424,6 +424,8 @@ HSH_Lookup(struct req *req) if (cache_param->diag_bitmap & 0x20) VSLb(req->vsl, SLT_Debug, "on waiting list <%p>", oh); + + wrk->stats.busy_sleep++; SES_Charge(req->wrk, req); /* * The objhead reference transfers to the sess, we get it @@ -480,6 +482,7 @@ hsh_rush(struct dstat *ds, struct objhead *oh) if (req == NULL) break; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + ds->busy_wakeup++; AZ(req->wrk); VTAILQ_REMOVE(&wl->list, req, w_list); DSL(0x20, SLT_Debug, req->sp->vsl_id, "off waiting list"); diff --git a/bin/varnishtest/tests/c00013.vtc b/bin/varnishtest/tests/c00013.vtc index fee1271..a79244b 100644 --- a/bin/varnishtest/tests/c00013.vtc +++ b/bin/varnishtest/tests/c00013.vtc @@ -33,3 +33,6 @@ client c2 { } -run client c1 -wait + +varnish v1 -expect busy_sleep == 1 +varnish v1 -expect busy_wakeup == 1 diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h index c3fd288..d9c78ce 100644 --- a/include/tbl/vsc_f_main.h +++ b/include/tbl/vsc_f_main.h @@ -249,6 +249,18 @@ VSC_F(thread_queue_len, uint64_t, 0, 'g', " See also param queue_max." ) +VSC_F(busy_sleep, uint64_t, 1, 'c', + "Number of requests sent to sleep on busy objhdr", + "Number of requests sent to sleep without a worker threads because" + " they found a busy object." +) + +VSC_F(busy_wakeup, uint64_t, 1, 'c', + "Number of requests woken after sleep on busy objhdr", + "Number of requests taken of the busy object sleep list and" + " and rescheduled." +) + VSC_F(sess_queued, uint64_t, 0, 'c', "Sessions queued for thread", "Number of times session was queued waiting for a thread." From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] dc79130 If people only specify the Transient storage, only run on the Transient storage. Message-ID: commit dc7913071e7bc19ddc7139b0bc9e13ca99357bda Author: Poul-Henning Kamp Date: Mon Aug 13 11:05:05 2012 +0000 If people only specify the Transient storage, only run on the Transient storage. Fixes #1176 diff --git a/bin/varnishtest/tests/r01176.vtc b/bin/varnishtest/tests/r01176.vtc new file mode 100644 index 0000000..d9a0a99 --- /dev/null +++ b/bin/varnishtest/tests/r01176.vtc @@ -0,0 +1,14 @@ +varnishtest "-s Transient=malloc crash" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -storage "-s Transient=malloc" -vcl+backend {} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] 58596b2 Sigh, this one is the working part of the fix: Message-ID: commit 58596b2780fdfdfeea4c000fe3e3912196369df2 Author: Poul-Henning Kamp Date: Mon Aug 13 11:05:39 2012 +0000 Sigh, this one is the working part of the fix: If people only specify Transient, run only on Transient. Fixes #1176 diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 910808f..cd1a316 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -144,6 +144,8 @@ stv_pick_stevedore(struct vsl_log *vsl, const char **hint) VSLb(vsl, SLT_Debug, "Storage hint not usable"); *hint = NULL; } + if (stv_next == NULL) + return (stv_transient); /* pick a stevedore and bump the head along */ stv = VTAILQ_NEXT(stv_next, list); if (stv == NULL) From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] 158b428 Not sure what I was thinking: Have SLT_SessOpen log the fd#, not the xid, we already have that. Message-ID: commit 158b428abfe2566c17eaf510bee2c4f4e1a1c0e7 Author: Poul-Henning Kamp Date: Tue Aug 14 06:52:42 2012 +0000 Not sure what I was thinking: Have SLT_SessOpen log the fd#, not the xid, we already have that. diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index c7b9a4a..33c63ba 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -197,8 +197,8 @@ ses_vsl_socket(struct sess *sp, const char *lsockname) strcpy(laddr, "-"); strcpy(lport, "-"); } - VSL(SLT_SessOpen, sp->vxid, "%u %s %s %s %s %s %.6f", - sp->vxid, sp->addr, sp->port, lsockname, laddr, lport, sp->t_open); + VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s %.6f %d", + sp->addr, sp->port, lsockname, laddr, lport, sp->t_open, sp->fd); } /*-------------------------------------------------------------------- diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index ae83bf7..9d970e1 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -73,6 +73,7 @@ SLTM(SessOpen, "Client connection opened", "lsock\n Listen socket\n\n" "laddr\n Local IPv4/6 address ('-' if !$log_local_addr)\n\n" "lport\n Local TCP port ('-' if !$log_local_addr)\n\n" + "fd\n File descriptor number" ) /* From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] f5fe417 Move INCOMPL() into the VAS family and apply a bit of polish while there. Message-ID: commit f5fe41770bc33069377a11a4032a32faa8452f9f Author: Poul-Henning Kamp Date: Tue Aug 14 07:03:18 2012 +0000 Move INCOMPL() into the VAS family and apply a bit of polish while there. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 35c2b52..add0150 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -984,13 +984,6 @@ void VSL_Flush(struct vsl_log *, int overflow); VSL((tag), (id), __VA_ARGS__); \ } while (0) -#define INCOMPL() do { \ - VSL(SLT_Debug, 0, "INCOMPLETE AT: %s(%d)", __func__, __LINE__); \ - fprintf(stderr, \ - "INCOMPLETE AT: %s(%d)\n", \ - (const char *)__func__, __LINE__); \ - abort(); \ - } while (0) #endif /* cache_response.c */ diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index c38da38..e6b0941 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -343,9 +343,9 @@ pan_backtrace(void) /*--------------------------------------------------------------------*/ -static void +static void __attribute__((__noreturn__)) pan_ic(const char *func, const char *file, int line, const char *cond, - int err, int xxx) + int err, enum vas_e kind) { const char *q; const struct req *req; @@ -353,23 +353,28 @@ pan_ic(const char *func, const char *file, int line, const char *cond, AZ(pthread_mutex_lock(&panicstr_mtx)); /* Won't be released, we're going to die anyway */ - switch(xxx) { - case 3: + switch(kind) { + case VAS_WRONG: VSB_printf(pan_vsp, "Wrong turn at %s:%d:\n%s\n", file, line, cond); break; - case 2: + case VAS_VCL: VSB_printf(pan_vsp, "Panic from VCL:\n %s\n", cond); break; - case 1: + case VAS_MISSING: VSB_printf(pan_vsp, "Missing errorhandling code in %s(), %s line %d:\n" " Condition(%s) not true.", func, file, line, cond); break; + case VAS_INCOMPLETE: + VSB_printf(pan_vsp, + "Incomplete code in %s(), %s line %d:\n", + func, file, line); + break; default: - case 0: + case VAS_ASSERT: VSB_printf(pan_vsp, "Assert error in %s(), %s line %d:\n" " Condition(%s) not true.\n", diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c index 402b49f..1fc3760 100644 --- a/bin/varnishtest/vtc_log.c +++ b/bin/varnishtest/vtc_log.c @@ -285,14 +285,14 @@ vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx, /**********************************************************************/ -static void +static void __attribute__((__noreturn__)) vtc_log_VAS_Fail(const char *func, const char *file, int line, - const char *cond, int err, int xxx) + const char *cond, int err, enum vas_e why) { struct vtclog *vl; (void)err; - (void)xxx; + (void)why; vl = pthread_getspecific(log_key); if (vl == NULL || vl->act) { fprintf(stderr, @@ -303,6 +303,7 @@ vtc_log_VAS_Fail(const char *func, const char *file, int line, vtc_log(vl, 0, "Assert error in %s(), %s line %d:" " Condition(%s) not true.\n", func, file, line, cond); } + abort(); } -vas_f *VAS_Fail = vtc_log_VAS_Fail; +vas_f *VAS_Fail __attribute__((__noreturn__)) = vtc_log_VAS_Fail; diff --git a/include/vas.h b/include/vas.h index ff20036..246acc1 100644 --- a/include/vas.h +++ b/include/vas.h @@ -38,24 +38,37 @@ #ifndef VAS_H_INCLUDED #define VAS_H_INCLUDED -typedef void vas_f(const char *, const char *, int, const char *, int, int); +enum vas_e { + VAS_WRONG, + VAS_MISSING, + VAS_ASSERT, + VAS_INCOMPLETE, + VAS_VCL, +}; -extern vas_f *VAS_Fail; +typedef void vas_f(const char *, const char *, int, const char *, int, + enum vas_e); + +extern vas_f *VAS_Fail __attribute__((__noreturn__)); #ifdef WITHOUT_ASSERTS #define assert(e) ((void)(e)) #else /* WITH_ASSERTS */ #define assert(e) \ do { \ - if (!(e)) \ - VAS_Fail(__func__, __FILE__, __LINE__, #e, errno, 0); \ + if (!(e)) { \ + VAS_Fail(__func__, __FILE__, __LINE__, \ + #e, errno, VAS_ASSERT); \ + } \ } while (0) #endif #define xxxassert(e) \ do { \ - if (!(e)) \ - VAS_Fail(__func__, __FILE__, __LINE__, #e, errno, 1); \ + if (!(e)) { \ + VAS_Fail(__func__, __FILE__, __LINE__, \ + #e, errno, VAS_MISSING); \ + } \ } while (0) /* Assert zero return value */ @@ -66,8 +79,13 @@ do { \ #define diagnostic(foo) assert(foo) #define WRONG(expl) \ do { \ - VAS_Fail(__func__, __FILE__, __LINE__, expl, errno, 3); \ - abort(); \ + VAS_Fail(__func__, __FILE__, __LINE__, expl, errno, VAS_WRONG); \ +} while (0) + +#define INCOMPL() \ +do { \ + VAS_Fail(__func__, __FILE__, __LINE__, \ + "", errno, VAS_INCOMPLETE); \ } while (0) #endif diff --git a/lib/libvarnish/vas.c b/lib/libvarnish/vas.c index 6f69928..d63763d 100644 --- a/lib/libvarnish/vas.c +++ b/lib/libvarnish/vas.c @@ -37,16 +37,24 @@ #include "vas.h" -static void +static void __attribute__((__noreturn__)) VAS_Fail_default(const char *func, const char *file, int line, - const char *cond, int err, int xxx) + const char *cond, int err, enum vas_e kind) { - if (xxx) { + if (kind == VAS_MISSING) { fprintf(stderr, "Missing errorhandling code in %s(), %s line %d:\n" " Condition(%s) not true.\n", func, file, line, cond); + } else if (kind == VAS_INCOMPLETE) { + fprintf(stderr, + "Incompelte code in %s(), %s line %d:\n", + func, file, line); + } else if (kind == VAS_WRONG) { + fprintf(stderr, + "Wrong turn in %s(), %s line %d:\n", + func, file, line); } else { fprintf(stderr, "Assert error in %s(), %s line %d:\n" @@ -59,4 +67,4 @@ VAS_Fail_default(const char *func, const char *file, int line, abort(); } -vas_f *VAS_Fail = VAS_Fail_default; +vas_f *VAS_Fail __attribute__((__noreturn__)) = VAS_Fail_default; diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index cd23473..fd1a135 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -45,7 +45,7 @@ vmod_panic(struct req *req, const char *str, ...) va_start(ap, str); b = VRT_String(req->http->ws, "PANIC: ", str, ap); va_end(ap); - VAS_Fail("VCL", "", 0, b, 0, 2); + VAS_Fail("VCL", "", 0, b, 0, VAS_VCL); } const char * __match_proto__(td_debug_author) From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] 3c53c1a Replace the fd# with the VXID in VSL records. Message-ID: commit 3c53c1affef3c7fdd0bb040c4f0f7c7f87694a61 Author: Poul-Henning Kamp Date: Tue Aug 14 10:16:23 2012 +0000 Replace the fd# with the VXID in VSL records. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index add0150..2441bd6 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -494,7 +494,6 @@ struct busyobj { * is recycled. */ unsigned refcount; - uint32_t vxid; uint8_t *vary; unsigned is_gzip; @@ -583,7 +582,6 @@ struct req { unsigned magic; #define REQ_MAGIC 0x2751aaa1 - uint32_t vxid; int restarts; int esi_level; int disable_esi; @@ -679,7 +677,6 @@ struct sess { enum sess_step sess_step; int fd; enum sess_close reason; - unsigned vsl_id; uint32_t vxid; /* Cross references ------------------------------------------*/ @@ -957,7 +954,7 @@ void SES_Charge(struct worker *, struct req *); struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no); void SES_DeletePool(struct sesspool *sp); int SES_ScheduleReq(struct req *); -struct req *SES_GetReq(struct sess *sp); +struct req *SES_GetReq(struct worker *, struct sess *); void SES_Handle(struct sess *sp, double now); void SES_ReleaseReq(struct req *); pool_func_t SES_pool_accept_task; diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index da2cdf8..c5831fd 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -251,7 +251,6 @@ VCA_SetupSess(struct worker *wrk, struct sess *sp) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CAST_OBJ_NOTNULL(wa, (void*)wrk->aws->f, WRK_ACCEPT_MAGIC); sp->fd = wa->acceptsock; - sp->vsl_id = wa->acceptsock | VSL_CLIENTMARKER ; wa->acceptsock = -1; retval = wa->acceptlsock->name; assert(wa->acceptaddrlen <= sp->sockaddrlen); diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 7b3101b..e8efb90 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -181,7 +181,6 @@ bes_conn_try(struct req *req, struct vbc *vc, const struct vdi_simple *vs) vc->addr = NULL; vc->addrlen = 0; } else { - vc->vsl_id = s | VSL_BACKENDMARKER; VTCP_myname(s, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); VSLb(req->vsl, SLT_BackendOpen, "%d %s %s %s ", vc->fd, vs->backend->display_name, abuf1, pbuf1); diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index 3baa0ca..b84e702 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -153,8 +153,6 @@ struct vbc { struct backend *backend; struct vdi_simple *vdis; struct vsl_log *vsl; - unsigned orig_vsl_id; - unsigned vsl_id; int fd; struct sockaddr_storage *addr; diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 8f0cc50..014e564 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -108,7 +108,6 @@ VBO_GetBusyObj(struct worker *wrk) AZ(bo->refcount); bo->refcount = 1; - bo->vxid = VXID_Get(&wrk->vxid_pool); p = (void*)(bo + 1); p = (void*)PRNDUP(p); @@ -129,6 +128,7 @@ VBO_GetBusyObj(struct worker *wrk) sz = cache_param->vsl_buffer; VSL_Setup(bo->vsl, p, sz); + bo->vsl->wid = VXID_Get(&wrk->vxid_pool) | VSL_BACKENDMARKER; p += sz; p = (void*)PRNDUP(p); assert(p < bo->end); diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c index 528a57c..ab43cf1 100644 --- a/bin/varnishd/cache/cache_dir.c +++ b/bin/varnishd/cache/cache_dir.c @@ -61,9 +61,7 @@ VDI_CloseFd(struct vbc **vbp) * before the OS reuses the FD */ VSL_Flush(vc->vsl, 0); - vc->vsl->wid = vc->orig_vsl_id; vc->vsl = NULL; - vc->orig_vsl_id = 0; VTCP_close(&vc->fd); VBE_DropRefConn(bp); @@ -92,9 +90,7 @@ VDI_RecycleFd(struct vbc **vbp) /* XXX: revisit this hack */ VSL_Flush(vc->vsl, 0); - vc->vsl->wid = vc->orig_vsl_id; vc->vsl = NULL; - vc->orig_vsl_id = 0; Lck_Lock(&bp->mtx); VSC_C_main->backend_recycle++; @@ -114,11 +110,8 @@ VDI_GetFd(const struct director *d, struct req *req) d = req->director; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); vc = d->getfd(d, req); - if (vc != NULL) { + if (vc != NULL) vc->vsl = req->busyobj->vsl; - vc->orig_vsl_id = vc->vsl->wid; - vc->vsl->wid = vc->vsl_id; - } return (vc); } diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index daef6a9..e7fee9b 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -59,7 +59,7 @@ ved_include(struct req *preq, const char *src, const char *host) /* Take a workspace snapshot */ wrk_ws_wm = WS_Snapshot(wrk->aws); /* XXX ? */ - req = SES_GetReq(preq->sp); + req = SES_GetReq(wrk, preq->sp); req->esi_level = preq->esi_level + 1; HTTP_Copy(req->http0, preq->http0); @@ -105,7 +105,7 @@ ved_include(struct req *preq, const char *src, const char *host) i = CNT_Request(wrk, req); if (i == 1) break; - DSL(0x20, SLT_Debug, req->sp->vsl_id, + DSL(0x20, SLT_Debug, req->vsl->wid, "loop waiting for ESI (%d)", i); assert(i == 2); AZ(req->wrk); diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index aed84d9..80149c6 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -391,7 +391,6 @@ VGZ_UpdateObj(const struct vgz *vg, struct object *obj) } /*-------------------------------------------------------------------- - * Passing a vsl_id of -1 means "use wrk->vbc->vsl_id" */ int diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index fd0bbe1..0db0d6b 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -449,7 +449,6 @@ HSH_Lookup(struct req *req) AZ(req->busyobj); req->busyobj = VBO_GetBusyObj(wrk); - req->busyobj->vsl->wid = req->sp->vsl_id; req->busyobj->refcount = 2; /* One for req, one for FetchBody */ VRY_Validate(req->vary_b); @@ -485,7 +484,7 @@ hsh_rush(struct dstat *ds, struct objhead *oh) ds->busy_wakeup++; AZ(req->wrk); VTAILQ_REMOVE(&wl->list, req, w_list); - DSL(0x20, SLT_Debug, req->sp->vsl_id, "off waiting list"); + DSL(0x20, SLT_Debug, req->vsl->wid, "off waiting list"); if (SES_ScheduleReq(req)) { /* * We could not schedule the session, leave the diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 3b9ae2e..ad29899 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -852,7 +852,7 @@ http_FilterReq(const struct req *req, unsigned how) else http_linkh(hp, req->http, HTTP_HDR_PROTO); http_filterfields(hp, req->http, how); - http_PrintfHeader(hp, "X-Varnish: %u", req->vxid); + http_PrintfHeader(hp, "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index d181d11..ab870ad 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -245,6 +245,13 @@ http1_dissect(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + /* + * Cache_req_fsm zeros the vxid once a requests is processed. + * Allocate a new one only now that we know will need it. + */ + if (req->vsl->wid == 0) + req->vsl->wid = VXID_Get(&wrk->vxid_pool) | VSL_CLIENTMARKER; + /* Borrow VCL reference from worker thread */ VCL_Refresh(&wrk->vcl); req->vcl = wrk->vcl; @@ -336,7 +343,6 @@ HTTP1_Session(struct worker *wrk, struct req *req) if (done == 2) return; assert(done == 1); - assert(req->vxid == 0); sdr = http1_cleanup(sp, wrk, req); switch (sdr) { case SESS_DONE_RET_GONE: diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index e6b0941..ee7dd78 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -236,8 +236,7 @@ pan_req(const struct req *req) VSB_printf(pan_vsp, "req = %p {\n", req); - VSB_printf(pan_vsp, " sp = %p, vxid = %u,", - req->sp, req->vxid); + VSB_printf(pan_vsp, " sp = %p, vxid = %u,", req->sp, req->vsl->wid); switch (req->req_step) { #define REQ_STEP(l, u, arg) case R_STP_##u: stp = "R_STP_" #u; break; @@ -294,8 +293,8 @@ pan_sess(const struct sess *sp) const char *stp; VSB_printf(pan_vsp, " sp = %p {\n", sp); - VSB_printf(pan_vsp, " fd = %d, id = %u,\n", - sp->fd, sp->vsl_id & VSL_IDENTMASK); + VSB_printf(pan_vsp, " fd = %d, vxid = %u,\n", + sp->fd, sp->vxid & VSL_IDENTMASK); VSB_printf(pan_vsp, " client = %s %s,\n", sp->addr ? sp->addr : "?.?.?.?", sp->port ? sp->port : "?"); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 6e03e4c..e99ed6c 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -270,7 +270,6 @@ cnt_error(struct worker *wrk, struct req *req) bo = VBO_GetBusyObj(wrk); req->busyobj = bo; - bo->vsl->wid = req->sp->vsl_id; AZ(bo->stats); bo->stats = &wrk->stats; req->objcore = HSH_NewObjCore(wrk); @@ -286,7 +285,7 @@ cnt_error(struct worker *wrk, struct req *req) return(1); } CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC); - req->obj->vxid = req->vxid; + req->obj->vxid = bo->vsl->wid; req->obj->exp.entered = req->t_req; h = req->obj->http; @@ -626,7 +625,7 @@ cnt_fetchbody(struct worker *wrk, struct req *req) VSB_delete(vary); } - req->obj->vxid = req->vxid; + req->obj->vxid = bo->vsl->wid; req->obj->response = req->err_code; WS_Assert(req->obj->ws_o); @@ -955,7 +954,6 @@ cnt_pass(struct worker *wrk, struct req *req) req->busyobj = VBO_GetBusyObj(wrk); bo = req->busyobj; - bo->vsl->wid = req->sp->vsl_id; bo->refcount = 2; HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); http_FilterReq(req, HTTPH_R_PASS); @@ -1015,7 +1013,6 @@ cnt_pipe(struct worker *wrk, struct req *req) req->acct_req.pipe++; req->busyobj = VBO_GetBusyObj(wrk); bo = req->busyobj; - bo->vsl->wid = req->sp->vsl_id; HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq); http_FilterReq(req, 0); @@ -1103,8 +1100,7 @@ cnt_recv(const struct worker *wrk, struct req *req) AZ(req->busyobj); /* Assign XID and log */ - VSLb(req->vsl, SLT_ReqStart, "%s %s %u", - req->sp->addr, req->sp->port, req->vxid); + VSLb(req->vsl, SLT_ReqStart, "%s %s", req->sp->addr, req->sp->port); if (req->err_code) { req->req_step = R_STP_ERROR; @@ -1187,8 +1183,8 @@ cnt_diag(struct req *req, const char *state) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - VSLb(req->vsl, SLT_Debug, "vsl_id %u STP_%s sp %p obj %p vcl %p", - req->sp->vsl_id, state, req->sp, req->obj, req->vcl); + VSLb(req->vsl, SLT_Debug, "vxid %u STP_%s sp %p obj %p vcl %p", + req->vsl->wid, state, req->sp, req->obj, req->vcl); VSL_Flush(req->vsl, 0); } @@ -1207,8 +1203,7 @@ CNT_Request(struct worker *wrk, struct req *req) req->req_step == R_STP_LOOKUP || req->req_step == R_STP_RECV); - if (req->req_step == R_STP_RECV) - req->vxid = VXID_Get(&wrk->vxid_pool); + AN(req->vsl->wid & VSL_CLIENTMARKER); req->wrk = wrk; @@ -1243,8 +1238,7 @@ CNT_Request(struct worker *wrk, struct req *req) VSLb(req->vsl, SLT_Length, "%ju", (uintmax_t)req->req_bodybytes); } - VSLb(req->vsl, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f", - req->vxid, + VSLb(req->vsl, SLT_ReqEnd, "%.9f %.9f %.9f %.9f %.9f", req->t_req, req->sp->t_idle, req->sp->t_idle - req->t_resp, @@ -1253,7 +1247,12 @@ CNT_Request(struct worker *wrk, struct req *req) /* done == 2 was charged by cache_hash.c */ SES_Charge(wrk, req); - req->vxid = 0; + + /* + * Nuke the VXID, cache_http1_fsm.c::http1_dissect() will + * allocate a new one when necessary. + */ + req->vsl->wid = 0; } req->wrk = NULL; diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c index 0a554f4..cb152a5 100644 --- a/bin/varnishd/cache/cache_response.c +++ b/bin/varnishd/cache/cache_response.c @@ -134,11 +134,13 @@ RES_BuildHttp(struct req *req) VTIM_format(VTIM_real(), time_str); http_PrintfHeader(req->resp, "Date: %s", time_str); - if (req->vxid != req->obj->vxid) + if (req->wrk->stats.cache_hit) http_PrintfHeader(req->resp, - "X-Varnish: %u %u", req->vxid, req->obj->vxid); + "X-Varnish: %u %u", req->vsl->wid & VSL_IDENTMASK, + req->obj->vxid & VSL_IDENTMASK); else - http_PrintfHeader(req->resp, "X-Varnish: %u", req->vxid); + http_PrintfHeader(req->resp, + "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK); http_PrintfHeader(req->resp, "Age: %.0f", req->obj->exp.age + req->t_resp - req->obj->exp.entered); diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 7f5fd35..d0d0d87 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -169,8 +169,8 @@ RFC2616_Ttl(struct busyobj *bo) /* calculated TTL, Our time, Date, Expires, max-age, age */ VSLb(bo->vsl, SLT_TTL, - "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u", - bo->vxid, expp->ttl, -1., -1., expp->entered, + "RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u", + expp->ttl, -1., -1., expp->entered, expp->age, h_date, h_expires, max_age); } diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 33c63ba..6cffa69 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -160,7 +160,7 @@ ses_sess_pool_task(struct worker *wrk, void *arg) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CAST_OBJ_NOTNULL(sp, arg, SESS_MAGIC); - req = SES_GetReq(sp); + req = SES_GetReq(wrk, sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); sp->sess_step = S_STP_NEWREQ; @@ -228,7 +228,7 @@ SES_pool_accept_task(struct worker *wrk, void *arg) sp->t_open = VTIM_real(); sp->t_idle = sp->t_open; - sp->vxid = VXID_Get(&wrk->vxid_pool); + sp->vxid = VXID_Get(&wrk->vxid_pool) | VSL_CLIENTMARKER; lsockname = VCA_SetupSess(wrk, sp); ses_vsl_socket(sp, lsockname); @@ -343,7 +343,7 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now) */ struct req * -SES_GetReq(struct sess *sp) +SES_GetReq(struct worker *wrk, struct sess *sp) { struct sesspool *pp; struct req *req; @@ -351,6 +351,7 @@ SES_GetReq(struct sess *sp) unsigned sz, hl; char *p, *e; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); pp = sp->sesspool; CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC); @@ -386,7 +387,7 @@ SES_GetReq(struct sess *sp) sz = cache_param->workspace_thread; VSL_Setup(req->vsl, p, sz); - req->vsl->wid = sp->vsl_id; + req->vsl->wid = VXID_Get(&wrk->vxid_pool) | VSL_CLIENTMARKER; p += sz; p = (void*)PRNDUP(p); diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 215e7b6..61000a1 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -422,11 +422,11 @@ VRT_DO_EXP(obj, req->obj->exp, keep, 0, vrt_wsp_exp(req, req->obj->vxid, &req->obj->exp);) VRT_DO_EXP(beresp, req->busyobj->exp, grace, 0, - vrt_wsp_exp(req, req->vxid, &req->busyobj->exp);) + vrt_wsp_exp(req, req->vsl->wid & VSL_IDENTMASK, &req->busyobj->exp);) VRT_DO_EXP(beresp, req->busyobj->exp, ttl, 0, - vrt_wsp_exp(req, req->vxid, &req->busyobj->exp);) + vrt_wsp_exp(req, req->vsl->wid & VSL_IDENTMASK, &req->busyobj->exp);) VRT_DO_EXP(beresp, req->busyobj->exp, keep, 0, - vrt_wsp_exp(req, req->vxid, &req->busyobj->exp);) + vrt_wsp_exp(req, req->vsl->wid & VSL_IDENTMASK, &req->busyobj->exp);) /*-------------------------------------------------------------------- * req.xid @@ -439,9 +439,9 @@ VRT_r_req_xid(const struct req *req) int size; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - size = snprintf(NULL, 0, "%u", req->vxid) + 1; + size = snprintf(NULL, 0, "%u", req->vsl->wid & VSL_IDENTMASK) + 1; AN(p = WS_Alloc(req->http->ws, size)); - assert(snprintf(p, size, "%u", req->vxid) < size); + assert(snprintf(p, size, "%u", req->vsl->wid & VSL_IDENTMASK) < size); return (p); } diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c index a7403d6..bc5c67f 100644 --- a/bin/varnishd/waiter/cache_waiter_kqueue.c +++ b/bin/varnishd/waiter/cache_waiter_kqueue.c @@ -80,7 +80,7 @@ vwk_kq_sess(struct vwk *vwk, struct sess *sp, short arm) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); assert(sp->fd >= 0); - DSL(0x04, SLT_Debug, sp->vsl_id, "KQ: EV_SET sp %p arm %x", sp, arm); + DSL(0x04, SLT_Debug, sp->vxid, "KQ: EV_SET sp %p arm %x", sp, arm); EV_SET(&vwk->ki[vwk->nki], sp->fd, EVFILT_READ, arm, 0, 0, sp); if (++vwk->nki == NKEV) vwk_kq_flush(vwk); @@ -121,12 +121,10 @@ vwk_sess_ev(struct vwk *vwk, const struct kevent *kp, double now) AN(kp->udata); assert(kp->udata != vwk->pipes); CAST_OBJ_NOTNULL(sp, kp->udata, SESS_MAGIC); - DSL(0x04, SLT_Debug, sp->vsl_id, "KQ: sp %p kev data %lu flags 0x%x%s", + DSL(0x04, SLT_Debug, sp->vxid, "KQ: sp %p kev data %lu flags 0x%x%s", sp, (unsigned long)kp->data, kp->flags, (kp->flags & EV_EOF) ? " EOF" : ""); - assert((sp->vsl_id & VSL_IDENTMASK) == kp->ident); - assert((sp->vsl_id & VSL_IDENTMASK) == sp->fd); if (kp->data > 0) { VTAILQ_REMOVE(&vwk->sesshead, sp, list); SES_Handle(sp, now); @@ -136,7 +134,7 @@ vwk_sess_ev(struct vwk *vwk, const struct kevent *kp, double now) SES_Delete(sp, SC_REM_CLOSE, now); return; } else { - VSL(SLT_Debug, sp->vsl_id, + VSL(SLT_Debug, sp->vxid, "KQ: sp %p kev data %lu flags 0x%x%s", sp, (unsigned long)kp->data, kp->flags, (kp->flags & EV_EOF) ? " EOF" : ""); diff --git a/bin/varnishtest/tests/b00003.vtc b/bin/varnishtest/tests/b00003.vtc index 177643c..55442b3 100644 --- a/bin/varnishtest/tests/b00003.vtc +++ b/bin/varnishtest/tests/b00003.vtc @@ -18,7 +18,7 @@ client c2 { txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1004 1001" + expect resp.http.X-Varnish == "1004 1002" } -run # Give varnish a chance to update stats diff --git a/bin/varnishtest/tests/b00010.vtc b/bin/varnishtest/tests/b00010.vtc index cfa3266..05b9e91 100644 --- a/bin/varnishtest/tests/b00010.vtc +++ b/bin/varnishtest/tests/b00010.vtc @@ -14,5 +14,5 @@ client c1 { txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.x-varnish == "1003 1001" + expect resp.http.x-varnish == "1003 1002" } -run diff --git a/bin/varnishtest/tests/b00012.vtc b/bin/varnishtest/tests/b00012.vtc index 302767b..0aa9cec 100644 --- a/bin/varnishtest/tests/b00012.vtc +++ b/bin/varnishtest/tests/b00012.vtc @@ -24,7 +24,7 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == 6 - expect resp.http.x-varnish == "1005 1003" + expect resp.http.x-varnish == "1005 1004" } -run varnish v1 -expect sess_pipeline == 2 diff --git a/bin/varnishtest/tests/b00013.vtc b/bin/varnishtest/tests/b00013.vtc index f0be1d9..d0b957a 100644 --- a/bin/varnishtest/tests/b00013.vtc +++ b/bin/varnishtest/tests/b00013.vtc @@ -26,7 +26,7 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == 6 - expect resp.http.x-varnish == "1005 1003" + expect resp.http.x-varnish == "1005 1004" } -run varnish v1 -expect sess_readahead == 2 diff --git a/bin/varnishtest/tests/b00015.vtc b/bin/varnishtest/tests/b00015.vtc index 2e94add..fc022c3 100644 --- a/bin/varnishtest/tests/b00015.vtc +++ b/bin/varnishtest/tests/b00015.vtc @@ -42,7 +42,7 @@ client c1 { txreq -url "/" rxresp expect resp.status == 302 - expect resp.http.X-varnish == "1012 1009" + expect resp.http.X-varnish == "1012 1010" } -run # Then check that a non-cacheable error from the backend can be @@ -71,5 +71,5 @@ client c1 { txreq -url "/2" rxresp expect resp.status == 502 - expect resp.http.X-varnish == "1017 1014" + expect resp.http.X-varnish == "1017 1015" } -run diff --git a/bin/varnishtest/tests/c00004.vtc b/bin/varnishtest/tests/c00004.vtc index cd79441..3074eb8 100644 --- a/bin/varnishtest/tests/c00004.vtc +++ b/bin/varnishtest/tests/c00004.vtc @@ -44,7 +44,7 @@ client c1 { txreq -hdr "Foobar: 1 " rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1009 1001" + expect resp.http.X-Varnish == "1009 1002" expect resp.http.snafu == "1" } -run diff --git a/bin/varnishtest/tests/c00010.vtc b/bin/varnishtest/tests/c00010.vtc index 1b6a5ec..238dada 100644 --- a/bin/varnishtest/tests/c00010.vtc +++ b/bin/varnishtest/tests/c00010.vtc @@ -25,7 +25,7 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == 7 - expect resp.http.x-varnish == "1003" + expect resp.http.x-varnish == "1003 1004" } client c1 -run diff --git a/bin/varnishtest/tests/c00013.vtc b/bin/varnishtest/tests/c00013.vtc index a79244b..7bc36ae 100644 --- a/bin/varnishtest/tests/c00013.vtc +++ b/bin/varnishtest/tests/c00013.vtc @@ -29,7 +29,7 @@ client c2 { rxresp expect resp.status == 200 expect resp.bodylen == 12 - expect resp.http.x-varnish == "1004 1001" + expect resp.http.x-varnish == "1004 1002" } -run client c1 -wait diff --git a/bin/varnishtest/tests/c00015.vtc b/bin/varnishtest/tests/c00015.vtc index 8490bf3..cbf5cca 100644 --- a/bin/varnishtest/tests/c00015.vtc +++ b/bin/varnishtest/tests/c00015.vtc @@ -46,7 +46,7 @@ client c3 { rxresp expect resp.status == 200 expect resp.bodylen == 6 - expect resp.http.x-varnish == "1007 1001" + expect resp.http.x-varnish == "1007 1002" } -run varnish v1 -cli "vcl.show vcl2" diff --git a/bin/varnishtest/tests/c00020.vtc b/bin/varnishtest/tests/c00020.vtc index 8e14f12..3713951 100644 --- a/bin/varnishtest/tests/c00020.vtc +++ b/bin/varnishtest/tests/c00020.vtc @@ -19,7 +19,7 @@ client c2 { txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1004 1001" + expect resp.http.X-Varnish == "1004 1002" } -run server s1 { @@ -39,7 +39,7 @@ client c2 { txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1008 1001" + expect resp.http.X-Varnish == "1008 1002" txreq -url "/bar" rxresp expect resp.status == 200 @@ -47,7 +47,7 @@ client c2 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1011 1006" + expect resp.http.X-Varnish == "1011 1007" } -run varnish v1 -expect sess_conn == 3 diff --git a/bin/varnishtest/tests/c00023.vtc b/bin/varnishtest/tests/c00023.vtc index a29908d..b57c8c1 100644 --- a/bin/varnishtest/tests/c00023.vtc +++ b/bin/varnishtest/tests/c00023.vtc @@ -95,55 +95,55 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == 1 - expect resp.http.X-Varnish == "1020 1001" + expect resp.http.X-Varnish == "1020 1002" txreq -url "/2" rxresp expect resp.bodylen == 2 expect resp.status == 200 - expect resp.http.X-Varnish == "1021 1003" + expect resp.http.X-Varnish == "1021 1004" txreq -url "/3" rxresp expect resp.bodylen == 3 expect resp.status == 200 - expect resp.http.X-Varnish == "1022 1005" + expect resp.http.X-Varnish == "1022 1006" txreq -url "/4" rxresp expect resp.bodylen == 4 expect resp.status == 200 - expect resp.http.X-Varnish == "1023 1007" + expect resp.http.X-Varnish == "1023 1008" txreq -url "/5" rxresp expect resp.bodylen == 5 expect resp.status == 200 - expect resp.http.X-Varnish == "1024 1009" + expect resp.http.X-Varnish == "1024 1010" txreq -url "/6" rxresp expect resp.bodylen == 6 expect resp.status == 200 - expect resp.http.X-Varnish == "1025 1011" + expect resp.http.X-Varnish == "1025 1012" txreq -url "/7" rxresp expect resp.bodylen == 7 expect resp.status == 200 - expect resp.http.X-Varnish == "1026 1013" + expect resp.http.X-Varnish == "1026 1014" txreq -url "/8" rxresp expect resp.bodylen == 8 expect resp.status == 200 - expect resp.http.X-Varnish == "1027 1015" + expect resp.http.X-Varnish == "1027 1016" txreq -url "/9" rxresp expect resp.bodylen == 9 expect resp.status == 200 - expect resp.http.X-Varnish == "1028 1017" + expect resp.http.X-Varnish == "1028 1018" } -run varnish v1 -cliok "hcb.dump" diff --git a/bin/varnishtest/tests/p00000.vtc b/bin/varnishtest/tests/p00000.vtc index 0a17be2..12fa676 100644 --- a/bin/varnishtest/tests/p00000.vtc +++ b/bin/varnishtest/tests/p00000.vtc @@ -29,12 +29,13 @@ varnish v1 -cliok "debug.persistent s0 sync" varnish v1 -stop varnish v1 -start +varnish v1 -cliok "debug.xid 1999" client c1 { txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1001" + expect resp.http.X-Varnish == "2001 1002" } -run # shell "rm -f /tmp/__v1/_.per" diff --git a/bin/varnishtest/tests/p00001.vtc b/bin/varnishtest/tests/p00001.vtc index 4afccf3..26ea975 100644 --- a/bin/varnishtest/tests/p00001.vtc +++ b/bin/varnishtest/tests/p00001.vtc @@ -32,12 +32,13 @@ varnish v1 -vcl+backend { server s1 -wait varnish v1 -start +varnish v1 -cliok "debug.xid 1999" client c1 { txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1001" + expect resp.http.X-Varnish == "2001 1002" expect resp.http.foo == "foo" } -run @@ -53,12 +54,13 @@ varnish v1 -vcl+backend { } delay 2 varnish v1 -start +varnish v1 -cliok "debug.xid 2999" client c1 { txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1001" + expect resp.http.X-Varnish == "3001" expect resp.http.foo == "bar" } -run diff --git a/bin/varnishtest/tests/p00004.vtc b/bin/varnishtest/tests/p00004.vtc index dab8ea4..343d7de 100644 --- a/bin/varnishtest/tests/p00004.vtc +++ b/bin/varnishtest/tests/p00004.vtc @@ -46,7 +46,7 @@ client c1 { txreq -url "/foo" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "2001 1001" + expect resp.http.X-Varnish == "2001 1002" expect resp.http.foo == "foo" } -run @@ -57,7 +57,7 @@ client c1 { txreq -url "/bar" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "2003 1004" + expect resp.http.X-Varnish == "2003 1005" expect resp.http.bar == "bar" } -run diff --git a/bin/varnishtest/tests/p00006.vtc b/bin/varnishtest/tests/p00006.vtc index 3334f36..0adb67b 100644 --- a/bin/varnishtest/tests/p00006.vtc +++ b/bin/varnishtest/tests/p00006.vtc @@ -43,13 +43,13 @@ client c1 { txreq -url "/foo" -hdr "foo: 1" -hdr "bar: 2" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "2001 1001" + expect resp.http.X-Varnish == "2001 1002" expect resp.http.foo == "foo1" txreq -url "/foo" -hdr "foo: 2" -hdr "bar: 1" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "2002 1003" + expect resp.http.X-Varnish == "2002 1004" expect resp.http.foo == "foo2" } -run diff --git a/bin/varnishtest/tests/r00102.vtc b/bin/varnishtest/tests/r00102.vtc index d2a3c13..0fa597b 100644 --- a/bin/varnishtest/tests/r00102.vtc +++ b/bin/varnishtest/tests/r00102.vtc @@ -28,7 +28,7 @@ client c1 { -body "123456789\n" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1003 1001" + expect resp.http.X-Varnish == "1003 1002" } client c1 -run diff --git a/bin/varnishtest/tests/r00262.vtc b/bin/varnishtest/tests/r00262.vtc index 1deb9ea..09ed730 100644 --- a/bin/varnishtest/tests/r00262.vtc +++ b/bin/varnishtest/tests/r00262.vtc @@ -19,7 +19,7 @@ client c1 { send "GET / HTTP/1.1\r\n\r\n" rxresp expect resp.status == 200 - expect resp.http.X-Varnish == "1003 1001" + expect resp.http.X-Varnish == "1003 1002" } client c1 -run diff --git a/bin/varnishtest/tests/s00002.vtc b/bin/varnishtest/tests/s00002.vtc index 31fb400..ab3736e 100644 --- a/bin/varnishtest/tests/s00002.vtc +++ b/bin/varnishtest/tests/s00002.vtc @@ -69,5 +69,5 @@ client c2 { rxresp expect resp.http.foo == "bar" expect resp.status == 200 - expect resp.http.x-varnish == "1004 1001" + expect resp.http.x-varnish == "1004 1002" } -run From martin at varnish-software.com Thu Dec 18 09:27:53 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] 5d3dc29 Include compat/srandomdev.h in cache_main.c to fix build on Linux. Message-ID: commit 5d3dc293ecbf1d1944bfb60701543d2f47cb4b1c Author: Martin Blix Grydeland Date: Thu Aug 16 10:05:16 2012 +0200 Include compat/srandomdev.h in cache_main.c to fix build on Linux. diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index 37aea3a..8af00f6 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -40,6 +40,10 @@ #include "waiter/waiter.h" #include "hash/hash_slinger.h" +#ifndef HAVE_SRANDOMDEV +#include "compat/srandomdev.h" +#endif + volatile struct params *cache_param; /*-------------------------------------------------------------------- From tfheen at varnish-software.com Thu Dec 18 09:27:53 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] a591999 Update changes for 3.0.2 rc 1 Message-ID: commit a5919990d92a2bd5645a3c8e8e475f8c80c12060 Author: Tollef Fog Heen Date: Thu Oct 6 10:09:43 2011 +0200 Update changes for 3.0.2 rc 1 diff --git a/doc/changes.rst b/doc/changes.rst index cbf4834..87df237 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,4 +1,64 @@ ================================ +Changes from 3.0.1 to 3.0.2 rc 1 +================================ + +Varnishd +-------- + +- Only log the first 20 bytes of extra headers to prevent overflows. + +- Fix crasher bug which sometimes happened if responses are queued and + the backend sends us Vary. `Bug #994`_. + +- Log correct size of compressed when uncompressing them for clients + that do not support compression. `Bug #996`_. + +- Only send Range responses if we are going to send a body. `Bug #1007`_. + +- When varnishd creates a storage file, also unlink it to avoid + leaking disk space over time. `Bug #1008`_. + +- The default size of the `-s file` parameter has been changed to + 100MB instead of 50% of the available disk space. + +- The limit on the number of objects we remove from the cache to make + room for a new one was mistakenly lowered to 10 in 3.0.1. This has + been raised back to 50. `Bug #1012`_. + +- `http_req_size` and `http_resp_size` have been increased to 8192 + bytes. This better matches what other HTTPds have. `Bug #1016`_. + +.. _bug #994: http://varnish-cache.org/trac/ticket/994 +.. _bug #992: http://varnish-cache.org/trac/ticket/992 +.. _bug #1007: http://varnish-cache.org/trac/ticket/1007 +.. _bug #1008: http://varnish-cache.org/trac/ticket/1008 +.. _bug #1012: http://varnish-cache.org/trac/ticket/1012 +.. _bug #1012: http://varnish-cache.org/trac/ticket/1016 + +VCL +--- + +- Allow relational comparisons of floating point types. + +- Make it possible for vmods to fail loading and so cause the VCL + loading to fail. + +varnishncsa +----------- + +- Fixed crash when client was sending illegal HTTP headers. + +- `%{Varnish:handling}` in format strings was broken, this has been + fixed. + +Other +----- + +- Documentation updates + +- Some Solaris portability updates. + +================================ Changes from 3.0.1 rc 1 to 3.0.1 ================================ From tfheen at varnish-software.com Thu Dec 18 09:27:53 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] ad169d8 Document 3.0.2 changes Message-ID: commit ad169d84763ac1c5900f58b00cbf3d0516b6aebe Author: Tollef Fog Heen Date: Fri Oct 21 10:48:09 2011 +0200 Document 3.0.2 changes diff --git a/doc/changes.rst b/doc/changes.rst index 87df237..7557393 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,4 +1,34 @@ ================================ +Changes from 3.0.2 rc 1 to 3.0.2 +================================ + +Varnishd +-------- + +- Make the size of the synthetic object workspace equal to + `http_resp_size` and add workaround to avoid a crash when setting + too long response strings for synthetic objects. + +- Ensure the ban lurker always sleeps the advertised 1 second when it + does not have anything to do. + +- Remove error from `vcl_deliver`. Previously this would assert while + it will now give a syntax error. + +varnishncsa +----------- + +- Add default values for some fields when logging incomplete records + and document the default values. + +Other +----- + +- Documentation updates + +- Some Solaris portability updates. + +================================ Changes from 3.0.1 to 3.0.2 rc 1 ================================ From tfheen at varnish-software.com Thu Dec 18 09:27:53 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] 7be15ba Changelog for 3.0.3 Message-ID: commit 7be15ba0d899c7a3de0e4459dda41e206588be97 Author: Andreas Plesner Jacobsen Date: Fri Jul 27 14:59:36 2012 +0200 Changelog for 3.0.3 diff --git a/doc/changes.rst b/doc/changes.rst index 7557393..e1c6979 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,3 +1,123 @@ +=========================== +Changes from 3.0.2 to 3.0.3 +=========================== + +Varnishd +-------- + +- Fix an issue where the bodybytes counter is not updated correctly on + gunzipped delivery. +- Fix a race on the n_sess counter. This race made varnish do excessive + session workspace allocations. `Bug #897`_. +- Fix some crashes in the gzip code when it runs out of memory. `Bug #1037`_. + `Bug #1043`_. `Bug #1044`_. +- Fix a bug where the regular expression parser could end up in an infinite + loop. `Bug #1047`_. +- Fix a memory leak in the regex code. +- DNS director now uses port 80 by default if not specified. +- Introduce `idle_send_timeout` and increase default value for `send_timeout` + to 600s. This allows a long send timeout for slow clients while still being + able to disconnect idle clients. +- Fix an issue where did not remove HTML comments. `Bug #1092`_. +- Fix a crash when passing with streaming on. +- Fix a crash in the idle session timeout code. +- Fix an issue where the poll waiter did not timeout clients if all clients + were idle. `Bug #1023`_. +- Log regex errors instead of crashing. +- Introduce `pcre_match_limit`, and `pcre_match_limit_recursion` parameters. +- Add CLI commands to manually control health state of a backend. +- Fix an issue where the s_bodybytes counter is not updated correctly on + gunzipped delivery. +- Fix a race on the n_sess counter that leads to increased session object + allocation. `Bug #897`_. +- Fix a crash when we couldn't allocate memory for a fetched object. + `Bug #1100`_. +- Fix an issue where objects could end up in the transient store with a + long TTL, when memory could not be allocated for them in the requested + store. `Bug #1140`_. +- Activate req.hash_ignore_busy when req.hash_always_miss is activated. + `Bug #1073`_. +- Reject invalid tcp port numbers for listen address. `Bug #1035`_. +- Enable JIT for better performing regular expressions. `Bug #1080`_. +- Return VCL errors in exit code when using -C. `Bug #1069`_. +- Stricter validation of acl syntax, to avoid a crash with 5-octet IPv4 + addresses. `Bug #1126`_. +- Fix a crash when first argument to regsub was null. `Bug #1125`_. +- Fix a case where varnish delivered corrupt gzip content when using ESI. + `Bug #1109`_. +- Fix a case where varnish didn't remove the old Date header and served + it alongside the varnish-generated Date header. `Bug #1104`_. +- Make saint mode work, for the case where we have no object with that hash. + `Bug #1091`_. +- Don't save the object body on hit-for-pass objects. +- n_ban_gone counter added to count the number of "gone" bans. +- Ban lurker rewritten to properly sleep when no bans are present, and + otherwise to process the list at the configured speed. +- Fix a case where varnish delivered wrong content for an uncompressed page + with compressed ESI child. `Bug #1029`_. +- Fix an issue where varnish runs out of thread workspace when processing + many ESI includes on an object. `Bug #1038`_. +- Fix a crash when streaming was enabled for an empty body. +- Better error reporting for some fetch errors. +- Small performance optimizations. + +.. _bug #897: http://varnish-cache.org/trac/ticket/897 +.. _bug #1023: http://varnish-cache.org/trac/ticket/1023 +.. _bug #1029: http://varnish-cache.org/trac/ticket/1029 +.. _bug #1023: http://varnish-cache.org/trac/ticket/1023 +.. _bug #1035: http://varnish-cache.org/trac/ticket/1035 +.. _bug #1037: http://varnish-cache.org/trac/ticket/1037 +.. _bug #1038: http://varnish-cache.org/trac/ticket/1038 +.. _bug #1043: http://varnish-cache.org/trac/ticket/1043 +.. _bug #1044: http://varnish-cache.org/trac/ticket/1044 +.. _bug #1047: http://varnish-cache.org/trac/ticket/1047 +.. _bug #1069: http://varnish-cache.org/trac/ticket/1069 +.. _bug #1073: http://varnish-cache.org/trac/ticket/1073 +.. _bug #1080: http://varnish-cache.org/trac/ticket/1080 +.. _bug #1091: http://varnish-cache.org/trac/ticket/1091 +.. _bug #1092: http://varnish-cache.org/trac/ticket/1092 +.. _bug #1100: http://varnish-cache.org/trac/ticket/1100 +.. _bug #1104: http://varnish-cache.org/trac/ticket/1104 +.. _bug #1109: http://varnish-cache.org/trac/ticket/1109 +.. _bug #1125: http://varnish-cache.org/trac/ticket/1125 +.. _bug #1126: http://varnish-cache.org/trac/ticket/1126 +.. _bug #1140: http://varnish-cache.org/trac/ticket/1140 + +varnishncsa +----------- + +- Support for \t\n in varnishncsa format strings. +- Add new format: %{VCL_Log:foo}x which output key:value from std.log() in + VCL. +- Add user-defined date formatting, using %{format}t. + +varnishtest +----------- + +- resp.body is now available for inspection. +- Make it possible to test for the absence of an HTTP header. `Bug #1062`_. +- Log the full panic message instead of shortening it to 512 characters. + +.. _bug #1062: http://varnish-cache.org/trac/ticket/1062 + +varnishstat +----------- + +- Add json output (-j). + +Other +----- + +- Documentation updates. +- Bump minimum number of threads to 50 in RPM packages. +- RPM packaging updates. +- Fix some compilation warnings on Solaris. +- Fix some build issues on Open/Net/DragonFly-BSD. +- Fix build on FreeBSD 10-current. +- Fix libedit detection on \*BSD OSes. `Bug #1003`_. + +.. _bug #1003: http://varnish-cache.org/trac/ticket/1003 + ================================ Changes from 3.0.2 rc 1 to 3.0.2 ================================ @@ -60,10 +180,11 @@ Varnishd .. _bug #994: http://varnish-cache.org/trac/ticket/994 .. _bug #992: http://varnish-cache.org/trac/ticket/992 +.. _bug #996: http://varnish-cache.org/trac/ticket/996 .. _bug #1007: http://varnish-cache.org/trac/ticket/1007 .. _bug #1008: http://varnish-cache.org/trac/ticket/1008 .. _bug #1012: http://varnish-cache.org/trac/ticket/1012 -.. _bug #1012: http://varnish-cache.org/trac/ticket/1016 +.. _bug #1016: http://varnish-cache.org/trac/ticket/1016 VCL --- From tfheen at varnish-software.com Thu Dec 18 09:27:53 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] 043e7dc Duplicate changelog entry Message-ID: commit 043e7dcf13ac6f40c2c55b3dbfdc0bd6dfc2b372 Author: Andreas Plesner Jacobsen Date: Sun Aug 5 23:51:31 2012 +0200 Duplicate changelog entry diff --git a/doc/changes.rst b/doc/changes.rst index e1c6979..27faf07 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -28,8 +28,6 @@ Varnishd - Add CLI commands to manually control health state of a backend. - Fix an issue where the s_bodybytes counter is not updated correctly on gunzipped delivery. -- Fix a race on the n_sess counter that leads to increased session object - allocation. `Bug #897`_. - Fix a crash when we couldn't allocate memory for a fetched object. `Bug #1100`_. - Fix an issue where objects could end up in the transient store with a From tfheen at varnish-software.com Thu Dec 18 09:27:53 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] 33f5bae Another duplicate changelog entry, thanks fgs Message-ID: commit 33f5bae99a64fc7b04fffe70eb2965346aadc3a7 Author: Andreas Plesner Jacobsen Date: Wed Aug 8 09:34:29 2012 +0200 Another duplicate changelog entry, thanks fgs diff --git a/doc/changes.rst b/doc/changes.rst index 27faf07..87a7648 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -5,8 +5,6 @@ Changes from 3.0.2 to 3.0.3 Varnishd -------- -- Fix an issue where the bodybytes counter is not updated correctly on - gunzipped delivery. - Fix a race on the n_sess counter. This race made varnish do excessive session workspace allocations. `Bug #897`_. - Fix some crashes in the gzip code when it runs out of memory. `Bug #1037`_. From phk at FreeBSD.org Thu Dec 18 09:27:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:53 +0100 Subject: [experimental-ims] 7c784d5 Add long time missing error handling of gunzip'ing fetched objects for ESI processing. Message-ID: commit 7c784d5c9d2dd959a5ea1a1bea5f7bbb4173437d Author: Poul-Henning Kamp Date: Tue Aug 21 08:19:44 2012 +0000 Add long time missing error handling of gunzip'ing fetched objects for ESI processing. Polish the VGZ code a bit while here anyway. Fixes #1184 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 2441bd6..43c104e 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -803,6 +803,13 @@ void Fetch_Init(void); /* cache_gzip.c */ struct vgz; +enum vgzret_e { + VGZ_ERROR = -1, + VGZ_OK = 0, + VGZ_END = 1, + VGZ_STUCK = 2, +}; + enum vgz_flag { VGZ_NORMAL, VGZ_ALIGN, VGZ_RESET, VGZ_FINISH }; struct vgz *VGZ_NewUngzip(struct vsl_log *vsl, const char *id); struct vgz *VGZ_NewGzip(struct vsl_log *vsl, const char *id); @@ -811,22 +818,16 @@ int VGZ_IbufEmpty(const struct vgz *vg); void VGZ_Obuf(struct vgz *, void *, ssize_t len); int VGZ_ObufFull(const struct vgz *vg); int VGZ_ObufStorage(struct busyobj *, struct vgz *vg); -int VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag); -int VGZ_Gunzip(struct vgz *, const void **, size_t *len); -int VGZ_Destroy(struct vgz **); +enum vgzret_e VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag); +enum vgzret_e VGZ_Gunzip(struct vgz *, const void **, size_t *len); +enum vgzret_e VGZ_Destroy(struct vgz **); void VGZ_UpdateObj(const struct vgz*, struct object *); int VGZ_WrwInit(struct vgz *vg); -int VGZ_WrwGunzip(struct req *, struct vgz *, const void *ibuf, +enum vgzret_e VGZ_WrwGunzip(struct req *, struct vgz *, const void *ibuf, ssize_t ibufl); void VGZ_WrwFlush(struct req *, struct vgz *vg); -/* Return values */ -#define VGZ_ERROR -1 -#define VGZ_OK 0 -#define VGZ_END 1 -#define VGZ_STUCK 2 - /* cache_http.c */ unsigned HTTP_estimate(unsigned nhttp); void HTTP_Copy(struct http *to, const struct http * const fm); diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index c9cc689..7dc74ae 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -119,7 +119,7 @@ vfp_esi_bytes_gu(struct busyobj *bo, const struct vef_priv *vef, { struct vgz *vg; ssize_t wl; - int i; + enum vgzret_e vr; size_t dl; const void *dp; @@ -137,10 +137,13 @@ vfp_esi_bytes_gu(struct busyobj *bo, const struct vef_priv *vef, } if (VGZ_ObufStorage(bo, vg)) return(-1); - i = VGZ_Gunzip(vg, &dp, &dl); - xxxassert(i == VGZ_OK || i == VGZ_END); - VEP_Parse(bo, dp, dl); - VFP_update_length(bo, dl); + vr = VGZ_Gunzip(vg, &dp, &dl); + if (vr < VGZ_OK) + return (-1); + if (dl > 0) { + VEP_Parse(bo, dp, dl); + VFP_update_length(bo, dl); + } } return (1); } @@ -265,7 +268,7 @@ vfp_esi_bytes_gg(const struct busyobj *bo, struct vef_priv *vef, ssize_t wl; size_t dl; const void *dp; - int i; + enum vgzret_e vr; CHECK_OBJ_NOTNULL(vef, VEF_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); @@ -280,9 +283,9 @@ vfp_esi_bytes_gg(const struct busyobj *bo, struct vef_priv *vef, do { wl = vef->ibuf_sz - (vef->ibuf_i - vef->ibuf); VGZ_Obuf(bo->vgz_rx, vef->ibuf_i, wl); - i = VGZ_Gunzip(bo->vgz_rx, &dp, &dl); - /* XXX: check i */ - assert(i >= VGZ_OK); + vr = VGZ_Gunzip(bo->vgz_rx, &dp, &dl); + if (vr < VGZ_OK) + return (-1); if (dl > 0 && vfp_vep_inject(bo, vef, dl)) return (-1); } while (!VGZ_IbufEmpty(bo->vgz_rx)); diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 80149c6..5a53cd8 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -218,7 +218,7 @@ VGZ_ObufStorage(struct busyobj *bo, struct vgz *vg) /*--------------------------------------------------------------------*/ -int +enum vgzret_e VGZ_Gunzip(struct vgz *vg, const void **pptr, size_t *plen) { int i; @@ -247,13 +247,13 @@ VGZ_Gunzip(struct vgz *vg, const void **pptr, size_t *plen) return (VGZ_END); if (i == Z_BUF_ERROR) return (VGZ_STUCK); - VSL(SLT_Debug, 0, "Unknown INFLATE=%d (%s)\n", i, vg->vz.msg); + VSLb(vg->vsl, SLT_Gzip, "Gunzip error: %d (%s)", i, vg->vz.msg); return (VGZ_ERROR); } /*--------------------------------------------------------------------*/ -int +enum vgzret_e VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, enum vgz_flag flags) { int i; @@ -285,12 +285,13 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, enum vgz_flag flags) } vg->last_i = i; if (i == Z_OK) - return (0); + return (VGZ_OK); if (i == Z_STREAM_END) - return (1); + return (VGZ_END); if (i == Z_BUF_ERROR) - return (2); - return (-1); + return (VGZ_STUCK); + VSLb(vg->vsl, SLT_Gzip, "Gzip error: %d (%s)", i, vg->vz.msg); + return (VGZ_ERROR); } /*-------------------------------------------------------------------- @@ -314,11 +315,11 @@ VGZ_WrwInit(struct vgz *vg) * Leave flushing to caller, more data may be coming. */ -int +enum vgzret_e VGZ_WrwGunzip(struct req *req, struct vgz *vg, const void *ibuf, ssize_t ibufl) { - int i; + enum vgzret_e vr; size_t dl; const void *dp; struct worker *wrk; @@ -333,16 +334,14 @@ VGZ_WrwGunzip(struct req *req, struct vgz *vg, const void *ibuf, return (VGZ_OK); do { if (vg->m_len == vg->m_sz) - i = VGZ_STUCK; + vr = VGZ_STUCK; else { - i = VGZ_Gunzip(vg, &dp, &dl); + vr = VGZ_Gunzip(vg, &dp, &dl); vg->m_len += dl; } - if (i < VGZ_OK) { - /* XXX: VSL ? */ - return (-1); - } - if (vg->m_len == vg->m_sz || i == VGZ_STUCK) { + if (vr < VGZ_OK) + return (vr); + if (vg->m_len == vg->m_sz || vr == VGZ_STUCK) { req->acct_req.bodybytes += vg->m_len; (void)WRW_Write(wrk, vg->m_buf, vg->m_len); (void)WRW_Flush(wrk); @@ -350,9 +349,9 @@ VGZ_WrwGunzip(struct req *req, struct vgz *vg, const void *ibuf, VGZ_Obuf(vg, vg->m_buf, vg->m_sz); } } while (!VGZ_IbufEmpty(vg)); - if (i == VGZ_STUCK) - i = VGZ_OK; - return (i); + if (vr == VGZ_STUCK) + vr = VGZ_OK; + return (vr); } /*--------------------------------------------------------------------*/ @@ -393,10 +392,11 @@ VGZ_UpdateObj(const struct vgz *vg, struct object *obj) /*-------------------------------------------------------------------- */ -int +enum vgzret_e VGZ_Destroy(struct vgz **vgp) { struct vgz *vg; + enum vgzret_e vr; int i; vg = *vgp; @@ -420,14 +420,18 @@ VGZ_Destroy(struct vgz **vgp) i = Z_STREAM_END; if (vg->m_buf) free(vg->m_buf); - FREE_OBJ(vg); if (i == Z_OK) - return (VGZ_OK); - if (i == Z_STREAM_END) - return (VGZ_END); - if (i == Z_BUF_ERROR) - return (VGZ_STUCK); - return (VGZ_ERROR); + vr = VGZ_OK; + else if (i == Z_STREAM_END) + vr = VGZ_END; + else if (i == Z_BUF_ERROR) + vr = VGZ_STUCK; + else { + VSLb(vg->vsl, SLT_Gzip, "G(un)zip error: %d (%s)", i, vg->vz.msg); + vr = VGZ_ERROR; + } + FREE_OBJ(vg); + return (vr); } /*-------------------------------------------------------------------- diff --git a/bin/varnishtest/tests/r01184.vtc b/bin/varnishtest/tests/r01184.vtc new file mode 100644 index 0000000..9a74f28 --- /dev/null +++ b/bin/varnishtest/tests/r01184.vtc @@ -0,0 +1,126 @@ +varnishtest "Corrupt gzip'ed ESI objects" + +# First, check that our data is actually good + +server s1 { + rxreq + expect req.url == "/" + txresp -nolen \ + -hdr "Content-Encoding: gzip" \ + -hdr "Transfer-Encoding: Chunked" + send "ed\r\n" + sendhex " 1f 8b 08 00 c2 39 33 50 02 03 45 90 4d 6a 03 31" + sendhex " 0c 85 f7 73 8a 47 2e 10 d9 f2 9f ca 34 d0 c2 64" + sendhex " 5b 08 bd 80 2d cb 10 28 5d 34 f4 fe f5 30 d0 68" + sendhex " 25 89 a7 f7 3e b4 be 6f d7 8f db 76 59 e8 28 d8" + sendhex " 10 45 f3 a9 83 b8 18 1c 7b c2 30 55 04 17 13 c4" + sendhex " 0f 07 5f 7a 38 f4 8e 50 b3 37 d4 3a 32 4a 34 07" + sendhex " 8a 9c d1 92 77 48 d4 0a 72 ea 06 5f b3 1c fa dd" + sendhex " 2b b9 88 20 99 e6 9a 3c 84 7c 85 8e 50 e0 59 2a" + sendhex " 42 b0 8a 34 0f 96 d5 1e f7 97 fb b7 7e fd 4e 87" + sendhex " c7 8f be 9e ce fb 74 3a 3f 51 89 a3 9b 7e b2 43" + sendhex " a4 86 a2 55 90 b9 29 4c 4b 83 b8 99 5f b5 bb 27" + sendhex " 6a d4 86 18 22 83 8a 26 f4 11 1a 5c eb 34 3b ca" + sendhex " 20 31 9e 12 29 ff a8 92 78 a2 e6 ec 61 55 12 fc" + sendhex " 68 84 6c 12 41 b9 cf 2f 30 3b f0 10 5e d6 b7 eb" + sendhex " e7 76 bb 2c 7f 8c 90 4a 14 4c 01 00 00" + send "\r\n" + chunkedlen 0 + rxreq + expect req.url == "/incl" + txresp -body "INCLUDED\n" +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.do_esi = true; + } +} -start + +client c1 { + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.bodylen == 315 +} -run + +# Second, corrupt them, while using the g-g path + +server s1 { + rxreq + expect req.url == "/c" + txresp -nolen \ + -hdr "Content-Encoding: gzip" \ + -hdr "Transfer-Encoding: Chunked" + send "ed\r\n" + sendhex " 1f 8b 08 00 c2 39 33 50 02 03 45 90 4d 6a 03 31" + sendhex " 0c 85 f7 73 8a 47 2e 10 d9 f2 9f ca 34 d0 c2 64" + sendhex " 5b 08 bd 80 2d cb 10 28 5d 34 f4 fe f5 30 d0 68" + sendhex " 25 89 a7 f7 3e b4 be 6f d7 8f db 76 59 e8 28 d8" + sendhex " 10 45 f3 a9 83 b8 18 1c 7b c2 30 55 04 17 13 c4" + sendhex " 0f 07 5f 7a 38 f4 8e 50 b3 37 d4 3a 32 4a 34 07" + sendhex " FF FF FF FF FF FF FF FF 72 ea 06 5f b3 1c fa dd" + sendhex " 2b b9 88 20 99 e6 9a 3c 84 7c 85 8e 50 e0 59 2a" + sendhex " 42 b0 8a 34 0f 96 d5 1e f7 97 fb b7 7e fd 4e 87" + sendhex " c7 8f be 9e ce fb 74 3a 3f 51 89 a3 9b 7e b2 43" + sendhex " a4 86 a2 55 90 b9 29 4c 4b 83 b8 99 5f b5 bb 27" + sendhex " 6a d4 86 18 22 83 8a 26 f4 11 1a 5c eb 34 3b ca" + sendhex " 20 31 9e 12 29 ff a8 92 78 a2 e6 ec 61 55 12 fc" + sendhex " 68 84 6c 12 41 b9 cf 2f 30 3b f0 10 5e d6 b7 eb" + sendhex " e7 76 bb 2c 7f 8c 90 4a 14 4c 01 00 00" + send "\r\n" + chunkedlen 0 +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.do_esi = true; + } +} + +client c1 { + txreq -url "/c" + rxresp + expect resp.status == 503 +} -run + +# Third, corrupt them, while using the g-u path + +server s1 { + rxreq + expect req.url == "/d" + txresp -nolen \ + -hdr "Content-Encoding: gzip" \ + -hdr "Transfer-Encoding: Chunked" + send "ed\r\n" + sendhex " 1f 8b 08 00 c2 39 33 50 02 03 45 90 4d 6a 03 31" + sendhex " 0c 85 f7 73 8a 47 2e 10 d9 f2 9f ca 34 d0 c2 64" + sendhex " 5b 08 bd 80 2d cb 10 28 5d 34 f4 fe f5 30 d0 68" + sendhex " 25 89 a7 f7 3e b4 be 6f d7 8f db 76 59 e8 28 d8" + sendhex " 10 45 f3 a9 83 b8 18 1c 7b c2 30 55 04 17 13 c4" + sendhex " 0f 07 5f 7a 38 f4 8e 50 b3 37 d4 3a 32 4a 34 07" + sendhex " FF FF FF FF FF FF FF FF 72 ea 06 5f b3 1c fa dd" + sendhex " 2b b9 88 20 99 e6 9a 3c 84 7c 85 8e 50 e0 59 2a" + sendhex " 42 b0 8a 34 0f 96 d5 1e f7 97 fb b7 7e fd 4e 87" + sendhex " c7 8f be 9e ce fb 74 3a 3f 51 89 a3 9b 7e b2 43" + sendhex " a4 86 a2 55 90 b9 29 4c 4b 83 b8 99 5f b5 bb 27" + sendhex " 6a d4 86 18 22 83 8a 26 f4 11 1a 5c eb 34 3b ca" + sendhex " 20 31 9e 12 29 ff a8 92 78 a2 e6 ec 61 55 12 fc" + sendhex " 68 84 6c 12 41 b9 cf 2f 30 3b f0 10 5e d6 b7 eb" + sendhex " e7 76 bb 2c 7f 8c 90 4a 14 4c 01 00 00" + send "\r\n" + chunkedlen 0 +} -start + +varnish v1 -vcl+backend { + sub vcl_fetch { + set beresp.do_esi = true; + set beresp.do_gunzip = true; + } +} + +client c1 { + txreq -url "/d" + rxresp + expect resp.status == 503 +} -run From perbu at varnish-software.com Thu Dec 18 09:27:54 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] a3eb213 Remove the FAQ from sphinx. It was never properly maintained. Message-ID: commit a3eb21309ffd22e3143f779ad18e8f8ac6562d18 Author: Per Buer Date: Tue Aug 21 10:28:52 2012 +0200 Remove the FAQ from sphinx. It was never properly maintained. diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index d2e5258..987c069 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -91,11 +91,6 @@ doctest: EXTRA_DIST = \ conf.py \ index.rst \ - faq/configuration.rst \ - faq/general.rst \ - faq/http.rst \ - faq/index.rst \ - faq/logging.rst \ glossary/index.rst \ installation/bugs.rst \ installation/help.rst \ diff --git a/doc/sphinx/faq/configuration.rst b/doc/sphinx/faq/configuration.rst deleted file mode 100644 index df06b07..0000000 --- a/doc/sphinx/faq/configuration.rst +++ /dev/null @@ -1,53 +0,0 @@ -%%%%%%%%%%%%%%% -Configuration -%%%%%%%%%%%%%%% - -.. _faq-vcl: - -VCL -=== - -**What is VCL?** - -VCL is an acronym for Varnish Configuration Language. In a VCL file, you configure how Varnish should behave. Sample VCL files will be included in this Wiki at a later stage. - -**Where is the documentation on VCL?** - -Please see ``man 7 vcl``. There are also some real-world examples on -the `wiki `_ - - -**How do I load VCL file while Varnish is running?** - -* Place the VCL file on the server -* Telnet into the managment port. -* do a "vcl.load " in managment interface. is whatever you would like to call your new configuration. -* do a "vcl.use " to start using your new config. - -**Should I use ''pipe'' or ''pass'' in my VCL code? What is the difference?** - -When varnish does a ``pass`` it acts like a normal HTTP proxy. It -reads the request and pushes it onto the backend. The next HTTP -request can then be handled like any other. - -``pipe`` is only used when Varnish for some reason can't handle the -``pass``. ``pipe`` reads the request, pushes in onty the backend -_only_ pushes bytes back and forth, with no other actions taken. - -Since most HTTP clients do pipeline several requests into one -connection this might give you an undesirable result - as every -subsequent request will reuse the existing ``pipe``. Please see `this -article ` for -more details and a workaround. - -Varnish versions prior to 2.0 does not support handling a request body -with ``pass`` mode, so in those releases ``pipe`` is required for -correct handling. - -In 2.0 and later, ``pass`` will handle the request body correctly. - -If you get 503 errors when making a request which is ``pass`` ed, make sure -that you're specifying the backend before returning from vcl_recv with ``pass``. - - - diff --git a/doc/sphinx/faq/general.rst b/doc/sphinx/faq/general.rst deleted file mode 100644 index b0c7e13..0000000 --- a/doc/sphinx/faq/general.rst +++ /dev/null @@ -1,238 +0,0 @@ -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -General questions -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -What is Varnish? -================ - -Please see ref:`tutorial-intro`. - -How... -====== - -**How much RAM/Disk do I need for Varnish?** - -That depends on pretty much everything. - -I think our best current guidance is that you go for a cost-effective -RAM configuration, something like 1-16GB, and an SSD. - -Unless you positively know that you will need it, there is -little point in spending a fortune on a hand-sewn motherboard -that can fit several TB in special RAM blocks, riveted together -by leftover watch-makers in Switzerland. - -On the other hand, if you plot your traffic in Gb/s, you probably -need all the RAM you can afford/get. - -**How can I limit Varnish to use less RAM?** - -You can not. Varnish operates in Virtual Memory and it is up to the -kernel to decide which process gets to use how much RAM to map the -virtual address-space of the process. - -**How do I instruct varnish to ignore the query parameters and only cache one instance of an object?** - -This can be achieved by removing the query parameters using a regexp:: - - sub vcl_recv { - set req.url = regsub(req.url, "\?.*", ""); - } - -**How can I force a refresh on a object cached by varnish?** - -Refreshing is often called `purging -`_ a document. You can -purge at least 2 different ways in Varnish: - -1. Command line - - From the command line you can write:: - - ban.url ^/$ - - to ban your / document. As you might see ban.url takes an `regular expression `_ - as its argument. Hence the ``^`` and ``$`` at the front and end. If the ``^`` is omitted, all the documents ending in a ``/`` in the cache would be deleted. - - So to delete all the documents in the cache, write:: - - ban.url . - - at the command line. - -2. HTTP PURGE - - VCL code to allow HTTP PURGE is to be found `here `_. Note that this method does not support wildcard purging. - -**How can I debug the requests of a single client?** - -The "varnishlog" utility may produce a horrendous amount of output. To be able debug our own traffic can be useful. - -The ReqStart token will include the client IP address. To see log entries matching this, type:: - - $ varnishlog -c -m ReqStart:192.0.2.123 - -To see the backend requests generated by a client IP address, we can match on the TxHeader token, since the IP address of the client is included in the X-Forwarded-For header in the request sent to the backend. - -At the shell command line, type:: - - $ varnishlog -b -m TxHeader:192.0.2.123 - -**How can I rewrite URLS before they are sent to the backend?** - -You can use the ``regsub()`` function to do this. Here's an example for zope, to rewrite URLs for the virtualhostmonster:: - - if (req.http.host ~ "^(www.)?example.com") { - set req.url = regsub(req.url, "^", "/VirtualHostBase/http/example.com:80/Sites/example.com/VirtualHostRoot"); - } - -**I have a site with many host names, how do I keep them from multiplying the cache?** - -You can do this by normalizing the ``Host`` header for all your host names. Here's a VCL example:: - - if (req.http.host ~ "^(www.)?example.com") { - set req.http.host = "example.com"; - } - -**How do I do to alter the request going to the backend?** -You can use the ``bereq`` object for altering requests going to the backend, but you can only 'set' values to it. Therefore use req.url to build the request:: - - sub vcl_miss { - set bereq.url = regsub(req.url,"stream/","/"); - return(fetch); - } - -**How can I customize the error messages that Varnish returns?** - -A custom error page can be generated by adding a ``vcl_error`` to your configuration file. The default error page looks like this:: - - sub vcl_error { - set obj.http.Content-Type = "text/html; charset=utf-8"; - - synthetic {" - - - - - "} + obj.status + " " + obj.response + {" - - -

Error "} + obj.status + " " + obj.response + {"

-

"} + obj.response + {"

-

Guru Meditation:

-

XID: "} + req.xid + {"

-
Varnish
- - - "}; - return(deliver); - } - -**How do I instruct varnish to ignore the query parameters and only cache one instance of an object?** - -This can be achieved by removing the query parameters using a regexp:: - - sub vcl_recv { - set req.url = regsub(req.url, "\?.*", ""); - } - - -Can I... -======== - -**Can I run Varnish on the same system as Apache?** - -Yes, and many people do that with good success. - -There will be competition for resources, but Apache is not particular -good at using RAM effectively and Varnish is, so this synergy usually -more than compensates for the competition. - -**Can I run multiple Varnish on the same system?** - -Yes, as long as you give them different TCP ports and different ``-n`` -arguments, you will be fine. - - -**Can I cache multiple virtual hosts with one Varnish?** - -Yes, that works right out of the box. - -**Can I see what is cached in Varnish?** - -That is not possible for several reasons. A command to list -all the contents of a Varnish cache with millions of objects would -bring your Varnish to a standstill while it traverses the index. - -Besides, the output is a lot less useful than you might think. - -**Can I use Varnish to do HTTPS?** - -Not at present, and while we keep an eye on this, there are no -current plans to add HTTPS support, until we can find a way where -it adds significant value, relative to running a stand-alone -HTTPS proxy such as nginx or pound. - -**Can Varnish load balance between multiple backends?** - -Yes, you need VCL code like this:: - - director foobar round-robin { - { .backend = { .host = "www1.example.com"; .port = "http"; } } - { .backend = { .host = "www2.example.com"; .port = "http"; } } - } - - sub vcl_recv { - set req.backend = foobar; - } - -Please see :ref:`tutorial-advanced_backend_servers-directors`. - - -Why ... -======= - -**Why does it look like Varnish sends all requests to the backend? I thought it was a cache?** - - -Please see :ref:`tutorial-increasing_your_hitrate`. - -**Why does Varnish require the system to have a C compiler?** - -The :ref:`VCL ` compiler generates C source as output (your -config file), and uses the systems C-compiler to compile that into a -shared library. If there is no C compiler, Varnish will not work. - -**Isn't that security problem?** - -The days when you could prevent people from running non-approved -programs by removing the C compiler from your system ended roughly -with the VAX 11/780 computer. - -Troubleshooting -=============== - -**Why am I getting a cache hit, but a request is still going to my backend?** - -Varnish has a feature called **hit for pass**, which is used when Varnish gets a response from the backend and finds out it cannot be cached. In such cases, Varnish will create a cache object that records that fact, so that the next request goes directly to "pass". - - Since Varnish bundles multiple requests for the same URL to the backend, a common case where a client will get a **hit for pass** is: - * Client 1 requests url /foo - * Client 2..N request url /foo - * Varnish tasks a worker to fetch /foo for Client 1 - * Client 2..N are now queued pending response from the worker - * Worker returns object to varnish which turns out to be non-cacheable. - * Client 2..N are now given the **hit for pass** object instructing them to go to the backend - -The **hit for pass** object will stay cached for the duration of its ttl. This means that subsequent clients requesting /foo will be sent straight to the backend as long as the **hit for pass** object exists. -The :command:`varnishstat` can tell you how many **hit for pass** objects varnish has served. The default vcl will set ttl for a hit_for_pass object to 120s. But you can override this, using the following logic:: - - sub vcl_fetch { - if (!obj.cacheable) { - # Limit the lifetime of all 'hit for pass' objects to 10 seconds - obj.ttl = 10s; - return(hit_for_pass); - } - } - diff --git a/doc/sphinx/faq/http.rst b/doc/sphinx/faq/http.rst deleted file mode 100644 index 1e276ba..0000000 --- a/doc/sphinx/faq/http.rst +++ /dev/null @@ -1,55 +0,0 @@ -%%%%%%%%%%% -HTTP -%%%%%%%%%%% - -**What is the purpose of the X-Varnish HTTP header?** - -The X-Varnish HTTP header allows you to find the correct log-entries for the transaction. For a cache hit, X-Varnish will contain both the ID of the current request and the ID of the request that populated the cache. It makes debugging Varnish a lot easier. - -**Does Varnish support compression?** - -This is a simple question with a complicated answer; see `WIKI `_. - -**How do I add a HTTP header?** - -To add a HTTP header, unless you want to add something about the client/request, it is best done in vcl_fetch as this means it will only be processed every time the object is fetched:: - - sub vcl_fetch { - # Add a unique header containing the cache servers IP address: - remove beresp.http.X-Varnish-IP; - set beresp.http.X-Varnish-IP = server.ip; - # Another header: - set beresp.http.Foo = "bar"; - } - -**How can I log the client IP address on the backend?** - -All I see is the IP address of the varnish server. How can I log the client IP address? - -We will need to add the IP address to a header used for the backend request, and configure the backend to log the content of this header instead of the address of the connecting client (which is the varnish server). - -Varnish configuration:: - - sub vcl_recv { - # Add a unique header containing the client address - remove req.http.X-Forwarded-For; - set req.http.X-Forwarded-For = client.ip; - # [...] - } - -For the apache configuration, we copy the "combined" log format to a new one we call "varnishcombined", for instance, and change the client IP field to use the content of the variable we set in the varnish configuration:: - - LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" varnishcombined - -And so, in our virtualhost, you need to specify this format instead of "combined" (or "common", or whatever else you use):: - - - ServerName www.example.com - # [...] - CustomLog /var/log/apache2/www.example.com/access.log varnishcombined - # [...] - - -The [http://www.openinfo.co.uk/apache/index.html mod_extract_forwarded Apache module] might also be useful. - - diff --git a/doc/sphinx/faq/index.rst b/doc/sphinx/faq/index.rst deleted file mode 100644 index a8136a8..0000000 --- a/doc/sphinx/faq/index.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _faq: - -%%%%%%%%%%%%%%%%%%%%%%%%%% -Frequently asked questions -%%%%%%%%%%%%%%%%%%%%%%%%%% - -The most frequently asked questions about Varnish in different contexts. - - -.. toctree:: - - general.rst - http.rst - configuration.rst - logging.rst - -.. todo:: - [V] - Now that the FAQ has been all reStructureized, we need to qualitycheck the content, update old info, and delete unrelevant info. - - diff --git a/doc/sphinx/faq/logging.rst b/doc/sphinx/faq/logging.rst deleted file mode 100644 index 89fdbde..0000000 --- a/doc/sphinx/faq/logging.rst +++ /dev/null @@ -1,8 +0,0 @@ -%%%%%%%%%%%%%%%%%%% -Logging -%%%%%%%%%%%%%%%%%%% - -**Where can I find the log files?** - -Varnish does not log to a file, but to shared memory log. Use the varnishlog utility to print the shared memory log or varnishncsa to present it in the Apache / NCSA "combined" log format. - diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst index 2558413..06eb52a 100644 --- a/doc/sphinx/index.rst +++ b/doc/sphinx/index.rst @@ -21,7 +21,6 @@ Contents: installation/index.rst tutorial/index.rst reference/index.rst - faq/index.rst phk/index.rst glossary/index.rst From perbu at varnish-software.com Thu Dec 18 09:27:54 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 77d35ab remove warning (invalid comment Message-ID: commit 77d35ab8812311b871956f53bf27fb26b6cf5e93 Author: Per Buer Date: Tue Aug 21 10:30:26 2012 +0200 remove warning (invalid comment diff --git a/doc/sphinx/glossary/index.rst b/doc/sphinx/glossary/index.rst index 7a2787d..3da5af3 100644 --- a/doc/sphinx/glossary/index.rst +++ b/doc/sphinx/glossary/index.rst @@ -7,8 +7,7 @@ Varnish Glossary .. glossary:: :sorted: - .. comment: - + .. This file will be sorted automagically during formatting, so we keep the source in subject order to make sure we cover all bases. From phk at FreeBSD.org Thu Dec 18 09:27:54 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] f656558 Try to make osx build happier. Message-ID: commit f656558cc4d0350cd9f8340960d88c90f0e2783e Author: Poul-Henning Kamp Date: Tue Aug 21 09:01:36 2012 +0000 Try to make osx build happier. And yes, why ptrdiff_t even exists and why it can be different, semantically or otherwise, from ssize_t is beyond me... diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c index 76ef874..6404a3e 100644 --- a/lib/libvcl/vcc_backend.c +++ b/lib/libvcl/vcc_backend.c @@ -708,8 +708,8 @@ vcc_ParseDirector(struct vcc *tl) ERRCHK(tl); if (tl->t->e - tl->t->b > 64) { VSB_printf(tl->sb, - "Name of %.*s too long (max 64, is %zd):\n", - PF(t_first), (tl->t->e - tl->t->b)); + "Name of %.*s too long (max 64, is %zu):\n", + PF(t_first), (size_t)(tl->t->e - tl->t->b)); vcc_ErrWhere(tl, tl->t); return; } From tfheen at varnish-software.com Thu Dec 18 09:27:54 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 7bced23 Quiet build by default Message-ID: commit 7bced23aecd08a94c953835b16c63ea0f5264652 Author: Tollef Fog Heen Date: Tue Aug 21 11:44:13 2012 +0200 Quiet build by default diff --git a/configure.ac b/configure.ac index e1311f0..13c38d1 100644 --- a/configure.ac +++ b/configure.ac @@ -12,7 +12,8 @@ OCFLAGS="$CFLAGS" AC_CANONICAL_SYSTEM AC_LANG(C) -AM_INIT_AUTOMAKE([foreign color-tests parallel-tests]) +AM_INIT_AUTOMAKE([1.11 foreign color-tests parallel-tests]) +AM_SILENT_RULES([yes]) AC_DISABLE_STATIC AC_PROG_LIBTOOL From perbu at varnish-software.com Thu Dec 18 09:27:54 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 652cdeb Showcase !~ in the regex example Message-ID: commit 652cdeb92ff3bd68d2485ae2f07f585dc75da1a4 Author: Per Buer Date: Wed Aug 22 09:13:36 2012 +0200 Showcase !~ in the regex example diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 736c7c0..eee9c0b 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -404,7 +404,8 @@ insensitivity* add the flag within parens following a question mark, like this: :: - if (req.http.host ~ "(?i)example.com$") { + # If host is NOT example dot com.. + if (req.http.host !~ "(?i)example.com$") { ... } From phk at FreeBSD.org Thu Dec 18 09:27:54 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 07d9e57 Pass -g to nm(1), it makes backtraces on OSX marginally less useless Message-ID: commit 07d9e57d2a2afaab66daf040c9b06699882b1021 Author: Poul-Henning Kamp Date: Wed Aug 22 08:59:47 2012 +0000 Pass -g to nm(1), it makes backtraces on OSX marginally less useless diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 7b42437..a760b40 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -246,7 +246,7 @@ Symbol_hack(const char *a0) uintptr_t a; struct symbols *s; - bprintf(buf, "nm -an %s 2>/dev/null", a0); + bprintf(buf, "nm -agn %s 2>/dev/null", a0); fi = popen(buf, "r"); if (fi == NULL) return; From phk at FreeBSD.org Thu Dec 18 09:27:54 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 83387b8 Initialize pthreads stuff before we use it. Message-ID: commit 83387b8ba36c9793370e99b79688c9cc9cae0ef7 Author: Poul-Henning Kamp Date: Wed Aug 22 09:07:09 2012 +0000 Initialize pthreads stuff before we use it. Spotted by: OSX pthread implementation diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 21a8a31..336c76a 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -418,6 +418,8 @@ pool_mkpool(unsigned pool_no) VTAILQ_INIT(&pp->back_queue); pp->sesspool = SES_NewPool(pp, pool_no); AN(pp->sesspool); + AZ(pthread_cond_init(&pp->herder_cond, NULL)); + AZ(pthread_create(&pp->herder_thr, NULL, pool_herder, pp)); VTAILQ_FOREACH(ls, &heritage.socks, list) { if (ls->sock < 0) @@ -430,9 +432,6 @@ pool_mkpool(unsigned pool_no) AZ(Pool_Task(pp, &ps->task, POOL_QUEUE_BACK)); } - AZ(pthread_cond_init(&pp->herder_cond, NULL)); - AZ(pthread_create(&pp->herder_thr, NULL, pool_herder, pp)); - return (pp); } From phk at FreeBSD.org Thu Dec 18 09:27:54 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 86619ad A couple of cleanups to get OSX to be more happy. Message-ID: commit 86619addc250bede7f6be2f635fe0cf1deb067ea Author: Poul-Henning Kamp Date: Wed Aug 22 09:31:21 2012 +0000 A couple of cleanups to get OSX to be more happy. diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h index a755b89..c817faf 100644 --- a/bin/varnishd/storage/storage_persistent.h +++ b/bin/varnishd/storage/storage_persistent.h @@ -34,7 +34,7 @@ */ #define ASSERT_SILO_THREAD(sc) \ - do {assert(pthread_self() == (sc)->thread);} while (0) + do {assert(pthread_equal(pthread_self(), (sc)->thread));} while (0) #define OC_F_NEEDFIXUP OC_F_PRIV diff --git a/bin/varnishtest/tests/r01035.vtc b/bin/varnishtest/tests/r01035.vtc index ccd6078..4bcb735 100644 --- a/bin/varnishtest/tests/r01035.vtc +++ b/bin/varnishtest/tests/r01035.vtc @@ -1,6 +1,6 @@ varnishtest "Test case for #1035" -varnish v1 -arg "-a 127.0.0.1:80 -b localhost:8080" +varnish v1 -arg "-a 127.0.0.1:80 -b 127.0.0.1:8080" varnish v1 -cliok "param.set listen_address 127.0.0.1:80" varnish v1 -clierr 106 "param.set listen_address 127.0.0.1:65540" varnish v1 -clierr 106 "param.set listen_address 127.0.0.1:65536" From phk at FreeBSD.org Thu Dec 18 09:27:54 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 4711b4b Fix a race condition, the child thread could get to the first assert before the parent carried out the assignment of the thread id to the variable. Message-ID: commit 4711b4b3187b7088c841aedaf56797997ffef6fe Author: Poul-Henning Kamp Date: Wed Aug 22 10:01:19 2012 +0000 Fix a race condition, the child thread could get to the first assert before the parent carried out the assignment of the thread id to the variable. Detected by: OSX diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index 6118fbf..b4e8c8c 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -275,6 +275,7 @@ smp_thread(struct worker *wrk, void *priv) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CAST_OBJ_NOTNULL(sc, priv, SMP_SC_MAGIC); + sc->thread = pthread_self(); /* First, load all the objects from all segments */ VTAILQ_FOREACH(sg, &sc->segments, list) @@ -305,6 +306,7 @@ static void smp_open(const struct stevedore *st) { struct smp_sc *sc; + pthread_t pt; ASSERT_CLI(); @@ -346,7 +348,7 @@ smp_open(const struct stevedore *st) smp_new_seg(sc); /* Start the worker silo worker thread, it will load the objects */ - WRK_BgThread(&sc->thread, "persistence", smp_thread, sc); + WRK_BgThread(&pt, "persistence", smp_thread, sc); VTAILQ_INSERT_TAIL(&silos, sc, list); Lck_Unlock(&sc->mtx); From phk at FreeBSD.org Thu Dec 18 09:27:54 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] bbdf093 Use expect_close after we munged the gzip stream, varnishd isn't going to like it... Message-ID: commit bbdf093f2a637cc7ef83ad4336849c363cebe501 Author: Poul-Henning Kamp Date: Wed Aug 22 12:07:55 2012 +0000 Use expect_close after we munged the gzip stream, varnishd isn't going to like it... Patch by: daghf diff --git a/bin/varnishtest/tests/r01184.vtc b/bin/varnishtest/tests/r01184.vtc index 9a74f28..3977eb1 100644 --- a/bin/varnishtest/tests/r01184.vtc +++ b/bin/varnishtest/tests/r01184.vtc @@ -60,16 +60,7 @@ server s1 { sendhex " 10 45 f3 a9 83 b8 18 1c 7b c2 30 55 04 17 13 c4" sendhex " 0f 07 5f 7a 38 f4 8e 50 b3 37 d4 3a 32 4a 34 07" sendhex " FF FF FF FF FF FF FF FF 72 ea 06 5f b3 1c fa dd" - sendhex " 2b b9 88 20 99 e6 9a 3c 84 7c 85 8e 50 e0 59 2a" - sendhex " 42 b0 8a 34 0f 96 d5 1e f7 97 fb b7 7e fd 4e 87" - sendhex " c7 8f be 9e ce fb 74 3a 3f 51 89 a3 9b 7e b2 43" - sendhex " a4 86 a2 55 90 b9 29 4c 4b 83 b8 99 5f b5 bb 27" - sendhex " 6a d4 86 18 22 83 8a 26 f4 11 1a 5c eb 34 3b ca" - sendhex " 20 31 9e 12 29 ff a8 92 78 a2 e6 ec 61 55 12 fc" - sendhex " 68 84 6c 12 41 b9 cf 2f 30 3b f0 10 5e d6 b7 eb" - sendhex " e7 76 bb 2c 7f 8c 90 4a 14 4c 01 00 00" - send "\r\n" - chunkedlen 0 + expect_close } -start varnish v1 -vcl+backend { @@ -100,16 +91,7 @@ server s1 { sendhex " 10 45 f3 a9 83 b8 18 1c 7b c2 30 55 04 17 13 c4" sendhex " 0f 07 5f 7a 38 f4 8e 50 b3 37 d4 3a 32 4a 34 07" sendhex " FF FF FF FF FF FF FF FF 72 ea 06 5f b3 1c fa dd" - sendhex " 2b b9 88 20 99 e6 9a 3c 84 7c 85 8e 50 e0 59 2a" - sendhex " 42 b0 8a 34 0f 96 d5 1e f7 97 fb b7 7e fd 4e 87" - sendhex " c7 8f be 9e ce fb 74 3a 3f 51 89 a3 9b 7e b2 43" - sendhex " a4 86 a2 55 90 b9 29 4c 4b 83 b8 99 5f b5 bb 27" - sendhex " 6a d4 86 18 22 83 8a 26 f4 11 1a 5c eb 34 3b ca" - sendhex " 20 31 9e 12 29 ff a8 92 78 a2 e6 ec 61 55 12 fc" - sendhex " 68 84 6c 12 41 b9 cf 2f 30 3b f0 10 5e d6 b7 eb" - sendhex " e7 76 bb 2c 7f 8c 90 4a 14 4c 01 00 00" - send "\r\n" - chunkedlen 0 + expect_close } -start varnish v1 -vcl+backend { From phk at FreeBSD.org Thu Dec 18 09:27:54 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 4f9c5b9 Trim chunk headers to force the bogus data into varnishd's processing Message-ID: commit 4f9c5b93d61d82dc48f71c9d7373c4d8eb5cc53b Author: Poul-Henning Kamp Date: Wed Aug 22 13:30:56 2012 +0000 Trim chunk headers to force the bogus data into varnishd's processing diff --git a/bin/varnishtest/tests/r01184.vtc b/bin/varnishtest/tests/r01184.vtc index 3977eb1..046b371 100644 --- a/bin/varnishtest/tests/r01184.vtc +++ b/bin/varnishtest/tests/r01184.vtc @@ -52,7 +52,7 @@ server s1 { txresp -nolen \ -hdr "Content-Encoding: gzip" \ -hdr "Transfer-Encoding: Chunked" - send "ed\r\n" + send "70\r\n" sendhex " 1f 8b 08 00 c2 39 33 50 02 03 45 90 4d 6a 03 31" sendhex " 0c 85 f7 73 8a 47 2e 10 d9 f2 9f ca 34 d0 c2 64" sendhex " 5b 08 bd 80 2d cb 10 28 5d 34 f4 fe f5 30 d0 68" @@ -83,7 +83,7 @@ server s1 { txresp -nolen \ -hdr "Content-Encoding: gzip" \ -hdr "Transfer-Encoding: Chunked" - send "ed\r\n" + send "70\r\n" sendhex " 1f 8b 08 00 c2 39 33 50 02 03 45 90 4d 6a 03 31" sendhex " 0c 85 f7 73 8a 47 2e 10 d9 f2 9f ca 34 d0 c2 64" sendhex " 5b 08 bd 80 2d cb 10 28 5d 34 f4 fe f5 30 d0 68" From perbu at varnish-software.com Thu Dec 18 09:27:54 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 7ad9ae5 Fix the CLI ref documentation (including a bug in the built in docs). Message-ID: commit 7ad9ae5cf540534f629b2c2eb14bf5eaabdb388b Author: Per Buer Date: Thu Aug 23 10:19:01 2012 +0200 Fix the CLI ref documentation (including a bug in the built in docs). diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c index 45b85d0..3f15ada 100644 --- a/bin/varnishd/cache/cache_backend_cfg.c +++ b/bin/varnishd/cache/cache_backend_cfg.c @@ -489,7 +489,7 @@ static struct cli_proto backend_cmds[] = { { "backend.list", "backend.list", "\tList all backends\n", 0, 1, "", cli_backend_list }, { "backend.set_health", "backend.set_health matcher state", - "\tShow a backend\n", 2, 2, "", cli_backend_set_health }, + "\tSet health status on a backend\n", 2, 2, "", cli_backend_set_health }, { NULL } }; diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index ad9cc9c..033be5a 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -76,108 +76,121 @@ be entered with the \\xnn syntax. Commands -------- -help [command] - Display a list of available commands. +.. glossary:: + :sorted: + help [command] + Display a list of available commands. If the command is specified, display help for this command. -param.set param value + param.set param value Set the parameter specified by param to the specified value. See Run-Time Parameters for a list of parame? ters. -param.show [-l] [param] + param.show [-l] [param] Display a list if run-time parameters and their values. - + If the -l option is specified, the list includes a brief explanation of each parameter. - + If a param is specified, display only the value and explanation for this parameter. -ping [timestamp] + ping [timestamp] Ping the Varnish cache process, keeping the connection alive. -ban *field operator argument* [&& field operator argument [...]] + ban *field operator argument* [&& field operator argument [...]] Immediately invalidate all documents matching the ban expression. See *Ban Expressions* for more documentation and examples. -ban.list + ban.list All requests for objects from the cache are matched against items on the ban list. If an object in the cache is older than a matching ban list item, it is considered "banned", and will be fetched from the backend instead. - + When a ban expression is older than all the objects in the cache, it is removed from the list. - + ban.list displays the ban list. The output looks something like this (broken into two lines): - + 0x7fea4fcb0580 1303835108.618863 131G req.http.host ~ www.myhost.com && req.url ~ /some/url - + The first field is the address of the ban. - + The second is the time of entry into the list, given as a high precision timestamp. - + The third field describes many objects point to this ban. When an object is compared to a ban the object is marked with a reference to the newest ban it was tested against. This isn't really useful unless you're debugging. - + A "G" marks that the ban is "Gone". Meaning it has been marked as a duplicate or it is no longer valid. It stays in the list for effiency reasons. - + Then follows the actual ban it self. -ban.url regexp + ban.url regexp Immediately invalidate all documents whose URL matches the specified regular expression. Please note that the Host part of the URL is ignored, so if you have several virtual hosts all of them will be banned. Use *ban* to specify a complete ban if you need to narrow it down. -quit + quit Close the connection to the varnish admin port. -start + start Start the Varnish cache process if it is not already running. -status + status Check the status of the Varnish cache process. -stop + stop Stop the Varnish cache process. -vcl.discard configname + vcl.discard configname Discard the configuration specified by configname. This will have no effect if the specified configuration has a non-zero reference count. -vcl.inline configname vcl + vcl.inline configname vcl Create a new configuration named configname with the VCL code specified by vcl, which must be a quoted string. -vcl.list + vcl.list List available configurations and their respective reference counts. The active configuration is indicated with an asterisk ("*"). -vcl.load configname filename + vcl.load configname filename Create a new configuration named configname with the contents of the specified file. -vcl.show configname + vcl.show configname Display the source code for the specified configuration. -vcl.use configname + vcl.use configname Start using the configuration specified by configname for all new requests. Existing requests will con? tinue using whichever configuration was in use when they arrived. + storage.list + Lists the defined storage backends. + + backend.list + Lists the defined backends including health state. + + backend.set_health matcher state + Sets the health state on a specific backend. This is useful if + you want to take a certain backend out of sirculations. + + Ban Expressions --------------- From tfheen at varnish-software.com Thu Dec 18 09:27:54 2014 From: tfheen at varnish-software.com (Tollef Fog Heen) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] e60db5b Pass -Wc, -fstack-protector to the libtool ld stage, to make the stack protector work on omnios Message-ID: commit e60db5be39c24347edb54de6df4130c4ee1f4c38 Author: Tollef Fog Heen Date: Thu Aug 23 11:40:33 2012 +0200 Pass -Wc,-fstack-protector to the libtool ld stage, to make the stack protector work on omnios diff --git a/configure.ac b/configure.ac index 13c38d1..e831b31 100644 --- a/configure.ac +++ b/configure.ac @@ -480,12 +480,16 @@ AC_ARG_ENABLE(stack-protector, if test "x$enable_stack_protector" != "xno"; then save_CFLAGS="$CFLAGS" CFLAGS="${CFLAGS} -fstack-protector-all" + save_AM_LT_LDFLAGS="$AM_LT_LDFLAGS" + AM_LT_LDFLAGS="${AM_LT_LDFLAGS} -Wc,-fstack-protector" AC_COMPILE_IFELSE( [AC_LANG_PROGRAM([],[],[])], [], [AC_MSG_WARN([-fstack-protector not supported, disabling]) - CFLAGS="$save_CFLAGS"]) + CFLAGS="$save_CFLAGS"] + AM_LT_LDFLAGS="$save_AM_LT_LDFLAGS") fi +AC_SUBST(AM_LT_LDFLAGS) # --enable-tests AC_ARG_ENABLE(tests, diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index ab2cc94..740ae16 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -1,8 +1,10 @@ INCLUDES = -I$(top_srcdir)/include @PCRE_CFLAGS@ +AM_LDFLAGS = $(AM_LT_LDFLAGS) + pkglib_LTLIBRARIES = libvarnish.la -libvarnish_la_LDFLAGS = -avoid-version +libvarnish_la_LDFLAGS = $(AM_LDFLAGS) -avoid-version libvarnish_la_SOURCES = \ vav.c \ diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index 492b10d..ee86abc 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -1,10 +1,12 @@ # +AM_LDFLAGS = $(AM_LT_LDFLAGS) + INCLUDES = -I$(top_srcdir)/include @PCRE_CFLAGS@ lib_LTLIBRARIES = libvarnishapi.la -libvarnishapi_la_LDFLAGS = -version-info 1:0:0 +libvarnishapi_la_LDFLAGS = $(AM_LDFLAGS) -version-info 1:0:0 libvarnishapi_la_SOURCES = \ vsm_api.h \ diff --git a/lib/libvarnishcompat/Makefile.am b/lib/libvarnishcompat/Makefile.am index 22c652e..33aac4a 100644 --- a/lib/libvarnishcompat/Makefile.am +++ b/lib/libvarnishcompat/Makefile.am @@ -1,10 +1,11 @@ # INCLUDES = -I$(top_srcdir)/include +AM_LDFLAGS = $(AM_LT_LDFLAGS) pkglib_LTLIBRARIES = libvarnishcompat.la -libvarnishcompat_la_LDFLAGS = -avoid-version +libvarnishcompat_la_LDFLAGS = $(AM_LDFLAGS) -avoid-version libvarnishcompat_la_SOURCES = \ daemon.c \ diff --git a/lib/libvcl/Makefile.am b/lib/libvcl/Makefile.am index 02a685f..f07facd 100644 --- a/lib/libvcl/Makefile.am +++ b/lib/libvcl/Makefile.am @@ -1,10 +1,12 @@ # +AM_LDFLAGS = $(AM_LT_LDFLAGS) + INCLUDES = -I$(top_srcdir)/include -I$(top_builddir)/include pkglib_LTLIBRARIES = libvcl.la -libvcl_la_LDFLAGS = -avoid-version +libvcl_la_LDFLAGS = $(AM_LDFLAGS) -avoid-version libvcl_la_SOURCES = \ vcc_compile.h \ diff --git a/lib/libvgz/Makefile.am b/lib/libvgz/Makefile.am index c04242c..7497aab 100644 --- a/lib/libvgz/Makefile.am +++ b/lib/libvgz/Makefile.am @@ -1,8 +1,9 @@ # +AM_LDFLAGS = $(AM_LT_LDFLAGS) pkglib_LTLIBRARIES = libvgz.la -libvgz_la_LDFLAGS = -avoid-version +libvgz_la_LDFLAGS = $(AM_LDFLAGS) -avoid-version libvgz_la_CFLAGS = -D_LARGEFILE64_SOURCE=1 $(libvgz_extra_cflags) libvgz_la_SOURCES = \ diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am index 23a6d53..1705cf9 100644 --- a/lib/libvmod_debug/Makefile.am +++ b/lib/libvmod_debug/Makefile.am @@ -1,4 +1,5 @@ # +AM_LDFLAGS = $(AM_LT_LDFLAGS) INCLUDES = \ -I$(top_srcdir)/include \ @@ -10,7 +11,7 @@ vmod_srcdir = $(top_srcdir)/lib/libvmod_debug vmodtool = $(top_srcdir)/lib/libvcl/vmodtool.py noinst_LTLIBRARIES = libvmod_debug.la -libvmod_debug_la_LDFLAGS = -module -export-dynamic -avoid-version -rpath /nowhere +libvmod_debug_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -rpath /nowhere libvmod_debug_la_SOURCES = \ vcc_if.c \ diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am index a872837..0a4736a 100644 --- a/lib/libvmod_std/Makefile.am +++ b/lib/libvmod_std/Makefile.am @@ -1,5 +1,7 @@ # +AM_LDFLAGS = $(AM_LT_LDFLAGS) + INCLUDES = \ -I$(top_srcdir)/include \ -I$(top_srcdir)/bin/varnishd \ @@ -12,7 +14,7 @@ vmod_srcdir = $(top_srcdir)/lib/libvmod_std vmodtool = $(top_srcdir)/lib/libvcl/vmodtool.py vmod_LTLIBRARIES = libvmod_std.la -libvmod_std_la_LDFLAGS = -module -export-dynamic -avoid-version +libvmod_std_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version libvmod_std_la_SOURCES = \ vcc_if.c \ From perbu at varnish-software.com Thu Dec 18 09:27:54 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 9d052cb Clarify storage backend usage Some more text describing the behaviour on the various storage backend + some reformatting for readability Message-ID: commit 9d052cb74759aa89d4358fe58632b4c0307137dc Author: Per Buer Date: Mon Aug 27 10:41:19 2012 +0200 Clarify storage backend usage Some more text describing the behaviour on the various storage backend + some reformatting for readability diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 17e6c03..e6857a8 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -156,10 +156,16 @@ Storage Types The following storage types are available: malloc[,size] - Storage for each object is allocated with malloc(3). - The size parameter specifies the maximum amount of memory varnishd will allocate. The size is assumed to - be in bytes, unless followed by one of the following suffixes: + Malloc is a memory based backend. Each object will be allocated + from memory. If your system runs low on memory swap will be + used. Be aware that the size limitation only limits the actual + storage and that approximately 1k of memory per object will be + used for various internal structures. + + The size parameter specifies the maximum amount of memory + varnishd will allocate. The size is assumed to be in bytes, + unless followed by one of the following suffixes: K, k The size is expressed in kibibytes. @@ -169,16 +175,24 @@ malloc[,size] T, t The size is expressed in tebibytes. - The default size is unlimited. + The default size is unlimited. + + Mallocs performance is bound by memory speed so it is very fast. file[,path[,size[,granularity]]] - Storage for each object is allocated from an arena backed by a file. This is the default. - The path parameter specifies either the path to the backing file or the path to a directory in which - varnishd will create the backing file. The default is /tmp. + The file backend stores objects in memory backed by a file on + disk with mmap. This is the default storage backend and unless + you specify another storage this one will used along with + Transient storage. - The size parameter specifies the size of the backing file. The size is assumed to be in bytes, unless fol? - lowed by one of the following suffixes: + The path parameter specifies either the path to the backing file + or the path to a directory in which varnishd will create the + backing file. The default is /tmp. + + The size parameter specifies the size of the backing file. The + size is assumed to be in bytes, unless fol? lowed by one of the + following suffixes: K, k The size is expressed in kibibytes. @@ -188,21 +202,29 @@ file[,path[,size[,granularity]]] T, t The size is expressed in tebibytes. - % The size is expressed as a percentage of the free space on the file system where it resides. + % The size is expressed as a percentage of the free space on the + file system where it resides. The default size is 50%. - If the backing file already exists, it will be truncated or expanded to the specified size. + If the backing file already exists, it will be truncated or + expanded to the specified size. + + Note that if varnishd has to create or expand the file, it will + not pre-allocate the added space, leading to fragmentation, + which may adversely impact performance. Pre-creating the + storage file using dd(1) will reduce fragmentation to a minimum. - Note that if varnishd has to create or expand the file, it will not pre-allocate the added space, leading - to fragmentation, which may adversely impact performance. Pre-creating the storage file using dd(1) will - reduce fragmentation to a minimum. + The granularity parameter specifies the granularity of + allocation. All allocations are rounded up to this size. The + size is assumed to be in bytes, unless followed by one of the + suffixes described for size except for %. - The granularity parameter specifies the granularity of allocation. All allocations are rounded up to this - size. The size is assumed to be in bytes, unless followed by one of the suffixes described for size except - for %. + The default size is the VM page size. The size should be + reduced if you have many small objects. - The default size is the VM page size. The size should be reduced if you have many small objects. + File performance is typically limited by the write speed of the + device, and depending on use, the seek time. persistent,path,size {experimental} @@ -241,38 +263,47 @@ Transient Storage Management Interface -------------------- -If the -T option was specified, varnishd will offer a command-line management interface on the specified address -and port. The recommended way of connecting to the command-line management interface is through varnishadm(1). +If the -T option was specified, varnishd will offer a command-line +management interface on the specified address and port. The +recommended way of connecting to the command-line management interface +is through varnishadm(1). The commands available are documented in varnish(7). Run-Time Parameters ------------------- -Runtime parameters are marked with shorthand flags to avoid repeating the same text over and over in the table -below. The meaning of the flags are: +Runtime parameters are marked with shorthand flags to avoid repeating +the same text over and over in the table below. The meaning of the +flags are: experimental - We have no solid information about good/bad/optimal values for this parameter. Feedback with experience - and observations are most welcome. + We have no solid information about good/bad/optimal values for + this parameter. Feedback with experience and observations are + most welcome. delayed - This parameter can be changed on the fly, but will not take effect immediately. + This parameter can be changed on the fly, but will not take + effect immediately. restart - The worker process must be stopped and restarted, before this parameter takes effect. + The worker process must be stopped and restarted, before this + parameter takes effect. reload The VCL programs must be reloaded for this parameter to take effect. -Here is a list of all parameters, current as of last time we remembered to update the manual page. This text is -produced from the same text you will find in the CLI if you use the param.show command, so should there be a new -parameter which is not listed here, you can find the description using the CLI commands. +Here is a list of all parameters, current as of last time we +remembered to update the manual page. This text is produced from the +same text you will find in the CLI if you use the param.show command, +so should there be a new parameter which is not listed here, you can +find the description using the CLI commands. Be aware that on 32 bit systems, certain default values, such as -workspace_client (=16k), thread_pool_workspace (=16k), http_resp_size (=8k), -http_req_size (=12k), gzip_stack_buffer (=4k) and thread_pool_stack (=64k) are -reduced relative to the values listed here, in order to conserve VM space. +workspace_client (=16k), thread_pool_workspace (=16k), http_resp_size +(=8k), http_req_size (=12k), gzip_stack_buffer (=4k) and +thread_pool_stack (=64k) are reduced relative to the values listed +here, in order to conserve VM space. .. include:: params.rst From perbu at varnish-software.com Thu Dec 18 09:27:54 2014 From: perbu at varnish-software.com (Per Buer) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 1ecffcd remove indent warning Message-ID: commit 1ecffcdc5d91c000faf5ae6e4c9bf38d2b27847c Author: Per Buer Date: Mon Aug 27 10:47:43 2012 +0200 remove indent warning diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index 033be5a..a4ec1b9 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -77,12 +77,12 @@ Commands -------- .. glossary:: - :sorted: + :sorted: help [command] Display a list of available commands. If the command is specified, display help for this command. - + param.set param value Set the parameter specified by param to the specified value. See Run-Time Parameters for a list of parame? ters. From phk at FreeBSD.org Thu Dec 18 09:27:54 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 71a1806 Add a "vsl_mask" parameter, that allows individual SLT tags to be suppressed so the never take up space/bandwidth in the VSL. Message-ID: commit 71a18061e026ecde6b319e798b1e61cf887f0829 Author: Poul-Henning Kamp Date: Mon Aug 27 10:30:46 2012 +0000 Add a "vsl_mask" parameter, that allows individual SLT tags to be suppressed so the never take up space/bandwidth in the VSL. Reimplement vcl_trace using this: param.set vsl_mask +VCL_trace param.set vsl_mask -VCL_trace XXX: doc-update needed diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 967d3bb..78f253c 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -61,6 +61,7 @@ varnishd_SOURCES = \ mgt/mgt_cli.c \ mgt/mgt_main.c \ mgt/mgt_param.c \ + mgt/mgt_param_vsl.c \ mgt/mgt_pool.c \ mgt/mgt_sandbox.c \ mgt/mgt_sandbox_solaris.c \ diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index f5cc879..167c25e 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -50,6 +50,18 @@ static uint32_t *vsl_ptr; struct VSC_C_main *VSC_C_main; +static inline int +vsl_tag_is_masked(enum VSL_tag_e tag) +{ + volatile uint8_t *bm = &cache_param->vsl_mask[0]; + uint8_t b; + + assert(tag < SLT_Reserved); + bm += ((unsigned)tag >> 3); + b = (0x80 >> ((unsigned)tag & 7)); + return (*bm & b); +} + static inline uint32_t vsl_w0(uint32_t type, uint32_t length) { @@ -159,6 +171,8 @@ VSL(enum VSL_tag_e tag, uint32_t vxid, const char *fmt, ...) unsigned n, mlen = cache_param->shm_reclen; char buf[mlen]; + if (vsl_tag_is_masked(tag)) + return; AN(fmt); va_start(ap, fmt); @@ -281,6 +295,8 @@ VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) va_list ap; AN(fmt); + if (vsl_tag_is_masked(tag)) + return; va_start(ap, fmt); wsl(vsl, tag, vsl->wid, fmt, ap); va_end(ap); @@ -293,6 +309,9 @@ VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) void VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t) { + + if (vsl_tag_is_masked(tag)) + return; wslr(vsl, tag, -1, t); } diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 8cc2920..477d337 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -75,9 +75,8 @@ VRT_count(struct req *req, unsigned u) if (req == NULL) return; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - if (cache_param->vcl_trace) - VSLb(req->vsl, SLT_VCL_trace, "%u %u.%u", u, - req->vcl->ref[u].line, req->vcl->ref[u].pos); + VSLb(req->vsl, SLT_VCL_trace, "%u %u.%u", u, + req->vcl->ref[u].line, req->vcl->ref[u].pos); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index a66bcf8..79de607 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -103,10 +103,6 @@ struct params { ssize_t fetch_maxchunksize; unsigned nuke_limit; - - /* VCL traces */ - unsigned vcl_trace; - unsigned accept_filter; /* Listen address */ @@ -204,4 +200,6 @@ struct params { struct poolparam req_pool; struct poolparam sess_pool; struct poolparam vbo_pool; + + uint8_t vsl_mask[256/8]; }; diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 2f090dc..026207c 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -872,14 +872,6 @@ static const struct parspec input_parspec[] = { "fragmentation.\n", EXPERIMENTAL, "256m", "bytes" }, - { "vcl_trace", tweak_bool, &mgt_param.vcl_trace, 0, 0, - "Trace VCL execution in the shmlog.\n" - "Enabling this will allow you to see the path each " - "request has taken through the VCL program.\n" - "This generates a lot of logrecords so it is off by " - "default.", - 0, - "off", "bool" }, { "accept_filter", tweak_bool, &mgt_param.accept_filter, 0, 0, "Enable kernel accept-filters, if supported by the kernel.", MUST_RESTART, @@ -1318,7 +1310,7 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv) continue; } pp->func(cli, pp, NULL); - if (pp->units != NULL) + if (pp->units != NULL && *pp->units != '\0') VCLI_Out(cli, " [%s]\n", pp->units); else VCLI_Out(cli, "\n"); @@ -1496,6 +1488,7 @@ MCF_ParamInit(struct cli *cli) MCF_AddParams(input_parspec); MCF_AddParams(WRK_parspec); + MCF_AddParams(VSL_parspec); /* XXX: We do this twice, to get past any interdependencies */ MCF_SetDefaults(NULL); diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h index ee1ad52..d25411d 100644 --- a/bin/varnishd/mgt/mgt_param.h +++ b/bin/varnishd/mgt/mgt_param.h @@ -57,5 +57,8 @@ void tweak_timeout_double(struct cli *cli, const struct parspec *par, const char *arg); void tweak_bytes(struct cli *cli, const struct parspec *par, const char *arg); +/* mgt_param_vsl.c */ +extern const struct parspec VSL_parspec[]; + /* mgt_pool.c */ extern const struct parspec WRK_parspec[]; diff --git a/bin/varnishd/mgt/mgt_param_vsl.c b/bin/varnishd/mgt/mgt_param_vsl.c new file mode 100644 index 0000000..6338d49 --- /dev/null +++ b/bin/varnishd/mgt/mgt_param_vsl.c @@ -0,0 +1,137 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2011 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include +#include + +#include "common/params.h" +#include "mgt/mgt.h" +#include "mgt/mgt_param.h" + +#include "vav.h" +#include "vcli.h" +#include "vcli_common.h" +#include "vcli_priv.h" + +#include "vapi/vsl_int.h" + +static const char * const VSL_tags[256] = { +# define SLTM(foo,sdesc,ldesc) [SLT_##foo] = #foo, +# include "tbl/vsl_tags.h" +# undef SLTM +}; + +enum bit_do {BSET, BCLR, BTST}; + +static int +vsl_bit(unsigned no, enum bit_do act) +{ + volatile uint8_t *bm = &mgt_param.vsl_mask[0]; + uint8_t b; + + assert(no < (unsigned)SLT_Reserved); + + bm += (no >> 3); + b = (0x80 >> (no & 7)); + if (act == BSET) + *bm |= b; + else if (act == BCLR) + *bm &= ~b; + return (*bm & b); +} + +static void +tweak_vsl_mask(struct cli *cli, const struct parspec *par, const char *arg) +{ + int i, n; + unsigned j; + const char *s; + char **av; + (void)par; + + if (arg != NULL) { + if (!strcmp(arg, "default")) { + (void)vsl_bit(SLT_VCL_trace, BSET); + (void)vsl_bit(SLT_WorkThread, BSET); + } else if (*arg != 0) { + av = VAV_Parse(arg, &n, ARGV_COMMA); + if (av[0] != NULL) { + VCLI_Out(cli, "Cannot parse: %s\n", av[0]); + VCLI_SetResult(cli, CLIS_PARAM); + return; + } + for (i = 1; av[i] != NULL; i++) { + s = av[i]; + if (*s != '-' && *s != '+') { + VCLI_Out(cli, + "Missing '+' or '-' (%s)\n", s); + VCLI_SetResult(cli, CLIS_PARAM); + VAV_Free(av); + return; + } + for (j = 0; j < 256; j++) + if (VSL_tags[j] != NULL && + !strcasecmp(s + 1, VSL_tags[j])) + break; + if (j == 256) { + VCLI_Out(cli, + "Unknown VSL tag (%s)\n", s); + VCLI_SetResult(cli, CLIS_PARAM); + VAV_Free(av); + return; + } + if (s[0] == '+') + (void)vsl_bit(j, BCLR); + else + (void)vsl_bit(j, BSET); + } + VAV_Free(av); + } + } else { + s = ""; + for (j = 0; j < 256; j++) { + if (vsl_bit(j, BTST)) { + VCLI_Out(cli, "%s-%s", s, VSL_tags[j]); + s = ","; + } + } + if (*s == '\0') + VCLI_Out(cli, "(all enabled)"); + } +} + +const struct parspec VSL_parspec[] = { + { "vsl_mask", tweak_vsl_mask, NULL, 0, 0, + "Mask individual VSL messages from being logged", + 0, "default", "" }, + { NULL, NULL, NULL } +}; diff --git a/bin/varnishtest/tests/c00005.vtc b/bin/varnishtest/tests/c00005.vtc index 7a7965c..846b283 100644 --- a/bin/varnishtest/tests/c00005.vtc +++ b/bin/varnishtest/tests/c00005.vtc @@ -9,7 +9,7 @@ server s1 { txresp -body "2222\n" } -start -varnish v1 -arg "-p vcl_trace=on" -vcl+backend { +varnish v1 -arg "-p vsl_mask=+VCL_trace" -vcl+backend { acl acl1 { "localhost"; } diff --git a/bin/varnishtest/tests/c00054.vtc b/bin/varnishtest/tests/c00054.vtc new file mode 100644 index 0000000..3299604 --- /dev/null +++ b/bin/varnishtest/tests/c00054.vtc @@ -0,0 +1,23 @@ +varnishtest "SLT masking" + + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend {} -start + +varnish v1 -cliok "param.show vsl_mask" +varnish v1 -cliok "param.set vsl_mask +VCL_trace" +varnish v1 -cliok "param.show vsl_mask" +varnish v1 -cliok "param.set vsl_mask -WorkThread,-TTL" +varnish v1 -cliok "param.show vsl_mask" +varnish v1 -cliok "param.set vsl_mask +WorkThread" +varnish v1 -clierr 106 "param.set vsl_mask FooBar" +varnish v1 -clierr 106 "param.set vsl_mask -FooBar" + +client c1 { + txreq + rxresp +} -run From phk at FreeBSD.org Thu Dec 18 09:27:54 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] cf4da22 Reimplement param log_hashstring using vsl_mask Message-ID: commit cf4da2286ef0f3321c5e0c12b7a4bee50eacc83c Author: Poul-Henning Kamp Date: Mon Aug 27 10:46:45 2012 +0000 Reimplement param log_hashstring using vsl_mask XXX: doc-update needed diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 0db0d6b..5e1b004 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -169,8 +169,7 @@ HSH_AddString(struct req *req, const char *str) SHA256_Update(req->sha256ctx, str, l); SHA256_Update(req->sha256ctx, "#", 1); - if (cache_param->log_hash) - VSLb(req->vsl, SLT_Hash, "%s", str); + VSLb(req->vsl, SLT_Hash, "%s", str); } /*--------------------------------------------------------------------- diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index 79de607..e2c9ec1 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -144,9 +144,6 @@ struct params { /* Control diagnostic code */ unsigned diag_bitmap; - /* Log hash string to shm */ - unsigned log_hash; - /* Log local socket address to shm */ unsigned log_local_addr; diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 026207c..9ce9470 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1033,10 +1033,6 @@ static const struct parspec input_parspec[] = { "more sessions take a detour around the waiter.", EXPERIMENTAL, "0.050", "seconds" }, - { "log_hashstring", tweak_bool, &mgt_param.log_hash, 0, 0, - "Log the hash string components to shared memory log.\n", - 0, - "on", "bool" }, { "log_local_address", tweak_bool, &mgt_param.log_local_addr, 0, 0, "Log the local address on the TCP connection in the " "SessionOpen VSL record.\n" diff --git a/bin/varnishd/mgt/mgt_param_vsl.c b/bin/varnishd/mgt/mgt_param_vsl.c index 6338d49..28ba7de 100644 --- a/bin/varnishd/mgt/mgt_param_vsl.c +++ b/bin/varnishd/mgt/mgt_param_vsl.c @@ -82,6 +82,7 @@ tweak_vsl_mask(struct cli *cli, const struct parspec *par, const char *arg) if (!strcmp(arg, "default")) { (void)vsl_bit(SLT_VCL_trace, BSET); (void)vsl_bit(SLT_WorkThread, BSET); + (void)vsl_bit(SLT_Hash, BSET); } else if (*arg != 0) { av = VAV_Parse(arg, &n, ARGV_COMMA); if (av[0] != NULL) { @@ -118,7 +119,7 @@ tweak_vsl_mask(struct cli *cli, const struct parspec *par, const char *arg) } } else { s = ""; - for (j = 0; j < 256; j++) { + for (j = 0; j < (unsigned)SLT_Reserved; j++) { if (vsl_bit(j, BTST)) { VCLI_Out(cli, "%s-%s", s, VSL_tags[j]); s = ","; From phk at FreeBSD.org Thu Dec 18 09:27:54 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 9232cd6 Generalize the param/bits code, we can do more with it that way. Message-ID: commit 9232cd638abebd9868c859c16dbbb4c64ae39391 Author: Poul-Henning Kamp Date: Mon Aug 27 16:46:01 2012 +0000 Generalize the param/bits code, we can do more with it that way. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 78f253c..3d35113 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -61,7 +61,7 @@ varnishd_SOURCES = \ mgt/mgt_cli.c \ mgt/mgt_main.c \ mgt/mgt_param.c \ - mgt/mgt_param_vsl.c \ + mgt/mgt_param_bits.c \ mgt/mgt_pool.c \ mgt/mgt_sandbox.c \ mgt/mgt_sandbox_solaris.c \ diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c new file mode 100644 index 0000000..1910f2b --- /dev/null +++ b/bin/varnishd/mgt/mgt_param_bits.c @@ -0,0 +1,168 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2011 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include +#include + +#include "common/params.h" +#include "mgt/mgt.h" +#include "mgt/mgt_param.h" + +#include "vav.h" +#include "vcli.h" +#include "vcli_common.h" +#include "vcli_priv.h" + +#include "vapi/vsl_int.h" + +/*-------------------------------------------------------------------- + */ + +enum bit_do {BSET, BCLR, BTST}; + +static int +bit(uint8_t *p, unsigned no, enum bit_do act) +{ + uint8_t b; + + p += (no >> 3); + b = (0x80 >> (no & 7)); + if (act == BSET) + *p |= b; + else if (act == BCLR) + *p &= ~b; + return (*p & b); +} + +/*-------------------------------------------------------------------- + */ + +static void +bit_tweak(struct cli *cli, uint8_t *p, unsigned l, const char *arg, + const char * const *tags, const char *desc, const char *sign) +{ + int i, n; + unsigned j; + char **av; + const char *s; + + av = VAV_Parse(arg, &n, ARGV_COMMA); + if (av[0] != NULL) { + VCLI_Out(cli, "Cannot parse: %s\n", av[0]); + VCLI_SetResult(cli, CLIS_PARAM); + VAV_Free(av); + return; + } + for (i = 1; av[i] != NULL; i++) { + s = av[i]; + if (*s != '-' && *s != '+') { + VCLI_Out(cli, "Missing '+' or '-' (%s)\n", s); + VCLI_SetResult(cli, CLIS_PARAM); + VAV_Free(av); + return; + } + for (j = 0; j < l; j++) { + if (tags[j] != NULL && !strcasecmp(s + 1, tags[j])) + break; + } + if (tags[j] == NULL) { + VCLI_Out(cli, "Unknown %s (%s)\n", desc, s); + VCLI_SetResult(cli, CLIS_PARAM); + VAV_Free(av); + return; + } + assert(j < l); + if (s[0] == *sign) + (void)bit(p, j, BSET); + else + (void)bit(p, j, BCLR); + } + VAV_Free(av); +} + + +/*-------------------------------------------------------------------- + * The vsl_mask parameter + */ + +static const char * const VSL_tags[256] = { +# define SLTM(foo,sdesc,ldesc) [SLT_##foo] = #foo, +# include "tbl/vsl_tags.h" +# undef SLTM + NULL +}; + +static void +tweak_vsl_mask(struct cli *cli, const struct parspec *par, const char *arg) +{ + unsigned j; + const char *s; + (void)par; + + if (arg != NULL) { + if (!strcmp(arg, "default")) { + memset(mgt_param.vsl_mask, + 0, sizeof mgt_param.vsl_mask); + (void)bit(mgt_param.vsl_mask, SLT_VCL_trace, BSET); + (void)bit(mgt_param.vsl_mask, SLT_WorkThread, BSET); + (void)bit(mgt_param.vsl_mask, SLT_Hash, BSET); + } else { + bit_tweak(cli, mgt_param.vsl_mask, + SLT_Reserved, arg, VSL_tags, + "VSL tag", "-"); + } + } else { + s = ""; + for (j = 0; j < (unsigned)SLT_Reserved; j++) { + if (bit(mgt_param.vsl_mask, j, BTST)) { + VCLI_Out(cli, "%s-%s", s, VSL_tags[j]); + s = ","; + } + } + if (*s == '\0') + VCLI_Out(cli, "(all enabled)"); + } +} + +/*-------------------------------------------------------------------- + * The parameter table itself + */ + +const struct parspec VSL_parspec[] = { + { "vsl_mask", tweak_vsl_mask, NULL, 0, 0, + "Mask individual VSL messages from being logged.\n" + "\tdefault\tSet default value\n" + "Use +/- prefixe in front of VSL tag name, to mask/unmask " + "individual VSL messages.", + 0, "default", "" }, + { NULL, NULL, NULL } +}; diff --git a/bin/varnishd/mgt/mgt_param_vsl.c b/bin/varnishd/mgt/mgt_param_vsl.c deleted file mode 100644 index 28ba7de..0000000 --- a/bin/varnishd/mgt/mgt_param_vsl.c +++ /dev/null @@ -1,138 +0,0 @@ -/*- - * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2011 Varnish Software AS - * All rights reserved. - * - * Author: Poul-Henning Kamp - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "config.h" - -#include -#include -#include - -#include "common/params.h" -#include "mgt/mgt.h" -#include "mgt/mgt_param.h" - -#include "vav.h" -#include "vcli.h" -#include "vcli_common.h" -#include "vcli_priv.h" - -#include "vapi/vsl_int.h" - -static const char * const VSL_tags[256] = { -# define SLTM(foo,sdesc,ldesc) [SLT_##foo] = #foo, -# include "tbl/vsl_tags.h" -# undef SLTM -}; - -enum bit_do {BSET, BCLR, BTST}; - -static int -vsl_bit(unsigned no, enum bit_do act) -{ - volatile uint8_t *bm = &mgt_param.vsl_mask[0]; - uint8_t b; - - assert(no < (unsigned)SLT_Reserved); - - bm += (no >> 3); - b = (0x80 >> (no & 7)); - if (act == BSET) - *bm |= b; - else if (act == BCLR) - *bm &= ~b; - return (*bm & b); -} - -static void -tweak_vsl_mask(struct cli *cli, const struct parspec *par, const char *arg) -{ - int i, n; - unsigned j; - const char *s; - char **av; - (void)par; - - if (arg != NULL) { - if (!strcmp(arg, "default")) { - (void)vsl_bit(SLT_VCL_trace, BSET); - (void)vsl_bit(SLT_WorkThread, BSET); - (void)vsl_bit(SLT_Hash, BSET); - } else if (*arg != 0) { - av = VAV_Parse(arg, &n, ARGV_COMMA); - if (av[0] != NULL) { - VCLI_Out(cli, "Cannot parse: %s\n", av[0]); - VCLI_SetResult(cli, CLIS_PARAM); - return; - } - for (i = 1; av[i] != NULL; i++) { - s = av[i]; - if (*s != '-' && *s != '+') { - VCLI_Out(cli, - "Missing '+' or '-' (%s)\n", s); - VCLI_SetResult(cli, CLIS_PARAM); - VAV_Free(av); - return; - } - for (j = 0; j < 256; j++) - if (VSL_tags[j] != NULL && - !strcasecmp(s + 1, VSL_tags[j])) - break; - if (j == 256) { - VCLI_Out(cli, - "Unknown VSL tag (%s)\n", s); - VCLI_SetResult(cli, CLIS_PARAM); - VAV_Free(av); - return; - } - if (s[0] == '+') - (void)vsl_bit(j, BCLR); - else - (void)vsl_bit(j, BSET); - } - VAV_Free(av); - } - } else { - s = ""; - for (j = 0; j < (unsigned)SLT_Reserved; j++) { - if (vsl_bit(j, BTST)) { - VCLI_Out(cli, "%s-%s", s, VSL_tags[j]); - s = ","; - } - } - if (*s == '\0') - VCLI_Out(cli, "(all enabled)"); - } -} - -const struct parspec VSL_parspec[] = { - { "vsl_mask", tweak_vsl_mask, NULL, 0, 0, - "Mask individual VSL messages from being logged", - 0, "default", "" }, - { NULL, NULL, NULL } -}; From phk at FreeBSD.org Thu Dec 18 09:27:54 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 73632b2 Add a debug parameter and start moving diag_bitmap stuff into it. Message-ID: commit 73632b2c74bfc12f15ec06d9e0ad0cea1acd0fe6 Author: Poul-Henning Kamp Date: Mon Aug 27 18:09:17 2012 +0000 Add a debug parameter and start moving diag_bitmap stuff into it. XXX: doc-update needed diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 43c104e..f3e32c3 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -976,12 +976,6 @@ void VSLbt(struct vsl_log *, enum VSL_tag_e tag, txt t); void VSL_Flush(struct vsl_log *, int overflow); -#define DSL(flag, tag, id, ...) \ - do { \ - if (cache_param->diag_bitmap & (flag)) \ - VSL((tag), (id), __VA_ARGS__); \ - } while (0) - #endif /* cache_response.c */ @@ -1122,3 +1116,16 @@ Tadd(txt *t, const char *p, int l) * extra timestamps in cache_pool.c. Hide this detail with a macro */ #define W_TIM_real(w) ((w)->lastused = VTIM_real()) + +static inline int +DO_DEBUG(enum debug_bits x) +{ + return (cache_param->debug_bits[(unsigned)x>>3] & + (0x80U >> ((unsigned)x & 7))); +} + +#define DSL(flag, tag, id, ...) \ + do { \ + if (cache_param->diag_bitmap & (flag)) \ + VSL((tag), (id), __VA_ARGS__); \ + } while (0) diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index e99ed6c..e12da55 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -1220,7 +1220,7 @@ CNT_Request(struct worker *wrk, struct req *req) switch (req->req_step) { #define REQ_STEP(l,u,arg) \ case R_STP_##u: \ - if (cache_param->diag_bitmap & 0x01) \ + if (DO_DEBUG(DBG_REQ_STATE)) \ cnt_diag(req, #u); \ done = cnt_##l arg; \ break; diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index e2c9ec1..6c3abb4 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -33,6 +33,13 @@ #define VSM_CLASS_PARAM "Params" +enum debug_bits { +#define DEBUG_BIT(U, l, p, d) DBG_##U, +#include "tbl/debug_bits.h" +#undef DEBUG_BIT + DBG_Reserved +}; + struct poolparam { unsigned min_pool; unsigned max_pool; @@ -198,5 +205,6 @@ struct params { struct poolparam sess_pool; struct poolparam vbo_pool; - uint8_t vsl_mask[256/8]; + uint8_t vsl_mask[256>>3]; + uint8_t debug_bits[(DBG_Reserved+7)>>3]; }; diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 9ce9470..7696292 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1046,7 +1046,6 @@ static const struct parspec input_parspec[] = { WAITER_DEFAULT, NULL }, { "diag_bitmap", tweak_diag_bitmap, 0, 0, 0, "Bitmap controlling diagnostics code:\n" - " 0x00000001 - CNT_Session states.\n" " 0x00000002 - workspace debugging.\n" " 0x00000004 - kqueue debugging.\n" " 0x00000008 - mutex logging.\n" diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c index 1910f2b..b238823 100644 --- a/bin/varnishd/mgt/mgt_param_bits.c +++ b/bin/varnishd/mgt/mgt_param_bits.c @@ -154,6 +154,45 @@ tweak_vsl_mask(struct cli *cli, const struct parspec *par, const char *arg) } /*-------------------------------------------------------------------- + * The debug parameter + */ + +static const char * const debug_tags[] = { +# define DEBUG_BIT(U,l,p,d) [DBG_##U] = #l, +# include "tbl/debug_bits.h" +# undef DEBUG_BIT + NULL +}; + +static void +tweak_debug(struct cli *cli, const struct parspec *par, const char *arg) +{ + const char *s; + unsigned j; + (void)par; + + if (arg != NULL) { + if (!strcmp(arg, "none")) { + memset(mgt_param.debug_bits, + 0, sizeof mgt_param.debug_bits); + } else { + bit_tweak(cli, mgt_param.debug_bits, + DBG_Reserved, arg, debug_tags, "debug bit", "+"); + } + } else { + s = ""; + for (j = 0; j < (unsigned)DBG_Reserved; j++) { + if (bit(mgt_param.debug_bits, j, BTST)) { + VCLI_Out(cli, "%s+%s", s, debug_tags[j]); + s = ","; + } + } + if (*s == '\0') + VCLI_Out(cli, "none"); + } +} + +/*-------------------------------------------------------------------- * The parameter table itself */ @@ -164,5 +203,13 @@ const struct parspec VSL_parspec[] = { "Use +/- prefixe in front of VSL tag name, to mask/unmask " "individual VSL messages.", 0, "default", "" }, + { "debug", tweak_debug, NULL, 0, 0, + "Enable/Disable various kinds of debugging.\n" + "\tnone\t\tDisable all debugging\n" + "Use +/- prefix to set/reset individual bits:\n" +#define DEBUG_BIT(U, l, p, d) "\t" #l "\t" p d "\n" +#include "tbl/debug_bits.h" +#undef DEBUG_BIT + , 0, "none", "" }, { NULL, NULL, NULL } }; diff --git a/bin/varnishtest/tests.disabled/t00000.vtc b/bin/varnishtest/tests.disabled/t00000.vtc index 57ab7c7..8265195 100644 --- a/bin/varnishtest/tests.disabled/t00000.vtc +++ b/bin/varnishtest/tests.disabled/t00000.vtc @@ -20,7 +20,7 @@ varnish v1 -vcl+backend { } } -start -varnish v1 -cliok "param.set diag_bitmap 1" +varnish v1 -cliok "param.set debug +req_state" client c1 { txreq -hdr "foo: /foo" diff --git a/bin/varnishtest/tests/r00902.vtc b/bin/varnishtest/tests/r00902.vtc index 67e4c96..8caf652 100644 --- a/bin/varnishtest/tests/r00902.vtc +++ b/bin/varnishtest/tests/r00902.vtc @@ -15,7 +15,7 @@ server s1 { varnish v1 -vcl+backend { } -start -varnish v1 -cliok "param.set diag_bitmap 1" +varnish v1 -cliok "param.set debug +req_state" client c1 { txreq -hdr "foo: /foo" diff --git a/include/Makefile.am b/include/Makefile.am index b70a825..7562f6c 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -6,6 +6,7 @@ nobase_pkginclude_HEADERS = \ tbl/backend_poll.h \ tbl/ban_vars.h \ tbl/body_status.h \ + tbl/debug_bits.h \ tbl/http_headers.h \ tbl/http_response.h \ tbl/locks.h \ diff --git a/include/tbl/debug_bits.h b/include/tbl/debug_bits.h new file mode 100644 index 0000000..7acf6d8 --- /dev/null +++ b/include/tbl/debug_bits.h @@ -0,0 +1,32 @@ +/*- + * Copyright (c) 2012 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Fields in the debug parameter + * + */ + +DEBUG_BIT(REQ_STATE, req_state, "", "Request state engine") From phk at FreeBSD.org Thu Dec 18 09:27:54 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:54 +0100 Subject: [experimental-ims] 4dbf7e7 Move 3 bits more from diag_bitmaps to debug Message-ID: commit 4dbf7e7f2a5721a76b9b46a10cb5638c4972d0a1 Author: Poul-Henning Kamp Date: Mon Aug 27 18:58:13 2012 +0000 Move 3 bits more from diag_bitmaps to debug XXX: doc-update needed diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index f3e32c3..30f8e99 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -1124,8 +1124,8 @@ DO_DEBUG(enum debug_bits x) (0x80U >> ((unsigned)x & 7))); } -#define DSL(flag, tag, id, ...) \ +#define DSL(debug_bit, id, ...) \ do { \ - if (cache_param->diag_bitmap & (flag)) \ - VSL((tag), (id), __VA_ARGS__); \ + if (DO_DEBUG(debug_bit)) \ + VSL(SLT_Debug, (id), __VA_ARGS__); \ } while (0) diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index e7fee9b..404409f 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -105,7 +105,7 @@ ved_include(struct req *preq, const char *src, const char *host) i = CNT_Request(wrk, req); if (i == 1) break; - DSL(0x20, SLT_Debug, req->vsl->wid, + DSL(DBG_WAITINGLIST, req->vsl->wid, "loop waiting for ESI (%d)", i); assert(i == 2); AZ(req->wrk); diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 5e1b004..fcf447a 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -420,7 +420,7 @@ HSH_Lookup(struct req *req) VTAILQ_INSERT_TAIL(&oh->waitinglist->list, req, w_list); } - if (cache_param->diag_bitmap & 0x20) + if (DO_DEBUG(DBG_WAITINGLIST)) VSLb(req->vsl, SLT_Debug, "on waiting list <%p>", oh); @@ -483,7 +483,7 @@ hsh_rush(struct dstat *ds, struct objhead *oh) ds->busy_wakeup++; AZ(req->wrk); VTAILQ_REMOVE(&wl->list, req, w_list); - DSL(0x20, SLT_Debug, req->vsl->wid, "off waiting list"); + DSL(DBG_WAITINGLIST, req->vsl->wid, "off waiting list"); if (SES_ScheduleReq(req)) { /* * We could not schedule the session, leave the diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index d84e937..a5251ff 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -37,7 +37,7 @@ WS_Assert(const struct ws *ws) { CHECK_OBJ_NOTNULL(ws, WS_MAGIC); - DSL(0x02, SLT_Debug, 0, "WS(%p = (%s, %p %u %u %u)", + DSL(DBG_WORKSPACE, 0, "WS(%p = (%s, %p %u %u %u)", ws, ws->id, ws->s, pdiff(ws->s, ws->f), ws->r == NULL ? 0 : pdiff(ws->f, ws->r), pdiff(ws->s, ws->e)); @@ -60,7 +60,7 @@ void WS_Init(struct ws *ws, const char *id, void *space, unsigned len) { - DSL(0x02, SLT_Debug, 0, + DSL(DBG_WORKSPACE, 0, "WS_Init(%p, \"%s\", %p, %u)", ws, id, space, len); assert(space != NULL); memset(ws, 0, sizeof *ws); @@ -83,7 +83,7 @@ WS_Reset(struct ws *ws, char *p) { WS_Assert(ws); - DSL(0x02, SLT_Debug, 0, "WS_Reset(%p, %p)", ws, p); + DSL(DBG_WORKSPACE, 0, "WS_Reset(%p, %p)", ws, p); assert(ws->r == NULL); if (p == NULL) ws->f = ws->s; @@ -111,7 +111,7 @@ WS_Alloc(struct ws *ws, unsigned bytes) } r = ws->f; ws->f += bytes; - DSL(0x02, SLT_Debug, 0, "WS_Alloc(%p, %u) = %p", ws, bytes, r); + DSL(DBG_WORKSPACE, 0, "WS_Alloc(%p, %u) = %p", ws, bytes, r); WS_Assert(ws); return (r); } @@ -122,7 +122,7 @@ WS_Snapshot(struct ws *ws) WS_Assert(ws); assert(ws->r == NULL); - DSL(0x02, SLT_Debug, 0, "WS_Snapshot(%p) = %p", ws, ws->f); + DSL(DBG_WORKSPACE, 0, "WS_Snapshot(%p) = %p", ws, ws->f); return (ws->f); } @@ -142,7 +142,7 @@ WS_Reserve(struct ws *ws, unsigned bytes) b2 = PRNDDN(b2); xxxassert(ws->f + b2 <= ws->e); ws->r = ws->f + b2; - DSL(0x02, SLT_Debug, 0, "WS_Reserve(%p, %u/%u) = %u", + DSL(DBG_WORKSPACE, 0, "WS_Reserve(%p, %u/%u) = %u", ws, b2, bytes, pdiff(ws->f, ws->r)); WS_Assert(ws); return (pdiff(ws->f, ws->r)); @@ -154,7 +154,7 @@ WS_Release(struct ws *ws, unsigned bytes) WS_Assert(ws); bytes = PRNDUP(bytes); assert(bytes <= ws->e - ws->f); - DSL(0x02, SLT_Debug, 0, "WS_Release(%p, %u)", ws, bytes); + DSL(DBG_WORKSPACE, 0, "WS_Release(%p, %u)", ws, bytes); assert(ws->r != NULL); assert(ws->f + bytes <= ws->r); ws->f += bytes; @@ -166,7 +166,7 @@ void WS_ReleaseP(struct ws *ws, char *ptr) { WS_Assert(ws); - DSL(0x02, SLT_Debug, 0, "WS_ReleaseP(%p, %p)", ws, ptr); + DSL(DBG_WORKSPACE, 0, "WS_ReleaseP(%p, %p)", ws, ptr); assert(ws->r != NULL); assert(ptr >= ws->f); assert(ptr <= ws->r); diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 7696292..dd084ed 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1046,11 +1046,8 @@ static const struct parspec input_parspec[] = { WAITER_DEFAULT, NULL }, { "diag_bitmap", tweak_diag_bitmap, 0, 0, 0, "Bitmap controlling diagnostics code:\n" - " 0x00000002 - workspace debugging.\n" - " 0x00000004 - kqueue debugging.\n" " 0x00000008 - mutex logging.\n" " 0x00000010 - mutex contests.\n" - " 0x00000020 - waiting list.\n" " 0x00000040 - object workspace.\n" " 0x00000080 - mutex timing.\n" " 0x00001000 - do not core-dump child process.\n" diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c index bc5c67f..8d37ad6 100644 --- a/bin/varnishd/waiter/cache_waiter_kqueue.c +++ b/bin/varnishd/waiter/cache_waiter_kqueue.c @@ -80,7 +80,7 @@ vwk_kq_sess(struct vwk *vwk, struct sess *sp, short arm) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); assert(sp->fd >= 0); - DSL(0x04, SLT_Debug, sp->vxid, "KQ: EV_SET sp %p arm %x", sp, arm); + DSL(DBG_WAITER, sp->vxid, "KQ: EV_SET sp %p arm %x", sp, arm); EV_SET(&vwk->ki[vwk->nki], sp->fd, EVFILT_READ, arm, 0, 0, sp); if (++vwk->nki == NKEV) vwk_kq_flush(vwk); @@ -121,7 +121,7 @@ vwk_sess_ev(struct vwk *vwk, const struct kevent *kp, double now) AN(kp->udata); assert(kp->udata != vwk->pipes); CAST_OBJ_NOTNULL(sp, kp->udata, SESS_MAGIC); - DSL(0x04, SLT_Debug, sp->vxid, "KQ: sp %p kev data %lu flags 0x%x%s", + DSL(DBG_WAITER, sp->vxid, "KQ: sp %p kev data %lu flags 0x%x%s", sp, (unsigned long)kp->data, kp->flags, (kp->flags & EV_EOF) ? " EOF" : ""); diff --git a/bin/varnishtest/tests/b00000.vtc b/bin/varnishtest/tests/b00000.vtc index 2018235..2e6d5ce 100644 --- a/bin/varnishtest/tests/b00000.vtc +++ b/bin/varnishtest/tests/b00000.vtc @@ -7,7 +7,7 @@ server s1 { varnish v1 -storage "-smalloc,1m" -vcl+backend {} -start -varnish v1 -cliok "param.set diag_bitmap 0x2" +varnish v1 -cliok "param.set debug +workspace" varnish v1 -expect n_object == 0 varnish v1 -expect sess_conn == 0 diff --git a/bin/varnishtest/tests/r00345.vtc b/bin/varnishtest/tests/r00345.vtc index 712aa98..6ebbc99 100644 --- a/bin/varnishtest/tests/r00345.vtc +++ b/bin/varnishtest/tests/r00345.vtc @@ -9,7 +9,7 @@ server s1 { txresp -body {DATA} } -start -varnish v1 -arg "-p diag_bitmap=0x20" -vcl+backend { +varnish v1 -arg "-p debug=+workspace" -vcl+backend { sub vcl_fetch { if (req.url == "/") { set beresp.do_esi = true; diff --git a/bin/varnishtest/tests/r00386.vtc b/bin/varnishtest/tests/r00386.vtc index f5f1907..c2fcc53 100644 --- a/bin/varnishtest/tests/r00386.vtc +++ b/bin/varnishtest/tests/r00386.vtc @@ -9,7 +9,7 @@ server s1 { txresp -body {} } -start -varnish v1 -arg "-p diag_bitmap=0x20" -vcl+backend { +varnish v1 -arg "-p debug=+workspace" -vcl+backend { sub vcl_fetch { if (req.url == "/") { set beresp.do_esi = true; diff --git a/include/tbl/debug_bits.h b/include/tbl/debug_bits.h index 7acf6d8..3300547 100644 --- a/include/tbl/debug_bits.h +++ b/include/tbl/debug_bits.h @@ -29,4 +29,7 @@ * */ -DEBUG_BIT(REQ_STATE, req_state, "", "Request state engine") +DEBUG_BIT(REQ_STATE, req_state, "", "VSL Request state engine") +DEBUG_BIT(WORKSPACE, workspace, "", "VSL Workspace operations") +DEBUG_BIT(WAITER, waiter, "", "VSL Waiter internals") +DEBUG_BIT(WAITINGLIST, waitinglist, "", "VSL Waitinglist events") From phk at FreeBSD.org Thu Dec 18 09:27:55 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:55 +0100 Subject: [experimental-ims] 609c67d Move another bit from diag_bitmap to debug Message-ID: commit 609c67d18aa90cebb8301cbafec592c66e110a38 Author: Poul-Henning Kamp Date: Mon Aug 27 19:21:23 2012 +0000 Move another bit from diag_bitmap to debug XXX: doc-update needed diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 167c25e..228dde8 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -240,7 +240,7 @@ wslr(struct vsl_log *vsl, enum VSL_tag_e tag, int id, txt t) vsl->wlp = VSL_END(vsl->wlp, l); assert(vsl->wlp < vsl->wle); vsl->wlr++; - if (cache_param->diag_bitmap & 0x10000) + if (DO_DEBUG(DBG_SYNCVSL)) VSL_Flush(vsl, 0); } @@ -281,7 +281,7 @@ wsl(struct vsl_log *vsl, enum VSL_tag_e tag, int id, const char *fmt, assert(vsl->wlp < vsl->wle); vsl->wlr++; } - if (cache_param->diag_bitmap & 0x10000) + if (DO_DEBUG(DBG_SYNCVSL)) VSL_Flush(vsl, 0); } diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index dd084ed..2c8ad86 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1053,7 +1053,6 @@ static const struct parspec input_parspec[] = { " 0x00001000 - do not core-dump child process.\n" " 0x00002000 - only short panic message.\n" " 0x00004000 - panic to stderr.\n" - " 0x00010000 - synchronize shmlog.\n" " 0x00020000 - synchronous start of persistence.\n" " 0x00040000 - release VCL early.\n" " 0x00080000 - ban-lurker debugging.\n" diff --git a/bin/varnishtest/tests/e00008.vtc b/bin/varnishtest/tests/e00008.vtc index 9c8770f..15b915a 100644 --- a/bin/varnishtest/tests/e00008.vtc +++ b/bin/varnishtest/tests/e00008.vtc @@ -63,7 +63,7 @@ varnish v1 -vcl+backend { varnish v1 -cliok "param.set esi_syntax 0x3e" -varnish v1 -cliok "param.set diag_bitmap 0x10000" +varnish v1 -cliok "param.set debug +syncvsl" client c1 { txreq diff --git a/bin/varnishtest/tests/e00019.vtc b/bin/varnishtest/tests/e00019.vtc index 2c1e82b..54c756f 100644 --- a/bin/varnishtest/tests/e00019.vtc +++ b/bin/varnishtest/tests/e00019.vtc @@ -38,7 +38,7 @@ varnish v1 -vcl+backend { varnish v1 -cliok "param.set esi_syntax 8" -varnish v1 -cliok "param.set diag_bitmap 0x10000" +varnish v1 -cliok "param.set debug +syncvsl" client c1 { txreq -url bar diff --git a/bin/varnishtest/tests/e00023.vtc b/bin/varnishtest/tests/e00023.vtc index 1373fc8..03d185b 100644 --- a/bin/varnishtest/tests/e00023.vtc +++ b/bin/varnishtest/tests/e00023.vtc @@ -44,7 +44,7 @@ varnish v1 -vcl+backend { varnish v1 -cliok "param.set http_gzip_support true" varnish v1 -cliok "param.set esi_syntax 0x3e" -varnish v1 -cliok "param.set diag_bitmap 0x10000" +varnish v1 -cliok "param.set debug +syncvsl" client c1 { txreq -hdr "Accept-Encoding: gzip" diff --git a/bin/varnishtest/tests/e00024.vtc b/bin/varnishtest/tests/e00024.vtc index f728b3c..87235c5 100644 --- a/bin/varnishtest/tests/e00024.vtc +++ b/bin/varnishtest/tests/e00024.vtc @@ -69,7 +69,7 @@ varnish v1 -vcl+backend { varnish v1 -cliok "param.set http_gzip_support true" varnish v1 -cliok "param.set esi_syntax 0x3e" -varnish v1 -cliok "param.set diag_bitmap 0x10000" +varnish v1 -cliok "param.set debug +syncvsl" client c1 { txreq -hdr "Accept-Encoding: gzip" diff --git a/bin/varnishtest/tests/e00026.vtc b/bin/varnishtest/tests/e00026.vtc index bcb05b0..fa6cf9a 100644 --- a/bin/varnishtest/tests/e00026.vtc +++ b/bin/varnishtest/tests/e00026.vtc @@ -37,7 +37,7 @@ varnish v1 -vcl+backend { varnish v1 -cliok "param.set esi_syntax 0x21" -varnish v1 -cliok "param.set diag_bitmap 0x10000" +varnish v1 -cliok "param.set debug +syncvsl" client c1 { txreq -url /foo -hdr "Accept-Encoding: gzip" diff --git a/bin/varnishtest/tests/g00004.vtc b/bin/varnishtest/tests/g00004.vtc index 0ab74ad..0a23c01 100644 --- a/bin/varnishtest/tests/g00004.vtc +++ b/bin/varnishtest/tests/g00004.vtc @@ -18,7 +18,6 @@ server s1 -repeat 2 { } -start varnish v1 \ - -arg {-p diag_bitmap=0x00010000} \ -vcl+backend { sub vcl_fetch { set beresp.do_stream = false; @@ -29,6 +28,8 @@ varnish v1 \ } } +varnish v1 -cliok "param.set debug +syncvsl" + varnish v1 -start client c1 { diff --git a/bin/varnishtest/tests/r00433.vtc b/bin/varnishtest/tests/r00433.vtc index f782f14..1bcefa8 100644 --- a/bin/varnishtest/tests/r00433.vtc +++ b/bin/varnishtest/tests/r00433.vtc @@ -37,7 +37,7 @@ varnish v1 -vcl+backend { } -start varnish v1 -cliok "param.set esi_syntax 4" -varnish v1 -cliok "param.set diag_bitmap 0x10000" +varnish v1 -cliok "param.set debug +syncvsl" varnish v1 -cliok "debug.fragfetch 32" client c1 { diff --git a/bin/varnishtest/tests/r00466.vtc b/bin/varnishtest/tests/r00466.vtc index 3d64b22..e509de8 100644 --- a/bin/varnishtest/tests/r00466.vtc +++ b/bin/varnishtest/tests/r00466.vtc @@ -23,7 +23,9 @@ varnish v1 -vcl+backend { return(pass); } } -} -start -cliok "param.set diag_bitmap 0x10000" +} -start + +varnish v1 -cliok "param.set debug +syncvsl" client c1 { txreq -url "/foo" -hdr "Range: 100-200" diff --git a/bin/varnishtest/tests/r00942.vtc b/bin/varnishtest/tests/r00942.vtc index 6bb31e1..1506709 100644 --- a/bin/varnishtest/tests/r00942.vtc +++ b/bin/varnishtest/tests/r00942.vtc @@ -25,13 +25,13 @@ server s1 { } -start varnish v1 \ - -arg {-p diag_bitmap=0x00010000} \ -vcl+backend { sub vcl_fetch { set beresp.do_stream = false; } } +varnish v1 -cliok "param.set debug +syncvsl" varnish v1 -start client c1 { diff --git a/include/tbl/debug_bits.h b/include/tbl/debug_bits.h index 3300547..ece0fc4 100644 --- a/include/tbl/debug_bits.h +++ b/include/tbl/debug_bits.h @@ -31,5 +31,6 @@ DEBUG_BIT(REQ_STATE, req_state, "", "VSL Request state engine") DEBUG_BIT(WORKSPACE, workspace, "", "VSL Workspace operations") -DEBUG_BIT(WAITER, waiter, "", "VSL Waiter internals") +DEBUG_BIT(WAITER, waiter, "\t","VSL Waiter internals") DEBUG_BIT(WAITINGLIST, waitinglist, "", "VSL Waitinglist events") +DEBUG_BIT(SYNCVSL, syncvsl, "\t","Make VSL synchronous") From phk at FreeBSD.org Thu Dec 18 09:27:55 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:55 +0100 Subject: [experimental-ims] 495b9f5 Move 3 more debug bits from diag_bitmaps to debug Message-ID: commit 495b9f51c6a7cc67e1c6b5609251cbcfb50ea3c2 Author: Poul-Henning Kamp Date: Mon Aug 27 20:19:33 2012 +0000 Move 3 more debug bits from diag_bitmaps to debug XXX: doc-update needed diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index c949acc..2054d3e 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -836,13 +836,13 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl, unsigned pass) b->flags &= ~BAN_F_LURK; b->flags |= pass; } - if (cache_param->diag_bitmap & 0x80000) + if (DO_DEBUG(DBG_LURKER)) VSLb(vsl, SLT_Debug, "lurker: %d actionable bans", i); if (i == 0) return (0); VTAILQ_FOREACH_REVERSE(b, &ban_head, banhead_s, list) { - if (cache_param->diag_bitmap & 0x80000) + if (DO_DEBUG(DBG_LURKER)) VSLb(vsl, SLT_Debug, "lurker doing %f %d", ban_time(b->spec), b->refcount); while (1) { @@ -851,7 +851,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl, unsigned pass) if (oc == NULL) break; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - if (cache_param->diag_bitmap & 0x80000) + if (DO_DEBUG(DBG_LURKER)) VSLb(vsl, SLT_Debug, "test: %p %u %u", oc, oc->flags & OC_F_LURK, pass); if ((oc->flags & OC_F_LURK) == pass) @@ -903,7 +903,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl, unsigned pass) */ o = oc_getobj(&wrk->stats, oc); i = ban_check_object(o, vsl, NULL); - if (cache_param->diag_bitmap & 0x80000) + if (DO_DEBUG(DBG_LURKER)) VSLb(vsl, SLT_Debug, "lurker got: %p %d", oc, i); if (i == -1) { @@ -915,7 +915,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl, unsigned pass) Lck_Unlock(&ban_mtx); } Lck_Unlock(&oh->mtx); - if (cache_param->diag_bitmap & 0x80000) + if (DO_DEBUG(DBG_LURKER)) VSLb(vsl, SLT_Debug, "lurker done: %p %u %u", oc, oc->flags & OC_F_LURK, pass); (void)HSH_Deref(&wrk->stats, NULL, &o); @@ -927,7 +927,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl, unsigned pass) b->flags |= BAN_F_GONE; VSC_C_main->bans_gone++; } - if (cache_param->diag_bitmap & 0x80000) + if (DO_DEBUG(DBG_LURKER)) VSLb(vsl, SLT_Debug, "lurker BAN %f now gone", ban_time(b->spec)); } @@ -1091,7 +1091,7 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv) VCLI_Out(cli, "Present bans:\n"); VTAILQ_FOREACH(b, &ban_head, list) { - if (b == bl && !(cache_param->diag_bitmap & 0x80000)) + if (b == bl && !DO_DEBUG(DBG_LURKER)) break; VCLI_Out(cli, "%10.6f %5u%s\t", ban_time(b->spec), bl == b ? b->refcount - 1 : b->refcount, @@ -1100,7 +1100,7 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv) VCLI_Out(cli, "\n"); if (VCLI_Overflow(cli)) break; - if (cache_param->diag_bitmap & 0x80000) { + if (DO_DEBUG(DBG_LURKER)) { Lck_Lock(&ban_mtx); struct objcore *oc; VTAILQ_FOREACH(oc, &b->objcore, ban_list) diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index fcf447a..932cb67 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -305,7 +305,7 @@ HSH_Lookup(struct req *req) AN(hash); hsh_prealloc(wrk); - if (cache_param->diag_bitmap & 0x80000000) + if (DO_DEBUG(DBG_HASHEDGE)) hsh_testmagic(req->digest); if (req->hash_objhead != NULL) { diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 6cffa69..5caf03f 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -140,10 +140,8 @@ ses_req_pool_task(struct worker *wrk, void *arg) HTTP1_Session(wrk, req); WS_Assert(wrk->aws); AZ(wrk->wrw); - if (cache_param->diag_bitmap & 0x00040000) { - if (wrk->vcl != NULL) - VCL_Rel(&wrk->vcl); - } + if (DO_DEBUG(DBG_VCLREL) && wrk->vcl != NULL) + VCL_Rel(&wrk->vcl); THR_SetRequest(NULL); } diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 2c8ad86..a6f75b8 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1054,9 +1054,6 @@ static const struct parspec input_parspec[] = { " 0x00002000 - only short panic message.\n" " 0x00004000 - panic to stderr.\n" " 0x00020000 - synchronous start of persistence.\n" - " 0x00040000 - release VCL early.\n" - " 0x00080000 - ban-lurker debugging.\n" - " 0x80000000 - do edge-detection on digest.\n" "\n" "Use 0x notation and do the bitor in your head :-)\n" "\n" diff --git a/bin/varnishtest/tests/c00023.vtc b/bin/varnishtest/tests/c00023.vtc index b57c8c1..0e6aff1 100644 --- a/bin/varnishtest/tests/c00023.vtc +++ b/bin/varnishtest/tests/c00023.vtc @@ -31,7 +31,7 @@ server s1 { } -start varnish v1 -arg "-hcritbit" -vcl+backend { } -start -varnish v1 -cliok "param.set diag_bitmap 0x80000000" +varnish v1 -cliok "param.set debug +hashedge" client c1 { txreq -url "/1" diff --git a/bin/varnishtest/tests/c00049.vtc b/bin/varnishtest/tests/c00049.vtc index 59e6c14..2e31705 100644 --- a/bin/varnishtest/tests/c00049.vtc +++ b/bin/varnishtest/tests/c00049.vtc @@ -36,7 +36,7 @@ varnish v1 -vcl+backend { } -start varnish v1 -cliok "param.set ban_lurker_sleep 0" -varnish v1 -cliok "param.set diag_bitmap 0x80000" +varnish v1 -cliok "param.set debug +lurker" varnish v1 -cliok "ban.list" diff --git a/bin/varnishtest/tests/m00001.vtc b/bin/varnishtest/tests/m00001.vtc index dd2b064..9f4a743 100644 --- a/bin/varnishtest/tests/m00001.vtc +++ b/bin/varnishtest/tests/m00001.vtc @@ -15,7 +15,7 @@ varnish v1 -arg "-pthread_pools=1" -vcl+backend { } } -start -varnish v1 -cliok "param.set diag_bitmap 0x40000" +varnish v1 -cliok "param.set debug +vclrel" client c1 { txreq -url "/bar" diff --git a/bin/varnishtest/tests/p00005.vtc b/bin/varnishtest/tests/p00005.vtc index b41ded3..e519cc1 100644 --- a/bin/varnishtest/tests/p00005.vtc +++ b/bin/varnishtest/tests/p00005.vtc @@ -8,7 +8,6 @@ server s1 { } -start varnish v1 \ - -arg "-pdiag_bitmap=0x30000" \ -storage "-spersistent,${tmpdir}/_.per,10m" \ -arg "-pban_lurker_sleep=0" \ -vcl+backend { @@ -17,6 +16,9 @@ varnish v1 \ } } -start +varnish v1 -cliok "param.set debug +syncvsl" +varnish v1 -cliok "param.set diag_bitmap 0x20000" + client c1 { txreq -url "/foo" rxresp diff --git a/include/tbl/debug_bits.h b/include/tbl/debug_bits.h index ece0fc4..72d298d 100644 --- a/include/tbl/debug_bits.h +++ b/include/tbl/debug_bits.h @@ -34,3 +34,6 @@ DEBUG_BIT(WORKSPACE, workspace, "", "VSL Workspace operations") DEBUG_BIT(WAITER, waiter, "\t","VSL Waiter internals") DEBUG_BIT(WAITINGLIST, waitinglist, "", "VSL Waitinglist events") DEBUG_BIT(SYNCVSL, syncvsl, "\t","Make VSL synchronous") +DEBUG_BIT(HASHEDGE, hashedge, "", "Edge cases in Hash") +DEBUG_BIT(VCLREL, vclrel, "\t","Rapid VCL release") +DEBUG_BIT(LURKER, lurker, "\t","VSL Ban lurker") From phk at FreeBSD.org Thu Dec 18 09:27:55 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:55 +0100 Subject: [experimental-ims] a1781ca Minor polish to bitmap params Message-ID: commit a1781ca5c65cf4bc81c0b98136ce93d0ff7c325b Author: Poul-Henning Kamp Date: Tue Aug 28 06:37:36 2012 +0000 Minor polish to bitmap params diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index a6f75b8..aa99683 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1048,7 +1048,6 @@ static const struct parspec input_parspec[] = { "Bitmap controlling diagnostics code:\n" " 0x00000008 - mutex logging.\n" " 0x00000010 - mutex contests.\n" - " 0x00000040 - object workspace.\n" " 0x00000080 - mutex timing.\n" " 0x00001000 - do not core-dump child process.\n" " 0x00002000 - only short panic message.\n" diff --git a/bin/varnishtest/tests/b00008.vtc b/bin/varnishtest/tests/b00008.vtc index 5d447b2..b8dcbc8 100644 --- a/bin/varnishtest/tests/b00008.vtc +++ b/bin/varnishtest/tests/b00008.vtc @@ -22,7 +22,7 @@ varnish v1 -cliok "param.show" varnish v1 -cliok "param.show diag_bitmap" -varnish v1 -cliok "param.set diag_bitmap 0x40" +varnish v1 -cliok "param.set diag_bitmap 0x80" varnish v1 -cliok "param.set diag_bitmap 0x0" diff --git a/bin/varnishtest/tests/c00054.vtc b/bin/varnishtest/tests/c00054.vtc index 3299604..e6e2260 100644 --- a/bin/varnishtest/tests/c00054.vtc +++ b/bin/varnishtest/tests/c00054.vtc @@ -1,4 +1,4 @@ -varnishtest "SLT masking" +varnishtest "bitmap params masking" server s1 { @@ -13,9 +13,14 @@ varnish v1 -cliok "param.set vsl_mask +VCL_trace" varnish v1 -cliok "param.show vsl_mask" varnish v1 -cliok "param.set vsl_mask -WorkThread,-TTL" varnish v1 -cliok "param.show vsl_mask" -varnish v1 -cliok "param.set vsl_mask +WorkThread" +varnish v1 -cliok "param.set vsl_mask +WorkThread,+TTL,+Hash" +varnish v1 -cliok "param.show vsl_mask" varnish v1 -clierr 106 "param.set vsl_mask FooBar" varnish v1 -clierr 106 "param.set vsl_mask -FooBar" +varnish v1 -clierr 106 {param.set vsl_mask \"} + +varnish v1 -cliok "param.set debug +workspace" +varnish v1 -cliok "param.show debug" client c1 { txreq From phk at FreeBSD.org Thu Dec 18 09:27:55 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:55 +0100 Subject: [experimental-ims] 13c8fed Slightly better coverage testing. Message-ID: commit 13c8fedd4bae89bda9c78670d0660213dcf8707f Author: Poul-Henning Kamp Date: Tue Aug 28 08:21:49 2012 +0000 Slightly better coverage testing. diff --git a/bin/varnishtest/tests/c00054.vtc b/bin/varnishtest/tests/c00054.vtc index d3a1072..266cc13 100644 --- a/bin/varnishtest/tests/c00054.vtc +++ b/bin/varnishtest/tests/c00054.vtc @@ -22,6 +22,9 @@ varnish v1 -clierr 106 {param.set vsl_mask \"} varnish v1 -cliok "param.set debug +workspace" varnish v1 -cliok "param.show debug" varnish v1 -cliok "param.show feature" +varnish v1 -cliok "param.set feature +short_panic" +varnish v1 -cliok "param.show feature" + client c1 { txreq From phk at FreeBSD.org Thu Dec 18 09:27:55 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:55 +0100 Subject: [experimental-ims] 0b7cc04 Add a new "feature" parameter and move certain bits from diag_bitmaps there. These bits are legitimate feature-ettes which might make sense in a production environment. Message-ID: commit 0b7cc04c35e917df74cfa976748ae3cae092f65a Author: Poul-Henning Kamp Date: Tue Aug 28 07:41:45 2012 +0000 Add a new "feature" parameter and move certain bits from diag_bitmaps there. These bits are legitimate feature-ettes which might make sense in a production environment. XXX: need doc-change diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 30f8e99..94db4b6 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -1118,6 +1118,13 @@ Tadd(txt *t, const char *p, int l) #define W_TIM_real(w) ((w)->lastused = VTIM_real()) static inline int +FEATURE(enum feature_bits x) +{ + return (cache_param->feature_bits[(unsigned)x>>3] & + (0x80U >> ((unsigned)x & 7))); +} + +static inline int DO_DEBUG(enum debug_bits x) { return (cache_param->debug_bits[(unsigned)x>>3] & diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index 8af00f6..ac36308 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -225,7 +225,7 @@ child_main(void) CLI_AddFuncs(debug_cmds); /* Wait for persistent storage to load if asked to */ - if (cache_param->diag_bitmap & 0x00020000) + if (FEATURE(FEATURE_WAIT_SILO)) SMP_Ready(); CLI_Run(); diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index ee7dd78..c93749f 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -392,7 +392,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond, pan_backtrace(); - if (!(cache_param->diag_bitmap & 0x2000)) { + if (!FEATURE(FEATURE_SHORT_PANIC)) { req = THR_GetRequest(); if (req != NULL) pan_req(req); @@ -400,10 +400,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond, VSB_printf(pan_vsp, "\n"); VSB_bcat(pan_vsp, "", 1); /* NUL termination */ - if (cache_param->diag_bitmap & 0x4000) - (void)fputs(heritage.panic_str, stderr); - - if (cache_param->diag_bitmap & 0x1000) + if (FEATURE(FEATURE_NO_COREDUMP)) exit(4); else abort(); diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index 6c3abb4..6abe0fc 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -40,6 +40,13 @@ enum debug_bits { DBG_Reserved }; +enum feature_bits { +#define FEATURE_BIT(U, l, p, d, ld) FEATURE_##U, +#include "tbl/feature_bits.h" +#undef FEATURE_BIT + FEATURE_Reserved +}; + struct poolparam { unsigned min_pool; unsigned max_pool; @@ -207,4 +214,5 @@ struct params { uint8_t vsl_mask[256>>3]; uint8_t debug_bits[(DBG_Reserved+7)>>3]; + uint8_t feature_bits[(FEATURE_Reserved+7)>>3]; }; diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index c869897..95d1794 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -88,6 +88,14 @@ static struct vlu *vlu; static struct vsb *child_panic = NULL; +static inline int +MGT_FEATURE(enum feature_bits x) +{ + return (mgt_param.feature_bits[(unsigned)x>>3] & + (0x80U >> ((unsigned)x & 7))); +} + + /*-------------------------------------------------------------------- * Track the highest file descriptor the parent knows is being used. * @@ -213,7 +221,7 @@ MGT_Child_Cli_Fail(void) return; REPORT(LOG_ERR, "Child (%jd) not responding to CLI, killing it.", (intmax_t)child_pid); - if (mgt_param.diag_bitmap & 0x1000) + if (MGT_FEATURE(FEATURE_NO_COREDUMP)) (void)kill(child_pid, SIGKILL); else (void)kill(child_pid, SIGQUIT); diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index aa99683..434082e 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -1049,10 +1049,6 @@ static const struct parspec input_parspec[] = { " 0x00000008 - mutex logging.\n" " 0x00000010 - mutex contests.\n" " 0x00000080 - mutex timing.\n" - " 0x00001000 - do not core-dump child process.\n" - " 0x00002000 - only short panic message.\n" - " 0x00004000 - panic to stderr.\n" - " 0x00020000 - synchronous start of persistence.\n" "\n" "Use 0x notation and do the bitor in your head :-)\n" "\n" diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c index b238823..a5996a2 100644 --- a/bin/varnishd/mgt/mgt_param_bits.c +++ b/bin/varnishd/mgt/mgt_param_bits.c @@ -115,7 +115,7 @@ bit_tweak(struct cli *cli, uint8_t *p, unsigned l, const char *arg, */ static const char * const VSL_tags[256] = { -# define SLTM(foo,sdesc,ldesc) [SLT_##foo] = #foo, +# define SLTM(foo,sdesc,ldesc) [SLT_##foo] = #foo, # include "tbl/vsl_tags.h" # undef SLTM NULL @@ -158,7 +158,7 @@ tweak_vsl_mask(struct cli *cli, const struct parspec *par, const char *arg) */ static const char * const debug_tags[] = { -# define DEBUG_BIT(U,l,p,d) [DBG_##U] = #l, +# define DEBUG_BIT(U,l,p,d) [DBG_##U] = #l, # include "tbl/debug_bits.h" # undef DEBUG_BIT NULL @@ -193,6 +193,46 @@ tweak_debug(struct cli *cli, const struct parspec *par, const char *arg) } /*-------------------------------------------------------------------- + * The feature parameter + */ + +static const char * const feature_tags[] = { +# define FEATURE_BIT(U,l,p,d, ld) [FEATURE_##U] = #l, +# include "tbl/feature_bits.h" +# undef FEATURE_BIT + NULL +}; + +static void +tweak_feature(struct cli *cli, const struct parspec *par, const char *arg) +{ + const char *s; + unsigned j; + (void)par; + + if (arg != NULL) { + if (!strcmp(arg, "none")) { + memset(mgt_param.feature_bits, + 0, sizeof mgt_param.feature_bits); + } else { + bit_tweak(cli, mgt_param.feature_bits, + FEATURE_Reserved, arg, feature_tags, + "feature bit", "+"); + } + } else { + s = ""; + for (j = 0; j < (unsigned)FEATURE_Reserved; j++) { + if (bit(mgt_param.feature_bits, j, BTST)) { + VCLI_Out(cli, "%s+%s", s, feature_tags[j]); + s = ","; + } + } + if (*s == '\0') + VCLI_Out(cli, "none"); + } +} + +/*-------------------------------------------------------------------- * The parameter table itself */ @@ -211,5 +251,13 @@ const struct parspec VSL_parspec[] = { #include "tbl/debug_bits.h" #undef DEBUG_BIT , 0, "none", "" }, + { "feature", tweak_feature, NULL, 0, 0, + "Enable/Disable various minor features.\n" + "\tnone\t\tDisable all features.\n" + "Use +/- prefix to enable/disable individual feature:\n" +#define FEATURE_BIT(U, l, p, d, ld) "\t" #l "\t" p d "\n" +#include "tbl/feature_bits.h" +#undef FEATURE_BIT + , 0, "none", "" }, { NULL, NULL, NULL } }; diff --git a/bin/varnishtest/tests/c00054.vtc b/bin/varnishtest/tests/c00054.vtc index e6e2260..d3a1072 100644 --- a/bin/varnishtest/tests/c00054.vtc +++ b/bin/varnishtest/tests/c00054.vtc @@ -21,6 +21,7 @@ varnish v1 -clierr 106 {param.set vsl_mask \"} varnish v1 -cliok "param.set debug +workspace" varnish v1 -cliok "param.show debug" +varnish v1 -cliok "param.show feature" client c1 { txreq diff --git a/bin/varnishtest/tests/p00000.vtc b/bin/varnishtest/tests/p00000.vtc index 12fa676..745abf5 100644 --- a/bin/varnishtest/tests/p00000.vtc +++ b/bin/varnishtest/tests/p00000.vtc @@ -8,7 +8,7 @@ server s1 { shell "rm -f ${tmpdir}/_.per" varnish v1 \ - -arg "-pdiag_bitmap=0x20000" \ + -arg "-pfeature=+wait_silo" \ -storage "-spersistent,${tmpdir}/_.per,10m" \ -vcl+backend { } -start diff --git a/bin/varnishtest/tests/p00001.vtc b/bin/varnishtest/tests/p00001.vtc index 26ea975..ac95149 100644 --- a/bin/varnishtest/tests/p00001.vtc +++ b/bin/varnishtest/tests/p00001.vtc @@ -8,7 +8,7 @@ server s1 { } -start varnish v1 \ - -arg "-pdiag_bitmap=0x20000" \ + -arg "-pfeature=+wait_silo" \ -arg "-pban_lurker_sleep=0" \ -storage "-spersistent,${tmpdir}/_.per,10m" \ -vcl+backend { } -start diff --git a/bin/varnishtest/tests/p00002.vtc b/bin/varnishtest/tests/p00002.vtc index e3ea052..dc6e584 100644 --- a/bin/varnishtest/tests/p00002.vtc +++ b/bin/varnishtest/tests/p00002.vtc @@ -8,7 +8,7 @@ server s1 { } -start varnish v1 \ - -arg "-pdiag_bitmap=0x20000" \ + -arg "-pfeature=+wait_silo" \ -arg "-pban_lurker_sleep=0" \ -storage "-spersistent,${tmpdir}/_.per1,10m" \ -storage "-spersistent,${tmpdir}/_.per2,10m" \ diff --git a/bin/varnishtest/tests/p00003.vtc b/bin/varnishtest/tests/p00003.vtc index f554536..4af3b14 100644 --- a/bin/varnishtest/tests/p00003.vtc +++ b/bin/varnishtest/tests/p00003.vtc @@ -8,7 +8,7 @@ server s1 { } -start varnish v1 \ - -arg "-pdiag_bitmap=0x20000" \ + -arg "-pfeature=+wait_silo" \ -storage "-spersistent,${tmpdir}/_.per,10m" \ -arg "-pban_lurker_sleep=0" \ -vcl+backend { } -start diff --git a/bin/varnishtest/tests/p00004.vtc b/bin/varnishtest/tests/p00004.vtc index 343d7de..f1e4368 100644 --- a/bin/varnishtest/tests/p00004.vtc +++ b/bin/varnishtest/tests/p00004.vtc @@ -10,7 +10,7 @@ server s1 { } -start varnish v1 \ - -arg "-pdiag_bitmap=0x20000" \ + -arg "-pfeature=+wait_silo" \ -storage "-spersistent,${tmpdir}/_.per,10m" \ -arg "-pban_lurker_sleep=0" \ -vcl+backend { } -start diff --git a/bin/varnishtest/tests/p00005.vtc b/bin/varnishtest/tests/p00005.vtc index e519cc1..cb0c4e2 100644 --- a/bin/varnishtest/tests/p00005.vtc +++ b/bin/varnishtest/tests/p00005.vtc @@ -17,7 +17,7 @@ varnish v1 \ } -start varnish v1 -cliok "param.set debug +syncvsl" -varnish v1 -cliok "param.set diag_bitmap 0x20000" +varnish v1 -cliok "param.set feature +wait_silo" client c1 { txreq -url "/foo" diff --git a/bin/varnishtest/tests/r00915.vtc b/bin/varnishtest/tests/r00915.vtc index 07a39d9..266c6b7 100644 --- a/bin/varnishtest/tests/r00915.vtc +++ b/bin/varnishtest/tests/r00915.vtc @@ -8,7 +8,7 @@ server s1 { shell "rm -f ${tmpdir}/_.per" varnish v1 \ - -arg "-pdiag_bitmap=0x20000" \ + -arg "-pfeature=+wait_silo" \ -storage "-spersistent,${tmpdir}/_.per,10m" \ -vcl+backend { diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc index aa0fb10..4331cd0 100644 --- a/bin/varnishtest/tests/r00962.vtc +++ b/bin/varnishtest/tests/r00962.vtc @@ -11,7 +11,7 @@ server s1 { shell "rm -f ${tmpdir}/_.per?" varnish v1 \ - -arg "-pdiag_bitmap=0x20000" \ + -arg "-pfeature=+wait_silo" \ -storage "-spersistent,${tmpdir}/_.per1,10m -spersistent,${tmpdir}/_.per2,10m" \ -vcl+backend { sub vcl_fetch { @@ -42,7 +42,7 @@ server s1 { varnish v2 \ - -arg "-pdiag_bitmap=0x20000" \ + -arg "-pfeature=+wait_silo" \ -storage "-spersistent,${tmpdir}/_.per2,10m -spersistent,${tmpdir}/_.per1,10m" \ -vcl+backend { } -start diff --git a/bin/varnishtest/tests/v00010.vtc b/bin/varnishtest/tests/v00010.vtc index 2e9a385..19405cc 100644 --- a/bin/varnishtest/tests/v00010.vtc +++ b/bin/varnishtest/tests/v00010.vtc @@ -37,7 +37,7 @@ varnish v1 -storage "-smalloc,1m" -vcl+backend { } } -start -varnish v1 -cliok "param.set diag_bitmap 0x00001000" +varnish v1 -cliok "param.set feature +no_coredump" # Force the (random) port selected to be used again after restart. varnish v1 -cliok "param.set listen_address ${v1_addr}:${v1_port}" diff --git a/include/Makefile.am b/include/Makefile.am index 7562f6c..15df374 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -7,6 +7,7 @@ nobase_pkginclude_HEADERS = \ tbl/ban_vars.h \ tbl/body_status.h \ tbl/debug_bits.h \ + tbl/feature_bits.h \ tbl/http_headers.h \ tbl/http_response.h \ tbl/locks.h \ diff --git a/include/tbl/feature_bits.h b/include/tbl/feature_bits.h new file mode 100644 index 0000000..9d99449 --- /dev/null +++ b/include/tbl/feature_bits.h @@ -0,0 +1,43 @@ +/*- + * Copyright (c) 2012 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Fields in the feature parameter + * + */ + +FEATURE_BIT(SHORT_PANIC, short_panic, "", + "Short panic message.", + "Reduce level of detail for panic messages." +) +FEATURE_BIT(WAIT_SILO, wait_silo, "", + "Wait for persistent silo.", + "Wait for persistent silos to load completely before serving requests." +) +FEATURE_BIT(NO_COREDUMP, no_coredump, "", + "No coredumps.", + "Don't attempt to coredump child process on panics." +) From phk at FreeBSD.org Thu Dec 18 09:27:55 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:55 +0100 Subject: [experimental-ims] a187a0a Elimiante diag_bitmaps and the mutex-profiling/measuring code. Message-ID: commit a187a0ad4f7552a65684649ffddd745c14c1c791 Author: Poul-Henning Kamp Date: Tue Aug 28 09:06:00 2012 +0000 Elimiante diag_bitmaps and the mutex-profiling/measuring code. XXX: doc-change needed. diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c index 993bb5e..bd34046 100644 --- a/bin/varnishd/cache/cache_lck.c +++ b/bin/varnishd/cache/cache_lck.c @@ -37,7 +37,6 @@ #include -#include "vtim.h" #include "cache.h" /*The constability of lck depends on platform pthreads implementation */ @@ -48,7 +47,6 @@ struct ilck { pthread_mutex_t mtx; int held; pthread_t owner; - double t0; VTAILQ_ENTRY(ilck) list; const char *w; struct VSC_C_lck *stat; @@ -63,37 +61,13 @@ void __match_proto__() Lck__Lock(struct lock *lck, const char *p, const char *f, int l) { struct ilck *ilck; - int r; - double t0 = 0, t; + + (void)p; + (void)f; + (void)l; CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC); - if (!(cache_param->diag_bitmap & 0x98)) { - AZ(pthread_mutex_lock(&ilck->mtx)); - AZ(ilck->held); - ilck->stat->locks++; - ilck->owner = pthread_self(); - ilck->held = 1; - return; - } - if (cache_param->diag_bitmap & 0x80) - t0 = VTIM_real(); - r = pthread_mutex_trylock(&ilck->mtx); - assert(r == 0 || r == EBUSY); - if (r) { - ilck->stat->colls++; - if (cache_param->diag_bitmap & 0x8) - VSL(SLT_Debug, 0, "MTX_CONTEST(%s,%s,%d,%s)", - p, f, l, ilck->w); - AZ(pthread_mutex_lock(&ilck->mtx)); - } else if (cache_param->diag_bitmap & 0x8) { - VSL(SLT_Debug, 0, "MTX_LOCK(%s,%s,%d,%s)", p, f, l, ilck->w); - } - if (cache_param->diag_bitmap & 0x80) { - t = VTIM_real(); - VSL(SLT_Debug, 0, "MTX_LOCKWAIT(%s,%s,%d,%s) %.9fs", - p, f, l, ilck->w, t - t0); - ilck->t0 = t; - } + AZ(pthread_mutex_lock(&ilck->mtx)); AZ(ilck->held); ilck->stat->locks++; ilck->owner = pthread_self(); @@ -105,6 +79,10 @@ Lck__Unlock(struct lock *lck, const char *p, const char *f, int l) { struct ilck *ilck; + (void)p; + (void)f; + (void)l; + CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC); assert(pthread_equal(ilck->owner, pthread_self())); AN(ilck->held); @@ -121,11 +99,6 @@ Lck__Unlock(struct lock *lck, const char *p, const char *f, int l) */ memset(&ilck->owner, 0, sizeof ilck->owner); AZ(pthread_mutex_unlock(&ilck->mtx)); - if (cache_param->diag_bitmap & 0x80) - VSL(SLT_Debug, 0, "MTX_UNLOCK(%s,%s,%d,%s) %.9fs", - p, f, l, ilck->w, VTIM_real() - ilck->t0); - else if (cache_param->diag_bitmap & 0x8) - VSL(SLT_Debug, 0, "MTX_UNLOCK(%s,%s,%d,%s)", p, f, l, ilck->w); } int __match_proto__() @@ -134,19 +107,18 @@ Lck__Trylock(struct lock *lck, const char *p, const char *f, int l) struct ilck *ilck; int r; + (void)p; + (void)f; + (void)l; + CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC); r = pthread_mutex_trylock(&ilck->mtx); assert(r == 0 || r == EBUSY); - if (cache_param->diag_bitmap & 0x8) - VSL(SLT_Debug, 0, - "MTX_TRYLOCK(%s,%s,%d,%s) = %d", p, f, l, ilck->w, r); if (r == 0) { AZ(ilck->held); ilck->held = 1; ilck->stat->locks++; ilck->owner = pthread_self(); - if (cache_param->diag_bitmap & 0x80) - ilck->t0 = VTIM_real(); } return (r); } diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index 6abe0fc..7c1ccd5 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -155,9 +155,6 @@ struct params { /* CLI buffer size */ unsigned cli_buffer; - /* Control diagnostic code */ - unsigned diag_bitmap; - /* Log local socket address to shm */ unsigned log_local_addr; diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 434082e..b0631a3 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -579,22 +579,6 @@ tweak_waiter(struct cli *cli, const struct parspec *par, const char *arg) /*--------------------------------------------------------------------*/ static void -tweak_diag_bitmap(struct cli *cli, const struct parspec *par, const char *arg) -{ - unsigned u; - - (void)par; - if (arg != NULL) { - u = strtoul(arg, NULL, 0); - mgt_param.diag_bitmap = u; - } else { - VCLI_Out(cli, "0x%x", mgt_param.diag_bitmap); - } -} - -/*--------------------------------------------------------------------*/ - -static void tweak_poolparam(struct cli *cli, const struct parspec *par, const char *arg) { volatile struct poolparam *pp, px; @@ -1044,19 +1028,6 @@ static const struct parspec input_parspec[] = { "Select the waiter kernel interface.\n", WIZARD | MUST_RESTART, WAITER_DEFAULT, NULL }, - { "diag_bitmap", tweak_diag_bitmap, 0, 0, 0, - "Bitmap controlling diagnostics code:\n" - " 0x00000008 - mutex logging.\n" - " 0x00000010 - mutex contests.\n" - " 0x00000080 - mutex timing.\n" - "\n" - "Use 0x notation and do the bitor in your head :-)\n" - "\n" - "Note: Mutex timing will add extra overhead and " - "synchronization between threads, consequently changing the " - "locking characteristics.\n", - 0, - "0", "bitmap" }, { "ban_dups", tweak_bool, &mgt_param.ban_dups, 0, 0, "Detect and eliminate duplicate bans.\n", 0, diff --git a/bin/varnishtest/tests/b00008.vtc b/bin/varnishtest/tests/b00008.vtc index b8dcbc8..1ce9b64 100644 --- a/bin/varnishtest/tests/b00008.vtc +++ b/bin/varnishtest/tests/b00008.vtc @@ -20,12 +20,6 @@ varnish v1 -clierr 105 "help 1 2 3" varnish v1 -cliok "param.show" -varnish v1 -cliok "param.show diag_bitmap" - -varnish v1 -cliok "param.set diag_bitmap 0x80" - -varnish v1 -cliok "param.set diag_bitmap 0x0" - varnish v1 -start varnish v1 -cliok "help" From phk at FreeBSD.org Thu Dec 18 09:27:55 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:55 +0100 Subject: [experimental-ims] 132e912 eliminate unused VSC field Message-ID: commit 132e912d5b1aa06cdd38b915901ae118d1895ea5 Author: Poul-Henning Kamp Date: Tue Aug 28 09:09:24 2012 +0000 eliminate unused VSC field diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h index ea70ffb..1ab16ff 100644 --- a/include/tbl/vsc_fields.h +++ b/include/tbl/vsc_fields.h @@ -73,10 +73,6 @@ VSC_F(locks, uint64_t, 0, 'a', "Lock Operations", "" ) -VSC_F(colls, uint64_t, 0, 'a', - "Collisions", - "" -) #endif From phk at FreeBSD.org Thu Dec 18 09:27:55 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Dec 2014 10:27:55 +0100 Subject: [experimental-ims] 57b4594 Don't overrun the VSL segment looking for the end. Message-ID: commit 57b4594c3ad86afc0f656943cf20070cb8d3af57 Author: Poul-Henning Kamp Date: Tue Aug 28 12:24:29 2012 +0000 Don't overrun the VSL segment looking for the end. diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index cd6034d..da9e5df 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -139,9 +139,12 @@ vsl_open(struct VSM_data *vd) vsl->log_end = vsl->vf.e; vsl->log_ptr = vsl->log_start + 1; if (!vsl->d_opt) { - while (*vsl->log_ptr != VSL_ENDMARKER) + while (vsl->log_ptr < vsl->log_end && + *vsl->log_ptr != VSL_ENDMARKER) vsl->log_ptr = VSL_NEXT(vsl->log_ptr); } + if (vsl->log_ptr >= vsl->log_end) + vsl->log_ptr = vsl->log_start + 1; return (0); } From arianna.aondio at varnish-software.com Fri Dec 19 09:05:25 2014 From: arianna.aondio at varnish-software.com (arianna-aondio) Date: Fri, 19 Dec 2014 10:05:25 +0100 Subject: [master] 2ddab72 %hh length modifier added for non-printabale chars. Message-ID: commit 2ddab727b9a3c7907804d0c3f6810b59a5fce562 Author: arianna-aondio Date: Fri Dec 19 10:04:00 2014 +0100 %hh length modifier added for non-printabale chars. diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index dcda6e5..6e01e60 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -222,7 +222,7 @@ vsb_esc_cat(struct vsb *sb, const char *b, const char *e) break; } } else - VSB_printf(sb, "\\x%02x", *b); + VSB_printf(sb, "\\x%02hhx", *b); } return (VSB_error(sb)); From fgsch at lodoss.net Sat Dec 27 09:43:11 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 27 Dec 2014 10:43:11 +0100 Subject: [master] 544749c Whitespace Message-ID: commit 544749c9618c2e3ba3265ddb4d2db020f432fc77 Author: Federico G. Schwindt Date: Fri Dec 19 10:59:33 2014 +0000 Whitespace diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 6e01e60..d61a421 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -215,8 +215,8 @@ vsb_esc_cat(struct vsb *sb, const char *b, const char *e) VSB_cat(sb, "\\\""); break; case '\\': - VSB_cat(sb, "\\\\"); - break; + VSB_cat(sb, "\\\\"); + break; default: VSB_putc(sb, *b); break; @@ -272,7 +272,7 @@ format_fragment(const struct format *format) if (format->string == NULL) return (-1); AZ(vsb_esc_cat(CTX.vsb, format->string, - format->string + strlen(format->string))); + format->string + strlen(format->string))); return (0); } AZ(vsb_fcat(CTX.vsb, format->frag, NULL)); diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 42b356a..01f13e8 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -233,7 +233,7 @@ vmod_cache_req_body(VRT_CTX, VCL_BYTES size) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); result = VRT_CacheReqBody(ctx, size); - VSLb(ctx->vsl, SLT_Debug,"VRT_CacheReqBody(%zu): %d", + VSLb(ctx->vsl, SLT_Debug, "VRT_CacheReqBody(%zu): %d", (size_t)size, result); } From fgsch at lodoss.net Sat Dec 27 09:43:11 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 27 Dec 2014 10:43:11 +0100 Subject: [master] 3e8cabc Correct synopsis Message-ID: commit 3e8cabc22b2663c05b6d85d745b14834c866a4fa Author: Federico G. Schwindt Date: Sat Dec 27 09:12:02 2014 +0000 Correct synopsis diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst index 54247fd..15a1446 100644 --- a/doc/sphinx/reference/varnishlog.rst +++ b/doc/sphinx/reference/varnishlog.rst @@ -12,7 +12,7 @@ SYNOPSIS ======== .. include:: ../include/varnishlog_synopsis.rst -varnishlog |synopsis| +varnishlog |synopsis| OPTIONS ======= From fgsch at lodoss.net Tue Dec 30 03:17:27 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 30 Dec 2014 04:17:27 +0100 Subject: [master] cd66576 Update tags Message-ID: commit cd665769c5915febad040a98062c8e7976051036 Author: Federico G. Schwindt Date: Tue Dec 30 03:16:45 2014 +0000 Update tags diff --git a/doc/sphinx/users-guide/operation-statistics.rst b/doc/sphinx/users-guide/operation-statistics.rst index 788e600..de25bd8 100644 --- a/doc/sphinx/users-guide/operation-statistics.rst +++ b/doc/sphinx/users-guide/operation-statistics.rst @@ -18,11 +18,10 @@ With suitable filtering using the -I, -i, -X and -x options, it can be used to display a ranking of requested documents, clients, user agents, or any other information which is recorded in the log. -``varnishtop -i rxurl`` will show you what URLs are being asked for -by the client. ``varnishtop -i txurl`` will show you what your backend -is being asked the most. ``varnishtop -i RxHeader -I -Accept-Encoding`` will show the most popular Accept-Encoding header -the client are sending you. +``varnishtop -i ReqURL`` will show you what URLs are being asked for by +the client. ``varnishtop -i BereqURL`` will show you what your backend +is being asked the most. ``varnishtop -I ReqHeader:Accept-Encoding`` will +show the most popular Accept-Encoding header the client are sending you. For more information please see :ref:`ref-varnishtop`.