From phk at FreeBSD.org Mon Sep 1 06:45:21 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 01 Sep 2014 08:45:21 +0200 Subject: [master] 81913d9 Add obj.age and beresp.age VCL variables Message-ID: commit 81913d93d978f45bfa49b65321cb49e4f1e8ad88 Author: Poul-Henning Kamp Date: Mon Sep 1 06:45:06 2014 +0000 Add obj.age and beresp.age VCL variables diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 93bbf04..451f516 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -537,6 +537,22 @@ VRT_DO_EXP_L(beresp, ctx->bo->fetch_objcore->exp, keep, 0) VRT_DO_EXP_R(beresp, ctx->bo->fetch_objcore->exp, keep, 0) /*-------------------------------------------------------------------- + */ + +#define VRT_DO_AGE_R(which, sexp) \ + \ +double \ +VRT_r_##which##_##age(const struct vrt_ctx *ctx) \ +{ \ + \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ + return(ctx->now - sexp.t_origin); \ +} + +VRT_DO_AGE_R(obj, ctx->req->objcore->exp) +VRT_DO_AGE_R(beresp, ctx->bo->fetch_objcore->exp) + +/*-------------------------------------------------------------------- * [be]req.xid */ diff --git a/bin/varnishtest/tests/r00956.vtc b/bin/varnishtest/tests/r00956.vtc index 71dc617..a00f001 100644 --- a/bin/varnishtest/tests/r00956.vtc +++ b/bin/varnishtest/tests/r00956.vtc @@ -10,12 +10,15 @@ varnish v1 -vcl+backend { set beresp.http.fooA = beresp.ttl; set beresp.ttl = 10s; set beresp.http.fooB = beresp.ttl; + set beresp.http.ageA = beresp.age; } sub vcl_hit { set req.http.foo = obj.ttl; + set req.http.ageB = obj.age; } sub vcl_deliver { set resp.http.foo = req.http.foo; + set resp.http.ageB = req.http.ageB; } } -start @@ -25,6 +28,7 @@ client c1 { expect resp.bodylen == 40 expect resp.http.fooA == 19.000 expect resp.http.fooB == 10.000 + expect resp.http.ageA == 4.000 delay 2 txreq @@ -33,6 +37,8 @@ client c1 { expect resp.http.fooA == 19.000 expect resp.http.fooB == 10.000 expect resp.http.foo <= 8.000 + expect resp.http.ageA == 4.000 + expect resp.http.ageB >= 6.000 delay 2 txreq @@ -41,4 +47,6 @@ client c1 { expect resp.http.fooA == 19.000 expect resp.http.fooB == 10.000 expect resp.http.foo <= 6.000 + expect resp.http.ageA == 4.000 + expect resp.http.ageB >= 8.000 } -run diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 6e3c91f..ca6f7c9 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -468,7 +468,13 @@ sp_variables = [ ( 'backend_response', 'backend_error'), ( 'backend_response', 'backend_error'), """ The object's remaining time to live, in seconds. - beresp.ttl is writable. + """ + ), + ('beresp.age', + 'DURATION', + ( 'backend_response', 'backend_error'), + ( ), """ + The age of the object. """ ), ('beresp.grace', @@ -553,6 +559,13 @@ sp_variables = [ The object's remaining time to live, in seconds. """ ), + ('obj.age', + 'DURATION', + ( 'hit', ), + ( ), """ + The age of the object. + """ + ), ('obj.grace', 'DURATION', ( 'hit', ), From phk at FreeBSD.org Mon Sep 1 07:17:01 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 01 Sep 2014 09:17:01 +0200 Subject: [master] 636ec48 Add another request to this test-case, hoping to catch info about #1520 Message-ID: commit 636ec484a6e24dafe4cf840c26a482c2e7b259e5 Author: Poul-Henning Kamp Date: Mon Sep 1 07:16:31 2014 +0000 Add another request to this test-case, hoping to catch info about #1520 diff --git a/bin/varnishtest/tests/m00011.vtc b/bin/varnishtest/tests/m00011.vtc index 9f17dfe..bd17059 100644 --- a/bin/varnishtest/tests/m00011.vtc +++ b/bin/varnishtest/tests/m00011.vtc @@ -4,6 +4,8 @@ varnishtest "Test std.ip" server s1 { rxreq txresp -body "1" + rxreq + txresp -body "22" } -start @@ -21,8 +23,12 @@ varnish v1 -vcl+backend { client c1 { txreq -url "/foo1" rxresp + expect resp.bodylen == 1 expect resp.http.foo0 == "127.0.0.1" expect resp.http.foo1 == "127.0.0.1" expect resp.http.foo2 == "127.0.0.2" expect resp.http.foo3 == "1.2.3.5" + txreq -url "/foo2" + rxresp + expect resp.bodylen == 2 } -run From phk at FreeBSD.org Mon Sep 1 07:35:40 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 01 Sep 2014 09:35:40 +0200 Subject: [master] 9dd5d27 Don't leak struct req if we cannot restart it from a waiting list. Message-ID: commit 9dd5d273b6b5aa44705778033701ff04be79bcf9 Author: Poul-Henning Kamp Date: Mon Sep 1 07:35:16 2014 +0000 Don't leak struct req if we cannot restart it from a waiting list. Fixes: #1466 diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 753a646..9bcc322 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -534,9 +534,9 @@ hsh_rush(struct dstat *ds, struct objhead *oh) DSL(DBG_WAITINGLIST, req->vsl->wid, "off waiting list"); if (SES_ScheduleReq(req)) { /* - * We could not schedule the session, leave the - * rest on the busy list. + * We could not schedule the session, put it back. */ + VTAILQ_INSERT_HEAD(&wl->list, req, w_list); break; } } From phk at FreeBSD.org Mon Sep 1 08:10:30 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 01 Sep 2014 10:10:30 +0200 Subject: [master] 69425d5 Reinstate this testcase, but make the servers failure to deliver non-fatal. Message-ID: commit 69425d5ec6b0c7870e0e967aa3b3c56cd4724c1c Author: Poul-Henning Kamp Date: Mon Sep 1 08:09:46 2014 +0000 Reinstate this testcase, but make the servers failure to deliver non-fatal. Fixes: #1414 diff --git a/bin/varnishtest/tests.disabled/r01391.vtc b/bin/varnishtest/tests.disabled/r01391.vtc deleted file mode 100644 index 4b98fda..0000000 --- a/bin/varnishtest/tests.disabled/r01391.vtc +++ /dev/null @@ -1,39 +0,0 @@ -varnishtest "client abandoning hit-for-pass" - - -server s1 { - rxreq - txresp -nolen -hdr "Transfer-Encoding: chunked" -hdr "Set-Cookie: foo=bar" - chunked "foo" - sema r1 sync 2 - chunked "bar" - delay .1 - chunkedlen 64 - delay .1 - chunkedlen 64 - chunkedlen 0 -} -start - -varnish v1 -vcl+backend { -} -start - - -client c1 { - txreq - rxhdrs - rxchunk - sema r1 sync 2 -} -run - -delay 2 - -server s1 { - rxreq - txresp -} -start - -client c1 { - txreq - rxresp - expect resp.status == 200 -} -run diff --git a/bin/varnishtest/tests/r01391.vtc b/bin/varnishtest/tests/r01391.vtc new file mode 100644 index 0000000..3858558 --- /dev/null +++ b/bin/varnishtest/tests/r01391.vtc @@ -0,0 +1,40 @@ +varnishtest "client abandoning hit-for-pass" + + +server s1 { + non-fatal + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" -hdr "Set-Cookie: foo=bar" + chunked "foo" + sema r1 sync 2 + chunked "bar" + delay .1 + chunkedlen 64 + delay .1 + chunkedlen 64 + chunkedlen 0 +} -start + +varnish v1 -vcl+backend { +} -start + + +client c1 { + txreq + rxresphdrs + rxchunk + sema r1 sync 2 +} -run + +delay 2 + +server s1 { + rxreq + txresp +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run From phk at FreeBSD.org Mon Sep 1 08:49:32 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 01 Sep 2014 10:49:32 +0200 Subject: [master] 6a989bc Don't expect the workspace length to be aligned. Message-ID: commit 6a989bc91aee29a653ab212c95afa9665abe5d3b Author: Poul-Henning Kamp Date: Mon Sep 1 08:48:47 2014 +0000 Don't expect the workspace length to be aligned. diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index 3930c4d..91613b5 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -75,8 +75,8 @@ WS_Init(struct ws *ws, const char *id, void *space, unsigned len) ws->magic = WS_MAGIC; ws->s = space; assert(PAOK(space)); + len = PRNDDN(len); ws->e = ws->s + len; - assert(PAOK(len)); ws->f = ws->s; assert(id[0] & 0x40); assert(strlen(id) < sizeof ws->id); From phk at FreeBSD.org Mon Sep 1 09:05:55 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 01 Sep 2014 11:05:55 +0200 Subject: [master] a175e2e Bump the minimum workspace_client to 9k. Message-ID: commit a175e2ed2c6022b27d5310b8cb111ba78114df48 Author: Poul-Henning Kamp Date: Mon Sep 1 08:57:24 2014 +0000 Bump the minimum workspace_client to 9k. diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index 55889bc..52d25dc 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -92,7 +92,7 @@ struct parspec mgt_parspec[] = { "384", "bytes" }, { "workspace_client", tweak_bytes_u, &mgt_param.workspace_client, - "3072", NULL, + "9k", NULL, "Bytes of HTTP protocol workspace for clients HTTP req/resp." " If larger than 4k, use a multiple of 4k for VM efficiency.", DELAYED_EFFECT, From phk at FreeBSD.org Mon Sep 1 09:17:21 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 01 Sep 2014 11:17:21 +0200 Subject: [master] 5bd0f78 These two disabled testcases don't make sense any more Message-ID: commit 5bd0f781482119a911e8e5b2d2f05f3a249de1f9 Author: Poul-Henning Kamp Date: Mon Sep 1 09:17:02 2014 +0000 These two disabled testcases don't make sense any more diff --git a/bin/varnishtest/tests.disabled/r00929.vtc b/bin/varnishtest/tests.disabled/r00929.vtc deleted file mode 100644 index 6f7729d..0000000 --- a/bin/varnishtest/tests.disabled/r00929.vtc +++ /dev/null @@ -1,26 +0,0 @@ -varnishtest "stream test" - - -server s1 { - rxreq - txresp -bodylen 400 -} -start - -varnish v1 -vcl+backend { - - sub vcl_backend_response { - set beresp.do_stream = true; - } - -} -start - -client c1 { - txreq - rxresp - expect resp.bodylen == 400 - txreq - rxresp - expect resp.bodylen == 400 -} -run - - diff --git a/bin/varnishtest/tests.disabled/r00978.vtc b/bin/varnishtest/tests.disabled/r00978.vtc deleted file mode 100644 index e02be57..0000000 --- a/bin/varnishtest/tests.disabled/r00978.vtc +++ /dev/null @@ -1,40 +0,0 @@ -varnishtest "r00978.vtc Test esi_level > 0 and do_stream" - -server s1 { - rxreq - expect req.url == "/" - txresp -body { - - Before include - - After include - } - rxreq - expect req.url == "/body1" - txresp -body { - Included file - } -} -start - -varnish v1 -vcl+backend { - sub vcl_recv { - if (req.esi_level > 0) { - set req.url = req.url + req.esi_level; - } - } - sub vcl_backend_response { - if (bereq.url == "/") { - set beresp.do_esi = true; - } - set beresp.do_stream = true; - } -} -start - -client c1 { - txreq - rxresp - expect resp.bodylen == 65 - expect resp.status == 200 -} -run - -varnish v1 -expect esi_errors == 0 From phk at FreeBSD.org Mon Sep 1 09:23:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 01 Sep 2014 11:23:45 +0200 Subject: [master] 9c2e787 Resurrect this test-case Message-ID: commit 9c2e787d2780aa6a3dfb6ed31eab66cd854af320 Author: Poul-Henning Kamp Date: Mon Sep 1 09:23:35 2014 +0000 Resurrect this test-case diff --git a/bin/varnishtest/tests/r00979.vtc b/bin/varnishtest/tests/r00979.vtc new file mode 100644 index 0000000..46d7fa9 --- /dev/null +++ b/bin/varnishtest/tests/r00979.vtc @@ -0,0 +1,31 @@ +varnishtest "r00979.vtc Test restart when do_stream in vcl_deliver" + +server s1 { + rxreq + txresp -status 200 -gzipbody "1" + + accept + rxreq + txresp -status 200 -body "11" +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_response { + set beresp.do_stream = true; + set beresp.uncacheable = true; + } + sub vcl_deliver { + if (req.restarts == 0) { + return (restart); + } + } +} -start + +varnish v1 -cliok "param.set debug +syncvsl" + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.bodylen == 2 +} -run From martin at varnish-software.com Mon Sep 1 12:01:50 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 01 Sep 2014 14:01:50 +0200 Subject: [master] 8098866 Fix logexpect cursor creation to use the proper cursor options Message-ID: commit 809886601fd77bc95faf65be82aade61dc0a5bfa Author: Martin Blix Grydeland Date: Mon Sep 1 13:06:22 2014 +0200 Fix logexpect cursor creation to use the proper cursor options diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 48c8f3d..d868a1b 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -330,7 +330,7 @@ logexp_start(struct logexp *le) } le->vsl = VSL_New(); AN(le->vsl); - c = VSL_CursorVSM(le->vsl, le->vsm, !le->d_arg); + c = VSL_CursorVSM(le->vsl, le->vsm, le->d_arg ? 0 : VSL_COPT_TAIL); if (c == NULL) { vtc_log(le->vl, 0, "VSL_CursorVSM: %s", VSL_Error(le->vsl)); logexp_close(le); From martin at varnish-software.com Mon Sep 1 12:23:21 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 01 Sep 2014 14:23:21 +0200 Subject: [master] 5f57983 Don't log SLT_ReqAcct when we don't have assigned a VXID yet. Message-ID: commit 5f57983a05d70b83d7f89e018541e7bfbdc8d781 Author: Martin Blix Grydeland Date: Mon Sep 1 14:22:34 2014 +0200 Don't log SLT_ReqAcct when we don't have assigned a VXID yet. This prevents SLT_ReqAcct log records with 0 as VXID. diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 14b45cf..bd9b696 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -934,7 +934,7 @@ CNT_AcctLogCharge(struct dstat *ds, struct req *req) a = &req->acct; - if (!(req->res_mode & RES_PIPE)) { + if (req->vsl->wid && !(req->res_mode & RES_PIPE)) { VSLb(req->vsl, SLT_ReqAcct, "%ju %ju %ju %ju %ju %ju", (uintmax_t)a->req_hdrbytes, (uintmax_t)a->req_bodybytes, From lkarsten at varnish-software.com Mon Sep 1 13:16:20 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 01 Sep 2014 15:16:20 +0200 Subject: [3.0] 50ab8ab Require a running Varnish before starting. Message-ID: commit 50ab8ab595df780bc2c5d6ff4f349e6476771990 Author: Lasse Karstensen Date: Mon Sep 1 15:15:40 2014 +0200 Require a running Varnish before starting. diff --git a/varnish.varnishlog.init b/varnish.varnishlog.init index ee8fd9c..1574ace 100644 --- a/varnish.varnishlog.init +++ b/varnish.varnishlog.init @@ -2,8 +2,8 @@ ### BEGIN INIT INFO # Provides: varnishlog -# Required-Start: $local_fs $remote_fs $network -# Required-Stop: $local_fs $remote_fs $network +# Required-Start: $local_fs $remote_fs $network varnish +# Required-Stop: $local_fs $remote_fs $network varnish # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Start HTTP accelerator log daemon diff --git a/varnish.varnishncsa.init b/varnish.varnishncsa.init index 8504fce..1d098ad 100644 --- a/varnish.varnishncsa.init +++ b/varnish.varnishncsa.init @@ -2,8 +2,8 @@ ### BEGIN INIT INFO # Provides: varnishncsa -# Required-Start: $local_fs $remote_fs $network -# Required-Stop: $local_fs $remote_fs $network +# Required-Start: $local_fs $remote_fs $network varnish +# Required-Stop: $local_fs $remote_fs $network varnish # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Start HTTP accelerator log daemon From lkarsten at varnish-software.com Mon Sep 1 13:16:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 01 Sep 2014 15:16:26 +0200 Subject: [master] 0241c9f Require a running Varnish before starting. Message-ID: commit 0241c9fb8e98fd8a06523bdbffbe85ed4ac8cd0f Author: Lasse Karstensen Date: Mon Sep 1 15:15:40 2014 +0200 Require a running Varnish before starting. diff --git a/varnish.varnishlog.init b/varnish.varnishlog.init index ee8fd9c..1574ace 100644 --- a/varnish.varnishlog.init +++ b/varnish.varnishlog.init @@ -2,8 +2,8 @@ ### BEGIN INIT INFO # Provides: varnishlog -# Required-Start: $local_fs $remote_fs $network -# Required-Stop: $local_fs $remote_fs $network +# Required-Start: $local_fs $remote_fs $network varnish +# Required-Stop: $local_fs $remote_fs $network varnish # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Start HTTP accelerator log daemon diff --git a/varnish.varnishncsa.init b/varnish.varnishncsa.init index 8504fce..1d098ad 100644 --- a/varnish.varnishncsa.init +++ b/varnish.varnishncsa.init @@ -2,8 +2,8 @@ ### BEGIN INIT INFO # Provides: varnishncsa -# Required-Start: $local_fs $remote_fs $network -# Required-Stop: $local_fs $remote_fs $network +# Required-Start: $local_fs $remote_fs $network varnish +# Required-Stop: $local_fs $remote_fs $network varnish # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Start HTTP accelerator log daemon From phk at FreeBSD.org Mon Sep 1 14:47:07 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 01 Sep 2014 16:47:07 +0200 Subject: [master] 8ced67a Resurrect another test-case Message-ID: commit 8ced67ad6fc6b8489148d2b5094ce2f53c2a6f5e Author: Poul-Henning Kamp Date: Mon Sep 1 09:30:30 2014 +0000 Resurrect another test-case diff --git a/bin/varnishtest/tests.disabled/r00979.vtc b/bin/varnishtest/tests.disabled/r00979.vtc deleted file mode 100644 index 432aea5..0000000 --- a/bin/varnishtest/tests.disabled/r00979.vtc +++ /dev/null @@ -1,29 +0,0 @@ -varnishtest "r00979.vtc Test restart when do_stream in vcl_deliver" - -server s1 { - rxreq - txresp -status 200 -gzipbody "1" - expect_close - - accept - rxreq - txresp -status 200 -body "11" -} -start - -varnish v1 -vcl+backend { - sub vcl_backend_response { - set beresp.do_stream = true; - } - sub vcl_deliver { - if (req.restarts == 0) { - return (restart); - } - } -} -start - -client c1 { - txreq - rxresp - expect resp.status == 200 - expect resp.bodylen == 2 -} -run diff --git a/bin/varnishtest/tests.disabled/r00980.vtc b/bin/varnishtest/tests.disabled/r00980.vtc deleted file mode 100644 index 006de3e..0000000 --- a/bin/varnishtest/tests.disabled/r00980.vtc +++ /dev/null @@ -1,29 +0,0 @@ -varnishtest "r00980 test gzip on fetch with content_length and do_stream" - -server s1 { - rxreq - expect req.url == "/foobar" - expect req.http.accept-encoding == "gzip" - txresp -bodylen 43 -} -start - -varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend { - - sub vcl_backend_response { - set beresp.do_gzip = true; - set beresp.do_stream = true; - } -} -start - -client c1 { - txreq -url /foobar -hdr "Accept-Encoding: gzip" - rxresp - expect resp.http.content-encoding == "gzip" - gunzip - expect resp.bodylen == 43 - - txreq -url /foobar - rxresp - expect resp.http.content-encoding == - expect resp.bodylen == 43 -} -run diff --git a/bin/varnishtest/tests/r00980.vtc b/bin/varnishtest/tests/r00980.vtc new file mode 100644 index 0000000..006de3e --- /dev/null +++ b/bin/varnishtest/tests/r00980.vtc @@ -0,0 +1,29 @@ +varnishtest "r00980 test gzip on fetch with content_length and do_stream" + +server s1 { + rxreq + expect req.url == "/foobar" + expect req.http.accept-encoding == "gzip" + txresp -bodylen 43 +} -start + +varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend { + + sub vcl_backend_response { + set beresp.do_gzip = true; + set beresp.do_stream = true; + } +} -start + +client c1 { + txreq -url /foobar -hdr "Accept-Encoding: gzip" + rxresp + expect resp.http.content-encoding == "gzip" + gunzip + expect resp.bodylen == 43 + + txreq -url /foobar + rxresp + expect resp.http.content-encoding == + expect resp.bodylen == 43 +} -run From phk at FreeBSD.org Mon Sep 1 14:47:07 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 01 Sep 2014 16:47:07 +0200 Subject: [master] 39805b6 Don't rename the VSM file into place until we have started the child process during startup. Message-ID: commit 39805b604e842b15059f31d56b71277cd8a4b7fb Author: Poul-Henning Kamp Date: Mon Sep 1 14:46:33 2014 +0000 Don't rename the VSM file into place until we have started the child process during startup. Fixes #1407 diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 4f4a8f6..c57ac27 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -107,6 +107,7 @@ void mgt_SHM_Init(void); void mgt_SHM_static_alloc(const void *, ssize_t size, const char *class, const char *type, const char *ident); void mgt_SHM_Create(void); +void mgt_SHM_Commit(void); void mgt_SHM_Destroy(int keep); void mgt_SHM_Size_Adjust(void); diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 1fe8473..7c777de 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -570,6 +570,7 @@ mgt_reap_child(void) mgt_SHM_Destroy(0); } mgt_SHM_Create(); + mgt_SHM_Commit(); if (child_state == CH_RUNNING) child_state = CH_DIED; @@ -752,13 +753,15 @@ MGT_Run(void) REPORT0(LOG_ERR, "No VCL loaded yet"); else if (!d_flag) { mgt_launch_child(NULL); - if (child_state == CH_STOPPED) { + if (child_state != CH_RUNNING) { // XXX correct? or 0? exit_status = 2; return; } } + mgt_SHM_Commit(); + i = vev_schedule(mgt_evb); if (i != 0) REPORT(LOG_ERR, "vev_schedule() = %d", i); diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c index ed020f7..dc1fa5c 100644 --- a/bin/varnishd/mgt/mgt_shmem.c +++ b/bin/varnishd/mgt/mgt_shmem.c @@ -246,17 +246,28 @@ mgt_SHM_Create(void) AN(VSC_C_mgt); *VSC_C_mgt = static_VSC_C_mgt; +#ifdef __OpenBSD__ + /* Commit changes, for OS's without coherent VM/buf */ + AZ(msync(p, getpagesize(), MS_SYNC)); +#endif +} + +/*-------------------------------------------------------------------- + * Commit the VSM + */ + +void +mgt_SHM_Commit(void) +{ + char fnbuf[64]; + + bprintf(fnbuf, "%s.%jd", VSM_FILENAME, (intmax_t)getpid()); if (rename(fnbuf, VSM_FILENAME)) { fprintf(stderr, "Rename failed %s -> %s: %s\n", fnbuf, VSM_FILENAME, strerror(errno)); (void)unlink(fnbuf); exit(1); } - -#ifdef __OpenBSD__ - /* Commit changes, for OS's without coherent VM/buf */ - AZ(msync(p, getpagesize(), MS_SYNC)); -#endif } /*-------------------------------------------------------------------- From lkarsten at varnish-software.com Tue Sep 2 10:42:07 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Tue, 02 Sep 2014 12:42:07 +0200 Subject: [master] bb9d787 Don't add a trailing dash to URL if missing path. Message-ID: commit bb9d78797a8e163f533ccaea3686b5340a19987d Author: Lasse Karstensen Date: Tue Sep 2 12:29:47 2014 +0200 Don't add a trailing dash to URL if missing path. Adding a dash if missing the path component does not make sense. Related bug: #1584 diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 685d869..59c1918 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -300,7 +300,7 @@ format_requestline(const struct format *format) AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_host], NULL)); } else AZ(VSB_cat(CTX.vsb, "http://localhost")); - AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_U], "-")); + AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_U], NULL)); AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_q], "")); AZ(VSB_putc(CTX.vsb, ' ')); AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_H], "HTTP/1.0")); From lkarsten at varnish-software.com Tue Sep 2 10:42:07 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Tue, 02 Sep 2014 12:42:07 +0200 Subject: [master] 39f6f36 Do not log garbage requests. Message-ID: commit 39f6f3622fd4b880c4149c576b44ce4ed475d6ba Author: Lasse Karstensen Date: Tue Sep 2 12:33:16 2014 +0200 Do not log garbage requests. Requests that end up in the hard "400 Bad Request" handling used to be logged with incomplete data. (no method, maybe no path, possibly no proto, and no response status) Port scans or anything sending a byte and a linefeed would be logged. Since this is used for logging access to a web site, it makes more sense to skip these garbage requests. Fixes: #1584 diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 59c1918..a6658e3 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -760,6 +760,9 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], e--; switch (tag) { + case SLT_HttpGarbage: + skip = 1; + break; case SLT_PipeAcct: frag_fields(b, e, 3, &CTX.frag[F_I], From lkarsten at varnish-software.com Tue Sep 2 11:48:38 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Tue, 02 Sep 2014 13:48:38 +0200 Subject: [master] 3ee5325 Fix assertion error introduced in bb9d7879. Message-ID: commit 3ee53259917b9aef37f4be08be273650b9a53968 Author: Lasse Karstensen Date: Tue Sep 2 13:45:28 2014 +0200 Fix assertion error introduced in bb9d7879. diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index a6658e3..162829d 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -300,7 +300,7 @@ format_requestline(const struct format *format) AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_host], NULL)); } else AZ(VSB_cat(CTX.vsb, "http://localhost")); - AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_U], NULL)); + AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_U], "")); AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_q], "")); AZ(VSB_putc(CTX.vsb, ' ')); AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_H], "HTTP/1.0")); From phk at FreeBSD.org Tue Sep 2 20:41:00 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 02 Sep 2014 22:41:00 +0200 Subject: [master] cf3e395 Remove noisy debugging Message-ID: commit cf3e395f69bfb95b44caaec924ee4eba7ed3b5b5 Author: Poul-Henning Kamp Date: Tue Sep 2 20:40:53 2014 +0000 Remove noisy debugging diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 754438a..f4de5f3 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -334,7 +334,6 @@ pool_stat_summ(struct worker *wrk, void *priv) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(wrk->pool, POOL_MAGIC); - VSL(SLT_Debug, 0, "STATSUMM"); AN(priv); src = priv; Lck_Lock(&wstat_mtx); From phk at FreeBSD.org Wed Sep 3 09:32:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 03 Sep 2014 11:32:47 +0200 Subject: [master] 9a7eb1a Pinpoint sanity-checks of vfp->ctx to only the two place it is still hackishly used. Message-ID: commit 9a7eb1a64371cdecd9f1b996b3e1ecaf66bca701 Author: Poul-Henning Kamp Date: Wed Sep 3 09:32:15 2014 +0000 Pinpoint sanity-checks of vfp->ctx to only the two place it is still hackishly used. diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index 2963b95..ce76cec 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -65,7 +65,6 @@ vfp_vep_callback(struct vfp_ctx *vc, void *priv, ssize_t l, enum vgz_flag flg) int i; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vc->bo, BUSYOBJ_MAGIC); CAST_OBJ_NOTNULL(vef, priv, VEF_MAGIC); assert(l >= 0); @@ -82,6 +81,7 @@ vfp_vep_callback(struct vfp_ctx *vc, void *priv, ssize_t l, enum vgz_flag flg) if (l == 0 && flg == VGZ_NORMAL) return (vef->tot); + CHECK_OBJ_NOTNULL(vc->bo, BUSYOBJ_MAGIC); VGZ_Ibuf(vef->vgz, vef->ibuf_o, l); do { dl = 0; diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index 0083510..3db8287 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -58,7 +58,6 @@ VFP_Error(struct vfp_ctx *vc, const char *fmt, ...) va_list ap; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - assert(vc->bo->state >= BOS_REQ_DONE); if (!vc->failed) { va_start(ap, fmt); VSLbv(vc->vsl, SLT_FetchError, fmt, ap); @@ -79,7 +78,6 @@ VFP_GetStorage(struct vfp_ctx *vc, ssize_t *sz, uint8_t **ptr) ssize_t l; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vc->bo, BUSYOBJ_MAGIC); AN(sz); assert(*sz >= 0); AN(ptr); diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index e05be30..99fc00f 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -409,6 +409,7 @@ ObjSetattr(const struct vfp_ctx *vc, enum obj_attr attr, ssize_t len, void *retval = NULL; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vc->bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(vc->oc, OBJCORE_MAGIC); o = obj_getobj(vc->oc, vc->stats); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); From phk at FreeBSD.org Wed Sep 3 10:03:07 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 03 Sep 2014 12:03:07 +0200 Subject: [master] 4fdd324 Protect this test-case against a streaming related race. Message-ID: commit 4fdd32471d6301f2c730e61693b6d9af8565d79f Author: Poul-Henning Kamp Date: Wed Sep 3 10:02:47 2014 +0000 Protect this test-case against a streaming related race. diff --git a/bin/varnishtest/tests/r01399.vtc b/bin/varnishtest/tests/r01399.vtc index 48a0bc0..5d7171a 100644 --- a/bin/varnishtest/tests/r01399.vtc +++ b/bin/varnishtest/tests/r01399.vtc @@ -23,6 +23,7 @@ server s1 { varnish v1 -vcl+backend { sub vcl_backend_response { + set beresp.do_stream = false; set beresp.ttl = 2s; set beresp.grace = 1800s; } From phk at FreeBSD.org Wed Sep 3 10:10:35 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 03 Sep 2014 12:10:35 +0200 Subject: [master] 3063060 Reduce reliance on busyobj in preference of vfp_ctx. Message-ID: commit 3063060f4a47d62a9217ab9df5bd665a25ecf9c0 Author: Poul-Henning Kamp Date: Wed Sep 3 10:10:13 2014 +0000 Reduce reliance on busyobj in preference of vfp_ctx. diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index 3db8287..b44abd4 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -190,10 +190,13 @@ VFP_Fetch_Body(struct busyobj *bo) uint8_t *ptr; enum vfp_status vfps = VFP_ERROR; ssize_t est; + struct vfp_ctx *vfc; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + vfc = bo->vfc; + CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC); - AN(bo->vfc->vfp_nxt); + AN(vfc->vfp_nxt); est = bo->content_length; if (est < 0) @@ -206,23 +209,23 @@ VFP_Fetch_Body(struct busyobj *bo) * We don't fail the fetch, in order for hit-for-pass * objects to be created. */ - AN(bo->fetch_objcore->flags & OC_F_PASS); - VSLb(bo->vsl, SLT_FetchError, + AN(vfc->oc->flags & OC_F_PASS); + VSLb(vfc->vsl, SLT_FetchError, "Pass delivery abandoned"); vfps = VFP_END; bo->doclose = SC_RX_BODY; break; } - AZ(bo->vfc->failed); + AZ(vfc->failed); l = est; assert(l >= 0); - if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK) { + if (VFP_GetStorage(vfc, &l, &ptr) != VFP_OK) { bo->doclose = SC_RX_BODY; break; } - AZ(bo->vfc->failed); - vfps = VFP_Suck(bo->vfc, ptr, &l); + AZ(vfc->failed); + vfps = VFP_Suck(vfc, ptr, &l); if (l > 0 && vfps != VFP_ERROR) { VBO_extend(bo, l); if (est >= l) @@ -233,15 +236,15 @@ VFP_Fetch_Body(struct busyobj *bo) } while (vfps == VFP_OK); if (vfps == VFP_ERROR) { - AN(bo->vfc->failed); - (void)VFP_Error(bo->vfc, "Fetch Pipeline failed to process"); + AN(vfc->failed); + (void)VFP_Error(vfc, "Fetch Pipeline failed to process"); bo->doclose = SC_RX_BODY; } - vfp_suck_fini(bo->vfc); + vfp_suck_fini(vfc); if (!bo->do_stream) - ObjTrimStore(bo->fetch_objcore, bo->stats); + ObjTrimStore(vfc->oc, bo->stats); } struct vfp_entry * From phk at FreeBSD.org Wed Sep 3 10:57:06 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 03 Sep 2014 12:57:06 +0200 Subject: [master] e6e9569 Repatriate VFP_Fetch_Body() with cache_fetch.c, it's not really a VFP function but rather a fetch function calling VFP. Message-ID: commit e6e956995d50a26a3d9703eecc4189ac1d70a5b1 Author: Poul-Henning Kamp Date: Wed Sep 3 10:56:23 2014 +0000 Repatriate VFP_Fetch_Body() with cache_fetch.c, it's not really a VFP function but rather a fetch function calling VFP. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 08e8714..4e7046f 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -910,7 +910,6 @@ void VBF_Fetch(struct worker *wrk, struct req *req, /* cache_fetch_proc.c */ enum vfp_status VFP_GetStorage(struct vfp_ctx *, ssize_t *sz, uint8_t **ptr); void VFP_Init(void); -void VFP_Fetch_Body(struct busyobj *bo); /* cache_gzip.c */ struct vgz; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 001cd79..b4c4457 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -393,6 +393,73 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) /*-------------------------------------------------------------------- */ +static void +vbf_fetch_body_helper(struct busyobj *bo) +{ + ssize_t l; + uint8_t *ptr; + enum vfp_status vfps = VFP_ERROR; + ssize_t est; + struct vfp_ctx *vfc; + + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + vfc = bo->vfc; + CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC); + + AN(vfc->vfp_nxt); + + est = bo->content_length; + if (est < 0) + est = 0; + + do { + if (bo->abandon) { + /* + * A pass object and delivery was terminted + * We don't fail the fetch, in order for hit-for-pass + * objects to be created. + */ + AN(vfc->oc->flags & OC_F_PASS); + VSLb(vfc->vsl, SLT_FetchError, + "Pass delivery abandoned"); + vfps = VFP_END; + bo->doclose = SC_RX_BODY; + break; + } + AZ(vfc->failed); + l = est; + assert(l >= 0); + if (VFP_GetStorage(vfc, &l, &ptr) != VFP_OK) { + bo->doclose = SC_RX_BODY; + break; + } + + AZ(vfc->failed); + vfps = VFP_Suck(vfc, ptr, &l); + if (l > 0 && vfps != VFP_ERROR) { + VBO_extend(bo, l); + if (est >= l) + est -= l; + else + est = 0; + } + } while (vfps == VFP_OK); + + if (vfps == VFP_ERROR) { + AN(vfc->failed); + (void)VFP_Error(vfc, "Fetch Pipeline failed to process"); + bo->doclose = SC_RX_BODY; + } + + VFP_Close(vfc); + + if (!bo->do_stream) + ObjTrimStore(vfc->oc, bo->stats); +} + +/*-------------------------------------------------------------------- + */ + static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { @@ -512,7 +579,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) if (bo->htc.body_status != BS_NONE) { assert(bo->htc.body_status != BS_ERROR); - VFP_Fetch_Body(bo); + vbf_fetch_body_helper(bo); bo->acct.beresp_bodybytes = bo->vfc->bodybytes; } diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index b44abd4..b8ee0b9 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -113,8 +113,8 @@ VFP_Setup(struct vfp_ctx *vc) /********************************************************************** */ -static void -vfp_suck_fini(struct vfp_ctx *vc) +void +VFP_Close(struct vfp_ctx *vc) { struct vfp_entry *vfe; @@ -136,7 +136,7 @@ VFP_Open(struct vfp_ctx *vc) if (vfe->closed != VFP_OK && vfe->closed != VFP_NULL) { (void)VFP_Error(vc, "Fetch filter %s failed to open", vfe->vfp->name); - vfp_suck_fini(vc); + VFP_Close(vc); return (-1); } } @@ -182,71 +182,6 @@ VFP_Suck(struct vfp_ctx *vc, void *p, ssize_t *lp) /*-------------------------------------------------------------------- */ - -void -VFP_Fetch_Body(struct busyobj *bo) -{ - ssize_t l; - uint8_t *ptr; - enum vfp_status vfps = VFP_ERROR; - ssize_t est; - struct vfp_ctx *vfc; - - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - vfc = bo->vfc; - CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC); - - AN(vfc->vfp_nxt); - - est = bo->content_length; - if (est < 0) - est = 0; - - do { - if (bo->abandon) { - /* - * A pass object and delivery was terminted - * We don't fail the fetch, in order for hit-for-pass - * objects to be created. - */ - AN(vfc->oc->flags & OC_F_PASS); - VSLb(vfc->vsl, SLT_FetchError, - "Pass delivery abandoned"); - vfps = VFP_END; - bo->doclose = SC_RX_BODY; - break; - } - AZ(vfc->failed); - l = est; - assert(l >= 0); - if (VFP_GetStorage(vfc, &l, &ptr) != VFP_OK) { - bo->doclose = SC_RX_BODY; - break; - } - - AZ(vfc->failed); - vfps = VFP_Suck(vfc, ptr, &l); - if (l > 0 && vfps != VFP_ERROR) { - VBO_extend(bo, l); - if (est >= l) - est -= l; - else - est = 0; - } - } while (vfps == VFP_OK); - - if (vfps == VFP_ERROR) { - AN(vfc->failed); - (void)VFP_Error(vfc, "Fetch Pipeline failed to process"); - bo->doclose = SC_RX_BODY; - } - - vfp_suck_fini(vfc); - - if (!bo->do_stream) - ObjTrimStore(vfc->oc, bo->stats); -} - struct vfp_entry * VFP_Push(struct vfp_ctx *vc, const struct vfp *vfp, int top) { diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 584bf77..a5a705a 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -64,6 +64,7 @@ extern const struct vfp vfp_esi_gzip; struct vfp_entry *VFP_Push(struct vfp_ctx *, const struct vfp *, int top); void VFP_Setup(struct vfp_ctx *vc); int VFP_Open(struct vfp_ctx *bo); +void VFP_Close(struct vfp_ctx *bo); enum vfp_status VFP_Suck(struct vfp_ctx *, void *p, ssize_t *lp); enum vfp_status VFP_Error(struct vfp_ctx *, const char *fmt, ...) __printflike(2, 3); From phk at FreeBSD.org Wed Sep 3 11:22:31 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 03 Sep 2014 13:22:31 +0200 Subject: [master] 233b392 push busyobj out of V1F fetching Message-ID: commit 233b39282d2a30b869e072d4cc38977d46aa1974 Author: Poul-Henning Kamp Date: Wed Sep 3 11:22:09 2014 +0000 push busyobj out of V1F fetching diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 4e7046f..4c0e6d2 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -809,7 +809,7 @@ void VBO_waitstate(struct busyobj *bo, enum busyobj_state_e want); /* cache_http1_fetch.c [V1F] */ int V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req); -void V1F_Setup_Fetch(struct busyobj *bo); +void V1F_Setup_Fetch(struct vfp_ctx *vfc, ssize_t cl, struct http_conn *htc); /* cache_http1_fsm.c [HTTP1] */ typedef int (req_body_iter_f)(struct req *, void *priv, void *ptr, size_t); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index b4c4457..24ba95d 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -556,7 +556,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) ObjSetFlag(bo->vfc, OF_IMSCAND, 1); if (bo->htc.body_status != BS_NONE) - V1F_Setup_Fetch(bo); + V1F_Setup_Fetch(bo->vfc, bo->content_length, &bo->htc); /* * Ready to fetch the body diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index b622c3c..b6657d5 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -153,39 +153,34 @@ static const struct vfp v1f_eof = { */ void -V1F_Setup_Fetch(struct busyobj *bo) +V1F_Setup_Fetch(struct vfp_ctx *vfc, ssize_t cl, struct http_conn *htc) { - struct http_conn *htc; struct vfp_entry *vfe; - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - htc = &bo->htc; + CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); - CHECK_OBJ_NOTNULL(bo->vbc, VBC_MAGIC); switch(htc->body_status) { case BS_EOF: - assert(bo->content_length == -1); - vfe = VFP_Push(bo->vfc, &v1f_eof, 0); - vfe->priv1 = &bo->htc; + assert(cl == -1); + vfe = VFP_Push(vfc, &v1f_eof, 0); vfe->priv2 = 0; break; case BS_LENGTH: - assert(bo->content_length > 0); - vfe = VFP_Push(bo->vfc, &v1f_straight, 0); - vfe->priv1 = &bo->htc; - vfe->priv2 = bo->content_length; + assert(cl > 0); + vfe = VFP_Push(vfc, &v1f_straight, 0); + vfe->priv2 = cl; break; case BS_CHUNKED: - assert(bo->content_length == -1); - vfe = VFP_Push(bo->vfc, &v1f_chunked, 0); - vfe->priv1 = &bo->htc; + assert(cl == -1); + vfe = VFP_Push(vfc, &v1f_chunked, 0); vfe->priv2 = -1; break; default: WRONG("Wrong body_status"); break; } + vfe->priv1 = htc; return; } From fgsch at lodoss.net Wed Sep 3 14:53:40 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 03 Sep 2014 16:53:40 +0200 Subject: [master] 5c798e2 Improve examples and reduce duplicated information Message-ID: commit 5c798e215dedf244249d4aa6d82a1aaa3fe453d9 Author: Federico G. Schwindt Date: Wed Sep 3 14:52:37 2014 +0100 Improve examples and reduce duplicated information diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 6828f99..90ac019 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -213,6 +213,8 @@ are available: Backends can be used with *directors*. Please see the vmod_directors(3) man page for more information. +.. _reference-vcl_probes: + Probes ------ diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst index 5d31e77..1a1f556 100644 --- a/doc/sphinx/users-guide/vcl-backends.rst +++ b/doc/sphinx/users-guide/vcl-backends.rst @@ -156,14 +156,14 @@ Health checks ------------- Lets set up a director with two backends and health checks. First let -us define the backends.:: +us define the backends:: backend server1 { .host = "server1.example.com"; .probe = { .url = "/"; + .timeout = 1s; .interval = 5s; - .timeout = 1 s; .window = 5; .threshold = 3; } @@ -173,40 +173,23 @@ us define the backends.:: .host = "server2.example.com"; .probe = { .url = "/"; + .timeout = 1s; .interval = 5s; - .timeout = 1 s; .window = 5; .threshold = 3; } } -What is new here is the ``probe``. Varnish will check the health of each -backend with a probe. The options are: - -url - The URL Varnish will use to send a probe request. - -interval - How often should we poll. - -timeout - What is the timeout of the probe. - -window - Varnish will retain up to this many probes when considering backend health. - -threshold - How many of the `.window` last polls must be good for the backend to be - declared healthy. - -initial - How many of the probes that needs to be succesful when Varnish starts. - Defaults to the same amount as the threshold. +What is new here is the ``probe``. In this example Varnish will check the +health of each backend every 5 seconds, timing out after 1 second. Each +poll will send a GET request to /. If 3 out of the last 5 polls succeeded +the backend is considered healthy, otherwise it will be marked as sick. +Refer to the :ref:`reference-vcl_probes` section in the +:ref:`reference-vcl` documentation for more information. Now we define the 'director':: - import directors; sub vcl_init { @@ -215,17 +198,16 @@ Now we define the 'director':: vdir.add_backend(server2); } +You use this `vdir` director as a backend_hint for requests, just like +you would with a simple backend. Varnish will not send traffic to hosts +that are marked as unhealthy. -You use this `vdir` director as a backend_hint for requests, just like you would -with a simple backend. Varnish will not send traffic to hosts that are marked as -unhealthy. - -Varnish can also serve stale content if all the backends are -down. See :ref:`users-guide-handling_misbehaving_servers` for more -information on how to enable this. - -Please note that Varnish will keep health probes running for all loaded VCLs. Varnish -will coalesce probes that seem identical - so be careful not to change the -probe config if you do a lot of VCL loading. Unloading the VCL will discard the -probes. For more information on how to do this please see ref:`reference-vcl-director`. +Varnish can also serve stale content if all the backends are down. See +:ref:`users-guide-handling_misbehaving_servers` for more information on +how to enable this. +Please note that Varnish will keep health probes running for all loaded +VCLs. Varnish will coalesce probes that seem identical - so be careful +not to change the probe config if you do a lot of VCL loading. Unloading +the VCL will discard the probes. For more information on how to do this +please see ref:`reference-vcl-director`. From lkarsten at varnish-software.com Wed Sep 3 15:22:59 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Wed, 03 Sep 2014 17:22:59 +0200 Subject: [master] b328d4e Double allocated varnishlog buffer space. Message-ID: commit b328d4e5771df5d7a2e34796d5aceaa5c2be440a Author: Lasse Karstensen Date: Wed Sep 3 17:18:08 2014 +0200 Double allocated varnishlog buffer space. Extensive test cases often lead to a lot of log output, which at times tripped the following assert: Assert error in vtc_log_emit(), vtc_log.c line 122: Condition(vtclog_left > l) not true. Double the amount of buffer space allowed to avoid this. Another 256KB memory use per concurrent running test should be fine. diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 823a0a2..6fdfb6f 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -260,7 +260,7 @@ start_test(void) ALLOC_OBJ(jp, JOB_MAGIC); AN(jp); - jp->bufsiz = 256*1024; /* XXX */ + jp->bufsiz = 512*1024; /* XXX */ jp->buf = mmap(NULL, jp->bufsiz, PROT_READ|PROT_WRITE, MAP_ANON | MAP_SHARED, -1, 0); From lkarsten at varnish-software.com Wed Sep 3 15:22:59 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Wed, 03 Sep 2014 17:22:59 +0200 Subject: [master] 92fd590 Merge branch 'master' of git.varnish-cache.org:varnish-cache Message-ID: commit 92fd590bf8940a97df5e90a3357ad40c18e16c0a Merge: b328d4e 5c798e2 Author: Lasse Karstensen Date: Wed Sep 3 17:22:38 2014 +0200 Merge branch 'master' of git.varnish-cache.org:varnish-cache From lkarsten at varnish-software.com Wed Sep 3 15:24:17 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Wed, 03 Sep 2014 17:24:17 +0200 Subject: [3.0] 9511eef Double allocated varnishlog buffer space. Message-ID: commit 9511eefcdafea9a6c87f83a46d5e2b1d41c1c2c9 Author: Lasse Karstensen Date: Wed Sep 3 17:18:08 2014 +0200 Double allocated varnishlog buffer space. Extensive test cases often lead to a lot of log output, which at times tripped the following assert: Assert error in vtc_log_emit(), vtc_log.c line 122: Condition(vtclog_left > l) not true. Double the amount of buffer space allowed to avoid this. Another 256KB memory use per concurrent running test should be fine. diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 76fe2b2..c1bf731 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -262,7 +262,7 @@ start_test(void) ALLOC_OBJ(jp, JOB_MAGIC); AN(jp); - jp->bufsiz = 256*1024; /* XXX */ + jp->bufsiz = 512*1024; /* XXX */ jp->buf = mmap(NULL, jp->bufsiz, PROT_READ|PROT_WRITE, MAP_ANON | MAP_SHARED, -1, 0); From lkarsten at varnish-software.com Thu Sep 4 11:00:04 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Thu, 04 Sep 2014 13:00:04 +0200 Subject: [master] d63c53e Squash warnings for warn_unused_result. Message-ID: commit d63c53e803b0022f43ead1d25acd64e0fad39949 Author: Lasse Karstensen Date: Thu Sep 4 12:56:47 2014 +0200 Squash warnings for warn_unused_result. diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c index 610ca8b..5498f11 100644 --- a/lib/libvarnish/vlu.c +++ b/lib/libvarnish/vlu.c @@ -96,6 +96,7 @@ vlu_dotelnet(struct vlu *l, char *p) char *e; char tno[3]; int i; + ssize_t r = 0; e = l->buf + l->bufp; assert(p >= l->buf && p < e); @@ -120,7 +121,9 @@ vlu_dotelnet(struct vlu *l, char *p) /* Return WONT for these */ memcpy(tno, p, 3); tno[1] = (char)252; - (void)write(l->telnet, tno, 3); + r = write(l->telnet, tno, 3); + if (r != 3) + return (1); i = 3; break; default: diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 6e0f20a..7319253 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -408,6 +408,7 @@ VSL_WriteOpen(struct VSL_data *vsl, const char *name, int append, int unbuf) { const char head[] = VSL_FILE_ID; FILE* f; + size_t r = 0; f = fopen(name, append ? "a" : "w"); if (f == NULL) { @@ -417,7 +418,12 @@ VSL_WriteOpen(struct VSL_data *vsl, const char *name, int append, int unbuf) if (unbuf) setbuf(f, NULL); if (0 == ftell(f)) - fwrite(head, 1, sizeof head, f); + r = fwrite(head, 1, sizeof head, f); + if (r != sizeof head) { + vsl_diag(vsl, "%s", strerror(errno)); + fclose(f); + return (NULL); + } return (f); } From lkarsten at varnish-software.com Thu Sep 4 12:14:33 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Thu, 04 Sep 2014 14:14:33 +0200 Subject: [master] 93b6f31 Clean up mistakes from d63c53e. Message-ID: commit 93b6f318841f14d3949587124ca3c10cba4c0355 Author: Lasse Karstensen Date: Thu Sep 4 14:04:34 2014 +0200 Clean up mistakes from d63c53e. Add missing braces, avoid unnecessary variables. Noticed by: fgs diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c index 5498f11..c983442 100644 --- a/lib/libvarnish/vlu.c +++ b/lib/libvarnish/vlu.c @@ -96,7 +96,6 @@ vlu_dotelnet(struct vlu *l, char *p) char *e; char tno[3]; int i; - ssize_t r = 0; e = l->buf + l->bufp; assert(p >= l->buf && p < e); @@ -121,8 +120,7 @@ vlu_dotelnet(struct vlu *l, char *p) /* Return WONT for these */ memcpy(tno, p, 3); tno[1] = (char)252; - r = write(l->telnet, tno, 3); - if (r != 3) + if (write(l->telnet, tno, 3) != 3) return (1); i = 3; break; diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 7319253..949028f 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -408,8 +408,6 @@ VSL_WriteOpen(struct VSL_data *vsl, const char *name, int append, int unbuf) { const char head[] = VSL_FILE_ID; FILE* f; - size_t r = 0; - f = fopen(name, append ? "a" : "w"); if (f == NULL) { vsl_diag(vsl, "%s", strerror(errno)); @@ -417,13 +415,13 @@ VSL_WriteOpen(struct VSL_data *vsl, const char *name, int append, int unbuf) } if (unbuf) setbuf(f, NULL); - if (0 == ftell(f)) - r = fwrite(head, 1, sizeof head, f); - if (r != sizeof head) { + if (0 == ftell(f)) { + if (fwrite(head, 1, sizeof head, f) != sizeof head) { vsl_diag(vsl, "%s", strerror(errno)); fclose(f); return (NULL); } + } return (f); } From lkarsten at varnish-software.com Thu Sep 4 14:06:16 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Thu, 04 Sep 2014 16:06:16 +0200 Subject: [master] 0b69439 Squash further warn_unused_result warnings. Message-ID: commit 0b69439ccbabe3b77fae444ef5796d138a453067 Author: Lasse Karstensen Date: Thu Sep 4 14:45:11 2014 +0200 Squash further warn_unused_result warnings. Not being able to fchown() is normal when running varnishd unprivileged, so don't report failure unless running as root. diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index b76b016..35e3806 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -248,7 +248,10 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag, unsigned *status) *status = 2; return (NULL); } - (void)fchown(sfd, mgt_param.uid, mgt_param.gid); + if (fchown(sfd, mgt_param.uid, mgt_param.gid) != 0) + if (geteuid() == 0) + VSB_printf(sb, "Failed to change owner on %s: %s\n", + sf, strerror(errno)); AZ(close(sfd)); @@ -286,7 +289,10 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag, unsigned *status) *status = 2; return (NULL); } - (void)fchown(i, mgt_param.uid, mgt_param.gid); + if (fchown(i, mgt_param.uid, mgt_param.gid) != 0) + if (geteuid() == 0) + VSB_printf(sb, "Failed to change owner on %s: %s\n", + of, strerror(errno)); AZ(close(i)); /* Build the C-compiler command line */ From fgsch at lodoss.net Thu Sep 4 18:20:50 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 04 Sep 2014 20:20:50 +0200 Subject: [master] e10d234 Relax the vmod abi check if we are not in master Message-ID: commit e10d234cfea61acbdbc8a56e8ed6450850dd051c Author: Federico G. Schwindt Date: Thu Sep 4 19:14:22 2014 +0100 Relax the vmod abi check if we are not in master For master we retain the more strict check in case we forget to bump the API version as discussed on irc. Implementation idea from scoof. Fixes #1538 diff --git a/include/Makefile.am b/include/Makefile.am index 766eb9a..09c237d 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -96,6 +96,7 @@ MAINTAINERCLEANFILES = vcs_version.h vcs_version.h: FORCE @if [ -d "$(top_srcdir)/.git" ]; then \ V="$$(git show -s --pretty=format:%h)" \ + B="$$(git rev-parse --abbrev-ref HEAD)" \ H="$$(head -n 1 vcs_version.h 2>/dev/null || true)"; \ if [ "/* $$V */" != "$$H" ]; then \ ( \ @@ -108,7 +109,8 @@ vcs_version.h: FORCE echo ' */' ;\ echo "/* $$V */" ;\ echo '' ;\ - echo "#define VCS_Version \"$$V\"" \ + echo "#define VCS_Version \"$$V\"" ; \ + echo "#define VCS_Branch \"$$B\"" \ ) > vcs_version.h ; \ fi \ else \ @@ -116,7 +118,8 @@ vcs_version.h: FORCE ( \ echo "/* NOGIT */" ; \ echo '/* No git commit ID available, see include/Makefile.am for explanation */' ; \ - echo '#define VCS_Version "NOGIT"' \ + echo '#define VCS_Version "NOGIT"' ; \ + echo '#define VCS_Branch "NOGIT"' \ ) > vcs_version.h ; \ fi \ fi diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 4f139de..7644621 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -34,6 +34,8 @@ #include "vcc_compile.h" +#include "vcs_version.h" + #include "vmod_abi.h" #include "vrt.h" @@ -123,6 +125,15 @@ vcc_ParseImport(struct vcc *tl) vcc_ErrWhere(tl, mod); return; } + if (strcmp(VCS_Branch, "master") == 0 && + strcmp(vmd->abi, VMOD_ABI_Version) != 0) { + VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); + VSB_printf(tl->sb, "\tFile name: %s\n", fn); + VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n", + VMOD_ABI_Version, vmd->abi); + vcc_ErrWhere(tl, mod); + return; + } if (vmd->vrt_major != VRT_MAJOR_VERSION || vmd->vrt_minor > VRT_MINOR_VERSION) { VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); @@ -154,15 +165,6 @@ vcc_ParseImport(struct vcc *tl) return; } - if (strcmp(vmd->abi, VMOD_ABI_Version) != 0) { - VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", fn); - VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n", - VMOD_ABI_Version, vmd->abi); - vcc_ErrWhere(tl, mod); - return; - } - ifp = New_IniFin(tl); VSB_printf(ifp->ini, "\tif (VRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod)); From phk at FreeBSD.org Mon Sep 8 08:01:55 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 08 Sep 2014 10:01:55 +0200 Subject: [master] 62080bc Move interpretation of Content-Length to cache_http.c since HTTP/2 will also use it. Message-ID: commit 62080bc09286e7ebee31e0aceb34dca8dba398a4 Author: Poul-Henning Kamp Date: Mon Sep 8 08:01:22 2014 +0000 Move interpretation of Content-Length to cache_http.c since HTTP/2 will also use it. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 4c0e6d2..b422d9b 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -966,6 +966,7 @@ int http_GetHdrToken(const struct http *hp, const char *hdr, int http_GetHdrField(const struct http *hp, const char *hdr, const char *field, char **ptr); double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field); +ssize_t http_GetContentLength(const struct http *hp); uint16_t http_GetStatus(const struct http *hp); int http_IsStatus(const struct http *hp, int); void http_SetStatus(struct http *to, uint16_t status); diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 177919e..7e2474c 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -525,6 +525,35 @@ http_GetHdrField(const struct http *hp, const char *hdr, /*--------------------------------------------------------------------*/ +ssize_t +http_GetContentLength(const struct http *hp) +{ + ssize_t cl, cll; + char *b; + + CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); + + if (!http_GetHdr(hp, H_Content_Length, &b)) + return (-1); + cl = 0; + if (!vct_isdigit(*b)) + return (-2); + for (;vct_isdigit(*b); b++) { + cll = cl; + cl *= 10; + cl += *b - '0'; + if (cll != cl / 10) + return (-2); + } + while (vct_islws(*b)) + b++; + if (*b != '\0') + return (-2); + return (cl); +} + +/*--------------------------------------------------------------------*/ + int http_HdrIs(const struct http *hp, const char *hdr, const char *val) { diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index b7737bc..131df15 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -35,7 +35,6 @@ #include "cache.h" #include "vtim.h" -#include "vct.h" /*-------------------------------------------------------------------- * TTL and Age calculation in Varnish @@ -190,7 +189,7 @@ RFC2616_Body(struct busyobj *bo, struct dstat *stats) { struct http *hp; char *b; - ssize_t cll; + ssize_t cl; hp = bo->beresp; @@ -243,37 +242,20 @@ RFC2616_Body(struct busyobj *bo, struct dstat *stats) return (BS_ERROR); } - if (http_GetHdr(hp, H_Content_Length, &b)) { - bo->content_length = 0; - if (!vct_isdigit(*b)) { - VSLb(bo->vsl, SLT_Error, "Empty Content-Length:"); - stats->fetch_bad++; - return (BS_ERROR); - } - for (;vct_isdigit(*b); b++) { - cll = bo->content_length; - bo->content_length *= 10; - bo->content_length += *b - '0'; - if (cll > bo->content_length) { - VSLb(bo->vsl, SLT_Error, - "Content-Length: too large"); - stats->fetch_bad++; - return (BS_ERROR); - } - } - while (vct_islws(*b)) - b++; - if (*b != '\0') { - VSLb(bo->vsl, SLT_Error, - "Illegal Content-Length: (0x%02x)", *b); - stats->fetch_bad++; - return (BS_ERROR); - } + cl = http_GetContentLength(hp); + if (cl == -2) { + VSLb(bo->vsl, SLT_Error, "Bad Content-Length:"); + stats->fetch_bad++; + return (BS_ERROR); + } + if (cl > 0) { + stats->fetch_length++; + bo->content_length = cl; + return (BS_LENGTH); + } + if (cl == 0) { stats->fetch_length++; - if (bo->content_length == 0) - return (BS_NONE); - else - return (BS_LENGTH); + return (BS_NONE); } if (http_HdrIs(hp, H_Connection, "keep-alive")) { From phk at FreeBSD.org Mon Sep 8 08:18:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 08 Sep 2014 10:18:38 +0200 Subject: [master] 972273f Set the protocol-specific body status in http_conn for both req and resp. Message-ID: commit 972273fbe4d32e70e1b85e1a43e4c46d2f9188df Author: Poul-Henning Kamp Date: Mon Sep 8 08:18:15 2014 +0000 Set the protocol-specific body status in http_conn for both req and resp. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index b422d9b..1bf2918 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -218,6 +218,7 @@ struct http_conn { struct ws *ws; txt rxbuf; txt pipeline; + ssize_t content_length; enum body_status body_status; }; @@ -1000,8 +1001,8 @@ enum http1_status_e HTTP1_Reinit(struct http_conn *htc); enum http1_status_e HTTP1_Rx(struct http_conn *htc); ssize_t HTTP1_Read(struct http_conn *htc, void *d, size_t len); enum http1_status_e HTTP1_Complete(struct http_conn *htc); -uint16_t HTTP1_DissectRequest(struct req *); -uint16_t HTTP1_DissectResponse(struct http *sp, const struct http_conn *htc); +uint16_t HTTP1_DissectRequest(struct http_conn *htc, struct http *hp); +uint16_t HTTP1_DissectResponse(struct http *sp, struct http_conn *htc); unsigned HTTP1_Write(const struct worker *w, const struct http *hp, const int*); #define HTTPH(a, b, c) extern char b[]; diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index b0c5299..af7442a 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -322,10 +322,11 @@ http1_dissect(struct worker *wrk, struct req *req) wrk->vcl = NULL; HTTP_Setup(req->http, req->ws, req->vsl, SLT_ReqMethod); - req->err_code = HTTP1_DissectRequest(req); + req->err_code = HTTP1_DissectRequest(req->htc, req->http); /* If we could not even parse the request, just close */ if (req->err_code != 0) { + VSLbt(req->vsl, SLT_HttpGarbage, req->htc->rxbuf); wrk->stats.client_req_400++; r = write(req->sp->fd, r_400, strlen(r_400)); if (r > 0) diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c index 03563fd..8b872fc 100644 --- a/bin/varnishd/cache/cache_http1_proto.c +++ b/bin/varnishd/cache/cache_http1_proto.c @@ -373,6 +373,61 @@ http1_splitline(struct http *hp, const struct http_conn *htc, const int *hf) /*--------------------------------------------------------------------*/ +static enum body_status +http1_body_status(const struct http *hp, struct http_conn *htc) +{ + ssize_t cl; + char *b; + + CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); + CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); + + htc->content_length = -1; + + if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) + return (BS_CHUNKED); + + if (http_GetHdr(hp, H_Transfer_Encoding, &b)) + return (BS_ERROR); + + cl = http_GetContentLength(hp); + if (cl == -2) + return (BS_ERROR); + if (cl >= 0) { + htc->content_length = cl; + return (cl == 0 ? BS_NONE : BS_LENGTH); + } + + if (http_HdrIs(hp, H_Connection, "keep-alive")) { + /* + * Keep alive with neither TE=Chunked or C-Len is impossible. + * We assume a zero length body. + */ + return (BS_NONE); + } + + if (http_HdrIs(hp, H_Connection, "close")) { + /* + * In this case, it is safe to just read what comes. + */ + return (BS_EOF); + } + + if (hp->protover < 11) { + /* + * With no Connection header, assume EOF. + */ + return (BS_EOF); + } + + /* + * Fall back to EOF transfer. + */ + return (BS_EOF); +} + +/*--------------------------------------------------------------------*/ + static uint16_t http1_request_check_host_hdr(const struct http *hp) { @@ -453,24 +508,17 @@ http1_DoConnection(struct http *hp) /*--------------------------------------------------------------------*/ uint16_t -HTTP1_DissectRequest(struct req *req) +HTTP1_DissectRequest(struct http_conn *htc, struct http *hp) { - struct http_conn *htc; - struct http *hp; uint16_t retval; char *b, *e; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - htc = req->htc; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); - hp = req->http; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); retval = http1_splitline(hp, htc, HTTP1_Req); - if (retval != 0) { - VSLbt(req->vsl, SLT_HttpGarbage, htc->rxbuf); + if (retval != 0) return (retval); - } http1_proto_ver(hp); retval = http1_request_check_host_hdr(hp); @@ -490,6 +538,8 @@ HTTP1_DissectRequest(struct req *req) } } + htc->body_status = http1_body_status(hp, htc); + hp->doclose = http1_DoConnection(hp); return (retval); @@ -498,7 +548,7 @@ HTTP1_DissectRequest(struct req *req) /*--------------------------------------------------------------------*/ uint16_t -HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc) +HTTP1_DissectResponse(struct http *hp, struct http_conn *htc) { uint16_t retval = 0; char *p; @@ -544,6 +594,8 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc) !Tlen(hp->hd[HTTP_HDR_REASON])) http_SetH(hp, HTTP_HDR_REASON, http_Status2Reason(hp->status)); + htc->body_status = http1_body_status(hp, htc); + hp->doclose = http1_DoConnection(hp); return (retval); From phk at FreeBSD.org Mon Sep 8 09:12:48 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 08 Sep 2014 11:12:48 +0200 Subject: [master] e7414b5 Use the protocol specific info to determine precense of request body. Message-ID: commit e7414b584f9085f7001730a80e993f84902d445a Author: Poul-Henning Kamp Date: Mon Sep 8 09:12:04 2014 +0000 Use the protocol specific info to determine precense of request body. diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index af7442a..b707e55 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -77,7 +77,6 @@ #include "hash/hash_slinger.h" #include "vcl.h" -#include "vct.h" #include "vtcp.h" #include "vtim.h" @@ -259,43 +258,11 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req) /*---------------------------------------------------------------------- */ -static enum req_body_state_e -http1_req_body_status(struct req *req) -{ - char *ptr, *endp; - - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - - if (http_GetHdr(req->http, H_Content_Length, &ptr)) { - AN(ptr); - if (*ptr == '\0') - return (REQ_BODY_FAIL); - req->req_bodybytes = strtoul(ptr, &endp, 10); - if (*endp != '\0' && !vct_islws(*endp)) - return (REQ_BODY_FAIL); - if (req->req_bodybytes == 0) - return (REQ_BODY_NONE); - req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done; - return (REQ_BODY_PRESENT); - } - if (http_HdrIs(req->http, H_Transfer_Encoding, "chunked")) { - req->h1.chunk_ctr = -1; - return (REQ_BODY_CHUNKED); - } - if (http_GetHdr(req->http, H_Transfer_Encoding, NULL)) - return (REQ_BODY_FAIL); - return (REQ_BODY_NONE); -} - -/*---------------------------------------------------------------------- - */ - static enum req_fsm_nxt http1_dissect(struct worker *wrk, struct req *req) { const char *r_100 = "HTTP/1.1 100 Continue\r\n\r\n"; const char *r_400 = "HTTP/1.1 400 Bad Request\r\n\r\n"; - const char *r_411 = "HTTP/1.1 411 Length Required\r\n\r\n"; const char *r_417 = "HTTP/1.1 417 Expectation Failed\r\n\r\n"; char *p; ssize_t r; @@ -335,18 +302,22 @@ http1_dissect(struct worker *wrk, struct req *req) return (REQ_FSM_DONE); } - if (req->req_body_status == REQ_BODY_INIT) - req->req_body_status = http1_req_body_status(req); - else + if (req->req_body_status != REQ_BODY_INIT) { assert(req->req_body_status == REQ_BODY_NONE); // ESI - - if (req->req_body_status == REQ_BODY_FAIL) { - wrk->stats.client_req_411++; - r = write(req->sp->fd, r_411, strlen(r_411)); - if (r > 0) - req->acct.resp_hdrbytes += r; - SES_Close(req->sp, SC_RX_JUNK); - return (REQ_FSM_DONE); + } else if (req->htc->body_status == BS_CHUNKED) { + req->h1.chunk_ctr = -1; + req->req_body_status = REQ_BODY_CHUNKED; + } else if (req->htc->body_status == BS_LENGTH) { + req->req_bodybytes = req->htc->content_length; + req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done; + req->req_body_status = REQ_BODY_PRESENT; + } else if (req->htc->body_status == BS_NONE) { + req->req_body_status = REQ_BODY_NONE; + } else if (req->htc->body_status == BS_EOF) { + /* XXX: We don't support EOF bodies in requests */ + req->req_body_status = REQ_BODY_NONE; + } else { + WRONG("Unknown req.body_length situation"); } if (http_GetHdr(req->http, H_Expect, &p)) { @@ -366,6 +337,7 @@ http1_dissect(struct worker *wrk, struct req *req) SES_Close(req->sp, SC_REM_CLOSE); return (REQ_FSM_DONE); } + http_Unset(req->http, H_Expect); } wrk->stats.client_req++; @@ -375,8 +347,6 @@ http1_dissect(struct worker *wrk, struct req *req) req->ws_req = WS_Snapshot(req->ws); req->doclose = req->http->doclose; - http_Unset(req->http, H_Expect); - assert(req->req_body_status != REQ_BODY_INIT); HTTP_Copy(req->http0, req->http); // For ESI & restart diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c index 8b872fc..b502fbd 100644 --- a/bin/varnishd/cache/cache_http1_proto.c +++ b/bin/varnishd/cache/cache_http1_proto.c @@ -522,9 +522,8 @@ HTTP1_DissectRequest(struct http_conn *htc, struct http *hp) http1_proto_ver(hp); retval = http1_request_check_host_hdr(hp); - if (retval != 0) { + if (retval != 0) return (retval); - } /* RFC2616, section 5.2, point 1 */ if (!strncasecmp(hp->hd[HTTP_HDR_URL].b, "http://", 7)) { @@ -540,6 +539,9 @@ HTTP1_DissectRequest(struct http_conn *htc, struct http *hp) htc->body_status = http1_body_status(hp, htc); + if (htc->body_status == BS_ERROR) + return (400); + hp->doclose = http1_DoConnection(hp); return (retval); diff --git a/bin/varnishtest/tests/r01356.vtc b/bin/varnishtest/tests/r01356.vtc index 341f2c8..543ee3e 100644 --- a/bin/varnishtest/tests/r01356.vtc +++ b/bin/varnishtest/tests/r01356.vtc @@ -10,13 +10,13 @@ varnish v1 -vcl+backend { } -start client c1 { txreq -req POST -nolen -hdr "Transfer-Encoding: carrier-pigeon" rxresp - expect resp.status == 411 + expect resp.status == 400 } -run client c1 { txreq -req POST -nolen -hdr "Content-Length: carrier-pigeon" rxresp - expect resp.status == 411 + expect resp.status == 400 } -run client c1 { From phk at FreeBSD.org Mon Sep 8 09:27:12 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 08 Sep 2014 11:27:12 +0200 Subject: [master] 49e7b9c Be consistent about bo-> vs bo. Message-ID: commit 49e7b9cfec83f3527add01447676b0d1ad112eb7 Author: Poul-Henning Kamp Date: Mon Sep 8 09:26:56 2014 +0000 Be consistent about bo-> vs bo. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 1bf2918..89d019f 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -509,7 +509,7 @@ struct busyobj { struct objcore *ims_oc; struct objcore *fetch_objcore; - struct http_conn htc; + struct http_conn htc[1]; struct pool_task fetch_task; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 24ba95d..ab71e87 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -316,9 +316,9 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) * headers are adultered by VCL * NB: Also sets other wrk variables */ - bo->htc.body_status = RFC2616_Body(bo, &wrk->stats); + bo->htc->body_status = RFC2616_Body(bo, &wrk->stats); - if (bo->htc.body_status == BS_ERROR) { + if (bo->htc->body_status == BS_ERROR) { AN (bo->vbc); VDI_CloseFd(&bo->vbc, &bo->acct); VSLb(bo->vsl, SLT_Error, "Body cannot be fetched"); @@ -526,7 +526,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) AN(bo->uncacheable); /* No reason to try streaming a non-existing body */ - if (bo->htc.body_status == BS_NONE) + if (bo->htc->body_status == BS_NONE) bo->do_stream = 0; if (VFP_Open(bo->vfc)) { @@ -555,8 +555,8 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) http_GetHdr(bo->beresp, H_ETag, &p))) ObjSetFlag(bo->vfc, OF_IMSCAND, 1); - if (bo->htc.body_status != BS_NONE) - V1F_Setup_Fetch(bo->vfc, bo->content_length, &bo->htc); + if (bo->htc->body_status != BS_NONE) + V1F_Setup_Fetch(bo->vfc, bo->content_length, bo->htc); /* * Ready to fetch the body @@ -574,11 +574,11 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) } VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", - bo->htc.body_status, body_status_2str(bo->htc.body_status), + bo->htc->body_status, body_status_2str(bo->htc->body_status), bo->do_stream ? "stream" : "-"); - if (bo->htc.body_status != BS_NONE) { - assert(bo->htc.body_status != BS_ERROR); + if (bo->htc->body_status != BS_NONE) { + assert(bo->htc->body_status != BS_ERROR); vbf_fetch_body_helper(bo); bo->acct.beresp_bodybytes = bo->vfc->bodybytes; } diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index b6657d5..4ab5d47 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -254,7 +254,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_ORNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - htc = &bo->htc; + htc = bo->htc; if (bo->director == NULL) { VSLb(bo->vsl, SLT_FetchError, "No backend"); @@ -335,7 +335,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req) cache_param->http_resp_size, cache_param->http_resp_hdr_len); CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); - CHECK_OBJ_NOTNULL(&bo->htc, HTTP_CONN_MAGIC); + CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC); VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout); diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 9d0330f..7574638 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -327,7 +327,7 @@ pan_busyobj(const struct busyobj *bo) #undef BO_FLAG VSB_printf(pan_vsp, " bodystatus = %d (%s),\n", - bo->htc.body_status, body_status_2str(bo->htc.body_status)); + bo->htc->body_status, body_status_2str(bo->htc->body_status)); if (!VTAILQ_EMPTY(&bo->vfc->vfp)) { VSB_printf(pan_vsp, " filters ="); VTAILQ_FOREACH(vfe, &bo->vfc->vfp, list) From phk at FreeBSD.org Mon Sep 8 13:18:30 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 08 Sep 2014 15:18:30 +0200 Subject: [master] 8535a3e Add asserts Message-ID: commit 8535a3ed28ba58b97117cd4cf04fd54a18ebed00 Author: Poul-Henning Kamp Date: Mon Sep 8 12:13:46 2014 +0000 Add asserts diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index b8ee0b9..fda8cb6 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -129,6 +129,8 @@ VFP_Open(struct vfp_ctx *vc) struct vfp_entry *vfe; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vc->http, HTTP_MAGIC); + AN(vc->vsl); VTAILQ_FOREACH_REVERSE(vfe, &vc->vfp, vfp_entry_s, list) { if (vfe->vfp->init == NULL) continue; @@ -188,6 +190,7 @@ VFP_Push(struct vfp_ctx *vc, const struct vfp *vfp, int top) struct vfp_entry *vfe; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vc->http, HTTP_MAGIC); vfe = (void*)WS_Alloc(vc->http->ws, sizeof *vfe); AN(vfe); vfe->magic = VFP_ENTRY_MAGIC; diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 74b11d7..66d7c89 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -303,6 +303,7 @@ VSLbv(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, va_list ap) unsigned n, mlen; txt t; + AN(vsl); AN(fmt); if (vsl_tag_is_masked(tag)) return; From phk at FreeBSD.org Mon Sep 8 13:18:30 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 08 Sep 2014 15:18:30 +0200 Subject: [master] 0c35ac8 Use the HTTP/1 VFP's for fetching the req.body Message-ID: commit 0c35ac8a7df799b53c31d8429206b928a9b9ca2b Author: Poul-Henning Kamp Date: Mon Sep 8 13:17:56 2014 +0000 Use the HTTP/1 VFP's for fetching the req.body diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 89d019f..1f0e512 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -605,12 +605,6 @@ struct req { volatile enum req_body_state_e req_body_status; struct body body[1]; - struct { - ssize_t bytes_done; - ssize_t bytes_yet; - intptr_t chunk_ctr; - } h1; /* HTTP1 specific */ - /* The busy objhead we sleep on */ struct objhead *hash_objhead; @@ -810,7 +804,7 @@ void VBO_waitstate(struct busyobj *bo, enum busyobj_state_e want); /* cache_http1_fetch.c [V1F] */ int V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req); -void V1F_Setup_Fetch(struct vfp_ctx *vfc, ssize_t cl, struct http_conn *htc); +void V1F_Setup_Fetch(struct vfp_ctx *vfc, struct http_conn *htc); /* cache_http1_fsm.c [HTTP1] */ typedef int (req_body_iter_f)(struct req *, void *priv, void *ptr, size_t); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index ab71e87..ba547ca 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -556,7 +556,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) ObjSetFlag(bo->vfc, OF_IMSCAND, 1); if (bo->htc->body_status != BS_NONE) - V1F_Setup_Fetch(bo->vfc, bo->content_length, bo->htc); + V1F_Setup_Fetch(bo->vfc, bo->htc); /* * Ready to fetch the body diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index 4ab5d47..4b76f46 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -153,7 +153,7 @@ static const struct vfp v1f_eof = { */ void -V1F_Setup_Fetch(struct vfp_ctx *vfc, ssize_t cl, struct http_conn *htc) +V1F_Setup_Fetch(struct vfp_ctx *vfc, struct http_conn *htc) { struct vfp_entry *vfe; @@ -162,17 +162,17 @@ V1F_Setup_Fetch(struct vfp_ctx *vfc, ssize_t cl, struct http_conn *htc) switch(htc->body_status) { case BS_EOF: - assert(cl == -1); + assert(htc->content_length == -1); vfe = VFP_Push(vfc, &v1f_eof, 0); vfe->priv2 = 0; break; case BS_LENGTH: - assert(cl > 0); + assert(htc->content_length > 0); vfe = VFP_Push(vfc, &v1f_straight, 0); - vfe->priv2 = cl; + vfe->priv2 = htc->content_length; break; case BS_CHUNKED: - assert(cl == -1); + assert(htc->content_length == -1); vfe = VFP_Push(vfc, &v1f_chunked, 0); vfe->priv2 = -1; break; @@ -181,7 +181,6 @@ V1F_Setup_Fetch(struct vfp_ctx *vfc, ssize_t cl, struct http_conn *htc) break; } vfe->priv1 = htc; - return; } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index b707e55..3a1a7cc 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -270,8 +270,6 @@ http1_dissect(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - memset(&req->h1, 0, sizeof req->h1); - /* * Cache_req_fsm zeros the vxid once a requests is processed. * Allocate a new one only now that we know will need it. @@ -305,11 +303,8 @@ http1_dissect(struct worker *wrk, struct req *req) if (req->req_body_status != REQ_BODY_INIT) { assert(req->req_body_status == REQ_BODY_NONE); // ESI } else if (req->htc->body_status == BS_CHUNKED) { - req->h1.chunk_ctr = -1; req->req_body_status = REQ_BODY_CHUNKED; } else if (req->htc->body_status == BS_LENGTH) { - req->req_bodybytes = req->htc->content_length; - req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done; req->req_body_status = REQ_BODY_PRESENT; } else if (req->htc->body_status == BS_NONE) { req->req_body_status = REQ_BODY_NONE; @@ -446,50 +441,6 @@ HTTP1_Session(struct worker *wrk, struct req *req) } } -static ssize_t -http1_iter_req_body(struct req *req, enum req_body_state_e bs, - void *buf, ssize_t len) -{ - const char *err; - - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - - AN(len); - AN(buf); - if (bs == REQ_BODY_PRESENT) { - AN(req->req_bodybytes); - if (len > req->req_bodybytes - req->h1.bytes_done) - len = req->req_bodybytes - req->h1.bytes_done; - if (len == 0) { - req->req_body_status = REQ_BODY_DONE; - return (0); - } - len = HTTP1_Read(req->htc, buf, len); - if (len <= 0) { - req->req_body_status = REQ_BODY_FAIL; - return (-1); - } - req->h1.bytes_done += len; - req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done; - req->acct.req_bodybytes += len; - return (len); - } else if (bs == REQ_BODY_CHUNKED) { - switch (HTTP1_Chunked(req->htc, &req->h1.chunk_ctr, &err, - &req->acct.req_bodybytes, buf, &len)) { - case H1CR_ERROR: - VSLb(req->vsl, SLT_Debug, "CHUNKERR: %s", err); - return (-1); - case H1CR_MORE: - return (len); - case H1CR_END: - return (0); - default: - WRONG("invalid HTTP1_Chunked return"); - } - } else - WRONG("Illegal req_bodystatus"); -} - /*---------------------------------------------------------------------- * Iterate over the req.body. * @@ -502,9 +453,10 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv) { char buf[8192]; struct storage *st; - enum req_body_state_e bs; ssize_t l; int i; + struct vfp_ctx vfc; + enum vfp_status vfps = VFP_ERROR; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AN(func); @@ -535,7 +487,6 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv) WRONG("Wrong req_body_status in HTTP1_IterateReqBody()"); } Lck_Lock(&req->sp->mtx); - bs = req->req_body_status; if (req->req_body_status == REQ_BODY_PRESENT || req->req_body_status == REQ_BODY_CHUNKED) { req->req_body_status = REQ_BODY_TAKEN; @@ -549,20 +500,33 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv) return (i); } + VFP_Setup(&vfc); + vfc.http = req->http; + vfc.vsl = req->vsl; + V1F_Setup_Fetch(&vfc, req->htc); + if (VFP_Open(&vfc) < 0) { + VSLb(req->vsl, SLT_FetchError, "Could not open Fetch Pipeline"); + return (-1); + } + do { - l = http1_iter_req_body(req, bs, buf, sizeof buf); - if (l < 0) { - req->doclose = SC_RX_BODY; + l = sizeof buf; + vfps = VFP_Suck(&vfc, buf, &l); + if (vfps == VFP_ERROR) { + req->req_body_status = REQ_BODY_FAIL; + l = -1; break; - } - if (l > 0) { - i = func(req, priv, buf, l); - if (i) { - l = i; + } else if (l > 0) { + req->req_bodybytes += l; + req->acct.req_bodybytes += l; + l = func(req, priv, buf, l); + if (l) { + req->req_body_status = REQ_BODY_FAIL; break; } } - } while (l > 0); + } while (vfps == VFP_OK); + VFP_Close(&vfc); VSLb_ts_req(req, "ReqBody", VTIM_real()); return (l); @@ -609,7 +573,9 @@ int HTTP1_CacheReqBody(struct req *req, ssize_t maxsize) { struct storage *st; - ssize_t l, l2; + ssize_t l, yet; + struct vfp_ctx vfc; + enum vfp_status vfps = VFP_ERROR; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -627,18 +593,30 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize) WRONG("Wrong req_body_status in HTTP1_CacheReqBody()"); } - if (req->req_bodybytes > maxsize) { + if (req->htc->content_length > maxsize) { req->req_body_status = REQ_BODY_FAIL; return (-1); } - l2 = 0; + VFP_Setup(&vfc); + vfc.http = req->http; + vfc.vsl = req->vsl; + V1F_Setup_Fetch(&vfc, req->htc); + + if (VFP_Open(&vfc) < 0) { + req->req_body_status = REQ_BODY_FAIL; + return (-1); + } + + + yet = req->htc->content_length; + if (yet < 0) + yet = 0; st = NULL; do { if (st == NULL) { st = STV_alloc_transient( - req->h1.bytes_yet ? - req->h1.bytes_yet : cache_param->fetch_chunksize); + yet ? yet : cache_param->fetch_chunksize); if (st == NULL) { req->req_body_status = REQ_BODY_FAIL; l = -1; @@ -647,39 +625,46 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize) VTAILQ_INSERT_TAIL(&req->body->list, st, list); } } - l = st->space - st->len; - l = http1_iter_req_body(req, req->req_body_status, - st->ptr + st->len, l); - if (l < 0) { - req->doclose = SC_RX_BODY; - break; - } - if (req->req_bodybytes > maxsize) { + vfps = VFP_Suck(&vfc, st->ptr + st->len, &l); + if (vfps == VFP_ERROR) { req->req_body_status = REQ_BODY_FAIL; l = -1; break; } if (l > 0) { - l2 += l; + req->req_bodybytes += l; + req->acct.req_bodybytes += l; + if (yet > 0) + yet -= l; st->len += l; if (st->space == st->len) st = NULL; + l = 0; } - } while (l > 0); + if (req->req_bodybytes > maxsize) { + req->req_body_status = REQ_BODY_FAIL; + l = -1; + break; + } + } while (vfps == VFP_OK); + VFP_Close(&vfc); + + if (l == 0) { - req->req_bodybytes = l2; - /* We must update also the "pristine" req.* copy */ - - http_Unset(req->http0, H_Content_Length); - http_Unset(req->http0, H_Transfer_Encoding); - http_PrintfHeader(req->http0, "Content-Length: %ju", - (uintmax_t)req->req_bodybytes); - - http_Unset(req->http, H_Content_Length); - http_Unset(req->http, H_Transfer_Encoding); - http_PrintfHeader(req->http, "Content-Length: %ju", - (uintmax_t)req->req_bodybytes); + + if (req->req_bodybytes != req->htc->content_length) { + /* We must update also the "pristine" req.* copy */ + http_Unset(req->http0, H_Content_Length); + http_Unset(req->http0, H_Transfer_Encoding); + http_PrintfHeader(req->http0, "Content-Length: %ju", + (uintmax_t)req->req_bodybytes); + + http_Unset(req->http, H_Content_Length); + http_Unset(req->http, H_Transfer_Encoding); + http_PrintfHeader(req->http, "Content-Length: %ju", + (uintmax_t)req->req_bodybytes); + } req->req_body_status = REQ_BODY_CACHED; } From phk at FreeBSD.org Mon Sep 8 18:14:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 08 Sep 2014 20:14:52 +0200 Subject: [master] a701b66 Eliminate a unused REQ_BODY state Message-ID: commit a701b66ca10af4a86d6c7a27cb327d22e965a2bd Author: Poul-Henning Kamp Date: Mon Sep 8 18:14:27 2014 +0000 Eliminate a unused REQ_BODY state diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index 4b76f46..6df8fcf 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -300,7 +300,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req) } else { i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk); } - if (req->req_body_status == REQ_BODY_DONE) { + if (req->req_body_status == REQ_BODY_TAKEN) { retry = -1; } else if (req->req_body_status == REQ_BODY_FAIL) { VSLb(bo->vsl, SLT_FetchError, diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 3a1a7cc..f29df20 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -474,7 +474,6 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv) case REQ_BODY_PRESENT: case REQ_BODY_CHUNKED: break; - case REQ_BODY_DONE: case REQ_BODY_TAKEN: VSLb(req->vsl, SLT_VCL_Error, "Uncached req.body can only be consumed once."); @@ -553,8 +552,6 @@ int HTTP1_DiscardReqBody(struct req *req) { - if (req->req_body_status == REQ_BODY_DONE) - return(0); if (req->req_body_status == REQ_BODY_FAIL) return(0); if (req->req_body_status == REQ_BODY_TAKEN) diff --git a/include/tbl/req_body.h b/include/tbl/req_body.h index 83744cc..9e58deb 100644 --- a/include/tbl/req_body.h +++ b/include/tbl/req_body.h @@ -34,7 +34,6 @@ REQ_BODY(PRESENT) REQ_BODY(CHUNKED) REQ_BODY(TAKEN) REQ_BODY(CACHED) -REQ_BODY(DONE) REQ_BODY(FAIL) REQ_BODY(NONE) From phk at FreeBSD.org Mon Sep 8 18:56:31 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 08 Sep 2014 20:56:31 +0200 Subject: [master] f40ffc8 Remove leftovers from ESI handling that now happens in req_fsm Message-ID: commit f40ffc8c9cd2989646f5b3dfbdba7dfde0ee2476 Author: Poul-Henning Kamp Date: Mon Sep 8 18:56:08 2014 +0000 Remove leftovers from ESI handling that now happens in req_fsm diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index f29df20..5fe9345 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -300,9 +300,9 @@ http1_dissect(struct worker *wrk, struct req *req) return (REQ_FSM_DONE); } - if (req->req_body_status != REQ_BODY_INIT) { - assert(req->req_body_status == REQ_BODY_NONE); // ESI - } else if (req->htc->body_status == BS_CHUNKED) { + assert (req->req_body_status == REQ_BODY_INIT); + + if (req->htc->body_status == BS_CHUNKED) { req->req_body_status = REQ_BODY_CHUNKED; } else if (req->htc->body_status == BS_LENGTH) { req->req_body_status = REQ_BODY_PRESENT; @@ -366,10 +366,8 @@ HTTP1_Session(struct worker *wrk, struct req *req) /* * Whenever we come in from the acceptor or waiter, we need to set - * blocking mode, but there is no point in setting it when we come from - * ESI or when a parked sessions returns. - * It would be simpler to do this in the acceptor or waiter, but we'd - * rather do the syscall in the worker thread. + * blocking mode. It would be simpler to do this in the acceptor + * or waiter, but we'd rather do the syscall in the worker thread. * On systems which return errors for ioctl, we close early */ if (sp->sess_step == S_STP_NEWREQ && VTCP_blocking(sp->fd)) { From phk at FreeBSD.org Mon Sep 8 19:52:56 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 08 Sep 2014 21:52:56 +0200 Subject: [master] d4b3c67 Set X-F-F early enough that it's present if we head directly to synth{} Message-ID: commit d4b3c674bce0e2ab607fdf9dc32222c88b9451dc Author: Poul-Henning Kamp Date: Mon Sep 8 19:52:31 2014 +0000 Set X-F-F early enough that it's present if we head directly to synth{} diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index bd9b696..032c88d 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -697,6 +697,21 @@ cnt_recv(struct worker *wrk, struct req *req) http_VSL_log(req->http); + if (req->restarts == 0) { + /* + * This really should be done earlier, but we want to capture + * it in the VSL log. + */ + if (http_GetHdr(req->http, H_X_Forwarded_For, &xff)) { + http_Unset(req->http, H_X_Forwarded_For); + http_PrintfHeader(req->http, "X-Forwarded-For: %s, %s", + xff, req->sp->client_addr_str); + } else { + http_PrintfHeader(req->http, "X-Forwarded-For: %s", + req->sp->client_addr_str); + } + } + if (req->err_code) { req->req_step = R_STP_SYNTH; return (REQ_FSM_MORE); @@ -712,25 +727,15 @@ cnt_recv(struct worker *wrk, struct req *req) req->hash_always_miss = 0; req->hash_ignore_busy = 0; req->client_identity = NULL; - if (req->restarts == 0) { - if (http_GetHdr(req->http, H_X_Forwarded_For, &xff)) { - http_Unset(req->http, H_X_Forwarded_For); - http_PrintfHeader(req->http, "X-Forwarded-For: %s, %s", xff, - req->sp->client_addr_str); - } else { - http_PrintfHeader(req->http, "X-Forwarded-For: %s", - req->sp->client_addr_str); - } - } http_CollectHdr(req->http, H_Cache_Control); VCL_recv_method(req->vcl, wrk, req, NULL, req->http->ws); /* Attempts to cache req.body may fail */ - if (req->req_body_status == REQ_BODY_FAIL) { + if (req->req_body_status == REQ_BODY_FAIL) return (REQ_FSM_DONE); - } + recv_handling = wrk->handling; /* We wash the A-E header here for the sake of VRY */ From phk at FreeBSD.org Tue Sep 9 06:35:38 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 09 Sep 2014 08:35:38 +0200 Subject: [master] 06cea07 Move HTTP1 VFPs to their own file, and integrate chunked helper function Message-ID: commit 06cea072ba587be9c538d34aac885c21d288fba4 Author: Poul-Henning Kamp Date: Tue Sep 9 06:35:16 2014 +0000 Move HTTP1 VFPs to their own file, and integrate chunked helper function diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index bcfe989..ddbd1d2 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -29,6 +29,7 @@ varnishd_SOURCES = \ cache/cache_http1_fetch.c \ cache/cache_http1_fsm.c \ cache/cache_http1_proto.c \ + cache/cache_http1_vfp.c \ cache/cache_lck.c \ cache/cache_main.c \ cache/cache_mempool.c \ diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 1f0e512..c006ca6 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -814,14 +814,6 @@ int HTTP1_CacheReqBody(struct req *req, ssize_t maxsize); int HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv); extern const int HTTP1_Req[3]; extern const int HTTP1_Resp[3]; -enum http1_chunked_ret { - H1CR_ERROR, - H1CR_MORE, - H1CR_END, -}; -enum http1_chunked_ret -HTTP1_Chunked(struct http_conn *htc, intptr_t *priv, const char **error, - uint64_t *statp, void *ptr, ssize_t *lp); /* cache_http1_deliver.c */ unsigned V1D_FlushReleaseAcct(struct req *req); diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index 6df8fcf..9555bbc 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -42,147 +42,6 @@ #include "vtcp.h" #include "vtim.h" -/*--------------------------------------------------------------------*/ - -static enum vfp_status __match_proto__(vfp_pull_f) -v1f_pull_straight(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, - ssize_t *lp) -{ - ssize_t l, lr; - struct http_conn *htc; - - CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); - CAST_OBJ_NOTNULL(htc, vfe->priv1, HTTP_CONN_MAGIC); - AN(p); - AN(lp); - - l = *lp; - *lp = 0; - - if (vfe->priv2 == 0) // XXX: Optimize Content-Len: 0 out earlier - return (VFP_END); - if (vfe->priv2 < l) - l = vfe->priv2; - lr = HTTP1_Read(htc, p, l); - vc->bodybytes += lr; - if (lr <= 0) - return (VFP_Error(vc, "straight insufficient bytes")); - *lp = lr; - vfe->priv2 -= lr; - if (vfe->priv2 == 0) - return (VFP_END); - return (VFP_OK); -} - -static const struct vfp v1f_straight = { - .name = "V1F_STRAIGHT", - .pull = v1f_pull_straight, -}; - -/*-------------------------------------------------------------------- - * Read a chunked HTTP object. - * - * XXX: Reading one byte at a time is pretty pessimal. - */ - -static enum vfp_status __match_proto__(vfp_pull_f) -v1f_pull_chunked(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, - ssize_t *lp) -{ - const char *err; - struct http_conn *htc; - - CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); - CAST_OBJ_NOTNULL(htc, vfe->priv1, HTTP_CONN_MAGIC); - AN(p); - AN(lp); - - switch (HTTP1_Chunked(htc, &vfe->priv2, &err, &vc->bodybytes, p, lp)) { - case H1CR_ERROR: - return (VFP_Error(vc, "%s", err)); - case H1CR_MORE: - return (VFP_OK); - case H1CR_END: - return (VFP_END); - default: - WRONG("invalid HTTP1_Chunked return"); - } -} - -static const struct vfp v1f_chunked = { - .name = "V1F_CHUNKED", - .pull = v1f_pull_chunked, -}; - -/*--------------------------------------------------------------------*/ - -static enum vfp_status __match_proto__(vfp_pull_f) -v1f_pull_eof(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, - ssize_t *lp) -{ - ssize_t l, lr; - struct http_conn *htc; - - CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); - CAST_OBJ_NOTNULL(htc, vfe->priv1, HTTP_CONN_MAGIC); - AN(p); - - AN(lp); - - l = *lp; - *lp = 0; - lr = HTTP1_Read(htc, p, l); - if (lr < 0) - return (VFP_Error(vc, "eof socket fail")); - if (lr == 0) - return (VFP_END); - *lp = lr; - vc->bodybytes += lr; - return (VFP_OK); -} - -static const struct vfp v1f_eof = { - .name = "V1F_EOF", - .pull = v1f_pull_eof, -}; - -/*-------------------------------------------------------------------- - */ - -void -V1F_Setup_Fetch(struct vfp_ctx *vfc, struct http_conn *htc) -{ - struct vfp_entry *vfe; - - CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); - - switch(htc->body_status) { - case BS_EOF: - assert(htc->content_length == -1); - vfe = VFP_Push(vfc, &v1f_eof, 0); - vfe->priv2 = 0; - break; - case BS_LENGTH: - assert(htc->content_length > 0); - vfe = VFP_Push(vfc, &v1f_straight, 0); - vfe->priv2 = htc->content_length; - break; - case BS_CHUNKED: - assert(htc->content_length == -1); - vfe = VFP_Push(vfc, &v1f_chunked, 0); - vfe->priv2 = -1; - break; - default: - WRONG("Wrong body_status"); - break; - } - vfe->priv1 = htc; -} - /*-------------------------------------------------------------------- * Pass the request body to the backend with chunks */ diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c index b502fbd..ab01b35 100644 --- a/bin/varnishd/cache/cache_http1_proto.c +++ b/bin/varnishd/cache/cache_http1_proto.c @@ -45,8 +45,6 @@ #include "config.h" -#include - #include "cache.h" #include "vct.h" @@ -623,106 +621,3 @@ HTTP1_Write(const struct worker *w, const struct http *hp, const int *hf) l += WRW_Write(w, "\r\n", -1); return (l); } - -/*-------------------------------------------------------------------- - * Read a chunked body. - * - * XXX: Reading one byte at a time is pretty pessimal. - */ - -enum http1_chunked_ret -HTTP1_Chunked(struct http_conn *htc, intptr_t *priv, const char **error, - uint64_t *statp, void *ptr, ssize_t *lp) -{ - int i; - char buf[20]; /* XXX: 20 is arbitrary */ - char *q; - unsigned u; - uintmax_t cll; - ssize_t cl, l, lr; - -#define ERR(x) do { *error = (x); return (H1CR_ERROR); } while (0) - - CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); - AN(priv); - AN(error); - AN(statp); - AN(ptr); - AN(lp); - l = *lp; - *lp = 0; - if (*priv == -1) { - /* Skip leading whitespace */ - do { - lr = HTTP1_Read(htc, buf, 1); - if (lr <= 0) - ERR("chunked read err"); - *statp += lr; - } while (vct_islws(buf[0])); - - if (!vct_ishex(buf[0])) - ERR("chunked header non-hex"); - - /* Collect hex digits, skipping leading zeros */ - for (u = 1; u < sizeof buf; u++) { - do { - lr = HTTP1_Read(htc, buf + u, 1); - if (lr <= 0) - ERR("chunked read err"); - *statp += lr; - } while (u == 1 && buf[0] == '0' && buf[u] == '0'); - if (!vct_ishex(buf[u])) - break; - } - - if (u >= sizeof buf) - ERR("chunked header too long"); - - /* Skip trailing white space */ - while(vct_islws(buf[u]) && buf[u] != '\n') { - lr = HTTP1_Read(htc, buf + u, 1); - if (lr <= 0) - ERR("chunked read err"); - *statp += lr; - } - - if (buf[u] != '\n') - ERR("chunked header no NL"); - - buf[u] = '\0'; - - cll = strtoumax(buf, &q, 16); - if (q == NULL || *q != '\0') - ERR("chunked header number syntax"); - cl = (ssize_t)cll; - if((uintmax_t)cl != cll) - ERR("bogusly large chunk size"); - - *priv = cl; - } - if (*priv > 0) { - if (*priv < l) - l = *priv; - lr = HTTP1_Read(htc, ptr, l); - if (lr <= 0) - ERR("straight insufficient bytes"); - *statp += lr; - *lp = lr; - *priv -= lr; - if (*priv == 0) - *priv = -1; - return (H1CR_MORE); - } - AZ(*priv); - i = HTTP1_Read(htc, buf, 1); - if (i <= 0) - ERR("chunked read err"); - *statp += i; - if (buf[0] == '\r' && HTTP1_Read(htc, buf, 1) <= 0) - ERR("chunked read err"); - if (buf[0] != '\n') - ERR("chunked tail no NL"); - return (H1CR_END); -#undef ERR -} - diff --git a/bin/varnishd/cache/cache_http1_vfp.c b/bin/varnishd/cache/cache_http1_vfp.c new file mode 100644 index 0000000..4e2fa29 --- /dev/null +++ b/bin/varnishd/cache/cache_http1_vfp.c @@ -0,0 +1,256 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2014 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * HTTP1 Fetch Filters + * + * These filters are used for both req.body and beresp.body to handle + * the HTTP/1 aspects (C-L/Chunked/EOF) + * + */ + +#include "config.h" + +#include + +#include "cache.h" + +#include "vct.h" + +/*-------------------------------------------------------------------- + * Read a chunked HTTP object. + * + * XXX: Reading one byte at a time is pretty pessimal. + */ + +static enum vfp_status __match_proto__(vfp_pull_f) +v1f_pull_chunked(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, + ssize_t *lp) +{ + struct http_conn *htc; + int i; + char buf[20]; /* XXX: 20 is arbitrary */ + char *q; + unsigned u; + uintmax_t cll; + ssize_t cl, l, lr; + + CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); + CAST_OBJ_NOTNULL(htc, vfe->priv1, HTTP_CONN_MAGIC); + AN(ptr); + AN(lp); + + AN(ptr); + AN(lp); + l = *lp; + *lp = 0; + if (vfe->priv2 == -1) { + /* Skip leading whitespace */ + do { + lr = HTTP1_Read(htc, buf, 1); + if (lr <= 0) + return (VFP_Error(vc, "chunked read err")); + vc->bodybytes += lr; + } while (vct_islws(buf[0])); + + if (!vct_ishex(buf[0])) + return (VFP_Error(vc, "chunked header non-hex")); + + /* Collect hex digits, skipping leading zeros */ + for (u = 1; u < sizeof buf; u++) { + do { + lr = HTTP1_Read(htc, buf + u, 1); + if (lr <= 0) + return (VFP_Error(vc, + "chunked read err")); + vc->bodybytes += lr; + } while (u == 1 && buf[0] == '0' && buf[u] == '0'); + if (!vct_ishex(buf[u])) + break; + } + + if (u >= sizeof buf) + return (VFP_Error(vc, "chunked header too long")); + + /* Skip trailing white space */ + while(vct_islws(buf[u]) && buf[u] != '\n') { + lr = HTTP1_Read(htc, buf + u, 1); + if (lr <= 0) + return (VFP_Error(vc, "chunked read err")); + vc->bodybytes += lr; + } + + if (buf[u] != '\n') + return (VFP_Error(vc, "chunked header no NL")); + + buf[u] = '\0'; + + cll = strtoumax(buf, &q, 16); + if (q == NULL || *q != '\0') + return (VFP_Error(vc, "chunked header number syntax")); + cl = (ssize_t)cll; + if((uintmax_t)cl != cll) + return (VFP_Error(vc, "bogusly large chunk size")); + + vfe->priv2 = cl; + } + if (vfe->priv2 > 0) { + if (vfe->priv2 < l) + l = vfe->priv2; + lr = HTTP1_Read(htc, ptr, l); + if (lr <= 0) + return (VFP_Error(vc, "straight insufficient bytes")); + vc->bodybytes += lr; + *lp = lr; + vfe->priv2 -= lr; + if (vfe->priv2 == 0) + vfe->priv2 = -1; + return (VFP_OK); + } + AZ(vfe->priv2); + i = HTTP1_Read(htc, buf, 1); + if (i <= 0) + return (VFP_Error(vc, "chunked read err")); + vc->bodybytes += i; + if (buf[0] == '\r' && HTTP1_Read(htc, buf, 1) <= 0) + return (VFP_Error(vc, "chunked read err")); + if (buf[0] != '\n') + return (VFP_Error(vc, "chunked tail no NL")); + return (VFP_END); +} + +static const struct vfp v1f_chunked = { + .name = "V1F_CHUNKED", + .pull = v1f_pull_chunked, +}; + + +/*--------------------------------------------------------------------*/ + +static enum vfp_status __match_proto__(vfp_pull_f) +v1f_pull_straight(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, + ssize_t *lp) +{ + ssize_t l, lr; + struct http_conn *htc; + + CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); + CAST_OBJ_NOTNULL(htc, vfe->priv1, HTTP_CONN_MAGIC); + AN(p); + AN(lp); + + l = *lp; + *lp = 0; + + if (vfe->priv2 == 0) // XXX: Optimize Content-Len: 0 out earlier + return (VFP_END); + if (vfe->priv2 < l) + l = vfe->priv2; + lr = HTTP1_Read(htc, p, l); + vc->bodybytes += lr; + if (lr <= 0) + return (VFP_Error(vc, "straight insufficient bytes")); + *lp = lr; + vfe->priv2 -= lr; + if (vfe->priv2 == 0) + return (VFP_END); + return (VFP_OK); +} + +static const struct vfp v1f_straight = { + .name = "V1F_STRAIGHT", + .pull = v1f_pull_straight, +}; + +/*--------------------------------------------------------------------*/ + +static enum vfp_status __match_proto__(vfp_pull_f) +v1f_pull_eof(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, + ssize_t *lp) +{ + ssize_t l, lr; + struct http_conn *htc; + + CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); + CAST_OBJ_NOTNULL(htc, vfe->priv1, HTTP_CONN_MAGIC); + AN(p); + + AN(lp); + + l = *lp; + *lp = 0; + lr = HTTP1_Read(htc, p, l); + if (lr < 0) + return (VFP_Error(vc, "eof socket fail")); + if (lr == 0) + return (VFP_END); + *lp = lr; + vc->bodybytes += lr; + return (VFP_OK); +} + +static const struct vfp v1f_eof = { + .name = "V1F_EOF", + .pull = v1f_pull_eof, +}; + +/*-------------------------------------------------------------------- + */ + +void +V1F_Setup_Fetch(struct vfp_ctx *vfc, struct http_conn *htc) +{ + struct vfp_entry *vfe; + + CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); + + switch(htc->body_status) { + case BS_EOF: + assert(htc->content_length == -1); + vfe = VFP_Push(vfc, &v1f_eof, 0); + vfe->priv2 = 0; + break; + case BS_LENGTH: + assert(htc->content_length > 0); + vfe = VFP_Push(vfc, &v1f_straight, 0); + vfe->priv2 = htc->content_length; + break; + case BS_CHUNKED: + assert(htc->content_length == -1); + vfe = VFP_Push(vfc, &v1f_chunked, 0); + vfe->priv2 = -1; + break; + default: + WRONG("Wrong body_status"); + break; + } + vfe->priv1 = htc; +} From phk at FreeBSD.org Tue Sep 9 08:02:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 09 Sep 2014 10:02:41 +0200 Subject: [master] 6fa596b Fail (400) on both buplicate Host: and Content-Length: headers Message-ID: commit 6fa596b91fc6d09a5dab29eb77ee571465e72280 Author: Poul-Henning Kamp Date: Tue Sep 9 08:02:10 2014 +0000 Fail (400) on both buplicate Host: and Content-Length: headers diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index c006ca6..59d8703 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -959,11 +959,11 @@ int http_IsStatus(const struct http *hp, int); void http_SetStatus(struct http *to, uint16_t status); const char *http_GetReq(const struct http *hp); int http_HdrIs(const struct http *hp, const char *hdr, const char *val); -int http_IsHdr(const txt *hh, const char *hdr); void http_CopyHome(const struct http *hp); void http_Unset(struct http *hp, const char *hdr); void http_MarkHeader(const struct http *, const char *hdr, unsigned hdrlen, uint8_t flag); +unsigned http_CountHdr(const struct http *hp, const char *hdr); void http_CollectHdr(struct http *hp, const char *hdr); void http_VSL_log(const struct http *hp); void HTTP_Merge(struct objcore *, struct dstat *, struct http *to); diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 7e2474c..17212a9 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -230,7 +230,7 @@ http_PutField(const struct http *to, int field, const char *string) /*--------------------------------------------------------------------*/ -int +static int http_IsHdr(const txt *hh, const char *hdr) { unsigned l; @@ -282,6 +282,26 @@ http_MarkHeader(const struct http *hp, const char *hdr, unsigned hdrlen, } /*-------------------------------------------------------------------- + * Count how many instances we have of this header + */ + +unsigned +http_CountHdr(const struct http *hp, const char *hdr) +{ + unsigned retval = 0; + unsigned u; + + CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); + + for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) { + Tcheck(hp->hd[u]); + if (http_IsHdr(&hp->hd[u], hdr)) + retval++; + } + return (retval); +} + +/*-------------------------------------------------------------------- * This function collapses multiple headerlines of the same name. * The lines are joined with a comma, according to [rfc2616, 4.2bot, p32] */ diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c index ab01b35..fc98f47 100644 --- a/bin/varnishd/cache/cache_http1_proto.c +++ b/bin/varnishd/cache/cache_http1_proto.c @@ -426,31 +426,6 @@ http1_body_status(const struct http *hp, struct http_conn *htc) /*--------------------------------------------------------------------*/ -static uint16_t -http1_request_check_host_hdr(const struct http *hp) -{ - int u; - int seen_host = 0; - - for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) { - if (hp->hd[u].b == NULL) - continue; - AN(hp->hd[u].b); - AN(hp->hd[u].e); - if (http_IsHdr(&hp->hd[u], H_Host)) { - if (seen_host) { - VSLb(hp->vsl, SLT_Error, - "Duplicated Host header"); - return (400); - } - seen_host = 1; - } - } - return (0); -} - -/*--------------------------------------------------------------------*/ - static void http1_proto_ver(struct http *hp) { @@ -519,9 +494,11 @@ HTTP1_DissectRequest(struct http_conn *htc, struct http *hp) return (retval); http1_proto_ver(hp); - retval = http1_request_check_host_hdr(hp); - if (retval != 0) - return (retval); + if (http_CountHdr(hp, H_Host) > 1) + return (400); + + if (http_CountHdr(hp, H_Content_Length) > 1) + return (400); /* RFC2616, section 5.2, point 1 */ if (!strncasecmp(hp->hd[HTTP_HDR_URL].b, "http://", 7)) { diff --git a/bin/varnishtest/tests/b00037.vtc b/bin/varnishtest/tests/b00037.vtc index 848bdd4..7112cff 100644 --- a/bin/varnishtest/tests/b00037.vtc +++ b/bin/varnishtest/tests/b00037.vtc @@ -10,6 +10,16 @@ varnish v1 -vcl+backend { client c1 { txreq -hdr "Host: foo" -hdr "Host: bar" + rxresp + expect resp.status == 400 } -run varnish v1 -expect client_req_400 == 1 + +client c1 { + txreq -hdr "Content-Length: 12" -bodylen 12 + rxresp + expect resp.status == 400 +} -run + +varnish v1 -expect client_req_400 == 2 diff --git a/bin/varnishtest/tests/r00102.vtc b/bin/varnishtest/tests/r00102.vtc index cf8a1bb..b8b56b3 100644 --- a/bin/varnishtest/tests/r00102.vtc +++ b/bin/varnishtest/tests/r00102.vtc @@ -17,14 +17,12 @@ varnish v1 -vcl+backend { client c1 { txreq -req POST -url "/" \ - -hdr "Content-Length: 10" \ -body "123456789\n" rxresp expect resp.status == 200 expect resp.http.X-Varnish == "1001" txreq -req POST -url "/" \ - -hdr "Content-Length: 10" \ -body "123456789\n" rxresp expect resp.status == 200 From phk at FreeBSD.org Tue Sep 9 08:17:34 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 09 Sep 2014 10:17:34 +0200 Subject: [master] 2ada3dc Connection: header handling is not version dependent. Message-ID: commit 2ada3dc3016a7edefc0c0f41695f7caf977d8988 Author: Poul-Henning Kamp Date: Tue Sep 9 08:17:01 2014 +0000 Connection: header handling is not version dependent. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 59d8703..34e51e7 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -196,7 +196,6 @@ struct http { uint16_t status; uint8_t protover; uint8_t conds; /* If-* headers present */ - enum sess_close doclose; }; /*-------------------------------------------------------------------- @@ -961,8 +960,6 @@ const char *http_GetReq(const struct http *hp); int http_HdrIs(const struct http *hp, const char *hdr, const char *val); void http_CopyHome(const struct http *hp); void http_Unset(struct http *hp, const char *hdr); -void http_MarkHeader(const struct http *, const char *hdr, unsigned hdrlen, - uint8_t flag); unsigned http_CountHdr(const struct http *hp, const char *hdr); void http_CollectHdr(struct http *hp, const char *hdr); void http_VSL_log(const struct http *hp); @@ -970,6 +967,7 @@ void HTTP_Merge(struct objcore *, struct dstat *, struct http *to); uint16_t HTTP_GetStatusPack(struct objcore *oc, struct dstat *ds); const char *HTTP_GetHdrPack(struct objcore *, struct dstat *, const char *hdr); +enum sess_close http_DoConnection(struct http *hp); /* cache_http1_proto.c */ diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 17212a9..2b2d61f 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -267,7 +267,7 @@ http_findhdr(const struct http *hp, unsigned l, const char *hdr) /*-------------------------------------------------------------------- */ -void +static void http_MarkHeader(const struct http *hp, const char *hdr, unsigned hdrlen, uint8_t flag) { @@ -572,6 +572,46 @@ http_GetContentLength(const struct http *hp) return (cl); } +/*-------------------------------------------------------------------- + */ + +enum sess_close +http_DoConnection(struct http *hp) +{ + char *p, *q; + enum sess_close retval; + unsigned u; + + if (hp->protover < 11) + retval = SC_REQ_HTTP10; + else + retval = SC_NULL; + + http_CollectHdr(hp, H_Connection); + if (!http_GetHdr(hp, H_Connection, &p)) + return (retval); + AN(p); + for (; *p; p++) { + if (vct_issp(*p)) + continue; + if (*p == ',') + continue; + for (q = p + 1; *q; q++) + if (*q == ',' || vct_issp(*q)) + break; + u = pdiff(p, q); + if (u == 5 && !strncasecmp(p, "close", u)) + retval = SC_REQ_CLOSE; + if (u == 10 && !strncasecmp(p, "keep-alive", u)) + retval = SC_NULL; + http_MarkHeader(hp, p, u, HDF_FILTER); + if (!*q) + break; + p = q; + } + return (retval); +} + /*--------------------------------------------------------------------*/ int diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index 9555bbc..3ad3eee 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -235,6 +235,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req) return (-1); } - bo->doclose = hp->doclose; + bo->doclose = http_DoConnection(hp); + return (0); } diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 5fe9345..8203987 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -340,7 +340,6 @@ http1_dissect(struct worker *wrk, struct req *req) AZ(req->err_code); req->ws_req = WS_Snapshot(req->ws); - req->doclose = req->http->doclose; assert(req->req_body_status != REQ_BODY_INIT); diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c index fc98f47..97b184e 100644 --- a/bin/varnishd/cache/cache_http1_proto.c +++ b/bin/varnishd/cache/cache_http1_proto.c @@ -437,47 +437,6 @@ http1_proto_ver(struct http *hp) hp->protover = 9; } -/*-------------------------------------------------------------------- - */ - -static enum sess_close -http1_DoConnection(struct http *hp) -{ - char *p, *q; - enum sess_close retval; - unsigned u; - - if (hp->protover < 11) - retval = SC_REQ_HTTP10; - else - retval = SC_NULL;; - - http_CollectHdr(hp, H_Connection); - if (!http_GetHdr(hp, H_Connection, &p)) - return (retval); - AN(p); - for (; *p; p++) { - if (vct_issp(*p)) - continue; - if (*p == ',') - continue; - for (q = p + 1; *q; q++) - if (*q == ',' || vct_issp(*q)) - break; - u = pdiff(p, q); - if (u == 5 && !strncasecmp(p, "close", u)) - retval = SC_REQ_CLOSE; - if (u == 10 && !strncasecmp(p, "keep-alive", u)) - retval = SC_NULL; - http_MarkHeader(hp, p, u, HDF_FILTER); - if (!*q) - break; - p = q; - } - return (retval); -} - - /*--------------------------------------------------------------------*/ uint16_t @@ -517,8 +476,6 @@ HTTP1_DissectRequest(struct http_conn *htc, struct http *hp) if (htc->body_status == BS_ERROR) return (400); - hp->doclose = http1_DoConnection(hp); - return (retval); } @@ -573,8 +530,6 @@ HTTP1_DissectResponse(struct http *hp, struct http_conn *htc) htc->body_status = http1_body_status(hp, htc); - hp->doclose = http1_DoConnection(hp); - return (retval); } diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 032c88d..1f0189e 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -717,6 +717,8 @@ cnt_recv(struct worker *wrk, struct req *req) return (REQ_FSM_MORE); } + req->doclose = http_DoConnection(req->http); + /* By default we use the first backend */ AZ(req->director_hint); req->director_hint = req->vcl->director[0]; From phk at FreeBSD.org Tue Sep 9 08:33:37 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 09 Sep 2014 10:33:37 +0200 Subject: [master] 8f3140f Rename http_GetReq() to http_GetMethod() Message-ID: commit 8f3140f77f3dced646eb0e06fdec213664fc3bc3 Author: Poul-Henning Kamp Date: Tue Sep 9 08:20:30 2014 +0000 Rename http_GetReq() to http_GetMethod() diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 34e51e7..519809d 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -956,7 +956,7 @@ ssize_t http_GetContentLength(const struct http *hp); uint16_t http_GetStatus(const struct http *hp); int http_IsStatus(const struct http *hp, int); void http_SetStatus(struct http *to, uint16_t status); -const char *http_GetReq(const struct http *hp); +const char *http_GetMethod(const struct http *hp); int http_HdrIs(const struct http *hp, const char *hdr, const char *val); void http_CopyHome(const struct http *hp); void http_Unset(struct http *hp, const char *hdr); diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 2b2d61f..27ab49f 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -671,7 +671,7 @@ http_SetStatus(struct http *to, uint16_t status) /*--------------------------------------------------------------------*/ const char * -http_GetReq(const struct http *hp) +http_GetMethod(const struct http *hp) { CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 131df15..dcd5102 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -193,7 +193,7 @@ RFC2616_Body(struct busyobj *bo, struct dstat *stats) hp = bo->beresp; - if (!strcasecmp(http_GetReq(bo->bereq), "head")) { + if (!strcasecmp(http_GetMethod(bo->bereq), "head")) { /* * A HEAD request can never have a body in the reply, * no matter what the headers might say. From phk at FreeBSD.org Tue Sep 9 08:33:37 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 09 Sep 2014 10:33:37 +0200 Subject: [master] 81b1736 Be more precise about when we take EOF bodies on requests. Message-ID: commit 81b17367de895fc888a94aa6797c7e28b69c2f28 Author: Poul-Henning Kamp Date: Tue Sep 9 08:33:15 2014 +0000 Be more precise about when we take EOF bodies on requests. diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 8203987..ede17bb 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -309,8 +309,7 @@ http1_dissect(struct worker *wrk, struct req *req) } else if (req->htc->body_status == BS_NONE) { req->req_body_status = REQ_BODY_NONE; } else if (req->htc->body_status == BS_EOF) { - /* XXX: We don't support EOF bodies in requests */ - req->req_body_status = REQ_BODY_NONE; + req->req_body_status = REQ_BODY_PRESENT; } else { WRONG("Unknown req.body_length situation"); } diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c index 97b184e..79288c6 100644 --- a/bin/varnishd/cache/cache_http1_proto.c +++ b/bin/varnishd/cache/cache_http1_proto.c @@ -443,6 +443,7 @@ uint16_t HTTP1_DissectRequest(struct http_conn *htc, struct http *hp) { uint16_t retval; + const char *p; char *b, *e; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); @@ -472,10 +473,21 @@ HTTP1_DissectRequest(struct http_conn *htc, struct http *hp) } htc->body_status = http1_body_status(hp, htc); - if (htc->body_status == BS_ERROR) return (400); + p = http_GetMethod(hp); + AN(p); + + /* We handle EOF bodies only for PUT and POST */ + if (htc->body_status == BS_EOF && + strcasecmp(p, "put") && strcasecmp(p, "post")) + htc->body_status = BS_NONE; + + /* HEAD with a body is a hard error */ + if (htc->body_status != BS_NONE && !strcasecmp(p, "head")) + return (400); + return (retval); } From phk at FreeBSD.org Tue Sep 9 09:28:00 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 09 Sep 2014 11:28:00 +0200 Subject: [master] 4fdf41a Add a vfp_ctx to http_conn Message-ID: commit 4fdf41a55afc22b560d49178cb930ba537d076b4 Author: Poul-Henning Kamp Date: Tue Sep 9 09:27:48 2014 +0000 Add a vfp_ctx to http_conn diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 519809d..6d17e44 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -199,6 +199,40 @@ struct http { }; /*-------------------------------------------------------------------- + * VFP filter state + */ + +struct vfp_entry { + unsigned magic; +#define VFP_ENTRY_MAGIC 0xbe32a027 + const struct vfp *vfp; + void *priv1; + intptr_t priv2; + enum vfp_status closed; + VTAILQ_ENTRY(vfp_entry) list; +}; + +VTAILQ_HEAD(vfp_entry_s, vfp_entry); + +struct vfp_ctx { + unsigned magic; +#define VFP_CTX_MAGIC 0x61d9d3e5 + struct busyobj *bo; + struct objcore *oc; + struct dstat *stats; + + int failed; + + struct vfp_entry_s vfp; + struct vfp_entry *vfp_nxt; + + struct vsl_log *vsl; + struct http *http; + struct http *esi_req; + uint64_t bodybytes; +}; + +/*-------------------------------------------------------------------- * HTTP Protocol connection structure * * This is the protocol independent object for a HTTP connection, used @@ -219,6 +253,7 @@ struct http_conn { txt pipeline; ssize_t content_length; enum body_status body_status; + struct vfp_ctx vfc[1]; }; /*--------------------------------------------------------------------*/ @@ -448,36 +483,6 @@ enum busyobj_state_e { BOS_FAILED, /* something went wrong */ }; -struct vfp_entry { - unsigned magic; -#define VFP_ENTRY_MAGIC 0xbe32a027 - const struct vfp *vfp; - void *priv1; - intptr_t priv2; - enum vfp_status closed; - VTAILQ_ENTRY(vfp_entry) list; -}; - -VTAILQ_HEAD(vfp_entry_s, vfp_entry); - -struct vfp_ctx { - unsigned magic; -#define VFP_CTX_MAGIC 0x61d9d3e5 - struct busyobj *bo; - struct objcore *oc; - struct dstat *stats; - - int failed; - - struct vfp_entry_s vfp; - struct vfp_entry *vfp_nxt; - - struct vsl_log *vsl; - struct http *http; - struct http *esi_req; - uint64_t bodybytes; -}; - struct busyobj { unsigned magic; #define BUSYOBJ_MAGIC 0x23b95567 diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index ede17bb..23bec6c 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -309,7 +309,7 @@ http1_dissect(struct worker *wrk, struct req *req) } else if (req->htc->body_status == BS_NONE) { req->req_body_status = REQ_BODY_NONE; } else if (req->htc->body_status == BS_EOF) { - req->req_body_status = REQ_BODY_PRESENT; + req->req_body_status = REQ_BODY_CHUNKED; } else { WRONG("Unknown req.body_length situation"); } @@ -451,7 +451,7 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv) struct storage *st; ssize_t l; int i; - struct vfp_ctx vfc; + struct vfp_ctx *vfc; enum vfp_status vfps = VFP_ERROR; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -495,18 +495,20 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv) return (i); } - VFP_Setup(&vfc); - vfc.http = req->http; - vfc.vsl = req->vsl; - V1F_Setup_Fetch(&vfc, req->htc); - if (VFP_Open(&vfc) < 0) { + CHECK_OBJ_NOTNULL(req->htc, HTTP_CONN_MAGIC); + vfc = req->htc->vfc; + VFP_Setup(vfc); + vfc->http = req->http; + vfc->vsl = req->vsl; + V1F_Setup_Fetch(vfc, req->htc); + if (VFP_Open(vfc) < 0) { VSLb(req->vsl, SLT_FetchError, "Could not open Fetch Pipeline"); return (-1); } do { l = sizeof buf; - vfps = VFP_Suck(&vfc, buf, &l); + vfps = VFP_Suck(vfc, buf, &l); if (vfps == VFP_ERROR) { req->req_body_status = REQ_BODY_FAIL; l = -1; @@ -521,7 +523,7 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv) } } } while (vfps == VFP_OK); - VFP_Close(&vfc); + VFP_Close(vfc); VSLb_ts_req(req, "ReqBody", VTIM_real()); return (l); @@ -548,11 +550,10 @@ int HTTP1_DiscardReqBody(struct req *req) { - if (req->req_body_status == REQ_BODY_FAIL) - return(0); - if (req->req_body_status == REQ_BODY_TAKEN) - return(0); - return(HTTP1_IterateReqBody(req, httpq_req_body_discard, NULL)); + if (req->req_body_status == REQ_BODY_PRESENT || + req->req_body_status == REQ_BODY_CHUNKED) + (void)HTTP1_IterateReqBody(req, httpq_req_body_discard, NULL); + return(0); } /*---------------------------------------------------------------------- @@ -567,7 +568,7 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize) { struct storage *st; ssize_t l, yet; - struct vfp_ctx vfc; + struct vfp_ctx *vfc; enum vfp_status vfps = VFP_ERROR; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -586,17 +587,20 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize) WRONG("Wrong req_body_status in HTTP1_CacheReqBody()"); } + CHECK_OBJ_NOTNULL(req->htc, HTTP_CONN_MAGIC); + vfc = req->htc->vfc; + if (req->htc->content_length > maxsize) { req->req_body_status = REQ_BODY_FAIL; return (-1); } - VFP_Setup(&vfc); - vfc.http = req->http; - vfc.vsl = req->vsl; - V1F_Setup_Fetch(&vfc, req->htc); + VFP_Setup(vfc); + vfc->http = req->http; + vfc->vsl = req->vsl; + V1F_Setup_Fetch(vfc, req->htc); - if (VFP_Open(&vfc) < 0) { + if (VFP_Open(vfc) < 0) { req->req_body_status = REQ_BODY_FAIL; return (-1); } @@ -619,7 +623,7 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize) } } l = st->space - st->len; - vfps = VFP_Suck(&vfc, st->ptr + st->len, &l); + vfps = VFP_Suck(vfc, st->ptr + st->len, &l); if (vfps == VFP_ERROR) { req->req_body_status = REQ_BODY_FAIL; l = -1; @@ -641,7 +645,7 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize) break; } } while (vfps == VFP_OK); - VFP_Close(&vfc); + VFP_Close(vfc); if (l == 0) { From martin at varnish-software.com Tue Sep 9 12:00:01 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 09 Sep 2014 14:00:01 +0200 Subject: [master] 71cd323 Make random/hash director exhaust the backend list when looking for healthy backend. Message-ID: commit 71cd323b23c221afa65ad20ac3f772b823a98ac7 Author: Martin Blix Grydeland Date: Fri Aug 29 14:47:21 2014 +0200 Make random/hash director exhaust the backend list when looking for healthy backend. Fixes: #1575 diff --git a/bin/varnishtest/tests/r01575.vtc b/bin/varnishtest/tests/r01575.vtc new file mode 100644 index 0000000..11c82c1 --- /dev/null +++ b/bin/varnishtest/tests/r01575.vtc @@ -0,0 +1,56 @@ +varnishtest "#1575 - random director exhaust backend list" + +# Add 5 backends to a random director, with the 5th having very low weight. +# Mark the first 4 sick, and make sure that the 5th will be selected. + +server s1 { + rxreq + txresp +} -start + +server s2 { + rxreq + txresp +} -start + +server s3 { + rxreq + txresp +} -start + +server s4 { + rxreq + txresp +} -start + +server s5 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + import ${vmod_directors}; + sub vcl_init { + new rd = directors.random(); + rd.add_backend(s1, 10000); + rd.add_backend(s2, 10000); + rd.add_backend(s3, 10000); + rd.add_backend(s4, 10000); + rd.add_backend(s5, 1); + } + + sub vcl_backend_fetch { + set bereq.backend = rd.backend(); + } +} -start + +varnish v1 -cliok "backend.set_health s1 sick" +varnish v1 -cliok "backend.set_health s2 sick" +varnish v1 -cliok "backend.set_health s3 sick" +varnish v1 -cliok "backend.set_health s4 sick" + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c index afef7ed..090039f 100644 --- a/lib/libvmod_directors/hash.c +++ b/lib/libvmod_directors/hash.c @@ -47,7 +47,7 @@ struct vmod_directors_hash { unsigned magic; #define VMOD_DIRECTORS_HASH_MAGIC 0xc08dd611 struct vdir *vd; - unsigned nloops; + unsigned n_backend; struct vbitmap *vbm; }; @@ -64,7 +64,6 @@ vmod_hash__init(const struct vrt_ctx *ctx, struct vmod_directors_hash **rrp, AN(rr); rr->vbm = vbit_init(8); AN(rr->vbm); - rr->nloops = 3; // *rrp = rr; vdir_new(&rr->vd, vcl_name, NULL, NULL, rr); } @@ -90,6 +89,7 @@ vmod_hash_add_backend(const struct vrt_ctx *ctx, CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_HASH_MAGIC); (void)vdir_add_backend(rr->vd, be, w); + rr->n_backend++; } VCL_BACKEND __match_proto__() @@ -120,6 +120,6 @@ vmod_hash_backend(const struct vrt_ctx *ctx, struct vmod_directors_hash *rr, r = vbe32dec(sha256); r = scalbn(r, -32); assert(r >= 0 && r <= 1.0); - be = vdir_pick_be(rr->vd, r, rr->nloops); + be = vdir_pick_be(rr->vd, r, rr->n_backend); return (be); } diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c index 22f0bb9..8ae36a7 100644 --- a/lib/libvmod_directors/random.c +++ b/lib/libvmod_directors/random.c @@ -45,7 +45,7 @@ struct vmod_directors_random { unsigned magic; #define VMOD_DIRECTORS_RANDOM_MAGIC 0x4732d092 struct vdir *vd; - unsigned nloops; + unsigned n_backend; struct vbitmap *vbm; }; @@ -68,7 +68,7 @@ vmod_rr_getfd(const struct director *dir, struct busyobj *bo) CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_RANDOM_MAGIC); r = scalbn(random(), -31); assert(r >= 0 && r < 1.0); - be = vdir_pick_be(rr->vd, r, rr->nloops); + be = vdir_pick_be(rr->vd, r, rr->n_backend); if (be == NULL) return (NULL); return (be->getfd(be, bo)); @@ -87,7 +87,6 @@ vmod_random__init(const struct vrt_ctx *ctx, struct vmod_directors_random **rrp, AN(rr); rr->vbm = vbit_init(8); AN(rr->vbm); - rr->nloops = 3; // *rrp = rr; vdir_new(&rr->vd, vcl_name, vmod_rr_healthy, vmod_rr_getfd, rr); } @@ -113,6 +112,7 @@ vmod_random_add_backend(const struct vrt_ctx *ctx, CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_RANDOM_MAGIC); (void)vdir_add_backend(rr->vd, be, w); + rr->n_backend++; } VCL_BACKEND __match_proto__() diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index 12268eb..aae7a14 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -186,7 +186,7 @@ vdir_pick_be(struct vdir *vd, double w, unsigned nloops) nbe = vd->n_backend; assert(w >= 0.0 && w < 1.0); vdir_lock(vd); - for (l = 0; nbe > 0 && tw > 0.0 && l 0 && tw > 0.0 && l < nloops; l++) { u = vdir_pick_by_weight(vd, w * tw, vbm); be = vd->backend[u]; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); From martin at varnish-software.com Tue Sep 9 12:29:15 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 09 Sep 2014 14:29:15 +0200 Subject: [master] 74d4f0f Enable batch record usage in logexpect Message-ID: commit 74d4f0f15ebf53220d93888c6be42462070e1ed4 Author: Martin Blix Grydeland Date: Mon Sep 1 13:13:30 2014 +0200 Enable batch record usage in logexpect This enables catching batch related log problems using logexpect. This commit will make several test cases start failing until #1588 has been fixed. diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index d868a1b..a2b6323 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -330,7 +330,8 @@ logexp_start(struct logexp *le) } le->vsl = VSL_New(); AN(le->vsl); - c = VSL_CursorVSM(le->vsl, le->vsm, le->d_arg ? 0 : VSL_COPT_TAIL); + c = VSL_CursorVSM(le->vsl, le->vsm, + (le->d_arg ? 0 : VSL_COPT_TAIL) | VSL_COPT_BATCH); if (c == NULL) { vtc_log(le->vl, 0, "VSL_CursorVSM: %s", VSL_Error(le->vsl)); logexp_close(le); From martin at varnish-software.com Tue Sep 9 12:29:15 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 09 Sep 2014 14:29:15 +0200 Subject: [master] 51e13a4 Allocate request VXID only when it's needed and delay all logging until it's been allocated. Message-ID: commit 51e13a4a5e3940c29730ed65aa0b7b482ba21e61 Author: Martin Blix Grydeland Date: Mon Sep 1 13:52:53 2014 +0200 Allocate request VXID only when it's needed and delay all logging until it's been allocated. This patch delays the VXID allocation until the time we know that we will actually be logging anything for this request (not completely all whitespace noise). This keeps the code in one place only (plus the ESI mock request handling), instead of separated between the linger wait, cache acceptor and pipelining code paths. Coincidentally fixes the logging of timestamps problem in #1588. Test case by daghf. Fixes: #1581 #1588 diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 55c8901..d272bfc 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -62,7 +62,8 @@ ved_include(struct req *preq, const char *src, const char *host) req = SES_GetReq(wrk, preq->sp); req->req_body_status = REQ_BODY_NONE; - AN(req->vsl->wid & VSL_CLIENTMARKER); + AZ(req->vsl->wid); + req->vsl->wid = VXID_Get(wrk, VSL_CLIENTMARKER); VSLb(req->vsl, SLT_Begin, "req %u esi", VXID(preq->vsl->wid)); VSLb(preq->vsl, SLT_Link, "req %u esi", VXID(req->vsl->wid)); req->esi_level = preq->esi_level + 1; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index ba547ca..1011946 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -889,8 +889,8 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, bo = VBO_GetBusyObj(wrk, req); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - VSLb(bo->vsl, SLT_Begin, "bereq %u %s ", VXID(req->vsl->wid), how); - VSLb(req->vsl, SLT_Link, "bereq %u %s ", VXID(bo->vsl->wid), how); + VSLb(bo->vsl, SLT_Begin, "bereq %u %s", VXID(req->vsl->wid), how); + VSLb(req->vsl, SLT_Link, "bereq %u %s", VXID(bo->vsl->wid), how); THR_SetBusyobj(bo); diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 23bec6c..bd80c79 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -121,9 +121,9 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req) if (hs == HTTP1_COMPLETE) { /* Got it, run with it */ if (isnan(req->t_first)) - VSLb_ts_req(req, "Start", now); - VSLb_ts_req(req, "Req", now); - req->t_req = req->t_prev; + req->t_first = now; + if (isnan(req->t_req)) + req->t_req = now; req->acct.req_hdrbytes += Tlen(req->htc->rxbuf); return (REQ_FSM_MORE); } else if (hs == HTTP1_ERROR_EOF) { @@ -150,8 +150,8 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req) } else { /* Working on it */ if (isnan(req->t_first)) - /* Timestamp Start on first byte received */ - VSLb_ts_req(req, "Start", now); + /* Record first byte received time stamp */ + req->t_first = now; when = sp->t_idle + cache_param->timeout_req; tmo = (int)(1e3 * (when - now)); if (when < now || tmo == 0) { @@ -238,13 +238,7 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req) if (HTTP1_Reinit(req->htc) == HTTP1_COMPLETE) { AZ(req->vsl->wid); - req->vsl->wid = VXID_Get(wrk, VSL_CLIENTMARKER); - VSLb(req->vsl, SLT_Begin, "req %u rxreq", VXID(req->sp->vxid)); - VSL(SLT_Link, req->sp->vxid, "req %u rxreq", - VXID(req->vsl->wid)); - VSLb_ts_req(req, "Start", sp->t_idle); - VSLb_ts_req(req, "Req", sp->t_idle); - req->t_req = req->t_prev; + req->t_first = req->t_req = sp->t_idle; wrk->stats.sess_pipeline++; req->acct.req_hdrbytes += Tlen(req->htc->rxbuf); return (SESS_DONE_RET_START); @@ -270,16 +264,17 @@ http1_dissect(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - /* - * Cache_req_fsm zeros the vxid once a requests is processed. - * Allocate a new one only now that we know will need it. - */ - if (req->vsl->wid == 0) { - req->vsl->wid = VXID_Get(wrk, VSL_CLIENTMARKER); - VSLb(req->vsl, SLT_Begin, "req %u rxreq", VXID(req->sp->vxid)); - VSL(SLT_Link, req->sp->vxid, "req %u rxreq", - VXID(req->vsl->wid)); - } + /* Allocate a new vxid now that we know we'll need it. */ + AZ(req->vsl->wid); + req->vsl->wid = VXID_Get(wrk, VSL_CLIENTMARKER); + + VSLb(req->vsl, SLT_Begin, "req %u rxreq", VXID(req->sp->vxid)); + VSL(SLT_Link, req->sp->vxid, "req %u rxreq", VXID(req->vsl->wid)); + AZ(isnan(req->t_first)); /* First byte timestamp set by http1_wait */ + AZ(isnan(req->t_req)); /* Complete req rcvd set by http1_wait */ + req->t_prev = req->t_first; + VSLb_ts_req(req, "Start", req->t_first); + VSLb_ts_req(req, "Req", req->t_req); /* Borrow VCL reference from worker thread */ VCL_Refresh(&wrk->vcl); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 3b22ce7..bd48b45 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -133,9 +133,6 @@ ses_sess_pool_task(struct worker *wrk, void *arg) req = SES_GetReq(wrk, sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - AN(req->vsl->wid & VSL_CLIENTMARKER); - VSLb(req->vsl, SLT_Begin, "req %u rxreq", VXID(sp->vxid)); - VSL(SLT_Link, sp->vxid, "req %u rxreq", VXID(req->vsl->wid)); sp->sess_step = S_STP_NEWREQ; ses_req_pool_task(wrk, req); @@ -362,7 +359,6 @@ SES_GetReq(struct worker *wrk, struct sess *sp) sz = cache_param->vsl_buffer; VSL_Setup(req->vsl, p, sz); - req->vsl->wid = VXID_Get(wrk, VSL_CLIENTMARKER); p += sz; p = (void*)PRNDUP(p); diff --git a/bin/varnishtest/tests/r01581.vtc b/bin/varnishtest/tests/r01581.vtc new file mode 100644 index 0000000..f04eb0e --- /dev/null +++ b/bin/varnishtest/tests/r01581.vtc @@ -0,0 +1,23 @@ +varnishtest "Duplcate SLT_Begin" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { +} -start + +logexpect l1 -v v1 -g request { + expect 0 1001 Begin + expect * = ReqStart + expect * = End +} -start + +client c1 { + delay 0.1 + send "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n" + rxresp +} -run + +logexpect l1 -wait From martin at varnish-software.com Tue Sep 9 12:54:33 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 09 Sep 2014 14:54:33 +0200 Subject: [4.0] d85dde6 Make random/hash director exhaust the backend list when looking for healthy backend. Message-ID: commit d85dde647a93e0d585b74910eeed121942f73ee7 Author: Martin Blix Grydeland Date: Fri Aug 29 14:47:21 2014 +0200 Make random/hash director exhaust the backend list when looking for healthy backend. Fixes: #1575 diff --git a/bin/varnishtest/tests/r01575.vtc b/bin/varnishtest/tests/r01575.vtc new file mode 100644 index 0000000..11c82c1 --- /dev/null +++ b/bin/varnishtest/tests/r01575.vtc @@ -0,0 +1,56 @@ +varnishtest "#1575 - random director exhaust backend list" + +# Add 5 backends to a random director, with the 5th having very low weight. +# Mark the first 4 sick, and make sure that the 5th will be selected. + +server s1 { + rxreq + txresp +} -start + +server s2 { + rxreq + txresp +} -start + +server s3 { + rxreq + txresp +} -start + +server s4 { + rxreq + txresp +} -start + +server s5 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + import ${vmod_directors}; + sub vcl_init { + new rd = directors.random(); + rd.add_backend(s1, 10000); + rd.add_backend(s2, 10000); + rd.add_backend(s3, 10000); + rd.add_backend(s4, 10000); + rd.add_backend(s5, 1); + } + + sub vcl_backend_fetch { + set bereq.backend = rd.backend(); + } +} -start + +varnish v1 -cliok "backend.set_health s1 sick" +varnish v1 -cliok "backend.set_health s2 sick" +varnish v1 -cliok "backend.set_health s3 sick" +varnish v1 -cliok "backend.set_health s4 sick" + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c index 35c8ed7..b0c10aa 100644 --- a/lib/libvmod_directors/hash.c +++ b/lib/libvmod_directors/hash.c @@ -47,7 +47,7 @@ struct vmod_directors_hash { unsigned magic; #define VMOD_DIRECTORS_HASH_MAGIC 0xc08dd611 struct vdir *vd; - unsigned nloops; + unsigned n_backend; struct vbitmap *vbm; }; @@ -64,7 +64,6 @@ vmod_hash__init(const struct vrt_ctx *ctx, struct vmod_directors_hash **rrp, AN(rr); rr->vbm = vbit_init(8); AN(rr->vbm); - rr->nloops = 3; // *rrp = rr; vdir_new(&rr->vd, vcl_name, NULL, NULL, rr); } @@ -90,6 +89,7 @@ vmod_hash_add_backend(const struct vrt_ctx *ctx, CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_HASH_MAGIC); (void)vdir_add_backend(rr->vd, be, w); + rr->n_backend++; } VCL_BACKEND __match_proto__() @@ -119,6 +119,6 @@ vmod_hash_backend(const struct vrt_ctx *ctx, struct vmod_directors_hash *rr, r = vbe32dec(sha256); r = scalbn(r, -32); assert(r >= 0 && r <= 1.0); - be = vdir_pick_be(rr->vd, r, rr->nloops); + be = vdir_pick_be(rr->vd, r, rr->n_backend); return (be); } diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c index 22f0bb9..8ae36a7 100644 --- a/lib/libvmod_directors/random.c +++ b/lib/libvmod_directors/random.c @@ -45,7 +45,7 @@ struct vmod_directors_random { unsigned magic; #define VMOD_DIRECTORS_RANDOM_MAGIC 0x4732d092 struct vdir *vd; - unsigned nloops; + unsigned n_backend; struct vbitmap *vbm; }; @@ -68,7 +68,7 @@ vmod_rr_getfd(const struct director *dir, struct busyobj *bo) CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_RANDOM_MAGIC); r = scalbn(random(), -31); assert(r >= 0 && r < 1.0); - be = vdir_pick_be(rr->vd, r, rr->nloops); + be = vdir_pick_be(rr->vd, r, rr->n_backend); if (be == NULL) return (NULL); return (be->getfd(be, bo)); @@ -87,7 +87,6 @@ vmod_random__init(const struct vrt_ctx *ctx, struct vmod_directors_random **rrp, AN(rr); rr->vbm = vbit_init(8); AN(rr->vbm); - rr->nloops = 3; // *rrp = rr; vdir_new(&rr->vd, vcl_name, vmod_rr_healthy, vmod_rr_getfd, rr); } @@ -113,6 +112,7 @@ vmod_random_add_backend(const struct vrt_ctx *ctx, CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_RANDOM_MAGIC); (void)vdir_add_backend(rr->vd, be, w); + rr->n_backend++; } VCL_BACKEND __match_proto__() diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index 12268eb..aae7a14 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -186,7 +186,7 @@ vdir_pick_be(struct vdir *vd, double w, unsigned nloops) nbe = vd->n_backend; assert(w >= 0.0 && w < 1.0); vdir_lock(vd); - for (l = 0; nbe > 0 && tw > 0.0 && l 0 && tw > 0.0 && l < nloops; l++) { u = vdir_pick_by_weight(vd, w * tw, vbm); be = vd->backend[u]; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); From martin at varnish-software.com Tue Sep 9 12:54:33 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 09 Sep 2014 14:54:33 +0200 Subject: [4.0] d2d17a6 Fix logexpect cursor creation to use the proper cursor options Message-ID: commit d2d17a64cae5c5c0c3f3fa10617c4e27303996c0 Author: Martin Blix Grydeland Date: Mon Sep 1 13:06:22 2014 +0200 Fix logexpect cursor creation to use the proper cursor options diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 48c8f3d..d868a1b 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -330,7 +330,7 @@ logexp_start(struct logexp *le) } le->vsl = VSL_New(); AN(le->vsl); - c = VSL_CursorVSM(le->vsl, le->vsm, !le->d_arg); + c = VSL_CursorVSM(le->vsl, le->vsm, le->d_arg ? 0 : VSL_COPT_TAIL); if (c == NULL) { vtc_log(le->vl, 0, "VSL_CursorVSM: %s", VSL_Error(le->vsl)); logexp_close(le); From martin at varnish-software.com Tue Sep 9 12:54:33 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 09 Sep 2014 14:54:33 +0200 Subject: [4.0] 4621557 Enable batch record usage in logexpect Message-ID: commit 4621557d63202c399c8c6f50b291f588868b324d Author: Martin Blix Grydeland Date: Mon Sep 1 13:13:30 2014 +0200 Enable batch record usage in logexpect This enables catching batch related log problems using logexpect. This commit will make several test cases start failing until #1588 has been fixed. diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index d868a1b..a2b6323 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -330,7 +330,8 @@ logexp_start(struct logexp *le) } le->vsl = VSL_New(); AN(le->vsl); - c = VSL_CursorVSM(le->vsl, le->vsm, le->d_arg ? 0 : VSL_COPT_TAIL); + c = VSL_CursorVSM(le->vsl, le->vsm, + (le->d_arg ? 0 : VSL_COPT_TAIL) | VSL_COPT_BATCH); if (c == NULL) { vtc_log(le->vl, 0, "VSL_CursorVSM: %s", VSL_Error(le->vsl)); logexp_close(le); From martin at varnish-software.com Tue Sep 9 12:54:33 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 09 Sep 2014 14:54:33 +0200 Subject: [4.0] f941069 Allocate request VXID only when it's needed and delay all logging until it's been allocated. Message-ID: commit f94106987fcdd958c1daf4cec6186da980c8ac82 Author: Martin Blix Grydeland Date: Mon Sep 1 13:52:53 2014 +0200 Allocate request VXID only when it's needed and delay all logging until it's been allocated. This patch delays the VXID allocation until the time we know that we will actually be logging anything for this request (not completely all whitespace noise). This keeps the code in one place only (plus the ESI mock request handling), instead of separated between the linger wait, cache acceptor and pipelining code paths. Coincidentally fixes the logging of timestamps problem in #1588. Test case by daghf. Fixes: #1581 #1588 Conflicts: bin/varnishd/cache/cache_esi_deliver.c bin/varnishd/cache/cache_fetch.c bin/varnishd/cache/cache_http1_fsm.c bin/varnishd/cache/cache_session.c diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index ee1b0e5..fee1240 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -62,7 +62,8 @@ ved_include(struct req *preq, const char *src, const char *host) req = SES_GetReq(wrk, preq->sp); req->req_body_status = REQ_BODY_NONE; - AN(req->vsl->wid & VSL_CLIENTMARKER); + AZ(req->vsl->wid); + req->vsl->wid = VXID_Get(&wrk->vxid_pool) | VSL_CLIENTMARKER; VSLb(req->vsl, SLT_Begin, "req %u esi", preq->vsl->wid & VSL_IDENTMASK); VSLb(preq->vsl, SLT_Link, "req %u esi", req->vsl->wid & VSL_IDENTMASK); req->esi_level = preq->esi_level + 1; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index a768170..3d1fbfe 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -873,9 +873,9 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, default: WRONG("Wrong fetch mode"); } - VSLb(bo->vsl, SLT_Begin, "bereq %u %s ", + VSLb(bo->vsl, SLT_Begin, "bereq %u %s", req->vsl->wid & VSL_IDENTMASK, how); - VSLb(req->vsl, SLT_Link, "bereq %u %s ", + VSLb(req->vsl, SLT_Link, "bereq %u %s", bo->vsl->wid & VSL_IDENTMASK, how); bo->refcount = 2; diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 454ec28..31b65e2 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -123,9 +123,9 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req) if (hs == HTTP1_COMPLETE) { /* Got it, run with it */ if (isnan(req->t_first)) - VSLb_ts_req(req, "Start", now); - VSLb_ts_req(req, "Req", now); - req->t_req = req->t_prev; + req->t_first = now; + if (isnan(req->t_req)) + req->t_req = now; req->acct.req_hdrbytes += Tlen(req->htc->rxbuf); return (REQ_FSM_MORE); } else if (hs == HTTP1_ERROR_EOF) { @@ -152,8 +152,8 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req) } else { /* Working on it */ if (isnan(req->t_first)) - /* Timestamp Start on first byte received */ - VSLb_ts_req(req, "Start", now); + /* Record first byte received time stamp */ + req->t_first = now; when = sp->t_idle + cache_param->timeout_req; tmo = (int)(1e3 * (when - now)); if (when < now || tmo == 0) { @@ -245,14 +245,7 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req) if (HTTP1_Reinit(req->htc) == HTTP1_COMPLETE) { AZ(req->vsl->wid); - req->vsl->wid = VXID_Get(&wrk->vxid_pool) | VSL_CLIENTMARKER; - VSLb(req->vsl, SLT_Begin, "req %u rxreq", - req->sp->vxid & VSL_IDENTMASK); - VSL(SLT_Link, req->sp->vxid, "req %u rxreq", - req->vsl->wid & VSL_IDENTMASK); - VSLb_ts_req(req, "Start", sp->t_idle); - VSLb_ts_req(req, "Req", sp->t_idle); - req->t_req = req->t_prev; + req->t_first = req->t_req = sp->t_idle; wrk->stats.sess_pipeline++; req->acct.req_hdrbytes += Tlen(req->htc->rxbuf); return (SESS_DONE_RET_START); @@ -312,17 +305,19 @@ http1_dissect(struct worker *wrk, struct req *req) memset(&req->h1, 0, sizeof req->h1); - /* - * Cache_req_fsm zeros the vxid once a requests is processed. - * Allocate a new one only now that we know will need it. - */ - if (req->vsl->wid == 0) { - req->vsl->wid = VXID_Get(&wrk->vxid_pool) | VSL_CLIENTMARKER; - VSLb(req->vsl, SLT_Begin, "req %u rxreq", - req->sp->vxid & VSL_IDENTMASK); - VSL(SLT_Link, req->sp->vxid, "req %u rxreq", - req->vsl->wid & VSL_IDENTMASK); - } + /* Allocate a new vxid now that we know we'll need it. */ + AZ(req->vsl->wid); + req->vsl->wid = VXID_Get(&wrk->vxid_pool) | VSL_CLIENTMARKER; + + VSLb(req->vsl, SLT_Begin, "req %u rxreq", + req->sp->vxid & VSL_IDENTMASK); + VSL(SLT_Link, req->sp->vxid, "req %u rxreq", + req->vsl->wid & VSL_IDENTMASK); + AZ(isnan(req->t_first)); /* First byte timestamp set by http1_wait */ + AZ(isnan(req->t_req)); /* Complete req rcvd set by http1_wait */ + req->t_prev = req->t_first; + VSLb_ts_req(req, "Start", req->t_first); + VSLb_ts_req(req, "Req", req->t_req); /* Borrow VCL reference from worker thread */ VCL_Refresh(&wrk->vcl); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index c938b98..7291a98 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -133,9 +133,6 @@ ses_sess_pool_task(struct worker *wrk, void *arg) req = SES_GetReq(wrk, sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - AN(req->vsl->wid & VSL_CLIENTMARKER); - VSLb(req->vsl, SLT_Begin, "req %u rxreq", sp->vxid & VSL_IDENTMASK); - VSL(SLT_Link, sp->vxid, "req %u rxreq", req->vsl->wid & VSL_IDENTMASK); sp->sess_step = S_STP_NEWREQ; ses_req_pool_task(wrk, req); @@ -362,7 +359,6 @@ SES_GetReq(struct worker *wrk, struct sess *sp) sz = cache_param->vsl_buffer; VSL_Setup(req->vsl, p, sz); - req->vsl->wid = VXID_Get(&wrk->vxid_pool) | VSL_CLIENTMARKER; p += sz; p = (void*)PRNDUP(p); diff --git a/bin/varnishtest/tests/r01581.vtc b/bin/varnishtest/tests/r01581.vtc new file mode 100644 index 0000000..f04eb0e --- /dev/null +++ b/bin/varnishtest/tests/r01581.vtc @@ -0,0 +1,23 @@ +varnishtest "Duplcate SLT_Begin" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { +} -start + +logexpect l1 -v v1 -g request { + expect 0 1001 Begin + expect * = ReqStart + expect * = End +} -start + +client c1 { + delay 0.1 + send "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n" + rxresp +} -run + +logexpect l1 -wait From martin at varnish-software.com Tue Sep 9 14:47:30 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 09 Sep 2014 16:47:30 +0200 Subject: [master] aa80275 Remove lingering VSL flush calls from when we used FD as log ID. Message-ID: commit aa802750fae04c36fb2ab6d23fec57dee701286c Author: Martin Blix Grydeland Date: Wed Aug 27 17:08:40 2014 +0200 Remove lingering VSL flush calls from when we used FD as log ID. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 6622f4b..9854b41 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -283,10 +283,6 @@ vbe_GetVbe(struct busyobj *bo, struct vdi_simple *vs) VSLb(bo->vsl, SLT_BackendClose, "%d %s toolate", vc->fd, bp->display_name); - /* Checkpoint log to flush all info related to this connection - before the OS reuses the FD */ - VSL_Flush(bo->vsl, 0); - VTCP_close(&vc->fd); VBE_DropRefConn(bp, NULL); vc->backend = NULL; diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c index 5d374ab..9687366 100644 --- a/bin/varnishd/cache/cache_dir.c +++ b/bin/varnishd/cache/cache_dir.c @@ -56,13 +56,7 @@ VDI_CloseFd(struct vbc **vbp, const struct acct_bereq *acct_bereq) VSLb(vc->vsl, SLT_BackendClose, "%d %s", vc->fd, bp->display_name); - /* - * Checkpoint log to flush all info related to this connection - * before the OS reuses the FD - */ - VSL_Flush(vc->vsl, 0); vc->vsl = NULL; - VTCP_close(&vc->fd); VBE_DropRefConn(bp, acct_bereq); vc->backend = NULL; @@ -88,8 +82,6 @@ VDI_RecycleFd(struct vbc **vbp, const struct acct_bereq *acct_bereq) VSLb(vc->vsl, SLT_BackendReuse, "%d %s", vc->fd, bp->display_name); - /* XXX: revisit this hack */ - VSL_Flush(vc->vsl, 0); vc->vsl = NULL; Lck_Lock(&bp->mtx); From fgsch at lodoss.net Tue Sep 9 21:59:15 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 09 Sep 2014 23:59:15 +0200 Subject: [master] f6619ba Update references to vcl_error Message-ID: commit f6619babcf166eb4baa008d67237522c32df3bec Author: Federico G. Schwindt Date: Tue Sep 9 18:26:35 2014 +0100 Update references to vcl_error Use vcl_synth or vcl_backend_error where appropriate. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 6d17e44..f514180 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -689,7 +689,7 @@ struct req { /* Temporary accounting */ struct acct_req acct; - /* Synth content in vcl_error */ + /* Synth content in vcl_synth */ struct vsb *synth_body; }; diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst index 36d51bd..deb8e42 100644 --- a/doc/sphinx/reference/vsl.rst +++ b/doc/sphinx/reference/vsl.rst @@ -95,7 +95,7 @@ Retry Backend request is being retried. Error - Backend request failed to vcl_error. + Backend request failed to vcl_backend_error. HISTORY From fgsch at lodoss.net Tue Sep 9 21:59:15 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 09 Sep 2014 23:59:15 +0200 Subject: [master] 9d69163 Spelling Message-ID: commit 9d691639a0148d4a78ba968b8f2589383cb46bc1 Author: Federico G. Schwindt Date: Tue Sep 9 18:28:24 2014 +0100 Spelling diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index a74e6a4..1aaec4c 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -225,7 +225,7 @@ vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label) CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); VSLb_ts_busyobj(ctx->bo, label, VTIM_real()); } else if (ctx->req != NULL) { - /* Called from request vcl methdos */ + /* Called from request vcl methods */ CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); VSLb_ts_req(ctx->req, label, VTIM_real()); } From fgsch at lodoss.net Tue Sep 9 21:59:15 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 09 Sep 2014 23:59:15 +0200 Subject: [master] fdee70d Remove duplicated line Message-ID: commit fdee70d70faba04569029fc04ae8352cdc79b71f Author: Federico G. Schwindt Date: Tue Sep 9 18:28:39 2014 +0100 Remove duplicated line diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 1011946..7e0bcb1 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -731,7 +731,6 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) bo->synth_body = NULL; if (bo->retries++ < cache_param->max_retries) return (F_STP_RETRY); - bo->synth_body = NULL; return (F_STP_FAIL); } From phk at FreeBSD.org Wed Sep 10 07:48:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 10 Sep 2014 09:48:52 +0200 Subject: [master] 7221cf7 Rename REQ_BODY_ modes to WITH[OUT]_LEN rather than transmission mode. Message-ID: commit 7221cf79675933c89c5db87e85c54d180126211e Author: Poul-Henning Kamp Date: Wed Sep 10 07:48:01 2014 +0000 Rename REQ_BODY_ modes to WITH[OUT]_LEN rather than transmission mode. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index f514180..8080082 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -1111,7 +1111,7 @@ void SES_Delete(struct sess *sp, enum sess_close reason, double now); struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no); void SES_DeletePool(struct sesspool *sp); int SES_ScheduleReq(struct req *); -struct req *SES_GetReq(struct worker *, struct sess *); +struct req *SES_GetReq(const struct worker *, struct sess *); void SES_Handle(struct sess *sp, double now); void SES_ReleaseReq(struct req *); pool_func_t SES_pool_accept_task; diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index 3ad3eee..7043ac5 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -139,7 +139,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req) if (!http_GetHdr(bo->bereq, H_Host, NULL)) VDI_AddHostHeader(bo->bereq, vc); - if (req != NULL && req->req_body_status == REQ_BODY_CHUNKED) { + if (req != NULL && req->req_body_status == REQ_BODY_WITHOUT_LEN) { http_PrintfHeader(hp, "Transfer-Encoding: chunked"); do_chunked = 1; } diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index bd80c79..ca6794b 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -298,13 +298,13 @@ http1_dissect(struct worker *wrk, struct req *req) assert (req->req_body_status == REQ_BODY_INIT); if (req->htc->body_status == BS_CHUNKED) { - req->req_body_status = REQ_BODY_CHUNKED; + req->req_body_status = REQ_BODY_WITHOUT_LEN; } else if (req->htc->body_status == BS_LENGTH) { - req->req_body_status = REQ_BODY_PRESENT; + req->req_body_status = REQ_BODY_WITH_LEN; } else if (req->htc->body_status == BS_NONE) { req->req_body_status = REQ_BODY_NONE; } else if (req->htc->body_status == BS_EOF) { - req->req_body_status = REQ_BODY_CHUNKED; + req->req_body_status = REQ_BODY_WITHOUT_LEN; } else { WRONG("Unknown req.body_length situation"); } @@ -462,8 +462,8 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv) return (0); case REQ_BODY_NONE: return (0); - case REQ_BODY_PRESENT: - case REQ_BODY_CHUNKED: + case REQ_BODY_WITH_LEN: + case REQ_BODY_WITHOUT_LEN: break; case REQ_BODY_TAKEN: VSLb(req->vsl, SLT_VCL_Error, @@ -477,8 +477,8 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv) WRONG("Wrong req_body_status in HTTP1_IterateReqBody()"); } Lck_Lock(&req->sp->mtx); - if (req->req_body_status == REQ_BODY_PRESENT || - req->req_body_status == REQ_BODY_CHUNKED) { + if (req->req_body_status == REQ_BODY_WITH_LEN || + req->req_body_status == REQ_BODY_WITHOUT_LEN) { req->req_body_status = REQ_BODY_TAKEN; i = 0; } else @@ -545,8 +545,8 @@ int HTTP1_DiscardReqBody(struct req *req) { - if (req->req_body_status == REQ_BODY_PRESENT || - req->req_body_status == REQ_BODY_CHUNKED) + if (req->req_body_status == REQ_BODY_WITH_LEN || + req->req_body_status == REQ_BODY_WITHOUT_LEN) (void)HTTP1_IterateReqBody(req, httpq_req_body_discard, NULL); return(0); } @@ -575,8 +575,8 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize) return (-1); case REQ_BODY_NONE: return (0); - case REQ_BODY_CHUNKED: - case REQ_BODY_PRESENT: + case REQ_BODY_WITHOUT_LEN: + case REQ_BODY_WITH_LEN: break; default: WRONG("Wrong req_body_status in HTTP1_CacheReqBody()"); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index bd48b45..b6d8049 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -315,7 +315,7 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now) */ struct req * -SES_GetReq(struct worker *wrk, struct sess *sp) +SES_GetReq(const struct worker *wrk, struct sess *sp) { struct sesspool *pp; struct req *req; diff --git a/bin/varnishtest/tests/r01581.vtc b/bin/varnishtest/tests/r01581.vtc index f04eb0e..9ba44b1 100644 --- a/bin/varnishtest/tests/r01581.vtc +++ b/bin/varnishtest/tests/r01581.vtc @@ -11,7 +11,7 @@ varnish v1 -vcl+backend { logexpect l1 -v v1 -g request { expect 0 1001 Begin expect * = ReqStart - expect * = End + expect * = End } -start client c1 { diff --git a/include/tbl/req_body.h b/include/tbl/req_body.h index 9e58deb..d28ada4 100644 --- a/include/tbl/req_body.h +++ b/include/tbl/req_body.h @@ -30,8 +30,8 @@ /*lint -save -e525 -e539 */ REQ_BODY(INIT) -REQ_BODY(PRESENT) -REQ_BODY(CHUNKED) +REQ_BODY(WITHOUT_LEN) +REQ_BODY(WITH_LEN) REQ_BODY(TAKEN) REQ_BODY(CACHED) REQ_BODY(FAIL) From phk at FreeBSD.org Wed Sep 10 08:03:13 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 10 Sep 2014 10:03:13 +0200 Subject: [master] e93ce21 Minor polish Message-ID: commit e93ce21341a23d2d52cdd20969c478535c32973e Author: Poul-Henning Kamp Date: Wed Sep 10 08:03:02 2014 +0000 Minor polish diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index ca6794b..499c7b9 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -571,6 +571,7 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize) assert (req->req_step == R_STP_RECV); switch(req->req_body_status) { case REQ_BODY_CACHED: + return (0); case REQ_BODY_FAIL: return (-1); case REQ_BODY_NONE: @@ -587,6 +588,7 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize) if (req->htc->content_length > maxsize) { req->req_body_status = REQ_BODY_FAIL; + (void)VFP_Error(vfc, "Request body too big to cache"); return (-1); } @@ -600,7 +602,6 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize) return (-1); } - yet = req->htc->content_length; if (yet < 0) yet = 0; @@ -642,7 +643,6 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize) } while (vfps == VFP_OK); VFP_Close(vfc); - if (l == 0) { if (req->req_bodybytes != req->htc->content_length) { From phk at FreeBSD.org Wed Sep 10 08:30:57 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 10 Sep 2014 10:30:57 +0200 Subject: [master] c337a6a Use WRW's facility for sending chunked req.body to backend Message-ID: commit c337a6a29bfad1eed7c1fe266e8af23ff9f6a47f Author: Poul-Henning Kamp Date: Wed Sep 10 08:30:33 2014 +0000 Use WRW's facility for sending chunked req.body to backend diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index 7043ac5..c593c32 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -43,31 +43,6 @@ #include "vtim.h" /*-------------------------------------------------------------------- - * Pass the request body to the backend with chunks - */ - -static int __match_proto__(req_body_iter_f) -vbf_iter_req_body_chunked(struct req *req, void *priv, void *ptr, size_t l) -{ - struct worker *wrk; - char buf[20]; - - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CAST_OBJ_NOTNULL(wrk, priv, WORKER_MAGIC); - - if (l > 0) { - bprintf(buf, "%jx\r\n", (uintmax_t)l); - VSLb(req->vsl, SLT_Debug, "WWWW: %s", buf); - (void)WRW_Write(wrk, buf, strlen(buf)); - (void)WRW_Write(wrk, ptr, l); - (void)WRW_Write(wrk, "\r\n", 2); - if (WRW_Flush(wrk)) - return (-1); - } - return (0); -} - -/*-------------------------------------------------------------------- * Pass the request body to the backend */ @@ -152,13 +127,10 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req) i = 0; if (req != NULL) { - if (do_chunked) { - i = HTTP1_IterateReqBody(req, - vbf_iter_req_body_chunked, wrk); - (void)WRW_Write(wrk, "0\r\n\r\n", 5); - } else { - i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk); - } + if (do_chunked) + WRW_Chunked(wrk); + i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk); + if (req->req_body_status == REQ_BODY_TAKEN) { retry = -1; } else if (req->req_body_status == REQ_BODY_FAIL) { @@ -168,6 +140,8 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req) req->doclose = SC_RX_BODY; retry = -1; } + if (do_chunked) + WRW_EndChunk(wrk); } j = WRW_FlushRelease(wrk, &bo->acct.bereq_hdrbytes); From fgsch at lodoss.net Thu Sep 11 12:24:55 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 11 Sep 2014 14:24:55 +0200 Subject: [master] 9d70a57 Fix reference Message-ID: commit 9d70a57e7212251e239d28c2f2202de02643f71d Author: Federico G. Schwindt Date: Thu Sep 11 13:24:24 2014 +0100 Fix reference Spotted by: Dridi Boukelmoune diff --git a/doc/sphinx/tutorial/now_what.rst b/doc/sphinx/tutorial/now_what.rst index 1e23e10..90e28f4 100644 --- a/doc/sphinx/tutorial/now_what.rst +++ b/doc/sphinx/tutorial/now_what.rst @@ -7,5 +7,5 @@ Now what? You've read through the tutorial. You should have Varnish up and running. You should know about the logs and you should have a rough idea of what VCL is. Next, you might want to have a look at -:ref:``users-guide-index``, where we go through the features of +:ref:`users-guide-index`, where we go through the features of Varnish in more detail. From phk at FreeBSD.org Thu Sep 11 13:56:37 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 11 Sep 2014 15:56:37 +0200 Subject: [master] 4be6020 Split beresp.body status determination into transport protocol dependent (is there a C-L header ?), and semantic (ie: HEAD request doesn't have body, even if the response has C-L etc.) Message-ID: commit 4be6020bd72be0f90d904f1bf5ccb56454015dcd Author: Poul-Henning Kamp Date: Thu Sep 11 13:18:10 2014 +0000 Split beresp.body status determination into transport protocol dependent (is there a C-L header ?), and semantic (ie: HEAD request doesn't have body, even if the response has C-L etc.) diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 8080082..b7591c8 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -1228,7 +1228,6 @@ void *WS_Printf(struct ws *ws, const char *fmt, ...) __printflike(2, 3); /* cache_rfc2616.c */ void RFC2616_Ttl(struct busyobj *, double now); -enum body_status RFC2616_Body(struct busyobj *, struct dstat *); unsigned RFC2616_Req_Gzip(const struct http *); int RFC2616_Do_Cond(const struct req *sp); void RFC2616_Weaken_Etag(struct http *hp); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 7e0bcb1..3d48293 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -314,9 +314,52 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) /* * Figure out how the fetch is supposed to happen, before the * headers are adultered by VCL - * NB: Also sets other wrk variables */ - bo->htc->body_status = RFC2616_Body(bo, &wrk->stats); + if (!strcasecmp(http_GetMethod(bo->bereq), "head")) { + /* + * A HEAD request can never have a body in the reply, + * no matter what the headers might say. + * [RFC2516 4.3 p33] + */ + wrk->stats.fetch_head++; + bo->htc->body_status = BS_NONE; + } else if (http_GetStatus(bo->beresp) <= 199) { + /* + * 1xx responses never have a body. + * [RFC2616 4.3 p33] + * ... but we should never see them. + */ + wrk->stats.fetch_1xx++; + bo->htc->body_status = BS_ERROR; + } else if (http_IsStatus(bo->beresp, 204)) { + /* + * 204 is "No Content", obviously don't expect a body. + * [RFC2616 10.2.5 p60] + */ + wrk->stats.fetch_204++; + bo->htc->body_status = BS_NONE; + } else if (http_IsStatus(bo->beresp, 304)) { + /* + * 304 is "Not Modified" it has no body. + * [RFC2616 10.3.5 p63] + */ + wrk->stats.fetch_304++; + bo->htc->body_status = BS_NONE; + } else if (bo->htc->body_status == BS_CHUNKED) { + wrk->stats.fetch_chunked++; + } else if (bo->htc->body_status == BS_LENGTH) { + assert(bo->htc->content_length > 0); + bo->content_length = bo->htc->content_length; + wrk->stats.fetch_length++; + } else if (bo->htc->body_status == BS_EOF) { + wrk->stats.fetch_eof++; + } else if (bo->htc->body_status == BS_ERROR) { + wrk->stats.fetch_bad++; + } else if (bo->htc->body_status == BS_NONE) { + wrk->stats.fetch_none++; + } else { + WRONG("wrong bodystatus"); + } if (bo->htc->body_status == BS_ERROR) { AN (bo->vbc); diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index dcd5102..51ba2c4 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -181,116 +181,6 @@ RFC2616_Ttl(struct busyobj *bo, double now) } /*-------------------------------------------------------------------- - * Body existence, fetch method and close policy. - */ - -enum body_status -RFC2616_Body(struct busyobj *bo, struct dstat *stats) -{ - struct http *hp; - char *b; - ssize_t cl; - - hp = bo->beresp; - - if (!strcasecmp(http_GetMethod(bo->bereq), "head")) { - /* - * A HEAD request can never have a body in the reply, - * no matter what the headers might say. - * [RFC2516 4.3 p33] - */ - stats->fetch_head++; - return (BS_NONE); - } - - if (http_GetStatus(hp) <= 199) { - /* - * 1xx responses never have a body. - * [RFC2616 4.3 p33] - * ... but we should never see them. - */ - stats->fetch_1xx++; - return (BS_ERROR); - } - - if (http_IsStatus(hp, 204)) { - /* - * 204 is "No Content", obviously don't expect a body. - * [RFC2616 10.2.5 p60] - */ - stats->fetch_204++; - return (BS_NONE); - } - - if (http_IsStatus(hp, 304)) { - /* - * 304 is "Not Modified" it has no body. - * [RFC2616 10.3.5 p63] - */ - stats->fetch_304++; - return (BS_NONE); - } - - if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) { - stats->fetch_chunked++; - return (BS_CHUNKED); - } - - if (http_GetHdr(hp, H_Transfer_Encoding, &b)) { - VSLb(bo->vsl, SLT_Error, "Illegal Transfer-Encoding:"); - stats->fetch_bad++; - return (BS_ERROR); - } - - cl = http_GetContentLength(hp); - if (cl == -2) { - VSLb(bo->vsl, SLT_Error, "Bad Content-Length:"); - stats->fetch_bad++; - return (BS_ERROR); - } - if (cl > 0) { - stats->fetch_length++; - bo->content_length = cl; - return (BS_LENGTH); - } - if (cl == 0) { - stats->fetch_length++; - return (BS_NONE); - } - - if (http_HdrIs(hp, H_Connection, "keep-alive")) { - /* - * Keep alive with neither TE=Chunked or C-Len is impossible. - * We assume a zero length body. - */ - stats->fetch_zero++; - return (BS_NONE); - } - - if (http_HdrIs(hp, H_Connection, "close")) { - /* - * In this case, it is safe to just read what comes. - */ - stats->fetch_close++; - return (BS_EOF); - } - - if (hp->protover < 11) { - /* - * With no Connection header, assume EOF. - */ - stats->fetch_oldhttp++; - return (BS_EOF); - } - - /* - * Fall back to EOF transfer. - */ - stats->fetch_eof++; - return (BS_EOF); -} - -/*-------------------------------------------------------------------- * Find out if the request can receive a gzip'ed response */ diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h index 845fbcb..72c2568 100644 --- a/include/tbl/vsc_f_main.h +++ b/include/tbl/vsc_f_main.h @@ -197,31 +197,23 @@ VSC_F(fetch_head, uint64_t, 1, 'c', info, ) VSC_F(fetch_length, uint64_t, 1, 'c', info, "Fetch with Length", - "beresp with Content-Length." + "beresp.body with Content-Length." ) VSC_F(fetch_chunked, uint64_t, 1, 'c', info, "Fetch chunked", - "beresp with Chunked." + "beresp.body with Chunked." ) VSC_F(fetch_eof, uint64_t, 1, 'c', info, "Fetch EOF", - "beresp with EOF from lack of other info." + "beresp.body with EOF." ) VSC_F(fetch_bad, uint64_t, 1, 'c', info, "Fetch bad T-E", - "beresp failed due to unknown Transfer-Encoding." + "beresp.body length/fetch could not be determined." ) -VSC_F(fetch_close, uint64_t, 1, 'c', info, - "Fetch wanted close", - "beresp with EOF due to Connection: Close." -) -VSC_F(fetch_oldhttp, uint64_t, 1, 'c', info, - "Fetch pre HTTP/1.1 closed", - "beresp with EOF due to HTTP < 1.1" -) -VSC_F(fetch_zero, uint64_t, 1, 'c', info, - "Fetch zero len body", - "beresp with EOF due to keep-live but neither Chunked or Len." +VSC_F(fetch_none, uint64_t, 1, 'c', info, + "Fetch no body", + "beresp.body empty" ) VSC_F(fetch_1xx, uint64_t, 1, 'c', info, "Fetch no body (1xx)", From lkarsten at varnish-software.com Thu Sep 11 15:35:41 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Thu, 11 Sep 2014 17:35:41 +0200 Subject: [master] 1b72b66 Start ignoring lower case ctags file. Message-ID: commit 1b72b668ac62f863bd9531265f9042d730669e9c Author: Lasse Karstensen Date: Thu Sep 11 14:54:32 2014 +0200 Start ignoring lower case ctags file. diff --git a/.gitignore b/.gitignore index b4ae99b..460e76a 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ Makefile.in /varnishapi.pc /build-aux TAGS +tags cscope.*out # /varnishapi-uninstalled.pc From nils.goroll at uplex.de Sun Sep 14 11:31:44 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 14 Sep 2014 13:31:44 +0200 Subject: [master] d2014e9 clarify that we will not see piped connections ever again Message-ID: commit d2014e93f1472d666a1b0d84cb308d7ce38aa537 Author: Nils Goroll Date: Sun Sep 14 13:31:35 2014 +0200 clarify that we will not see piped connections ever again diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst index 5280eb4..9c9c690 100644 --- a/doc/sphinx/users-guide/vcl-built-in-subs.rst +++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst @@ -43,11 +43,12 @@ of the following keywords: vcl_pipe ~~~~~~~~ -Called upon entering pipe mode. In this mode, the request is passed -on to the backend, and any further data from either client or backend +Called upon entering pipe mode. In this mode, the request is passed on +to the backend, and any further data from both the client and backend is passed on unaltered until either end closes the connection. Basically, Varnish will degrade into a simple TCP proxy, -shuffling bytes back and forth. +shuffling bytes back and forth. For a connection in pipe mode, no +other VCL subroutine will ever get called after `vcl_pipe`. The `vcl_pipe` subroutine may terminate with calling ``return()`` with one of the following keywords: From phk at FreeBSD.org Mon Sep 15 06:37:56 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 08:37:56 +0200 Subject: [master] 287e405 Eliminate bo->content_length Message-ID: commit 287e405a7f79d2d702a355751a4d5658b076be3f Author: Poul-Henning Kamp Date: Thu Sep 11 20:21:53 2014 +0000 Eliminate bo->content_length diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index b7591c8..7c18ae9 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -517,8 +517,6 @@ struct busyobj { struct pool_task fetch_task; - ssize_t content_length; - enum sess_close doclose; #define BO_FLAG(l, r, w, d) unsigned l:1; diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index a3f4092..a0c0666 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -149,7 +149,6 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req) VCL_Ref(bo->vcl); bo->t_first = bo->t_prev = NAN; - bo->content_length = -1; bo->doclose = SC_NULL; return (bo); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 3d48293..79db1dd 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -349,7 +349,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) wrk->stats.fetch_chunked++; } else if (bo->htc->body_status == BS_LENGTH) { assert(bo->htc->content_length > 0); - bo->content_length = bo->htc->content_length; wrk->stats.fetch_length++; } else if (bo->htc->body_status == BS_EOF) { wrk->stats.fetch_eof++; @@ -451,7 +450,7 @@ vbf_fetch_body_helper(struct busyobj *bo) AN(vfc->vfp_nxt); - est = bo->content_length; + est = bo->htc->content_length; if (est < 0) est = 0; From phk at FreeBSD.org Mon Sep 15 06:37:56 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 08:37:56 +0200 Subject: [master] f6c846c Add statistics to VFP's: number of calls and number of bytes returned. Message-ID: commit f6c846c384336d7d6555f55da342f0a08a4b7d35 Author: Poul-Henning Kamp Date: Mon Sep 15 06:37:06 2014 +0000 Add statistics to VFP's: number of calls and number of bytes returned. Now I just need to figure out where to stuff them. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 7c18ae9..df8b562 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -210,6 +210,8 @@ struct vfp_entry { intptr_t priv2; enum vfp_status closed; VTAILQ_ENTRY(vfp_entry) list; + uint64_t calls; + uint64_t bytes_out; }; VTAILQ_HEAD(vfp_entry_s, vfp_entry); @@ -229,7 +231,6 @@ struct vfp_ctx { struct vsl_log *vsl; struct http *http; struct http *esi_req; - uint64_t bodybytes; }; /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index fda8cb6..bec9225 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -165,19 +165,23 @@ VFP_Suck(struct vfp_ctx *vc, void *p, ssize_t *lp) vc->vfp_nxt = VTAILQ_NEXT(vfe, list); if (vfe->closed == VFP_NULL) { + /* Layer asked to be bypassed when opened */ vp = VFP_Suck(vc, p, lp); } else if (vfe->closed == VFP_OK) { vp = vfe->vfp->pull(vc, vfe, p, lp); - if (vp == VFP_END || vp == VFP_ERROR) { - vfe->closed = vp; - } else if (vp != VFP_OK) + if (vp != VFP_OK && vp != VFP_END && vp != VFP_ERROR) { (void)VFP_Error(vc, "Fetch filter %s returned %d", vfe->vfp->name, vp); + vp = VFP_ERROR; + } + vfe->closed = vp; } else { /* Already closed filter */ *lp = 0; vp = vfe->closed; } + vfe->calls++; + vfe->bytes_out = *lp; vc->vfp_nxt = vfe; return (vp); } From phk at FreeBSD.org Mon Sep 15 06:45:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 08:45:41 +0200 Subject: [master] dfc78bf Centralize counting of beresp.bodybytes Message-ID: commit dfc78bfab0e481816dfa30a8035c9bf0d606cc73 Author: Poul-Henning Kamp Date: Mon Sep 15 06:45:24 2014 +0000 Centralize counting of beresp.bodybytes diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 79db1dd..c4b4332 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -479,6 +479,7 @@ vbf_fetch_body_helper(struct busyobj *bo) AZ(vfc->failed); vfps = VFP_Suck(vfc, ptr, &l); if (l > 0 && vfps != VFP_ERROR) { + bo->acct.beresp_bodybytes += l; VBO_extend(bo, l); if (est >= l) est -= l; @@ -622,7 +623,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) if (bo->htc->body_status != BS_NONE) { assert(bo->htc->body_status != BS_ERROR); vbf_fetch_body_helper(bo); - bo->acct.beresp_bodybytes = bo->vfc->bodybytes; } if (bo->vfc->failed && !bo->do_stream) { diff --git a/bin/varnishd/cache/cache_http1_vfp.c b/bin/varnishd/cache/cache_http1_vfp.c index 4e2fa29..33a48c5 100644 --- a/bin/varnishd/cache/cache_http1_vfp.c +++ b/bin/varnishd/cache/cache_http1_vfp.c @@ -75,7 +75,6 @@ v1f_pull_chunked(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, lr = HTTP1_Read(htc, buf, 1); if (lr <= 0) return (VFP_Error(vc, "chunked read err")); - vc->bodybytes += lr; } while (vct_islws(buf[0])); if (!vct_ishex(buf[0])) @@ -88,7 +87,6 @@ v1f_pull_chunked(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, if (lr <= 0) return (VFP_Error(vc, "chunked read err")); - vc->bodybytes += lr; } while (u == 1 && buf[0] == '0' && buf[u] == '0'); if (!vct_ishex(buf[u])) break; @@ -102,7 +100,6 @@ v1f_pull_chunked(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, lr = HTTP1_Read(htc, buf + u, 1); if (lr <= 0) return (VFP_Error(vc, "chunked read err")); - vc->bodybytes += lr; } if (buf[u] != '\n') @@ -125,7 +122,6 @@ v1f_pull_chunked(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, lr = HTTP1_Read(htc, ptr, l); if (lr <= 0) return (VFP_Error(vc, "straight insufficient bytes")); - vc->bodybytes += lr; *lp = lr; vfe->priv2 -= lr; if (vfe->priv2 == 0) @@ -136,7 +132,6 @@ v1f_pull_chunked(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, i = HTTP1_Read(htc, buf, 1); if (i <= 0) return (VFP_Error(vc, "chunked read err")); - vc->bodybytes += i; if (buf[0] == '\r' && HTTP1_Read(htc, buf, 1) <= 0) return (VFP_Error(vc, "chunked read err")); if (buf[0] != '\n') @@ -173,7 +168,6 @@ v1f_pull_straight(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, if (vfe->priv2 < l) l = vfe->priv2; lr = HTTP1_Read(htc, p, l); - vc->bodybytes += lr; if (lr <= 0) return (VFP_Error(vc, "straight insufficient bytes")); *lp = lr; @@ -212,7 +206,6 @@ v1f_pull_eof(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, if (lr == 0) return (VFP_END); *lp = lr; - vc->bodybytes += lr; return (VFP_OK); } From phk at FreeBSD.org Mon Sep 15 07:59:04 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 09:59:04 +0200 Subject: [master] 2f057a3 Report VFP statistics in their own suppressed SLT_VfpAcct until we know what we really want with them. Message-ID: commit 2f057a329074d281b6031c413a0d1a044fc54840 Author: Poul-Henning Kamp Date: Mon Sep 15 07:58:27 2014 +0000 Report VFP statistics in their own suppressed SLT_VfpAcct until we know what we really want with them. diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index bec9225..9cf0aa4 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -118,9 +118,12 @@ VFP_Close(struct vfp_ctx *vc) { struct vfp_entry *vfe; - VTAILQ_FOREACH(vfe, &vc->vfp, list) + VTAILQ_FOREACH(vfe, &vc->vfp, list) { if(vfe->vfp->fini != NULL) vfe->vfp->fini(vc, vfe); + VSLb(vc->vsl, SLT_VfpAcct, "%s %ju %ju", + vfe->vfp->name, vfe->calls, vfe->bytes_out); + } } int @@ -131,6 +134,7 @@ VFP_Open(struct vfp_ctx *vc) CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vc->http, HTTP_MAGIC); AN(vc->vsl); + VTAILQ_FOREACH_REVERSE(vfe, &vc->vfp, vfp_entry_s, list) { if (vfe->vfp->init == NULL) continue; @@ -142,6 +146,7 @@ VFP_Open(struct vfp_ctx *vc) return (-1); } } + return (0); } @@ -175,13 +180,13 @@ VFP_Suck(struct vfp_ctx *vc, void *p, ssize_t *lp) vp = VFP_ERROR; } vfe->closed = vp; + vfe->calls++; + vfe->bytes_out += *lp; } else { /* Already closed filter */ *lp = 0; vp = vfe->closed; } - vfe->calls++; - vfe->bytes_out = *lp; vc->vfp_nxt = vfe; return (vp); } diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c index 270ed1f..32a4593 100644 --- a/bin/varnishd/mgt/mgt_param_bits.c +++ b/bin/varnishd/mgt/mgt_param_bits.c @@ -130,6 +130,7 @@ tweak_vsl_mask(struct vsb *vsb, const struct parspec *par, const char *arg) (void)bit(mgt_param.vsl_mask, SLT_VCL_trace, BSET); (void)bit(mgt_param.vsl_mask, SLT_WorkThread, BSET); (void)bit(mgt_param.vsl_mask, SLT_Hash, BSET); + (void)bit(mgt_param.vsl_mask, SLT_VfpAcct, BSET); } else { return (bit_tweak(vsb, mgt_param.vsl_mask, SLT__Reserved, arg, VSL_tags, diff --git a/bin/varnishtest/tests/g00002.vtc b/bin/varnishtest/tests/g00002.vtc index 6c1597c..1418212 100644 --- a/bin/varnishtest/tests/g00002.vtc +++ b/bin/varnishtest/tests/g00002.vtc @@ -23,6 +23,7 @@ varnish v1 \ } -start varnish v1 -cliok "param.set fetch_chunksize 4k" +varnish v1 -cliok "param.set vsl_mask +VfpAcct" client c1 { txreq -url /foo -hdr "Accept-Encoding: gzip" diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 18b93f8..0fc3b86 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -499,4 +499,16 @@ SLTM(BereqAcct, 0, "Backend request accounting", "\n" ) +SLTM(VfpAcct, 0, "Fetch filter accounting", + "Contains name of VFP and statistics.\n\n" + "The format is::\n\n" + "\t%s %d %d\n" + "\t| | |\n" + "\t| | +- Total bytes produced\n" + "\t| +---- Number of calls made\n" + "\t+------- Name of filter\n" + "\n" + NODEF_NOTICE +) + #undef NODEF_NOTICE From phk at FreeBSD.org Mon Sep 15 08:50:32 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 10:50:32 +0200 Subject: [master] 036bd79 Move the req.body stuff to its own source-file, its not HTTP1 specific. Message-ID: commit 036bd79f6e6d851bef2af330b909ba769e0d9917 Author: Poul-Henning Kamp Date: Mon Sep 15 08:50:14 2014 +0000 Move the req.body stuff to its own source-file, its not HTTP1 specific. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index ddbd1d2..edb0e31 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -37,6 +37,7 @@ varnishd_SOURCES = \ cache/cache_panic.c \ cache/cache_pipe.c \ cache/cache_pool.c \ + cache/cache_req_body.c \ cache/cache_req_fsm.c \ cache/cache_http1_deliver.c \ cache/cache_rfc2616.c \ diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index df8b562..c76f7c1 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -812,9 +812,6 @@ void V1F_Setup_Fetch(struct vfp_ctx *vfc, struct http_conn *htc); /* cache_http1_fsm.c [HTTP1] */ typedef int (req_body_iter_f)(struct req *, void *priv, void *ptr, size_t); void HTTP1_Session(struct worker *, struct req *); -int HTTP1_DiscardReqBody(struct req *req); -int HTTP1_CacheReqBody(struct req *req, ssize_t maxsize); -int HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv); extern const int HTTP1_Req[3]; extern const int HTTP1_Resp[3]; @@ -823,6 +820,11 @@ unsigned V1D_FlushReleaseAcct(struct req *req); void V1D_Deliver(struct req *, struct busyobj *); void V1D_Deliver_Synth(struct req *req); +/* cache_req_body.c */ +int VRB_Ignore(struct req *req); +int VRB_Cache(struct req *req, ssize_t maxsize); +int VRB_Iterate(struct req *req, req_body_iter_f *func, void *priv); +void VRB_Free(struct req *req); static inline int VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len) diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index c593c32..3874fb5 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -129,7 +129,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req) if (req != NULL) { if (do_chunked) WRW_Chunked(wrk); - i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk); + i = VRB_Iterate(req, vbf_iter_req_body, wrk); if (req->req_body_status == REQ_BODY_TAKEN) { retry = -1; diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 499c7b9..2a6b781 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -431,235 +431,3 @@ HTTP1_Session(struct worker *wrk, struct req *req) } } } - -/*---------------------------------------------------------------------- - * Iterate over the req.body. - * - * This can be done exactly once if uncached, and multiple times if the - * req.body is cached. - */ - -int -HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv) -{ - char buf[8192]; - struct storage *st; - ssize_t l; - int i; - struct vfp_ctx *vfc; - enum vfp_status vfps = VFP_ERROR; - - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - AN(func); - - switch(req->req_body_status) { - case REQ_BODY_CACHED: - VTAILQ_FOREACH(st, &req->body->list, list) { - i = func(req, priv, st->ptr, st->len); - if (i) - return (i); - } - return (0); - case REQ_BODY_NONE: - return (0); - case REQ_BODY_WITH_LEN: - case REQ_BODY_WITHOUT_LEN: - break; - case REQ_BODY_TAKEN: - VSLb(req->vsl, SLT_VCL_Error, - "Uncached req.body can only be consumed once."); - return (-1); - case REQ_BODY_FAIL: - VSLb(req->vsl, SLT_FetchError, - "Had failed reading req.body before."); - return (-1); - default: - WRONG("Wrong req_body_status in HTTP1_IterateReqBody()"); - } - Lck_Lock(&req->sp->mtx); - if (req->req_body_status == REQ_BODY_WITH_LEN || - req->req_body_status == REQ_BODY_WITHOUT_LEN) { - req->req_body_status = REQ_BODY_TAKEN; - i = 0; - } else - i = -1; - Lck_Unlock(&req->sp->mtx); - if (i) { - VSLb(req->vsl, SLT_VCL_Error, - "Multiple attempts to access non-cached req.body"); - return (i); - } - - CHECK_OBJ_NOTNULL(req->htc, HTTP_CONN_MAGIC); - vfc = req->htc->vfc; - VFP_Setup(vfc); - vfc->http = req->http; - vfc->vsl = req->vsl; - V1F_Setup_Fetch(vfc, req->htc); - if (VFP_Open(vfc) < 0) { - VSLb(req->vsl, SLT_FetchError, "Could not open Fetch Pipeline"); - return (-1); - } - - do { - l = sizeof buf; - vfps = VFP_Suck(vfc, buf, &l); - if (vfps == VFP_ERROR) { - req->req_body_status = REQ_BODY_FAIL; - l = -1; - break; - } else if (l > 0) { - req->req_bodybytes += l; - req->acct.req_bodybytes += l; - l = func(req, priv, buf, l); - if (l) { - req->req_body_status = REQ_BODY_FAIL; - break; - } - } - } while (vfps == VFP_OK); - VFP_Close(vfc); - VSLb_ts_req(req, "ReqBody", VTIM_real()); - - return (l); -} - -/*---------------------------------------------------------------------- - * DiscardReqBody() is a dedicated function, because we might - * be able to disuade or terminate its transmission in some protocols. - * For HTTP1 we have no such luck, and we just iterate it into oblivion. - */ - -static int __match_proto__(req_body_iter_f) -httpq_req_body_discard(struct req *req, void *priv, void *ptr, size_t len) -{ - - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - (void)priv; - (void)ptr; - (void)len; - return (0); -} - -int -HTTP1_DiscardReqBody(struct req *req) -{ - - if (req->req_body_status == REQ_BODY_WITH_LEN || - req->req_body_status == REQ_BODY_WITHOUT_LEN) - (void)HTTP1_IterateReqBody(req, httpq_req_body_discard, NULL); - return(0); -} - -/*---------------------------------------------------------------------- - * Cache the req.body if it is smaller than the given size - * - * This function must be called before any backend fetches are kicked - * off to prevent parallelism. - */ - -int -HTTP1_CacheReqBody(struct req *req, ssize_t maxsize) -{ - struct storage *st; - ssize_t l, yet; - struct vfp_ctx *vfc; - enum vfp_status vfps = VFP_ERROR; - - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - - assert (req->req_step == R_STP_RECV); - switch(req->req_body_status) { - case REQ_BODY_CACHED: - return (0); - case REQ_BODY_FAIL: - return (-1); - case REQ_BODY_NONE: - return (0); - case REQ_BODY_WITHOUT_LEN: - case REQ_BODY_WITH_LEN: - break; - default: - WRONG("Wrong req_body_status in HTTP1_CacheReqBody()"); - } - - CHECK_OBJ_NOTNULL(req->htc, HTTP_CONN_MAGIC); - vfc = req->htc->vfc; - - if (req->htc->content_length > maxsize) { - req->req_body_status = REQ_BODY_FAIL; - (void)VFP_Error(vfc, "Request body too big to cache"); - return (-1); - } - - VFP_Setup(vfc); - vfc->http = req->http; - vfc->vsl = req->vsl; - V1F_Setup_Fetch(vfc, req->htc); - - if (VFP_Open(vfc) < 0) { - req->req_body_status = REQ_BODY_FAIL; - return (-1); - } - - yet = req->htc->content_length; - if (yet < 0) - yet = 0; - st = NULL; - do { - if (st == NULL) { - st = STV_alloc_transient( - yet ? yet : cache_param->fetch_chunksize); - if (st == NULL) { - req->req_body_status = REQ_BODY_FAIL; - l = -1; - break; - } else { - VTAILQ_INSERT_TAIL(&req->body->list, st, list); - } - } - l = st->space - st->len; - vfps = VFP_Suck(vfc, st->ptr + st->len, &l); - if (vfps == VFP_ERROR) { - req->req_body_status = REQ_BODY_FAIL; - l = -1; - break; - } - if (l > 0) { - req->req_bodybytes += l; - req->acct.req_bodybytes += l; - if (yet > 0) - yet -= l; - st->len += l; - if (st->space == st->len) - st = NULL; - l = 0; - } - if (req->req_bodybytes > maxsize) { - req->req_body_status = REQ_BODY_FAIL; - l = -1; - break; - } - } while (vfps == VFP_OK); - VFP_Close(vfc); - - if (l == 0) { - - if (req->req_bodybytes != req->htc->content_length) { - /* We must update also the "pristine" req.* copy */ - http_Unset(req->http0, H_Content_Length); - http_Unset(req->http0, H_Transfer_Encoding); - http_PrintfHeader(req->http0, "Content-Length: %ju", - (uintmax_t)req->req_bodybytes); - - http_Unset(req->http, H_Content_Length); - http_Unset(req->http, H_Transfer_Encoding); - http_PrintfHeader(req->http, "Content-Length: %ju", - (uintmax_t)req->req_bodybytes); - } - - req->req_body_status = REQ_BODY_CACHED; - } - VSLb_ts_req(req, "ReqBody", VTIM_real()); - return (l); -} diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c new file mode 100644 index 0000000..6c96f74 --- /dev/null +++ b/bin/varnishd/cache/cache_req_body.c @@ -0,0 +1,288 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2014 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include "config.h" + +#include +#include + +#include "cache.h" +#include "vtim.h" + +/*---------------------------------------------------------------------- + * Iterate over the req.body. + * + * This can be done exactly once if uncached, and multiple times if the + * req.body is cached. + */ + +int +VRB_Iterate(struct req *req, req_body_iter_f *func, void *priv) +{ + char buf[8192]; + struct storage *st; + ssize_t l; + int i; + struct vfp_ctx *vfc; + enum vfp_status vfps = VFP_ERROR; + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + AN(func); + + switch(req->req_body_status) { + case REQ_BODY_CACHED: + VTAILQ_FOREACH(st, &req->body->list, list) { + i = func(req, priv, st->ptr, st->len); + if (i) + return (i); + } + return (0); + case REQ_BODY_NONE: + return (0); + case REQ_BODY_WITH_LEN: + case REQ_BODY_WITHOUT_LEN: + break; + case REQ_BODY_TAKEN: + VSLb(req->vsl, SLT_VCL_Error, + "Uncached req.body can only be consumed once."); + return (-1); + case REQ_BODY_FAIL: + VSLb(req->vsl, SLT_FetchError, + "Had failed reading req.body before."); + return (-1); + default: + WRONG("Wrong req_body_status in VRB_IterateReqBody()"); + } + Lck_Lock(&req->sp->mtx); + if (req->req_body_status == REQ_BODY_WITH_LEN || + req->req_body_status == REQ_BODY_WITHOUT_LEN) { + req->req_body_status = REQ_BODY_TAKEN; + i = 0; + } else + i = -1; + Lck_Unlock(&req->sp->mtx); + if (i) { + VSLb(req->vsl, SLT_VCL_Error, + "Multiple attempts to access non-cached req.body"); + return (i); + } + + CHECK_OBJ_NOTNULL(req->htc, HTTP_CONN_MAGIC); + vfc = req->htc->vfc; + VFP_Setup(vfc); + vfc->http = req->http; + vfc->vsl = req->vsl; + V1F_Setup_Fetch(vfc, req->htc); + if (VFP_Open(vfc) < 0) { + VSLb(req->vsl, SLT_FetchError, "Could not open Fetch Pipeline"); + return (-1); + } + + do { + l = sizeof buf; + vfps = VFP_Suck(vfc, buf, &l); + if (vfps == VFP_ERROR) { + req->req_body_status = REQ_BODY_FAIL; + l = -1; + break; + } else if (l > 0) { + req->req_bodybytes += l; + req->acct.req_bodybytes += l; + l = func(req, priv, buf, l); + if (l) { + req->req_body_status = REQ_BODY_FAIL; + break; + } + } + } while (vfps == VFP_OK); + VFP_Close(vfc); + VSLb_ts_req(req, "ReqBody", VTIM_real()); + + return (l); +} + +/*---------------------------------------------------------------------- + * DiscardReqBody() is a dedicated function, because we might + * be able to disuade or terminate its transmission in some protocols. + * For HTTP1 we have no such luck, and we just iterate it into oblivion. + */ + +static int __match_proto__(req_body_iter_f) +httpq_req_body_discard(struct req *req, void *priv, void *ptr, size_t len) +{ + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + (void)priv; + (void)ptr; + (void)len; + return (0); +} + +int +VRB_Ignore(struct req *req) +{ + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + if (req->req_body_status == REQ_BODY_WITH_LEN || + req->req_body_status == REQ_BODY_WITHOUT_LEN) + (void)VRB_Iterate(req, httpq_req_body_discard, NULL); + return(0); +} + +/*---------------------------------------------------------------------- + */ + +void +VRB_Free(struct req *req) +{ + struct storage *st; + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + while (!VTAILQ_EMPTY(&req->body->list)) { + st = VTAILQ_FIRST(&req->body->list); + VTAILQ_REMOVE(&req->body->list, st, list); + STV_free(st); + } +} + +/*---------------------------------------------------------------------- + * Cache the req.body if it is smaller than the given size + * + * This function must be called before any backend fetches are kicked + * off to prevent parallelism. + */ + +int +VRB_Cache(struct req *req, ssize_t maxsize) +{ + struct storage *st; + ssize_t l, yet; + struct vfp_ctx *vfc; + enum vfp_status vfps = VFP_ERROR; + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + assert (req->req_step == R_STP_RECV); + switch(req->req_body_status) { + case REQ_BODY_CACHED: + return (0); + case REQ_BODY_FAIL: + return (-1); + case REQ_BODY_NONE: + return (0); + case REQ_BODY_WITHOUT_LEN: + case REQ_BODY_WITH_LEN: + break; + default: + WRONG("Wrong req_body_status in VRB_CacheReqBody()"); + } + + CHECK_OBJ_NOTNULL(req->htc, HTTP_CONN_MAGIC); + vfc = req->htc->vfc; + + if (req->htc->content_length > maxsize) { + req->req_body_status = REQ_BODY_FAIL; + (void)VFP_Error(vfc, "Request body too big to cache"); + return (-1); + } + + VFP_Setup(vfc); + vfc->http = req->http; + vfc->vsl = req->vsl; + V1F_Setup_Fetch(vfc, req->htc); + + if (VFP_Open(vfc) < 0) { + req->req_body_status = REQ_BODY_FAIL; + return (-1); + } + + yet = req->htc->content_length; + if (yet < 0) + yet = 0; + st = NULL; + do { + if (st == NULL) { + st = STV_alloc_transient( + yet ? yet : cache_param->fetch_chunksize); + if (st == NULL) { + req->req_body_status = REQ_BODY_FAIL; + l = -1; + break; + } else { + VTAILQ_INSERT_TAIL(&req->body->list, st, list); + } + } + l = st->space - st->len; + vfps = VFP_Suck(vfc, st->ptr + st->len, &l); + if (vfps == VFP_ERROR) { + req->req_body_status = REQ_BODY_FAIL; + l = -1; + break; + } + if (l > 0) { + req->req_bodybytes += l; + req->acct.req_bodybytes += l; + if (yet > 0) + yet -= l; + st->len += l; + if (st->space == st->len) + st = NULL; + l = 0; + } + if (req->req_bodybytes > maxsize) { + req->req_body_status = REQ_BODY_FAIL; + l = -1; + break; + } + } while (vfps == VFP_OK); + VFP_Close(vfc); + + if (l == 0) { + + if (req->req_bodybytes != req->htc->content_length) { + /* We must update also the "pristine" req.* copy */ + http_Unset(req->http0, H_Content_Length); + http_Unset(req->http0, H_Transfer_Encoding); + http_PrintfHeader(req->http0, "Content-Length: %ju", + (uintmax_t)req->req_bodybytes); + + http_Unset(req->http, H_Content_Length); + http_Unset(req->http, H_Transfer_Encoding); + http_PrintfHeader(req->http, "Content-Length: %ju", + (uintmax_t)req->req_bodybytes); + } + + req->req_body_status = REQ_BODY_CACHED; + } + VSLb_ts_req(req, "ReqBody", VTIM_real()); + return (l); +} diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 1f0189e..1125e36 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -259,7 +259,7 @@ cnt_synth(struct worker *wrk, struct req *req) req->doclose = SC_RESP_CLOSE; /* Discard any lingering request body before delivery */ - (void)HTTP1_DiscardReqBody(req); + (void)VRB_Ignore(req); V1D_Deliver_Synth(req); @@ -296,7 +296,7 @@ cnt_fetch(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); wrk->stats.s_fetch++; - (void)HTTP1_DiscardReqBody(req); + (void)VRB_Ignore(req); if (req->objcore->flags & OC_F_FAILED) { req->err_code = 503; @@ -420,7 +420,7 @@ cnt_lookup(struct worker *wrk, struct req *req) AZ(boc->busyobj); VBF_Fetch(wrk, req, boc, oc, VBF_BACKGROUND); } else { - (void)HTTP1_DiscardReqBody(req);// XXX: handle err + (void)VRB_Ignore(req);// XXX: handle err } wrk->stats.cache_hit++; req->req_step = R_STP_DELIVER; @@ -866,7 +866,6 @@ enum req_fsm_nxt CNT_Request(struct worker *wrk, struct req *req) { enum req_fsm_nxt nxt; - struct storage *st; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -920,11 +919,7 @@ CNT_Request(struct worker *wrk, struct req *req) VSLb(req->vsl, SLT_ESI_BodyBytes, "%ju", (uintmax_t)req->resp_bodybytes); - while (!VTAILQ_EMPTY(&req->body->list)) { - st = VTAILQ_FIRST(&req->body->list); - VTAILQ_REMOVE(&req->body->list, st, list); - STV_free(st); - } + VRB_Free(req); req->wrk = NULL; } assert(WRW_IsReleased(wrk)); diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index f2c2065..dcb07ba 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -488,7 +488,7 @@ VRT_CacheReqBody(const struct vrt_ctx *ctx, long long maxsize) "req.body can only be cached in vcl_recv{}"); return (0); } - return (HTTP1_CacheReqBody(ctx->req, maxsize)); + return (VRB_Cache(ctx->req, maxsize)); } /*-------------------------------------------------------------------- From phk at FreeBSD.org Mon Sep 15 09:22:26 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 11:22:26 +0200 Subject: [master] 52a745b Reduce struct storage visibility. Message-ID: commit 52a745bc8617f60c2a00c83fdcf9a7cb874b8465 Author: Poul-Henning Kamp Date: Mon Sep 15 09:21:56 2014 +0000 Reduce struct storage visibility. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index c76f7c1..7373815 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -384,22 +384,6 @@ struct lru { unsigned n_objcore; }; -/* Storage -----------------------------------------------------------*/ - -struct storage { - unsigned magic; -#define STORAGE_MAGIC 0x1a4e51c0 - - - VTAILQ_ENTRY(storage) list; - struct stevedore *stevedore; - void *priv; - - unsigned char *ptr; - unsigned len; - unsigned space; -}; - /* Stored object ----------------------------------------------------- * Pointer to a stored object, and the methods it supports */ diff --git a/bin/varnishd/cache/cache_cli.c b/bin/varnishd/cache/cache_cli.c index 77a041c..9e1fa4e 100644 --- a/bin/varnishd/cache/cache_cli.c +++ b/bin/varnishd/cache/cache_cli.c @@ -48,6 +48,7 @@ #include "vcli_common.h" #include "vcli_priv.h" #include "vcli_serve.h" +#include "storage/storage.h" // struct storage pthread_t cli_thread; static struct lock cli_mtx; diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index 6c96f74..d39c51a 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -35,6 +35,7 @@ #include "cache.h" #include "vtim.h" +#include "storage/storage.h" /*---------------------------------------------------------------------- * Iterate over the req.body. diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index 1f83998..5dda954 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -39,6 +39,23 @@ struct objcore; struct worker; struct lru; +/* Storage -----------------------------------------------------------*/ + +struct storage { + unsigned magic; +#define STORAGE_MAGIC 0x1a4e51c0 + + + VTAILQ_ENTRY(storage) list; + struct stevedore *stevedore; + void *priv; + + unsigned char *ptr; + unsigned len; + unsigned space; +}; + + typedef void storage_init_f(struct stevedore *, int ac, char * const *av); typedef void storage_open_f(const struct stevedore *); typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); From phk at FreeBSD.org Mon Sep 15 10:25:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 12:25:43 +0200 Subject: [master] e9e8d42 Eliminate bo->ws_o Message-ID: commit e9e8d4283b6c66379068224330bc842884aa3b8f Author: Poul-Henning Kamp Date: Mon Sep 15 10:25:04 2014 +0000 Eliminate bo->ws_o diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 7373815..e6d5369 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -527,9 +527,6 @@ struct busyobj { struct vsl_log vsl[1]; struct dstat *stats; - /* Workspace for object only needed during fetch */ - struct ws ws_o[1]; - struct vsb *synth_body; }; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index c4b4332..04df133 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -137,7 +137,6 @@ vbf_beresp2obj(struct busyobj *bo) } AZ(ObjSetU32(bo->vfc, OA_VXID, VXID(bo->vsl->wid))); - WS_Assert(bo->ws_o); /* for HTTP_Encode() VSLH call */ bo->beresp->logtag = SLT_ObjMethod; @@ -607,8 +606,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) assert(bo->refcount >= 1); - AZ(WS_Overflowed(bo->ws_o)); - assert (bo->state == BOS_REQ_DONE); if (bo->do_stream) { @@ -680,7 +677,6 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) AZ(ObjCopyAttr(bo->vfc, bo->ims_oc, OA_FLAGS)); AZ(ObjCopyAttr(bo->vfc, bo->ims_oc, OA_GZIPBITS)); - AZ(WS_Overflowed(bo->ws_o)); if (bo->do_stream) { HSH_Unbusy(&wrk->stats, bo->fetch_objcore); VBO_setstate(bo, BOS_STREAM); diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 99fc00f..1d6399f 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -407,12 +407,14 @@ ObjSetattr(const struct vfp_ctx *vc, enum obj_attr attr, ssize_t len, { struct object *o; void *retval = NULL; + struct storage *st; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vc->bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(vc->oc, OBJCORE_MAGIC); o = obj_getobj(vc->oc, vc->stats); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); + st = o->objstore; switch (attr) { case OA_ESIDATA: o->esidata = objallocwithnuke(o->body->stevedore, vc->vsl, @@ -431,8 +433,10 @@ ObjSetattr(const struct vfp_ctx *vc, enum obj_attr attr, ssize_t len, retval = o->oa_gzipbits; break; case OA_HEADERS: - o->oa_http = (void*)WS_Alloc(vc->bo->ws_o, len); - AN(o->oa_http); + len = PRNDUP(len); + assert(st->len + len <= st->space); + o->oa_http = (void*)(st->ptr + st->len); + st->len += len; retval = o->oa_http; break; case OA_LASTMODIFIED: @@ -440,8 +444,10 @@ ObjSetattr(const struct vfp_ctx *vc, enum obj_attr attr, ssize_t len, retval = o->oa_lastmodified; break; case OA_VARY: - o->oa_vary = (void*)WS_Alloc(vc->bo->ws_o, len); - AN(o->oa_vary); + len = PRNDUP(len); + assert(st->len + len <= st->space); + o->oa_vary = (void*)(st->ptr + st->len); + st->len += len; retval = o->oa_vary; break; case OA_VXID: diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 7574638..c666d8a 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -342,7 +342,6 @@ pan_busyobj(const struct busyobj *bo) pan_http("bereq", bo->bereq, 4); if (bo->beresp->ws != NULL) pan_http("beresp", bo->beresp, 4); - pan_ws(bo->ws_o, 4); if (bo->fetch_objcore) pan_objcore("FETCH", bo->fetch_objcore); if (bo->ims_oc) diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index fbdcda1..7e4cea4 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -223,10 +223,6 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo, l = PRNDDN(ltot - sizeof *o); assert(l >= soc->wsl); - WS_Init(bo->ws_o, "obj", o + 1, l); - WS_Assert(bo->ws_o); - assert(bo->ws_o->e <= (char*)ptr + ltot); - VTAILQ_INIT(&o->body->list); bo->fetch_objcore->stobj->magic = STOREOBJ_MAGIC; @@ -260,9 +256,10 @@ stv_default_allocobj(struct stevedore *stv, struct busyobj *bo, stv->free(st); return (NULL); } - ltot = st->len = st->space; + ltot = st->space; o = STV_MkObject(stv, bo, st->ptr, ltot, soc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); + st->len = sizeof(*o); o->objstore = st; bo->stats->n_object++; return (o); diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index 281561b..3e54208 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -532,13 +532,14 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo, return (NULL); assert(st->space >= ltot); - ltot = st->len = st->space; + ltot = st->space; o = STV_MkObject(stv, bo, st->ptr, ltot, soc); AN(bo->fetch_objcore->stobj->stevedore); assert(bo->fetch_objcore->stobj->stevedore == stv); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); o->objstore = st; + st->len = sizeof(*o); bo->stats->n_object++; oc = bo->fetch_objcore; From phk at FreeBSD.org Mon Sep 15 10:48:45 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 12:48:45 +0200 Subject: [master] 80459a2 Clean up stuff that were needed for ws_o Message-ID: commit 80459a2c56b30356f6471dfc12d09734701fe6be Author: Poul-Henning Kamp Date: Mon Sep 15 10:48:24 2014 +0000 Clean up stuff that were needed for ws_o diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 7e4cea4..fa84d56 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -180,17 +180,6 @@ STV_alloc(struct stevedore *stv, size_t size) return (st); } -/*-------------------------------------------------------------------* - * Structure used to transport internal knowledge from STV_NewObject() - * to STV_MkObject(). Nobody else should mess with this struct. - */ - -struct stv_objsecrets { - unsigned magic; -#define STV_OBJ_SECRETES_MAGIC 0x78c87247 - unsigned wsl; -}; - /*-------------------------------------------------------------------- * This function is called by stevedores ->allocobj() method, which * very often will be stv_default_allocobj() below, to convert a slab @@ -200,29 +189,20 @@ struct stv_objsecrets { */ struct object * -STV_MkObject(struct stevedore *stv, struct busyobj *bo, - void *ptr, unsigned ltot, const struct stv_objsecrets *soc) +STV_MkObject(struct stevedore *stv, struct busyobj *bo, void *ptr) { struct object *o; - unsigned l; CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC); CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC); assert(PAOK(ptr)); - assert(PAOK(soc->wsl)); - - assert(ltot >= sizeof *o + soc->wsl); o = ptr; memset(o, 0, sizeof *o); o->magic = OBJECT_MAGIC; - l = PRNDDN(ltot - sizeof *o); - assert(l >= soc->wsl); - VTAILQ_INIT(&o->body->list); bo->fetch_objcore->stobj->magic = STOREOBJ_MAGIC; @@ -241,13 +221,11 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo, */ struct object * -stv_default_allocobj(struct stevedore *stv, struct busyobj *bo, - unsigned ltot, const struct stv_objsecrets *soc) +stv_default_allocobj(struct stevedore *stv, struct busyobj *bo, unsigned ltot) { struct object *o; struct storage *st; - CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); st = stv->alloc(stv, ltot); if (st == NULL) @@ -256,8 +234,7 @@ stv_default_allocobj(struct stevedore *stv, struct busyobj *bo, stv->free(st); return (NULL); } - ltot = st->space; - o = STV_MkObject(stv, bo, st->ptr, ltot, soc); + o = STV_MkObject(stv, bo, st->ptr); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); st->len = sizeof(*o); o->objstore = st; @@ -277,7 +254,6 @@ STV_NewObject(struct busyobj *bo, const char *hint, unsigned wsl) struct object *o; struct stevedore *stv, *stv0; unsigned ltot; - struct stv_objsecrets soc; int i; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); @@ -285,20 +261,16 @@ STV_NewObject(struct busyobj *bo, const char *hint, unsigned wsl) assert(wsl > 0); wsl = PRNDUP(wsl); - memset(&soc, 0, sizeof soc); - soc.magic = STV_OBJ_SECRETES_MAGIC; - soc.wsl = wsl; - ltot = sizeof *o + wsl; stv = stv0 = stv_pick_stevedore(bo->vsl, &hint); AN(stv->allocobj); - o = stv->allocobj(stv, bo, ltot, &soc); + o = stv->allocobj(stv, bo, ltot); if (o == NULL && hint == NULL) { do { stv = stv_pick_stevedore(bo->vsl, &hint); AN(stv->allocobj); - o = stv->allocobj(stv, bo, ltot, &soc); + o = stv->allocobj(stv, bo, ltot); } while (o == NULL && stv != stv0); } if (o == NULL) { @@ -306,7 +278,7 @@ STV_NewObject(struct busyobj *bo, const char *hint, unsigned wsl) for (i = 0; o == NULL && i < cache_param->nuke_limit; i++) { if (EXP_NukeOne(bo->vsl, bo->stats, stv->lru) == -1) break; - o = stv->allocobj(stv, bo, ltot, &soc); + o = stv->allocobj(stv, bo, ltot); } } diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index 5dda954..95d8eae 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -62,7 +62,7 @@ typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); typedef void storage_trim_f(struct storage *, size_t size, int move_ok); typedef void storage_free_f(struct storage *); typedef struct object *storage_allocobj_f(struct stevedore *, struct busyobj *, - unsigned ltot, const struct stv_objsecrets *); + unsigned ltot); typedef void storage_close_f(const struct stevedore *); typedef void storage_signal_close_f(const struct stevedore *); typedef int storage_baninfo_f(const struct stevedore *, enum baninfo event, @@ -123,7 +123,7 @@ int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx); uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity, const char *ctx); struct object *STV_MkObject(struct stevedore *stv, struct busyobj *bo, - void *ptr, unsigned ltot, const struct stv_objsecrets *soc); + void *ptr); struct lru *LRU_Alloc(void); void LRU_Free(struct lru *lru); diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index 3e54208..f85dc12 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -505,8 +505,7 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size, */ static struct object * -smp_allocobj(struct stevedore *stv, struct busyobj *bo, - unsigned ltot, const struct stv_objsecrets *soc) +smp_allocobj(struct stevedore *stv, struct busyobj *bo, unsigned ltot) { struct object *o; struct storage *st; @@ -532,9 +531,8 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo, return (NULL); assert(st->space >= ltot); - ltot = st->space; - o = STV_MkObject(stv, bo, st->ptr, ltot, soc); + o = STV_MkObject(stv, bo, st->ptr); AN(bo->fetch_objcore->stobj->stevedore); assert(bo->fetch_objcore->stobj->stevedore == stv); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); From phk at FreeBSD.org Mon Sep 15 12:20:47 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 14:20:47 +0200 Subject: [master] 35a9e75 Push busyobj further backwards Message-ID: commit 35a9e755c64b717b5c61d4cda5b24a24dd2a6bc9 Author: Poul-Henning Kamp Date: Mon Sep 15 12:20:33 2014 +0000 Push busyobj further backwards diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index fa84d56..9f42422 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -189,13 +189,12 @@ STV_alloc(struct stevedore *stv, size_t size) */ struct object * -STV_MkObject(struct stevedore *stv, struct busyobj *bo, void *ptr) +STV_MkObject(struct stevedore *stv, struct objcore *oc, void *ptr) { struct object *o; CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); assert(PAOK(ptr)); @@ -205,13 +204,12 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo, void *ptr) VTAILQ_INIT(&o->body->list); - bo->fetch_objcore->stobj->magic = STOREOBJ_MAGIC; - bo->fetch_objcore->stobj->stevedore = stv; + oc->stobj->magic = STOREOBJ_MAGIC; + oc->stobj->stevedore = stv; o->body->stevedore = stv; AN(stv->methods); - bo->fetch_objcore->stobj->priv = o; - bo->fetch_objcore->stobj->priv2 = (uintptr_t)stv; - VSLb(bo->vsl, SLT_Storage, "%s %s", stv->name, stv->ident); + oc->stobj->priv = o; + oc->stobj->priv2 = (uintptr_t)stv; return (o); } @@ -221,12 +219,12 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo, void *ptr) */ struct object * -stv_default_allocobj(struct stevedore *stv, struct busyobj *bo, unsigned ltot) +stv_default_allocobj(struct stevedore *stv, struct objcore *oc, unsigned ltot) { struct object *o; struct storage *st; - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); st = stv->alloc(stv, ltot); if (st == NULL) return (NULL); @@ -234,11 +232,10 @@ stv_default_allocobj(struct stevedore *stv, struct busyobj *bo, unsigned ltot) stv->free(st); return (NULL); } - o = STV_MkObject(stv, bo, st->ptr); + o = STV_MkObject(stv, oc, st->ptr); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); st->len = sizeof(*o); o->objstore = st; - bo->stats->n_object++; return (o); } @@ -252,12 +249,14 @@ int STV_NewObject(struct busyobj *bo, const char *hint, unsigned wsl) { struct object *o; + struct objcore *oc; struct stevedore *stv, *stv0; unsigned ltot; int i; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC); + oc = bo->fetch_objcore; + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); assert(wsl > 0); wsl = PRNDUP(wsl); @@ -265,12 +264,12 @@ STV_NewObject(struct busyobj *bo, const char *hint, unsigned wsl) stv = stv0 = stv_pick_stevedore(bo->vsl, &hint); AN(stv->allocobj); - o = stv->allocobj(stv, bo, ltot); + o = stv->allocobj(stv, oc, ltot); if (o == NULL && hint == NULL) { do { stv = stv_pick_stevedore(bo->vsl, &hint); AN(stv->allocobj); - o = stv->allocobj(stv, bo, ltot); + o = stv->allocobj(stv, oc, ltot); } while (o == NULL && stv != stv0); } if (o == NULL) { @@ -278,13 +277,16 @@ STV_NewObject(struct busyobj *bo, const char *hint, unsigned wsl) for (i = 0; o == NULL && i < cache_param->nuke_limit; i++) { if (EXP_NukeOne(bo->vsl, bo->stats, stv->lru) == -1) break; - o = stv->allocobj(stv, bo, ltot); + o = stv->allocobj(stv, oc, ltot); } } if (o == NULL) return (0); + bo->stats->n_object++; + VSLb(bo->vsl, SLT_Storage, "%s %s", + oc->stobj->stevedore->name, oc->stobj->stevedore->ident); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(o->objstore, STORAGE_MAGIC); AN(stv->methods); diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index 95d8eae..5108e47 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -61,7 +61,7 @@ typedef void storage_open_f(const struct stevedore *); typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); typedef void storage_trim_f(struct storage *, size_t size, int move_ok); typedef void storage_free_f(struct storage *); -typedef struct object *storage_allocobj_f(struct stevedore *, struct busyobj *, +typedef struct object *storage_allocobj_f(struct stevedore *, struct objcore *, unsigned ltot); typedef void storage_close_f(const struct stevedore *); typedef void storage_signal_close_f(const struct stevedore *); @@ -122,8 +122,7 @@ extern struct stevedore *stv_transient; int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx); uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity, const char *ctx); -struct object *STV_MkObject(struct stevedore *stv, struct busyobj *bo, - void *ptr); +struct object *STV_MkObject(struct stevedore *, struct objcore *, void *ptr); struct lru *LRU_Alloc(void); void LRU_Free(struct lru *lru); diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index f85dc12..037d23b 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -505,23 +505,20 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size, */ static struct object * -smp_allocobj(struct stevedore *stv, struct busyobj *bo, unsigned ltot) +smp_allocobj(struct stevedore *stv, struct objcore *oc, unsigned ltot) { struct object *o; struct storage *st; struct smp_sc *sc; struct smp_seg *sg; struct smp_object *so; - struct objcore *oc; unsigned objidx; - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC); /* Don't entertain already dead objects */ - if ((bo->fetch_objcore->exp.ttl + - bo->fetch_objcore->exp.grace + - bo->fetch_objcore->exp.keep) <= 0.) + if ((oc->exp.ttl + oc->exp.grace + oc->exp.keep) <= 0.) return (NULL); ltot = IRNUP(sc, ltot); @@ -532,16 +529,12 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo, unsigned ltot) assert(st->space >= ltot); - o = STV_MkObject(stv, bo, st->ptr); - AN(bo->fetch_objcore->stobj->stevedore); - assert(bo->fetch_objcore->stobj->stevedore == stv); + o = STV_MkObject(stv, oc, st->ptr); + AN(oc->stobj->stevedore); + assert(oc->stobj->stevedore == stv); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); o->objstore = st; st->len = sizeof(*o); - bo->stats->n_object++; - - oc = bo->fetch_objcore; - CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); Lck_Lock(&sc->mtx); sg->nfixed++; @@ -550,7 +543,7 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo, unsigned ltot) /* We have to do this somewhere, might as well be here... */ assert(sizeof so->hash == DIGEST_LEN); memcpy(so->hash, oc->objhead->digest, DIGEST_LEN); - so->exp = bo->fetch_objcore->exp; + so->exp = oc->exp; so->ptr = (uint8_t*)o - sc->base; so->ban = BAN_Time(oc->ban); From phk at FreeBSD.org Mon Sep 15 12:32:05 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 14:32:05 +0200 Subject: [master] baf8541 Don't needlessly return struct object Message-ID: commit baf854187adf4eb7bbb4dbef08468723201ef43f Author: Poul-Henning Kamp Date: Mon Sep 15 12:31:52 2014 +0000 Don't needlessly return struct object diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 9f42422..c1fb802 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -218,7 +218,7 @@ STV_MkObject(struct stevedore *stv, struct objcore *oc, void *ptr) * implement persistent storage can rely on. */ -struct object * +int stv_default_allocobj(struct stevedore *stv, struct objcore *oc, unsigned ltot) { struct object *o; @@ -227,16 +227,16 @@ stv_default_allocobj(struct stevedore *stv, struct objcore *oc, unsigned ltot) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); st = stv->alloc(stv, ltot); if (st == NULL) - return (NULL); + return (0); if (st->space < ltot) { stv->free(st); - return (NULL); + return (0); } o = STV_MkObject(stv, oc, st->ptr); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); st->len = sizeof(*o); o->objstore = st; - return (o); + return (1); } /*------------------------------------------------------------------- @@ -248,11 +248,10 @@ stv_default_allocobj(struct stevedore *stv, struct objcore *oc, unsigned ltot) int STV_NewObject(struct busyobj *bo, const char *hint, unsigned wsl) { - struct object *o; struct objcore *oc; struct stevedore *stv, *stv0; unsigned ltot; - int i; + int i, j; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); oc = bo->fetch_objcore; @@ -260,36 +259,33 @@ STV_NewObject(struct busyobj *bo, const char *hint, unsigned wsl) assert(wsl > 0); wsl = PRNDUP(wsl); - ltot = sizeof *o + wsl; + ltot = sizeof(struct object) + wsl; stv = stv0 = stv_pick_stevedore(bo->vsl, &hint); AN(stv->allocobj); - o = stv->allocobj(stv, oc, ltot); - if (o == NULL && hint == NULL) { + j = stv->allocobj(stv, oc, ltot); + if (j == 0 && hint == NULL) { do { stv = stv_pick_stevedore(bo->vsl, &hint); AN(stv->allocobj); - o = stv->allocobj(stv, oc, ltot); - } while (o == NULL && stv != stv0); + j = stv->allocobj(stv, oc, ltot); + } while (j == 0 && stv != stv0); } - if (o == NULL) { + if (j == 0) { /* no luck; try to free some space and keep trying */ - for (i = 0; o == NULL && i < cache_param->nuke_limit; i++) { + for (i = 0; j == 0 && i < cache_param->nuke_limit; i++) { if (EXP_NukeOne(bo->vsl, bo->stats, stv->lru) == -1) break; - o = stv->allocobj(stv, oc, ltot); + j = stv->allocobj(stv, oc, ltot); } } - if (o == NULL) + if (j == 0) return (0); bo->stats->n_object++; VSLb(bo->vsl, SLT_Storage, "%s %s", oc->stobj->stevedore->name, oc->stobj->stevedore->ident); - CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - CHECK_OBJ_NOTNULL(o->objstore, STORAGE_MAGIC); - AN(stv->methods); return (1); } diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index 5108e47..56c72b8 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -61,7 +61,7 @@ typedef void storage_open_f(const struct stevedore *); typedef struct storage *storage_alloc_f(struct stevedore *, size_t size); typedef void storage_trim_f(struct storage *, size_t size, int move_ok); typedef void storage_free_f(struct storage *); -typedef struct object *storage_allocobj_f(struct stevedore *, struct objcore *, +typedef int storage_allocobj_f(struct stevedore *, struct objcore *, unsigned ltot); typedef void storage_close_f(const struct stevedore *); typedef void storage_signal_close_f(const struct stevedore *); diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index 037d23b..411fc9a 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -504,7 +504,7 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size, * Allocate an object */ -static struct object * +static int smp_allocobj(struct stevedore *stv, struct objcore *oc, unsigned ltot) { struct object *o; @@ -519,13 +519,13 @@ smp_allocobj(struct stevedore *stv, struct objcore *oc, unsigned ltot) /* Don't entertain already dead objects */ if ((oc->exp.ttl + oc->exp.grace + oc->exp.keep) <= 0.) - return (NULL); + return (0); ltot = IRNUP(sc, ltot); st = smp_allocx(stv, ltot, ltot, &so, &objidx, &sg); if (st == NULL) - return (NULL); + return (0); assert(st->space >= ltot); @@ -550,7 +550,7 @@ smp_allocobj(struct stevedore *stv, struct objcore *oc, unsigned ltot) smp_init_oc(oc, sg, objidx); Lck_Unlock(&sc->mtx); - return (o); + return (1); } /*-------------------------------------------------------------------- From phk at FreeBSD.org Mon Sep 15 12:46:42 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 14:46:42 +0200 Subject: [master] 245ea1d Push busyobj out of STV_NewObject() Message-ID: commit 245ea1daf39d54f48baced99eb8d3ccdea0f28cf Author: Poul-Henning Kamp Date: Mon Sep 15 12:46:27 2014 +0000 Push busyobj out of STV_NewObject() diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index e6d5369..fc7a2d9 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -1216,7 +1216,8 @@ void RFC2616_Weaken_Etag(struct http *hp); void RFC2616_Vary_AE(struct http *hp); /* stevedore.c */ -int STV_NewObject(struct busyobj *, const char *hint, unsigned len); +int STV_NewObject(struct objcore *, struct vsl_log *, struct dstat *, + const char *hint, unsigned len); struct storage *STV_alloc(struct stevedore *, size_t size); void STV_trim(struct storage *st, size_t size, int move_ok); void STV_free(struct storage *st); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 04df133..b245b82 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -66,7 +66,8 @@ vbf_allocobj(struct busyobj *bo, unsigned l) bo->storage_hint = NULL; - if (STV_NewObject(bo, storage_hint, l)) + if (STV_NewObject(bo->fetch_objcore, bo->vsl, bo->stats, + storage_hint, l)) return (1); if (storage_hint != NULL && !strcmp(storage_hint, TRANSIENT_STORAGE)) @@ -81,7 +82,8 @@ vbf_allocobj(struct busyobj *bo, unsigned l) oc->exp.ttl = cache_param->shortlived; oc->exp.grace = 0.0; oc->exp.keep = 0.0; - return (STV_NewObject(bo, TRANSIENT_STORAGE, l)); + return (STV_NewObject(bo->fetch_objcore, bo->vsl, bo->stats, + TRANSIENT_STORAGE, l)); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index c1fb802..58ab16f 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -246,27 +246,27 @@ stv_default_allocobj(struct stevedore *stv, struct objcore *oc, unsigned ltot) */ int -STV_NewObject(struct busyobj *bo, const char *hint, unsigned wsl) +STV_NewObject(struct objcore *oc, struct vsl_log *vsl, struct dstat *stats, + const char *hint, unsigned wsl) { - struct objcore *oc; struct stevedore *stv, *stv0; unsigned ltot; int i, j; - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - oc = bo->fetch_objcore; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + AN(vsl); + AN(stats); assert(wsl > 0); wsl = PRNDUP(wsl); ltot = sizeof(struct object) + wsl; - stv = stv0 = stv_pick_stevedore(bo->vsl, &hint); + stv = stv0 = stv_pick_stevedore(vsl, &hint); AN(stv->allocobj); j = stv->allocobj(stv, oc, ltot); if (j == 0 && hint == NULL) { do { - stv = stv_pick_stevedore(bo->vsl, &hint); + stv = stv_pick_stevedore(vsl, &hint); AN(stv->allocobj); j = stv->allocobj(stv, oc, ltot); } while (j == 0 && stv != stv0); @@ -274,7 +274,7 @@ STV_NewObject(struct busyobj *bo, const char *hint, unsigned wsl) if (j == 0) { /* no luck; try to free some space and keep trying */ for (i = 0; j == 0 && i < cache_param->nuke_limit; i++) { - if (EXP_NukeOne(bo->vsl, bo->stats, stv->lru) == -1) + if (EXP_NukeOne(vsl, stats, stv->lru) == -1) break; j = stv->allocobj(stv, oc, ltot); } @@ -283,8 +283,8 @@ STV_NewObject(struct busyobj *bo, const char *hint, unsigned wsl) if (j == 0) return (0); - bo->stats->n_object++; - VSLb(bo->vsl, SLT_Storage, "%s %s", + stats->n_object++; + VSLb(vsl, SLT_Storage, "%s %s", oc->stobj->stevedore->name, oc->stobj->stevedore->ident); return (1); } From martin at varnish-software.com Mon Sep 15 13:52:50 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 15 Sep 2014 15:52:50 +0200 Subject: [master] 2d303b0 Harden vtcp_sa_to_ascii Message-ID: commit 2d303b0732694e8bb4dca3711f6c8ecf47ff751f Author: Martin Blix Grydeland Date: Mon Sep 15 15:43:25 2014 +0200 Harden vtcp_sa_to_ascii Require that buffer arguments have greater than zero length to allow for zero termination. Make sure that NULL buffers are not touched. diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 26191fe..624dbb8 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -61,6 +61,8 @@ vtcp_sa_to_ascii(const void *sa, socklen_t l, char *abuf, unsigned alen, { int i; + assert(abuf == NULL || alen > 0); + assert(pbuf == NULL || plen > 0); i = getnameinfo(sa, l, abuf, alen, pbuf, plen, NI_NUMERICHOST | NI_NUMERICSERV); if (i) { @@ -69,12 +71,14 @@ vtcp_sa_to_ascii(const void *sa, socklen_t l, char *abuf, unsigned alen, * for the gai_strerror in the bufffer :-( */ printf("getnameinfo = %d %s\n", i, gai_strerror(i)); - (void)snprintf(abuf, alen, "Conversion"); - (void)snprintf(pbuf, plen, "Failed"); + if (abuf != NULL) + (void)snprintf(abuf, alen, "Conversion"); + if (pbuf != NULL) + (void)snprintf(pbuf, plen, "Failed"); return; } /* XXX dirty hack for v4-to-v6 mapped addresses */ - if (strncmp(abuf, "::ffff:", 7) == 0) { + if (abuf != NULL && strncmp(abuf, "::ffff:", 7) == 0) { for (i = 0; abuf[i + 7]; ++i) abuf[i] = abuf[i + 7]; abuf[i] = '\0'; From martin at varnish-software.com Mon Sep 15 13:52:50 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 15 Sep 2014 15:52:50 +0200 Subject: [master] 6382a1e Bail out on workspace exhaustion in VRT_IP_string Message-ID: commit 6382a1e67d7f2fc4131a80bb453e7f839a83659d Author: Martin Blix Grydeland Date: Mon Sep 15 15:47:26 2014 +0200 Bail out on workspace exhaustion in VRT_IP_string Fixes: #1592 diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index dcb07ba..e8c5d9a 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -307,6 +307,10 @@ VRT_IP_string(const struct vrt_ctx *ctx, VCL_IP ip) if (ip == NULL) return (NULL); len = WS_Reserve(ctx->ws, 0); + if (len == 0) { + WS_Release(ctx->ws, 0); + return (NULL); + } p = ctx->ws->f; VTCP_name(ip, p, len, NULL, 0); WS_Release(ctx->ws, strlen(p) + 1); From martin at varnish-software.com Mon Sep 15 14:50:31 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 15 Sep 2014 16:50:31 +0200 Subject: [master] 4e750aa Delay allocating log line data until it's known to be needed in varnishtop Message-ID: commit 4e750aa19841e76011549aabdfb3ede5d980beab Author: Martin Blix Grydeland Date: Mon Sep 15 16:49:19 2014 +0200 Delay allocating log line data until it's known to be needed in varnishtop diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c index 2c195ef..d29aeb8 100644 --- a/bin/varnishtop/varnishtop.c +++ b/bin/varnishtop/varnishtop.c @@ -117,7 +117,6 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], void *priv) { struct top *tp, t; - char *rd; unsigned int u; unsigned tag; const char *b, *e, *p; @@ -148,12 +147,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], t.hash = u; t.tag = tag; t.clen = len; - t.count = 0; - rd = calloc(len+1, 1); - AN(rd); - memcpy(rd, VSL_CDATA(tr->c->rec.ptr), len); - rd[len] = '\0'; - t.rec_data = rd; + t.rec_data = (char *)VSL_CDATA(tr->c->rec.ptr); AZ(pthread_mutex_lock(&mtx)); tp = VRB_FIND(top_tree, &top_tree_head, &t); @@ -162,7 +156,6 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], tp->count += 1.0; /* Reinsert to rebalance */ VRB_INSERT(top_tree, &top_tree_head, tp); - free(rd); } else { ntop++; tp = calloc(sizeof *tp, 1); @@ -171,7 +164,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], tp->count = 1.0; tp->clen = len; tp->tag = tag; - tp->rec_data = rd; + tp->rec_data = strdup(t.rec_data); + AN(tp->rec_data); VRB_INSERT(top_tree, &top_tree_head, tp); } AZ(pthread_mutex_unlock(&mtx)); From martin at varnish-software.com Mon Sep 15 14:50:31 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 15 Sep 2014 16:50:31 +0200 Subject: [master] 1161711 Keep two trees in varnishtop, one for order and one for uniqueness. Message-ID: commit 1161711e1cb090e37c33d9f26d7682d2a97acf25 Author: Martin Blix Grydeland Date: Mon Sep 15 16:49:53 2014 +0200 Keep two trees in varnishtop, one for order and one for uniqueness. Fixes: #1591 diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c index d29aeb8..45e8205 100644 --- a/bin/varnishtop/varnishtop.c +++ b/bin/varnishtop/varnishtop.c @@ -69,19 +69,37 @@ struct top { char *rec_data; int clen; unsigned hash; - VRB_ENTRY(top) entry; + VRB_ENTRY(top) e_order; + VRB_ENTRY(top) e_key; double count; }; -static int -top_cmp(const struct top *tp, const struct top *tp2); +static VRB_HEAD(t_order, top) h_order = VRB_INITIALIZER(&h_order); +static VRB_HEAD(t_key, top) h_key = VRB_INITIALIZER(&h_key); -static VRB_HEAD(top_tree, top) top_tree_head = VRB_INITIALIZER(&top_tree_head); -VRB_PROTOTYPE(top_tree, top, entry, top_cmp); +static inline int +cmp_key(const struct top *a, const struct top *b) +{ + if (a->hash != b->hash) + return (a->hash - b->hash); + if (a->tag != b->tag) + return (a->tag - b->tag); + if (a->clen != b->clen) + return (a->clen - b->clen); + return (memcmp(a->rec_data, b->rec_data, a->clen)); +} -static unsigned ntop; +static inline int +cmp_order(const struct top *a, const struct top *b) +{ + if (a->count > b->count) + return (-1); + else if (a->count < b->count) + return (1); + return (cmp_key(a, b)); +} -/*--------------------------------------------------------------------*/ +static unsigned ntop; static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; @@ -89,28 +107,10 @@ static int f_flag = 0; static unsigned maxfieldlen = 0; -VRB_GENERATE(top_tree, top, entry, top_cmp); - -static int -top_cmp(const struct top *tp, const struct top *tp2) -{ - if (tp->count == tp2->count || tp->count == 0.0) { - if (tp->hash != tp2->hash) - return (tp->hash - tp2->hash); - if (tp->tag != tp2->tag) - return (tp->tag - tp2->tag); - if (tp->clen != tp2->clen) - return (tp->clen - tp2->clen); - else - return (memcmp(tp->rec_data, tp2->rec_data, tp->clen)); - } else { - if (tp->count > tp2->count) - return -1; - else - return 1; - } -} - +VRB_PROTOTYPE(t_order, top, e_order, cmp_order); +VRB_GENERATE(t_order, top, e_order, cmp_order); +VRB_PROTOTYPE(t_key, top, e_key, cmp_key); +VRB_GENERATE(t_key, top, e_key, cmp_key); static int __match_proto__(VSLQ_dispatch_f) accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], @@ -150,12 +150,12 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], t.rec_data = (char *)VSL_CDATA(tr->c->rec.ptr); AZ(pthread_mutex_lock(&mtx)); - tp = VRB_FIND(top_tree, &top_tree_head, &t); + tp = VRB_FIND(t_key, &h_key, &t); if (tp) { - VRB_REMOVE(top_tree, &top_tree_head, tp); + VRB_REMOVE(t_order, &h_order, tp); tp->count += 1.0; /* Reinsert to rebalance */ - VRB_INSERT(top_tree, &top_tree_head, tp); + VRB_INSERT(t_order, &h_order, tp); } else { ntop++; tp = calloc(sizeof *tp, 1); @@ -166,7 +166,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], tp->tag = tag; tp->rec_data = strdup(t.rec_data); AN(tp->rec_data); - VRB_INSERT(top_tree, &top_tree_head, tp); + VRB_INSERT(t_key, &h_key, tp); + VRB_INSERT(t_order, &h_order, tp); } AZ(pthread_mutex_unlock(&mtx)); @@ -201,8 +202,8 @@ update(int p) else AC(mvprintw(0, COLS - 1 - strlen(VUT.name), "%s", VUT.name)); AC(mvprintw(0, 0, "list length %u", ntop)); - for (tp = VRB_MIN(top_tree, &top_tree_head); tp != NULL; tp = tp2) { - tp2 = VRB_NEXT(top_tree, &top_tree_head, tp); + for (tp = VRB_MIN(t_order, &h_order); tp != NULL; tp = tp2) { + tp2 = VRB_NEXT(t_order, &h_order, tp); if (++l < LINES) { len = tp->clen; @@ -218,7 +219,8 @@ update(int p) continue; tp->count += (1.0/3.0 - tp->count) / (double)n; if (tp->count * 10 < t || l > LINES * 10) { - VRB_REMOVE(top_tree, &top_tree_head, tp); + VRB_REMOVE(t_key, &h_key, tp); + VRB_REMOVE(t_order, &h_order, tp); free(tp->rec_data); free(tp); ntop--; @@ -290,8 +292,8 @@ static void dump(void) { struct top *tp, *tp2; - for (tp = VRB_MIN(top_tree, &top_tree_head); tp != NULL; tp = tp2) { - tp2 = VRB_NEXT(top_tree, &top_tree_head, tp); + for (tp = VRB_MIN(t_order, &h_order); tp != NULL; tp = tp2) { + tp2 = VRB_NEXT(t_order, &h_order, tp); if (tp->count <= 1.0) break; printf("%9.2f %s %*.*s\n", From phk at FreeBSD.org Mon Sep 15 14:59:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 16:59:46 +0200 Subject: [master] a2d16c1 Store cached req.body in a vacuous objcore to avoid a special-case interface to stevedores. Message-ID: commit a2d16c17db1f51ccf27c280dea6ef6a6053de8cd Author: Poul-Henning Kamp Date: Mon Sep 15 14:58:48 2014 +0000 Store cached req.body in a vacuous objcore to avoid a special-case interface to stevedores. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index fc7a2d9..75c4bdc 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -587,7 +587,7 @@ struct req { VTAILQ_ENTRY(req) w_list; volatile enum req_body_state_e req_body_status; - struct body body[1]; + struct objcore *body_oc; /* The busy objhead we sleep on */ struct objhead *hash_objhead; @@ -1225,7 +1225,6 @@ void STV_open(void); void STV_close(void); int STV_BanInfo(enum baninfo event, const uint8_t *ban, unsigned len); void STV_BanExport(const uint8_t *bans, unsigned len); -struct storage *STV_alloc_transient(size_t size); /* storage_persistent.c */ void SMP_Init(void); diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 1d6399f..f57bd0b 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -96,7 +96,8 @@ ObjIterBegin(struct worker *wrk, struct objcore *oc) oi->oc = oc; oi->obj = obj; oi->wrk = wrk; - oi->bo = HSH_RefBusy(oc); + if (oc->objhead != NULL) + oi->bo = HSH_RefBusy(oc); return (oi); } diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index d39c51a..6eb2590 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -36,6 +36,7 @@ #include "cache.h" #include "vtim.h" #include "storage/storage.h" +#include "hash/hash_slinger.h" /*---------------------------------------------------------------------- * Iterate over the req.body. @@ -48,23 +49,28 @@ int VRB_Iterate(struct req *req, req_body_iter_f *func, void *priv) { char buf[8192]; - struct storage *st; ssize_t l; + void *p; int i; struct vfp_ctx *vfc; enum vfp_status vfps = VFP_ERROR; + struct objiter *oi; + enum objiter_status ois; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AN(func); switch(req->req_body_status) { case REQ_BODY_CACHED: - VTAILQ_FOREACH(st, &req->body->list, list) { - i = func(req, priv, st->ptr, st->len); - if (i) - return (i); - } - return (0); + oi = ObjIterBegin(req->wrk, req->body_oc); + AN(oi); + do { + ois = ObjIter(oi, &p, &l); + if (l > 0 && func(req, priv, p, l)) + break; + } while (ois == OIS_DATA); + ObjIterEnd(&oi); + return (ois == OIS_DONE ? 0 : -1); case REQ_BODY_NONE: return (0); case REQ_BODY_WITH_LEN: @@ -164,14 +170,13 @@ VRB_Ignore(struct req *req) void VRB_Free(struct req *req) { - struct storage *st; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - while (!VTAILQ_EMPTY(&req->body->list)) { - st = VTAILQ_FIRST(&req->body->list); - VTAILQ_REMOVE(&req->body->list, st, list); - STV_free(st); + if (req->body_oc != NULL) { + ObjFreeObj(req->body_oc, &req->wrk->stats); + FREE_OBJ(req->body_oc); + req->body_oc = NULL; } } @@ -185,9 +190,9 @@ VRB_Free(struct req *req) int VRB_Cache(struct req *req, ssize_t maxsize) { - struct storage *st; ssize_t l, yet; struct vfp_ctx *vfc; + uint8_t *ptr; enum vfp_status vfps = VFP_ERROR; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -216,9 +221,15 @@ VRB_Cache(struct req *req, ssize_t maxsize) return (-1); } + req->body_oc = HSH_NewObjCore(req->wrk); + XXXAN(STV_NewObject(req->body_oc, req->vsl, &req->wrk->stats, + TRANSIENT_STORAGE, 8)); + VFP_Setup(vfc); vfc->http = req->http; vfc->vsl = req->vsl; + vfc->oc = req->body_oc; + vfc->stats = &req->wrk->stats; V1F_Setup_Fetch(vfc, req->htc); if (VFP_Open(vfc) < 0) { @@ -229,45 +240,27 @@ VRB_Cache(struct req *req, ssize_t maxsize) yet = req->htc->content_length; if (yet < 0) yet = 0; - st = NULL; do { - if (st == NULL) { - st = STV_alloc_transient( - yet ? yet : cache_param->fetch_chunksize); - if (st == NULL) { - req->req_body_status = REQ_BODY_FAIL; - l = -1; - break; - } else { - VTAILQ_INSERT_TAIL(&req->body->list, st, list); - } - } - l = st->space - st->len; - vfps = VFP_Suck(vfc, st->ptr + st->len, &l); - if (vfps == VFP_ERROR) { - req->req_body_status = REQ_BODY_FAIL; - l = -1; + AZ(vfc->failed); + l = yet; + if (VFP_GetStorage(vfc, &l, &ptr) != VFP_OK) break; - } - if (l > 0) { + AZ(vfc->failed); + AN(ptr); + AN(l); + vfps = VFP_Suck(vfc, ptr, &l); + if (l > 0 && vfps != VFP_ERROR) { req->req_bodybytes += l; req->acct.req_bodybytes += l; - if (yet > 0) + if (yet >= l) yet -= l; - st->len += l; - if (st->space == st->len) - st = NULL; - l = 0; - } - if (req->req_bodybytes > maxsize) { - req->req_body_status = REQ_BODY_FAIL; - l = -1; - break; + ObjExtend(req->body_oc, &req->wrk->stats, l); } + } while (vfps == VFP_OK); VFP_Close(vfc); - if (l == 0) { + if (vfps == VFP_END) { if (req->req_bodybytes != req->htc->content_length) { /* We must update also the "pristine" req.* copy */ @@ -283,7 +276,9 @@ VRB_Cache(struct req *req, ssize_t maxsize) } req->req_body_status = REQ_BODY_CACHED; + } else { + req->req_body_status = REQ_BODY_FAIL; } VSLb_ts_req(req, "ReqBody", VTIM_real()); - return (l); + return (vfps == VFP_END ? req->req_bodybytes : 0); } diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index b6d8049..6aa07b4 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -374,8 +374,6 @@ SES_GetReq(const struct worker *wrk, struct sess *sp) req->t_prev = NAN; req->t_req = NAN; - VTAILQ_INIT(&req->body->list); - return (req); } diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 58ab16f..2a5e692 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -291,13 +291,6 @@ STV_NewObject(struct objcore *oc, struct vsl_log *vsl, struct dstat *stats, /*-------------------------------------------------------------------*/ -struct storage * -STV_alloc_transient(size_t size) -{ - - return (STV_alloc(stv_transient, size)); -} - void STV_trim(struct storage *st, size_t size, int move_ok) { From phk at FreeBSD.org Mon Sep 15 19:07:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 21:07:46 +0200 Subject: [master] f7e2c15 Reintegrate struct body into struct object. Message-ID: commit f7e2c15615bd35e20f35c4050f6a2295210b0502 Author: Poul-Henning Kamp Date: Mon Sep 15 19:07:31 2014 +0000 Reintegrate struct body into struct object. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 75c4bdc..86f3c34 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -104,7 +104,6 @@ enum { struct SHA256Context; struct VSC_C_lck; struct ban; -struct body; struct busyobj; struct cli; struct cli_proto; @@ -391,7 +390,7 @@ struct lru { struct storeobj { unsigned magic; #define STOREOBJ_MAGIC 0x6faed850 - const struct stevedore *stevedore; + struct stevedore *stevedore; void *priv; uintptr_t priv2; }; @@ -534,12 +533,6 @@ struct busyobj { VTAILQ_HEAD(storagehead, storage); -struct body { - struct stevedore *stevedore; - struct storagehead list; - ssize_t len; -}; - enum obj_attr { #define OBJ_ATTR(U, l) OA_##U, #include "tbl/obj_attr.h" @@ -564,7 +557,9 @@ struct object { char oa_gzipbits[24]; char oa_lastmodified[8]; - struct body body[1]; + // struct stevedore *stevedore; + struct storagehead list; + ssize_t len; struct storage *esidata; }; diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index f57bd0b..81e65cf 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -116,7 +116,7 @@ ObjIter(struct objiter *oi, void **p, ssize_t *l) if (oi->bo == NULL) { if (oi->st == NULL) - oi->st = VTAILQ_FIRST(&oi->obj->body->list); + oi->st = VTAILQ_FIRST(&oi->obj->list); else oi->st = VTAILQ_NEXT(oi->st, list); while(oi->st != NULL && oi->st->len == 0) @@ -140,8 +140,8 @@ ObjIter(struct objiter *oi, void **p, ssize_t *l) return (OIS_ERROR); } Lck_Lock(&oi->bo->mtx); - AZ(VTAILQ_EMPTY(&oi->obj->body->list)); - VTAILQ_FOREACH(oi->st, &oi->obj->body->list, list) { + AZ(VTAILQ_EMPTY(&oi->obj->list)); + VTAILQ_FOREACH(oi->st, &oi->obj->list, list) { if (oi->st->len > ol) { *p = oi->st->ptr + ol; *l = oi->st->len - ol; @@ -231,7 +231,7 @@ ObjGetSpace(struct objcore *oc, struct vsl_log *vsl, struct dstat *ds, o = obj_getobj(oc, ds); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - st = VTAILQ_LAST(&o->body->list, storagehead); + st = VTAILQ_LAST(&o->list, storagehead); if (st != NULL && st->len < st->space) { *sz = st->space - st->len; *ptr = st->ptr + st->len; @@ -239,17 +239,17 @@ ObjGetSpace(struct objcore *oc, struct vsl_log *vsl, struct dstat *ds, return (1); } - st = objallocwithnuke(o->body->stevedore, vsl, ds, *sz); + st = objallocwithnuke(oc->stobj->stevedore, vsl, ds, *sz); if (st == NULL) return (0); if (oc->busyobj != NULL) { CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC); Lck_Lock(&oc->busyobj->mtx); - VTAILQ_INSERT_TAIL(&o->body->list, st, list); + VTAILQ_INSERT_TAIL(&o->list, st, list); Lck_Unlock(&oc->busyobj->mtx); } else { - VTAILQ_INSERT_TAIL(&o->body->list, st, list); + VTAILQ_INSERT_TAIL(&o->list, st, list); } *sz = st->space - st->len; assert (*sz > 0); @@ -269,11 +269,11 @@ ObjExtend(struct objcore *oc, struct dstat *ds, ssize_t l) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); o = obj_getobj(oc, ds); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - st = VTAILQ_LAST(&o->body->list, storagehead); + st = VTAILQ_LAST(&o->list, storagehead); CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); assert(st->len + l <= st->space); st->len += l; - o->body->len += l; + o->len += l; } /*-------------------------------------------------------------------- @@ -292,11 +292,11 @@ ObjTrimStore(struct objcore *oc, struct dstat *ds) CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); o = obj_getobj(oc, ds); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - st = VTAILQ_LAST(&o->body->list, storagehead); + st = VTAILQ_LAST(&o->list, storagehead); if (st == NULL) return; if (st->len == 0) { - VTAILQ_REMOVE(&o->body->list, st, list); + VTAILQ_REMOVE(&o->list, st, list); STV_free(st); } else if (st->len < st->space) { STV_trim(st, st->len, 1); @@ -322,9 +322,9 @@ ObjSlim(struct objcore *oc, struct dstat *ds) STV_free(o->esidata); o->esidata = NULL; } - VTAILQ_FOREACH_SAFE(st, &o->body->list, list, stn) { + VTAILQ_FOREACH_SAFE(st, &o->list, list, stn) { CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); - VTAILQ_REMOVE(&o->body->list, st, list); + VTAILQ_REMOVE(&o->list, st, list); STV_free(st); } } @@ -418,7 +418,7 @@ ObjSetattr(const struct vfp_ctx *vc, enum obj_attr attr, ssize_t len, st = o->objstore; switch (attr) { case OA_ESIDATA: - o->esidata = objallocwithnuke(o->body->stevedore, vc->vsl, + o->esidata = objallocwithnuke(vc->oc->stobj->stevedore, vc->vsl, vc->stats, len); if (o->esidata == NULL) return (NULL); @@ -498,7 +498,7 @@ ObjGetLen(struct objcore *oc, struct dstat *ds) o = obj_getobj(oc, ds); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - return (o->body->len); + return (o->len); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 2a5e692..11c8828 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -202,11 +202,10 @@ STV_MkObject(struct stevedore *stv, struct objcore *oc, void *ptr) memset(o, 0, sizeof *o); o->magic = OBJECT_MAGIC; - VTAILQ_INIT(&o->body->list); + VTAILQ_INIT(&o->list); oc->stobj->magic = STOREOBJ_MAGIC; oc->stobj->stevedore = stv; - o->body->stevedore = stv; AN(stv->methods); oc->stobj->priv = o; oc->stobj->priv2 = (uintptr_t)stv; diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c index 440446a..190843f 100644 --- a/bin/varnishd/storage/storage_persistent_silo.c +++ b/bin/varnishd/storage/storage_persistent_silo.c @@ -432,13 +432,13 @@ smp_oc_getobj(struct dstat *ds, struct objcore *oc) bad = 0; l = 0; - VTAILQ_FOREACH(st, &o->body->list, list) { + VTAILQ_FOREACH(st, &o->list, list) { bad |= smp_loaded_st(sg->sc, sg, st); if (bad) break; l += st->len; } - if (l != o->body->len) + if (l != o->len) bad |= 0x100; if(bad) { From phk at FreeBSD.org Mon Sep 15 19:31:32 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 21:31:32 +0200 Subject: [master] 5da2d31 Move struct object and its methods to storage.h Message-ID: commit 5da2d3192731ee86d9efb42daa46fe690700443a Author: Poul-Henning Kamp Date: Mon Sep 15 19:14:48 2014 +0000 Move struct object and its methods to storage.h diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 86f3c34..3add9ef 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -395,18 +395,6 @@ struct storeobj { uintptr_t priv2; }; -typedef struct object *getobj_f(struct dstat *ds, struct objcore *oc); -typedef void updatemeta_f(struct objcore *oc, struct dstat *); -typedef void freeobj_f(struct dstat *ds, struct objcore *oc); -typedef struct lru *getlru_f(const struct objcore *oc); - -struct storeobj_methods { - getobj_f *getobj; - updatemeta_f *updatemeta; - freeobj_f *freeobj; - getlru_f *getlru; -}; - /* Object core structure --------------------------------------------- * Objects have sideways references in the binary heap and the LRU list * and we want to avoid paging in a lot of objects just to move them up @@ -531,8 +519,6 @@ struct busyobj { /* Object structure --------------------------------------------------*/ -VTAILQ_HEAD(storagehead, storage); - enum obj_attr { #define OBJ_ATTR(U, l) OA_##U, #include "tbl/obj_attr.h" @@ -545,25 +531,6 @@ enum obj_flags { #undef OBJ_FLAG }; -struct object { - unsigned magic; -#define OBJECT_MAGIC 0x32851d42 - struct storage *objstore; - - char oa_vxid[4]; - uint8_t *oa_vary; - uint8_t *oa_http; - uint8_t oa_flags[1]; - char oa_gzipbits[24]; - char oa_lastmodified[8]; - - // struct stevedore *stevedore; - struct storagehead list; - ssize_t len; - - struct storage *esidata; -}; - /*--------------------------------------------------------------------*/ struct req { diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index 56c72b8..e9a12be 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -34,6 +34,7 @@ struct stv_objsecrets; struct stevedore; struct sess; +struct dstat; struct busyobj; struct objcore; struct worker; @@ -55,6 +56,43 @@ struct storage { unsigned space; }; +/* Object ------------------------------------------------------------*/ + +VTAILQ_HEAD(storagehead, storage); + +struct object { + unsigned magic; +#define OBJECT_MAGIC 0x32851d42 + struct storage *objstore; + + char oa_vxid[4]; + uint8_t *oa_vary; + uint8_t *oa_http; + uint8_t oa_flags[1]; + char oa_gzipbits[24]; + char oa_lastmodified[8]; + + struct storagehead list; + ssize_t len; + + struct storage *esidata; +}; + +/* -------------------------------------------------------------------*/ + +typedef struct object *getobj_f(struct dstat *ds, struct objcore *oc); +typedef void updatemeta_f(struct objcore *oc, struct dstat *); +typedef void freeobj_f(struct dstat *ds, struct objcore *oc); +typedef struct lru *getlru_f(const struct objcore *oc); + +struct storeobj_methods { + getobj_f *getobj; + updatemeta_f *updatemeta; + freeobj_f *freeobj; + getlru_f *getlru; +}; + +/* Prototypes --------------------------------------------------------*/ typedef void storage_init_f(struct stevedore *, int ac, char * const *av); typedef void storage_open_f(const struct stevedore *); From phk at FreeBSD.org Mon Sep 15 19:31:32 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 21:31:32 +0200 Subject: [master] e7a1289 Minor cleanup. Message-ID: commit e7a12895014b6e3b6514c803c3da60d63bb5cf42 Author: Poul-Henning Kamp Date: Mon Sep 15 19:31:19 2014 +0000 Minor cleanup. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 3add9ef..f463397 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -403,6 +403,18 @@ struct storeobj { * housekeeping fields parts of an object. */ +enum obj_attr { +#define OBJ_ATTR(U, l) OA_##U, +#include "tbl/obj_attr.h" +#undef OBJ_ATTR +}; + +enum obj_flags { +#define OBJ_FLAG(U, l, v) OF_##U = v, +#include "tbl/obj_attr.h" +#undef OBJ_FLAG +}; + struct objcore { unsigned magic; #define OBJCORE_MAGIC 0x4d301302 @@ -517,19 +529,6 @@ struct busyobj { struct vsb *synth_body; }; -/* Object structure --------------------------------------------------*/ - -enum obj_attr { -#define OBJ_ATTR(U, l) OA_##U, -#include "tbl/obj_attr.h" -#undef OBJ_ATTR -}; - -enum obj_flags { -#define OBJ_FLAG(U, l, v) OF_##U = v, -#include "tbl/obj_attr.h" -#undef OBJ_FLAG -}; /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 81e65cf..6013818 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -64,7 +64,7 @@ obj_getobj(struct objcore *oc, struct dstat *ds) AN(ds); AN(m->getobj); - return (m->getobj(ds, oc)); + return (m->getobj(oc, ds)); } /*-------------------------------------------------------------------- @@ -346,7 +346,7 @@ ObjFreeObj(struct objcore *oc, struct dstat *ds) AN(ds); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AN(m->freeobj); - m->freeobj(ds, oc); + m->freeobj(oc, ds); } struct lru * diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 11c8828..03f599a 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -49,7 +49,7 @@ static const struct stevedore * volatile stv_next; */ static struct object * __match_proto__(getobj_f) -default_oc_getobj(struct dstat *ds, struct objcore *oc) +default_oc_getobj(struct objcore *oc, struct dstat *ds) { struct object *o; @@ -61,7 +61,7 @@ default_oc_getobj(struct dstat *ds, struct objcore *oc) } static void __match_proto__(freeobj_f) -default_oc_freeobj(struct dstat *ds, struct objcore *oc) +default_oc_freeobj(struct objcore *oc, struct dstat *ds) { struct object *o; diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index e9a12be..1e42e23 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -80,9 +80,9 @@ struct object { /* -------------------------------------------------------------------*/ -typedef struct object *getobj_f(struct dstat *ds, struct objcore *oc); +typedef struct object *getobj_f(struct objcore *oc, struct dstat *); typedef void updatemeta_f(struct objcore *oc, struct dstat *); -typedef void freeobj_f(struct dstat *ds, struct objcore *oc); +typedef void freeobj_f(struct objcore *oc, struct dstat *); typedef struct lru *getlru_f(const struct objcore *oc); struct storeobj_methods { diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c index 190843f..b0dbff9 100644 --- a/bin/varnishd/storage/storage_persistent_silo.c +++ b/bin/varnishd/storage/storage_persistent_silo.c @@ -388,7 +388,7 @@ smp_loaded_st(const struct smp_sc *sc, const struct smp_seg *sg, */ static struct object * -smp_oc_getobj(struct dstat *ds, struct objcore *oc) +smp_oc_getobj(struct objcore *oc, struct dstat *ds) { struct object *o; struct smp_seg *sg; @@ -464,7 +464,7 @@ smp_oc_updatemeta(struct objcore *oc, struct dstat *ds) struct smp_object *so; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - o = smp_oc_getobj(ds, oc); + o = smp_oc_getobj(oc, ds); AN(o); CAST_OBJ_NOTNULL(sg, oc->stobj->priv, SMP_SEG_MAGIC); @@ -484,7 +484,7 @@ smp_oc_updatemeta(struct objcore *oc, struct dstat *ds) } static void __match_proto__(freeobj_f) -smp_oc_freeobj(struct dstat *ds, struct objcore *oc) +smp_oc_freeobj(struct objcore *oc, struct dstat *ds) { struct smp_seg *sg; struct smp_object *so; From phk at FreeBSD.org Mon Sep 15 20:25:59 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 22:25:59 +0200 Subject: [master] 6c9c80e Start to make all the Obj* functions overridable by stevedores, begining with the ObjIter set. Have them consistently take objcore as first argument and replace "struct objiter*" into a void* cookie of the stevedores choice. Message-ID: commit 6c9c80e2ef28e9365b729892578c3f0a5404dfe7 Author: Poul-Henning Kamp Date: Mon Sep 15 20:24:39 2014 +0000 Start to make all the Obj* functions overridable by stevedores, begining with the ObjIter set. Have them consistently take objcore as first argument and replace "struct objiter*" into a void* cookie of the stevedores choice. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index f463397..62e3485 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -113,7 +113,6 @@ struct mempool; struct objcore; struct object; struct objhead; -struct objiter; struct pool; struct poolparam; struct req; @@ -991,9 +990,9 @@ enum objiter_status { OIS_STREAM, OIS_ERROR, }; -struct objiter *ObjIterBegin(struct worker *, struct objcore *); -enum objiter_status ObjIter(struct objiter *, void **, ssize_t *); -void ObjIterEnd(struct objiter **); +void *ObjIterBegin(struct objcore *, struct worker *); +enum objiter_status ObjIter(struct objcore *, void *, void **, ssize_t *); +void ObjIterEnd(struct objcore *, void **); int ObjGetSpace(struct objcore *, struct vsl_log *vsl, struct dstat *, ssize_t *sz, uint8_t **ptr); void ObjExtend(struct objcore *, struct dstat *, ssize_t l); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index d272bfc..e315e82 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -261,7 +261,7 @@ ESI_Deliver(struct req *req) ssize_t dl; const void *dp; int i; - struct objiter *oi; + void *oi; enum objiter_status ois; void *sp; uint8_t *pp; @@ -309,8 +309,8 @@ ESI_Deliver(struct req *req) AZ(dl); } - oi = ObjIterBegin(req->wrk, req->objcore); - ois = ObjIter(oi, &sp, &sl); + oi = ObjIterBegin(req->objcore, req->wrk); + ois = ObjIter(req->objcore, oi, &sp, &sl); assert(ois != OIS_ERROR); pp = sp; @@ -380,7 +380,8 @@ ESI_Deliver(struct req *req) } pp += l2; if (sl == 0) { - ois = ObjIter(oi, &sp, &sl); + ois = ObjIter(req->objcore, oi, + &sp, &sl); assert(ois != OIS_ERROR); pp = sp; } @@ -404,7 +405,8 @@ ESI_Deliver(struct req *req) l -= l2; pp += l2; if (sl == 0) { - ois = ObjIter(oi, &sp, &sl); + ois = ObjIter(req->objcore, oi, + &sp, &sl); assert(ois != OIS_ERROR); pp = sp; } @@ -455,7 +457,7 @@ ESI_Deliver(struct req *req) req->resp_bodybytes += WRW_Write(req->wrk, tailbuf, 13); } (void)WRW_Flush(req->wrk); - ObjIterEnd(&oi); + ObjIterEnd(req->objcore, &oi); } /*--------------------------------------------------------------------- @@ -475,7 +477,7 @@ ESI_DeliverChild(struct req *req) uint8_t *pp; uint8_t tailbuf[8]; enum objiter_status ois; - struct objiter *oi; + void *oi; void *sp; ssize_t sl, ll, dl; @@ -483,13 +485,13 @@ ESI_DeliverChild(struct req *req) CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); if (!ObjCheckFlag(req->objcore, &req->wrk->stats, OF_GZIPED)) { - oi = ObjIterBegin(req->wrk, req->objcore); + oi = ObjIterBegin(req->objcore, req->wrk); do { - ois = ObjIter(oi, &sp, &sl); + ois = ObjIter(req->objcore, oi, &sp, &sl); if (sl > 0) ved_pretend_gzip(req, sp, sl); } while (ois == OIS_DATA || ois == OIS_STREAM); - ObjIterEnd(&oi); + ObjIterEnd(req->objcore, &oi); return; } /* @@ -523,9 +525,9 @@ ESI_DeliverChild(struct req *req) dbits = (void*)WS_Alloc(req->ws, 8); AN(dbits); ll = 0; - oi = ObjIterBegin(req->wrk, req->objcore); + oi = ObjIterBegin(req->objcore, req->wrk); do { - ois = ObjIter(oi, &sp, &sl); + ois = ObjIter(req->objcore, oi, &sp, &sl); pp = sp; if (sl > 0) { /* Skip over the GZIP header */ @@ -651,7 +653,7 @@ ESI_DeliverChild(struct req *req) } } } while (ois == OIS_DATA || ois == OIS_STREAM); - ObjIterEnd(&oi); + ObjIterEnd(req->objcore, &oi); icrc = vle32dec(tailbuf); ilen = vle32dec(tailbuf + 4); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index b245b82..56e4eed 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -661,7 +661,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) static enum fetch_step vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) { - struct objiter *oi; + void *oi; void *sp; ssize_t sl, al, l; uint8_t *ptr; @@ -686,9 +686,9 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) al = 0; ol = ObjGetLen(bo->ims_oc, bo->stats); - oi = ObjIterBegin(wrk, bo->ims_oc); + oi = ObjIterBegin(bo->ims_oc, wrk); do { - ois = ObjIter(oi, &sp, &sl); + ois = ObjIter(bo->ims_oc, oi, &sp, &sl); while (sl > 0) { l = ol - al; if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK) @@ -702,7 +702,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) sl -= l; } } while (!bo->vfc->failed && (ois == OIS_DATA || ois == OIS_STREAM)); - ObjIterEnd(&oi); + ObjIterEnd(bo->ims_oc, &oi); if (bo->vfc->failed) return (F_STP_FAIL); diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c index e4d26b1..1d21ecc 100644 --- a/bin/varnishd/cache/cache_http1_deliver.c +++ b/bin/varnishd/cache/cache_http1_deliver.c @@ -169,16 +169,16 @@ v1d_WriteDirObj(struct req *req) { enum objiter_status ois; ssize_t len; - struct objiter *oi; + void *oi; void *ptr; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - oi = ObjIterBegin(req->wrk, req->objcore); + oi = ObjIterBegin(req->objcore, req->wrk); XXXAN(oi); do { - ois = ObjIter(oi, &ptr, &len); + ois = ObjIter(req->objcore, oi, &ptr, &len); switch(ois) { case OIS_DONE: AZ(len); @@ -196,7 +196,7 @@ v1d_WriteDirObj(struct req *req) } } while (ois == OIS_DATA || ois == OIS_STREAM); (void)VDP_bytes(req, VDP_FINISH, NULL, 0); - ObjIterEnd(&oi); + ObjIterEnd(req->objcore, &oi); return (ois); } diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 6013818..af69ed3 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -67,7 +67,19 @@ obj_getobj(struct objcore *oc, struct dstat *ds) return (m->getobj(oc, ds)); } -/*-------------------------------------------------------------------- +/*==================================================================== + * ObjIterBegin() + * ObjIter() + * ObjIterEnd() + * + * These three allow iteration over the body of an object. + * The ObjIterBegin() returns a magic cookie which must be passed to + * ObjIter() and which ObjIterEnd() will obliterate again. + * + * These functions get slightly complicated due to unbusy but not + * yet completed objects (ie: when streaming). Exactly how they + * interact with ObjExtend(), especially with respect to locking, + * is entirely up to the implementation. */ struct objiter { @@ -81,11 +93,15 @@ struct objiter { ssize_t len; }; -struct objiter * -ObjIterBegin(struct worker *wrk, struct objcore *oc) +void * +ObjIterBegin(struct objcore *oc, struct worker *wrk) { struct objiter *oi; struct object *obj; + const struct storeobj_methods *om = obj_getmethods(oc); + + if (om->objiterbegin != NULL) + return (om->objiterbegin(oc, wrk)); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); obj = obj_getobj(oc, &wrk->stats); @@ -102,12 +118,17 @@ ObjIterBegin(struct worker *wrk, struct objcore *oc) } enum objiter_status -ObjIter(struct objiter *oi, void **p, ssize_t *l) +ObjIter(struct objcore *oc, void *oix, void **p, ssize_t *l) { + struct objiter *oi; ssize_t ol; ssize_t nl; + const struct storeobj_methods *om = obj_getmethods(oc); - CHECK_OBJ_NOTNULL(oi, OBJITER_MAGIC); + if (om->objiter != NULL) + return (om->objiter(oc, oix, p, l)); + + CAST_OBJ_NOTNULL(oi, oix, OBJITER_MAGIC); CHECK_OBJ_NOTNULL(oi->obj, OBJECT_MAGIC); AN(p); AN(l); @@ -165,19 +186,27 @@ ObjIter(struct objiter *oi, void **p, ssize_t *l) } void -ObjIterEnd(struct objiter **oi) +ObjIterEnd(struct objcore *oc, void **oix) { + struct objiter *oi; + const struct storeobj_methods *om = obj_getmethods(oc); + + if (om->objiterend != NULL) { + om->objiterend(oc, oix); + return; + } - AN(oi); - CHECK_OBJ_NOTNULL((*oi), OBJITER_MAGIC); - CHECK_OBJ_NOTNULL((*oi)->obj, OBJECT_MAGIC); - if ((*oi)->bo != NULL) { - if ((*oi)->oc->flags & OC_F_PASS) - (*oi)->bo->abandon = 1; - VBO_DerefBusyObj((*oi)->wrk, &(*oi)->bo); + AN(oc); + AN(oix); + CAST_OBJ_NOTNULL(oi, (*oix), OBJITER_MAGIC); + *oix = NULL; + CHECK_OBJ_NOTNULL(oi->obj, OBJECT_MAGIC); + if (oi->bo != NULL) { + if (oi->oc->flags & OC_F_PASS) + oi->bo->abandon = 1; + VBO_DerefBusyObj(oi->wrk, &oi->bo); } - FREE_OBJ((*oi)); - *oi = NULL; + FREE_OBJ(oi); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index 6eb2590..dadd25c 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -54,7 +54,7 @@ VRB_Iterate(struct req *req, req_body_iter_f *func, void *priv) int i; struct vfp_ctx *vfc; enum vfp_status vfps = VFP_ERROR; - struct objiter *oi; + void *oi; enum objiter_status ois; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -62,14 +62,14 @@ VRB_Iterate(struct req *req, req_body_iter_f *func, void *priv) switch(req->req_body_status) { case REQ_BODY_CACHED: - oi = ObjIterBegin(req->wrk, req->body_oc); + oi = ObjIterBegin(req->body_oc, req->wrk); AN(oi); do { - ois = ObjIter(oi, &p, &l); + ois = ObjIter(req->body_oc, oi, &p, &l); if (l > 0 && func(req, priv, p, l)) break; } while (ois == OIS_DATA); - ObjIterEnd(&oi); + ObjIterEnd(req->body_oc, &oi); return (ois == OIS_DONE ? 0 : -1); case REQ_BODY_NONE: return (0); diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index 1e42e23..60faf9d 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -78,18 +78,44 @@ struct object { struct storage *esidata; }; -/* -------------------------------------------------------------------*/ +/* Methods on objcore ------------------------------------------------*/ -typedef struct object *getobj_f(struct objcore *oc, struct dstat *); typedef void updatemeta_f(struct objcore *oc, struct dstat *); typedef void freeobj_f(struct objcore *oc, struct dstat *); typedef struct lru *getlru_f(const struct objcore *oc); +/* + * Stevedores can either be simple, and provide just this method: + */ + +typedef struct object *getobj_f(struct objcore *oc, struct dstat *); + +/* + * Or the can be "complex" and provide all of these methods: + * (Described in comments in cache_obj.c) + */ +typedef void *objiterbegin_f(struct objcore *oc, struct worker *wrk); +typedef enum objiter_status objiter_f(struct objcore *oc, void *oix, + void **p, ssize_t *l); +typedef void objiterend_f(struct objcore *oc, void **oix); +/* objgetspace */ +/* objtrimstore */ +/* objslim */ +/* objgetattr */ +/* objsetattr */ +/* objextend */ +/* objgetlen */ + struct storeobj_methods { - getobj_f *getobj; - updatemeta_f *updatemeta; freeobj_f *freeobj; getlru_f *getlru; + updatemeta_f *updatemeta; + + getobj_f *getobj; + + objiterbegin_f *objiterbegin; + objiter_f *objiter; + objiterend_f *objiterend; }; /* Prototypes --------------------------------------------------------*/ From phk at FreeBSD.org Mon Sep 15 20:41:30 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 22:41:30 +0200 Subject: [master] aee0b24 Methodify ObjGetSpace() and ObjExtend() Message-ID: commit aee0b243ebf134270a7d4e58c45632d62f9147e2 Author: Poul-Henning Kamp Date: Mon Sep 15 20:41:03 2014 +0000 Methodify ObjGetSpace() and ObjExtend() diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index af69ed3..60b38df 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -242,7 +242,13 @@ objallocwithnuke(struct stevedore *stv, struct vsl_log *vsl, struct dstat *ds, return (st); } -/*-------------------------------------------------------------------- +/*==================================================================== + * ObjGetSpace() + * + * This function returns a pointer and length of free space. If there + * is no free space, some will be added first. + * + * The "sz" argument is an input hint of how much space is desired. */ int @@ -251,6 +257,10 @@ ObjGetSpace(struct objcore *oc, struct vsl_log *vsl, struct dstat *ds, { struct object *o; struct storage *st; + const struct storeobj_methods *om = obj_getmethods(oc); + + if (om->objgetspace != NULL) + return (om->objgetspace(oc, vsl, ds, sz, ptr)); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AN(ds); @@ -286,7 +296,11 @@ ObjGetSpace(struct objcore *oc, struct vsl_log *vsl, struct dstat *ds, return (1); } -/*-------------------------------------------------------------------- +/*==================================================================== + * ObjExtend() + * + * This function extends the used part of the object a number of bytes + * into the last space returned by ObjGetSpace() */ void @@ -294,6 +308,12 @@ ObjExtend(struct objcore *oc, struct dstat *ds, ssize_t l) { struct object *o; struct storage *st; + const struct storeobj_methods *om = obj_getmethods(oc); + + if (om->objextend != NULL) { + om->objextend(oc, ds, l); + return; + } CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); o = obj_getobj(oc, ds); diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index 60faf9d..a5376a9 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -39,6 +39,7 @@ struct busyobj; struct objcore; struct worker; struct lru; +struct vsl_log; /* Storage -----------------------------------------------------------*/ @@ -98,7 +99,10 @@ typedef void *objiterbegin_f(struct objcore *oc, struct worker *wrk); typedef enum objiter_status objiter_f(struct objcore *oc, void *oix, void **p, ssize_t *l); typedef void objiterend_f(struct objcore *oc, void **oix); -/* objgetspace */ +typedef int objgetspace_f(struct objcore *oc, struct vsl_log *vsl, + struct dstat *ds, ssize_t *sz, uint8_t **ptr); +typedef void objextend_f(struct objcore *oc, struct dstat *ds, ssize_t l); + /* objtrimstore */ /* objslim */ /* objgetattr */ @@ -116,6 +120,8 @@ struct storeobj_methods { objiterbegin_f *objiterbegin; objiter_f *objiter; objiterend_f *objiterend; + objgetspace_f *objgetspace; + objextend_f *objextend; }; /* Prototypes --------------------------------------------------------*/ From phk at FreeBSD.org Mon Sep 15 21:12:37 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Sep 2014 23:12:37 +0200 Subject: [master] 666ede5 Methodify the rest of the Obj* functions. Message-ID: commit 666ede56d8a218b3cc1b5c75f656360c6b374467 Author: Poul-Henning Kamp Date: Mon Sep 15 21:12:20 2014 +0000 Methodify the rest of the Obj* functions. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 62e3485..f1da513 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -402,18 +402,6 @@ struct storeobj { * housekeeping fields parts of an object. */ -enum obj_attr { -#define OBJ_ATTR(U, l) OA_##U, -#include "tbl/obj_attr.h" -#undef OBJ_ATTR -}; - -enum obj_flags { -#define OBJ_FLAG(U, l, v) OF_##U = v, -#include "tbl/obj_attr.h" -#undef OBJ_FLAG -}; - struct objcore { unsigned magic; #define OBJCORE_MAGIC 0x4d301302 diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 60b38df..53b686f 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -325,7 +325,32 @@ ObjExtend(struct objcore *oc, struct dstat *ds, ssize_t l) o->len += l; } -/*-------------------------------------------------------------------- +/*==================================================================== + * ObjGetlen() + * + * This is a separate function because it may need locking + */ + +uint64_t +ObjGetLen(struct objcore *oc, struct dstat *ds) +{ + struct object *o; + const struct storeobj_methods *om = obj_getmethods(oc); + + if (om->objgetlen != NULL) + return (om->objgetlen(oc, ds)); + + o = obj_getobj(oc, ds); + CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); + return (o->len); +} + + +/*==================================================================== + * ObjTrimStore() + * + * Release any surplus space allocated, we promise not to call ObjExtend() + * any more. */ void @@ -334,6 +359,12 @@ ObjTrimStore(struct objcore *oc, struct dstat *ds) const struct stevedore *stv; struct storage *st; struct object *o; + const struct storeobj_methods *om = obj_getmethods(oc); + + if (om->objtrimstore != NULL) { + om->objtrimstore(oc, ds); + return; + } CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AN(ds); @@ -352,8 +383,11 @@ ObjTrimStore(struct objcore *oc, struct dstat *ds) } } -/*-------------------------------------------------------------------- - * Early disposal of storage from soon to be killed object. +/*==================================================================== + * ObjSlim() + * + * Free the whatever storage can be freed, without freeing the actual + * object yet. */ void @@ -361,6 +395,12 @@ ObjSlim(struct objcore *oc, struct dstat *ds) { struct object *o; struct storage *st, *stn; + const struct storeobj_methods *om = obj_getmethods(oc); + + if (om->objslim != NULL) { + om->objslim(oc, ds); + return; + } CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AN(ds); @@ -378,6 +418,8 @@ ObjSlim(struct objcore *oc, struct dstat *ds) } } +/*==================================================================== + */ void ObjUpdateMeta(struct objcore *oc, struct dstat *ds) { @@ -387,6 +429,8 @@ ObjUpdateMeta(struct objcore *oc, struct dstat *ds) m->updatemeta(oc, ds); } +/*==================================================================== + */ void ObjFreeObj(struct objcore *oc, struct dstat *ds) { @@ -398,6 +442,8 @@ ObjFreeObj(struct objcore *oc, struct dstat *ds) m->freeobj(oc, ds); } +/*==================================================================== + */ struct lru * ObjGetLRU(const struct objcore *oc) { @@ -408,12 +454,22 @@ ObjGetLRU(const struct objcore *oc) return (m->getlru(oc)); } +/*==================================================================== + * ObjGetattr() + * + * Get an attribute of the object. + */ + void * ObjGetattr(struct objcore *oc, struct dstat *ds, enum obj_attr attr, ssize_t *len) { struct object *o; ssize_t dummy; + const struct storeobj_methods *om = obj_getmethods(oc); + + if (om->objgetattr != NULL) + return (om->objgetattr(oc, ds, attr, len)); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AN(ds); @@ -451,6 +507,13 @@ ObjGetattr(struct objcore *oc, struct dstat *ds, enum obj_attr attr, WRONG("Unsupported OBJ_ATTR"); } +/*==================================================================== + * ObjSetattr() + * + * If ptr is Non-NULL, it points to the new content which is copied into + * the attribute. Otherwise the caller will have to do the copying. + */ + void * ObjSetattr(const struct vfp_ctx *vc, enum obj_attr attr, ssize_t len, const void *ptr) @@ -458,6 +521,10 @@ ObjSetattr(const struct vfp_ctx *vc, enum obj_attr attr, ssize_t len, struct object *o; void *retval = NULL; struct storage *st; + const struct storeobj_methods *om = obj_getmethods(vc->oc); + + if (om->objsetattr != NULL) + return (om->objsetattr(vc, attr, len, ptr)); CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vc->bo, BUSYOBJ_MAGIC); @@ -513,6 +580,10 @@ ObjSetattr(const struct vfp_ctx *vc, enum obj_attr attr, ssize_t len, return (retval); } +/*==================================================================== + * Utility functions which work on top of the previous ones + */ + int ObjCopyAttr(const struct vfp_ctx *vc, struct objcore *ocs, enum obj_attr attr) { @@ -540,16 +611,6 @@ ObjGetXID(struct objcore *oc, struct dstat *ds) return (u); } -uint64_t -ObjGetLen(struct objcore *oc, struct dstat *ds) -{ - struct object *o; - - o = obj_getobj(oc, ds); - CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - return (o->len); -} - /*-------------------------------------------------------------------- * There is no well-defined byteorder for IEEE-754 double and the * correct solution (frexp(3) and manual encoding) is more work diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h index 7c98526..17b68b1 100644 --- a/bin/varnishd/common/common.h +++ b/bin/varnishd/common/common.h @@ -51,6 +51,18 @@ enum baninfo { BI_DROP }; +enum obj_attr { +#define OBJ_ATTR(U, l) OA_##U, +#include "tbl/obj_attr.h" +#undef OBJ_ATTR +}; + +enum obj_flags { +#define OBJ_FLAG(U, l, v) OF_##U = v, +#include "tbl/obj_attr.h" +#undef OBJ_FLAG +}; + struct cli; /********************************************************************** diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index a5376a9..9ddd770 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -31,7 +31,6 @@ * */ -struct stv_objsecrets; struct stevedore; struct sess; struct dstat; @@ -40,6 +39,7 @@ struct objcore; struct worker; struct lru; struct vsl_log; +struct vfp_ctx; /* Storage -----------------------------------------------------------*/ @@ -102,13 +102,13 @@ typedef void objiterend_f(struct objcore *oc, void **oix); typedef int objgetspace_f(struct objcore *oc, struct vsl_log *vsl, struct dstat *ds, ssize_t *sz, uint8_t **ptr); typedef void objextend_f(struct objcore *oc, struct dstat *ds, ssize_t l); - -/* objtrimstore */ -/* objslim */ -/* objgetattr */ -/* objsetattr */ -/* objextend */ -/* objgetlen */ +typedef void objtrimstore_f(struct objcore *oc, struct dstat *ds); +typedef void objslim_f(struct objcore *oc, struct dstat *ds); +typedef void *objgetattr_f(struct objcore *oc, struct dstat *ds, + enum obj_attr attr, ssize_t *len); +typedef void *objsetattr_f(const struct vfp_ctx *vc, enum obj_attr attr, + ssize_t len, const void *ptr); +typedef uint64_t objgetlen_f(struct objcore *oc, struct dstat *ds); struct storeobj_methods { freeobj_f *freeobj; @@ -122,6 +122,11 @@ struct storeobj_methods { objiterend_f *objiterend; objgetspace_f *objgetspace; objextend_f *objextend; + objgetlen_f *objgetlen; + objtrimstore_f *objtrimstore; + objslim_f *objslim; + objgetattr_f *objgetattr; + objsetattr_f *objsetattr; }; /* Prototypes --------------------------------------------------------*/ From phk at FreeBSD.org Tue Sep 16 07:49:56 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Sep 2014 09:49:56 +0200 Subject: [master] c739268 Change wrk->stats.bla to wrk->stats->bla since it trips me up every single time, and it is inconsistent with how we do it elsewhere. Message-ID: commit c739268039fb958ae729e4c7a7d97d4cce04621d Author: Poul-Henning Kamp Date: Tue Sep 16 07:49:11 2014 +0000 Change wrk->stats.bla to wrk->stats->bla since it trips me up every single time, and it is inconsistent with how we do it elsewhere. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index f1da513..b16edbb 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -348,7 +348,7 @@ struct worker { struct waitinglist *nwaitinglist; struct busyobj *nbo; void *nhashpriv; - struct dstat stats; + struct dstat stats[1]; struct pool_task task; diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 7afcef6..f178d10 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -331,7 +331,7 @@ VCA_FailSess(struct worker *wrk) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CAST_OBJ_NOTNULL(wa, (void*)wrk->aws->f, WRK_ACCEPT_MAGIC); AZ(close(wa->acceptsock)); - wrk->stats.sess_drop++; + wrk->stats->sess_drop++; vca_pace_bad(); WS_Release(wrk->aws, 0); } @@ -355,7 +355,7 @@ VCA_SetupSess(struct worker *wrk, struct sess *sp) assert(wa->acceptaddrlen <= vsa_suckaddr_len); AN(VSA_Build(sess_remote_addr(sp), &wa->acceptaddr, wa->acceptaddrlen)); vca_pace_good(); - wrk->stats.sess_conn++; + wrk->stats->sess_conn++; WS_Release(wrk->aws, 0); if (need_test) { diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index 6fe3bba..a7fd546 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -939,7 +939,7 @@ BAN_CheckObject(struct worker *wrk, struct objcore *oc, struct req *req) CHECK_OBJ_NOTNULL(b, BAN_MAGIC); if (b->flags & BANS_FLAG_COMPLETED) continue; - if (ban_evaluate(b->spec, oc, &wrk->stats, req->http, &tests)) + if (ban_evaluate(b->spec, oc, wrk->stats, req->http, &tests)) break; } @@ -958,12 +958,12 @@ BAN_CheckObject(struct worker *wrk, struct objcore *oc, struct req *req) if (b == oc->ban) { /* not banned */ oc->ban = b0; - ObjUpdateMeta(oc, &wrk->stats); + ObjUpdateMeta(oc, wrk->stats); return (0); } else { oc->ban = NULL; VSLb(vsl, SLT_ExpBan, "%u banned lookup", - ObjGetXID(oc, &wrk->stats)); + ObjGetXID(oc, wrk->stats)); VSC_C_main->bans_obj_killed++; EXP_Rearm(oc, oc->exp.t_origin, 0, 0, 0); // XXX fake now return (1); @@ -1088,7 +1088,7 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt, continue; } tests = 0; - i = ban_evaluate(bl->spec, oc, &wrk->stats, NULL, &tests); + i = ban_evaluate(bl->spec, oc, wrk->stats, NULL, &tests); VSC_C_main->bans_lurker_tested++; VSC_C_main->bans_lurker_tests_tested += tests; if (i) @@ -1096,11 +1096,11 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt, } if (i) { VSLb(vsl, SLT_ExpBan, "%u banned by lurker", - ObjGetXID(oc, &wrk->stats)); + ObjGetXID(oc, wrk->stats)); EXP_Rearm(oc, oc->exp.t_origin, 0, 0, 0); // XXX fake now VSC_C_main->bans_lurker_obj_killed++; } - (void)HSH_DerefObjCore(&wrk->stats, &oc); + (void)HSH_DerefObjCore(wrk->stats, &oc); } } diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index a0c0666..dd7834f 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -197,7 +197,7 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo) if (bo->fetch_objcore != NULL) { AN(wrk); - (void)HSH_DerefObjCore(&wrk->stats, &bo->fetch_objcore); + (void)HSH_DerefObjCore(wrk->stats, &bo->fetch_objcore); } VCL_Rel(&bo->vcl); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index e315e82..9f2059e 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -269,7 +269,7 @@ ESI_Deliver(struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); - p = ObjGetattr(req->objcore, &req->wrk->stats, OA_ESIDATA, &l); + p = ObjGetattr(req->objcore, req->wrk->stats, OA_ESIDATA, &l); AN(p); assert(l > 0); e = p + l; @@ -484,7 +484,7 @@ ESI_DeliverChild(struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); - if (!ObjCheckFlag(req->objcore, &req->wrk->stats, OF_GZIPED)) { + if (!ObjCheckFlag(req->objcore, req->wrk->stats, OF_GZIPED)) { oi = ObjIterBegin(req->objcore, req->wrk); do { ois = ObjIter(req->objcore, oi, &sp, &sl); @@ -500,13 +500,13 @@ ESI_DeliverChild(struct req *req) * padding it, as necessary, to a byte boundary. */ - p = ObjGetattr(req->objcore, &req->wrk->stats, OA_GZIPBITS, &l); + p = ObjGetattr(req->objcore, req->wrk->stats, OA_GZIPBITS, &l); AN(p); assert(l == 24); start = vbe64dec(p); last = vbe64dec(p + 8); stop = vbe64dec(p + 16); - olen = ObjGetLen(req->objcore, &req->wrk->stats); + olen = ObjGetLen(req->objcore, req->wrk->stats); assert(start > 0 && start < olen * 8); assert(last > 0 && last < olen * 8); assert(stop > 0 && stop < olen * 8); diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index 1cc223b..0b9772e 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -386,13 +386,13 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now) binheap_delete(ep->heap, oc->timer_idx); } assert(oc->timer_idx == BINHEAP_NOIDX); - (void)HSH_DerefObjCore(&ep->wrk->stats, &oc); + (void)HSH_DerefObjCore(ep->wrk->stats, &oc); return; } if (flags & OC_EF_MOVE) { oc->timer_when = EXP_When(&oc->exp); - ObjUpdateMeta(oc, &ep->wrk->stats); + ObjUpdateMeta(oc, ep->wrk->stats); } VSLb(&ep->vsl, SLT_ExpKill, "EXP_When p=%p e=%.9f f=0x%x", oc, @@ -464,9 +464,9 @@ exp_expire(struct exp_priv *ep, double now) CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); VSLb(&ep->vsl, SLT_ExpKill, "EXP_Expired x=%u t=%.0f", - ObjGetXID(oc, &ep->wrk->stats), + ObjGetXID(oc, ep->wrk->stats), EXP_Ttl(NULL, &oc->exp) - now); - (void)HSH_DerefObjCore(&ep->wrk->stats, &oc); + (void)HSH_DerefObjCore(ep->wrk->stats, &oc); return (0); } diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 56e4eed..e8dc047 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -322,7 +322,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) * no matter what the headers might say. * [RFC2516 4.3 p33] */ - wrk->stats.fetch_head++; + wrk->stats->fetch_head++; bo->htc->body_status = BS_NONE; } else if (http_GetStatus(bo->beresp) <= 199) { /* @@ -330,33 +330,33 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) * [RFC2616 4.3 p33] * ... but we should never see them. */ - wrk->stats.fetch_1xx++; + wrk->stats->fetch_1xx++; bo->htc->body_status = BS_ERROR; } else if (http_IsStatus(bo->beresp, 204)) { /* * 204 is "No Content", obviously don't expect a body. * [RFC2616 10.2.5 p60] */ - wrk->stats.fetch_204++; + wrk->stats->fetch_204++; bo->htc->body_status = BS_NONE; } else if (http_IsStatus(bo->beresp, 304)) { /* * 304 is "Not Modified" it has no body. * [RFC2616 10.3.5 p63] */ - wrk->stats.fetch_304++; + wrk->stats->fetch_304++; bo->htc->body_status = BS_NONE; } else if (bo->htc->body_status == BS_CHUNKED) { - wrk->stats.fetch_chunked++; + wrk->stats->fetch_chunked++; } else if (bo->htc->body_status == BS_LENGTH) { assert(bo->htc->content_length > 0); - wrk->stats.fetch_length++; + wrk->stats->fetch_length++; } else if (bo->htc->body_status == BS_EOF) { - wrk->stats.fetch_eof++; + wrk->stats->fetch_eof++; } else if (bo->htc->body_status == BS_ERROR) { - wrk->stats.fetch_bad++; + wrk->stats->fetch_bad++; } else if (bo->htc->body_status == BS_NONE) { - wrk->stats.fetch_none++; + wrk->stats->fetch_none++; } else { WRONG("wrong bodystatus"); } @@ -611,7 +611,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) assert (bo->state == BOS_REQ_DONE); if (bo->do_stream) { - HSH_Unbusy(&wrk->stats, bo->fetch_objcore); + HSH_Unbusy(wrk->stats, bo->fetch_objcore); VBO_setstate(bo, BOS_STREAM); } @@ -638,7 +638,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) assert(bo->state == BOS_STREAM); else { assert(bo->state == BOS_REQ_DONE); - HSH_Unbusy(&wrk->stats, bo->fetch_objcore); + HSH_Unbusy(wrk->stats, bo->fetch_objcore); } /* Recycle the backend connection before setting BOS_FINISHED to @@ -680,7 +680,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) AZ(ObjCopyAttr(bo->vfc, bo->ims_oc, OA_GZIPBITS)); if (bo->do_stream) { - HSH_Unbusy(&wrk->stats, bo->fetch_objcore); + HSH_Unbusy(wrk->stats, bo->fetch_objcore); VBO_setstate(bo, BOS_STREAM); } @@ -707,7 +707,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) return (F_STP_FAIL); if (!bo->do_stream) - HSH_Unbusy(&wrk->stats, bo->fetch_objcore); + HSH_Unbusy(wrk->stats, bo->fetch_objcore); assert(al == ol); assert(ObjGetLen(bo->fetch_objcore, bo->stats) == al); @@ -801,7 +801,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) VSB_delete(bo->synth_body); bo->synth_body = NULL; - HSH_Unbusy(&wrk->stats, bo->fetch_objcore); + HSH_Unbusy(wrk->stats, bo->fetch_objcore); VBO_setstate(bo, BOS_FINISHED); return (F_STP_DONE); } @@ -824,7 +824,7 @@ vbf_stp_fail(struct worker *wrk, struct busyobj *bo) EXP_Rearm(bo->fetch_objcore, bo->fetch_objcore->exp.t_origin, 0, 0, 0); } - wrk->stats.fetch_failed++; + wrk->stats->fetch_failed++; VBO_setstate(bo, BOS_FAILED); return (F_STP_DONE); } @@ -857,7 +857,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv) assert(isnan(bo->t_prev)); VSLb_ts_busyobj(bo, "Start", W_TIM_real(wrk)); - bo->stats = &wrk->stats; + bo->stats = wrk->stats; while (stp != F_STP_DONE) { CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); @@ -895,7 +895,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv) AZ(bo->fetch_objcore->busyobj); if (bo->ims_oc != NULL) - (void)HSH_DerefObjCore(&wrk->stats, &bo->ims_oc); + (void)HSH_DerefObjCore(wrk->stats, &bo->ims_oc); bo->stats = NULL; @@ -951,7 +951,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, AZ(bo->ims_oc); if (oldoc != NULL && - ObjCheckFlag(oldoc, &req->wrk->stats, OF_IMSCAND)) { + ObjCheckFlag(oldoc, req->wrk->stats, OF_IMSCAND)) { assert(oldoc->refcnt > 0); HSH_Ref(oldoc); bo->ims_oc = oldoc; diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 9bcc322..b102928 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -75,7 +75,7 @@ HSH_NewObjCore(struct worker *wrk) ALLOC_OBJ(oc, OBJCORE_MAGIC); XXXAN(oc); - wrk->stats.n_objectcore++; + wrk->stats->n_objectcore++; oc->flags |= OC_F_BUSY; return (oc); } @@ -110,7 +110,7 @@ hsh_prealloc(struct worker *wrk) if (wrk->nobjhead == NULL) { wrk->nobjhead = hsh_newobjhead(); - wrk->stats.n_objecthead++; + wrk->stats->n_objecthead++; } CHECK_OBJ_NOTNULL(wrk->nobjhead, OBJHEAD_MAGIC); @@ -119,7 +119,7 @@ hsh_prealloc(struct worker *wrk) XXXAN(wl); VTAILQ_INIT(&wl->list); wrk->nwaitinglist = wl; - wrk->stats.n_waitinglist++; + wrk->stats->n_waitinglist++; } CHECK_OBJ_NOTNULL(wrk->nwaitinglist, WAITINGLIST_MAGIC); @@ -155,19 +155,19 @@ HSH_Cleanup(struct worker *wrk) if (wrk->nobjcore != NULL) { FREE_OBJ(wrk->nobjcore); - wrk->stats.n_objectcore--; + wrk->stats->n_objectcore--; wrk->nobjcore = NULL; } if (wrk->nobjhead != NULL) { Lck_Delete(&wrk->nobjhead->mtx); FREE_OBJ(wrk->nobjhead); wrk->nobjhead = NULL; - wrk->stats.n_objecthead--; + wrk->stats->n_objecthead--; } if (wrk->nwaitinglist != NULL) { FREE_OBJ(wrk->nwaitinglist); wrk->nwaitinglist = NULL; - wrk->stats.n_waitinglist--; + wrk->stats->n_waitinglist--; } if (wrk->nhashpriv != NULL) { /* XXX: If needed, add slinger method for this */ @@ -306,8 +306,8 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc) /* NB: do not deref objhead the new object inherits our reference */ oc->objhead = oh; Lck_Unlock(&oh->mtx); - wrk->stats.n_objectcore++; - wrk->stats.n_vampireobject++; + wrk->stats->n_objectcore++; + wrk->stats->n_vampireobject++; } /*--------------------------------------------------------------------- @@ -424,7 +424,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, if (BAN_CheckObject(wrk, oc, req)) continue; - vary = ObjGetattr(oc, &wrk->stats, OA_VARY, NULL); + vary = ObjGetattr(oc, wrk->stats, OA_VARY, NULL); if (vary != NULL && !VRY_Match(req, vary)) continue; @@ -437,7 +437,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, if (oh->hits < LONG_MAX) oh->hits++; Lck_Unlock(&oh->mtx); - assert(HSH_DerefObjHead(&wrk->stats, &oh)); + assert(HSH_DerefObjHead(wrk->stats, &oh)); *ocp = oc; return (HSH_HIT); } @@ -465,7 +465,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, oh->hits++; Lck_Unlock(&oh->mtx); if (retval == HSH_EXP) - assert(HSH_DerefObjHead(&wrk->stats, &oh)); + assert(HSH_DerefObjHead(wrk->stats, &oh)); *ocp = exp_oc; return (retval); } @@ -497,7 +497,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, VSLb(req->vsl, SLT_Debug, "hit busy obj <%p>", oh); } - wrk->stats.busy_sleep++; + wrk->stats->busy_sleep++; /* * The objhead reference transfers to the sess, we get it * back when the sess comes off the waiting list and @@ -602,7 +602,7 @@ double keep) oc = ocp[n]; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); EXP_Rearm(oc, now, ttl, grace, keep); - (void)HSH_DerefObjCore(&wrk->stats, &oc); + (void)HSH_DerefObjCore(wrk->stats, &oc); } } while (more); WS_Release(wrk->aws, 0); diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c index 1d21ecc..f58ea70 100644 --- a/bin/varnishd/cache/cache_http1_deliver.c +++ b/bin/varnishd/cache/cache_http1_deliver.c @@ -97,9 +97,9 @@ v1d_dorange(struct req *req, struct busyobj *bo, const char *r) /* We must snapshot the length if we're streaming from the backend */ if (bo != NULL) - len = VBO_waitlen(bo, &req->wrk->stats, -1); + len = VBO_waitlen(bo, req->wrk->stats, -1); else - len = ObjGetLen(req->objcore, &req->wrk->stats); + len = ObjGetLen(req->objcore, req->wrk->stats); if (strncmp(r, "bytes=", 6)) return; @@ -239,7 +239,7 @@ V1D_Deliver(struct req *req, struct busyobj *bo) req->res_mode = 0; - if (!req->disable_esi && ObjGetattr(req->objcore, &req->wrk->stats, + if (!req->disable_esi && ObjGetattr(req->objcore, req->wrk->stats, OA_ESIDATA, NULL) != NULL) { /* In ESI mode, we can't know the aggregate length */ req->res_mode &= ~RES_LEN; @@ -252,11 +252,11 @@ V1D_Deliver(struct req *req, struct busyobj *bo) /* XXX: Not happy with this convoluted test */ req->res_mode |= RES_LEN; if (!(req->objcore->flags & OC_F_PASS) || - ObjGetLen(req->objcore, &req->wrk->stats) != 0) { + ObjGetLen(req->objcore, req->wrk->stats) != 0) { http_Unset(req->resp, H_Content_Length); http_PrintfHeader(req->resp, "Content-Length: %ju", (uintmax_t)ObjGetLen( - req->objcore, &req->wrk->stats)); + req->objcore, req->wrk->stats)); } } @@ -267,7 +267,7 @@ V1D_Deliver(struct req *req, struct busyobj *bo) } if (cache_param->http_gzip_support && - ObjCheckFlag(req->objcore, &req->wrk->stats, OF_GZIPED) && + ObjCheckFlag(req->objcore, req->wrk->stats, OF_GZIPED) && !RFC2616_Req_Gzip(req->http)) { /* * We don't know what it uncompresses to @@ -340,13 +340,13 @@ V1D_Deliver(struct req *req, struct busyobj *bo) l = -1; while (req->objcore->busyobj) { assert(bo != NULL); - l = VBO_waitlen(bo, &req->wrk->stats, l); + l = VBO_waitlen(bo, req->wrk->stats, l); } ESI_DeliverChild(req); } else if (req->res_mode & RES_GUNZIP || (req->res_mode & RES_ESI_CHILD && !req->gzip_resp && - ObjCheckFlag(req->objcore, &req->wrk->stats, OF_GZIPED))) { + ObjCheckFlag(req->objcore, req->wrk->stats, OF_GZIPED))) { VDP_push(req, VDP_gunzip); req->vgz = VGZ_NewUngzip(req->vsl, "U D -"); AZ(VGZ_WrwInit(req->vgz)); diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 2a6b781..970ea24 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -142,7 +142,7 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req) when = sp->t_idle + cache_param->timeout_linger; tmo = (int)(1e3 * (when - now)); if (when < now || tmo == 0) { - wrk->stats.sess_herd++; + wrk->stats->sess_herd++; SES_ReleaseReq(req); WAIT_Enter(sp); return (REQ_FSM_DONE); @@ -161,7 +161,7 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req) } } req->acct.req_hdrbytes += Tlen(req->htc->rxbuf); - CNT_AcctLogCharge(&wrk->stats, req); + CNT_AcctLogCharge(wrk->stats, req); SES_ReleaseReq(req); assert(why != SC_NULL); SES_Delete(sp, why, now); @@ -202,7 +202,7 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req) /* Charge and log byte counters */ AN(req->vsl->wid); - CNT_AcctLogCharge(&wrk->stats, req); + CNT_AcctLogCharge(wrk->stats, req); req->req_bodybytes = 0; req->resp_hdrbytes = 0; req->resp_bodybytes = 0; @@ -226,7 +226,7 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req) SES_Close(sp, req->doclose); if (sp->fd < 0) { - wrk->stats.sess_closed++; + wrk->stats->sess_closed++; AZ(req->vcl); SES_ReleaseReq(req); SES_Delete(sp, SC_NULL, NAN); @@ -239,12 +239,12 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req) if (HTTP1_Reinit(req->htc) == HTTP1_COMPLETE) { AZ(req->vsl->wid); req->t_first = req->t_req = sp->t_idle; - wrk->stats.sess_pipeline++; + wrk->stats->sess_pipeline++; req->acct.req_hdrbytes += Tlen(req->htc->rxbuf); return (SESS_DONE_RET_START); } else { if (Tlen(req->htc->rxbuf)) - wrk->stats.sess_readahead++; + wrk->stats->sess_readahead++; return (SESS_DONE_RET_WAIT); } } @@ -287,7 +287,7 @@ http1_dissect(struct worker *wrk, struct req *req) /* If we could not even parse the request, just close */ if (req->err_code != 0) { VSLbt(req->vsl, SLT_HttpGarbage, req->htc->rxbuf); - wrk->stats.client_req_400++; + wrk->stats->client_req_400++; r = write(req->sp->fd, r_400, strlen(r_400)); if (r > 0) req->acct.resp_hdrbytes += r; @@ -311,7 +311,7 @@ http1_dissect(struct worker *wrk, struct req *req) if (http_GetHdr(req->http, H_Expect, &p)) { if (strcasecmp(p, "100-continue")) { - wrk->stats.client_req_417++; + wrk->stats->client_req_417++; req->err_code = 417; r = write(req->sp->fd, r_417, strlen(r_417)); if (r > 0) @@ -329,8 +329,8 @@ http1_dissect(struct worker *wrk, struct req *req) http_Unset(req->http, H_Expect); } - wrk->stats.client_req++; - wrk->stats.s_req++; + wrk->stats->client_req++; + wrk->stats->s_req++; AZ(req->err_code); req->ws_req = WS_Snapshot(req->ws); @@ -378,7 +378,7 @@ HTTP1_Session(struct worker *wrk, struct req *req) */ if (req->req_step == R_STP_LOOKUP && VTCP_check_hup(sp->fd)) { AN(req->hash_objhead); - (void)HSH_DerefObjHead(&wrk->stats, &req->hash_objhead); + (void)HSH_DerefObjHead(wrk->stats, &req->hash_objhead); AZ(req->hash_objhead); SES_Close(sp, SC_REM_CLOSE); sdr = http1_cleanup(sp, wrk, req); diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 53b686f..f846541 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -104,7 +104,7 @@ ObjIterBegin(struct objcore *oc, struct worker *wrk) return (om->objiterbegin(oc, wrk)); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - obj = obj_getobj(oc, &wrk->stats); + obj = obj_getobj(oc, wrk->stats); CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC); ALLOC_OBJ(oi, OBJITER_MAGIC); if (oi == NULL) @@ -152,7 +152,7 @@ ObjIter(struct objcore *oc, void *oix, void **p, ssize_t *l) } else { ol = oi->len; while (1) { - nl = VBO_waitlen(oi->bo, &oi->wrk->stats, ol); + nl = VBO_waitlen(oi->bo, oi->wrk->stats, ol); if (nl != ol) break; if (oi->bo->state == BOS_FINISHED) diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index f4de5f3..fc41856 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -101,23 +101,23 @@ pool_sumstat(const struct dstat *src) } void -Pool_Sumstat(struct worker *w) +Pool_Sumstat(struct worker *wrk) { Lck_Lock(&wstat_mtx); - pool_sumstat(&w->stats); + pool_sumstat(wrk->stats); Lck_Unlock(&wstat_mtx); - memset(&w->stats, 0, sizeof w->stats); + memset(wrk->stats, 0, sizeof *wrk->stats); } static int -Pool_TrySumstat(struct worker *w) +Pool_TrySumstat(struct worker *wrk) { if (Lck_Trylock(&wstat_mtx)) return (0); - pool_sumstat(&w->stats); + pool_sumstat(wrk->stats); Lck_Unlock(&wstat_mtx); - memset(&w->stats, 0, sizeof w->stats); + memset(wrk->stats, 0, sizeof *wrk->stats); return (1); } @@ -222,7 +222,7 @@ pool_accept(struct worker *wrk, void *arg) return; } if (VCA_Accept(ps->lsock, wa) < 0) { - wrk->stats.sess_fail++; + wrk->stats->sess_fail++; /* We're going to pace in vca anyway... */ (void)Pool_TrySumstat(wrk); continue; @@ -374,12 +374,12 @@ Pool_Work_Thread(void *priv, struct worker *wrk) VTAILQ_REMOVE(&pp->back_queue, tp, list); } - if ((tp == NULL && wrk->stats.summs > 0) || - (wrk->stats.summs >= cache_param->wthread_stats_rate)) - pool_addstat(pp->a_stat, &wrk->stats); + if ((tp == NULL && wrk->stats->summs > 0) || + (wrk->stats->summs >= cache_param->wthread_stats_rate)) + pool_addstat(pp->a_stat, wrk->stats); if (tp != NULL) { - wrk->stats.summs++; + wrk->stats->summs++; } else if (pp->b_stat != NULL && pp->a_stat->summs) { /* Nothing to do, push pool stats into global pool */ tps.func = pool_stat_summ; @@ -401,7 +401,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk) VCL_Rel(&wrk->vcl); } while (wrk->task.func == NULL); tp = &wrk->task; - wrk->stats.summs++; + wrk->stats->summs++; } Lck_Unlock(&pp->mtx); diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index dadd25c..1befd7d 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -174,7 +174,7 @@ VRB_Free(struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (req->body_oc != NULL) { - ObjFreeObj(req->body_oc, &req->wrk->stats); + ObjFreeObj(req->body_oc, req->wrk->stats); FREE_OBJ(req->body_oc); req->body_oc = NULL; } @@ -222,14 +222,14 @@ VRB_Cache(struct req *req, ssize_t maxsize) } req->body_oc = HSH_NewObjCore(req->wrk); - XXXAN(STV_NewObject(req->body_oc, req->vsl, &req->wrk->stats, + XXXAN(STV_NewObject(req->body_oc, req->vsl, req->wrk->stats, TRANSIENT_STORAGE, 8)); VFP_Setup(vfc); vfc->http = req->http; vfc->vsl = req->vsl; vfc->oc = req->body_oc; - vfc->stats = &req->wrk->stats; + vfc->stats = req->wrk->stats; V1F_Setup_Fetch(vfc, req->htc); if (VFP_Open(vfc) < 0) { @@ -254,7 +254,7 @@ VRB_Cache(struct req *req, ssize_t maxsize) req->acct.req_bodybytes += l; if (yet >= l) yet -= l; - ObjExtend(req->body_oc, &req->wrk->stats, l); + ObjExtend(req->body_oc, req->wrk->stats, l); } } while (vfps == VFP_OK); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 1125e36..457b97e 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -103,13 +103,13 @@ cnt_deliver(struct worker *wrk, struct req *req) HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod); AZ(HTTP_Decode(req->resp, - ObjGetattr(req->objcore, &req->wrk->stats, OA_HEADERS, NULL))); + ObjGetattr(req->objcore, req->wrk->stats, OA_HEADERS, NULL))); http_ForceField(req->resp, HTTP_HDR_PROTO, "HTTP/1.1"); - if (req->wrk->stats.cache_hit) + if (req->wrk->stats->cache_hit) http_PrintfHeader(req->resp, "X-Varnish: %u %u", VXID(req->vsl->wid), - ObjGetXID(req->objcore, &wrk->stats)); + ObjGetXID(req->objcore, wrk->stats)); else http_PrintfHeader(req->resp, "X-Varnish: %u", VXID(req->vsl->wid)); @@ -127,7 +127,7 @@ cnt_deliver(struct worker *wrk, struct req *req) http_SetHeader(req->resp, "Via: 1.1 varnish-v4"); if (cache_param->http_gzip_support && - ObjCheckFlag(req->objcore, &req->wrk->stats, OF_GZIPED) && + ObjCheckFlag(req->objcore, req->wrk->stats, OF_GZIPED) && !RFC2616_Req_Gzip(req->http)) RFC2616_Weaken_Etag(req->resp); @@ -140,7 +140,7 @@ cnt_deliver(struct worker *wrk, struct req *req) if (wrk->handling != VCL_RET_DELIVER) { assert(req->objcore == req->objcore); - (void)HSH_DerefObjCore(&wrk->stats, &req->objcore); + (void)HSH_DerefObjCore(wrk->stats, &req->objcore); http_Teardown(req->resp); switch (wrk->handling) { @@ -188,11 +188,11 @@ cnt_deliver(struct worker *wrk, struct req *req) */ while (req->objcore->busyobj != NULL) (void)usleep(100000); - ObjSlim(req->objcore, &wrk->stats); + ObjSlim(req->objcore, wrk->stats); } assert(WRW_IsReleased(wrk)); - (void)HSH_DerefObjCore(&wrk->stats, &req->objcore); + (void)HSH_DerefObjCore(wrk->stats, &req->objcore); http_Teardown(req->resp); return (REQ_FSM_DONE); } @@ -220,7 +220,7 @@ cnt_synth(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - wrk->stats.s_synth++; + wrk->stats->s_synth++; now = W_TIM_real(wrk); VSLb_ts_req(req, "Process", now); @@ -295,13 +295,13 @@ cnt_fetch(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); - wrk->stats.s_fetch++; + wrk->stats->s_fetch++; (void)VRB_Ignore(req); if (req->objcore->flags & OC_F_FAILED) { req->err_code = 503; req->req_step = R_STP_SYNTH; - (void)HSH_DerefObjCore(&wrk->stats, &req->objcore); + (void)HSH_DerefObjCore(wrk->stats, &req->objcore); AZ(req->objcore); return (REQ_FSM_MORE); } @@ -399,16 +399,16 @@ cnt_lookup(struct worker *wrk, struct req *req) /* Found a hit-for-pass */ VSLb(req->vsl, SLT_Debug, "XXXX HIT-FOR-PASS"); VSLb(req->vsl, SLT_HitPass, "%u", - ObjGetXID(req->objcore, &wrk->stats)); + ObjGetXID(req->objcore, wrk->stats)); AZ(boc); - (void)HSH_DerefObjCore(&wrk->stats, &req->objcore); - wrk->stats.cache_hitpass++; + (void)HSH_DerefObjCore(wrk->stats, &req->objcore); + wrk->stats->cache_hitpass++; req->req_step = R_STP_PASS; return (REQ_FSM_MORE); } VSLb(req->vsl, SLT_Hit, "%u", - ObjGetXID(req->objcore, &wrk->stats)); + ObjGetXID(req->objcore, wrk->stats)); VCL_hit_method(req->vcl, wrk, req, NULL, req->http->ws); @@ -422,7 +422,7 @@ cnt_lookup(struct worker *wrk, struct req *req) } else { (void)VRB_Ignore(req);// XXX: handle err } - wrk->stats.cache_hit++; + wrk->stats->cache_hit++; req->req_step = R_STP_DELIVER; return (REQ_FSM_MORE); case VCL_RET_FETCH: @@ -431,7 +431,7 @@ cnt_lookup(struct worker *wrk, struct req *req) req->ims_oc = oc; req->req_step = R_STP_MISS; } else { - (void)HSH_DerefObjCore(&wrk->stats, &req->objcore); + (void)HSH_DerefObjCore(wrk->stats, &req->objcore); /* * We don't have a busy object, so treat this * like a pass @@ -449,7 +449,7 @@ cnt_lookup(struct worker *wrk, struct req *req) req->req_step = R_STP_SYNTH; break; case VCL_RET_PASS: - wrk->stats.cache_hit++; + wrk->stats->cache_hit++; req->req_step = R_STP_PASS; break; default: @@ -457,10 +457,10 @@ cnt_lookup(struct worker *wrk, struct req *req) } /* Drop our object, we won't need it */ - (void)HSH_DerefObjCore(&wrk->stats, &req->objcore); + (void)HSH_DerefObjCore(wrk->stats, &req->objcore); if (boc != NULL) { - (void)HSH_DerefObjCore(&wrk->stats, &boc); + (void)HSH_DerefObjCore(wrk->stats, &boc); VRY_Clear(req); } @@ -493,11 +493,11 @@ cnt_miss(struct worker *wrk, struct req *req) VCL_miss_method(req->vcl, wrk, req, NULL, req->http->ws); switch (wrk->handling) { case VCL_RET_FETCH: - wrk->stats.cache_miss++; + wrk->stats->cache_miss++; VBF_Fetch(wrk, req, req->objcore, req->ims_oc, VBF_NORMAL); req->req_step = R_STP_FETCH; if (req->ims_oc != NULL) - (void)HSH_DerefObjCore(&wrk->stats, &req->ims_oc); + (void)HSH_DerefObjCore(wrk->stats, &req->ims_oc); return (REQ_FSM_MORE); case VCL_RET_SYNTH: req->req_step = R_STP_SYNTH; @@ -513,8 +513,8 @@ cnt_miss(struct worker *wrk, struct req *req) } VRY_Clear(req); if (req->ims_oc != NULL) - (void)HSH_DerefObjCore(&wrk->stats, &req->ims_oc); - AZ(HSH_DerefObjCore(&wrk->stats, &req->objcore)); + (void)HSH_DerefObjCore(wrk->stats, &req->ims_oc); + AZ(HSH_DerefObjCore(wrk->stats, &req->objcore)); return (REQ_FSM_MORE); } @@ -553,7 +553,7 @@ cnt_pass(struct worker *wrk, struct req *req) req->req_step = R_STP_RESTART; break; case VCL_RET_FETCH: - wrk->stats.s_pass++; + wrk->stats->s_pass++; req->objcore = HSH_Private(wrk); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); VBF_Fetch(wrk, req, req->objcore, NULL, VBF_PASS); @@ -592,7 +592,7 @@ cnt_pipe(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC); - wrk->stats.s_pipe++; + wrk->stats->s_pipe++; bo = VBO_GetBusyObj(wrk, req); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); VSLb(bo->vsl, SLT_Begin, "bereq %u pipe", VXID(req->vsl->wid)); @@ -828,7 +828,7 @@ cnt_purge(struct worker *wrk, struct req *req) HSH_Purge(wrk, boc->objhead, 0, 0, 0); - AZ(HSH_DerefObjCore(&wrk->stats, &boc)); + AZ(HSH_DerefObjCore(wrk->stats, &boc)); VCL_purge_method(req->vcl, wrk, req, NULL, req->http->ws); switch (wrk->handling) { diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 51ba2c4..6f27269 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -225,7 +225,7 @@ RFC2616_Do_Cond(const struct req *req) ims = VTIM_parse(p); if (ims > req->t_req) /* [RFC2616 14.25] */ return (0); - AZ(ObjGetDouble(req->objcore, &req->wrk->stats, + AZ(ObjGetDouble(req->objcore, req->wrk->stats, OA_LASTMODIFIED, &lm)); if (lm > ims) return (0); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 6aa07b4..ca4574f 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -200,7 +200,7 @@ SES_pool_accept_task(struct worker *wrk, void *arg) VCA_FailSess(wrk); return; } - wrk->stats.s_sess++; + wrk->stats->s_sess++; sp->t_open = VTIM_real(); sp->t_idle = sp->t_open; diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index e8c5d9a..13c04dc 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -128,7 +128,7 @@ VRT_GetHdr(const struct vrt_ctx *ctx, const struct gethdr_s *hs) CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); return(HTTP_GetHdrPack(ctx->req->objcore, - &ctx->req->wrk->stats, hs->what)); + ctx->req->wrk->stats, hs->what)); } hp = vrt_selecthttp(ctx, hs->where); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 451f516..59b3c6f 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -148,7 +148,7 @@ VRT_r_obj_status(const struct vrt_ctx *ctx) CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); - return (HTTP_GetStatusPack(ctx->req->objcore, &ctx->req->wrk->stats)); + return (HTTP_GetStatusPack(ctx->req->objcore, ctx->req->wrk->stats)); } const char * @@ -159,7 +159,7 @@ VRT_r_obj_proto(const struct vrt_ctx *ctx) CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); return (HTTP_GetHdrPack(ctx->req->objcore, - &ctx->req->wrk->stats, ":proto")); + ctx->req->wrk->stats, ":proto")); } const char * @@ -170,7 +170,7 @@ VRT_r_obj_reason(const struct vrt_ctx *ctx) CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); return (HTTP_GetHdrPack(ctx->req->objcore, - &ctx->req->wrk->stats, ":reason")); + ctx->req->wrk->stats, ":reason")); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c index 4c7e4b5..0f7b7ec 100644 --- a/bin/varnishd/hash/hash_critbit.c +++ b/bin/varnishd/hash/hash_critbit.c @@ -363,7 +363,7 @@ hcb_cleaner(struct worker *wrk, void *priv) } VTAILQ_FOREACH_SAFE(oh, &dead_h, hoh_list, oh2) { VTAILQ_REMOVE(&dead_h, oh, hoh_list); - HSH_DeleteObjHead(&wrk->stats, oh); + HSH_DeleteObjHead(wrk->stats, oh); } Lck_Lock(&hcb_mtx); VSTAILQ_CONCAT(&dead_y, &cool_y); @@ -432,7 +432,7 @@ hcb_lookup(struct worker *wrk, const void *digest, struct objhead **noh) /* First try in read-only mode without holding a lock */ - wrk->stats.hcb_nolock++; + wrk->stats->hcb_nolock++; oh = hcb_insert(wrk, &hcb_root, digest, NULL); if (oh != NULL) { Lck_Lock(&oh->mtx); From phk at FreeBSD.org Tue Sep 16 07:59:34 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Sep 2014 09:59:34 +0200 Subject: [master] f724b36 Lend the worker a pointer to req/bo's vsl to allow us to simplify some function arguments. Message-ID: commit f724b3638243d0b4b41c7d464f6558950acb22fa Author: Poul-Henning Kamp Date: Tue Sep 16 07:59:01 2014 +0000 Lend the worker a pointer to req/bo's vsl to allow us to simplify some function arguments. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index b16edbb..d68e05f 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -349,6 +349,7 @@ struct worker { struct busyobj *nbo; void *nhashpriv; struct dstat stats[1]; + struct vsl_log *vsl; // borrowed from req/bo struct pool_task task; @@ -468,6 +469,7 @@ struct busyobj { unsigned refcount; int retries; struct req *req; + struct worker *wrk; uint8_t *vary; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index e8dc047..99654ae 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -858,6 +858,8 @@ vbf_fetch_thread(struct worker *wrk, void *priv) VSLb_ts_busyobj(bo, "Start", W_TIM_real(wrk)); bo->stats = wrk->stats; + bo->wrk = wrk; + wrk->vsl = bo->vsl; while (stp != F_STP_DONE) { CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); @@ -899,6 +901,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv) bo->stats = NULL; + wrk->vsl = NULL; VBO_DerefBusyObj(wrk, &bo); THR_SetBusyobj(NULL); } diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index fc41856..77524b3 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -363,6 +363,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); WS_Reset(wrk->aws, NULL); + AZ(wrk->vsl); tp = VTAILQ_FIRST(&pp->front_queue); if (tp != NULL) { diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 457b97e..f0c418d 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -880,6 +880,7 @@ CNT_Request(struct worker *wrk, struct req *req) AN(req->vsl->wid & VSL_CLIENTMARKER); req->wrk = wrk; + wrk->vsl = req->vsl; for (nxt = REQ_FSM_MORE; nxt == REQ_FSM_MORE; ) { /* @@ -913,6 +914,7 @@ CNT_Request(struct worker *wrk, struct req *req) WS_Assert(wrk->aws); CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); } + wrk->vsl = NULL; if (nxt == REQ_FSM_DONE) { AN(req->vsl->wid); if (req->res_mode & (RES_ESI|RES_ESI_CHILD)) From phk at FreeBSD.org Tue Sep 16 08:41:05 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Sep 2014 10:41:05 +0200 Subject: [master] 0ff51bd Trying to clean up various function arguments by moving towards struct worker for "usual stuff" like vsl, stats etc. Message-ID: commit 0ff51bd69a8b4ac2803b3071b648a962ab02b4b7 Author: Poul-Henning Kamp Date: Tue Sep 16 08:40:25 2014 +0000 Trying to clean up various function arguments by moving towards struct worker for "usual stuff" like vsl, stats etc. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index d68e05f..df62953 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -218,15 +218,14 @@ struct vfp_ctx { unsigned magic; #define VFP_CTX_MAGIC 0x61d9d3e5 struct busyobj *bo; + struct worker *wrk; struct objcore *oc; - struct dstat *stats; int failed; struct vfp_entry_s vfp; struct vfp_entry *vfp_nxt; - struct vsl_log *vsl; struct http *http; struct http *esi_req; }; @@ -513,7 +512,6 @@ struct busyobj { struct VCL_conf *vcl; struct vsl_log vsl[1]; - struct dstat *stats; struct vsb *synth_body; }; @@ -819,7 +817,7 @@ void EXP_Init(void); void EXP_Rearm(struct objcore *, double now, double ttl, double grace, double keep); void EXP_Touch(struct objcore *oc, double now); -int EXP_NukeOne(struct vsl_log *vsl, struct dstat *ds, struct lru *lru); +int EXP_NukeOne(struct worker *wrk, struct lru *lru); /* cache_fetch.c */ enum vbf_fetch_mode_e { @@ -983,8 +981,7 @@ enum objiter_status { void *ObjIterBegin(struct objcore *, struct worker *); enum objiter_status ObjIter(struct objcore *, void *, void **, ssize_t *); void ObjIterEnd(struct objcore *, void **); -int ObjGetSpace(struct objcore *, struct vsl_log *vsl, - struct dstat *, ssize_t *sz, uint8_t **ptr); +int ObjGetSpace(struct objcore *, struct worker *, ssize_t *sz, uint8_t **ptr); void ObjExtend(struct objcore *, struct dstat *, ssize_t l); void ObjTrimStore(struct objcore *, struct dstat *); unsigned ObjGetXID(struct objcore *, struct dstat *); @@ -1166,7 +1163,7 @@ void RFC2616_Weaken_Etag(struct http *hp); void RFC2616_Vary_AE(struct http *hp); /* stevedore.c */ -int STV_NewObject(struct objcore *, struct vsl_log *, struct dstat *, +int STV_NewObject(struct objcore *, struct worker *, const char *hint, unsigned len); struct storage *STV_alloc(struct stevedore *, size_t size); void STV_trim(struct storage *st, size_t size, int move_ok); diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index dd7834f..53f5abe 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -224,7 +224,7 @@ VBO_extend(struct busyobj *bo, ssize_t l) return; assert(l > 0); Lck_Lock(&bo->mtx); - ObjExtend(bo->vfc->oc, bo->stats, l); + ObjExtend(bo->vfc->oc, bo->wrk->stats, l); AZ(pthread_cond_broadcast(&bo->cond)); Lck_Unlock(&bo->mtx); } diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index ce76cec..8a73566 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -150,7 +150,7 @@ vfp_esi_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe) ALLOC_OBJ(vef, VEF_MAGIC); if (vef == NULL) return (VFP_ERROR); - vef->vgz = VGZ_NewGzip(vc->vsl, "G F E"); + vef->vgz = VGZ_NewGzip(vc->wrk->vsl, "G F E"); vef->vep = VEP_Init(vc, vc->esi_req, vfp_vep_callback, vef); vef->ibuf_sz = cache_param->gzip_buffer; vef->ibuf = calloc(1L, vef->ibuf_sz); diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c index 1860b8f..ab2a4f5 100644 --- a/bin/varnishd/cache/cache_esi_parse.c +++ b/bin/varnishd/cache/cache_esi_parse.c @@ -196,7 +196,7 @@ vep_error(const struct vep_state *vep, const char *p) VSC_C_main->esi_errors++; l = (intmax_t)(vep->ver_p - vep->hack_p); - VSLb(vep->vc->vsl, SLT_ESI_xmlerror, "ERR at %jd %s", l, p); + VSLb(vep->vc->wrk->vsl, SLT_ESI_xmlerror, "ERR at %jd %s", l, p); } @@ -211,7 +211,7 @@ vep_warn(const struct vep_state *vep, const char *p) VSC_C_main->esi_warnings++; l = (intmax_t)(vep->ver_p - vep->hack_p); - VSLb(vep->vc->vsl, SLT_ESI_xmlerror, "WARN at %jd %s", l, p); + VSLb(vep->vc->wrk->vsl, SLT_ESI_xmlerror, "WARN at %jd %s", l, p); } @@ -634,14 +634,14 @@ VEP_Parse(struct vep_state *vep, const char *p, size_t l) p++; vep->state = VEP_STARTTAG; } else if (p < e && *p == '\xeb') { - VSLb(vep->vc->vsl, SLT_ESI_xmlerror, + VSLb(vep->vc->wrk->vsl, SLT_ESI_xmlerror, "No ESI processing, " "first char not '<' but BOM." " (See feature esi_remove_bom)" ); vep->state = VEP_NOTXML; } else if (p < e) { - VSLb(vep->vc->vsl, SLT_ESI_xmlerror, + VSLb(vep->vc->wrk->vsl, SLT_ESI_xmlerror, "No ESI processing, " "first char not '<'." " (See feature esi_disable_xml_check)" diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index 0b9772e..6753669 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -286,20 +286,19 @@ EXP_Rearm(struct objcore *oc, double now, double ttl, double grace, double keep) */ int -EXP_NukeOne(struct vsl_log *vsl, struct dstat *ds, struct lru *lru) +EXP_NukeOne(struct worker *wrk, struct lru *lru) { struct objcore *oc, *oc2; struct objhead *oh; - AN(vsl); - AN(ds); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(lru, LRU_MAGIC); /* Find the first currently unused object on the LRU. */ Lck_Lock(&lru->mtx); VTAILQ_FOREACH_SAFE(oc, &lru->lru_head, lru_list, oc2) { CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - VSLb(vsl, SLT_ExpKill, "LRU_Cand p=%p f=0x%x r=%d", + VSLb(wrk->vsl, SLT_ExpKill, "LRU_Cand p=%p f=0x%x r=%d", oc, oc->flags, oc->refcnt); AZ(oc->exp_flags & OC_EF_OFFLRU); @@ -331,17 +330,17 @@ EXP_NukeOne(struct vsl_log *vsl, struct dstat *ds, struct lru *lru) Lck_Unlock(&lru->mtx); if (oc == NULL) { - VSLb(vsl, SLT_ExpKill, "LRU_Fail"); + VSLb(wrk->vsl, SLT_ExpKill, "LRU_Fail"); return (-1); } /* XXX: We could grab and return one storage segment to our caller */ - ObjSlim(oc, ds); + ObjSlim(oc, wrk->stats); exp_mail_it(oc); - VSLb(vsl, SLT_ExpKill, "LRU x=%u", ObjGetXID(oc, ds)); - (void)HSH_DerefObjCore(ds, &oc); + VSLb(wrk->vsl, SLT_ExpKill, "LRU x=%u", ObjGetXID(oc, wrk->stats)); + (void)HSH_DerefObjCore(wrk->stats, &oc); return (1); } diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 99654ae..e6bc239 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -66,8 +66,7 @@ vbf_allocobj(struct busyobj *bo, unsigned l) bo->storage_hint = NULL; - if (STV_NewObject(bo->fetch_objcore, bo->vsl, bo->stats, - storage_hint, l)) + if (STV_NewObject(bo->fetch_objcore, bo->wrk, storage_hint, l)) return (1); if (storage_hint != NULL && !strcmp(storage_hint, TRANSIENT_STORAGE)) @@ -82,7 +81,7 @@ vbf_allocobj(struct busyobj *bo, unsigned l) oc->exp.ttl = cache_param->shortlived; oc->exp.grace = 0.0; oc->exp.keep = 0.0; - return (STV_NewObject(bo->fetch_objcore, bo->vsl, bo->stats, + return (STV_NewObject(bo->fetch_objcore, bo->wrk, TRANSIENT_STORAGE, l)); } @@ -191,11 +190,12 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo) } if (bo->ims_oc != NULL) { - q = HTTP_GetHdrPack(bo->ims_oc, bo->stats, H_Last_Modified); + q = HTTP_GetHdrPack(bo->ims_oc, bo->wrk->stats, + H_Last_Modified); if (q != NULL) http_PrintfHeader(bo->bereq0, "If-Modified-Since: %s", q); - q = HTTP_GetHdrPack(bo->ims_oc, bo->stats, H_ETag); + q = HTTP_GetHdrPack(bo->ims_oc, bo->wrk->stats, H_ETag); if (q != NULL) http_PrintfHeader(bo->bereq0, "If-None-Match: %s", q); @@ -381,7 +381,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) AZ(bo->do_esi); if (bo->ims_oc != NULL && http_IsStatus(bo->beresp, 304)) { - if (ObjCheckFlag(bo->ims_oc, bo->stats, OF_CHGGZIP)) { + if (ObjCheckFlag(bo->ims_oc, bo->wrk->stats, OF_CHGGZIP)) { /* * If we changed the gzip status of the object * the stored Content_Encoding controls we @@ -390,7 +390,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) http_Unset(bo->beresp, H_Content_Encoding); RFC2616_Weaken_Etag(bo->beresp); } - HTTP_Merge(bo->ims_oc, bo->stats, bo->beresp); + HTTP_Merge(bo->ims_oc, bo->wrk->stats, bo->beresp); assert(http_IsStatus(bo->beresp, 200)); do_ims = 1; } else @@ -399,10 +399,9 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) VFP_Setup(bo->vfc); bo->vfc->bo = bo; bo->vfc->oc = bo->fetch_objcore; - bo->vfc->stats = bo->stats; + bo->vfc->wrk = bo->wrk; bo->vfc->http = bo->beresp; bo->vfc->esi_req = bo->bereq; - bo->vfc->vsl = bo->vsl; VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws); @@ -463,7 +462,7 @@ vbf_fetch_body_helper(struct busyobj *bo) * objects to be created. */ AN(vfc->oc->flags & OC_F_PASS); - VSLb(vfc->vsl, SLT_FetchError, + VSLb(vfc->wrk->vsl, SLT_FetchError, "Pass delivery abandoned"); vfps = VFP_END; bo->doclose = SC_RX_BODY; @@ -498,7 +497,7 @@ vbf_fetch_body_helper(struct busyobj *bo) VFP_Close(vfc); if (!bo->do_stream) - ObjTrimStore(vfc->oc, bo->stats); + ObjTrimStore(vfc->oc, bo->wrk->stats); } /*-------------------------------------------------------------------- @@ -627,7 +626,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) if (bo->vfc->failed && !bo->do_stream) { assert(bo->state < BOS_STREAM); if (bo->fetch_objcore != NULL) - ObjFreeObj(bo->fetch_objcore, bo->stats); + ObjFreeObj(bo->fetch_objcore, bo->wrk->stats); return (F_STP_ERROR); } @@ -673,7 +672,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) AZ(vbf_beresp2obj(bo)); - if (ObjGetattr(bo->ims_oc, bo->stats, OA_ESIDATA, NULL) != NULL) + if (ObjGetattr(bo->ims_oc, bo->wrk->stats, OA_ESIDATA, NULL) != NULL) AZ(ObjCopyAttr(bo->vfc, bo->ims_oc, OA_ESIDATA)); AZ(ObjCopyAttr(bo->vfc, bo->ims_oc, OA_FLAGS)); @@ -685,7 +684,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) } al = 0; - ol = ObjGetLen(bo->ims_oc, bo->stats); + ol = ObjGetLen(bo->ims_oc, bo->wrk->stats); oi = ObjIterBegin(bo->ims_oc, wrk); do { ois = ObjIter(bo->ims_oc, oi, &sp, &sl); @@ -710,7 +709,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) HSH_Unbusy(wrk->stats, bo->fetch_objcore); assert(al == ol); - assert(ObjGetLen(bo->fetch_objcore, bo->stats) == al); + assert(ObjGetLen(bo->fetch_objcore, bo->wrk->stats) == al); EXP_Rearm(bo->ims_oc, bo->ims_oc->exp.t_origin, 0, 0, 0); /* Recycle the backend connection before setting BOS_FINISHED to @@ -778,11 +777,10 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) VFP_Setup(bo->vfc); bo->vfc->bo = bo; + bo->vfc->wrk = bo->wrk; bo->vfc->oc = bo->fetch_objcore; - bo->vfc->stats = bo->stats; bo->vfc->http = bo->beresp; bo->vfc->esi_req = bo->bereq; - bo->vfc->vsl = bo->vsl; if (vbf_beresp2obj(bo)) return (F_STP_FAIL); @@ -857,7 +855,6 @@ vbf_fetch_thread(struct worker *wrk, void *priv) assert(isnan(bo->t_prev)); VSLb_ts_busyobj(bo, "Start", W_TIM_real(wrk)); - bo->stats = wrk->stats; bo->wrk = wrk; wrk->vsl = bo->vsl; @@ -892,14 +889,13 @@ vbf_fetch_thread(struct worker *wrk, void *priv) AZ(bo->fetch_objcore->flags & OC_F_FAILED); HSH_Complete(bo->fetch_objcore); VSLb(bo->vsl, SLT_Length, "%ju", - (uintmax_t)ObjGetLen(bo->fetch_objcore, bo->stats)); + (uintmax_t)ObjGetLen(bo->fetch_objcore, bo->wrk->stats)); } AZ(bo->fetch_objcore->busyobj); if (bo->ims_oc != NULL) (void)HSH_DerefObjCore(wrk->stats, &bo->ims_oc); - bo->stats = NULL; wrk->vsl = NULL; VBO_DerefBusyObj(wrk, &bo); diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index 9cf0aa4..3e3aad6 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -60,7 +60,7 @@ VFP_Error(struct vfp_ctx *vc, const char *fmt, ...) CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); if (!vc->failed) { va_start(ap, fmt); - VSLbv(vc->vsl, SLT_FetchError, fmt, ap); + VSLbv(vc->wrk->vsl, SLT_FetchError, fmt, ap); va_end(ap); vc->failed = 1; } @@ -77,19 +77,19 @@ VFP_GetStorage(struct vfp_ctx *vc, ssize_t *sz, uint8_t **ptr) { ssize_t l; + CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); AN(sz); assert(*sz >= 0); AN(ptr); - AN(vc->stats); l = fetchfrag; if (l == 0) l = *sz; if (l == 0) l = cache_param->fetch_chunksize; *sz = l; - if (!ObjGetSpace(vc->oc, vc->vsl, vc->stats, sz, ptr)) { + if (!ObjGetSpace(vc->oc, vc->wrk, sz, ptr)) { *sz = 0; *ptr = NULL; return (VFP_Error(vc, "Could not get storage")); @@ -121,7 +121,7 @@ VFP_Close(struct vfp_ctx *vc) VTAILQ_FOREACH(vfe, &vc->vfp, list) { if(vfe->vfp->fini != NULL) vfe->vfp->fini(vc, vfe); - VSLb(vc->vsl, SLT_VfpAcct, "%s %ju %ju", + VSLb(vc->wrk->vsl, SLT_VfpAcct, "%s %ju %ju", vfe->vfp->name, vfe->calls, vfe->bytes_out); } } @@ -133,7 +133,8 @@ VFP_Open(struct vfp_ctx *vc) CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vc->http, HTTP_MAGIC); - AN(vc->vsl); + CHECK_OBJ_NOTNULL(vc->wrk, WORKER_MAGIC); + AN(vc->wrk->vsl); VTAILQ_FOREACH_REVERSE(vfe, &vc->vfp, vfp_entry_s, list) { if (vfe->vfp->init == NULL) diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 1e5b806..a4b3034 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -469,11 +469,11 @@ vfp_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe) if (vfe->vfp->priv2 == VFP_GZIP) { if (http_GetHdr(vc->http, H_Content_Encoding, NULL)) return (VFP_NULL); - vg = VGZ_NewGzip(vc->vsl, vfe->vfp->priv1); + vg = VGZ_NewGzip(vc->wrk->vsl, vfe->vfp->priv1); } else { if (!http_HdrIs(vc->http, H_Content_Encoding, "gzip")) return (VFP_NULL); - vg = VGZ_NewUngzip(vc->vsl, vfe->vfp->priv1); + vg = VGZ_NewUngzip(vc->wrk->vsl, vfe->vfp->priv1); } if (vg == NULL) return (VFP_ERROR); diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index f846541..2c5335c 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -213,8 +213,7 @@ ObjIterEnd(struct objcore *oc, void **oix) */ static struct storage * -objallocwithnuke(struct stevedore *stv, struct vsl_log *vsl, struct dstat *ds, - size_t size) +objallocwithnuke(struct stevedore *stv, struct worker *wrk, size_t size) { struct storage *st = NULL; unsigned fail; @@ -235,7 +234,7 @@ objallocwithnuke(struct stevedore *stv, struct vsl_log *vsl, struct dstat *ds, /* no luck; try to free some space and keep trying */ if (fail < cache_param->nuke_limit && - EXP_NukeOne(vsl, ds, stv->lru) == -1) + EXP_NukeOne(wrk, stv->lru) == -1) break; } CHECK_OBJ_ORNULL(st, STORAGE_MAGIC); @@ -252,22 +251,21 @@ objallocwithnuke(struct stevedore *stv, struct vsl_log *vsl, struct dstat *ds, */ int -ObjGetSpace(struct objcore *oc, struct vsl_log *vsl, struct dstat *ds, - ssize_t *sz, uint8_t **ptr) +ObjGetSpace(struct objcore *oc, struct worker *wrk, ssize_t *sz, uint8_t **ptr) { struct object *o; struct storage *st; const struct storeobj_methods *om = obj_getmethods(oc); if (om->objgetspace != NULL) - return (om->objgetspace(oc, vsl, ds, sz, ptr)); + return (om->objgetspace(oc, wrk, sz, ptr)); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - AN(ds); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AN(sz); AN(ptr); assert(*sz > 0); - o = obj_getobj(oc, ds); + o = obj_getobj(oc, wrk->stats); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); st = VTAILQ_LAST(&o->list, storagehead); @@ -278,7 +276,7 @@ ObjGetSpace(struct objcore *oc, struct vsl_log *vsl, struct dstat *ds, return (1); } - st = objallocwithnuke(oc->stobj->stevedore, vsl, ds, *sz); + st = objallocwithnuke(oc->stobj->stevedore, wrk, *sz); if (st == NULL) return (0); @@ -529,13 +527,13 @@ ObjSetattr(const struct vfp_ctx *vc, enum obj_attr attr, ssize_t len, CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vc->bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(vc->oc, OBJCORE_MAGIC); - o = obj_getobj(vc->oc, vc->stats); + o = obj_getobj(vc->oc, vc->wrk->stats); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); st = o->objstore; switch (attr) { case OA_ESIDATA: - o->esidata = objallocwithnuke(vc->oc->stobj->stevedore, vc->vsl, - vc->stats, len); + o->esidata = objallocwithnuke(vc->oc->stobj->stevedore, + vc->wrk, len); if (o->esidata == NULL) return (NULL); o->esidata->len = len; @@ -592,7 +590,7 @@ ObjCopyAttr(const struct vfp_ctx *vc, struct objcore *ocs, enum obj_attr attr) CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - vps = ObjGetattr(ocs, vc->stats, attr, &l); + vps = ObjGetattr(ocs, vc->wrk->stats, attr, &l); // XXX: later we want to have zero-length OA's too if (vps == NULL || l <= 0) return (-1); diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index 1befd7d..049e850 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -105,7 +105,7 @@ VRB_Iterate(struct req *req, req_body_iter_f *func, void *priv) vfc = req->htc->vfc; VFP_Setup(vfc); vfc->http = req->http; - vfc->vsl = req->vsl; + vfc->wrk = req->wrk; V1F_Setup_Fetch(vfc, req->htc); if (VFP_Open(vfc) < 0) { VSLb(req->vsl, SLT_FetchError, "Could not open Fetch Pipeline"); @@ -222,14 +222,13 @@ VRB_Cache(struct req *req, ssize_t maxsize) } req->body_oc = HSH_NewObjCore(req->wrk); - XXXAN(STV_NewObject(req->body_oc, req->vsl, req->wrk->stats, - TRANSIENT_STORAGE, 8)); + AN(req->body_oc); + XXXAN(STV_NewObject(req->body_oc, req->wrk, TRANSIENT_STORAGE, 8)); VFP_Setup(vfc); vfc->http = req->http; - vfc->vsl = req->vsl; + vfc->wrk = req->wrk; vfc->oc = req->body_oc; - vfc->stats = req->wrk->stats; V1F_Setup_Fetch(vfc, req->htc); if (VFP_Open(vfc) < 0) { @@ -237,6 +236,7 @@ VRB_Cache(struct req *req, ssize_t maxsize) return (-1); } + AN(req->htc); yet = req->htc->content_length; if (yet < 0) yet = 0; diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 03f599a..2e02e0a 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -245,7 +245,7 @@ stv_default_allocobj(struct stevedore *stv, struct objcore *oc, unsigned ltot) */ int -STV_NewObject(struct objcore *oc, struct vsl_log *vsl, struct dstat *stats, +STV_NewObject(struct objcore *oc, struct worker *wrk, const char *hint, unsigned wsl) { struct stevedore *stv, *stv0; @@ -253,19 +253,18 @@ STV_NewObject(struct objcore *oc, struct vsl_log *vsl, struct dstat *stats, int i, j; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - AN(vsl); - AN(stats); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); assert(wsl > 0); wsl = PRNDUP(wsl); ltot = sizeof(struct object) + wsl; - stv = stv0 = stv_pick_stevedore(vsl, &hint); + stv = stv0 = stv_pick_stevedore(wrk->vsl, &hint); AN(stv->allocobj); j = stv->allocobj(stv, oc, ltot); if (j == 0 && hint == NULL) { do { - stv = stv_pick_stevedore(vsl, &hint); + stv = stv_pick_stevedore(wrk->vsl, &hint); AN(stv->allocobj); j = stv->allocobj(stv, oc, ltot); } while (j == 0 && stv != stv0); @@ -273,7 +272,7 @@ STV_NewObject(struct objcore *oc, struct vsl_log *vsl, struct dstat *stats, if (j == 0) { /* no luck; try to free some space and keep trying */ for (i = 0; j == 0 && i < cache_param->nuke_limit; i++) { - if (EXP_NukeOne(vsl, stats, stv->lru) == -1) + if (EXP_NukeOne(wrk, stv->lru) == -1) break; j = stv->allocobj(stv, oc, ltot); } @@ -282,8 +281,8 @@ STV_NewObject(struct objcore *oc, struct vsl_log *vsl, struct dstat *stats, if (j == 0) return (0); - stats->n_object++; - VSLb(vsl, SLT_Storage, "%s %s", + wrk->stats->n_object++; + VSLb(wrk->vsl, SLT_Storage, "%s %s", oc->stobj->stevedore->name, oc->stobj->stevedore->ident); return (1); } diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index 9ddd770..cc3420a 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -99,8 +99,8 @@ typedef void *objiterbegin_f(struct objcore *oc, struct worker *wrk); typedef enum objiter_status objiter_f(struct objcore *oc, void *oix, void **p, ssize_t *l); typedef void objiterend_f(struct objcore *oc, void **oix); -typedef int objgetspace_f(struct objcore *oc, struct vsl_log *vsl, - struct dstat *ds, ssize_t *sz, uint8_t **ptr); +typedef int objgetspace_f(struct objcore *oc, struct worker *, + ssize_t *sz, uint8_t **ptr); typedef void objextend_f(struct objcore *oc, struct dstat *ds, ssize_t l); typedef void objtrimstore_f(struct objcore *oc, struct dstat *ds); typedef void objslim_f(struct objcore *oc, struct dstat *ds); From martin at varnish-software.com Tue Sep 16 08:49:18 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 16 Sep 2014 10:49:18 +0200 Subject: [master] 31b4844 Don't mix const and non-const pointers. This should make jenkins happy again. Message-ID: commit 31b4844e3c34cb823bb3c70f3c88faa86d28c7f6 Author: Martin Blix Grydeland Date: Tue Sep 16 10:48:52 2014 +0200 Don't mix const and non-const pointers. This should make jenkins happy again. diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c index 45e8205..15835cb 100644 --- a/bin/varnishtop/varnishtop.c +++ b/bin/varnishtop/varnishtop.c @@ -66,7 +66,8 @@ static int end_of_file = 0; struct top { uint8_t tag; - char *rec_data; + const char *rec_data; + char *rec_buf; int clen; unsigned hash; VRB_ENTRY(top) e_order; @@ -147,7 +148,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], t.hash = u; t.tag = tag; t.clen = len; - t.rec_data = (char *)VSL_CDATA(tr->c->rec.ptr); + t.rec_data = VSL_CDATA(tr->c->rec.ptr); AZ(pthread_mutex_lock(&mtx)); tp = VRB_FIND(t_key, &h_key, &t); @@ -164,7 +165,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], tp->count = 1.0; tp->clen = len; tp->tag = tag; - tp->rec_data = strdup(t.rec_data); + tp->rec_buf = strdup(t.rec_data); + tp->rec_data = tp->rec_buf; AN(tp->rec_data); VRB_INSERT(t_key, &h_key, tp); VRB_INSERT(t_order, &h_order, tp); @@ -221,7 +223,7 @@ update(int p) if (tp->count * 10 < t || l > LINES * 10) { VRB_REMOVE(t_key, &h_key, tp); VRB_REMOVE(t_order, &h_order, tp); - free(tp->rec_data); + free(tp->rec_buf); free(tp); ntop--; } From phk at FreeBSD.org Tue Sep 16 10:28:57 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Sep 2014 12:28:57 +0200 Subject: [master] faf9ba4 More refactoring: HSH_* functions take struct worker as first argument Message-ID: commit faf9ba4883647829ca43727000349ac756a91754 Author: Poul-Henning Kamp Date: Tue Sep 16 10:28:29 2014 +0000 More refactoring: HSH_* functions take struct worker as first argument diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index a7fd546..9311b75 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -1100,7 +1100,7 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt, EXP_Rearm(oc, oc->exp.t_origin, 0, 0, 0); // XXX fake now VSC_C_main->bans_lurker_obj_killed++; } - (void)HSH_DerefObjCore(wrk->stats, &oc); + (void)HSH_DerefObjCore(wrk, &oc); } } diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 53f5abe..f381e08 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -197,7 +197,7 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo) if (bo->fetch_objcore != NULL) { AN(wrk); - (void)HSH_DerefObjCore(wrk->stats, &bo->fetch_objcore); + (void)HSH_DerefObjCore(wrk, &bo->fetch_objcore); } VCL_Rel(&bo->vcl); diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index 6753669..3d8caa5 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -340,7 +340,7 @@ EXP_NukeOne(struct worker *wrk, struct lru *lru) exp_mail_it(oc); VSLb(wrk->vsl, SLT_ExpKill, "LRU x=%u", ObjGetXID(oc, wrk->stats)); - (void)HSH_DerefObjCore(wrk->stats, &oc); + (void)HSH_DerefObjCore(wrk, &oc); return (1); } @@ -385,7 +385,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now) binheap_delete(ep->heap, oc->timer_idx); } assert(oc->timer_idx == BINHEAP_NOIDX); - (void)HSH_DerefObjCore(ep->wrk->stats, &oc); + (void)HSH_DerefObjCore(ep->wrk, &oc); return; } @@ -465,7 +465,7 @@ exp_expire(struct exp_priv *ep, double now) VSLb(&ep->vsl, SLT_ExpKill, "EXP_Expired x=%u t=%.0f", ObjGetXID(oc, ep->wrk->stats), EXP_Ttl(NULL, &oc->exp) - now); - (void)HSH_DerefObjCore(ep->wrk->stats, &oc); + (void)HSH_DerefObjCore(ep->wrk, &oc); return (0); } diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index e6bc239..4080a8e 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -610,7 +610,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) assert (bo->state == BOS_REQ_DONE); if (bo->do_stream) { - HSH_Unbusy(wrk->stats, bo->fetch_objcore); + HSH_Unbusy(wrk, bo->fetch_objcore); VBO_setstate(bo, BOS_STREAM); } @@ -637,7 +637,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) assert(bo->state == BOS_STREAM); else { assert(bo->state == BOS_REQ_DONE); - HSH_Unbusy(wrk->stats, bo->fetch_objcore); + HSH_Unbusy(wrk, bo->fetch_objcore); } /* Recycle the backend connection before setting BOS_FINISHED to @@ -679,7 +679,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) AZ(ObjCopyAttr(bo->vfc, bo->ims_oc, OA_GZIPBITS)); if (bo->do_stream) { - HSH_Unbusy(wrk->stats, bo->fetch_objcore); + HSH_Unbusy(wrk, bo->fetch_objcore); VBO_setstate(bo, BOS_STREAM); } @@ -706,7 +706,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) return (F_STP_FAIL); if (!bo->do_stream) - HSH_Unbusy(wrk->stats, bo->fetch_objcore); + HSH_Unbusy(wrk, bo->fetch_objcore); assert(al == ol); assert(ObjGetLen(bo->fetch_objcore, bo->wrk->stats) == al); @@ -799,7 +799,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) VSB_delete(bo->synth_body); bo->synth_body = NULL; - HSH_Unbusy(wrk->stats, bo->fetch_objcore); + HSH_Unbusy(wrk, bo->fetch_objcore); VBO_setstate(bo, BOS_FINISHED); return (F_STP_DONE); } @@ -894,7 +894,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv) AZ(bo->fetch_objcore->busyobj); if (bo->ims_oc != NULL) - (void)HSH_DerefObjCore(wrk->stats, &bo->ims_oc); + (void)HSH_DerefObjCore(wrk, &bo->ims_oc); wrk->vsl = NULL; diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index b102928..f8c4149 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -177,13 +177,13 @@ HSH_Cleanup(struct worker *wrk) } void -HSH_DeleteObjHead(struct dstat *ds, struct objhead *oh) +HSH_DeleteObjHead(struct worker *wrk, struct objhead *oh) { AZ(oh->refcnt); assert(VTAILQ_EMPTY(&oh->objcs)); Lck_Delete(&oh->mtx); - ds->n_objecthead--; + wrk->stats->n_objecthead--; FREE_OBJ(oh); } @@ -437,7 +437,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, if (oh->hits < LONG_MAX) oh->hits++; Lck_Unlock(&oh->mtx); - assert(HSH_DerefObjHead(wrk->stats, &oh)); + assert(HSH_DerefObjHead(wrk, &oh)); *ocp = oc; return (HSH_HIT); } @@ -465,7 +465,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, oh->hits++; Lck_Unlock(&oh->mtx); if (retval == HSH_EXP) - assert(HSH_DerefObjHead(wrk->stats, &oh)); + assert(HSH_DerefObjHead(wrk, &oh)); *ocp = exp_oc; return (retval); } @@ -513,12 +513,13 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, */ static void -hsh_rush(struct dstat *ds, struct objhead *oh) +hsh_rush(struct worker *wrk, struct objhead *oh) { unsigned u; struct req *req; struct waitinglist *wl; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); Lck_AssertHeld(&oh->mtx); wl = oh->waitinglist; @@ -528,7 +529,7 @@ hsh_rush(struct dstat *ds, struct objhead *oh) if (req == NULL) break; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - ds->busy_wakeup++; + wrk->stats->busy_wakeup++; AZ(req->wrk); VTAILQ_REMOVE(&wl->list, req, w_list); DSL(DBG_WAITINGLIST, req->vsl->wid, "off waiting list"); @@ -543,7 +544,7 @@ hsh_rush(struct dstat *ds, struct objhead *oh) if (VTAILQ_EMPTY(&wl->list)) { oh->waitinglist = NULL; FREE_OBJ(wl); - ds->n_waitinglist--; + wrk->stats->n_waitinglist--; } } @@ -602,7 +603,7 @@ double keep) oc = ocp[n]; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); EXP_Rearm(oc, now, ttl, grace, keep); - (void)HSH_DerefObjCore(wrk->stats, &oc); + (void)HSH_DerefObjCore(wrk, &oc); } } while (more); WS_Release(wrk->aws, 0); @@ -658,10 +659,11 @@ HSH_Complete(struct objcore *oc) */ void -HSH_Unbusy(struct dstat *ds, struct objcore *oc) +HSH_Unbusy(struct worker *wrk, struct objcore *oc) { struct objhead *oh; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); oh = oc->objhead; CHECK_OBJ(oh, OBJHEAD_MAGIC); @@ -685,7 +687,7 @@ HSH_Unbusy(struct dstat *ds, struct objcore *oc) VTAILQ_INSERT_HEAD(&oh->objcs, oc, list); oc->flags &= ~OC_F_BUSY; if (oh->waitinglist != NULL) - hsh_rush(ds, oh); + hsh_rush(wrk, oh); Lck_Unlock(&oh->mtx); } @@ -737,17 +739,17 @@ HSH_RefBusy(const struct objcore *oc) */ int -HSH_DerefObjCore(struct dstat *ds, struct objcore **ocp) +HSH_DerefObjCore(struct worker *wrk, struct objcore **ocp) { struct objcore *oc; struct objhead *oh; unsigned r; AN(ocp); - AN(ds); oc = *ocp; *ocp = NULL; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); assert(oc->refcnt > 0); @@ -760,7 +762,7 @@ HSH_DerefObjCore(struct dstat *ds, struct objcore **ocp) if (!r) VTAILQ_REMOVE(&oh->objcs, oc, list); if (oh->waitinglist != NULL) - hsh_rush(ds, oh); + hsh_rush(wrk, oh); Lck_Unlock(&oh->mtx); if (r != 0) return (r); @@ -769,23 +771,23 @@ HSH_DerefObjCore(struct dstat *ds, struct objcore **ocp) AZ(oc->ban); if (oc->stobj->stevedore != NULL) - ObjFreeObj(oc, ds); + ObjFreeObj(oc, wrk->stats); FREE_OBJ(oc); - ds->n_objectcore--; + wrk->stats->n_objectcore--; /* Drop our ref on the objhead */ assert(oh->refcnt > 0); - (void)HSH_DerefObjHead(ds, &oh); + (void)HSH_DerefObjHead(wrk, &oh); return (0); } int -HSH_DerefObjHead(struct dstat *ds, struct objhead **poh) +HSH_DerefObjHead(struct worker *wrk, struct objhead **poh) { struct objhead *oh; int r; - AN(ds); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AN(poh); oh = *poh; *poh = NULL; @@ -802,7 +804,7 @@ HSH_DerefObjHead(struct dstat *ds, struct objhead **poh) assert(oh->refcnt > 0); r = hash->deref(oh); if (!r) - HSH_DeleteObjHead(ds, oh); + HSH_DeleteObjHead(wrk, oh); return (r); } diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 970ea24..0f68cd1 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -378,7 +378,7 @@ HTTP1_Session(struct worker *wrk, struct req *req) */ if (req->req_step == R_STP_LOOKUP && VTCP_check_hup(sp->fd)) { AN(req->hash_objhead); - (void)HSH_DerefObjHead(wrk->stats, &req->hash_objhead); + (void)HSH_DerefObjHead(wrk, &req->hash_objhead); AZ(req->hash_objhead); SES_Close(sp, SC_REM_CLOSE); sdr = http1_cleanup(sp, wrk, req); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index f0c418d..3ea3452 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -140,7 +140,7 @@ cnt_deliver(struct worker *wrk, struct req *req) if (wrk->handling != VCL_RET_DELIVER) { assert(req->objcore == req->objcore); - (void)HSH_DerefObjCore(wrk->stats, &req->objcore); + (void)HSH_DerefObjCore(wrk, &req->objcore); http_Teardown(req->resp); switch (wrk->handling) { @@ -192,7 +192,7 @@ cnt_deliver(struct worker *wrk, struct req *req) } assert(WRW_IsReleased(wrk)); - (void)HSH_DerefObjCore(wrk->stats, &req->objcore); + (void)HSH_DerefObjCore(wrk, &req->objcore); http_Teardown(req->resp); return (REQ_FSM_DONE); } @@ -301,7 +301,7 @@ cnt_fetch(struct worker *wrk, struct req *req) if (req->objcore->flags & OC_F_FAILED) { req->err_code = 503; req->req_step = R_STP_SYNTH; - (void)HSH_DerefObjCore(wrk->stats, &req->objcore); + (void)HSH_DerefObjCore(wrk, &req->objcore); AZ(req->objcore); return (REQ_FSM_MORE); } @@ -401,7 +401,7 @@ cnt_lookup(struct worker *wrk, struct req *req) VSLb(req->vsl, SLT_HitPass, "%u", ObjGetXID(req->objcore, wrk->stats)); AZ(boc); - (void)HSH_DerefObjCore(wrk->stats, &req->objcore); + (void)HSH_DerefObjCore(wrk, &req->objcore); wrk->stats->cache_hitpass++; req->req_step = R_STP_PASS; return (REQ_FSM_MORE); @@ -431,7 +431,7 @@ cnt_lookup(struct worker *wrk, struct req *req) req->ims_oc = oc; req->req_step = R_STP_MISS; } else { - (void)HSH_DerefObjCore(wrk->stats, &req->objcore); + (void)HSH_DerefObjCore(wrk, &req->objcore); /* * We don't have a busy object, so treat this * like a pass @@ -457,10 +457,10 @@ cnt_lookup(struct worker *wrk, struct req *req) } /* Drop our object, we won't need it */ - (void)HSH_DerefObjCore(wrk->stats, &req->objcore); + (void)HSH_DerefObjCore(wrk, &req->objcore); if (boc != NULL) { - (void)HSH_DerefObjCore(wrk->stats, &boc); + (void)HSH_DerefObjCore(wrk, &boc); VRY_Clear(req); } @@ -497,7 +497,7 @@ cnt_miss(struct worker *wrk, struct req *req) VBF_Fetch(wrk, req, req->objcore, req->ims_oc, VBF_NORMAL); req->req_step = R_STP_FETCH; if (req->ims_oc != NULL) - (void)HSH_DerefObjCore(wrk->stats, &req->ims_oc); + (void)HSH_DerefObjCore(wrk, &req->ims_oc); return (REQ_FSM_MORE); case VCL_RET_SYNTH: req->req_step = R_STP_SYNTH; @@ -513,8 +513,8 @@ cnt_miss(struct worker *wrk, struct req *req) } VRY_Clear(req); if (req->ims_oc != NULL) - (void)HSH_DerefObjCore(wrk->stats, &req->ims_oc); - AZ(HSH_DerefObjCore(wrk->stats, &req->objcore)); + (void)HSH_DerefObjCore(wrk, &req->ims_oc); + AZ(HSH_DerefObjCore(wrk, &req->objcore)); return (REQ_FSM_MORE); } @@ -828,7 +828,7 @@ cnt_purge(struct worker *wrk, struct req *req) HSH_Purge(wrk, boc->objhead, 0, 0, 0); - AZ(HSH_DerefObjCore(wrk->stats, &boc)); + AZ(HSH_DerefObjCore(wrk, &boc)); VCL_purge_method(req->vcl, wrk, req, NULL, req->http->ws); switch (wrk->handling) { diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c index 0f7b7ec..78d5576 100644 --- a/bin/varnishd/hash/hash_critbit.c +++ b/bin/varnishd/hash/hash_critbit.c @@ -363,7 +363,7 @@ hcb_cleaner(struct worker *wrk, void *priv) } VTAILQ_FOREACH_SAFE(oh, &dead_h, hoh_list, oh2) { VTAILQ_REMOVE(&dead_h, oh, hoh_list); - HSH_DeleteObjHead(wrk->stats, oh); + HSH_DeleteObjHead(wrk, oh); } Lck_Lock(&hcb_mtx); VSTAILQ_CONCAT(&dead_y, &cool_y); diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index 0b1fca0..db722da 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -111,11 +111,11 @@ struct objhead { }; void HSH_Fail(struct objcore *); -void HSH_Unbusy(struct dstat *, struct objcore *); +void HSH_Unbusy(struct worker *, struct objcore *); void HSH_Complete(struct objcore *oc); -void HSH_DeleteObjHead(struct dstat *, struct objhead *oh); -int HSH_DerefObjHead(struct dstat *, struct objhead **poh); -int HSH_DerefObjCore(struct dstat *, struct objcore **ocp); +void HSH_DeleteObjHead(struct worker *, struct objhead *oh); +int HSH_DerefObjHead(struct worker *, struct objhead **poh); +int HSH_DerefObjCore(struct worker *, struct objcore **ocp); #endif /* VARNISH_CACHE_CHILD */ extern const struct hash_slinger hsl_slinger; From phk at FreeBSD.org Tue Sep 16 10:52:37 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Sep 2014 12:52:37 +0200 Subject: [master] 9da7a7b Move Obj* API towards taking a struct worker as first argument. Message-ID: commit 9da7a7bcc9e8a37611215886faea1a26af769f68 Author: Poul-Henning Kamp Date: Tue Sep 16 10:52:13 2014 +0000 Move Obj* API towards taking a struct worker as first argument. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index df62953..57fb42f 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -729,7 +729,7 @@ struct busyobj *VBO_GetBusyObj(struct worker *, const struct req *); void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj); void VBO_Free(struct busyobj **vbo); void VBO_extend(struct busyobj *, ssize_t); -ssize_t VBO_waitlen(struct busyobj *, struct dstat *, ssize_t l); +ssize_t VBO_waitlen(struct worker *, struct busyobj *, ssize_t l); void VBO_setstate(struct busyobj *bo, enum busyobj_state_e next); void VBO_waitstate(struct busyobj *bo, enum busyobj_state_e want); @@ -978,14 +978,14 @@ enum objiter_status { OIS_STREAM, OIS_ERROR, }; -void *ObjIterBegin(struct objcore *, struct worker *); +void *ObjIterBegin(struct worker *, struct objcore *); enum objiter_status ObjIter(struct objcore *, void *, void **, ssize_t *); void ObjIterEnd(struct objcore *, void **); -int ObjGetSpace(struct objcore *, struct worker *, ssize_t *sz, uint8_t **ptr); +int ObjGetSpace(struct worker *, struct objcore *, ssize_t *sz, uint8_t **ptr); void ObjExtend(struct objcore *, struct dstat *, ssize_t l); -void ObjTrimStore(struct objcore *, struct dstat *); -unsigned ObjGetXID(struct objcore *, struct dstat *); -uint64_t ObjGetLen(struct objcore *oc, struct dstat *ds); +void ObjTrimStore(struct worker *, struct objcore *); +unsigned ObjGetXID(struct worker *, struct objcore *); +uint64_t ObjGetLen(struct worker *, struct objcore *oc); void ObjUpdateMeta(struct objcore *, struct dstat *); void ObjFreeObj(struct objcore *, struct dstat *); void ObjSlim(struct objcore *oc, struct dstat *ds); diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index 9311b75..26c1f59 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -962,8 +962,7 @@ BAN_CheckObject(struct worker *wrk, struct objcore *oc, struct req *req) return (0); } else { oc->ban = NULL; - VSLb(vsl, SLT_ExpBan, "%u banned lookup", - ObjGetXID(oc, wrk->stats)); + VSLb(vsl, SLT_ExpBan, "%u banned lookup", ObjGetXID(wrk, oc)); VSC_C_main->bans_obj_killed++; EXP_Rearm(oc, oc->exp.t_origin, 0, 0, 0); // XXX fake now return (1); @@ -1096,7 +1095,7 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt, } if (i) { VSLb(vsl, SLT_ExpBan, "%u banned by lurker", - ObjGetXID(oc, wrk->stats)); + ObjGetXID(wrk, oc)); EXP_Rearm(oc, oc->exp.t_origin, 0, 0, 0); // XXX fake now VSC_C_main->bans_lurker_obj_killed++; } diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index f381e08..9fa5cf9 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -230,20 +230,20 @@ VBO_extend(struct busyobj *bo, ssize_t l) } ssize_t -VBO_waitlen(struct busyobj *bo, struct dstat *ds, ssize_t l) +VBO_waitlen(struct worker *wrk, struct busyobj *bo, ssize_t l) { ssize_t rv; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - AN(ds); Lck_Lock(&bo->mtx); - rv = ObjGetLen(bo->fetch_objcore, ds); + rv = ObjGetLen(wrk, bo->fetch_objcore); while (1) { assert(l <= rv || bo->state == BOS_FAILED); if (rv > l || bo->state >= BOS_FINISHED) break; (void)Lck_CondWait(&bo->cond, &bo->mtx, 0); - rv = ObjGetLen(bo->fetch_objcore, ds); + rv = ObjGetLen(wrk, bo->fetch_objcore); } Lck_Unlock(&bo->mtx); return (rv); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 9f2059e..09e8a27 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -309,7 +309,7 @@ ESI_Deliver(struct req *req) AZ(dl); } - oi = ObjIterBegin(req->objcore, req->wrk); + oi = ObjIterBegin(req->wrk, req->objcore); ois = ObjIter(req->objcore, oi, &sp, &sl); assert(ois != OIS_ERROR); pp = sp; @@ -485,7 +485,7 @@ ESI_DeliverChild(struct req *req) CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); if (!ObjCheckFlag(req->objcore, req->wrk->stats, OF_GZIPED)) { - oi = ObjIterBegin(req->objcore, req->wrk); + oi = ObjIterBegin(req->wrk, req->objcore); do { ois = ObjIter(req->objcore, oi, &sp, &sl); if (sl > 0) @@ -506,7 +506,7 @@ ESI_DeliverChild(struct req *req) start = vbe64dec(p); last = vbe64dec(p + 8); stop = vbe64dec(p + 16); - olen = ObjGetLen(req->objcore, req->wrk->stats); + olen = ObjGetLen(req->wrk, req->objcore); assert(start > 0 && start < olen * 8); assert(last > 0 && last < olen * 8); assert(stop > 0 && stop < olen * 8); @@ -525,7 +525,7 @@ ESI_DeliverChild(struct req *req) dbits = (void*)WS_Alloc(req->ws, 8); AN(dbits); ll = 0; - oi = ObjIterBegin(req->objcore, req->wrk); + oi = ObjIterBegin(req->wrk, req->objcore); do { ois = ObjIter(req->objcore, oi, &sp, &sl); pp = sp; diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index 3d8caa5..116a73e 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -339,7 +339,7 @@ EXP_NukeOne(struct worker *wrk, struct lru *lru) exp_mail_it(oc); - VSLb(wrk->vsl, SLT_ExpKill, "LRU x=%u", ObjGetXID(oc, wrk->stats)); + VSLb(wrk->vsl, SLT_ExpKill, "LRU x=%u", ObjGetXID(wrk, oc)); (void)HSH_DerefObjCore(wrk, &oc); return (1); } @@ -463,8 +463,7 @@ exp_expire(struct exp_priv *ep, double now) CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC); VSLb(&ep->vsl, SLT_ExpKill, "EXP_Expired x=%u t=%.0f", - ObjGetXID(oc, ep->wrk->stats), - EXP_Ttl(NULL, &oc->exp) - now); + ObjGetXID(ep->wrk, oc), EXP_Ttl(NULL, &oc->exp) - now); (void)HSH_DerefObjCore(ep->wrk, &oc); return (0); } diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 4080a8e..3c0ba92 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -497,7 +497,7 @@ vbf_fetch_body_helper(struct busyobj *bo) VFP_Close(vfc); if (!bo->do_stream) - ObjTrimStore(vfc->oc, bo->wrk->stats); + ObjTrimStore(bo->wrk, vfc->oc); } /*-------------------------------------------------------------------- @@ -684,8 +684,8 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) } al = 0; - ol = ObjGetLen(bo->ims_oc, bo->wrk->stats); - oi = ObjIterBegin(bo->ims_oc, wrk); + ol = ObjGetLen(bo->wrk, bo->ims_oc); + oi = ObjIterBegin(wrk, bo->ims_oc); do { ois = ObjIter(bo->ims_oc, oi, &sp, &sl); while (sl > 0) { @@ -709,7 +709,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) HSH_Unbusy(wrk, bo->fetch_objcore); assert(al == ol); - assert(ObjGetLen(bo->fetch_objcore, bo->wrk->stats) == al); + assert(ObjGetLen(bo->wrk, bo->fetch_objcore) == al); EXP_Rearm(bo->ims_oc, bo->ims_oc->exp.t_origin, 0, 0, 0); /* Recycle the backend connection before setting BOS_FINISHED to @@ -889,7 +889,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv) AZ(bo->fetch_objcore->flags & OC_F_FAILED); HSH_Complete(bo->fetch_objcore); VSLb(bo->vsl, SLT_Length, "%ju", - (uintmax_t)ObjGetLen(bo->fetch_objcore, bo->wrk->stats)); + (uintmax_t)ObjGetLen(bo->wrk, bo->fetch_objcore)); } AZ(bo->fetch_objcore->busyobj); diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index 3e3aad6..331efce 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -89,7 +89,7 @@ VFP_GetStorage(struct vfp_ctx *vc, ssize_t *sz, uint8_t **ptr) if (l == 0) l = cache_param->fetch_chunksize; *sz = l; - if (!ObjGetSpace(vc->oc, vc->wrk, sz, ptr)) { + if (!ObjGetSpace(vc->wrk, vc->oc, sz, ptr)) { *sz = 0; *ptr = NULL; return (VFP_Error(vc, "Could not get storage")); diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c index f58ea70..99e9a24 100644 --- a/bin/varnishd/cache/cache_http1_deliver.c +++ b/bin/varnishd/cache/cache_http1_deliver.c @@ -97,9 +97,9 @@ v1d_dorange(struct req *req, struct busyobj *bo, const char *r) /* We must snapshot the length if we're streaming from the backend */ if (bo != NULL) - len = VBO_waitlen(bo, req->wrk->stats, -1); + len = VBO_waitlen(req->wrk, bo, -1); else - len = ObjGetLen(req->objcore, req->wrk->stats); + len = ObjGetLen(req->wrk, req->objcore); if (strncmp(r, "bytes=", 6)) return; @@ -174,7 +174,7 @@ v1d_WriteDirObj(struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - oi = ObjIterBegin(req->objcore, req->wrk); + oi = ObjIterBegin(req->wrk, req->objcore); XXXAN(oi); do { @@ -252,11 +252,11 @@ V1D_Deliver(struct req *req, struct busyobj *bo) /* XXX: Not happy with this convoluted test */ req->res_mode |= RES_LEN; if (!(req->objcore->flags & OC_F_PASS) || - ObjGetLen(req->objcore, req->wrk->stats) != 0) { + ObjGetLen(req->wrk, req->objcore) != 0) { http_Unset(req->resp, H_Content_Length); http_PrintfHeader(req->resp, "Content-Length: %ju", (uintmax_t)ObjGetLen( - req->objcore, req->wrk->stats)); + req->wrk, req->objcore)); } } @@ -340,7 +340,7 @@ V1D_Deliver(struct req *req, struct busyobj *bo) l = -1; while (req->objcore->busyobj) { assert(bo != NULL); - l = VBO_waitlen(bo, req->wrk->stats, l); + l = VBO_waitlen(req->wrk, bo, l); } ESI_DeliverChild(req); } else if (req->res_mode & RES_GUNZIP || diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 2c5335c..1d8f578 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -94,14 +94,14 @@ struct objiter { }; void * -ObjIterBegin(struct objcore *oc, struct worker *wrk) +ObjIterBegin(struct worker *wrk, struct objcore *oc) { struct objiter *oi; struct object *obj; const struct storeobj_methods *om = obj_getmethods(oc); if (om->objiterbegin != NULL) - return (om->objiterbegin(oc, wrk)); + return (om->objiterbegin(wrk, oc)); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); obj = obj_getobj(oc, wrk->stats); @@ -152,7 +152,7 @@ ObjIter(struct objcore *oc, void *oix, void **p, ssize_t *l) } else { ol = oi->len; while (1) { - nl = VBO_waitlen(oi->bo, oi->wrk->stats, ol); + nl = VBO_waitlen(oi->wrk, oi->bo, ol); if (nl != ol) break; if (oi->bo->state == BOS_FINISHED) @@ -251,14 +251,14 @@ objallocwithnuke(struct stevedore *stv, struct worker *wrk, size_t size) */ int -ObjGetSpace(struct objcore *oc, struct worker *wrk, ssize_t *sz, uint8_t **ptr) +ObjGetSpace(struct worker *wrk, struct objcore *oc, ssize_t *sz, uint8_t **ptr) { struct object *o; struct storage *st; const struct storeobj_methods *om = obj_getmethods(oc); if (om->objgetspace != NULL) - return (om->objgetspace(oc, wrk, sz, ptr)); + return (om->objgetspace(wrk, oc, sz, ptr)); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -330,15 +330,15 @@ ObjExtend(struct objcore *oc, struct dstat *ds, ssize_t l) */ uint64_t -ObjGetLen(struct objcore *oc, struct dstat *ds) +ObjGetLen(struct worker *wrk, struct objcore *oc) { struct object *o; const struct storeobj_methods *om = obj_getmethods(oc); if (om->objgetlen != NULL) - return (om->objgetlen(oc, ds)); + return (om->objgetlen(wrk, oc)); - o = obj_getobj(oc, ds); + o = obj_getobj(oc, wrk->stats); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); return (o->len); } @@ -352,7 +352,7 @@ ObjGetLen(struct objcore *oc, struct dstat *ds) */ void -ObjTrimStore(struct objcore *oc, struct dstat *ds) +ObjTrimStore(struct worker *wrk, struct objcore *oc) { const struct stevedore *stv; struct storage *st; @@ -360,15 +360,15 @@ ObjTrimStore(struct objcore *oc, struct dstat *ds) const struct storeobj_methods *om = obj_getmethods(oc); if (om->objtrimstore != NULL) { - om->objtrimstore(oc, ds); + om->objtrimstore(wrk, oc); return; } CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - AN(ds); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); stv = oc->stobj->stevedore; CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); - o = obj_getobj(oc, ds); + o = obj_getobj(oc, wrk->stats); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); st = VTAILQ_LAST(&o->list, storagehead); if (st == NULL) @@ -601,11 +601,11 @@ ObjCopyAttr(const struct vfp_ctx *vc, struct objcore *ocs, enum obj_attr attr) } unsigned -ObjGetXID(struct objcore *oc, struct dstat *ds) +ObjGetXID(struct worker *wrk, struct objcore *oc) { uint32_t u; - AZ(ObjGetU32(oc, ds, OA_VXID, &u)); + AZ(ObjGetU32(oc, wrk->stats, OA_VXID, &u)); return (u); } diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index 049e850..16bd00e 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -62,7 +62,7 @@ VRB_Iterate(struct req *req, req_body_iter_f *func, void *priv) switch(req->req_body_status) { case REQ_BODY_CACHED: - oi = ObjIterBegin(req->body_oc, req->wrk); + oi = ObjIterBegin(req->wrk, req->body_oc); AN(oi); do { ois = ObjIter(req->body_oc, oi, &p, &l); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 3ea3452..4a34a17 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -109,7 +109,7 @@ cnt_deliver(struct worker *wrk, struct req *req) if (req->wrk->stats->cache_hit) http_PrintfHeader(req->resp, "X-Varnish: %u %u", VXID(req->vsl->wid), - ObjGetXID(req->objcore, wrk->stats)); + ObjGetXID(wrk, req->objcore)); else http_PrintfHeader(req->resp, "X-Varnish: %u", VXID(req->vsl->wid)); @@ -399,7 +399,7 @@ cnt_lookup(struct worker *wrk, struct req *req) /* Found a hit-for-pass */ VSLb(req->vsl, SLT_Debug, "XXXX HIT-FOR-PASS"); VSLb(req->vsl, SLT_HitPass, "%u", - ObjGetXID(req->objcore, wrk->stats)); + ObjGetXID(wrk, req->objcore)); AZ(boc); (void)HSH_DerefObjCore(wrk, &req->objcore); wrk->stats->cache_hitpass++; @@ -407,8 +407,7 @@ cnt_lookup(struct worker *wrk, struct req *req) return (REQ_FSM_MORE); } - VSLb(req->vsl, SLT_Hit, "%u", - ObjGetXID(req->objcore, wrk->stats)); + VSLb(req->vsl, SLT_Hit, "%u", ObjGetXID(wrk, req->objcore)); VCL_hit_method(req->vcl, wrk, req, NULL, req->http->ws); diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index cc3420a..06d6d22 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -95,20 +95,20 @@ typedef struct object *getobj_f(struct objcore *oc, struct dstat *); * Or the can be "complex" and provide all of these methods: * (Described in comments in cache_obj.c) */ -typedef void *objiterbegin_f(struct objcore *oc, struct worker *wrk); +typedef void *objiterbegin_f(struct worker *, struct objcore *oc); typedef enum objiter_status objiter_f(struct objcore *oc, void *oix, void **p, ssize_t *l); -typedef void objiterend_f(struct objcore *oc, void **oix); -typedef int objgetspace_f(struct objcore *oc, struct worker *, +typedef void objiterend_f(struct objcore *, void **oix); +typedef int objgetspace_f(struct worker *, struct objcore *, ssize_t *sz, uint8_t **ptr); -typedef void objextend_f(struct objcore *oc, struct dstat *ds, ssize_t l); -typedef void objtrimstore_f(struct objcore *oc, struct dstat *ds); -typedef void objslim_f(struct objcore *oc, struct dstat *ds); -typedef void *objgetattr_f(struct objcore *oc, struct dstat *ds, +typedef void objextend_f(struct objcore *, struct dstat *ds, ssize_t l); +typedef void objtrimstore_f(struct worker *, struct objcore *); +typedef void objslim_f(struct objcore *, struct dstat *ds); +typedef void *objgetattr_f(struct objcore *, struct dstat *ds, enum obj_attr attr, ssize_t *len); -typedef void *objsetattr_f(const struct vfp_ctx *vc, enum obj_attr attr, +typedef void *objsetattr_f(const struct vfp_ctx *, enum obj_attr attr, ssize_t len, const void *ptr); -typedef uint64_t objgetlen_f(struct objcore *oc, struct dstat *ds); +typedef uint64_t objgetlen_f(struct worker *, struct objcore *); struct storeobj_methods { freeobj_f *freeobj; From phk at FreeBSD.org Tue Sep 16 11:30:43 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Sep 2014 13:30:43 +0200 Subject: [master] 7816fee More refactoring to make struct worker the default context for Obj*() Message-ID: commit 7816feec3164b47081ae1df6a64ac8e14f15583d Author: Poul-Henning Kamp Date: Tue Sep 16 11:30:20 2014 +0000 More refactoring to make struct worker the default context for Obj*() diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 57fb42f..74f2d9f 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -898,9 +898,9 @@ void http_Unset(struct http *hp, const char *hdr); unsigned http_CountHdr(const struct http *hp, const char *hdr); void http_CollectHdr(struct http *hp, const char *hdr); void http_VSL_log(const struct http *hp); -void HTTP_Merge(struct objcore *, struct dstat *, struct http *to); -uint16_t HTTP_GetStatusPack(struct objcore *oc, struct dstat *ds); -const char *HTTP_GetHdrPack(struct objcore *, struct dstat *, +void HTTP_Merge(struct worker *, struct objcore *, struct http *to); +uint16_t HTTP_GetStatusPack(struct worker *, struct objcore *oc); +const char *HTTP_GetHdrPack(struct worker *, struct objcore *, const char *hdr); enum sess_close http_DoConnection(struct http *hp); @@ -982,15 +982,15 @@ void *ObjIterBegin(struct worker *, struct objcore *); enum objiter_status ObjIter(struct objcore *, void *, void **, ssize_t *); void ObjIterEnd(struct objcore *, void **); int ObjGetSpace(struct worker *, struct objcore *, ssize_t *sz, uint8_t **ptr); -void ObjExtend(struct objcore *, struct dstat *, ssize_t l); +void ObjExtend(struct worker *, struct objcore *, ssize_t l); void ObjTrimStore(struct worker *, struct objcore *); unsigned ObjGetXID(struct worker *, struct objcore *); uint64_t ObjGetLen(struct worker *, struct objcore *oc); -void ObjUpdateMeta(struct objcore *, struct dstat *); -void ObjFreeObj(struct objcore *, struct dstat *); -void ObjSlim(struct objcore *oc, struct dstat *ds); +void ObjUpdateMeta(struct worker *, struct objcore *); +void ObjFreeObj(struct worker *, struct objcore *); +void ObjSlim(struct worker *, struct objcore *oc); struct lru *ObjGetLRU(const struct objcore *); -void *ObjGetattr(struct objcore *oc, struct dstat *ds, enum obj_attr attr, +void *ObjGetattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr, ssize_t *len); void *ObjSetattr(const struct vfp_ctx *, enum obj_attr attr, ssize_t len, const void *); @@ -1000,11 +1000,11 @@ int ObjSetDouble(const struct vfp_ctx*, enum obj_attr, double); int ObjSetU32(const struct vfp_ctx *, enum obj_attr, uint32_t); int ObjSetU64(const struct vfp_ctx *, enum obj_attr, uint64_t); -int ObjGetDouble(struct objcore *, struct dstat *, enum obj_attr, double *); -int ObjGetU32(struct objcore *, struct dstat *, enum obj_attr, uint32_t *); -int ObjGetU64(struct objcore *, struct dstat *, enum obj_attr, uint64_t *); +int ObjGetDouble(struct worker *, struct objcore *, enum obj_attr, double *); +int ObjGetU32(struct worker *, struct objcore *, enum obj_attr, uint32_t *); +int ObjGetU64(struct worker *, struct objcore *, enum obj_attr, uint64_t *); -int ObjCheckFlag(struct objcore *oc, struct dstat *ds, enum obj_flags of); +int ObjCheckFlag(struct worker *, struct objcore *oc, enum obj_flags of); void ObjSetFlag(const struct vfp_ctx *vc, enum obj_flags of, int val); /* cache_panic.c */ diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index 26c1f59..daa040f 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -827,7 +827,7 @@ BAN_Compile(void) */ static int -ban_evaluate(const uint8_t *bs, struct objcore *oc, struct dstat *ds, +ban_evaluate(struct worker *wrk, const uint8_t *bs, struct objcore *oc, const struct http *reqhttp, unsigned *tests) { struct ban_test bt; @@ -852,10 +852,10 @@ ban_evaluate(const uint8_t *bs, struct objcore *oc, struct dstat *ds, arg1 = p; break; case BANS_ARG_OBJHTTP: - arg1 = HTTP_GetHdrPack(oc, ds, bt.arg1_spec); + arg1 = HTTP_GetHdrPack(wrk, oc, bt.arg1_spec); break; case BANS_ARG_OBJSTATUS: - arg1 = HTTP_GetHdrPack(oc, ds, ":status"); + arg1 = HTTP_GetHdrPack(wrk, oc, ":status"); break; default: WRONG("Wrong BAN_ARG code"); @@ -939,7 +939,7 @@ BAN_CheckObject(struct worker *wrk, struct objcore *oc, struct req *req) CHECK_OBJ_NOTNULL(b, BAN_MAGIC); if (b->flags & BANS_FLAG_COMPLETED) continue; - if (ban_evaluate(b->spec, oc, wrk->stats, req->http, &tests)) + if (ban_evaluate(wrk, b->spec, oc, req->http, &tests)) break; } @@ -958,7 +958,7 @@ BAN_CheckObject(struct worker *wrk, struct objcore *oc, struct req *req) if (b == oc->ban) { /* not banned */ oc->ban = b0; - ObjUpdateMeta(oc, wrk->stats); + ObjUpdateMeta(wrk, oc); return (0); } else { oc->ban = NULL; @@ -1087,7 +1087,7 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt, continue; } tests = 0; - i = ban_evaluate(bl->spec, oc, wrk->stats, NULL, &tests); + i = ban_evaluate(wrk, bl->spec, oc, NULL, &tests); VSC_C_main->bans_lurker_tested++; VSC_C_main->bans_lurker_tests_tested += tests; if (i) diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 9fa5cf9..97b6645 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -224,7 +224,7 @@ VBO_extend(struct busyobj *bo, ssize_t l) return; assert(l > 0); Lck_Lock(&bo->mtx); - ObjExtend(bo->vfc->oc, bo->wrk->stats, l); + ObjExtend(bo->wrk, bo->vfc->oc, l); AZ(pthread_cond_broadcast(&bo->cond)); Lck_Unlock(&bo->mtx); } diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 09e8a27..eca1eee 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -269,7 +269,7 @@ ESI_Deliver(struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); - p = ObjGetattr(req->objcore, req->wrk->stats, OA_ESIDATA, &l); + p = ObjGetattr(req->wrk, req->objcore, OA_ESIDATA, &l); AN(p); assert(l > 0); e = p + l; @@ -484,7 +484,7 @@ ESI_DeliverChild(struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); - if (!ObjCheckFlag(req->objcore, req->wrk->stats, OF_GZIPED)) { + if (!ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED)) { oi = ObjIterBegin(req->wrk, req->objcore); do { ois = ObjIter(req->objcore, oi, &sp, &sl); @@ -500,7 +500,7 @@ ESI_DeliverChild(struct req *req) * padding it, as necessary, to a byte boundary. */ - p = ObjGetattr(req->objcore, req->wrk->stats, OA_GZIPBITS, &l); + p = ObjGetattr(req->wrk, req->objcore, OA_GZIPBITS, &l); AN(p); assert(l == 24); start = vbe64dec(p); diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index 116a73e..79a8d2f 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -335,7 +335,7 @@ EXP_NukeOne(struct worker *wrk, struct lru *lru) } /* XXX: We could grab and return one storage segment to our caller */ - ObjSlim(oc, wrk->stats); + ObjSlim(wrk, oc); exp_mail_it(oc); @@ -391,7 +391,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now) if (flags & OC_EF_MOVE) { oc->timer_when = EXP_When(&oc->exp); - ObjUpdateMeta(oc, ep->wrk->stats); + ObjUpdateMeta(ep->wrk, oc); } VSLb(&ep->vsl, SLT_ExpKill, "EXP_When p=%p e=%.9f f=0x%x", oc, diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 3c0ba92..caa74c3 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -190,12 +190,11 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo) } if (bo->ims_oc != NULL) { - q = HTTP_GetHdrPack(bo->ims_oc, bo->wrk->stats, - H_Last_Modified); + q = HTTP_GetHdrPack(bo->wrk, bo->ims_oc, H_Last_Modified); if (q != NULL) http_PrintfHeader(bo->bereq0, "If-Modified-Since: %s", q); - q = HTTP_GetHdrPack(bo->ims_oc, bo->wrk->stats, H_ETag); + q = HTTP_GetHdrPack(bo->wrk, bo->ims_oc, H_ETag); if (q != NULL) http_PrintfHeader(bo->bereq0, "If-None-Match: %s", q); @@ -381,7 +380,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) AZ(bo->do_esi); if (bo->ims_oc != NULL && http_IsStatus(bo->beresp, 304)) { - if (ObjCheckFlag(bo->ims_oc, bo->wrk->stats, OF_CHGGZIP)) { + if (ObjCheckFlag(bo->wrk, bo->ims_oc, OF_CHGGZIP)) { /* * If we changed the gzip status of the object * the stored Content_Encoding controls we @@ -390,7 +389,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) http_Unset(bo->beresp, H_Content_Encoding); RFC2616_Weaken_Etag(bo->beresp); } - HTTP_Merge(bo->ims_oc, bo->wrk->stats, bo->beresp); + HTTP_Merge(bo->wrk, bo->ims_oc, bo->beresp); assert(http_IsStatus(bo->beresp, 200)); do_ims = 1; } else @@ -626,7 +625,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) if (bo->vfc->failed && !bo->do_stream) { assert(bo->state < BOS_STREAM); if (bo->fetch_objcore != NULL) - ObjFreeObj(bo->fetch_objcore, bo->wrk->stats); + ObjFreeObj(bo->wrk, bo->fetch_objcore); return (F_STP_ERROR); } @@ -672,7 +671,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) AZ(vbf_beresp2obj(bo)); - if (ObjGetattr(bo->ims_oc, bo->wrk->stats, OA_ESIDATA, NULL) != NULL) + if (ObjGetattr(bo->wrk, bo->ims_oc, OA_ESIDATA, NULL) != NULL) AZ(ObjCopyAttr(bo->vfc, bo->ims_oc, OA_ESIDATA)); AZ(ObjCopyAttr(bo->vfc, bo->ims_oc, OA_FLAGS)); @@ -949,8 +948,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, bo->fetch_objcore = oc; AZ(bo->ims_oc); - if (oldoc != NULL && - ObjCheckFlag(oldoc, req->wrk->stats, OF_IMSCAND)) { + if (oldoc != NULL && ObjCheckFlag(req->wrk, oldoc, OF_IMSCAND)) { assert(oldoc->refcnt > 0); HSH_Ref(oldoc); bo->ims_oc = oldoc; diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index f8c4149..7fe3d86 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -424,7 +424,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, if (BAN_CheckObject(wrk, oc, req)) continue; - vary = ObjGetattr(oc, wrk->stats, OA_VARY, NULL); + vary = ObjGetattr(wrk, oc, OA_VARY, NULL); if (vary != NULL && !VRY_Match(req, vary)) continue; @@ -771,7 +771,7 @@ HSH_DerefObjCore(struct worker *wrk, struct objcore **ocp) AZ(oc->ban); if (oc->stobj->stevedore != NULL) - ObjFreeObj(oc, wrk->stats); + ObjFreeObj(wrk, oc); FREE_OBJ(oc); wrk->stats->n_objectcore--; diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 27ab49f..031b7dd 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -820,10 +820,10 @@ HTTP_Decode(struct http *to, uint8_t *fm) /*--------------------------------------------------------------------*/ uint16_t -HTTP_GetStatusPack(struct objcore *oc, struct dstat *ds) +HTTP_GetStatusPack(struct worker *wrk, struct objcore *oc) { const char *ptr; - ptr = ObjGetattr(oc, ds, OA_HEADERS, NULL); + ptr = ObjGetattr(wrk, oc, OA_HEADERS, NULL); AN(ptr); return(vbe16dec(ptr + 2)); @@ -832,16 +832,16 @@ HTTP_GetStatusPack(struct objcore *oc, struct dstat *ds) /*--------------------------------------------------------------------*/ const char * -HTTP_GetHdrPack(struct objcore *oc, struct dstat *ds, const char *hdr) +HTTP_GetHdrPack(struct worker *wrk, struct objcore *oc, const char *hdr) { const char *ptr; unsigned l; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - AN(ds); AN(hdr); - ptr = ObjGetattr(oc, ds, OA_HEADERS, NULL); + ptr = ObjGetattr(wrk, oc, OA_HEADERS, NULL); AN(ptr); /* Skip nhd and status */ @@ -883,17 +883,17 @@ HTTP_GetHdrPack(struct objcore *oc, struct dstat *ds, const char *hdr) */ void -HTTP_Merge(struct objcore *oc, struct dstat *ds, struct http *to) +HTTP_Merge(struct worker *wrk, struct objcore *oc, struct http *to) { const char *ptr; unsigned u; const char *p; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); - AN(ds); - ptr = ObjGetattr(oc, ds, OA_HEADERS, NULL); + ptr = ObjGetattr(wrk, oc, OA_HEADERS, NULL); AN(ptr); to->status = vbe16dec(ptr + 2); diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c index 99e9a24..3eb10d6 100644 --- a/bin/varnishd/cache/cache_http1_deliver.c +++ b/bin/varnishd/cache/cache_http1_deliver.c @@ -239,7 +239,7 @@ V1D_Deliver(struct req *req, struct busyobj *bo) req->res_mode = 0; - if (!req->disable_esi && ObjGetattr(req->objcore, req->wrk->stats, + if (!req->disable_esi && ObjGetattr(req->wrk, req->objcore, OA_ESIDATA, NULL) != NULL) { /* In ESI mode, we can't know the aggregate length */ req->res_mode &= ~RES_LEN; @@ -267,7 +267,7 @@ V1D_Deliver(struct req *req, struct busyobj *bo) } if (cache_param->http_gzip_support && - ObjCheckFlag(req->objcore, req->wrk->stats, OF_GZIPED) && + ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) && !RFC2616_Req_Gzip(req->http)) { /* * We don't know what it uncompresses to @@ -346,7 +346,7 @@ V1D_Deliver(struct req *req, struct busyobj *bo) } else if (req->res_mode & RES_GUNZIP || (req->res_mode & RES_ESI_CHILD && !req->gzip_resp && - ObjCheckFlag(req->objcore, req->wrk->stats, OF_GZIPED))) { + ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED))) { VDP_push(req, VDP_gunzip); req->vgz = VGZ_NewUngzip(req->vsl, "U D -"); AZ(VGZ_WrwInit(req->vgz)); diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 1d8f578..4db9524 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -58,13 +58,14 @@ obj_getmethods(const struct objcore *oc) } static struct object * -obj_getobj(struct objcore *oc, struct dstat *ds) +obj_getobj(struct worker *wrk, struct objcore *oc) { const struct storeobj_methods *m = obj_getmethods(oc); - AN(ds); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AN(m->getobj); - return (m->getobj(oc, ds)); + return (m->getobj(wrk, oc)); } /*==================================================================== @@ -104,7 +105,7 @@ ObjIterBegin(struct worker *wrk, struct objcore *oc) return (om->objiterbegin(wrk, oc)); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - obj = obj_getobj(oc, wrk->stats); + obj = obj_getobj(wrk, oc); CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC); ALLOC_OBJ(oi, OBJITER_MAGIC); if (oi == NULL) @@ -265,7 +266,7 @@ ObjGetSpace(struct worker *wrk, struct objcore *oc, ssize_t *sz, uint8_t **ptr) AN(sz); AN(ptr); assert(*sz > 0); - o = obj_getobj(oc, wrk->stats); + o = obj_getobj(wrk, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); st = VTAILQ_LAST(&o->list, storagehead); @@ -302,19 +303,20 @@ ObjGetSpace(struct worker *wrk, struct objcore *oc, ssize_t *sz, uint8_t **ptr) */ void -ObjExtend(struct objcore *oc, struct dstat *ds, ssize_t l) +ObjExtend(struct worker *wrk, struct objcore *oc, ssize_t l) { struct object *o; struct storage *st; const struct storeobj_methods *om = obj_getmethods(oc); if (om->objextend != NULL) { - om->objextend(oc, ds, l); + om->objextend(wrk, oc, l); return; } + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - o = obj_getobj(oc, ds); + o = obj_getobj(wrk, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); st = VTAILQ_LAST(&o->list, storagehead); CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); @@ -338,7 +340,7 @@ ObjGetLen(struct worker *wrk, struct objcore *oc) if (om->objgetlen != NULL) return (om->objgetlen(wrk, oc)); - o = obj_getobj(oc, wrk->stats); + o = obj_getobj(wrk, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); return (o->len); } @@ -368,7 +370,7 @@ ObjTrimStore(struct worker *wrk, struct objcore *oc) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); stv = oc->stobj->stevedore; CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); - o = obj_getobj(oc, wrk->stats); + o = obj_getobj(wrk, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); st = VTAILQ_LAST(&o->list, storagehead); if (st == NULL) @@ -389,20 +391,20 @@ ObjTrimStore(struct worker *wrk, struct objcore *oc) */ void -ObjSlim(struct objcore *oc, struct dstat *ds) +ObjSlim(struct worker *wrk, struct objcore *oc) { struct object *o; struct storage *st, *stn; const struct storeobj_methods *om = obj_getmethods(oc); if (om->objslim != NULL) { - om->objslim(oc, ds); + om->objslim(wrk, oc); return; } + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - AN(ds); - o = obj_getobj(oc, ds); + o = obj_getobj(wrk, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); if (o->esidata != NULL) { @@ -419,25 +421,25 @@ ObjSlim(struct objcore *oc, struct dstat *ds) /*==================================================================== */ void -ObjUpdateMeta(struct objcore *oc, struct dstat *ds) +ObjUpdateMeta(struct worker *wrk, struct objcore *oc) { const struct storeobj_methods *m = obj_getmethods(oc); if (m->updatemeta != NULL) - m->updatemeta(oc, ds); + m->updatemeta(wrk, oc); } /*==================================================================== */ void -ObjFreeObj(struct objcore *oc, struct dstat *ds) +ObjFreeObj(struct worker *wrk, struct objcore *oc) { const struct storeobj_methods *m = obj_getmethods(oc); - AN(ds); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AN(m->freeobj); - m->freeobj(oc, ds); + m->freeobj(wrk, oc); } /*==================================================================== @@ -459,7 +461,7 @@ ObjGetLRU(const struct objcore *oc) */ void * -ObjGetattr(struct objcore *oc, struct dstat *ds, enum obj_attr attr, +ObjGetattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr, ssize_t *len) { struct object *o; @@ -467,13 +469,13 @@ ObjGetattr(struct objcore *oc, struct dstat *ds, enum obj_attr attr, const struct storeobj_methods *om = obj_getmethods(oc); if (om->objgetattr != NULL) - return (om->objgetattr(oc, ds, attr, len)); + return (om->objgetattr(wrk, oc, attr, len)); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - AN(ds); if (len == NULL) len = &dummy; - o = obj_getobj(oc, ds); + o = obj_getobj(wrk, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); switch (attr) { case OA_ESIDATA: @@ -527,7 +529,7 @@ ObjSetattr(const struct vfp_ctx *vc, enum obj_attr attr, ssize_t len, CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vc->bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(vc->oc, OBJCORE_MAGIC); - o = obj_getobj(vc->oc, vc->wrk->stats); + o = obj_getobj(vc->wrk, vc->oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); st = o->objstore; switch (attr) { @@ -590,7 +592,7 @@ ObjCopyAttr(const struct vfp_ctx *vc, struct objcore *ocs, enum obj_attr attr) CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - vps = ObjGetattr(ocs, vc->wrk->stats, attr, &l); + vps = ObjGetattr(vc->wrk, ocs, attr, &l); // XXX: later we want to have zero-length OA's too if (vps == NULL || l <= 0) return (-1); @@ -605,7 +607,7 @@ ObjGetXID(struct worker *wrk, struct objcore *oc) { uint32_t u; - AZ(ObjGetU32(oc, wrk->stats, OA_VXID, &u)); + AZ(ObjGetU32(wrk, oc, OA_VXID, &u)); return (u); } @@ -633,14 +635,14 @@ ObjSetDouble(const struct vfp_ctx *vc, enum obj_attr a, double t) } int -ObjGetDouble(struct objcore *oc, struct dstat *ds, enum obj_attr a, double *d) +ObjGetDouble(struct worker *wrk, struct objcore *oc, enum obj_attr a, double *d) { void *vp; uint64_t u; ssize_t l; assert(sizeof *d == sizeof u); - vp = ObjGetattr(oc, ds, a, &l); + vp = ObjGetattr(wrk, oc, a, &l); if (vp == NULL) return (-1); if (d != NULL) { @@ -667,12 +669,12 @@ ObjSetU64(const struct vfp_ctx *vc, enum obj_attr a, uint64_t t) } int -ObjGetU64(struct objcore *oc, struct dstat *ds, enum obj_attr a, uint64_t *d) +ObjGetU64(struct worker *wrk, struct objcore *oc, enum obj_attr a, uint64_t *d) { void *vp; ssize_t l; - vp = ObjGetattr(oc, ds, a, &l); + vp = ObjGetattr(wrk, oc, a, &l); if (vp == NULL || l != sizeof *d) return (-1); if (d != NULL) @@ -693,12 +695,12 @@ ObjSetU32(const struct vfp_ctx *vc, enum obj_attr a, uint32_t t) } int -ObjGetU32(struct objcore *oc, struct dstat *ds, enum obj_attr a, uint32_t *d) +ObjGetU32(struct worker *wrk, struct objcore *oc, enum obj_attr a, uint32_t *d) { void *vp; ssize_t l; - vp = ObjGetattr(oc, ds, a, &l); + vp = ObjGetattr(wrk, oc, a, &l); if (vp == NULL || l != sizeof *d) return (-1); if (d != NULL) @@ -710,11 +712,11 @@ ObjGetU32(struct objcore *oc, struct dstat *ds, enum obj_attr a, uint32_t *d) */ int -ObjCheckFlag(struct objcore *oc, struct dstat *ds, enum obj_flags of) +ObjCheckFlag(struct worker *wrk, struct objcore *oc, enum obj_flags of) { uint8_t *fp; - fp = ObjGetattr(oc, ds, OA_FLAGS, NULL); + fp = ObjGetattr(wrk, oc, OA_FLAGS, NULL); AN(fp); return ((*fp) & of); } diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index 16bd00e..223b728 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -174,7 +174,7 @@ VRB_Free(struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); if (req->body_oc != NULL) { - ObjFreeObj(req->body_oc, req->wrk->stats); + ObjFreeObj(req->wrk,req->body_oc); FREE_OBJ(req->body_oc); req->body_oc = NULL; } @@ -254,7 +254,7 @@ VRB_Cache(struct req *req, ssize_t maxsize) req->acct.req_bodybytes += l; if (yet >= l) yet -= l; - ObjExtend(req->body_oc, req->wrk->stats, l); + ObjExtend(req->wrk, req->body_oc, l); } } while (vfps == VFP_OK); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 4a34a17..3d20e43 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -103,7 +103,7 @@ cnt_deliver(struct worker *wrk, struct req *req) HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod); AZ(HTTP_Decode(req->resp, - ObjGetattr(req->objcore, req->wrk->stats, OA_HEADERS, NULL))); + ObjGetattr(req->wrk, req->objcore, OA_HEADERS, NULL))); http_ForceField(req->resp, HTTP_HDR_PROTO, "HTTP/1.1"); if (req->wrk->stats->cache_hit) @@ -127,7 +127,7 @@ cnt_deliver(struct worker *wrk, struct req *req) http_SetHeader(req->resp, "Via: 1.1 varnish-v4"); if (cache_param->http_gzip_support && - ObjCheckFlag(req->objcore, req->wrk->stats, OF_GZIPED) && + ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) && !RFC2616_Req_Gzip(req->http)) RFC2616_Weaken_Etag(req->resp); @@ -188,7 +188,7 @@ cnt_deliver(struct worker *wrk, struct req *req) */ while (req->objcore->busyobj != NULL) (void)usleep(100000); - ObjSlim(req->objcore, wrk->stats); + ObjSlim(wrk, req->objcore); } assert(WRW_IsReleased(wrk)); diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 6f27269..99de23f 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -225,8 +225,7 @@ RFC2616_Do_Cond(const struct req *req) ims = VTIM_parse(p); if (ims > req->t_req) /* [RFC2616 14.25] */ return (0); - AZ(ObjGetDouble(req->objcore, req->wrk->stats, - OA_LASTMODIFIED, &lm)); + AZ(ObjGetDouble(req->wrk, req->objcore, OA_LASTMODIFIED, &lm)); if (lm > ims) return (0); do_cond = 1; diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 13c04dc..8422f79 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -127,8 +127,8 @@ VRT_GetHdr(const struct vrt_ctx *ctx, const struct gethdr_s *hs) if (hs->where == HDR_OBJ) { CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); - return(HTTP_GetHdrPack(ctx->req->objcore, - ctx->req->wrk->stats, hs->what)); + return(HTTP_GetHdrPack(ctx->req->wrk, ctx->req->objcore, + hs->what)); } hp = vrt_selecthttp(ctx, hs->where); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 59b3c6f..439ef46 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -148,7 +148,7 @@ VRT_r_obj_status(const struct vrt_ctx *ctx) CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); - return (HTTP_GetStatusPack(ctx->req->objcore, ctx->req->wrk->stats)); + return (HTTP_GetStatusPack(ctx->req->wrk, ctx->req->objcore)); } const char * @@ -158,8 +158,7 @@ VRT_r_obj_proto(const struct vrt_ctx *ctx) CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); - return (HTTP_GetHdrPack(ctx->req->objcore, - ctx->req->wrk->stats, ":proto")); + return (HTTP_GetHdrPack(ctx->req->wrk, ctx->req->objcore, ":proto")); } const char * @@ -169,8 +168,7 @@ VRT_r_obj_reason(const struct vrt_ctx *ctx) CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); - return (HTTP_GetHdrPack(ctx->req->objcore, - ctx->req->wrk->stats, ":reason")); + return (HTTP_GetHdrPack(ctx->req->wrk, ctx->req->objcore, ":reason")); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 2e02e0a..ec03354 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -49,11 +49,12 @@ static const struct stevedore * volatile stv_next; */ static struct object * __match_proto__(getobj_f) -default_oc_getobj(struct objcore *oc, struct dstat *ds) +default_oc_getobj(struct worker *wrk, struct objcore *oc) { struct object *o; - (void)ds; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); if (oc->stobj->priv == NULL) return (NULL); CAST_OBJ_NOTNULL(o, oc->stobj->priv, OBJECT_MAGIC); @@ -61,13 +62,13 @@ default_oc_getobj(struct objcore *oc, struct dstat *ds) } static void __match_proto__(freeobj_f) -default_oc_freeobj(struct objcore *oc, struct dstat *ds) +default_oc_freeobj(struct worker *wrk, struct objcore *oc) { struct object *o; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - AN(ds); - ObjSlim(oc, ds); + ObjSlim(wrk, oc); CAST_OBJ_NOTNULL(o, oc->stobj->priv, OBJECT_MAGIC); oc->stobj->priv = NULL; oc->stobj->stevedore = NULL; @@ -75,7 +76,7 @@ default_oc_freeobj(struct objcore *oc, struct dstat *ds) STV_free(o->objstore); - ds->n_object--; + wrk->stats->n_object--; } static struct lru * __match_proto__(getlru_f) diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index 06d6d22..c3d6158 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -81,15 +81,15 @@ struct object { /* Methods on objcore ------------------------------------------------*/ -typedef void updatemeta_f(struct objcore *oc, struct dstat *); -typedef void freeobj_f(struct objcore *oc, struct dstat *); +typedef void updatemeta_f(struct worker *, struct objcore *oc); +typedef void freeobj_f(struct worker *, struct objcore *oc); typedef struct lru *getlru_f(const struct objcore *oc); /* * Stevedores can either be simple, and provide just this method: */ -typedef struct object *getobj_f(struct objcore *oc, struct dstat *); +typedef struct object *getobj_f(struct worker *, struct objcore *oc); /* * Or the can be "complex" and provide all of these methods: @@ -101,10 +101,10 @@ typedef enum objiter_status objiter_f(struct objcore *oc, void *oix, typedef void objiterend_f(struct objcore *, void **oix); typedef int objgetspace_f(struct worker *, struct objcore *, ssize_t *sz, uint8_t **ptr); -typedef void objextend_f(struct objcore *, struct dstat *ds, ssize_t l); +typedef void objextend_f(struct worker *, struct objcore *, ssize_t l); typedef void objtrimstore_f(struct worker *, struct objcore *); -typedef void objslim_f(struct objcore *, struct dstat *ds); -typedef void *objgetattr_f(struct objcore *, struct dstat *ds, +typedef void objslim_f(struct worker *, struct objcore *); +typedef void *objgetattr_f(struct worker *, struct objcore *, enum obj_attr attr, ssize_t *len); typedef void *objsetattr_f(const struct vfp_ctx *, enum obj_attr attr, ssize_t len, const void *ptr); diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c index b0dbff9..2dbfb0a 100644 --- a/bin/varnishd/storage/storage_persistent_silo.c +++ b/bin/varnishd/storage/storage_persistent_silo.c @@ -388,7 +388,7 @@ smp_loaded_st(const struct smp_sc *sc, const struct smp_seg *sg, */ static struct object * -smp_oc_getobj(struct objcore *oc, struct dstat *ds) +smp_oc_getobj(struct worker *wrk, struct objcore *oc) { struct object *o; struct smp_seg *sg; @@ -400,9 +400,8 @@ smp_oc_getobj(struct objcore *oc, struct dstat *ds) /* Some calls are direct, but they should match anyway */ assert(oc->stobj->stevedore->methods->getobj == smp_oc_getobj); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - if (ds == NULL) - AZ(oc->stobj->priv2 & NEED_FIXUP); CAST_OBJ_NOTNULL(sg, oc->stobj->priv, SMP_SEG_MAGIC); so = smp_find_so(sg, oc->stobj->priv2); @@ -424,7 +423,6 @@ smp_oc_getobj(struct objcore *oc, struct dstat *ds) if (!(oc->stobj->priv2 & NEED_FIXUP)) return (o); - AN(ds); Lck_Lock(&sg->sc->mtx); /* Check again, we might have raced. */ if (oc->stobj->priv2 & NEED_FIXUP) { @@ -447,8 +445,8 @@ smp_oc_getobj(struct objcore *oc, struct dstat *ds) } sg->nfixed++; - ds->n_object++; - ds->n_vampireobject--; + wrk->stats->n_object++; + wrk->stats->n_vampireobject--; oc->stobj->priv2 &= ~NEED_FIXUP; } Lck_Unlock(&sg->sc->mtx); @@ -457,14 +455,15 @@ smp_oc_getobj(struct objcore *oc, struct dstat *ds) } static void -smp_oc_updatemeta(struct objcore *oc, struct dstat *ds) +smp_oc_updatemeta(struct worker *wrk, struct objcore *oc) { struct object *o; struct smp_seg *sg; struct smp_object *so; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - o = smp_oc_getobj(oc, ds); + o = smp_oc_getobj(wrk, oc); AN(o); CAST_OBJ_NOTNULL(sg, oc->stobj->priv, SMP_SEG_MAGIC); @@ -484,12 +483,12 @@ smp_oc_updatemeta(struct objcore *oc, struct dstat *ds) } static void __match_proto__(freeobj_f) -smp_oc_freeobj(struct objcore *oc, struct dstat *ds) +smp_oc_freeobj(struct worker *wrk, struct objcore *oc) { struct smp_seg *sg; struct smp_object *so; - AN(ds); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CAST_OBJ_NOTNULL(sg, oc->stobj->priv, SMP_SEG_MAGIC); @@ -502,11 +501,11 @@ smp_oc_freeobj(struct objcore *oc, struct dstat *ds) assert(sg->nobj > 0); sg->nobj--; if (oc->stobj->priv2 & NEED_FIXUP) { - ds->n_vampireobject--; + wrk->stats->n_vampireobject--; } else { assert(sg->nfixed > 0); sg->nfixed--; - ds->n_object--; + wrk->stats->n_object--; } Lck_Unlock(&sg->sc->mtx); From phk at FreeBSD.org Tue Sep 16 11:49:39 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Sep 2014 13:49:39 +0200 Subject: [master] 58db58b Final refactoring of struct worker as context for Obj*() Message-ID: commit 58db58bed260ea52342e5456f7772668f8e73781 Author: Poul-Henning Kamp Date: Tue Sep 16 11:49:14 2014 +0000 Final refactoring of struct worker as context for Obj*() diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 74f2d9f..b4e79c4 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -992,20 +992,21 @@ void ObjSlim(struct worker *, struct objcore *oc); struct lru *ObjGetLRU(const struct objcore *); void *ObjGetattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr, ssize_t *len); -void *ObjSetattr(const struct vfp_ctx *, enum obj_attr attr, ssize_t len, - const void *); -int ObjCopyAttr(const struct vfp_ctx *, struct objcore *, enum obj_attr attr); +void *ObjSetattr(struct worker *, struct objcore *, enum obj_attr attr, + ssize_t len, const void *); +int ObjCopyAttr(struct worker *, struct objcore *, struct objcore *, + enum obj_attr attr); -int ObjSetDouble(const struct vfp_ctx*, enum obj_attr, double); -int ObjSetU32(const struct vfp_ctx *, enum obj_attr, uint32_t); -int ObjSetU64(const struct vfp_ctx *, enum obj_attr, uint64_t); +int ObjSetDouble(struct worker *, struct objcore *, enum obj_attr, double); +int ObjSetU32(struct worker *, struct objcore *, enum obj_attr, uint32_t); +int ObjSetU64(struct worker *, struct objcore *, enum obj_attr, uint64_t); int ObjGetDouble(struct worker *, struct objcore *, enum obj_attr, double *); int ObjGetU32(struct worker *, struct objcore *, enum obj_attr, uint32_t *); int ObjGetU64(struct worker *, struct objcore *, enum obj_attr, uint64_t *); int ObjCheckFlag(struct worker *, struct objcore *oc, enum obj_flags of); -void ObjSetFlag(const struct vfp_ctx *vc, enum obj_flags of, int val); +void ObjSetFlag(struct worker *, struct objcore *, enum obj_flags of, int val); /* cache_panic.c */ void PAN_Init(void); diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index 8a73566..a2aadb7 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -118,7 +118,8 @@ vfp_esi_end(struct vfp_ctx *vc, struct vef_priv *vef, if (retval == VFP_END) { l = VSB_len(vsb); assert(l > 0); - p = ObjSetattr(vc, OA_ESIDATA, l, VSB_data(vsb)); + p = ObjSetattr(vc->wrk, vc->oc, + OA_ESIDATA, l, VSB_data(vsb)); if (p == NULL) { retval = VFP_Error(vc, "Could not allocate storage for esidata"); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index caa74c3..3dbd83a 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -133,25 +133,25 @@ vbf_beresp2obj(struct busyobj *bo) return (-1); if (vary != NULL) { - b = ObjSetattr(bo->vfc, OA_VARY, varyl, VSB_data(vary)); + b = ObjSetattr(bo->wrk, bo->fetch_objcore, OA_VARY, varyl, VSB_data(vary)); VSB_delete(vary); } - AZ(ObjSetU32(bo->vfc, OA_VXID, VXID(bo->vsl->wid))); + AZ(ObjSetU32(bo->wrk, bo->fetch_objcore, OA_VXID, VXID(bo->vsl->wid))); /* for HTTP_Encode() VSLH call */ bo->beresp->logtag = SLT_ObjMethod; /* Filter into object */ - bp = ObjSetattr(bo->vfc, OA_HEADERS, l2, NULL); + bp = ObjSetattr(bo->wrk, bo->fetch_objcore, OA_HEADERS, l2, NULL); AN(bp); HTTP_Encode(bo->beresp, bp, l2, bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS); if (http_GetHdr(bo->beresp, H_Last_Modified, &b)) - AZ(ObjSetDouble(bo->vfc, OA_LASTMODIFIED, VTIM_parse(b))); + AZ(ObjSetDouble(bo->wrk, bo->fetch_objcore, OA_LASTMODIFIED, VTIM_parse(b))); else - AZ(ObjSetDouble(bo->vfc, OA_LASTMODIFIED, + AZ(ObjSetDouble(bo->wrk, bo->fetch_objcore, OA_LASTMODIFIED, floor(bo->fetch_objcore->exp.t_origin))); return (0); @@ -587,15 +587,15 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip)) - ObjSetFlag(bo->vfc, OF_GZIPED, 1); + ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_GZIPED, 1); if (bo->do_gzip || bo->do_gunzip) - ObjSetFlag(bo->vfc, OF_CHGGZIP, 1); + ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_CHGGZIP, 1); if (http_IsStatus(bo->beresp, 200) && ( http_GetHdr(bo->beresp, H_Last_Modified, &p) || http_GetHdr(bo->beresp, H_ETag, &p))) - ObjSetFlag(bo->vfc, OF_IMSCAND, 1); + ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1); if (bo->htc->body_status != BS_NONE) V1F_Setup_Fetch(bo->vfc, bo->htc); @@ -672,10 +672,11 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) AZ(vbf_beresp2obj(bo)); if (ObjGetattr(bo->wrk, bo->ims_oc, OA_ESIDATA, NULL) != NULL) - AZ(ObjCopyAttr(bo->vfc, bo->ims_oc, OA_ESIDATA)); + AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->ims_oc, + OA_ESIDATA)); - AZ(ObjCopyAttr(bo->vfc, bo->ims_oc, OA_FLAGS)); - AZ(ObjCopyAttr(bo->vfc, bo->ims_oc, OA_GZIPBITS)); + AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->ims_oc, OA_FLAGS)); + AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->ims_oc, OA_GZIPBITS)); if (bo->do_stream) { HSH_Unbusy(wrk, bo->fetch_objcore); diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index a4b3034..5e2125b 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -396,7 +396,7 @@ VGZ_UpdateObj(const struct vfp_ctx *vc, const struct vgz *vg) char *p; CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC); - p = ObjSetattr(vc, OA_GZIPBITS, 24, NULL); + p = ObjSetattr(vc->wrk, vc->oc, OA_GZIPBITS, 24, NULL); AN(p); vbe64enc(p, vg->vz.start_bit); vbe64enc(p + 8, vg->vz.last_bit); diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 4db9524..73eff9c 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -515,27 +515,25 @@ ObjGetattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr, */ void * -ObjSetattr(const struct vfp_ctx *vc, enum obj_attr attr, ssize_t len, - const void *ptr) +ObjSetattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr, + ssize_t len, const void *ptr) { struct object *o; void *retval = NULL; struct storage *st; - const struct storeobj_methods *om = obj_getmethods(vc->oc); + const struct storeobj_methods *om = obj_getmethods(oc); if (om->objsetattr != NULL) - return (om->objsetattr(vc, attr, len, ptr)); + return (om->objsetattr(wrk, oc, attr, len, ptr)); - CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vc->bo, BUSYOBJ_MAGIC); - CHECK_OBJ_NOTNULL(vc->oc, OBJCORE_MAGIC); - o = obj_getobj(vc->wrk, vc->oc); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + o = obj_getobj(wrk, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); st = o->objstore; switch (attr) { case OA_ESIDATA: - o->esidata = objallocwithnuke(vc->oc->stobj->stevedore, - vc->wrk, len); + o->esidata = objallocwithnuke(oc->stobj->stevedore, wrk, len); if (o->esidata == NULL) return (NULL); o->esidata->len = len; @@ -585,18 +583,20 @@ ObjSetattr(const struct vfp_ctx *vc, enum obj_attr attr, ssize_t len, */ int -ObjCopyAttr(const struct vfp_ctx *vc, struct objcore *ocs, enum obj_attr attr) +ObjCopyAttr(struct worker *wrk, struct objcore *oc, struct objcore *ocs, + enum obj_attr attr) { void *vps, *vpd; ssize_t l; - CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - vps = ObjGetattr(vc->wrk, ocs, attr, &l); + vps = ObjGetattr(wrk, ocs, attr, &l); // XXX: later we want to have zero-length OA's too if (vps == NULL || l <= 0) return (-1); - vpd = ObjSetattr(vc, attr, l, vps); + vpd = ObjSetattr(wrk, oc, attr, l, vps); if (vpd == NULL) return (-1); return (0); @@ -620,14 +620,14 @@ ObjGetXID(struct worker *wrk, struct objcore *oc) */ int -ObjSetDouble(const struct vfp_ctx *vc, enum obj_attr a, double t) +ObjSetDouble(struct worker *wrk, struct objcore *oc, enum obj_attr a, double t) { void *vp; uint64_t u; assert(sizeof t == sizeof u); memcpy(&u, &t, sizeof u); - vp = ObjSetattr(vc, a, sizeof u, NULL); + vp = ObjSetattr(wrk, oc, a, sizeof u, NULL); if (vp == NULL) return (-1); vbe64enc(vp, u); @@ -657,11 +657,11 @@ ObjGetDouble(struct worker *wrk, struct objcore *oc, enum obj_attr a, double *d) */ int -ObjSetU64(const struct vfp_ctx *vc, enum obj_attr a, uint64_t t) +ObjSetU64(struct worker *wrk, struct objcore *oc, enum obj_attr a, uint64_t t) { void *vp; - vp = ObjSetattr(vc, a, sizeof t, NULL); + vp = ObjSetattr(wrk, oc, a, sizeof t, NULL); if (vp == NULL) return (-1); vbe64enc(vp, t); @@ -683,11 +683,11 @@ ObjGetU64(struct worker *wrk, struct objcore *oc, enum obj_attr a, uint64_t *d) } int -ObjSetU32(const struct vfp_ctx *vc, enum obj_attr a, uint32_t t) +ObjSetU32(struct worker *wrk, struct objcore *oc, enum obj_attr a, uint32_t t) { void *vp; - vp = ObjSetattr(vc, a, sizeof t, NULL); + vp = ObjSetattr(wrk, oc, a, sizeof t, NULL); if (vp == NULL) return (-1); vbe32enc(vp, t); @@ -722,11 +722,11 @@ ObjCheckFlag(struct worker *wrk, struct objcore *oc, enum obj_flags of) } void -ObjSetFlag(const struct vfp_ctx *vc, enum obj_flags of, int val) +ObjSetFlag(struct worker *wrk, struct objcore *oc, enum obj_flags of, int val) { uint8_t *fp; - fp = ObjSetattr(vc, OA_FLAGS, 1, NULL); + fp = ObjSetattr(wrk, oc, OA_FLAGS, 1, NULL); AN(fp); if (val) (*fp) |= of; diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index c3d6158..0d162ca 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -33,7 +33,6 @@ struct stevedore; struct sess; -struct dstat; struct busyobj; struct objcore; struct worker; @@ -106,8 +105,8 @@ typedef void objtrimstore_f(struct worker *, struct objcore *); typedef void objslim_f(struct worker *, struct objcore *); typedef void *objgetattr_f(struct worker *, struct objcore *, enum obj_attr attr, ssize_t *len); -typedef void *objsetattr_f(const struct vfp_ctx *, enum obj_attr attr, - ssize_t len, const void *ptr); +typedef void *objsetattr_f(struct worker *, struct objcore *, + enum obj_attr attr, ssize_t len, const void *ptr); typedef uint64_t objgetlen_f(struct worker *, struct objcore *); struct storeobj_methods { From lkarsten at varnish-software.com Wed Sep 17 14:18:36 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Wed, 17 Sep 2014 16:18:36 +0200 Subject: [3.0] cf06737 Add that empty string VCL truth value changed. Message-ID: commit cf067372747b01aba1832c9d4b68f694be63e381 Author: Lasse Karstensen Date: Wed Sep 17 16:17:04 2014 +0200 Add that empty string VCL truth value changed. Solves: #1406 diff --git a/doc/changes.rst b/doc/changes.rst index 1f0aa34..247547c 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -149,6 +149,9 @@ Other Changes from 3.0.2 to 3.0.3 =========================== +Note: In between 3.0.2 and 3.0.3 the VCL truth value for empty strings changed. +Please see `Bug #1406`_ for the details. + Varnishd -------- @@ -225,6 +228,7 @@ Varnishd .. _bug #1125: http://varnish-cache.org/trac/ticket/1125 .. _bug #1126: http://varnish-cache.org/trac/ticket/1126 .. _bug #1140: http://varnish-cache.org/trac/ticket/1140 +.. _bug #1406: http://varnish-cache.org/trac/ticket/1406 varnishncsa ----------- From nils.goroll at uplex.de Wed Sep 17 15:02:53 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 17 Sep 2014 17:02:53 +0200 Subject: [master] e1dcc19 remove some unused declarations Message-ID: commit e1dcc194fa5508678fbd04416e326ca486f1dbf5 Author: Nils Goroll Date: Wed Sep 17 17:02:21 2014 +0200 remove some unused declarations Fixes #1546 diff --git a/include/vrt.h b/include/vrt.h index 90e85ea..1bb0b66 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -169,38 +169,6 @@ struct vrt_backend { }; /* - * A director with an unpredictable reply - */ - -struct vrt_dir_random_entry { - int host; - double weight; -}; - -struct vrt_dir_random { - const char *name; - unsigned retries; - unsigned nmember; - const struct vrt_dir_random_entry *members; -}; - -/* - * A director with dns-based selection - */ - -struct vrt_dir_dns_entry { - int host; -}; - -struct vrt_dir_dns { - const char *name; - const char *suffix; - const double ttl; - unsigned nmember; - const struct vrt_dir_dns_entry *members; -}; - -/* * other stuff. * XXX: document when bored */ From nils.goroll at uplex.de Fri Sep 19 09:38:52 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 19 Sep 2014 11:38:52 +0200 Subject: [master] 7e5ec4c add VCL_BYTES for vmods Message-ID: commit 7e5ec4c6dcdf9dedf2fb81d838dbdf1e7eae6949 Author: Nils Goroll Date: Fri Sep 19 11:38:37 2014 +0200 add VCL_BYTES for vmods diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 0553015..2861b40 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -49,6 +49,7 @@ ctypes = { 'BACKEND': "VCL_BACKEND", 'BLOB': "VCL_BLOB", 'BOOL': "VCL_BOOL", + 'BYTES': "VCL_BYTES", 'DURATION': "VCL_DURATION", 'ENUM': "VCL_ENUM", 'HEADER': "VCL_HEADER", From nils.goroll at uplex.de Fri Sep 19 09:41:45 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 19 Sep 2014 11:41:45 +0200 Subject: [master] cc2b4c4 Add std.cache_req_body to call VRT_CacheReqBody from VCL Message-ID: commit cc2b4c4d1a14dcdfe1a9f01a9797cbea396c581b Author: Nils Goroll Date: Fri Sep 19 11:40:22 2014 +0200 Add std.cache_req_body to call VRT_CacheReqBody from VCL Initial patch by Meng Zhnag - jammy.linux at gmail.com diff --git a/bin/varnishtest/tests/c00055.vtc b/bin/varnishtest/tests/c00055.vtc index 0344d02..ef568ea 100644 --- a/bin/varnishtest/tests/c00055.vtc +++ b/bin/varnishtest/tests/c00055.vtc @@ -11,8 +11,10 @@ server s1 { } -start varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend { + import ${vmod_std}; + sub vcl_recv { - C{ VRT_CacheReqBody(ctx, 1000); }C + std.cache_req_body(1KB); return (pass); } sub vcl_deliver { diff --git a/bin/varnishtest/tests/c00067.vtc b/bin/varnishtest/tests/c00067.vtc index d6d4c02..3c77433 100644 --- a/bin/varnishtest/tests/c00067.vtc +++ b/bin/varnishtest/tests/c00067.vtc @@ -31,8 +31,10 @@ client c1 { delay .2 varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend { + import ${vmod_std}; + sub vcl_recv { - C{ VRT_CacheReqBody(ctx, 1000); }C + std.cache_req_body(1000B); } } diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index 5bcc403..fbc082b 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -199,6 +199,14 @@ Description Example set req.url = std.querysort(req.url); +$Function VOID cache_req_body(BYTES) + +Description + Cache the req.body if it is smaller than the given size +Example + std.cache_req_body(1KB); + + This will cache the req.body if its size is smaller than 1KB. SEE ALSO diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 1aaec4c..55a574e 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -230,3 +230,12 @@ vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label) VSLb_ts_req(ctx->req, label, VTIM_real()); } } + +VCL_VOID __match_proto__(td_std_cache_req_body) +vmod_cache_req_body(const struct vrt_ctx *ctx, VCL_BYTES size) +{ + int result; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + result = VRT_CacheReqBody(ctx, size); + VSLb(ctx->vsl, SLT_Debug,"VRT_CacheReqBody[size: %zu] result: %d", (size_t)size, result); +} From phk at FreeBSD.org Fri Sep 19 10:29:15 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 19 Sep 2014 12:29:15 +0200 Subject: [master] 890a45a Make server.hostname and server.identity available in all VCL methods. Message-ID: commit 890a45ae8af9d6e70a2365f209707370af1bc574 Author: Poul-Henning Kamp Date: Fri Sep 19 10:28:48 2014 +0000 Make server.hostname and server.identity available in all VCL methods. Closes #1594 diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 77524b3..29fc2f3 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -471,14 +471,13 @@ pool_herder(void *priv) AZ(pthread_attr_init(&tp_attr)); while (1) { + AZ(pthread_attr_destroy(&tp_attr)); + AZ(pthread_attr_init(&tp_attr)); + /* Set the stacksize for worker threads we create */ if (cache_param->wthread_stacksize != UINT_MAX) AZ(pthread_attr_setstacksize(&tp_attr, cache_param->wthread_stacksize)); - else { - AZ(pthread_attr_destroy(&tp_attr)); - AZ(pthread_attr_init(&tp_attr)); - } /* Make more threads if needed and allowed */ if (pp->nthr < cache_param->wthread_min || diff --git a/bin/varnishtest/tests/v00001.vtc b/bin/varnishtest/tests/v00001.vtc index 1798ec9..822c847 100644 --- a/bin/varnishtest/tests/v00001.vtc +++ b/bin/varnishtest/tests/v00001.vtc @@ -32,6 +32,8 @@ varnish v1 -vcl+backend { set beresp.proto = "HTTP/1.2"; set beresp.reason = "For circular files"; set beresp.status = 903; + set beresp.http.y-served-by-hostname = server.hostname; + set beresp.http.y-served-by-identity = server.identity; } sub vcl_deliver { set resp.proto = "HTTP/1.2"; From nils.goroll at uplex.de Fri Sep 19 10:41:44 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 19 Sep 2014 12:41:44 +0200 Subject: [master] f57bc38 No inline-C required for these tests any more Message-ID: commit f57bc389fa4a4391f8017be300123b6d18631e81 Author: Nils Goroll Date: Fri Sep 19 12:39:53 2014 +0200 No inline-C required for these tests any more Thanks to fgs for noticing. diff --git a/bin/varnishtest/tests/c00055.vtc b/bin/varnishtest/tests/c00055.vtc index ef568ea..ebac77e 100644 --- a/bin/varnishtest/tests/c00055.vtc +++ b/bin/varnishtest/tests/c00055.vtc @@ -10,7 +10,7 @@ server s1 { txresp -status 200 -hdr "Foo: Foo" -body "56" } -start -varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend { +varnish v1 -vcl+backend { import ${vmod_std}; sub vcl_recv { diff --git a/bin/varnishtest/tests/c00067.vtc b/bin/varnishtest/tests/c00067.vtc index 3c77433..427042b 100644 --- a/bin/varnishtest/tests/c00067.vtc +++ b/bin/varnishtest/tests/c00067.vtc @@ -30,7 +30,7 @@ client c1 { delay .2 -varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend { +varnish v1 -vcl+backend { import ${vmod_std}; sub vcl_recv { From nils.goroll at uplex.de Fri Sep 19 10:52:04 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 19 Sep 2014 12:52:04 +0200 Subject: [master] f7c2c4c More canonical debug format for vmod_cache_req_body as suggested by fgs Message-ID: commit f7c2c4cb316dfb1fd17243aeaa4bb0b379f27b98 Author: Nils Goroll Date: Fri Sep 19 12:43:54 2014 +0200 More canonical debug format for vmod_cache_req_body as suggested by fgs diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 55a574e..8872a1c 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -237,5 +237,5 @@ vmod_cache_req_body(const struct vrt_ctx *ctx, VCL_BYTES size) int result; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); result = VRT_CacheReqBody(ctx, size); - VSLb(ctx->vsl, SLT_Debug,"VRT_CacheReqBody[size: %zu] result: %d", (size_t)size, result); + VSLb(ctx->vsl, SLT_Debug,"VRT_CacheReqBody(%zu): %d", (size_t)size, result); } From lkarsten at varnish-software.com Fri Sep 19 11:33:58 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Fri, 19 Sep 2014 13:33:58 +0200 Subject: [3.0] 5cb77b2 EL7 chroot seem not to have python by default. Message-ID: commit 5cb77b2f13b5bb45711a24cd33eaeff63d6f59ba Author: Lasse Karstensen Date: Fri Sep 19 13:33:36 2014 +0200 EL7 chroot seem not to have python by default. diff --git a/redhat/varnish.spec b/redhat/varnish.spec index 39f0141..bc4b4f3 100644 --- a/redhat/varnish.spec +++ b/redhat/varnish.spec @@ -14,6 +14,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) # You will need at least automake autoconf libtool python-docutils #BuildRequires: automake autoconf libtool python-docutils BuildRequires: ncurses-devel libxslt groff pcre-devel pkgconfig libedit-devel +BuildRequires: python Requires: varnish-libs = %{version}-%{release} Requires: logrotate Requires: ncurses From lkarsten at varnish-software.com Fri Sep 19 13:05:29 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Fri, 19 Sep 2014 15:05:29 +0200 Subject: [master] 114ba63 Increase accepted counter levels slightly. Message-ID: commit 114ba63f90240c6b370ca9326dcb5e9fbefc2e71 Author: Lasse Karstensen Date: Fri Sep 19 15:01:35 2014 +0200 Increase accepted counter levels slightly. Make the test case pass again on i386. (el5, el6, osx) This started to fail on September 15th, assumed after the changes to the Obj* API. diff --git a/bin/varnishtest/tests/c00045.vtc b/bin/varnishtest/tests/c00045.vtc index 57eef39..022e904 100644 --- a/bin/varnishtest/tests/c00045.vtc +++ b/bin/varnishtest/tests/c00045.vtc @@ -32,7 +32,7 @@ client c1 { varnish v1 -expect SMA.Transient.g_bytes == 0 varnish v1 -expect SMA.s0.g_bytes > 1000000 -varnish v1 -expect SMA.s0.g_space < 190 +varnish v1 -expect SMA.s0.g_space < 193 varnish v1 -expect SMA.s1.g_bytes == 0 varnish v1 -expect SMA.s1.g_space > 1000000 varnish v1 -expect SMA.s2.g_bytes == 0 @@ -48,7 +48,7 @@ client c1 { varnish v1 -expect n_lru_nuked == 1 varnish v1 -expect SMA.Transient.g_bytes == 0 varnish v1 -expect SMA.s0.g_bytes > 1000000 -varnish v1 -expect SMA.s0.g_space < 1190 +varnish v1 -expect SMA.s0.g_space < 1192 varnish v1 -expect SMA.s1.g_bytes == 0 varnish v1 -expect SMA.s1.g_space > 1000000 varnish v1 -expect SMA.s2.g_bytes == 0 @@ -64,7 +64,7 @@ client c1 { varnish v1 -expect n_lru_nuked == 2 varnish v1 -expect SMA.Transient.g_bytes == 0 varnish v1 -expect SMA.s0.g_bytes > 1000000 -varnish v1 -expect SMA.s0.g_space < 1192 +varnish v1 -expect SMA.s0.g_space < 1194 varnish v1 -expect SMA.s1.g_bytes == 0 varnish v1 -expect SMA.s1.g_space > 1000000 varnish v1 -expect SMA.s2.g_bytes == 0 From fgsch at lodoss.net Fri Sep 19 15:38:53 2014 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 19 Sep 2014 17:38:53 +0200 Subject: [master] 9c72a93 Revert "Make server.hostname and server.identity available in all VCL methods." Message-ID: commit 9c72a934cbc127419e3bb46f1bc25278f4eb9131 Author: Federico G. Schwindt Date: Fri Sep 19 16:27:58 2014 +0100 Revert "Make server.hostname and server.identity available in all VCL methods." This reverts commit 890a45ae8af9d6e70a2365f209707370af1bc574 since it broke master. diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 29fc2f3..77524b3 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -471,13 +471,14 @@ pool_herder(void *priv) AZ(pthread_attr_init(&tp_attr)); while (1) { - AZ(pthread_attr_destroy(&tp_attr)); - AZ(pthread_attr_init(&tp_attr)); - /* Set the stacksize for worker threads we create */ if (cache_param->wthread_stacksize != UINT_MAX) AZ(pthread_attr_setstacksize(&tp_attr, cache_param->wthread_stacksize)); + else { + AZ(pthread_attr_destroy(&tp_attr)); + AZ(pthread_attr_init(&tp_attr)); + } /* Make more threads if needed and allowed */ if (pp->nthr < cache_param->wthread_min || diff --git a/bin/varnishtest/tests/v00001.vtc b/bin/varnishtest/tests/v00001.vtc index 822c847..1798ec9 100644 --- a/bin/varnishtest/tests/v00001.vtc +++ b/bin/varnishtest/tests/v00001.vtc @@ -32,8 +32,6 @@ varnish v1 -vcl+backend { set beresp.proto = "HTTP/1.2"; set beresp.reason = "For circular files"; set beresp.status = 903; - set beresp.http.y-served-by-hostname = server.hostname; - set beresp.http.y-served-by-identity = server.identity; } sub vcl_deliver { set resp.proto = "HTTP/1.2"; From phk at FreeBSD.org Fri Sep 19 16:12:04 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 19 Sep 2014 18:12:04 +0200 Subject: [master] f2cd8dc Oops, forgot this bit of previous commit Message-ID: commit f2cd8dc1827266bede68180b1ff2d4b1ef512b60 Author: Poul-Henning Kamp Date: Fri Sep 19 10:48:32 2014 +0000 Oops, forgot this bit of previous commit diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index ca6f7c9..4fffa57 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -182,14 +182,14 @@ sp_variables = [ ), ('server.hostname', 'STRING', - ( 'client',), + ( 'all',), ( ), """ The host name of the server. """ ), ('server.identity', 'STRING', - ( 'client',), + ( 'all',), ( ), """ The identity of the server, as set by the -i parameter. If the -i parameter is not passed to varnishd, From phk at FreeBSD.org Fri Sep 19 16:12:04 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 19 Sep 2014 18:12:04 +0200 Subject: [master] 0d0a6ce Try again to make server.hostname and server.identity available in all VCL_methods. Message-ID: commit 0d0a6ce7dfd74ca07ee42b561e92a5a2b3e30e17 Author: Poul-Henning Kamp Date: Fri Sep 19 16:11:24 2014 +0000 Try again to make server.hostname and server.identity available in all VCL_methods. Apologies for the previous breakage. diff --git a/bin/varnishtest/tests/v00001.vtc b/bin/varnishtest/tests/v00001.vtc index 1798ec9..822c847 100644 --- a/bin/varnishtest/tests/v00001.vtc +++ b/bin/varnishtest/tests/v00001.vtc @@ -32,6 +32,8 @@ varnish v1 -vcl+backend { set beresp.proto = "HTTP/1.2"; set beresp.reason = "For circular files"; set beresp.status = 903; + set beresp.http.y-served-by-hostname = server.hostname; + set beresp.http.y-served-by-identity = server.identity; } sub vcl_deliver { set resp.proto = "HTTP/1.2"; From phk at FreeBSD.org Mon Sep 22 08:01:28 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 22 Sep 2014 10:01:28 +0200 Subject: [master] 3261a9f We always have a bo->fetch_objcore, don't confuse Coverity with a NULL test in the middle of things. Message-ID: commit 3261a9fe46ed0c3014f11c76a028d3128583899f Author: Poul-Henning Kamp Date: Mon Sep 22 08:00:40 2014 +0000 We always have a bo->fetch_objcore, don't confuse Coverity with a NULL test in the middle of things. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 3dbd83a..8de0923 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -509,6 +509,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC); assert(wrk->handling == VCL_RET_DELIVER); @@ -624,8 +625,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) if (bo->vfc->failed && !bo->do_stream) { assert(bo->state < BOS_STREAM); - if (bo->fetch_objcore != NULL) - ObjFreeObj(bo->wrk, bo->fetch_objcore); + ObjFreeObj(bo->wrk, bo->fetch_objcore); return (F_STP_ERROR); } From perbu at varnish-software.com Mon Sep 22 09:05:49 2014 From: perbu at varnish-software.com (Per Buer) Date: Mon, 22 Sep 2014 11:05:49 +0200 Subject: [master] 5aa97f1 add std.strstr Message-ID: commit 5aa97f16f416019f25a591902c4e704c4ff1d372 Author: Per Buer Date: Sat Sep 20 17:57:18 2014 +0200 add std.strstr diff --git a/bin/varnishtest/tests/m00018.vtc b/bin/varnishtest/tests/m00018.vtc new file mode 100644 index 0000000..b51ed18 --- /dev/null +++ b/bin/varnishtest/tests/m00018.vtc @@ -0,0 +1,34 @@ +varnishtest "Test substring matching in std" + +server s1 { + rxreq + txresp + + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + import ${vmod_std}; + + sub vcl_deliver { + if (std.strstr(req.url, "foo")) { + set resp.http.sub = "found"; + } else { + set resp.http.sub = "not found"; + } + + } +} -start + +client c1 { + txreq -url "/foobar" + rxresp + expect resp.http.sub == "found" + + + txreq -url "/quux" + rxresp + expect resp.http.sub == "not found" +} -run + diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index fbc082b..455485a 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -195,7 +195,6 @@ $Function STRING querysort(STRING) Description Sorts the querystring for cache normalization purposes. - Example set req.url = std.querysort(req.url); @@ -208,6 +207,17 @@ Example This will cache the req.body if its size is smaller than 1KB. +$Function STRING strstr(STRING, STRING) + +Description + Returns true if the second string is a substring of the first + string. Note that the comparison is case sensitive. You can + use the tolower function on both strings if you want case + insensitivity. +Example + if (std.strstr(req.url, req.http.x-restrict)) + + SEE ALSO ======== diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 8872a1c..898e8dd 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -239,3 +239,14 @@ vmod_cache_req_body(const struct vrt_ctx *ctx, VCL_BYTES size) result = VRT_CacheReqBody(ctx, size); VSLb(ctx->vsl, SLT_Debug,"VRT_CacheReqBody(%zu): %d", (size_t)size, result); } + +VCL_STRING __match_proto__(td_std_strstr) +vmod_strstr(const struct vrt_ctx *ctx, VCL_STRING mstr, VCL_STRING msubstr) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + char *match = strstr(mstr, msubstr); + + return(match); +} + From perbu at varnish-software.com Mon Sep 22 09:05:49 2014 From: perbu at varnish-software.com (Per Buer) Date: Mon, 22 Sep 2014 11:05:49 +0200 Subject: [master] c8485f8 Nitpicks Message-ID: commit c8485f8dd6913338cefa92ee1dad597918f5bf17 Author: Per Buer Date: Mon Sep 22 11:00:51 2014 +0200 Nitpicks diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index 455485a..ad832d1 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -210,10 +210,14 @@ Example $Function STRING strstr(STRING, STRING) Description - Returns true if the second string is a substring of the first + Returns the substring if the second string is a substring of the first string. Note that the comparison is case sensitive. You can use the tolower function on both strings if you want case insensitivity. + + If there is no match a NULL pointer is returned which would + evaluate to false in an if-test. + Example if (std.strstr(req.url, req.http.x-restrict)) diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 898e8dd..a3635e9 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -242,11 +242,12 @@ vmod_cache_req_body(const struct vrt_ctx *ctx, VCL_BYTES size) VCL_STRING __match_proto__(td_std_strstr) vmod_strstr(const struct vrt_ctx *ctx, VCL_STRING mstr, VCL_STRING msubstr) -{ - +{ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - char *match = strstr(mstr, msubstr); - return(match); + if ((mstr == NULL) || (msubstr == NULL)) + return (NULL); + else + return(strstr(mstr, msubstr)); } From phk at FreeBSD.org Mon Sep 22 09:44:24 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 22 Sep 2014 11:44:24 +0200 Subject: [master] 5125065 Use a bigger buffer for the stdout/stderr to avoid annoying linebreaks in panic messages. Message-ID: commit 5125065cc868a069cce3b2f4b09f0d5a217a2fd9 Author: Poul-Henning Kamp Date: Mon Sep 22 09:35:55 2014 +0000 Use a bigger buffer for the stdout/stderr to avoid annoying linebreaks in panic messages. diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 00c96c6..c0be978 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -328,7 +328,7 @@ static void * varnish_thread(void *priv) { struct varnish *v; - char buf[BUFSIZ]; + char buf[65536]; struct pollfd *fds, fd; int i; From phk at FreeBSD.org Mon Sep 22 09:44:24 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 22 Sep 2014 11:44:24 +0200 Subject: [master] c522f7a Record the start and end of the worker thread stack. Message-ID: commit c522f7af7abaf8d847d62f8edaa46a2f5c0178e3 Author: Poul-Henning Kamp Date: Mon Sep 22 09:44:03 2014 +0000 Record the start and end of the worker thread stack. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index b4e79c4..289ea28 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -367,6 +367,9 @@ struct worker { unsigned cur_method; unsigned seen_methods; unsigned handling; + + uintptr_t stack_start; + uintptr_t stack_end; }; /* LRU ---------------------------------------------------------------*/ @@ -1020,7 +1023,7 @@ void PipeRequest(struct req *req, struct busyobj *bo); /* cache_pool.c */ void Pool_Init(void); void Pool_Accept(void); -void Pool_Work_Thread(void *priv, struct worker *w); +void Pool_Work_Thread(struct pool *, struct worker *w); int Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how); void Pool_Sumstat(struct worker *w); void Pool_PurgeStat(unsigned nobj); @@ -1136,7 +1139,7 @@ void WAIT_Write_Session(struct sess *sp, int fd); /* cache_wrk.c */ -void *WRK_thread(void *priv); +void WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace); typedef void *bgthread_t(struct worker *, void *priv); void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv); diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index c666d8a..625f314 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -277,6 +277,8 @@ pan_wrk(const struct worker *wrk) const char *p; VSB_printf(pan_vsp, " worker = %p {\n", wrk); + VSB_printf(pan_vsp, " stack = {0x%jx -> 0x%jx}\n", + wrk->stack_start, wrk->stack_end); pan_ws(wrk->aws, 4); m = wrk->cur_method; diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 77524b3..916c4a3 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -348,14 +348,13 @@ pool_stat_summ(struct worker *wrk, void *priv) */ void -Pool_Work_Thread(void *priv, struct worker *wrk) +Pool_Work_Thread(struct pool *pp, struct worker *wrk) { - struct pool *pp; struct pool_task *tp; struct pool_task tps; int i; - CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC); + CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); wrk->pool = pp; while (1) { Lck_Lock(&pp->mtx); @@ -416,15 +415,48 @@ Pool_Work_Thread(void *priv, struct worker *wrk) } /*-------------------------------------------------------------------- - * Create another thread. + * Create another worker thread. */ +struct pool_info { + unsigned magic; +#define POOL_INFO_MAGIC 0x4e4442d3 + size_t stacksize; + struct pool *qp; +}; + +static void * +pool_thread(void *priv) +{ + struct pool_info *pi; + + CAST_OBJ_NOTNULL(pi, priv, POOL_INFO_MAGIC); + WRK_Thread(pi->qp, pi->stacksize, cache_param->workspace_thread); + FREE_OBJ(pi); + return (NULL); +} + static void -pool_breed(struct pool *qp, const pthread_attr_t *tp_attr) +pool_breed(struct pool *qp) { pthread_t tp; + pthread_attr_t tp_attr; + struct pool_info *pi; + + AZ(pthread_attr_init(&tp_attr)); + AZ(pthread_attr_setdetachstate(&tp_attr, PTHREAD_CREATE_DETACHED)); + + /* Set the stacksize for worker threads we create */ + if (cache_param->wthread_stacksize != UINT_MAX) + AZ(pthread_attr_setstacksize(&tp_attr, + cache_param->wthread_stacksize)); + + ALLOC_OBJ(pi, POOL_INFO_MAGIC); + AN(pi); + AZ(pthread_attr_getstacksize(&tp_attr, &pi->stacksize)); + pi->qp = qp; - if (pthread_create(&tp, tp_attr, WRK_thread, qp)) { + if (pthread_create(&tp, &tp_attr, pool_thread, pi)) { VSL(SLT_Debug, 0, "Create worker thread failed %d %s", errno, strerror(errno)); Lck_Lock(&pool_mtx); @@ -432,7 +464,6 @@ pool_breed(struct pool *qp, const pthread_attr_t *tp_attr) Lck_Unlock(&pool_mtx); VTIM_sleep(cache_param->wthread_fail_delay); } else { - AZ(pthread_detach(tp)); qp->dry = 0; qp->nthr++; Lck_Lock(&pool_mtx); @@ -441,6 +472,8 @@ pool_breed(struct pool *qp, const pthread_attr_t *tp_attr) Lck_Unlock(&pool_mtx); VTIM_sleep(cache_param->wthread_add_delay); } + + AZ(pthread_attr_destroy(&tp_attr)); } /*-------------------------------------------------------------------- @@ -463,27 +496,16 @@ pool_herder(void *priv) { struct pool *pp; struct pool_task *pt; - pthread_attr_t tp_attr; double t_idle; struct worker *wrk; CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC); - AZ(pthread_attr_init(&tp_attr)); while (1) { - /* Set the stacksize for worker threads we create */ - if (cache_param->wthread_stacksize != UINT_MAX) - AZ(pthread_attr_setstacksize(&tp_attr, - cache_param->wthread_stacksize)); - else { - AZ(pthread_attr_destroy(&tp_attr)); - AZ(pthread_attr_init(&tp_attr)); - } - /* Make more threads if needed and allowed */ if (pp->nthr < cache_param->wthread_min || (pp->dry && pp->nthr < cache_param->wthread_max)) { - pool_breed(pp, &tp_attr); + pool_breed(pp); continue; } diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 38a662b..26930ff 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -32,6 +32,7 @@ #include "config.h" #include +#include #include #include "cache.h" @@ -84,11 +85,16 @@ WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv) /*--------------------------------------------------------------------*/ -static void * -wrk_thread_real(void *priv, unsigned thread_workspace) +void +WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace) { struct worker *w, ww; unsigned char ws[thread_workspace]; + uintptr_t u; + + AN(qp); + AN(stacksize); + AN(thread_workspace); THR_SetName("cache-worker"); w = &ww; @@ -99,9 +105,17 @@ wrk_thread_real(void *priv, unsigned thread_workspace) WS_Init(w->aws, "wrk", ws, thread_workspace); + u = getpagesize(); + AN(u); + u -= 1U; + w->stack_start = (((uintptr_t)&qp) + u) & ~u; + + /* XXX: assuming stack grows down. */ + w->stack_end = w->stack_start - stacksize; + VSL(SLT_WorkThread, 0, "%p start", w); - Pool_Work_Thread(priv, w); + Pool_Work_Thread(qp, w); AZ(w->pool); VSL(SLT_WorkThread, 0, "%p end", w); @@ -112,12 +126,4 @@ wrk_thread_real(void *priv, unsigned thread_workspace) VBO_Free(&w->nbo); HSH_Cleanup(w); Pool_Sumstat(w); - return (NULL); -} - -void * -WRK_thread(void *priv) -{ - - return (wrk_thread_real(priv, cache_param->workspace_thread)); } From lkarsten at varnish-software.com Mon Sep 22 14:38:21 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:21 +0200 Subject: [4.0] 0bf6512 White space nit Message-ID: commit 0bf651265f867bf506a922211e8e0b99034d08d6 Author: Poul-Henning Kamp Date: Tue Jun 24 10:44:03 2014 +0000 White space nit diff --git a/bin/varnishtest/tests/c00067.vtc b/bin/varnishtest/tests/c00067.vtc index 73bcb34..d6d4c02 100644 --- a/bin/varnishtest/tests/c00067.vtc +++ b/bin/varnishtest/tests/c00067.vtc @@ -34,7 +34,7 @@ varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend { sub vcl_recv { C{ VRT_CacheReqBody(ctx, 1000); }C } -} +} client c1 { txreq -req POST -nolen -hdr "Transfer-encoding: chunked" From lkarsten at varnish-software.com Mon Sep 22 14:38:21 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:21 +0200 Subject: [4.0] ab252f0 Explicitly round down for timestamps Message-ID: commit ab252f042c612ac14384e09c437103489fb02ce3 Author: Poul-Henning Kamp Date: Tue Jun 24 10:44:30 2014 +0000 Explicitly round down for timestamps diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index 8ca0dd7..cdcd3cb 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -209,7 +209,7 @@ vmod_time2integer(const struct vrt_ctx *ctx, VCL_TIME t) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - return (t); + return ((long)floor(t)); } VCL_REAL __match_proto__(td_std_time2real) From lkarsten at varnish-software.com Mon Sep 22 14:38:21 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:21 +0200 Subject: [4.0] 8d496cc I know it's the OCD talking here, but neither a fixed 32 param limit or an string-insertion-sort feels right for me. Message-ID: commit 8d496cc9c1434d3daec6952bcbfef3e9cb5639fe Author: Poul-Henning Kamp Date: Tue Jun 24 10:44:42 2014 +0000 I know it's the OCD talking here, but neither a fixed 32 param limit or an string-insertion-sort feels right for me. Using reserved workspace to tracking the params and qsort(3) to sort them solves both problems. diff --git a/lib/libvmod_std/vmod_std_querysort.c b/lib/libvmod_std/vmod_std_querysort.c index 8aac9df..8f7f93b 100644 --- a/lib/libvmod_std/vmod_std_querysort.c +++ b/lib/libvmod_std/vmod_std_querysort.c @@ -28,97 +28,105 @@ #include "config.h" +#include + #include "vrt.h" #include "cache/cache.h" #include "vcc_if.h" -#define QS_MAX_PARAM_COUNT 32 -#define QS_EQUALS(a, b) \ - ((a) == (b) || ((a) == '\0' && (b) == '&') || ((a) == '&' && (b) == '\0')) - -static ssize_t -param_compare(const char *s, const char *t) +static int +compa(const void *a, const void *b) { - for (; QS_EQUALS(*s, *t); s++, t++) { - if (*s == '&' || *s == '\0') - return (0); - } - return (*s - *t); -} - -static size_t -param_copy(char *dst, const char *src) -{ - size_t len; - len = strcspn(src, "&"); - memcpy(dst, src, len); - return (len); + const char * const *pa = a; + const char * const *pb = b; + const char *a1, *b1; + + for(a1 = pa[0], b1 = pb[0]; a1 < pa[1] && b1 < pb[1]; a1++, b1++) + if (*a1 != *b1) + return (*a1 - *b1); + return (0); } VCL_STRING __match_proto__(td_std_querysort) vmod_querysort(const struct vrt_ctx *ctx, VCL_STRING url) { - char *param, *params[QS_MAX_PARAM_COUNT]; + const char *cq, *cu; char *p, *r; - size_t len; - int param_count; - int i, n; + const char **pp; + const char **pe; + int np; + int i; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (url == NULL) return (NULL); - p = strchr(url, '?'); - if (p == NULL) + /* Split :query from :url */ + cu = strchr(url, '?'); + if (cu == NULL) return (url); - param_count = 0; - params[param_count++] = ++p; - len = p - url; - - while ((p = strchr(p, '&')) != NULL) { - param = ++p; - - for (i = 0; i < param_count; i++) { - if (param[0] < params[i][0] || - param_compare(param, params[i]) < 0) { - for (n = param_count; n > i; n--) - params[n] = params[n - 1]; - break; - } - } - params[i] = param; - param_count++; - - if (param_count == QS_MAX_PARAM_COUNT) - return (url); - } - - if (param_count == 1) + /* Spot single-param queries */ + cq = strchr(cu, '&'); + if (cq == NULL) return (url); - r = WS_Alloc(ctx->ws, strchr(param, '\0') - url + 1); + r = WS_Copy(ctx->ws, url, -1); if (r == NULL) return (url); - p = memcpy(r, url, len); - p += len; + (void)WS_Reserve(ctx->ws, 0); + /* We trust cache_ws.c to align sensibly */ + pp = (const char**)(void*)(ctx->ws->f); + pe = (const char**)(void*)(ctx->ws->e); - for (i = 0; i < param_count - 1; i++) { - if (params[i][0] != '\0' && params[i][0] != '&') - break; + if (pp + 4 > pe) { + WS_Release(ctx->ws, 0); + WS_MarkOverflow(ctx->ws); + return (url); } - for (; i < param_count - 1; i++) { - p += param_copy(p, params[i]); - *p++ = '&'; + /* Collect params as pointer pairs */ + np = 0; + pp[np++] = 1 + cu; + for (cq = 1 + cu; *cq != '\0'; cq++) { + if (*cq == '&') { + if (pp + 3 > pe) { + WS_Release(ctx->ws, 0); + WS_MarkOverflow(ctx->ws); + return (url); + } + pp[np++] = cq; + /* Skip trivially empty params */ + while(cq[1] == '&') + cq++; + pp[np++] = cq + 1; + } + } + pp[np++] = cq; + assert(!(np & 1)); + + qsort(pp, np / 2, sizeof(*pp) * 2, compa); + + /* Emit sorted params */ + p = 1 + r + (cu - url); + cq = ""; + for (i = 0; i < np; i += 2) { + /* Ignore any edge-case zero length params */ + if (pp[i + 1] == pp[i]) + continue; + assert(pp[i + 1] > pp[i]); + if (*cq) + *p++ = *cq; + memcpy(p, pp[i], pp[i + 1] - pp[i]); + p += pp[i + 1] - pp[i]; + cq = "&"; } - - p += param_copy(p, params[i]); *p = '\0'; + WS_Release(ctx->ws, 0); return (r); } From lkarsten at varnish-software.com Mon Sep 22 14:38:21 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:21 +0200 Subject: [4.0] 3900cc1 Remove comment which breaks markup Message-ID: commit 3900cc1aec0c06bf757ead9667dbe87d837cb47d Author: Andreas Plesner Date: Tue Jun 24 12:56:42 2014 +0200 Remove comment which breaks markup diff --git a/doc/sphinx/installation/help.rst b/doc/sphinx/installation/help.rst index 897324a..2d04c5f 100644 --- a/doc/sphinx/installation/help.rst +++ b/doc/sphinx/installation/help.rst @@ -92,8 +92,6 @@ The following companies offer commercial Varnish support, and are listed here for your convenience. If you want your company listed here, drop an email to phk at FreeBSD.org. -.. XXX: Should we perhaps enhance this to explain Varnish Plus? benc - Varnish Software sales at varnish-software.com From lkarsten at varnish-software.com Mon Sep 22 14:38:21 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:21 +0200 Subject: [4.0] 30a8ad0 A couple more corner cases Message-ID: commit 30a8ad0a3265f4d02444ebceafab0c2088916a47 Author: Poul-Henning Kamp Date: Tue Jun 24 10:44:15 2014 +0000 A couple more corner cases diff --git a/bin/varnishtest/tests/m00014.vtc b/bin/varnishtest/tests/m00014.vtc index 260280f..28c4c79 100644 --- a/bin/varnishtest/tests/m00014.vtc +++ b/bin/varnishtest/tests/m00014.vtc @@ -1,6 +1,6 @@ varnishtest "Test std.querysort" -server s1 -repeat 5 { +server s1 -repeat 6 { rxreq txresp } -start @@ -18,18 +18,30 @@ client c1 { rxresp expect resp.http.url == "/foo/bar?b=0&c=5&p=0&t=0" + delay .1 + txreq -url "/foo/bar?coa=0&co=0" rxresp expect resp.http.url == "/foo/bar?co=0&coa=0" + delay .1 + txreq -url "/foo/bar?a=0&&&&&" rxresp expect resp.http.url == "/foo/bar?a=0" + txreq -url "/foo/bar?&a=0&&&&&z&w&x&" + rxresp + expect resp.http.url == "/foo/bar?a=0&w&x&z" + + delay .1 + txreq -url "/foo/bar?&" rxresp expect resp.http.url == "/foo/bar?" + delay .1 + txreq -url "/foo/bar" rxresp expect resp.http.url == "/foo/bar" From lkarsten at varnish-software.com Mon Sep 22 14:38:21 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:21 +0200 Subject: [4.0] c4d99ff Message 850 is not helpful Message-ID: commit c4d99ffe0392f751d09c6f2bc2c539d898b796f2 Author: Poul-Henning Kamp Date: Wed Jun 25 08:25:24 2014 +0000 Message 850 is not helpful diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 9ad0543..be84816 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -122,6 +122,9 @@ -e728 // Symbol ... not explicitly initialized -e716 // while(1) ... -e785 // Too few initializers for aggregate +-e850 // for loop index variable '___' whose type category is '___' + // is modified in body of the for loop that began at '___' + -esym(765, vcc_ProcAction) // could be made static -esym(759, vcc_ProcAction) // could be moved to module From lkarsten at varnish-software.com Mon Sep 22 14:38:21 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:21 +0200 Subject: [4.0] b1001cd Fix whitespace in compiled vcl Message-ID: commit b1001cd3a8663dc3e83b3a871583f4001f8bc5b4 Author: Poul-Henning Kamp Date: Wed Jun 25 10:02:36 2014 +0000 Fix whitespace in compiled vcl diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c index 080f50e..9b1d22d 100644 --- a/lib/libvcc/vcc_action.c +++ b/lib/libvcc/vcc_action.c @@ -291,10 +291,10 @@ parse_hash_data(struct vcc *tl) vcc_NextToken(tl); SkipToken(tl, '('); - Fb(tl, 1, "VRT_hashdata(ctx, "); + Fb(tl, 1, "VRT_hashdata(ctx,\n "); vcc_Expr(tl, STRING_LIST); ERRCHK(tl); - Fb(tl, 0, ");\n"); + Fb(tl, 1, ");\n"); SkipToken(tl, ')'); } From lkarsten at varnish-software.com Mon Sep 22 14:38:21 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:21 +0200 Subject: [4.0] 02cf2c5 Disassociate obj->http from bo->ws once the object is setup Message-ID: commit 02cf2c5b60ef57311db84bfc2f4c21544646d987 Author: Poul-Henning Kamp Date: Wed Jun 25 10:17:22 2014 +0000 Disassociate obj->http from bo->ws once the object is setup diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 3d1fbfe..e5c7921 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -164,6 +164,9 @@ vbf_beresp2obj(struct busyobj *bo) else obj->last_modified = floor(bo->exp.t_origin); + /* Disassociate the obj from the bo's workspace */ + hp2->ws = NULL; + return (0); } From lkarsten at varnish-software.com Mon Sep 22 14:38:21 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:21 +0200 Subject: [4.0] 7e8177b Split BuildRequires into multiple lines. Message-ID: commit 7e8177bc5bff1345b5b9f09e3d90c51af31ad0df Author: Lasse Karstensen Date: Fri Jul 4 15:11:26 2014 +0200 Split BuildRequires into multiple lines. Adhere to Fedora packaging standard. diff --git a/redhat/varnish.spec b/redhat/varnish.spec index 7e7e87f..8611e2e 100644 --- a/redhat/varnish.spec +++ b/redhat/varnish.spec @@ -17,7 +17,11 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) # To build from git, start with a make dist, see redhat/README.redhat # You will need at least automake autoconf libtool #BuildRequires: automake autoconf libtool -BuildRequires: ncurses-devel groff pcre-devel pkgconfig libedit-devel jemalloc-devel +BuildRequires: ncurses-devel +BuildRequires: pcre-devel +BuildRequires: pkgconfig +BuildRequires: libedit-devel +BuildRequires: jemalloc-devel BuildRequires: python-docutils >= 0.6 Requires: varnish-libs = %{version}-%{release} Requires: logrotate From lkarsten at varnish-software.com Mon Sep 22 14:38:21 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:21 +0200 Subject: [4.0] 71b6a5d Correct AC_CHECK_PROGS usage Message-ID: commit 71b6a5da74278a7814459a48a2ebbd6dbf60f5d5 Author: Federico G. Schwindt Date: Tue Jul 8 10:59:20 2014 +0100 Correct AC_CHECK_PROGS usage diff --git a/configure.ac b/configure.ac index 118f743..ced8495 100644 --- a/configure.ac +++ b/configure.ac @@ -321,7 +321,10 @@ else fi AM_MISSING_HAS_RUN -AC_CHECK_PROGS(PYTHON, [python3 python3.1 python3.2 python2.7 python2.6 python2.5 python2 python], [AC_MSG_ERROR([Python is needed to build Varnish, please install python.])]) +AC_CHECK_PROGS(PYTHON, [python3 python3.1 python3.2 python2.7 python2.6 python2.5 python2 python], "no") +if test "x$PYTHON" = "xno"; then + AC_MSG_ERROR([Python is needed to build Varnish, please install python.]) +fi AC_CHECK_DECL([SO_ACCEPTFILTER], AC_DEFINE(HAVE_ACCEPT_FILTERS,1,[Define to 1 if you have accept filters]), From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] 7233081 Sort the list of build requirements. Message-ID: commit 7233081cc87f539e736a8f1e8c7f0524d116b538 Author: Lasse Karstensen Date: Wed Jul 9 12:54:21 2014 +0200 Sort the list of build requirements. diff --git a/redhat/varnish.spec b/redhat/varnish.spec index 8611e2e..eda06ba 100644 --- a/redhat/varnish.spec +++ b/redhat/varnish.spec @@ -14,21 +14,21 @@ URL: http://www.varnish-cache.org/ Source0: %{name}-%{version}%{?vd_rc}.tar.gz #Source0: %{name}-trunk.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) -# To build from git, start with a make dist, see redhat/README.redhat -# You will need at least automake autoconf libtool -#BuildRequires: automake autoconf libtool +BuildRequires: automake +BuildRequires: autoconf +BuildRequires: jemalloc-devel +BuildRequires: libedit-devel +BuildRequires: libtool BuildRequires: ncurses-devel BuildRequires: pcre-devel BuildRequires: pkgconfig -BuildRequires: libedit-devel -BuildRequires: jemalloc-devel BuildRequires: python-docutils >= 0.6 -Requires: varnish-libs = %{version}-%{release} +Requires: jemalloc +Requires: libedit Requires: logrotate Requires: ncurses Requires: pcre -Requires: libedit -Requires: jemalloc +Requires: varnish-libs = %{version}-%{release} Requires(pre): shadow-utils Requires(post): /sbin/chkconfig, /usr/bin/uuidgen Requires(preun): /sbin/chkconfig From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] 9246cfa Update list of documented build requirements. Message-ID: commit 9246cfa99ef37ed117580beb7c7e52d652a6fab9 Author: Lasse Karstensen Date: Wed Jul 9 12:59:37 2014 +0200 Update list of documented build requirements. The list of documented build requirement have not been updated for a while, and was outdated. Update the lists with recent data. Added (commented-out) commands that extracts and formats this information directly from the packaging files. Reported by: frakt diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst index 883cbf9..c9dd0c5 100644 --- a/doc/sphinx/installation/install.rst +++ b/doc/sphinx/installation/install.rst @@ -76,21 +76,21 @@ Build dependencies on Debian / Ubuntu In order to build Varnish from source you need a number of packages installed. On a Debian or Ubuntu system these are: -* `autoconf` -* `automake1.1` +.. grep-dctrl -n -sBuild-Depends -r ^ ../../../../varnish-cache-debian/control | tr -d '\n' | awk -F,\ '{ for (i = 0; ++i <= NF;) { sub (/ .*/, "", $i); print "* `" $i "`"; }}' + +* `automake` * `autotools-dev` -* `groff-base` -* `make` +* `debhelper` * `libedit-dev` +* `libjemalloc-dev` * `libncurses-dev` * `libpcre3-dev` +* `libreadline-dev` * `libtool` * `pkg-config` * `python-docutils` +* `python-sphinx` -If you're building from git, you also need the following: - -* `python-sphinx` (optional, if you want to build the documentation) Build dependencies on Red Hat / CentOS -------------------------------------- @@ -98,19 +98,19 @@ Build dependencies on Red Hat / CentOS To build Varnish on a Red Hat or CentOS system you need the following packages installed: -* `automake` +.. gawk '/^BuildRequires/ {print "* `" $2 "`"}' ../../../redhat/varnish.spec | sort | uniq + * `autoconf` -* `groff` +* `automake` +* `jemalloc-devel` * `libedit-devel` * `libtool` * `ncurses-devel` * `pcre-devel` * `pkgconfig` * `python-docutils` +* `systemd-units` -If you're building from git, you also need the following: - -* `python-sphinx` (optional, if you want to build the documentation) Compiling Varnish ----------------- From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] d8a5c8b Filter the list and add forgotten python-sphinx. Message-ID: commit d8a5c8b9d3ebce1bda2a70290c148f721f9fdb31 Author: Lasse Karstensen Date: Wed Jul 9 14:56:07 2014 +0200 Filter the list and add forgotten python-sphinx. We only need debhelper when building the debian packages, and the systemd-units is within a conditional that awk ignores. Updated the commands to not include these in the future, and also the Redhat spec file so it should work on a fresh installation. Noticed by: fgs diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst index c9dd0c5..d93e2b5 100644 --- a/doc/sphinx/installation/install.rst +++ b/doc/sphinx/installation/install.rst @@ -76,11 +76,10 @@ Build dependencies on Debian / Ubuntu In order to build Varnish from source you need a number of packages installed. On a Debian or Ubuntu system these are: -.. grep-dctrl -n -sBuild-Depends -r ^ ../../../../varnish-cache-debian/control | tr -d '\n' | awk -F,\ '{ for (i = 0; ++i <= NF;) { sub (/ .*/, "", $i); print "* `" $i "`"; }}' +.. grep-dctrl -n -sBuild-Depends -r ^ ../../../../varnish-cache-debian/control | tr -d '\n' | awk -F,\ '{ for (i = 0; ++i <= NF;) { sub (/ .*/, "", $i); print "* `" $i "`"; }}' | egrep -v '(debhelper)' * `automake` * `autotools-dev` -* `debhelper` * `libedit-dev` * `libjemalloc-dev` * `libncurses-dev` @@ -98,7 +97,7 @@ Build dependencies on Red Hat / CentOS To build Varnish on a Red Hat or CentOS system you need the following packages installed: -.. gawk '/^BuildRequires/ {print "* `" $2 "`"}' ../../../redhat/varnish.spec | sort | uniq +.. gawk '/^BuildRequires/ {print "* `" $2 "`"}' ../../../redhat/varnish.spec | sort | uniq | egrep -v '(systemd)' * `autoconf` * `automake` @@ -109,7 +108,7 @@ packages installed: * `pcre-devel` * `pkgconfig` * `python-docutils` -* `systemd-units` +* `python-sphinx` Compiling Varnish diff --git a/redhat/varnish.spec b/redhat/varnish.spec index eda06ba..df75267 100644 --- a/redhat/varnish.spec +++ b/redhat/varnish.spec @@ -23,6 +23,7 @@ BuildRequires: ncurses-devel BuildRequires: pcre-devel BuildRequires: pkgconfig BuildRequires: python-docutils >= 0.6 +BuildRequires: python-sphinx Requires: jemalloc Requires: libedit Requires: logrotate From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] a8d7884 Readline does not belong on this list. Message-ID: commit a8d788465705259431177c3bd3de8a36a1d40e35 Author: Lasse Karstensen Date: Wed Jul 9 15:12:29 2014 +0200 Readline does not belong on this list. This reflects the debian build files, editline will suffice. diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst index d93e2b5..bbae1e6 100644 --- a/doc/sphinx/installation/install.rst +++ b/doc/sphinx/installation/install.rst @@ -84,7 +84,6 @@ installed. On a Debian or Ubuntu system these are: * `libjemalloc-dev` * `libncurses-dev` * `libpcre3-dev` -* `libreadline-dev` * `libtool` * `pkg-config` * `python-docutils` From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] 1d2edc2 Remove nitpicking mode (-n) for sphinx-build. Message-ID: commit 1d2edc2cf0e8d30dbdf76030f3deeb3ad58b64d7 Author: Lasse Karstensen Date: Tue Jul 15 14:28:12 2014 +0200 Remove nitpicking mode (-n) for sphinx-build. sphinx-build v0.6.6 which is on RHEL6/CentOS6 does not know about -n, so we'll have to do without it for now. Reported in: #1535 diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index f1b9bfc..0d681e8 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -3,7 +3,7 @@ # You can set these variables from the command line. SPHINXOPTS = -SPHINXBUILD = sphinx-build -W -N -n +SPHINXBUILD = sphinx-build -W -N PAPER = a4 BUILDDIR = build From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] a32c45f systemd support for rhel7 Message-ID: commit a32c45f07a40287ac97fb3021934233829387549 Author: Federico G. Schwindt Date: Fri Jul 18 19:22:29 2014 +0100 systemd support for rhel7 Submitted by: Gauthier Delacroix via github diff --git a/redhat/varnish.spec b/redhat/varnish.spec index df75267..ad2a914 100644 --- a/redhat/varnish.spec +++ b/redhat/varnish.spec @@ -37,7 +37,7 @@ Requires(preun): /sbin/service %if %{undefined suse_version} Requires(preun): initscripts %endif -%if 0%{?fedora} >= 17 +%if 0%{?fedora} >= 17 || 0%{?rhel} >= 7 Requires(post): systemd-units Requires(post): systemd-sysv Requires(preun): systemd-units @@ -168,7 +168,7 @@ install -D -m 0644 etc/example.vcl %{buildroot}%{_sysconfdir}/varnish/default.vc install -D -m 0644 redhat/varnish.logrotate %{buildroot}%{_sysconfdir}/logrotate.d/varnish # systemd support -%if 0%{?fedora} >= 17 +%if 0%{?fedora} >= 17 || 0%{?rhel} >= 7 mkdir -p %{buildroot}%{_unitdir} install -D -m 0644 redhat/varnish.service %{buildroot}%{_unitdir}/varnish.service install -D -m 0644 redhat/varnish.params %{buildroot}%{_sysconfdir}/varnish/varnish.params @@ -204,8 +204,8 @@ rm -rf %{buildroot} %config(noreplace) %{_sysconfdir}/varnish/default.vcl %config(noreplace) %{_sysconfdir}/logrotate.d/varnish -# systemd from fedora 17 -%if 0%{?fedora} >= 17 +# systemd from fedora 17 and rhel 7 +%if 0%{?fedora} >= 17 || 0%{?rhel} >= 7 %{_unitdir}/varnish.service %{_unitdir}/varnishncsa.service %{_unitdir}/varnishlog.service @@ -259,7 +259,7 @@ getent passwd varnish >/dev/null || \ exit 0 %post -%if 0%{?fedora} >= 17 +%if 0%{?fedora} >= 17 || 0%{?rhel} >= 7 /bin/systemctl daemon-reload >/dev/null 2>&1 || : %else /sbin/chkconfig --add varnish @@ -284,7 +284,7 @@ test -f /etc/varnish/secret || (uuidgen > /etc/varnish/secret && chmod 0600 /etc %preun if [ $1 -lt 1 ]; then # Package removal, not upgrade - %if 0%{?fedora} >= 17 + %if 0%{?fedora} >= 17 || 0%{?rhel} >= 7 /bin/systemctl --no-reload disable varnish.service > /dev/null 2>&1 || : /bin/systemctl stop varnish.service > /dev/null 2>&1 || : %else From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] db226a9 Purge has changed. Message-ID: commit db226a9ebce0964b58d106ed37d92464a8cdb341 Author: Lasse Karstensen Date: Tue Jul 22 11:31:13 2014 +0200 Purge has changed. diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst index f48548f..c02ae0f 100644 --- a/doc/sphinx/whats-new/upgrading.rst +++ b/doc/sphinx/whats-new/upgrading.rst @@ -136,6 +136,12 @@ client.port, and server.port replaced by respectively std.port(client.ip) and st as an IP address by default. You need to use the `std.port()` function to get the port number. +Invalidation with purge +~~~~~~~~~~~~~~~~~~~~~~~ + +Cache invalidation with purges is now done via `return(purge)` from `vcl_recv`. +The `purge;` keyword has been retired. + obj is now read-only ~~~~~~~~~~~~~~~~~~~~ From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] 016d66a Document the new return(retry). Message-ID: commit 016d66a084a9b2c2840135fce300962d1f1127f6 Author: Lasse Karstensen Date: Tue Jul 22 11:28:22 2014 +0200 Document the new return(retry). Backend restarts are now return(retry). Fixes: #1503 diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst index c02ae0f..cb36b08 100644 --- a/doc/sphinx/whats-new/upgrading.rst +++ b/doc/sphinx/whats-new/upgrading.rst @@ -164,6 +164,18 @@ following has changed: - `vcl_hash` must now return `lookup` instead of `hash` - `vcl_pass` must now return `fetch` instead of `pass` + +Backend restarts are now retry +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In 3.0 it was possible to do `return(restart)` after noticing that +the backend response was wrong, to change to a different backend. + +This is now called `return(retry)`, and jumps back up to `vcl_backend_fetch`. + +This only influences the backend fetch thread, client-side handling is not affected. + + default/builtin VCL changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~ From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] 40b8066 Non-breaking space broke the control sequence. Message-ID: commit 40b8066d99cfafe16b458b5dd9c80537eb5c48e9 Author: Lasse Karstensen Date: Tue Jul 22 13:10:42 2014 +0200 Non-breaking space broke the control sequence. diff --git a/redhat/varnish.initrc b/redhat/varnish.initrc index 0db3ab6..082c770 100755 --- a/redhat/varnish.initrc +++ b/redhat/varnish.initrc @@ -61,7 +61,7 @@ start() { ulimit -u ${NPROCS:-unlimited} # If defined, set maximum core size. - if [ -n?"${DAEMON_COREFILE_LIMIT}" ] + if [ -n "${DAEMON_COREFILE_LIMIT}" ] then ulimit -c ${DAEMON_COREFILE_LIMIT} fi From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] a68f58a Give the CLI command vcl.show a -v flag which outputs all the source files involved in a given VCL. Message-ID: commit a68f58ab76fa0572af14619e588c952b76e90a31 Author: Poul-Henning Kamp Date: Thu Jul 24 07:00:26 2014 +0000 Give the CLI command vcl.show a -v flag which outputs all the source files involved in a given VCL. When using -v, the individual source files are output like this: // VCL.SHOW %d %d %s\n%s\n First field is the source file index [0...]. Second field is the number of bytes in the source file. Third field is the name of the source file. Fourth field is the source file content. Inspired by a patch from daghf diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index e6dd179..63e8781 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -372,6 +372,40 @@ ccf_config_use(struct cli *cli, const char * const *av, void *priv) VBE_UseHealth(vcl->conf->director[i]); } +static void +ccf_config_show(struct cli *cli, const char * const *av, void *priv) +{ + struct vcls *vcl; + int verbose = 0; + int i; + + (void)priv; + if (!strcmp(av[2], "-v")) { + verbose = 1; + vcl = vcl_find(av[3]); + } else if (av[3] != NULL) { + VCLI_Out(cli, "Unknown options '%s'", av[2]); + VCLI_SetResult(cli, CLIS_PARAM); + return; + } else + vcl = vcl_find(av[2]); + + if (vcl == NULL) { + VCLI_Out(cli, "No VCL named '%s'", av[2]); + VCLI_SetResult(cli, CLIS_PARAM); + return; + } + if (verbose) { + for (i = 0; i < vcl->conf->nsrc; i++) + VCLI_Out(cli, "// VCL.SHOW %d %zd %s\n%s\n", + i, strlen(vcl->conf->srcbody[i]), + vcl->conf->srcname[i], + vcl->conf->srcbody[i]); + } else { + VCLI_Out(cli, "%s", vcl->conf->srcbody[0]); + } +} + /*-------------------------------------------------------------------- * Method functions to call into VCL programs. * @@ -451,6 +485,7 @@ static struct cli_proto vcl_cmds[] = { { CLI_VCL_LIST, "i", ccf_config_list }, { CLI_VCL_DISCARD, "i", ccf_config_discard }, { CLI_VCL_USE, "i", ccf_config_use }, + { CLI_VCL_SHOW, "i", ccf_config_show }, { NULL } }; diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c index 3d71a14..e7d7aa2 100644 --- a/bin/varnishd/mgt/mgt_cli.c +++ b/bin/varnishd/mgt/mgt_cli.c @@ -99,7 +99,6 @@ static struct cli_proto cli_proto[] = { { CLI_VCL_USE, "", mcf_config_use, NULL }, { CLI_VCL_DISCARD, "", mcf_config_discard, NULL }, { CLI_VCL_LIST, "", mcf_config_list, NULL }, - { CLI_VCL_SHOW, "", mcf_config_show, NULL }, { CLI_PARAM_SHOW, "", mcf_param_show, NULL }, { CLI_PARAM_SET, "", mcf_param_set, NULL }, { CLI_PANIC_SHOW, "", mcf_panic_show, NULL }, diff --git a/bin/varnishd/mgt/mgt_cli.h b/bin/varnishd/mgt/mgt_cli.h index 5d3ce8b..4d04535 100644 --- a/bin/varnishd/mgt/mgt_cli.h +++ b/bin/varnishd/mgt/mgt_cli.h @@ -44,7 +44,6 @@ cli_func_t mcf_config_inline; cli_func_t mcf_config_use; cli_func_t mcf_config_discard; cli_func_t mcf_config_list; -cli_func_t mcf_config_show; /* stevedore.c */ extern struct cli_proto cli_stv[]; diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 2c2ffd9..4d97de6 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -672,35 +672,3 @@ mcf_config_list(struct cli *cli, const char * const *av, void *priv) } } } - -/* - * XXX: This should take an option argument to show all (include) files - * XXX: This violates the principle of not loading VCL's in the master - * XXX: process. - */ -void -mcf_config_show(struct cli *cli, const char * const *av, void *priv) -{ - struct vclprog *vp; - void *dlh, *sym; - const char **src; - - (void)priv; - if ((vp = mcf_find_vcl(cli, av[2])) != NULL) { - if ((dlh = dlopen(vp->fname, RTLD_NOW | RTLD_LOCAL)) == NULL) { - VCLI_Out(cli, "failed to load %s: %s\n", - vp->name, dlerror()); - VCLI_SetResult(cli, CLIS_CANT); - } else if ((sym = dlsym(dlh, "srcbody")) == NULL) { - VCLI_Out(cli, "failed to locate source for %s: %s\n", - vp->name, dlerror()); - VCLI_SetResult(cli, CLIS_CANT); - AZ(dlclose(dlh)); - } else { - src = sym; - VCLI_Out(cli, "%s", src[0]); - /* VCLI_Out(cli, src[1]); */ - AZ(dlclose(dlh)); - } - } -} diff --git a/bin/varnishtest/tests/c00015.vtc b/bin/varnishtest/tests/c00015.vtc index 5b09c8d..63edd3c 100644 --- a/bin/varnishtest/tests/c00015.vtc +++ b/bin/varnishtest/tests/c00015.vtc @@ -50,5 +50,6 @@ client c3 { } -run varnish v1 -cli "vcl.show vcl2" +varnish v1 -cli "vcl.show -v vcl2" varnish v1 -cli "vcl.discard vcl2" varnish v1 -cli "vcl.list" diff --git a/include/vcli.h b/include/vcli.h index 3294dd7..f6b75da 100644 --- a/include/vcli.h +++ b/include/vcli.h @@ -99,7 +99,7 @@ "vcl.show", \ "vcl.show ", \ "\tDisplay the source code for the specified configuration.", \ - 1, 1 + 1, 2 #define CLI_VCL_USE \ "vcl.use", \ From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] 9d50fa6 Un-copy&paste leading whitespace in test-cases Message-ID: commit 9d50fa6e8e5d79a6d80605922455dc3889b3b19c Author: Poul-Henning Kamp Date: Thu Jul 24 07:28:20 2014 +0000 Un-copy&paste leading whitespace in test-cases diff --git a/bin/varnishtest/tests/a00010.vtc b/bin/varnishtest/tests/a00010.vtc index 60b0234..ce6b819 100644 --- a/bin/varnishtest/tests/a00010.vtc +++ b/bin/varnishtest/tests/a00010.vtc @@ -1,7 +1,7 @@ varnishtest "simply test that the framework support \0" server s1 { - rxreq + rxreq expect req.url == "/" txresp -body {a\0bc} } diff --git a/bin/varnishtest/tests/a00012.vtc b/bin/varnishtest/tests/a00012.vtc index 07e51c1..9b044fc 100644 --- a/bin/varnishtest/tests/a00012.vtc +++ b/bin/varnishtest/tests/a00012.vtc @@ -1,13 +1,13 @@ varnishtest "Ensure that we can test non-existence of headers (#1062)" server s1 { - rxreq - txresp + rxreq + txresp } -start client c1 -connect ${s1_sock} { - txreq - rxresp - expect resp.http.X-Test == + txreq + rxresp + expect resp.http.X-Test == } -run diff --git a/bin/varnishtest/tests/c00020.vtc b/bin/varnishtest/tests/c00020.vtc index ec54fe3..91fcd12 100644 --- a/bin/varnishtest/tests/c00020.vtc +++ b/bin/varnishtest/tests/c00020.vtc @@ -1,67 +1,67 @@ varnishtest "Test -h critbit a bit" server s1 { - rxreq + rxreq expect req.url == "/" - txresp -hdr "ID: slash" -hdr "Connection: close" -body "012345\n" + txresp -hdr "ID: slash" -hdr "Connection: close" -body "012345\n" } -start varnish v1 -arg "-hcritbit" -vcl+backend { } -start client c1 { - txreq -url "/" - rxresp - expect resp.status == 200 - expect resp.http.X-Varnish == "1001" - expect resp.http.ID == "slash" + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.http.X-Varnish == "1001" + expect resp.http.ID == "slash" } -run delay .1 client c2 { - txreq -url "/" - rxresp - expect resp.status == 200 - expect resp.http.X-Varnish == "1004 1002" - expect resp.http.ID == "slash" + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.http.X-Varnish == "1004 1002" + expect resp.http.ID == "slash" } -run delay .1 server s1 { - rxreq + rxreq expect req.url == "/foo" - txresp -hdr "ID: foo" -body "012345\n" - rxreq + txresp -hdr "ID: foo" -body "012345\n" + rxreq expect req.url == "/bar" - txresp -hdr "ID: bar" -body "012345\n" + txresp -hdr "ID: bar" -body "012345\n" } -start client c1 { - txreq -url "/foo" - rxresp - expect resp.status == 200 - expect resp.http.X-Varnish == "1006" - expect resp.http.ID == "foo" + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.http.X-Varnish == "1006" + expect resp.http.ID == "foo" delay .1 - txreq -url "/" - rxresp - expect resp.status == 200 - expect resp.http.X-Varnish == "1008 1002" - expect resp.http.ID == "slash" + txreq -url "/" + rxresp + expect resp.status == 200 + expect resp.http.X-Varnish == "1008 1002" + expect resp.http.ID == "slash" delay .1 - txreq -url "/bar" - rxresp - expect resp.status == 200 - expect resp.http.X-Varnish == "1009" - expect resp.http.ID == "bar" + txreq -url "/bar" + rxresp + expect resp.status == 200 + expect resp.http.X-Varnish == "1009" + expect resp.http.ID == "bar" delay .1 - txreq -url "/foo" - rxresp - expect resp.status == 200 - expect resp.http.X-Varnish == "1011 1007" - expect resp.http.ID == "foo" + txreq -url "/foo" + rxresp + expect resp.status == 200 + expect resp.http.X-Varnish == "1011 1007" + expect resp.http.ID == "foo" } -run varnish v1 -expect sess_conn == 3 diff --git a/bin/varnishtest/tests/c00023.vtc b/bin/varnishtest/tests/c00023.vtc index 11e8c72..f8b6a84 100644 --- a/bin/varnishtest/tests/c00023.vtc +++ b/bin/varnishtest/tests/c00023.vtc @@ -1,149 +1,149 @@ varnishtest "Test -h critbit for digest edges" server s1 { - rxreq + rxreq expect req.url == "/1" - txresp -body "\n" - rxreq + txresp -body "\n" + rxreq expect req.url == "/2" - txresp -body "x\n" - rxreq + txresp -body "x\n" + rxreq expect req.url == "/3" - txresp -body "xx\n" - rxreq + txresp -body "xx\n" + rxreq expect req.url == "/4" - txresp -body "xxx\n" - rxreq + txresp -body "xxx\n" + rxreq expect req.url == "/5" - txresp -body "xxxx\n" - rxreq + txresp -body "xxxx\n" + rxreq expect req.url == "/6" - txresp -body "xxxxx\n" - rxreq + txresp -body "xxxxx\n" + rxreq expect req.url == "/7" - txresp -body "xxxxxx\n" - rxreq + txresp -body "xxxxxx\n" + rxreq expect req.url == "/8" - txresp -body "xxxxxxx\n" - rxreq + txresp -body "xxxxxxx\n" + rxreq expect req.url == "/9" - txresp -body "xxxxxxxx\n" + txresp -body "xxxxxxxx\n" } -start varnish v1 -arg "-hcritbit" -vcl+backend { } -start varnish v1 -cliok "param.set debug +hashedge" client c1 { - txreq -url "/1" - rxresp - expect resp.status == 200 - expect resp.bodylen == 1 - expect resp.http.X-Varnish == "1001" - - txreq -url "/2" - rxresp - expect resp.bodylen == 2 - expect resp.status == 200 - expect resp.http.X-Varnish == "1003" - - txreq -url "/3" - rxresp - expect resp.bodylen == 3 - expect resp.status == 200 - expect resp.http.X-Varnish == "1005" - - txreq -url "/4" - rxresp - expect resp.bodylen == 4 - expect resp.status == 200 - expect resp.http.X-Varnish == "1007" - - txreq -url "/5" - rxresp - expect resp.bodylen == 5 - expect resp.status == 200 - expect resp.http.X-Varnish == "1009" - - txreq -url "/6" - rxresp - expect resp.bodylen == 6 - expect resp.status == 200 - expect resp.http.X-Varnish == "1011" - - txreq -url "/7" - rxresp - expect resp.bodylen == 7 - expect resp.status == 200 - expect resp.http.X-Varnish == "1013" - - txreq -url "/8" - rxresp - expect resp.bodylen == 8 - expect resp.status == 200 - expect resp.http.X-Varnish == "1015" - - txreq -url "/9" - rxresp - expect resp.bodylen == 9 - expect resp.status == 200 - expect resp.http.X-Varnish == "1017" + txreq -url "/1" + rxresp + expect resp.status == 200 + expect resp.bodylen == 1 + expect resp.http.X-Varnish == "1001" + + txreq -url "/2" + rxresp + expect resp.bodylen == 2 + expect resp.status == 200 + expect resp.http.X-Varnish == "1003" + + txreq -url "/3" + rxresp + expect resp.bodylen == 3 + expect resp.status == 200 + expect resp.http.X-Varnish == "1005" + + txreq -url "/4" + rxresp + expect resp.bodylen == 4 + expect resp.status == 200 + expect resp.http.X-Varnish == "1007" + + txreq -url "/5" + rxresp + expect resp.bodylen == 5 + expect resp.status == 200 + expect resp.http.X-Varnish == "1009" + + txreq -url "/6" + rxresp + expect resp.bodylen == 6 + expect resp.status == 200 + expect resp.http.X-Varnish == "1011" + + txreq -url "/7" + rxresp + expect resp.bodylen == 7 + expect resp.status == 200 + expect resp.http.X-Varnish == "1013" + + txreq -url "/8" + rxresp + expect resp.bodylen == 8 + expect resp.status == 200 + expect resp.http.X-Varnish == "1015" + + txreq -url "/9" + rxresp + expect resp.bodylen == 9 + expect resp.status == 200 + expect resp.http.X-Varnish == "1017" } -run client c1 { - txreq -url "/1" - rxresp - expect resp.status == 200 - expect resp.bodylen == 1 - expect resp.http.X-Varnish == "1020 1002" - - txreq -url "/2" - rxresp - expect resp.bodylen == 2 - expect resp.status == 200 - expect resp.http.X-Varnish == "1021 1004" - - txreq -url "/3" - rxresp - expect resp.bodylen == 3 - expect resp.status == 200 - expect resp.http.X-Varnish == "1022 1006" - - txreq -url "/4" - rxresp - expect resp.bodylen == 4 - expect resp.status == 200 - expect resp.http.X-Varnish == "1023 1008" - - txreq -url "/5" - rxresp - expect resp.bodylen == 5 - expect resp.status == 200 - expect resp.http.X-Varnish == "1024 1010" - - txreq -url "/6" - rxresp - expect resp.bodylen == 6 - expect resp.status == 200 - expect resp.http.X-Varnish == "1025 1012" - - txreq -url "/7" - rxresp - expect resp.bodylen == 7 - expect resp.status == 200 - expect resp.http.X-Varnish == "1026 1014" - - txreq -url "/8" - rxresp - expect resp.bodylen == 8 - expect resp.status == 200 - expect resp.http.X-Varnish == "1027 1016" - - txreq -url "/9" - rxresp - expect resp.bodylen == 9 - expect resp.status == 200 - expect resp.http.X-Varnish == "1028 1018" + txreq -url "/1" + rxresp + expect resp.status == 200 + expect resp.bodylen == 1 + expect resp.http.X-Varnish == "1020 1002" + + txreq -url "/2" + rxresp + expect resp.bodylen == 2 + expect resp.status == 200 + expect resp.http.X-Varnish == "1021 1004" + + txreq -url "/3" + rxresp + expect resp.bodylen == 3 + expect resp.status == 200 + expect resp.http.X-Varnish == "1022 1006" + + txreq -url "/4" + rxresp + expect resp.bodylen == 4 + expect resp.status == 200 + expect resp.http.X-Varnish == "1023 1008" + + txreq -url "/5" + rxresp + expect resp.bodylen == 5 + expect resp.status == 200 + expect resp.http.X-Varnish == "1024 1010" + + txreq -url "/6" + rxresp + expect resp.bodylen == 6 + expect resp.status == 200 + expect resp.http.X-Varnish == "1025 1012" + + txreq -url "/7" + rxresp + expect resp.bodylen == 7 + expect resp.status == 200 + expect resp.http.X-Varnish == "1026 1014" + + txreq -url "/8" + rxresp + expect resp.bodylen == 8 + expect resp.status == 200 + expect resp.http.X-Varnish == "1027 1016" + + txreq -url "/9" + rxresp + expect resp.bodylen == 9 + expect resp.status == 200 + expect resp.http.X-Varnish == "1028 1018" } -run varnish v1 -cliok "hcb.dump" diff --git a/bin/varnishtest/tests/c00024.vtc b/bin/varnishtest/tests/c00024.vtc index 80b3c59..0fd0034 100644 --- a/bin/varnishtest/tests/c00024.vtc +++ b/bin/varnishtest/tests/c00024.vtc @@ -1,8 +1,8 @@ varnishtest "Test restart in vcl_synth" server s1 { - rxreq - txresp + rxreq + txresp } -start varnish v1 -vcl+backend { @@ -21,7 +21,7 @@ varnish v1 -vcl+backend { } -start client c1 { - txreq -url "/" - rxresp - expect resp.status == 200 + txreq -url "/" + rxresp + expect resp.status == 200 } -run diff --git a/bin/varnishtest/tests/c00028.vtc b/bin/varnishtest/tests/c00028.vtc index 0ad1466..10caf43 100644 --- a/bin/varnishtest/tests/c00028.vtc +++ b/bin/varnishtest/tests/c00028.vtc @@ -14,7 +14,7 @@ varnish v1 -vcl { } -start client c1 { - txreq -url "/" - rxresp - expect resp.status == 503 + txreq -url "/" + rxresp + expect resp.status == 503 } -run diff --git a/bin/varnishtest/tests/c00037.vtc b/bin/varnishtest/tests/c00037.vtc index 48ea499..019ed0b 100644 --- a/bin/varnishtest/tests/c00037.vtc +++ b/bin/varnishtest/tests/c00037.vtc @@ -1,10 +1,10 @@ varnishtest "Test req.hash_always_miss in vcl_recv" server s1 { - rxreq - txresp -hdr "Inc: 1" - rxreq - txresp -hdr "Inc: 2" + rxreq + txresp -hdr "Inc: 1" + rxreq + txresp -hdr "Inc: 2" } -start varnish v1 -vcl+backend { diff --git a/bin/varnishtest/tests/e00008.vtc b/bin/varnishtest/tests/e00008.vtc index e72ec56..0345844 100644 --- a/bin/varnishtest/tests/e00008.vtc +++ b/bin/varnishtest/tests/e00008.vtc @@ -28,7 +28,7 @@ server s1 { 23 25 + > 25 26 27 28 diff --git a/bin/varnishtest/tests/e00012.vtc b/bin/varnishtest/tests/e00012.vtc index afc23c6..cb0a9e4 100644 --- a/bin/varnishtest/tests/e00012.vtc +++ b/bin/varnishtest/tests/e00012.vtc @@ -1,45 +1,45 @@ varnishtest "ESI includes for pre HTTP/1.1 cannot used chunked encoding" server s1 { - rxreq - expect req.url == "/foo/bar" - txresp -body { - - Before include - - After include - } - rxreq - expect req.url == "/foo/body" - txresp -body { - Included file - } + rxreq + expect req.url == "/foo/bar" + txresp -body { + + Before include + + After include + } + rxreq + expect req.url == "/foo/body" + txresp -body { + Included file + } } -start varnish v1 -vcl+backend { - sub vcl_backend_response { - set beresp.do_esi = true; - } + sub vcl_backend_response { + set beresp.do_esi = true; + } } -start client c1 { - txreq -url /foo/bar -proto HTTP/1.1 - rxresp - expect resp.status == 200 - expect resp.bodylen == 151 + txreq -url /foo/bar -proto HTTP/1.1 + rxresp + expect resp.status == 200 + expect resp.bodylen == 67 } -run client c1 { - txreq -url /foo/bar -proto HTTP/1.0 - rxresp - expect resp.status == 200 - expect resp.bodylen == 151 + txreq -url /foo/bar -proto HTTP/1.0 + rxresp + expect resp.status == 200 + expect resp.bodylen == 67 } -run client c1 { - txreq -url /foo/bar -proto "" - rxresp - expect resp.status == 200 - expect resp.bodylen == 151 + txreq -url /foo/bar -proto "" + rxresp + expect resp.status == 200 + expect resp.bodylen == 67 } -run varnish v1 -expect esi_errors == 0 diff --git a/bin/varnishtest/tests/e00013.vtc b/bin/varnishtest/tests/e00013.vtc index a8a6028..d6db08c 100644 --- a/bin/varnishtest/tests/e00013.vtc +++ b/bin/varnishtest/tests/e00013.vtc @@ -1,23 +1,23 @@ varnishtest "All white-space object, in multiple storage segments" server s1 { - rxreq - expect req.url == "/foo" - txresp -nolen -hdr "Connection: close" - send { } + rxreq + expect req.url == "/foo" + txresp -nolen -hdr "Connection: close" + send { } } -start varnish v1 -vcl+backend { - sub vcl_backend_response { - set beresp.do_esi = true; - } + sub vcl_backend_response { + set beresp.do_esi = true; + } } -start varnish v1 -cliok "debug.fragfetch 4" client c1 { - txreq -url /foo - rxresp + txreq -url /foo + rxresp } -run varnish v1 -expect esi_errors == 0 diff --git a/bin/varnishtest/tests/e00014.vtc b/bin/varnishtest/tests/e00014.vtc index ec6d16b..0097f13 100644 --- a/bin/varnishtest/tests/e00014.vtc +++ b/bin/varnishtest/tests/e00014.vtc @@ -1,25 +1,25 @@ varnishtest "Check } + send { } } -start varnish v1 -vcl+backend { - sub vcl_backend_response { - set beresp.do_esi = true; - } + sub vcl_backend_response { + set beresp.do_esi = true; + } } -start varnish v1 -cliok "debug.fragfetch 4" client c1 { - txreq -url /foo - rxresp + txreq -url /foo + rxresp expect resp.bodylen == 49 } -run diff --git a/bin/varnishtest/tests/e00015.vtc b/bin/varnishtest/tests/e00015.vtc index 3a41237..f5a948b 100644 --- a/bin/varnishtest/tests/e00015.vtc +++ b/bin/varnishtest/tests/e00015.vtc @@ -34,7 +34,7 @@ varnish v1 -vcl+backend { } -start client c1 { - txreq + txreq rxresp expect resp.bodylen == 73 expect resp.status == 200 diff --git a/bin/varnishtest/tests/e00016.vtc b/bin/varnishtest/tests/e00016.vtc index 7f0a71b..b760284 100644 --- a/bin/varnishtest/tests/e00016.vtc +++ b/bin/varnishtest/tests/e00016.vtc @@ -39,7 +39,7 @@ varnish v1 -vcl+backend { } -start client c1 { - txreq + txreq rxresp expect resp.bodylen == 105 expect resp.status == 200 diff --git a/bin/varnishtest/tests/r00878.vtc b/bin/varnishtest/tests/r00878.vtc index 8df049a..5725393 100644 --- a/bin/varnishtest/tests/r00878.vtc +++ b/bin/varnishtest/tests/r00878.vtc @@ -6,7 +6,7 @@ server s1 { } -start varnish v1 -vcl+backend { - import ${vmod_debug}; + import ${vmod_debug}; sub vcl_deliver { set resp.http.who = debug.author(phk); } @@ -18,7 +18,7 @@ client c1 { rxresp } -run varnish v1 -vcl+backend { - import ${vmod_debug}; + import ${vmod_debug}; sub vcl_deliver { set resp.http.who = debug.author(des); } @@ -30,7 +30,7 @@ client c1 { } -run varnish v1 -vcl+backend { - import ${vmod_debug}; + import ${vmod_debug}; sub vcl_deliver { set resp.http.who = debug.author(kristian); } diff --git a/bin/varnishtest/tests/r00965.vtc b/bin/varnishtest/tests/r00965.vtc index e094e3b..7ef8bc7 100644 --- a/bin/varnishtest/tests/r00965.vtc +++ b/bin/varnishtest/tests/r00965.vtc @@ -10,7 +10,7 @@ varnish v1 -vcl+backend { if (req.http.X-Banned == "check") { unset req.http.X-Banned; } elseif (req.restarts == 0) { - set req.http.X-Banned = "check"; + set req.http.X-Banned = "check"; if (req.http.x-pass) { return (pass); } else { diff --git a/bin/varnishtest/tests/r01145.vtc b/bin/varnishtest/tests/r01145.vtc index f87238d..cb26a2f 100644 --- a/bin/varnishtest/tests/r01145.vtc +++ b/bin/varnishtest/tests/r01145.vtc @@ -18,7 +18,7 @@ varnish v1 -vcl+backend { import ${vmod_std}; sub vcl_deliver { - set resp.http.foo = std.fileread("${tmpdir}" + req.url); + set resp.http.foo = std.fileread("${tmpdir}" + req.url); } } -start diff --git a/bin/varnishtest/tests/r01184.vtc b/bin/varnishtest/tests/r01184.vtc index 6eb07ed..8cfb741 100644 --- a/bin/varnishtest/tests/r01184.vtc +++ b/bin/varnishtest/tests/r01184.vtc @@ -9,21 +9,21 @@ server s1 { -hdr "Content-Encoding: gzip" \ -hdr "Transfer-Encoding: Chunked" send "ed\r\n" - sendhex " 1f 8b 08 00 c2 39 33 50 02 03 45 90 4d 6a 03 31" - sendhex " 0c 85 f7 73 8a 47 2e 10 d9 f2 9f ca 34 d0 c2 64" - sendhex " 5b 08 bd 80 2d cb 10 28 5d 34 f4 fe f5 30 d0 68" - sendhex " 25 89 a7 f7 3e b4 be 6f d7 8f db 76 59 e8 28 d8" - sendhex " 10 45 f3 a9 83 b8 18 1c 7b c2 30 55 04 17 13 c4" - sendhex " 0f 07 5f 7a 38 f4 8e 50 b3 37 d4 3a 32 4a 34 07" - sendhex " 8a 9c d1 92 77 48 d4 0a 72 ea 06 5f b3 1c fa dd" - sendhex " 2b b9 88 20 99 e6 9a 3c 84 7c 85 8e 50 e0 59 2a" - sendhex " 42 b0 8a 34 0f 96 d5 1e f7 97 fb b7 7e fd 4e 87" - sendhex " c7 8f be 9e ce fb 74 3a 3f 51 89 a3 9b 7e b2 43" - sendhex " a4 86 a2 55 90 b9 29 4c 4b 83 b8 99 5f b5 bb 27" - sendhex " 6a d4 86 18 22 83 8a 26 f4 11 1a 5c eb 34 3b ca" - sendhex " 20 31 9e 12 29 ff a8 92 78 a2 e6 ec 61 55 12 fc" - sendhex " 68 84 6c 12 41 b9 cf 2f 30 3b f0 10 5e d6 b7 eb" - sendhex " e7 76 bb 2c 7f 8c 90 4a 14 4c 01 00 00" + sendhex " 1f 8b 08 00 c2 39 33 50 02 03 45 90 4d 6a 03 31" + sendhex " 0c 85 f7 73 8a 47 2e 10 d9 f2 9f ca 34 d0 c2 64" + sendhex " 5b 08 bd 80 2d cb 10 28 5d 34 f4 fe f5 30 d0 68" + sendhex " 25 89 a7 f7 3e b4 be 6f d7 8f db 76 59 e8 28 d8" + sendhex " 10 45 f3 a9 83 b8 18 1c 7b c2 30 55 04 17 13 c4" + sendhex " 0f 07 5f 7a 38 f4 8e 50 b3 37 d4 3a 32 4a 34 07" + sendhex " 8a 9c d1 92 77 48 d4 0a 72 ea 06 5f b3 1c fa dd" + sendhex " 2b b9 88 20 99 e6 9a 3c 84 7c 85 8e 50 e0 59 2a" + sendhex " 42 b0 8a 34 0f 96 d5 1e f7 97 fb b7 7e fd 4e 87" + sendhex " c7 8f be 9e ce fb 74 3a 3f 51 89 a3 9b 7e b2 43" + sendhex " a4 86 a2 55 90 b9 29 4c 4b 83 b8 99 5f b5 bb 27" + sendhex " 6a d4 86 18 22 83 8a 26 f4 11 1a 5c eb 34 3b ca" + sendhex " 20 31 9e 12 29 ff a8 92 78 a2 e6 ec 61 55 12 fc" + sendhex " 68 84 6c 12 41 b9 cf 2f 30 3b f0 10 5e d6 b7 eb" + sendhex " e7 76 bb 2c 7f 8c 90 4a 14 4c 01 00 00" send "\r\n" chunkedlen 0 rxreq @@ -53,13 +53,13 @@ server s1 { -hdr "Content-Encoding: gzip" \ -hdr "Transfer-Encoding: Chunked" send "70\r\n" - sendhex " 1f 8b 08 00 c2 39 33 50 02 03 45 90 4d 6a 03 31" - sendhex " 0c 85 f7 73 8a 47 2e 10 d9 f2 9f ca 34 d0 c2 64" - sendhex " 5b 08 bd 80 2d cb 10 28 5d 34 f4 fe f5 30 d0 68" - sendhex " 25 89 a7 f7 3e b4 be 6f d7 8f db 76 59 e8 28 d8" - sendhex " 10 45 f3 a9 83 b8 18 1c 7b c2 30 55 04 17 13 c4" - sendhex " 0f 07 5f 7a 38 f4 8e 50 b3 37 d4 3a 32 4a 34 07" - sendhex " FF FF FF FF FF FF FF FF 72 ea 06 5f b3 1c fa dd" + sendhex " 1f 8b 08 00 c2 39 33 50 02 03 45 90 4d 6a 03 31" + sendhex " 0c 85 f7 73 8a 47 2e 10 d9 f2 9f ca 34 d0 c2 64" + sendhex " 5b 08 bd 80 2d cb 10 28 5d 34 f4 fe f5 30 d0 68" + sendhex " 25 89 a7 f7 3e b4 be 6f d7 8f db 76 59 e8 28 d8" + sendhex " 10 45 f3 a9 83 b8 18 1c 7b c2 30 55 04 17 13 c4" + sendhex " 0f 07 5f 7a 38 f4 8e 50 b3 37 d4 3a 32 4a 34 07" + sendhex " FF FF FF FF FF FF FF FF 72 ea 06 5f b3 1c fa dd" expect_close } -start @@ -84,13 +84,13 @@ server s1 { -hdr "Content-Encoding: gzip" \ -hdr "Transfer-Encoding: Chunked" send "70\r\n" - sendhex " 1f 8b 08 00 c2 39 33 50 02 03 45 90 4d 6a 03 31" - sendhex " 0c 85 f7 73 8a 47 2e 10 d9 f2 9f ca 34 d0 c2 64" - sendhex " 5b 08 bd 80 2d cb 10 28 5d 34 f4 fe f5 30 d0 68" - sendhex " 25 89 a7 f7 3e b4 be 6f d7 8f db 76 59 e8 28 d8" - sendhex " 10 45 f3 a9 83 b8 18 1c 7b c2 30 55 04 17 13 c4" - sendhex " 0f 07 5f 7a 38 f4 8e 50 b3 37 d4 3a 32 4a 34 07" - sendhex " FF FF FF FF FF FF FF FF 72 ea 06 5f b3 1c fa dd" + sendhex " 1f 8b 08 00 c2 39 33 50 02 03 45 90 4d 6a 03 31" + sendhex " 0c 85 f7 73 8a 47 2e 10 d9 f2 9f ca 34 d0 c2 64" + sendhex " 5b 08 bd 80 2d cb 10 28 5d 34 f4 fe f5 30 d0 68" + sendhex " 25 89 a7 f7 3e b4 be 6f d7 8f db 76 59 e8 28 d8" + sendhex " 10 45 f3 a9 83 b8 18 1c 7b c2 30 55 04 17 13 c4" + sendhex " 0f 07 5f 7a 38 f4 8e 50 b3 37 d4 3a 32 4a 34 07" + sendhex " FF FF FF FF FF FF FF FF 72 ea 06 5f b3 1c fa dd" expect_close } -start diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc index dfe69ae..3198e89 100644 --- a/bin/varnishtest/tests/v00016.vtc +++ b/bin/varnishtest/tests/v00016.vtc @@ -31,9 +31,9 @@ varnish v1 -vcl { varnish v1 -errvcl {include not followed by string constant.} { /* token test */ - error lookup hash pipe pass fetch deliver discard keep restart - include - if else elseif elsif + error lookup hash pipe pass fetch deliver discard keep restart + include + if else elseif elsif ++ -- && || <= == != >= >> << += -= *= /= { } ( ) * + - / % > < = ; ! & . | ~ , } diff --git a/bin/varnishtest/tests/v00036.vtc b/bin/varnishtest/tests/v00036.vtc index 7bb1be8..2a7009c 100644 --- a/bin/varnishtest/tests/v00036.vtc +++ b/bin/varnishtest/tests/v00036.vtc @@ -1,18 +1,18 @@ varnishtest "Test fallback director" server s1 { - rxreq - txresp -hdr "Foo: 1" + rxreq + txresp -hdr "Foo: 1" } -start server s2 { - rxreq - txresp -hdr "Foo: 2" + rxreq + txresp -hdr "Foo: 2" } -start server s3 { - rxreq - txresp -hdr "Foo: 3" + rxreq + txresp -hdr "Foo: 3" } -start varnish v1 -vcl+backend { @@ -25,12 +25,12 @@ varnish v1 -vcl+backend { fb1.add_backend(s3); } - sub vcl_recv { - return (pass); - } - sub vcl_backend_fetch { - set bereq.backend = fb1.backend(); - } + sub vcl_recv { + return (pass); + } + sub vcl_backend_fetch { + set bereq.backend = fb1.backend(); + } } -start @@ -38,24 +38,24 @@ varnish v1 -cliok "backend.set_health s1 sick" varnish v1 -cliok "backend.set_health s2 sick" client c1 { - # s1 & s2 are both sick, expect response from s3 - txreq - rxresp - expect resp.http.foo == "3" + # s1 & s2 are both sick, expect response from s3 + txreq + rxresp + expect resp.http.foo == "3" } -run varnish v1 -cliok "backend.set_health s2 healthy" client c1 { - txreq - rxresp - expect resp.http.foo == "2" + txreq + rxresp + expect resp.http.foo == "2" } -run varnish v1 -cliok "backend.set_health s1 healthy" client c1 { - txreq - rxresp - expect resp.http.foo == "1" + txreq + rxresp + expect resp.http.foo == "1" } -run From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] da49eb7 White space copy&paste fixups Message-ID: commit da49eb78009bff98156092dc8abda2756d020fc4 Author: Poul-Henning Kamp Date: Thu Jul 24 08:01:56 2014 +0000 White space copy&paste fixups diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h index a88a12d..845fbcb 100644 --- a/include/tbl/vsc_f_main.h +++ b/include/tbl/vsc_f_main.h @@ -573,11 +573,11 @@ VSC_F(bans_lurker_contention, uint64_t, 0, 'c', diag, ) VSC_F(bans_persisted_bytes, uint64_t, 0, 'g', diag, "Bytes used by the persisted ban lists", - "Number of bytes used by the persisted ban lists." + "Number of bytes used by the persisted ban lists." ) VSC_F(bans_persisted_fragmentation, uint64_t, 0, 'g', diag, "Extra bytes in persisted ban lists due to fragmentation", - "Number of extra bytes accumulated through dropped and" + "Number of extra bytes accumulated through dropped and" " completed bans in the persistent ban lists." ) diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h index 45c1aaf..a4f1b27 100644 --- a/include/vapi/vsl.h +++ b/include/vapi/vsl.h @@ -268,8 +268,8 @@ void VSL_ResetError(struct VSL_data *vsl); struct VSL_cursor *VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, unsigned options); /* - * Set the cursor pointed to by cursor up as a raw cursor in the - * log. Cursor points at the current log head. + * Set the cursor pointed to by cursor up as a raw cursor in the + * log. Cursor points at the current log head. * * Options: * VSL_COPT_TAIL Start cursor at log tail @@ -279,7 +279,7 @@ struct VSL_cursor *VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, * Return values: * non-NULL: Pointer to cursor * NULL: Error, see VSL_Error - */ + */ struct VSL_cursor *VSL_CursorFile(struct VSL_data *vsl, const char *name, unsigned options); diff --git a/include/vqueue.h b/include/vqueue.h index 8ffd820..f725f96 100644 --- a/include/vqueue.h +++ b/include/vqueue.h @@ -240,7 +240,7 @@ struct { \ #define VSTAILQ_LAST(head, type, field) \ (VSTAILQ_EMPTY((head)) ? \ NULL : \ - ((struct type *)(void *) \ + ((struct type *)(void *) \ ((char *)((head)->vstqh_last) - \ __offsetof(struct type, field)))) diff --git a/include/vtree.h b/include/vtree.h index cf4e061..d251d62 100644 --- a/include/vtree.h +++ b/include/vtree.h @@ -150,7 +150,7 @@ static __inline struct type * \ name##_VSPLAY_MIN_MAX(struct name *head, int val) \ { \ name##_VSPLAY_MINMAX(head, val); \ - return (VSPLAY_ROOT(head)); \ + return (VSPLAY_ROOT(head)); \ } /* Main splay operation. From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] 3ffbaed White-space copy&paste fixups Message-ID: commit 3ffbaed8203327eea93023e5eb48da46bd9ad174 Author: Poul-Henning Kamp Date: Thu Jul 24 08:03:00 2014 +0000 White-space copy&paste fixups diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 2109f20..26191fe 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -324,5 +324,5 @@ VTCP_check_hup(int sock) if (poll(&pfd, 1, 0) == 1 && pfd.revents & POLLHUP) return (1); - return (0); + return (0); } diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 0d39073..c2f9818 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1025,7 +1025,7 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, enum var_type fmt) } if ((*e)->fmt == STRING && (tl->t->tok == '~' || tl->t->tok == T_NOMATCH)) { - not = tl->t->tok == '~' ? "" : "!"; + not = tl->t->tok == '~' ? "" : "!"; vcc_NextToken(tl); ExpectErr(tl, CSTR); re = vcc_regexp(tl); @@ -1037,7 +1037,7 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, enum var_type fmt) } if ((*e)->fmt == IP && (tl->t->tok == '~' || tl->t->tok == T_NOMATCH)) { - not = tl->t->tok == '~' ? "" : "!"; + not = tl->t->tok == '~' ? "" : "!"; vcc_NextToken(tl); ExpectErr(tl, ID); vcc_AddRef(tl, tl->t, SYM_ACL); diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index b2f3313..744b10d 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -93,7 +93,7 @@ vmod_test_priv_vcl(const struct vrt_ctx *ctx, struct vmod_priv *priv) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - assert(!strcmp(priv->priv, "FOO")); + assert(!strcmp(priv->priv, "FOO")); } VCL_BLOB From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] 92004db Make it possible to return synth from vcl_deliver{} Message-ID: commit 92004db2c20fef4d11c15d7cf4b678796042927e Author: Poul-Henning Kamp Date: Thu Jul 24 08:40:02 2014 +0000 Make it possible to return synth from vcl_deliver{} This is now possible because synth responses do not go through deliver any more, as error responses used to. Patch by: Nils Goroll Much appreciated diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 32b81a1..d57de9a 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -137,11 +137,22 @@ cnt_deliver(struct worker *wrk, struct req *req) if (req->restarts >= cache_param->max_restarts) wrk->handling = VCL_RET_DELIVER; - if (wrk->handling == VCL_RET_RESTART) { + if (wrk->handling != VCL_RET_DELIVER) { (void)HSH_DerefObj(&wrk->stats, &req->obj); AZ(req->obj); http_Teardown(req->resp); - req->req_step = R_STP_RESTART; + + switch (wrk->handling) { + case VCL_RET_RESTART: + req->req_step = R_STP_RESTART; + break; + case VCL_RET_SYNTH: + req->req_step = R_STP_SYNTH; + break; + default: + INCOMPL(); + } + return (REQ_FSM_MORE); } diff --git a/bin/varnishtest/tests/c00068.vtc b/bin/varnishtest/tests/c00068.vtc new file mode 100644 index 0000000..817ec32 --- /dev/null +++ b/bin/varnishtest/tests/c00068.vtc @@ -0,0 +1,60 @@ +varnishtest "synth in deliver" + +server s1 { + rxreq + txresp -status 200 + rxreq + txresp -status 200 + rxreq + txresp -status 200 +} -start + +varnish v1 -vcl+backend { + sub vcl_deliver { + if (req.url == "/332") { + return (synth(332, "FOO")); + } else if (req.url == "/333") { + return (synth(333, "FOO")); + } else { + return (synth(334, "BAR")); + } + } + + sub vcl_synth { + if (resp.status == 333) { + set resp.http.connection = "close"; + } else if (resp.status == 332) { + if (req.restarts == 0) { + return (restart); + } else { + set resp.http.restarts = req.restarts; + synthetic(req.restarts); + } + } + return (deliver); + } +} -start + +client c1 { + txreq -url /334 + rxresp + expect resp.status == 334 + + # cache hit + txreq -url /334 + rxresp + expect resp.status == 334 + + txreq -url /333 + rxresp + expect resp.status == 333 + expect_close +} -run + +client c2 { + txreq -url /332 + rxresp + expect resp.status == 332 + expect resp.http.restarts == 1 + expect resp.bodylen == 1 +} -run diff --git a/bin/varnishtest/tests/r01027.vtc b/bin/varnishtest/tests/r01027.vtc deleted file mode 100644 index 716e6f0..0000000 --- a/bin/varnishtest/tests/r01027.vtc +++ /dev/null @@ -1,9 +0,0 @@ -varnishtest "Test if you can error in vcl_deliver" - -varnish v1 -errvcl {Invalid return "synth"} { - backend b { .host = "127.0.0.1"; } - sub vcl_deliver { - return (synth(201,"ok")); - } -} - diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index a1fd2d7..2906588 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -112,7 +112,7 @@ returns =( ), ('deliver', "C", - ('restart', 'deliver',) + ('synth', 'restart', 'deliver',) ), ('synth', "C", From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] 1b57f4c Add a VCC variable type "HTTP" to refer to an entire HTTP message. Presently supported "req", "bereq", "resp", "beresp". Message-ID: commit 1b57f4cb1956c16d4597e19fc6182b37b6f5c339 Author: Poul-Henning Kamp Date: Thu Jul 24 11:00:10 2014 +0000 Add a VCC variable type "HTTP" to refer to an entire HTTP message. Presently supported "req", "bereq", "resp", "beresp". diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index b74e101..47897e1 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -608,3 +608,19 @@ VRT_r_obj_uncacheable(const struct vrt_ctx *ctx) CHECK_OBJ_NOTNULL(ctx->req->obj, OBJECT_MAGIC); return (ctx->req->obj->objcore->flags & OC_F_PASS ? 1 : 0); } + +/*--------------------------------------------------------------------*/ + +#define HTTP_VAR(x) \ +struct http * \ +VRT_r_##x(const struct vrt_ctx *ctx) \ +{ \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ + CHECK_OBJ_NOTNULL(ctx->http_##x, HTTP_MAGIC); \ + return (ctx->http_##x); \ +} + +HTTP_VAR(req) +HTTP_VAR(resp) +HTTP_VAR(bereq) +HTTP_VAR(beresp) diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index 5704a1f..b3ac748 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -2,6 +2,7 @@ varnishtest "Test std & debug vmod" server s1 { rxreq + expect req.http.encrypted == "ROT52" txresp -hdr "foo: bAr" -hdr "bar: fOo" -bodylen 4 } -start @@ -9,6 +10,10 @@ varnish v1 -vcl+backend { import ${vmod_std}; import ${vmod_debug}; + sub vcl_recv { + debug.rot52(req); + } + sub vcl_deliver { set resp.http.foo = std.toupper(resp.http.foo); set resp.http.bar = std.tolower(resp.http.bar); @@ -17,6 +22,7 @@ varnish v1 -vcl+backend { debug.test_priv_vcl(); std.log("VCL initiated log"); std.syslog(8 + 7, "Somebody runs varnishtest"); + debug.rot52(resp); } } -start @@ -27,6 +33,7 @@ client c1 { expect resp.bodylen == "4" expect resp.http.foo == "BAR" expect resp.http.bar == "foo" + expect resp.http.encrypted == "ROT52" } -run varnish v1 -errvcl {Wrong enum value. Expected one of:} { diff --git a/include/vrt.h b/include/vrt.h index 066d6c8..efc9c94 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -41,7 +41,7 @@ #define VRT_MAJOR_VERSION 1U -#define VRT_MINOR_VERSION 1U +#define VRT_MINOR_VERSION 2U /***********************************************************************/ @@ -59,21 +59,23 @@ struct suckaddr; /*********************************************************************** * This is the central definition of the mapping from VCL types to * C-types. The python scripts read these from here. + * (alphabetic order) */ typedef struct director * VCL_BACKEND; +typedef const struct vmod_priv * VCL_BLOB; typedef unsigned VCL_BOOL; typedef double VCL_BYTES; typedef double VCL_DURATION; typedef const char * VCL_ENUM; typedef const struct gethdr_s * VCL_HEADER; +typedef struct http * VCL_HTTP; typedef long VCL_INT; typedef const struct suckaddr * VCL_IP; typedef double VCL_REAL; typedef const char * VCL_STRING; typedef double VCL_TIME; typedef void VCL_VOID; -typedef const struct vmod_priv * VCL_BLOB; /*********************************************************************** * This is the composite argument we pass to compiled VCL and VRT diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 2906588..21173c4 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -197,6 +197,13 @@ sp_variables = [ specified by the -n parameter. """ ), + ('req', + 'HTTP', + ( 'client',), + ( ), """ + The entire request HTTP data structure + """ + ), ('req.method', 'STRING', ( 'client',), @@ -294,6 +301,13 @@ sp_variables = [ always (re)fetch from the backend. """ ), + ('bereq', + 'HTTP', + ( 'backend',), + ( ), """ + The entire backend request HTTP data structure + """ + ), ('bereq.xid', 'STRING', ( 'backend',), @@ -370,6 +384,13 @@ sp_variables = [ backend. Not available in pipe mode. """ ), + ('beresp', + 'HTTP', + ( 'backend_response', 'backend_error'), + ( ), """ + The entire backend response HTTP data structure + """ + ), ('beresp.proto', 'STRING', ( 'backend_response', 'backend_error'), @@ -553,6 +574,13 @@ sp_variables = [ ( ), """ """ ), + ('resp', + 'HTTP', + ( 'deliver', 'synth'), + ( ), """ + The entire response HTTP data structure + """ + ), ('resp.proto', 'STRING', ( 'deliver', 'synth', ), @@ -597,17 +625,17 @@ aliases = [ stv_variables = ( ('free_space', 'BYTES', "0.", 'storage..free_space', """ - Free space available in the named stevedore. Only available for - the malloc stevedore. - """), + Free space available in the named stevedore. Only available for + the malloc stevedore. + """), ('used_space', 'BYTES', "0.", 'storage..used_space', """ - Used space in the named stevedore. Only available for the malloc - stevedore. - """), + Used space in the named stevedore. Only available for the malloc + stevedore. + """), ('happy', 'BOOL', "0", 'storage..happy', """ - Health status for the named stevedore. Not available in any of the - current stevedores. - """), + Health status for the named stevedore. Not available in any of the + current stevedores. + """), ) ####################################################################### @@ -1188,11 +1216,11 @@ hdr="storage" fp_vclvar.write("\n" + hdr + "\n"); fp_vclvar.write("~" * len(hdr) + "\n"); for i in stv_variables: - fp_vclvar.write("\n" + i[3] + "\n\n") - fp_vclvar.write("\tType: " + i[1] + "\n\n") - fp_vclvar.write("\tReadable from: client, backend\n\n") - for j in i[4].split("\n"): - fp_vclvar.write("\t%s\n" % j.strip()) + fp_vclvar.write("\n" + i[3] + "\n\n") + fp_vclvar.write("\tType: " + i[1] + "\n\n") + fp_vclvar.write("\tReadable from: client, backend\n\n") + for j in i[4].split("\n"): + fp_vclvar.write("\t%s\n" % j.strip()) fp_vclvar.close() diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 21c6c6d..25a901f 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -47,10 +47,12 @@ from pprint import pprint, pformat ctypes = { 'BACKEND': "VCL_BACKEND", + 'BLOB': "VCL_BLOB", 'BOOL': "VCL_BOOL", 'DURATION': "VCL_DURATION", 'ENUM': "VCL_ENUM", 'HEADER': "VCL_HEADER", + 'HTTP': "VCL_HTTP", 'INT': "VCL_INT", 'IP': "VCL_IP", 'PRIV_CALL': "struct vmod_priv *", @@ -60,7 +62,6 @@ ctypes = { 'STRING_LIST': "const char *, ...", 'TIME': "VCL_TIME", 'VOID': "VCL_VOID", - 'BLOB': "VCL_BLOB", } ####################################################################### diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 8667298..ca1b307 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -80,3 +80,8 @@ Foo indeed. $Method TIME .date() You never know when you need a date. + +$Function VOID rot52(HTTP) + +Encrypt the HTTP header with quad-ROT13 encryption, +(this is approx 33% better than triple-DES). diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 744b10d..b952f64 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -138,3 +138,12 @@ vmod_no_backend(const struct vrt_ctx *ctx) return (NULL); } +VCL_VOID __match_proto__(td_debug_rot52) +vmod_rot52(const struct vrt_ctx *ctx, VCL_HTTP hp) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); + + http_PrintfHeader(hp, "Encrypted: ROT52"); +} From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] c24c26e Expose WS_MarkOverflow() Message-ID: commit c24c26e05968cb61c4cc183fb55aadc6d9d902d6 Author: Poul-Henning Kamp Date: Tue Jun 24 10:38:22 2014 +0000 Expose WS_MarkOverflow() diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index b95bc65..7ce01db 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -1235,6 +1235,7 @@ void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void WS_Init(struct ws *ws, const char *id, void *space, unsigned len); unsigned WS_Reserve(struct ws *ws, unsigned bytes); +void WS_MarkOverflow(struct ws *ws); void WS_Release(struct ws *ws, unsigned bytes); void WS_ReleaseP(struct ws *ws, char *ptr); void WS_Assert(const struct ws *ws); diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index 8ef7aca..ccc887c 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -85,7 +85,7 @@ WS_Init(struct ws *ws, const char *id, void *space, unsigned len) } -static void +void WS_MarkOverflow(struct ws *ws) { CHECK_OBJ_NOTNULL(ws, WS_MAGIC); From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] 63ab036 Remove a couple of #ifdefs that serve no purpose anymore. Message-ID: commit 63ab0363be2923f1d7984e4483ce0749422028d2 Author: Poul-Henning Kamp Date: Mon Jul 28 07:27:56 2014 +0000 Remove a couple of #ifdefs that serve no purpose anymore. diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index be862e2..a338e34 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -534,21 +534,13 @@ struct parspec mgt_parspec[] = { "Directory from which relative VCL filenames (vcl.load and " "include) are opened.", 0, -#ifdef VARNISH_VCL_DIR VARNISH_VCL_DIR, -#else - ".", -#endif NULL }, { "vmod_dir", tweak_string, &mgt_vmod_dir, NULL, NULL, "Directory where VCL modules are to be found.", 0, -#ifdef VARNISH_VMOD_DIR VARNISH_VMOD_DIR, -#else - ".", -#endif NULL }, { "vcc_err_unref", tweak_bool, &mgt_vcc_err_unref, From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] fcc5fe3 Define VARNISH_VMOD_DIR and VARNISH_VCL_DIR Message-ID: commit fcc5fe39fb34d349960d72ce046342cece2f897b Author: Poul-Henning Kamp Date: Mon Jul 28 07:53:32 2014 +0000 Define VARNISH_VMOD_DIR and VARNISH_VCL_DIR diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh index 63b406d..4f471cf 100755 --- a/bin/varnishd/flint.sh +++ b/bin/varnishd/flint.sh @@ -16,6 +16,8 @@ flexelint \ -I../.. \ -I/usr/local/include \ -DVARNISH_STATE_DIR=\"foo\" \ + -DVARNISH_VMOD_DIR=\"foo\" \ + -DVARNISH_VCL_DIR=\"foo\" \ cache/*.c \ common/*.c \ storage/*.c \ From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] 14a9cbc Rename shm_reclen to vsl_reclen for consistency. Message-ID: commit 14a9cbc1eab47addf113b05375fb2013c85cf5eb Author: Poul-Henning Kamp Date: Mon Jul 28 07:57:33 2014 +0000 Rename shm_reclen to vsl_reclen for consistency. Leave shm_reclen as a parameter alias for now. Parameter vsl_buffer must be 12 bytes larger than vsl_reclen in order to avoid a panic when we try to put 12 pounds of VSL into a 5 pound vsl_buffer sack. Tweak the opposite parameter Minimum or Maximum value when we set one of of these parameters. Fixes #1547 diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index d0733d7..7c364c1 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -179,7 +179,7 @@ vslr(enum VSL_tag_e tag, uint32_t vxid, const char *b, unsigned len) uint32_t *p; unsigned mlen; - mlen = cache_param->shm_reclen; + mlen = cache_param->vsl_reclen; /* Truncate */ if (len > mlen) @@ -209,7 +209,7 @@ void VSL(enum VSL_tag_e tag, uint32_t vxid, const char *fmt, ...) { va_list ap; - unsigned n, mlen = cache_param->shm_reclen; + unsigned n, mlen = cache_param->vsl_reclen; char buf[mlen]; AN(fmt); @@ -267,7 +267,7 @@ VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t) Tcheck(t); if (vsl_tag_is_masked(tag)) return; - mlen = cache_param->shm_reclen; + mlen = cache_param->vsl_reclen; /* Truncate */ l = Tlen(t); @@ -321,7 +321,7 @@ VSLbv(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, va_list ap) return; } - mlen = cache_param->shm_reclen; + mlen = cache_param->vsl_reclen; /* Flush if we cannot fit a full size record */ if (VSL_END(vsl->wlp, mlen + 1) >= vsl->wle) diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index 7e208af..35c0a7a 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -103,7 +103,7 @@ struct params { unsigned http_resp_hdr_len; unsigned http_max_hdr; - unsigned shm_reclen; + unsigned vsl_reclen; double timeout_linger; double timeout_idle; diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index b5aec87..e8a1763 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -484,6 +484,7 @@ MCF_SetMinimum(const char *param, const char *new_min) { struct parspec *pp; + AN(new_min); pp = mcf_findpar(param); AN(pp); pp->min = new_min; @@ -495,6 +496,7 @@ MCF_SetMaximum(const char *param, const char *new_max) { struct parspec *pp; + AN(new_max); pp = mcf_findpar(param); AN(pp); pp->max = new_max; diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h index 3263537..f60180b 100644 --- a/bin/varnishd/mgt/mgt_param.h +++ b/bin/varnishd/mgt/mgt_param.h @@ -63,6 +63,8 @@ tweak_t tweak_timeout; tweak_t tweak_uint; tweak_t tweak_user; tweak_t tweak_waiter; +tweak_t tweak_vsl_buffer; +tweak_t tweak_vsl_reclen; int tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg, const char *min, const char *max); diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index a338e34..37d9a66 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -160,23 +160,27 @@ struct parspec mgt_parspec[] = { 0, "64", "header lines" }, { "vsl_buffer", - tweak_bytes_u, &mgt_param.vsl_buffer, + tweak_vsl_buffer, &mgt_param.vsl_buffer, "1024", NULL, "Bytes of (req-/backend-)workspace dedicated to buffering" " VSL records.\n" - "At a bare minimum, this must be longer than" - " the longest HTTP header to be logged.\n" "Setting this too high costs memory, setting it too low" " will cause more VSL flushes and likely increase" - " lock-contention on the VSL mutex.\n" - "Minimum is 1k bytes.", + " lock-contention on the VSL mutex.\n\n" + "The minimum tracks the vsl_reclen parameter + 12 bytes.", 0, "4k", "bytes" }, + { "vsl_reclen", + tweak_vsl_reclen, &mgt_param.vsl_reclen, + "16", "65535", + "Maximum number of bytes in SHM log record.\n\n" + "The maximum tracks the vsl_buffer parameter - 12 bytes.", + 0, + "255", "bytes" }, { "shm_reclen", - tweak_bytes_u, &mgt_param.shm_reclen, + tweak_vsl_reclen, &mgt_param.vsl_reclen, "16", "65535", - "Maximum number of bytes in SHM log record.\n" - "Maximum is 65535 bytes.", + "Old name for vsl_reclen, use that instead.", 0, "255", "bytes" }, { "timeout_idle", tweak_timeout, &mgt_param.timeout_idle, diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c index 4e2645f..7d9fb4f 100644 --- a/bin/varnishd/mgt/mgt_param_tweak.c +++ b/bin/varnishd/mgt/mgt_param_tweak.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -330,6 +331,45 @@ tweak_bytes_u(struct vsb *vsb, const struct parspec *par, const char *arg) } /*-------------------------------------------------------------------- + * vsl_buffer and vsl_reclen have dependencies. + */ + +int +tweak_vsl_buffer(struct vsb *vsb, const struct parspec *par, const char *arg) +{ + volatile unsigned *d1; + volatile ssize_t dest; + char buf[20]; + + d1 = par->priv; + dest = *d1; + if (tweak_generic_bytes(vsb, &dest, arg, par->min, par->max)) + return (-1); + *d1 = dest; + bprintf(buf, "%u", *d1 - 12); + MCF_SetMaximum("vsl_reclen", strdup(buf)); + MCF_SetMaximum("shm_reclen", strdup(buf)); + return (0); +} + +int +tweak_vsl_reclen(struct vsb *vsb, const struct parspec *par, const char *arg) +{ + volatile unsigned *d1; + volatile ssize_t dest; + char buf[20]; + + d1 = par->priv; + dest = *d1; + if (tweak_generic_bytes(vsb, &dest, arg, par->min, par->max)) + return (-1); + *d1 = dest; + bprintf(buf, "%u", *d1 + 12); + MCF_SetMinimum("vsl_buffer", strdup(buf)); + return (0); +} + +/*-------------------------------------------------------------------- * XXX: slightly magic. We want to initialize to "nobody" (XXX: shouldn't * XXX: that be something autocrap found for us ?) but we don't want to * XXX: fail initialization if that user doesn't exists, even though we From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] cd7bd2a Un-copy&paste all the spaces back into TABs Message-ID: commit cd7bd2af67d8bc21045b141c8fa08e9d787a87aa Author: Poul-Henning Kamp Date: Mon Jul 28 08:04:28 2014 +0000 Un-copy&paste all the spaces back into TABs diff --git a/lib/libvarnishapi/generate.py b/lib/libvarnishapi/generate.py index f8cc671..82f7c59 100755 --- a/lib/libvarnishapi/generate.py +++ b/lib/libvarnishapi/generate.py @@ -37,121 +37,121 @@ import copy srcroot = "../.." buildroot = "../.." if len(sys.argv) == 3: - srcroot = sys.argv[1] - buildroot = sys.argv[2] + srcroot = sys.argv[1] + buildroot = sys.argv[2] ####################################################################### # These are our tokens tokens = { - # Numerical comparisons - "T_EQ": "==", - "T_NEQ": "!=", - "T_LEQ": "<=", - "T_GEQ": ">=", + # Numerical comparisons + "T_EQ": "==", + "T_NEQ": "!=", + "T_LEQ": "<=", + "T_GEQ": ">=", - # String comparisons - "T_SEQ": "eq", - "T_SNEQ": "ne", + # String comparisons + "T_SEQ": "eq", + "T_SNEQ": "ne", - # Regular expression matching - "T_NOMATCH": "!~", + # Regular expression matching + "T_NOMATCH": "!~", - # Boolean operators - "T_AND": "and", - "T_OR": "or", - "T_NOT": "not", + # Boolean operators + "T_AND": "and", + "T_OR": "or", + "T_NOT": "not", - # Miscellaneous - None: "<>~[]{}():,", + # Miscellaneous + None: "<>~[]{}():,", - # These have handwritten recognizers - "VAL": None, - "EOI": None, + # These have handwritten recognizers + "VAL": None, + "EOI": None, - # Special - "T_TRUE": None, + # Special + "T_TRUE": None, } ####################################################################### # Emit a function to recognize tokens in a string def emit_vxp_fixed_token(fo, tokens): - recog = list() - emit = dict() - for i in tokens: - j = tokens[i] - if (j != None): - recog.append(j) - emit[j] = i - - recog.sort() - rrecog = copy.copy(recog) - rrecog.sort(key = lambda x: -len(x)) - - fo.write(""" + recog = list() + emit = dict() + for i in tokens: + j = tokens[i] + if (j != None): + recog.append(j) + emit[j] = i + + recog.sort() + rrecog = copy.copy(recog) + rrecog.sort(key = lambda x: -len(x)) + + fo.write(""" unsigned vxp_fixed_token(const char *p, const char **q) { \tswitch (p[0]) { """) - last_initial = None - for i in recog: - if (i[0] == last_initial): - continue - last_initial = i[0] - fo.write("\tcase '%s':\n" % last_initial) - for j in rrecog: - if (j[0] != last_initial): - continue - - fo.write("\t\tif (") - k = 1 - l = len(j) - while (k < l): - fo.write("p[%d] == '%s'" % (k, j[k])) - fo.write(" &&\n\t\t ") - k += 1 - fo.write("(isword(p[%d]) ? !isword(p[%d]) : 1)) {\n" % - (l - 1, l)) - fo.write("\t\t\t*q = p + %d;\n" % l) - fo.write("\t\t\treturn (%s);\n" % emit[j]) - fo.write("\t\t}\n"); - fo.write("\t\treturn (0);\n") - - fo.write("\tdefault:\n\t\treturn (0);\n\t}\n}\n") + last_initial = None + for i in recog: + if (i[0] == last_initial): + continue + last_initial = i[0] + fo.write("\tcase '%s':\n" % last_initial) + for j in rrecog: + if (j[0] != last_initial): + continue + + fo.write("\t\tif (") + k = 1 + l = len(j) + while (k < l): + fo.write("p[%d] == '%s'" % (k, j[k])) + fo.write(" &&\n\t\t ") + k += 1 + fo.write("(isword(p[%d]) ? !isword(p[%d]) : 1)) {\n" % + (l - 1, l)) + fo.write("\t\t\t*q = p + %d;\n" % l) + fo.write("\t\t\treturn (%s);\n" % emit[j]) + fo.write("\t\t}\n"); + fo.write("\t\treturn (0);\n") + + fo.write("\tdefault:\n\t\treturn (0);\n\t}\n}\n") ####################################################################### # Emit the vxp_tnames (token->string) conversion array def emit_vxp_tnames(fo, tokens): - fo.write("\nconst char * const vxp_tnames[256] = {\n") - l = list(tokens.keys()) - l.sort() - for i in l: - j = tokens[i] - if j == None: - j = i - if i[0] == "'": - j = i - fo.write("\t[%s] = \"%s\",\n" % (i, j)) - fo.write("};\n") + fo.write("\nconst char * const vxp_tnames[256] = {\n") + l = list(tokens.keys()) + l.sort() + for i in l: + j = tokens[i] + if j == None: + j = i + if i[0] == "'": + j = i + fo.write("\t[%s] = \"%s\",\n" % (i, j)) + fo.write("};\n") ####################################################################### def polish_tokens(tokens): - # Expand single char tokens - st = tokens[None] - del tokens[None] + # Expand single char tokens + st = tokens[None] + del tokens[None] - for i in st: - tokens["'" + i + "'"] = i + for i in st: + tokens["'" + i + "'"] = i ####################################################################### def file_header(fo): - fo.write("""/* + fo.write("""/* * NB: This file is machine generated, DO NOT EDIT! * * Edit and run generate.py instead @@ -170,11 +170,11 @@ j = 128 l = list(tokens.keys()) l.sort() for i in l: - if i[0] == "'": - continue - fo.write("#define\t%s %d\n" % (i, j)) - j += 1 - assert j < 256 + if i[0] == "'": + continue + fo.write("#define\t%s %d\n" % (i, j)) + j += 1 + assert j < 256 fo.close() From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] a5e6d3b Varnishd needs to run the systems C-compiler to compile the VCL code. Message-ID: commit a5e6d3b1936d0ae87bc079520b9f1a6fa816baf5 Author: Poul-Henning Kamp Date: Mon Jul 28 09:03:06 2014 +0000 Varnishd needs to run the systems C-compiler to compile the VCL code. For security reasons, we run the C-compiler in a sandbox process which by default uses the same (non-)privileges as the other sandboxes (VCL compiler, test-loader process and the worker process). On some systems access to the C-compiler is limited, also for reasons of security, and varnishd will fail to compile VCL code, unless all the sandboxes are given access to the C-compiler. Add a new parameter "group_cc" which adds a single gid to the grouplist of the sandbox which executes the cc_command, for the benefit of such systems. Do some slightly related polishing of the docs/help-texts in this area while here anyway. Fixes #1521 diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h index 35c0a7a..2ce95c6 100644 --- a/bin/varnishd/common/params.h +++ b/bin/varnishd/common/params.h @@ -63,6 +63,10 @@ struct params { char *group; gid_t gid; + /* Extra group for compiler access */ + char *group_cc; + gid_t gid_cc; + /* TTL used for lack of anything better */ double default_ttl; diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index e8a1763..9feea1a 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -90,6 +90,10 @@ static const char PROTECTED_TEXT[] = "\n\n" "NB: This parameter is protected and can not be changed."; +static const char ONLY_ROOT_TEXT[] = + "\n\n" + "NB: This parameter only works of varnishd is run as root."; + /*--------------------------------------------------------------------*/ @@ -265,6 +269,8 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv) mcf_wrap(cli, WIZARD_TEXT); if (pp->flags & PROTECTED) mcf_wrap(cli, PROTECTED_TEXT); + if (pp->flags & ONLY_ROOT) + mcf_wrap(cli, ONLY_ROOT_TEXT); VCLI_Out(cli, "\n\n"); } } @@ -556,6 +562,10 @@ MCF_DumpRstParam(void) printf("%swizard", q); q = ", "; } + if (pp->flags & ONLY_ROOT) { + printf("%sonly_root", q); + q = ", "; + } printf("\n"); } printf("\n"); diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h index f60180b..1b8204a 100644 --- a/bin/varnishd/mgt/mgt_param.h +++ b/bin/varnishd/mgt/mgt_param.h @@ -47,6 +47,7 @@ struct parspec { #define WIZARD (1<<4) #define PROTECTED (1<<5) #define OBJ_STICKY (1<<6) +#define ONLY_ROOT (1<<7) const char *def; const char *units; }; @@ -56,6 +57,7 @@ tweak_t tweak_bytes; tweak_t tweak_bytes_u; tweak_t tweak_double; tweak_t tweak_group; +tweak_t tweak_group_cc; tweak_t tweak_listen_address; tweak_t tweak_poolparam; tweak_t tweak_string; diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index 37d9a66..1e12691 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -53,11 +53,19 @@ struct parspec mgt_parspec[] = { { "user", tweak_user, NULL, NULL, NULL, "The unprivileged user to run as.", - MUST_RESTART, + MUST_RESTART | ONLY_ROOT, "" }, { "group", tweak_group, NULL, NULL, NULL, "The unprivileged group to run as.", - MUST_RESTART, + MUST_RESTART | ONLY_ROOT, + "" }, + { "group_cc", tweak_group_cc, NULL, NULL, NULL, + "On some systems the C-compiler is restricted so not" + " everybody can run it. This parameter makes it possible" + " to add an extra group to the sandbox process which runs the" + " cc_command, in order to gain access to such a restricted" + " c-compiler.", + ONLY_ROOT, "" }, { "default_ttl", tweak_timeout, &mgt_param.default_ttl, "0", NULL, diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c index 7d9fb4f..e757ec6 100644 --- a/bin/varnishd/mgt/mgt_param_tweak.c +++ b/bin/varnishd/mgt/mgt_param_tweak.c @@ -433,6 +433,38 @@ tweak_group(struct vsb *vsb, const struct parspec *par, const char *arg) return (0); } +/*-------------------------------------------------------------------- + * XXX: see comment for tweak_user, same thing here. + */ + +int +tweak_group_cc(struct vsb *vsb, const struct parspec *par, const char *arg) +{ + struct group *gr; + + (void)par; + if (arg != NULL) { + if (*arg != '\0') { + gr = getgrnam(arg); + if (gr == NULL) { + VSB_printf(vsb, "Unknown group"); + return(-1); + } + REPLACE(mgt_param.group_cc, gr->gr_name); + mgt_param.gid_cc = gr->gr_gid; + } else { + REPLACE(mgt_param.group_cc, ""); + mgt_param.gid_cc = 0; + } + } else if (strlen(mgt_param.group_cc) > 0) { + VSB_printf(vsb, "%s (%d)", + mgt_param.group_cc, (int)mgt_param.gid_cc); + } else { + VSB_printf(vsb, ""); + } + return (0); +} + /*--------------------------------------------------------------------*/ static void diff --git a/bin/varnishd/mgt/mgt_sandbox.c b/bin/varnishd/mgt/mgt_sandbox.c index 83a2aeb..973ec83 100644 --- a/bin/varnishd/mgt/mgt_sandbox.c +++ b/bin/varnishd/mgt/mgt_sandbox.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include "mgt/mgt.h" @@ -62,14 +63,27 @@ static void __match_proto__(mgt_sandbox_f) mgt_sandbox_unix(enum sandbox_e who) { - (void)who; - if (geteuid() == 0) { - XXXAZ(setgid(mgt_param.gid)); - XXXAZ(initgroups(mgt_param.user, mgt_param.gid)); - XXXAZ(setuid(mgt_param.uid)); - } else { +#define NGID 10 + gid_t gid_list[NGID]; + int i; + + if (geteuid() != 0) { REPORT0(LOG_INFO, "Not running as root, no priv-sep"); + return; } + + XXXAZ(setgid(mgt_param.gid)); + XXXAZ(initgroups(mgt_param.user, mgt_param.gid)); + + if (who == SANDBOX_CC && strlen(mgt_param.group_cc) > 0) { + /* Add the optional extra group for the C-compiler access */ + i = getgroups(NGID, gid_list); + assert(i >= 0); + gid_list[i++] = mgt_param.gid_cc; + XXXAZ(setgroups(i, gid_list)); + } + + XXXAZ(setuid(mgt_param.uid)); } #endif diff --git a/bin/varnishtest/tests/r01518.vtc b/bin/varnishtest/tests/r01518.vtc new file mode 100644 index 0000000..11791ae --- /dev/null +++ b/bin/varnishtest/tests/r01518.vtc @@ -0,0 +1,30 @@ +varnishtest "304 body handling with ESI" + +server s1 { + rxreq + txresp -hdr "Last-Modified: Wed, 04 Jun 2014 08:48:52 GMT" \ + -body {} + + rxreq + expect req.url == "/bar" + txresp -body {} + +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_response { + set beresp.do_esi = true; + } +} -start + +client c1 { + txreq + rxresp + expect resp.bodylen == 12 + expect resp.status == 200 + + txreq -hdr "If-Modified-Since: Wed, 04 Jun 2014 08:48:52 GMT" + rxresp + expect resp.status == 304 + expect resp.bodylen == 0 +} -run diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 633ea43..e1d02bb 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -226,6 +226,15 @@ restart reload The VCL programs must be reloaded for this parameter to take effect. +experimental + We're not really sure about this parameter, tell us what you find. + +wizard + Do not touch unless you *really* know what you're doing. + +only_root + Only works if varnishd is running as root. + Here is a list of all parameters, current as of last time we remembered to update the manual page. This text is produced from the same text you will find in the CLI if you use the param.show command, From lkarsten at varnish-software.com Mon Sep 22 14:38:22 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:22 +0200 Subject: [4.0] a36e406 Remove a comment overtaken by events. Message-ID: commit a36e406624a30a0faafd056c34d22c7fe7e414c2 Author: Poul-Henning Kamp Date: Mon Jul 28 09:12:42 2014 +0000 Remove a comment overtaken by events. diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index 1e12691..9135d8f 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -44,12 +44,6 @@ "\tmax_pool\tmaximum size of free pool.\n" \ "\tmax_age\tmax age of free element." -/* - * Remember to update varnishd.1 whenever you add / remove a parameter or - * change its default value. - * XXX: we should generate the relevant section of varnishd.1 from here. - */ - struct parspec mgt_parspec[] = { { "user", tweak_user, NULL, NULL, NULL, "The unprivileged user to run as.", @@ -64,7 +58,7 @@ struct parspec mgt_parspec[] = { " everybody can run it. This parameter makes it possible" " to add an extra group to the sandbox process which runs the" " cc_command, in order to gain access to such a restricted" - " c-compiler.", + " C-compiler.", ONLY_ROOT, "" }, { "default_ttl", tweak_timeout, &mgt_param.default_ttl, From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] c9a5dd2 Tweak these two test cases for 32 bit systems Message-ID: commit c9a5dd21bca7abe3d6cc685dd294e2f10816ca55 Author: Poul-Henning Kamp Date: Mon Jul 28 11:31:23 2014 +0000 Tweak these two test cases for 32 bit systems diff --git a/bin/varnishtest/tests/c00044.vtc b/bin/varnishtest/tests/c00044.vtc index 091e8a8..587bf38 100644 --- a/bin/varnishtest/tests/c00044.vtc +++ b/bin/varnishtest/tests/c00044.vtc @@ -42,7 +42,7 @@ varnish v1 -expect SMA.Transient.g_bytes == 0 varnish v1 -expect SMA.s0.g_bytes == 0 varnish v1 -expect SMA.s0.g_space > 1000000 varnish v1 -expect SMA.s1.g_bytes > 1000000 -varnish v1 -expect SMA.s1.g_space < 180 +varnish v1 -expect SMA.s1.g_space < 200 varnish v1 -expect SMA.s2.g_bytes == 0 varnish v1 -expect SMA.s2.g_space > 1000000 @@ -57,9 +57,9 @@ varnish v1 -expect SMA.Transient.g_bytes == 0 varnish v1 -expect SMA.s0.g_bytes == 0 varnish v1 -expect SMA.s0.g_space > 1000000 varnish v1 -expect SMA.s1.g_bytes > 1000000 -varnish v1 -expect SMA.s1.g_space < 180 +varnish v1 -expect SMA.s1.g_space < 200 varnish v1 -expect SMA.s2.g_bytes > 1000000 -varnish v1 -expect SMA.s2.g_space < 180 +varnish v1 -expect SMA.s2.g_space < 200 client c1 { txreq -url /burp @@ -70,11 +70,11 @@ client c1 { varnish v1 -expect SMA.Transient.g_bytes == 0 varnish v1 -expect SMA.s0.g_bytes > 1000000 -varnish v1 -expect SMA.s0.g_space < 180 +varnish v1 -expect SMA.s0.g_space < 200 varnish v1 -expect SMA.s1.g_bytes > 1000000 -varnish v1 -expect SMA.s1.g_space < 180 +varnish v1 -expect SMA.s1.g_space < 200 varnish v1 -expect SMA.s2.g_bytes > 1000000 -varnish v1 -expect SMA.s2.g_space < 180 +varnish v1 -expect SMA.s2.g_space < 200 client c1 { txreq -url /foo1 diff --git a/bin/varnishtest/tests/c00045.vtc b/bin/varnishtest/tests/c00045.vtc index 33b7b9e..77425cb 100644 --- a/bin/varnishtest/tests/c00045.vtc +++ b/bin/varnishtest/tests/c00045.vtc @@ -48,7 +48,7 @@ client c1 { varnish v1 -expect n_lru_nuked == 1 varnish v1 -expect SMA.Transient.g_bytes == 0 varnish v1 -expect SMA.s0.g_bytes > 1000000 -varnish v1 -expect SMA.s0.g_space < 1180 +varnish v1 -expect SMA.s0.g_space < 1190 varnish v1 -expect SMA.s1.g_bytes == 0 varnish v1 -expect SMA.s1.g_space > 1000000 varnish v1 -expect SMA.s2.g_bytes == 0 @@ -64,7 +64,7 @@ client c1 { varnish v1 -expect n_lru_nuked == 2 varnish v1 -expect SMA.Transient.g_bytes == 0 varnish v1 -expect SMA.s0.g_bytes > 1000000 -varnish v1 -expect SMA.s0.g_space < 1182 +varnish v1 -expect SMA.s0.g_space < 1192 varnish v1 -expect SMA.s1.g_bytes == 0 varnish v1 -expect SMA.s1.g_space > 1000000 varnish v1 -expect SMA.s2.g_bytes == 0 From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] cf3b7c2 Typo Message-ID: commit cf3b7c241478b51f2de5e1a46b6db1e233c72a4d Author: Federico G. Schwindt Date: Mon Jul 28 14:03:22 2014 +0100 Typo diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 9feea1a..500916e 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -92,7 +92,7 @@ static const char PROTECTED_TEXT[] = static const char ONLY_ROOT_TEXT[] = "\n\n" - "NB: This parameter only works of varnishd is run as root."; + "NB: This parameter only works if varnishd is run as root."; /*--------------------------------------------------------------------*/ From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] a36579b Zap space Message-ID: commit a36579baf58de25458103fb8eeeffe586ba90cd8 Author: Federico G. Schwindt Date: Mon Jul 28 14:03:41 2014 +0100 Zap space diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index bb0a85e..66dbc9b 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -240,7 +240,7 @@ RFC2616_Body(struct busyobj *bo, struct dstat *stats) } if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) { - stats->fetch_chunked++; + stats->fetch_chunked++; return (BS_CHUNKED); } From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 7a4fe10 Remove text - it no longer applies in V4 Message-ID: commit 7a4fe1005bbe23e08377809d622994a764866c49 Author: Federico G. Schwindt Date: Mon Jul 28 14:04:04 2014 +0100 Remove text - it no longer applies in V4 diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 21173c4..4a3e820 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -436,8 +436,6 @@ sp_variables = [ Deliver the object to the client directly without fetching the whole object into varnish. If this request is pass'ed it will not be stored in memory. - As of Varnish Cache 3.0 the object will marked as busy - as it is delivered so only client can access the object. """ ), ('beresp.do_gzip', From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 0bd10ea Correctly fail bad reads in req.body Message-ID: commit 0bd10ea0556d872f6115f6008233477e9ae1e62a Author: Poul-Henning Kamp Date: Tue Jul 29 11:22:16 2014 +0000 Correctly fail bad reads in req.body Spotted and fixed by: Nils Goroll Fixes #1562 diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 31b65e2..4bf643d 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -559,6 +559,10 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv) VSLb(req->vsl, SLT_VCL_Error, "Uncached req.body can only be consumed once."); return (-1); + case REQ_BODY_FAIL: + VSLb(req->vsl, SLT_FetchError, + "Had failed reading req.body before."); + return (-1); default: WRONG("Wrong req_body_status in HTTP1_IterateReqBody()"); } diff --git a/bin/varnishtest/tests/r01562.vtc b/bin/varnishtest/tests/r01562.vtc new file mode 100644 index 0000000..af979ab --- /dev/null +++ b/bin/varnishtest/tests/r01562.vtc @@ -0,0 +1,49 @@ +varnishtest "retrying a short client body read should not panic varnish" + +server s1 { + non-fatal + rxreq + txresp -status 200 -hdr "Foo: BAR" -body "1234" +} -start + +server s2 { + non-fatal + rxreq + txresp -status 200 -hdr "Foo: Foo" -body "56" +} -start + +varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend { + sub vcl_recv { + return (pass); + } + sub vcl_backend_fetch { + if (bereq.retries >= 1) { + set bereq.backend = s2; + } else { + set bereq.backend = s1; + } + } + sub vcl_backend_error { + return (retry); + } +} -start + +varnish v1 -cliok "param.set debug +syncvsl" + +client c1 { + txreq -req "POST" -nolen -hdr "Content-Length: 10000" -bodylen 9999 +} -run + +delay .4 + +server s1 { + rxreq + txresp -status 200 -bodylen 11 +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.bodylen == 11 +} -run From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 1026d14 Rearrange some deck-chairs so panic messages can get better. Message-ID: commit 1026d14191a584e0a9d624218765e8de83c15aa1 Author: Poul-Henning Kamp Date: Tue Jul 29 17:09:17 2014 +0000 Rearrange some deck-chairs so panic messages can get better. diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 4a3e820..a2947c6 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -911,15 +911,19 @@ typedef void vcl_fini_f(struct cli *); typedef int vcl_func_f(const struct vrt_ctx *ctx); """) +def tbl40(a, b): + while len(a.expandtabs()) < 40: + a += "\t" + return a + b fo.write("\n/* VCL Methods */\n") -n = 0 +n = 1 for i in returns: - fo.write("#define VCL_MET_%s\t\t(1U << %d)\n" % (i[0].upper(), n)) + fo.write(tbl40("#define VCL_MET_%s" % i[0].upper(), "(1U << %d)\n" % n)) n += 1 -fo.write("\n#define VCL_MET_MAX\t\t%d\n" % n) -fo.write("\n#define VCL_MET_MASK\t\t0x%x\n" % ((1 << n) - 1)) +fo.write("\n" + tbl40("#define VCL_MET_MAX", "%d\n" % n)) +fo.write("\n" + tbl40("#define VCL_MET_MASK", "0x%x\n" % ((1 << n) - 1))) fo.write("\n/* VCL Returns */\n") @@ -927,10 +931,10 @@ n = 0 l = list(rets.keys()) l.sort() for i in l: - fo.write("#define VCL_RET_%s\t\t%d\n" % (i.upper(), n)) + fo.write(tbl40("#define VCL_RET_%s" % i.upper(), "%d\n" % n)) n += 1 -fo.write("\n#define VCL_RET_MAX\t\t%d\n" % n) +fo.write("\n" + tbl40("#define VCL_RET_MAX", "%d\n" % n)) fo.write(""" diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 31212aa..4077c1c 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -65,6 +65,7 @@ #include "vfil.h" struct method method_tab[] = { + { "none", 0U, 0}, #define VCL_MET_MAC(l,U,m) { "vcl_"#l, m, VCL_MET_##U }, #include "tbl/vcl_returns.h" #undef VCL_MET_MAC @@ -666,7 +667,7 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp) return (vcc_DestroyTokenList(tl, NULL)); /* Emit method functions */ - for (i = 0; i < VCL_MET_MAX; i++) { + for (i = 1; i < VCL_MET_MAX; i++) { Fh(tl, 1, "\nint __match_proto__(vcl_func_f)\n"); Fh(tl, 1, "VGC_function_%s(const struct vrt_ctx *ctx);\n", From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] f1ca0b6 Use SI case for modifiers, fix spelling of (E)xa Message-ID: commit f1ca0b62566a22e2ea7e30c1671838aba91fef8e Author: Andreas Plesner Date: Wed Jul 30 10:55:22 2014 +0200 Use SI case for modifiers, fix spelling of (E)xa diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index e1d02bb..dbe5db8 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -76,10 +76,10 @@ OPTIONS -l shl[,free[,fill]] Specifies size of shmlog file. shl is the store for the - shared memory log records [80m], free is the store for other - allocations [1m] and fill determines how the log is [+]. - Scaling suffixes like 'k', 'm' can be used up to - (e)tabytes. Default is 80 Megabytes. + shared memory log records [80M], free is the store for other + allocations [1M] and fill determines how the log is [+]. + Scaling suffixes like 'k', 'M' can be used up to + (E)xabytes. Default is 80 Megabytes. -M address:port Connect to this port and offer the command line interface. From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 68b8768 Restrict charset used for file_id. Message-ID: commit 68b87684cbc1ff48174eb6bb3a58d453fbfb55af Author: Poul-Henning Kamp Date: Wed Jul 30 16:39:37 2014 +0000 Restrict charset used for file_id. Suggested by: scoof diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 25a901f..0553015 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -239,7 +239,7 @@ class Vmod(object): # fo.write("\t.file_id = \"") for i in range(32): - fo.write("%c" % random.randint(0x23,0x5b)) + fo.write("%c" % random.randint(0x40,0x5a)) fo.write("\",\n") fo.write("};\n") From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 47bb3c7 Update description Message-ID: commit 47bb3c7890ae95900f76bd02253332e5e03cf48d Author: Federico G. Schwindt Date: Wed Jul 30 23:36:23 2014 +0100 Update description obj.{grace,ttl} are read-only. diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index a2947c6..3d56bc5 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -550,14 +550,13 @@ sp_variables = [ ( 'hit', ), ( ), """ The object's remaining time to live, in seconds. - obj.ttl is writable. """ ), ('obj.grace', 'DURATION', ( 'hit', ), ( ), """ - The object's grace period in seconds. obj.grace is writable. + The object's grace period in seconds. """ ), ('obj.keep', From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 13ff334 Fix printf formats for MacOS 10.9.4 Message-ID: commit 13ff334d699311a3e62b7db215797ea15cbe5b7d Author: Poul-Henning Kamp Date: Thu Jul 31 09:16:58 2014 +0000 Fix printf formats for MacOS 10.9.4 Reported by: Meng Zhang diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 4bf643d..d83d522 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -706,12 +706,12 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize) http_Unset(req->http0, H_Content_Length); http_Unset(req->http0, H_Transfer_Encoding); http_PrintfHeader(req->http0, "Content-Length: %ju", - req->req_bodybytes); + (uintmax_t)req->req_bodybytes); http_Unset(req->http, H_Content_Length); http_Unset(req->http, H_Transfer_Encoding); http_PrintfHeader(req->http, "Content-Length: %ju", - req->req_bodybytes); + (uintmax_t)req->req_bodybytes); req->req_body_status = REQ_BODY_CACHED; } From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 0d26b91 Bump the limit on how many groups we can deal with. Message-ID: commit 0d26b91f11f45e1ee0aead24729ca2919cbe57c3 Author: Poul-Henning Kamp Date: Thu Jul 31 09:48:11 2014 +0000 Bump the limit on how many groups we can deal with. diff --git a/bin/varnishd/mgt/mgt_sandbox.c b/bin/varnishd/mgt/mgt_sandbox.c index 973ec83..69ce5fa 100644 --- a/bin/varnishd/mgt/mgt_sandbox.c +++ b/bin/varnishd/mgt/mgt_sandbox.c @@ -63,9 +63,9 @@ static void __match_proto__(mgt_sandbox_f) mgt_sandbox_unix(enum sandbox_e who) { -#define NGID 10 - gid_t gid_list[NGID]; +#define NGID 2000 int i; + gid_t gid_list[NGID]; if (geteuid() != 0) { REPORT0(LOG_INFO, "Not running as root, no priv-sep"); From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 873a6a0 Default the http timeout to half of the total testduration. Message-ID: commit 873a6a03c71cdd64681d5a78a4a0100eb158914e Author: Poul-Henning Kamp Date: Thu Jul 31 09:49:04 2014 +0000 Default the http timeout to half of the total testduration. Fixes #1563 diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 28b7340..f2998d4 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1227,7 +1227,7 @@ http_process(struct vtclog *vl, const char *spec, int sock, int *sfd) ALLOC_OBJ(hp, HTTP_MAGIC); AN(hp); hp->fd = sock; - hp->timeout = 15000; + hp->timeout = vtc_maxdur * 1000 / 2; hp->nrxbuf = 2048*1024; hp->vsb = VSB_new_auto(); hp->rxbuf = malloc(hp->nrxbuf); /* XXX */ From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 0d73493 Fix breakage Message-ID: commit 0d7349364e17674e701a752cdb4e8490669ce0fc Author: Federico G. Schwindt Date: Thu Jul 31 12:12:33 2014 +0100 Fix breakage Un-static-fy vtc_maxdur diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index a31c83f..c70b267 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -52,6 +52,7 @@ volatile sig_atomic_t vtc_error; /* Error encountered */ int vtc_stop; /* Stops current test without error */ pthread_t vtc_thread; static struct vtclog *vltop; +unsigned vtc_maxdur; /* Max duration of any test */ /********************************************************************** * Macro facility diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 6c56926..b877e26 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -69,6 +69,7 @@ extern volatile sig_atomic_t vtc_error; /* Error, bail out */ extern int vtc_stop; /* Abandon current test, no error */ extern pthread_t vtc_thread; extern int iflg; +extern unsigned vtc_maxdur; void init_sema(void); diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index de08003..823a0a2 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -73,12 +73,12 @@ struct vtc_job { }; int iflg = 0; +unsigned vtc_maxdur = 60; static VTAILQ_HEAD(, vtc_tst) tst_head = VTAILQ_HEAD_INITIALIZER(tst_head); static struct vev_base *vb; static int njob = 0; static int npar = 1; /* Number of parallel tests */ -static unsigned vtc_maxdur = 60; /* Max duration of any test */ static int vtc_continue; /* Continue on error */ static int vtc_verbosity = 1; /* Verbosity Level */ static int vtc_good; From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 02ac125 Ensure bereq changes are not lost across vcl_backend_xxx methods Message-ID: commit 02ac12513cb5b7e4fda7e12c3a88f3000dd198e1 Author: Federico G. Schwindt Date: Thu Jul 31 12:48:03 2014 +0100 Ensure bereq changes are not lost across vcl_backend_xxx methods Add std.rollback() to rollback req or bereq and mark the builtin rollback as deprecated. Original diff by Nils Goroll. Tweaks and reworked after phk comments by me. Fixes #1512 Conflicts: bin/varnishd/cache/cache_fetch.c diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 7ce01db..e455048 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -545,6 +545,7 @@ struct busyobj { enum busyobj_state_e state; struct ws ws[1]; + char *ws_bo; struct vbc *vbc; struct http *bereq0; struct http *bereq; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index e5c7921..92168e8 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -213,6 +213,10 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo) } } + HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod); + bo->ws_bo = WS_Snapshot(bo->ws); + HTTP_Copy(bo->bereq, bo->bereq0); + VBO_setstate(bo, BOS_REQ_DONE); return (F_STP_STARTFETCH); } @@ -267,12 +271,10 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) else AZ(bo->req); - HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod); - HTTP_Copy(bo->bereq, bo->bereq0); - http_PrintfHeader(bo->bereq, "X-Varnish: %u", bo->vsl->wid & VSL_IDENTMASK); + VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws); bo->uncacheable = bo->do_pass; diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 4038693..edfa2a5 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -357,13 +357,22 @@ VRT_BOOL_string(unsigned val) /*--------------------------------------------------------------------*/ void -VRT_Rollback(const struct vrt_ctx *ctx) +VRT_Rollback(const struct vrt_ctx *ctx, const struct http *hp) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - HTTP_Copy(ctx->req->http, ctx->req->http0); - WS_Reset(ctx->req->ws, ctx->req->ws_req); + CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); + if (hp == ctx->http_req) { + CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); + HTTP_Copy(ctx->req->http, ctx->req->http0); + WS_Reset(ctx->req->ws, ctx->req->ws_req); + } else if (hp == ctx->http_bereq) { + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + HTTP_Copy(ctx->bo->bereq, ctx->bo->bereq0); + WS_Reset(ctx->bo->bereq->ws, ctx->bo->ws_bo); + WS_Reset(ctx->bo->ws, ctx->bo->ws_bo); + } else + WRONG("VRT_Rollback 'hp' invalid"); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishtest/tests/m00017.vtc b/bin/varnishtest/tests/m00017.vtc new file mode 100644 index 0000000..5622ec0 --- /dev/null +++ b/bin/varnishtest/tests/m00017.vtc @@ -0,0 +1,69 @@ +varnishtest "Test std.rollback" + +server s1 { + rxreq + expect req.url == "/foo" + expect req.http.foobar == "bar" + txresp -status 400 + accept + rxreq + expect req.url == "/bar" + expect req.http.foobar == "foo" + txresp + accept + rxreq + expect req.url == "/baz" + expect req.http.foobar == "qux" + txresp -status 400 + accept + rxreq + expect req.url == "/qux" + expect req.http.foobar == "baz" + txresp +} -start + +varnish v1 -vcl+backend { + import ${vmod_std}; + + sub vcl_recv { + if (req.url == "/foo") { + set req.http.foobar = "bar"; + } + } + + sub vcl_deliver { + if (resp.status == 400) { + std.rollback(req); + set req.url = "/bar"; + return (restart); + } + } +} -start + +client c1 { + txreq -url "/foo" -hdr "foobar: foo" + rxresp +} -run + +varnish v1 -vcl+backend { + import ${vmod_std}; + + sub vcl_backend_fetch { + if (bereq.url == "/baz") { + set bereq.http.foobar = "qux"; + } + } + + sub vcl_backend_response { + if (beresp.status == 400) { + std.rollback(bereq); + set bereq.url = "/qux"; + return (retry); + } + } +} + +client c1 { + txreq -url "/baz" -hdr "foobar: baz" + rxresp +} -run diff --git a/bin/varnishtest/tests/r01512.vtc b/bin/varnishtest/tests/r01512.vtc new file mode 100644 index 0000000..71496a2 --- /dev/null +++ b/bin/varnishtest/tests/r01512.vtc @@ -0,0 +1,56 @@ +varnishtest "Regression test for #1512" + +# First test bereq changes across v_b_r and v_b_f + +server s1 { + rxreq + txresp -status 700 +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_fetch { + if (bereq.http.x-abandon == "1") { + return (abandon); + } + } + sub vcl_backend_response { + if (beresp.status == 700) { + set bereq.http.x-abandon = "1"; + return (retry); + } + } + sub vcl_synth { + set resp.status = 701; + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 701 +} -run + +# Then across v_b_e and v_b_f + +varnish v1 -vcl { + backend bad { .host = "${bad_ip}"; } + sub vcl_backend_fetch { + set bereq.backend = bad; + if (bereq.http.x-abandon == "2") { + return (abandon); + } + } + sub vcl_backend_error { + set bereq.http.x-abandon = "2"; + return (retry); + } + sub vcl_synth { + set resp.status = 702; + } +} + +client c1 { + txreq + rxresp + expect resp.status == 702 +} -run diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 743689e..6828f99 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -369,7 +369,8 @@ return() in the request handling state machine. rollback() - Restore request HTTP headers to their original state. + Restore *req* HTTP headers to their original state. This function is + deprecated. Use std.rollback() instead. synthetic(STRING) Prepare a synthetic response body containing the STRING. Available in diff --git a/include/vrt.h b/include/vrt.h index efc9c94..ead55d3 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -249,7 +249,7 @@ void VRT_hashdata(const struct vrt_ctx *, const char *str, ...); int VRT_strcmp(const char *s1, const char *s2); void VRT_memmove(void *dst, const void *src, unsigned len); -void VRT_Rollback(const struct vrt_ctx *); +void VRT_Rollback(const struct vrt_ctx *, const struct http *); /* Synthetic pages */ void VRT_synth_page(const struct vrt_ctx *, const char *, ...); diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c index 9b1d22d..1b0c4b2 100644 --- a/lib/libvcc/vcc_action.c +++ b/lib/libvcc/vcc_action.c @@ -376,7 +376,7 @@ parse_rollback(struct vcc *tl) { vcc_NextToken(tl); - Fb(tl, 1, "VRT_Rollback(ctx);\n"); + Fb(tl, 1, "VRT_Rollback(ctx, VRT_r_req(ctx));\n"); } /*--------------------------------------------------------------------*/ diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index fbee007..5bcc403 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -174,6 +174,13 @@ $Function INT port(IP) Description Returns the port number of an IP address. +$Function VOID rollback(HTTP) + +Description + Restore *h* HTTP headers to their original state. +Example + std.rollback(bereq); + $Function VOID timestamp(STRING) Description diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 718fb71..c5d0cfe 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -205,6 +205,12 @@ vmod_port(const struct vrt_ctx *ctx, VCL_IP ip) return (VSA_Port(ip)); } +VCL_VOID __match_proto__(td_std_rollback) +vmod_rollback(const struct vrt_ctx *ctx, VCL_HTTP hp) +{ + VRT_Rollback(ctx, hp); +} + VCL_VOID __match_proto__(td_std_timestamp) vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label) { From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] a973287 More coverage of VRT variables Message-ID: commit a97328796ce7ee1f94f92c9c6348858231259d5a Author: Poul-Henning Kamp Date: Thu Jul 31 14:58:32 2014 +0000 More coverage of VRT variables diff --git a/bin/varnishtest/tests/v00025.vtc b/bin/varnishtest/tests/v00025.vtc index 1be49af..67ac41a 100644 --- a/bin/varnishtest/tests/v00025.vtc +++ b/bin/varnishtest/tests/v00025.vtc @@ -5,16 +5,29 @@ server s1 { txresp } -start -varnish v1 -vcl+backend { +varnish v1 -arg "-i J.F.Nobody" -vcl+backend { import ${vmod_std}; + sub vcl_recv { + set client.identity = "Samuel B. Nobody"; + } + sub vcl_deliver { set resp.http.server_port = std.port(server.ip); + set resp.http.id = server.identity; + set resp.http.esi = req.esi; + set resp.http.be = req.backend_hint; + set resp.http.c_id = client.identity; } sub vcl_backend_response { set beresp.http.backend = bereq.backend; + set beresp.http.keep = beresp.keep; + set beresp.http.hint = beresp.storage_hint; + set beresp.http.be_ip = beresp.backend.ip; + set beresp.http.be_nm = beresp.backend.name; + set beresp.http.unc = bereq.uncacheable; if (beresp.ttl > 3 s) { set beresp.http.ttl = "long"; } else { @@ -23,6 +36,10 @@ varnish v1 -vcl+backend { } sub vcl_hit { + if (obj.proto) { } + if (obj.reason) { } + if (obj.uncacheable) { } + if (obj.keep > 1m) { } if (obj.grace < 3m) { return (deliver); } @@ -32,6 +49,7 @@ varnish v1 -vcl+backend { } sub vcl_backend_fetch { + set bereq.uncacheable = false; if (bereq.between_bytes_timeout < 10s) { set bereq.http.quick = "please"; } From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 6d58b5e Do a "backend.list" to try to provoke any pending panics better, this CLI command goes to the varnish worker process if it is running and gives an error (which we ignore) if it isn't. Message-ID: commit 6d58b5e039bc2e0bce2c81097eb192d1205360db Author: Poul-Henning Kamp Date: Tue Aug 5 10:06:16 2014 +0000 Do a "backend.list" to try to provoke any pending panics better, this CLI command goes to the varnish worker process if it is running and gives an error (which we ignore) if it isn't. diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index c559810..469effd 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -576,11 +576,13 @@ varnish_wait(struct varnish *v) void *p; int status, r; struct rusage ru; + char *resp; if (v->cli_fd < 0) return; if (vtc_error) (void)sleep(1); /* give panic messages a chance */ + varnish_ask_cli(v, "backend.list", &resp); varnish_stop(v); vtc_log(v->vl, 2, "Wait"); AZ(close(v->cli_fd)); From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 2a5ad6f Varnishstat hitrate implementation Message-ID: commit 2a5ad6f78667b1c93e9763d7417b7755ff107b36 Author: arianna Date: Fri Aug 8 12:56:55 2014 +0200 Varnishstat hitrate implementation diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index fd80580..8ce35f1 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -91,6 +91,15 @@ struct pt { struct ma ma_10, ma_100, ma_1000; }; +struct hitrate { + double lt; + uint64_t lhit, lmiss; + struct ma hr_10; + struct ma hr_100; + struct ma hr_1000; +}; +static struct hitrate hitrate; + static VTAILQ_HEAD(, pt) ptlist = VTAILQ_HEAD_INITIALIZER(ptlist); static int n_ptlist = 0; static int n_ptarray = 0; @@ -119,6 +128,19 @@ static double t_sample = 0.; static double interval = 1.; static void +init_hitrate(void) +{ + memset(&hitrate, 0, sizeof (struct hitrate)); + if (VSC_C_main != NULL) { + hitrate.lhit = VSC_C_main->cache_hit; + hitrate.lmiss = VSC_C_main->cache_miss; + } + hitrate.hr_10.nmax = 10; + hitrate.hr_100.nmax = 100; + hitrate.hr_1000.nmax = 1000; +} + +static void update_ma(struct ma *ma, double val) { AN(ma); @@ -348,10 +370,6 @@ sample_points(void) { struct pt *pt; - t_sample = VTIM_mono(); - sample = 0; - redraw = 1; - VTAILQ_FOREACH(pt, &ptlist, list) { AN(pt->vpt); AN(pt->ptr); @@ -390,6 +408,46 @@ sample_points(void) } static void +sample_hitrate(void) +{ + double tv,dt; + double hr, mr, ratio; + uint64_t hit, miss; + + if (VSC_C_mgt == NULL) + return; + + tv = VTIM_mono(); + dt = tv - hitrate.lt; + hitrate.lt= tv; + + hit = VSC_C_main->cache_hit; + miss = VSC_C_main->cache_miss; + hr = (hit - hitrate.lhit) / dt; + mr = (miss - hitrate.lmiss) / dt; + hitrate.lhit = hit; + hitrate.lmiss = miss; + + if (hr + mr != 0) + ratio = hr / (hr + mr); + else + ratio = 0; + update_ma(&hitrate.hr_10, ratio); + update_ma(&hitrate.hr_100, ratio); + update_ma(&hitrate.hr_1000, ratio); +} + +static void +sample_data(void) +{ + t_sample = VTIM_mono(); + sample = 0; + redraw = 1; + sample_points(); + sample_hitrate(); +} + +static void make_windows(void) { int Y, X; @@ -489,8 +547,18 @@ draw_status(void) AN(w_status); werase(w_status); - if (VSC_C_mgt != NULL) + + if (VSC_C_mgt != NULL) { up_mgt = VSC_C_mgt->uptime; + if( COLS > 70) { + mvwprintw(w_status, 0, (getmaxx (w_status) - 37), + "Hitrate n: %8u %8u %8u", hitrate.hr_10.n, hitrate.hr_100.n, + hitrate.hr_1000.n); + mvwprintw(w_status, 1, (getmaxx (w_status) - 37), + " avg(n): %8.4f %8.4f %8.4f", hitrate.hr_10.acc, + hitrate.hr_100.acc, hitrate.hr_1000.acc); + } + } if (VSC_C_main != NULL) up_chld = VSC_C_main->uptime; @@ -866,8 +934,10 @@ do_curses(struct VSM_data *vd, int delay) VSC_C_mgt = VSC_Mgt(vd, &f_mgt); VSC_C_main = VSC_Main(vd, &f_main); + init_hitrate(); while (keep_running) { if (VSM_Abandoned(vd)) { + init_hitrate(); delete_pt_list(); VSM_Close(vd); VSM_Open(vd); @@ -881,7 +951,7 @@ do_curses(struct VSM_data *vd, int delay) if (now - t_sample > interval) sample = 1; if (sample) - sample_points(); + sample_data(); if (rebuild) build_pt_array(); if (redraw) From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 1c965a1 Fix compilation if SO_{SND, RCV}TIMEO_WORKS is not defined Message-ID: commit 1c965a1a612f3ae048241b0eccc735c0c6c079ae Author: Federico G. Schwindt Date: Mon Aug 11 16:14:12 2014 +0100 Fix compilation if SO_{SND,RCV}TIMEO_WORKS is not defined Fixes #1567 diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index bad4ed9..af86b2b 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -126,13 +126,13 @@ vca_tcp_opt_init(void) { int n; int one = 1; - // int zero = 0; struct tcp_opt *to; struct timeval tv; int chg = 0; -#ifdef HAVE_TCP_KEEP int x; -#endif + + (void)tv; + (void)x; for (n = 0; n < n_tcp_opts; n++) { to = &tcp_opts[n]; From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 25ad44e Strictly speaking Message-ID: commit 25ad44eabf0b857eb00383e08a6bbe5d83969405 Author: Poul-Henning Kamp Date: Tue Aug 12 08:27:54 2014 +0000 Strictly speaking int foo() { int bar; (void)bar; [...] } Isn't valid C because bar has an undefined value when you cast its value away. Yes, many thanks to ISO-C for that gem. Memset the variables to zero instead. diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index af86b2b..7afcef6 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -131,8 +131,8 @@ vca_tcp_opt_init(void) int chg = 0; int x; - (void)tv; - (void)x; + memset(&tv, 0, sizeof tv); + memset(&x, 0, sizeof x); for (n = 0; n < n_tcp_opts; n++) { to = &tcp_opts[n]; From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 0715bd7 Update the XHTML in the default guru page. Message-ID: commit 0715bd73c63087076357ce24fb639fddbcbf3973 Author: Poul-Henning Kamp Date: Wed Aug 13 17:15:58 2014 +0000 Update the XHTML in the default guru page. Submitted by: mattrobenolt Approved by: mithradir diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl index cfb2238..6ead64e 100644 --- a/bin/varnishd/builtin.vcl +++ b/bin/varnishd/builtin.vcl @@ -126,10 +126,7 @@ sub vcl_deliver { sub vcl_synth { set resp.http.Content-Type = "text/html; charset=utf-8"; set resp.http.Retry-After = "5"; - synthetic( {" - - + synthetic( {" "} + resp.status + " " + resp.reason + {" @@ -173,10 +170,7 @@ sub vcl_backend_response { sub vcl_backend_error { set beresp.http.Content-Type = "text/html; charset=utf-8"; set beresp.http.Retry-After = "5"; - synthetic( {" - - + synthetic( {" "} + beresp.status + " " + beresp.reason + {" From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 2852b2d Skip NULL and empty arguments when hashing. Hash all other arguments. Message-ID: commit 2852b2d729ef0122ea6f08af2350f49ff49848f1 Author: Nils Goroll Date: Mon Aug 18 12:32:19 2014 +0200 Skip NULL and empty arguments when hashing. Hash all other arguments. When multiple arguments were passed to vmod_hash_backend, only the first argument was re-used for the number of arguments passed. Fixes #1568 diff --git a/bin/varnishtest/tests/v00026.vtc b/bin/varnishtest/tests/v00026.vtc index e1b77b9..711861d 100644 --- a/bin/varnishtest/tests/v00026.vtc +++ b/bin/varnishtest/tests/v00026.vtc @@ -12,6 +12,10 @@ server s2 { txresp -hdr "Foo: 2" -body "2" rxreq txresp -hdr "Foo: 4" -body "4" + rxreq + txresp -hdr "Foo: 6" -body "6" + rxreq + txresp -hdr "Foo: 8" -body "8" } -start @@ -28,7 +32,15 @@ varnish v1 -vcl+backend { return(pass); } sub vcl_backend_fetch { - set bereq.backend = h1.backend(bereq.url); + if (bereq.url == "/nohdr") { + set bereq.backend = h1.backend(bereq.http.Void); + } else if (bereq.url == "/emptystring") { + set bereq.backend = h1.backend(""); + } else if (bereq.url == "/13") { + set bereq.backend = h1.backend(bereq.http.Void + "" + bereq.url); + } else { + set bereq.backend = h1.backend(bereq.url); + } } } -start @@ -51,5 +63,13 @@ client c1 { rxresp expect resp.http.foo == "4" + txreq -url /emptystring + rxresp + expect resp.http.foo == "6" + + txreq -url /nohdr + rxresp + expect resp.http.foo == "8" + } -run diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c index b0c10aa..090039f 100644 --- a/lib/libvmod_directors/hash.c +++ b/lib/libvmod_directors/hash.c @@ -110,7 +110,8 @@ vmod_hash_backend(const struct vrt_ctx *ctx, struct vmod_directors_hash *rr, va_start(ap, arg); p = arg; while (p != vrt_magic_string_end) { - SHA256_Update(&sha_ctx, arg, strlen(arg)); + if (p != NULL && *p != '\0') + SHA256_Update(&sha_ctx, p, strlen(p)); p = va_arg(ap, const char *); } va_end(ap); From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] f7abaec Describe bereq.retries Message-ID: commit f7abaec620ebb028d69e140fa3cee023a15e6fb5 Author: Federico G. Schwindt Date: Mon Aug 18 19:35:06 2014 +0100 Describe bereq.retries diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 3d56bc5..6e3c91f 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -319,6 +319,7 @@ sp_variables = [ 'INT', ( 'backend',), ( ), """ + A count of how many times this request has been retried. """ ), ('bereq.backend', From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 76dd3b7 Rework autocrap configuration for libedit/libreadline Message-ID: commit 76dd3b7be9e27b48e5673949606795df03e2201e Author: Nils Goroll Date: Mon Aug 18 21:26:17 2014 +0200 Rework autocrap configuration for libedit/libreadline As before, libedit is preferred over libreadline. Fail configure if neither is found or if libedit includes are missing. Require readline history support (as varnishadm needs it). Previously, if only libreadline was found, all binaries were linked against it. Now only binaries getting linked with LIBEDIT_LIBS will get linked aginst readline if libedit is not found. If configure succeeds, a build should not fail any longer due to libedit/libreadline headers missing. Fixes #1555 diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index 0c2ecc1..26019a5 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -34,15 +34,17 @@ #include -#ifdef HAVE_EDIT_READLINE_READLINE_H -# include +#ifdef HAVE_LIBEDIT +# include #elif HAVE_READLINE_READLINE_H # include # ifdef HAVE_READLINE_HISTORY_H # include +# else +# error missing history.h - this should have got caught in configure # endif #else -# include +# error missing readline.h - this should have got caught in configure #endif #include diff --git a/configure.ac b/configure.ac index ced8495..c8a36f5 100644 --- a/configure.ac +++ b/configure.ac @@ -140,11 +140,25 @@ AC_SUBST(PCRE_CFLAGS) AC_SUBST(PCRE_LIBS) PKG_CHECK_MODULES([LIBEDIT], [libedit], - [AC_DEFINE([HAVE_LIBEDIT], [1], [Define we have libedit])], - [AX_LIB_READLINE]) -if test "$ac_cv_have_readline" = no; then - AC_MSG_ERROR([libedit or readline not found]) -fi + # having the module does not imply having the header + [AC_CHECK_HEADERS([editline/readline.h], + [AC_DEFINE([HAVE_LIBEDIT], [1], [Define if we have libedit])], + [AC_MSG_ERROR([Found libedit, but header file is missing. Hint: Install dev package?])])], + [ + # AX_LIB_READLINE overwrites LIBS which leads to every binary getting + # linked against libreadline uselessly. So we re-use LIBEDIT_LIBS which + # we have for libedit to add the lib specifically where needed + save_LIBS="${LIBS}" + AX_LIB_READLINE + LIBS="${save_LIBS}" + if test "$ax_cv_lib_readline" = "no"; then + AC_MSG_ERROR([neither libedit nor another readline compatible library found]) + fi + if test "x$ax_cv_lib_readline_history" != "xyes"; then + AC_MSG_ERROR([need readline history support]) + fi + LIBEDIT_LIBS="$ax_cv_lib_readline" + ]) # Checks for header files. AC_HEADER_STDC From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] 2911138 wrong kind of white space Message-ID: commit 2911138adb86f8cc5c189e3f3d83d884c479f653 Author: Nils Goroll Date: Mon Aug 18 21:30:10 2014 +0200 wrong kind of white space diff --git a/bin/varnishtest/tests/v00026.vtc b/bin/varnishtest/tests/v00026.vtc index 711861d..2b52238 100644 --- a/bin/varnishtest/tests/v00026.vtc +++ b/bin/varnishtest/tests/v00026.vtc @@ -33,7 +33,7 @@ varnish v1 -vcl+backend { } sub vcl_backend_fetch { if (bereq.url == "/nohdr") { - set bereq.backend = h1.backend(bereq.http.Void); + set bereq.backend = h1.backend(bereq.http.Void); } else if (bereq.url == "/emptystring") { set bereq.backend = h1.backend(""); } else if (bereq.url == "/13") { From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] f54e14f Tweak size-sensitive test case for 32bit systems Message-ID: commit f54e14f64007f3fbeda2ce155620e1c065c3ada7 Author: Poul-Henning Kamp Date: Mon Aug 18 22:48:18 2014 +0000 Tweak size-sensitive test case for 32bit systems Conflicts: bin/varnishtest/tests/r01140.vtc diff --git a/bin/varnishtest/tests/r01140.vtc b/bin/varnishtest/tests/r01140.vtc index 11e2a72..098a653 100644 --- a/bin/varnishtest/tests/r01140.vtc +++ b/bin/varnishtest/tests/r01140.vtc @@ -4,7 +4,7 @@ server s1 { # This response should almost completely fill the storage rxreq expect req.url == /url1 - txresp -bodylen 1048208 + txresp -bodylen 1048408 # The next one should not fit in the storage, ending up in transient # with zero ttl (=shortlived) @@ -31,7 +31,7 @@ client c1 { txreq -url /url1 rxresp expect resp.status == 200 - expect resp.bodylen == 1048208 + expect resp.bodylen == 1048408 } -run delay .1 From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] c1acdee Pass the right string to syslog() Message-ID: commit c1acdee858db4719676e8bf5046a2a2c2595714e Author: Federico G. Schwindt Date: Tue Aug 19 10:42:46 2014 +0100 Pass the right string to syslog() Fixes #1574 diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index c5d0cfe..a74e6a4 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -156,18 +156,18 @@ vmod_log(const struct vrt_ctx *ctx, const char *fmt, ...) VCL_VOID __match_proto__(td_std_syslog) vmod_syslog(const struct vrt_ctx *ctx, VCL_INT fac, const char *fmt, ...) { - char *p; unsigned u; va_list ap; + txt t; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); u = WS_Reserve(ctx->ws, 0); - p = ctx->ws->f; + t.b = ctx->ws->f; va_start(ap, fmt); - p = VRT_StringList(p, u, fmt, ap); + t.e = VRT_StringList(t.b, u, fmt, ap); va_end(ap); - if (p != NULL) - syslog((int)fac, "%s", p); + if (t.e != NULL) + syslog((int)fac, "%s", t.b); WS_Release(ctx->ws, 0); } From lkarsten at varnish-software.com Mon Sep 22 14:38:23 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:23 +0200 Subject: [4.0] e6d50a6 Set default ttl (according to rfc2616 rules) to max-age, do not use Age twice Message-ID: commit e6d50a6b980bfbffce14df81a0093136bd61e55d Author: Nils Goroll Date: Mon Aug 25 12:53:49 2014 +0200 Set default ttl (according to rfc2616 rules) to max-age, do not use Age twice With 160927b690dd076702601b7407eb71bd423c62dd t_origin got corrected by Age, so we already took the age the object had at fetch time into consideration for all other calculations. Fixes #1578 diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 66dbc9b..b0ebaf3 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -135,10 +135,7 @@ RFC2616_Ttl(struct busyobj *bo, double now) else max_age = strtoul(p, NULL, 0); - if (age > max_age) - expp->ttl = 0; - else - expp->ttl = max_age - age; + expp->ttl = max_age; break; } diff --git a/bin/varnishtest/tests/r01578.vtc b/bin/varnishtest/tests/r01578.vtc new file mode 100644 index 0000000..3086b6b --- /dev/null +++ b/bin/varnishtest/tests/r01578.vtc @@ -0,0 +1,41 @@ +varnishtest "max-age and age" + +server s1 { + rxreq + txresp -hdr "Cache-Control: max-age=23" -hdr "Age: 4" -bodylen 40 +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_response { + set beresp.http.x-ttl = beresp.ttl; + } + sub vcl_hit { + set req.http.x-remaining-ttl = obj.ttl; + } + sub vcl_deliver { + set resp.http.x-remaining-ttl = req.http.x-remaining-ttl; + } +} -start + +client c1 { + txreq + rxresp + expect resp.bodylen == 40 + expect resp.http.x-ttl == 23.000 + expect resp.http.Age == 4 + + + delay 2 + txreq + rxresp + expect resp.bodylen == 40 + expect resp.http.x-ttl == 23.000 + expect resp.http.x-remaining-ttl ~ ^17\.0|^16\.9 + + delay 2 + txreq + rxresp + expect resp.bodylen == 40 + expect resp.http.x-ttl == 23.000 + expect resp.http.x-remaining-ttl ~ ^15\.0|^14\.9 +} -run From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 98c949a Document why updating VMODs on the fly does not work. Message-ID: commit 98c949abdd49330ed0106b1b5dbf957e8b43b1f6 Author: Poul-Henning Kamp Date: Tue Aug 26 08:20:36 2014 +0000 Document why updating VMODs on the fly does not work. diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index 033e04b..5ad685f 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -325,3 +325,33 @@ unless they access VMOD specific global state, shared with other VCLs. Traffic in other VCLs which also import this VMOD, will be happening while housekeeping is going on. + +Updating VMODs +============== + +A compiled VMOD is a shared library file which Varnish dlopen(3)'s +using flags RTLD_NOW | RTLD_LOCAL. + +As a general rule, once a file is opened with dlopen(3) you should +never modify it, but it is safe to rename it and put a new file +under the name it had, which is how most tools installs and updates +shared libraries. + +However, when you call dlopen(3) with the same filename multiple +times it will give you the same single copy of the shared library +file, without checking if it was updated in the meantime. + +This is obviously an oversight in the design of the dlopen(3) library +function, but back in the late 1980ies nobody could imagine why a +program would ever want to have multiple different versions of the +same shared library mapped at the same time. + +Varnish does that, and therefore you must restart the worker process +before Varnish will discover an updated VMOD. + +If you want to test a new version of a VMOD, while being able to +instantly switch back to the old version, you will have to install +each version with a distinct filename or in a distinct subdirectory +and use ``import foo from "...";`` to reference it in your VCL. + +We're not happy about this, but have found no sensible workarounds. From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] e0d4d83 some words about exit codes Message-ID: commit e0d4d83e1d5c871096b4b42c3ecfc358b74c1a20 Author: Nils Goroll Date: Tue Aug 26 16:47:07 2014 +0200 some words about exit codes diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index dbe5db8..75b50ac 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -249,6 +249,23 @@ here, in order to conserve VM space. .. include:: ../include/params.rst +EXIT CODES +========== + +Varnish and bundled tools will, in most cases, exit with one of the +following codes + +* `0` OK +* `1` Configuration / parameter error +* `2` Some other error which could be system-dependend and/or transient + +In addition to these, when coredumps are disabled, `varnishd` will +exit with status code `4` in a panic situation. + +The `varnishd` master process may also OR its exit code with `0x40` +when the `varnishd` child process was terminated by a signal and with +`0x80` when a core was dumped. + SEE ALSO ======== From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 7988a31 be more precise about status codes Message-ID: commit 7988a315329eb250adfb9edc8cf2575806e28918 Author: Nils Goroll Date: Tue Aug 26 16:59:13 2014 +0200 be more precise about status codes diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 75b50ac..3375f3f 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -259,12 +259,19 @@ following codes * `1` Configuration / parameter error * `2` Some other error which could be system-dependend and/or transient -In addition to these, when coredumps are disabled, `varnishd` will -exit with status code `4` in a panic situation. +In addition to these, the `varnishd` child process will exit with +status code `4` -The `varnishd` master process may also OR its exit code with `0x40` -when the `varnishd` child process was terminated by a signal and with -`0x80` when a core was dumped. +* in a panic situation when coredumps are disabled and +* in panic-like situations when triggering an actual panic is not + possible. + +The `varnishd` master process may also OR its exit code + +* with `0x20` when the `varnishd` child process died, +* with `0x40` when the `varnishd` child process was terminated by a + signal and +* with `0x80` when a core was dumped. SEE ALSO ======== From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 2ce0a6a Make this test 0.1 second more tolerant of timing on loaded systems Message-ID: commit 2ce0a6a880005f172c4ca639bebb7ce7ded7a54f Author: Poul-Henning Kamp Date: Tue Aug 26 20:45:27 2014 +0000 Make this test 0.1 second more tolerant of timing on loaded systems diff --git a/bin/varnishtest/tests/r01578.vtc b/bin/varnishtest/tests/r01578.vtc index 3086b6b..d0835d5 100644 --- a/bin/varnishtest/tests/r01578.vtc +++ b/bin/varnishtest/tests/r01578.vtc @@ -30,12 +30,12 @@ client c1 { rxresp expect resp.bodylen == 40 expect resp.http.x-ttl == 23.000 - expect resp.http.x-remaining-ttl ~ ^17\.0|^16\.9 + expect resp.http.x-remaining-ttl ~ ^17\.0|^16\.[89] delay 2 txreq rxresp expect resp.bodylen == 40 expect resp.http.x-ttl == 23.000 - expect resp.http.x-remaining-ttl ~ ^15\.0|^14\.9 + expect resp.http.x-remaining-ttl ~ ^15\.0|^14\.[89] } -run From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 1db617d White space nit Message-ID: commit 1db617d1b9c1d3d9d1dd7db1bcd4c173538ed01c Author: Poul-Henning Kamp Date: Tue Aug 26 20:48:00 2014 +0000 White space nit diff --git a/bin/varnishtest/tests/r01578.vtc b/bin/varnishtest/tests/r01578.vtc index d0835d5..a72a3c9 100644 --- a/bin/varnishtest/tests/r01578.vtc +++ b/bin/varnishtest/tests/r01578.vtc @@ -7,7 +7,7 @@ server s1 { varnish v1 -vcl+backend { sub vcl_backend_response { - set beresp.http.x-ttl = beresp.ttl; + set beresp.http.x-ttl = beresp.ttl; } sub vcl_hit { set req.http.x-remaining-ttl = obj.ttl; From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 7d99e1b Rework how we infer types in the VCC expresion evaluator. Message-ID: commit 7d99e1be1c0b772fb84db18bce12eb69c09a22f1 Author: Poul-Henning Kamp Date: Tue Aug 26 20:48:08 2014 +0000 Rework how we infer types in the VCC expresion evaluator. This mainly (only ?) changes things when we ask for an expression of type STRING or STTRING_LIST. Previously addition and subtraction would follow the general rule, "first argument determines intial type" and that would make an expression like this fail: set resp.http.server_port_foo = std.port(server.ip) + "_foo"; Because the addition tries to evaluate "INT + STRING". The idea was that people would rewrite this as: set resp.http.server_port_foo = "" + std.port(server.ip) + "_foo"; To make the first argument STRING and everything will then just work. However, this is needlessly strict in the case where we are actively desiring the expression to evaluate to STRING -- like above where resp.http.* has type STRING. With the new code, when an impossible addition is encountered, it will look at the type requested of the expression, and if it is STRING, convert the current subexpression to STRING and continue adding. A large number of subtests which are designed to fail started working with this change, they have been fixed by not using *.http.* variables as scaffolding.. You can still get into some weird corner-cases like the difference between: set resp.http.foobar = req.ttl + 1s + "_" + req.ttl + 1s; and set resp.http.foobar = req.ttl + 1s + "_" + req.ttl + 1; and set resp.http.foobar = req.ttl + 1s + "_" + (req.ttl + 1s); (Hint: The last one is the only one which does what you think) Fixes #1579 diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index e63be3f..f9be7ec 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -74,7 +74,7 @@ varnish v1 -errvcl {Operator * not possible on type STRING.} { varnish v1 -errvcl {DURATION + INT not possible.} { sub vcl_backend_response { - set req.http.foo = req.ttl + beresp.status; + set req.ttl = req.ttl + beresp.status; } } @@ -85,7 +85,7 @@ varnish v1 -errvcl {'!' must be followed by BOOL, found DURATION.} { } } -varnish v1 -errvcl {Operator + not possible on type BOOL.} { +varnish v1 -errvcl {BOOL + BOOL not possible.} { sub vcl_backend_response { if (beresp.do_gzip + beresp.do_gunzip) { } @@ -111,7 +111,7 @@ varnish v1 -vcl { } # XXX: not the most clear error message -varnish v1 -errvcl {Expected ';' got '-'} { +varnish v1 -errvcl {STRING - STRING not possible.} { backend b { .host = "127.0.0.1"; } sub vcl_recv { set req.http.foo = "foo" - "bar"; @@ -121,14 +121,14 @@ varnish v1 -errvcl {Expected ';' got '-'} { varnish v1 -errvcl {TIME + STRING not possible.} { backend b { .host = "127.0.0.1"; } sub vcl_recv { - set req.http.foo = now + "foo"; + set req.ttl = now + "foo"; } } varnish v1 -errvcl {TIME + TIME not possible.} { backend b { .host = "127.0.0.1"; } sub vcl_recv { - set req.http.foo = now + now; + set req.ttl = now + now; } } @@ -142,15 +142,15 @@ varnish v1 -errvcl {Expected ID got ';'} { varnish v1 -errvcl {INT + STRING not possible.} { backend b { .host = "127.0.0.1"; } - sub vcl_recv { - set req.http.foo = 1 + "foo"; + sub vcl_backend_response { + set beresp.status = 1 + "foo"; } } varnish v1 -errvcl {INT + TIME not possible.} { backend b { .host = "127.0.0.1"; } - sub vcl_recv { - set req.http.foo = 1 + now; + sub vcl_backend_response { + set beresp.status = 1 + now; } } diff --git a/bin/varnishtest/tests/v00025.vtc b/bin/varnishtest/tests/v00025.vtc index 67ac41a..2ac4b9d 100644 --- a/bin/varnishtest/tests/v00025.vtc +++ b/bin/varnishtest/tests/v00025.vtc @@ -15,6 +15,9 @@ varnish v1 -arg "-i J.F.Nobody" -vcl+backend { sub vcl_deliver { set resp.http.server_port = std.port(server.ip); + set resp.http.server_port_foo = std.port(server.ip) + "_foo"; + set resp.http.ttl1 = (req.ttl + 10s); + set resp.http.ttl2 = req.ttl + 10s; set resp.http.id = server.identity; set resp.http.esi = req.esi; set resp.http.be = req.backend_hint; diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index c2f9818..c6c75fe 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -660,6 +660,7 @@ vcc_Eval_SymFunc(struct vcc *tl, struct expr **e, const struct symbol *sym) * SYNTAX: * Expr4: * '(' Expr0 ')' + * symbol * CNUM * CSTR */ @@ -704,6 +705,11 @@ vcc_expr4(struct vcc *tl, struct expr **e, enum var_type fmt) AN(sym->eval); AZ(*e); sym->eval(tl, e, sym); + /* Unless asked for a HEADER, fold to string here */ + if (*e && fmt != HEADER && (*e)->fmt == HEADER) { + vcc_expr_tostring(tl, e, STRING); + ERRCHK(tl); + } return; default: break; @@ -822,16 +828,21 @@ vcc_expr_mul(struct vcc *tl, struct expr **e, enum var_type fmt) */ static void -vcc_expr_string_add(struct vcc *tl, struct expr **e) +vcc_expr_string_add(struct vcc *tl, struct expr **e, struct expr *e2) { - struct expr *e2; enum var_type f2; + AN(e); + AN(*e); + AN(e2); f2 = (*e)->fmt; + assert (f2 == STRING || f2 == STRING_LIST); - while (tl->t->tok == '+') { - vcc_NextToken(tl); - vcc_expr_mul(tl, &e2, STRING); + while (e2 != NULL || tl->t->tok == '+') { + if (e2 == NULL) { + vcc_NextToken(tl); + vcc_expr_mul(tl, &e2, STRING); + } ERRCHK(tl); if (e2->fmt != STRING && e2->fmt != STRING_LIST) { vcc_expr_tostring(tl, &e2, f2); @@ -858,6 +869,7 @@ vcc_expr_string_add(struct vcc *tl, struct expr **e) *e = vcc_expr_edit(STRING_LIST, "\v1,\n\v2", *e, e2); (*e)->constant = EXPR_VAR; } + e2 = NULL; } } @@ -873,49 +885,36 @@ vcc_expr_add(struct vcc *tl, struct expr **e, enum var_type fmt) ERRCHK(tl); f2 = (*e)->fmt; - /* Unless we specifically ask for a HEADER, fold them to string here */ - if (fmt != HEADER && f2 == HEADER) { - vcc_expr_tostring(tl, e, STRING); - ERRCHK(tl); - f2 = (*e)->fmt; - assert(f2 == STRING); - } - - if (tl->t->tok != '+' && tl->t->tok != '-') - return; - - switch(f2) { - case STRING: - case STRING_LIST: - vcc_expr_string_add(tl, e); - return; - case INT: break; - case TIME: break; - case DURATION: break; - case BYTES: break; - default: - VSB_printf(tl->sb, "Operator %.*s not possible on type %s.\n", - PF(tl->t), vcc_Type(f2)); - vcc_ErrWhere(tl, tl->t); - return; - } - while (tl->t->tok == '+' || tl->t->tok == '-') { - if (f2 == TIME) - f2 = DURATION; tk = tl->t; vcc_NextToken(tl); - vcc_expr_mul(tl, &e2, f2); + if (f2 == TIME) + vcc_expr_mul(tl, &e2, DURATION); + else + vcc_expr_mul(tl, &e2, f2); ERRCHK(tl); if (tk->tok == '-' && (*e)->fmt == TIME && e2->fmt == TIME) { /* OK */ } else if ((*e)->fmt == TIME && e2->fmt == DURATION) { f2 = TIME; /* OK */ - } else if (tk->tok == '-' && - (*e)->fmt == BYTES && e2->fmt == BYTES) { + } else if ((*e)->fmt == BYTES && e2->fmt == BYTES) { + /* OK */ + } else if ((*e)->fmt == INT && e2->fmt == INT) { + /* OK */ + } else if ((*e)->fmt == DURATION && e2->fmt == DURATION) { /* OK */ - } else if (e2->fmt != f2) { + } else if (tk->tok == '+' && + (*e)->fmt == STRING && e2->fmt == STRING) { + vcc_expr_string_add(tl, e, e2); + return; + } else if (tk->tok == '+' && + (fmt == STRING || fmt == STRING_LIST)) { + /* Time to fold and add as string */ + vcc_expr_tostring(tl, e, STRING); + vcc_expr_string_add(tl, e, e2); + return; + } else { VSB_printf(tl->sb, "%s %.*s %s not possible.\n", vcc_Type((*e)->fmt), PF(tk), vcc_Type(e2->fmt)); vcc_ErrWhere2(tl, tk, tl->t); From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 5a01820 Fix an issue where the order of symbol definition determined if code could compile or not. Message-ID: commit 5a01820c20ef91a26c09a274588d41cd5713e11d Author: Poul-Henning Kamp Date: Wed Aug 27 07:29:49 2014 +0000 Fix an issue where the order of symbol definition determined if code could compile or not. Fixes #1569 diff --git a/bin/varnishtest/tests/r01569.vtc b/bin/varnishtest/tests/r01569.vtc new file mode 100644 index 0000000..7a05938 --- /dev/null +++ b/bin/varnishtest/tests/r01569.vtc @@ -0,0 +1,19 @@ +varnishtest "symbol lookup order issue" + +varnish v1 -vcl { + vcl 4.0; + import ${vmod_debug}; + + backend debug { + .host = "127.0.0.1"; + .port = "80"; + } + + sub debug { + set req.backend_hint = debug; + } + + sub vcl_recv { + call debug; + } +} diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index c6c75fe..3557a4b 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -688,7 +688,15 @@ vcc_expr4(struct vcc *tl, struct expr **e, enum var_type fmt) * XXX: what if var and func/proc had same name ? * XXX: look for SYM_VAR first for consistency ? */ - sym = VCC_FindSymbol(tl, tl->t, SYM_NONE); + sym = NULL; + if (fmt == BACKEND) + sym = VCC_FindSymbol(tl, tl->t, SYM_BACKEND); + if (sym == NULL) + sym = VCC_FindSymbol(tl, tl->t, SYM_VAR); + if (sym == NULL) + sym = VCC_FindSymbol(tl, tl->t, SYM_FUNC); + if (sym == NULL) + sym = VCC_FindSymbol(tl, tl->t, SYM_NONE); if (sym == NULL || sym->eval == NULL) { VSB_printf(tl->sb, "Symbol not found: "); vcc_ErrToken(tl, tl->t); diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index c3d8a13..e9d39f0 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -50,7 +50,6 @@ VCC_SymKind(struct vcc *tl, const struct symbol *s) } } - static struct symbol * vcc_AddSymbol(struct vcc *tl, const char *nb, int l, enum symkind kind) { From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] a716ea3 Fix readline finding on FreeBSD -current Message-ID: commit a716ea33147eb465904139e6ccbfb03c0349212d Author: Poul-Henning Kamp Date: Wed Aug 27 11:29:50 2014 +0000 Fix readline finding on FreeBSD -current diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index 26019a5..a17961d 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -34,9 +34,11 @@ #include -#ifdef HAVE_LIBEDIT +#if defined(HAVE_EDIT_READLINE_READLINE_H) +# include +#elif defined(HAVE_LIBEDIT) # include -#elif HAVE_READLINE_READLINE_H +#elif defined (HAVE_READLINE_READLINE_H) # include # ifdef HAVE_READLINE_HISTORY_H # include diff --git a/configure.ac b/configure.ac index c8a36f5..d4f5c5b 100644 --- a/configure.ac +++ b/configure.ac @@ -164,6 +164,7 @@ PKG_CHECK_MODULES([LIBEDIT], [libedit], AC_HEADER_STDC AC_HEADER_SYS_WAIT AC_HEADER_TIME +AC_CHECK_HEADERS([edit/readline/readline.h]) AC_CHECK_HEADERS([sys/param.h]) AC_CHECK_HEADERS([sys/types.h]) AC_CHECK_HEADERS([sys/endian.h]) From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 746e41c Disable streaming in this test Message-ID: commit 746e41c3a9818e2b7c43b9c40c30ff7851ea3ea0 Author: Poul-Henning Kamp Date: Wed Aug 27 12:14:25 2014 +0000 Disable streaming in this test diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc index 513038c..23910de 100644 --- a/bin/varnishtest/tests/c00034.vtc +++ b/bin/varnishtest/tests/c00034.vtc @@ -6,6 +6,10 @@ server s1 { } -start varnish v1 -vcl+backend { + + sub vcl_backend_response { + set beresp.do_stream = false; + } } -start varnish v1 -cliok "param.set http_range_support off" @@ -18,6 +22,7 @@ client c1 { varnish v1 -cliok "param.set http_range_support on" +varnish v1 -expect s_resp_bodybytes == 100 client c1 { txreq -hdr "Range: bytes =0-9" @@ -40,6 +45,12 @@ client c1 { expect resp.status == 200 expect resp.bodylen == 100 +} -run + +varnish v1 -expect s_resp_bodybytes == 500 + +client c1 { + txreq -hdr "Range: bytes=0-9" rxresp expect resp.status == 206 From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 43cc282 Give vrt_ctx a "double now" timestamp for use in VCL/VRT/VMODs Message-ID: commit 43cc2827c8f2be290fc95d18df2010060990fe0a Author: Poul-Henning Kamp Date: Wed Aug 27 12:15:23 2014 +0000 Give vrt_ctx a "double now" timestamp for use in VCL/VRT/VMODs Conflicts: bin/varnishd/cache/cache_vcl.c include/vrt.h diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 63e8781..43c28c4 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -42,6 +42,7 @@ #include "vrt.h" #include "vcli.h" #include "vcli_priv.h" +#include "vtim.h" struct vcls { unsigned magic; @@ -436,6 +437,7 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo, ctx.req = req; if (req->obj) ctx.http_obj = req->obj->http; + ctx.now = req->t_prev; } if (bo != NULL) { // AZ(req); @@ -446,7 +448,10 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo, ctx.http_bereq = bo->bereq; ctx.http_beresp = bo->beresp; ctx.bo = bo; + ctx.now = bo->t_prev; } + if (ctx.now == 0) + ctx.now = VTIM_real(); ctx.ws = ws; ctx.method = method; ctx.handling = &wrk->handling; diff --git a/include/vrt.h b/include/vrt.h index ead55d3..e49926d 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -102,6 +102,7 @@ struct vrt_ctx { struct http *http_bereq; struct http *http_beresp; + double now; }; /***********************************************************************/ From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 6e068ca Update filename references Message-ID: commit 6e068ca750a7070e7cdd1dec68beda3c89fd15fe Author: Federico G. Schwindt Date: Wed Aug 27 13:11:44 2014 +0100 Update filename references diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index e455048..ddc071c 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -926,7 +926,7 @@ void CLI_AddFuncs(struct cli_proto *p); extern pthread_t cli_thread; #define ASSERT_CLI() do {assert(pthread_self() == cli_thread);} while (0) -/* cache_expiry.c */ +/* cache_expire.c */ void EXP_Clr(struct exp *e); double EXP_Ttl(const struct req *, const struct object*); @@ -1247,7 +1247,7 @@ char *WS_Snapshot(struct ws *ws); int WS_Overflowed(const struct ws *ws); void *WS_Printf(struct ws *ws, const char *fmt, ...) __printflike(2, 3); -/* rfc2616.c */ +/* cache_rfc2616.c */ void RFC2616_Ttl(struct busyobj *, double now); enum body_status RFC2616_Body(struct busyobj *, struct dstat *); unsigned RFC2616_Req_Gzip(const struct http *); From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] d8f1646 WS_Reserve shouldn't round down if we're not asking for too much Message-ID: commit d8f16461ed6745dbefdc9d60bb5150d57445b3b8 Author: Nils Goroll Date: Wed Aug 27 18:07:43 2014 +0200 WS_Reserve shouldn't round down if we're not asking for too much diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index ccc887c..956fef4 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -202,13 +202,12 @@ WS_Reserve(struct ws *ws, unsigned bytes) WS_Assert(ws); assert(ws->r == NULL); - if (bytes == 0) - b2 = ws->e - ws->f; - else if (bytes > ws->e - ws->f) - b2 = ws->e - ws->f; - else - b2 = bytes; - b2 = PRNDDN(b2); + + b2 = PRNDDN(ws->e - ws->f); + if ((bytes != 0) && + (bytes < b2)) + b2 = PRNDUP(bytes); + xxxassert(ws->f + b2 <= ws->e); ws->r = ws->f + b2; DSL(DBG_WORKSPACE, 0, "WS_Reserve(%p, %u/%u) = %u", From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 6111bc3 fight ocpd Message-ID: commit 6111bc31d353baf6d2330ce613e8a7e73294b653 Author: Nils Goroll Date: Wed Aug 27 19:30:51 2014 +0200 fight ocpd diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index 956fef4..3930c4d 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -204,8 +204,7 @@ WS_Reserve(struct ws *ws, unsigned bytes) assert(ws->r == NULL); b2 = PRNDDN(ws->e - ws->f); - if ((bytes != 0) && - (bytes < b2)) + if (bytes != 0 && bytes < b2) b2 = PRNDUP(bytes); xxxassert(ws->f + b2 <= ws->e); From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] a382e0d Change exit code documentation so highest is most severe. Dont mention exit(4) Message-ID: commit a382e0d02785d71f2fa5c2d47f21ce5e0d6ba7c0 Author: Nils Goroll Date: Wed Aug 27 22:02:59 2014 +0200 Change exit code documentation so highest is most severe. Dont mention exit(4) After discussion with phk: exit(4) is a real exception, we should abort() whenever possible. The error codes patch is still to be committed. diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 3375f3f..6014637 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -256,15 +256,9 @@ Varnish and bundled tools will, in most cases, exit with one of the following codes * `0` OK -* `1` Configuration / parameter error -* `2` Some other error which could be system-dependend and/or transient - -In addition to these, the `varnishd` child process will exit with -status code `4` - -* in a panic situation when coredumps are disabled and -* in panic-like situations when triggering an actual panic is not - possible. +* `1` Some error which could be system-dependend and/or transient +* `2` Serious configuration / parameter error - retrying with the same + configuration / parameters is most likely useless The `varnishd` master process may also OR its exit code From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 52083bf update gzip docs Message-ID: commit 52083bf48324854d46f3ff596c23707fa55902cd Author: Nils Goroll Date: Thu Aug 28 11:06:06 2014 +0200 update gzip docs diff --git a/doc/sphinx/users-guide/compression.rst b/doc/sphinx/users-guide/compression.rst index bff1ebb..e7cfda2 100644 --- a/doc/sphinx/users-guide/compression.rst +++ b/doc/sphinx/users-guide/compression.rst @@ -9,24 +9,47 @@ encoding. *Before* 3.0, Varnish would never compress objects. In Varnish 4.0 compression defaults to "on", meaning that it tries to be smart and do the sensible thing. -.. XXX:Heavy refactoring to Varnish 4 above. benc - If you don't want Varnish tampering with the encoding you can disable -compression all together by setting the parameter 'http_gzip_support' to +compression all together by setting the parameter `http_gzip_support` to false. Please see man :ref:`ref-varnishd` for details. Default behaviour ~~~~~~~~~~~~~~~~~ -The default behaviour for Varnish is to check if the client supports our -compression scheme (gzip) and if it does it will override the -'Accept-Encoding' header and set it to "gzip". +The default behaviour is active when the `http_gzip_support` parameter +is set to "on" and neither `beresp.do_gzip` nor `beresp.do_gunzip` are +used in VCL. + +Unless returning from `vcl_recv` with `pipe` or `pass`, varnish sets +`req.http.Accept-Encoding` to "gzip". It removes the header otherwise. + +Unless the request is a `pass`, Varnish will set +`bereq.http.Accept-Encoding` to "gzip" before `vcl_backend_fetch` +runs, so the header can be changed in VCL. + +If the server responds with gzip'ed content it will be stored in +memory in its compressed form and `Accept-Encoding` will be added to +the `Vary` header. + +To clients supporting gzip, compressed content is delivered +unmodified. + +For clients not supporting gzip, compressed content gets decompressed +on the fly while delivering. The `Content-Encoding` response header +gets removed and any `Etag` gets weakened (by prepending "W/"). -When Varnish then issues a backend request the 'Accept-Encoding' will -then only consist of "gzip". If the server responds with gzip'ed -content it will be stored in memory in its compressed form. If the -backend sends content in clear text it will be stored in clear text. +For Vary Lookups, `Accept-Encoding` is ignored. + +Turning off gzip support +~~~~~~~~~~~~~~~~~~~~~~~~ + +When the `http_gzip_support` parameter is set to "off", Varnish does +not do any of the header alterations documented above and handles +`Vary: Accept-Encoding` like it would for any other `Vary` value. + +Compressing content if backends don't +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can make Varnish compress content before storing it in cache in `vcl_backend_response` by setting 'do_gzip' to true, like this:: @@ -37,15 +60,41 @@ You can make Varnish compress content before storing it in cache in } } -Please make sure that you don't try to compress content that is -uncompressable, like jpgs, gifs and mp3. You'll only waste CPU -cycles. You can also uncompress objects before storing it in memory by -setting 'do_gunzip' to true but that will usually not be the most sensible thing to do. +With `do_gzip` Varnish will make the following alterations to the +headers of the resulting object which cannot be modified in a backend +VCL (but in `vcl_deliver`): + +* set `obj.http.Content-Encoding` to "gzip" +* add "Accept-Encoding" to `obj.http.Content-Encoding`, unless already + present +* weaken any `Etag` (by prepending "W/") + Generally, Varnish doesn't use much CPU so it might make more sense to have Varnish spend CPU cycles compressing content than doing it in your web- or application servers, which are more likely to be CPU-bound. +Please make sure that you don't try to compress content that is +uncompressable, like jpgs, gifs and mp3. You'll only waste CPU cycles. + +Uncompressing content before entering the cache +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can also uncompress objects before storing it in memory by setting +`do_gunzip` to true. One use case for this feature is to work around +badly configured backends uselessly compressing already compressed +content like JPG images (but fixing the misbehaving backend is always +the better option). + +With `do_gzip` Varnish will make the following alterations to the +headers of the resulting object which cannot be modified in a backend +VCL (but in `vcl_deliver`): + +* remove `obj.http.Content-Encoding` to "gzip" +* remove any "Accept-Encoding" from `obj.http.Content-Encoding` + (XXX review when closing #940) +* weaken any `Etag` (by prepending "W/") + GZIP and ESI ~~~~~~~~~~~~ @@ -54,20 +103,6 @@ and GZIP work together really well. Varnish will magically decompress the content to do the ESI-processing, then recompress it for efficient storage and delivery. - -Clients that don't support gzip -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If the client does not support gzip the 'Accept-Encoding' header is left -alone then we'll end up serving whatever we get from the backend -server. Remember that the backend might tell Varnish to *Vary* on the -'Accept-Encoding'. - -If the client does not support gzip but we've already got a compressed -version of the page in memory Varnish will automatically decompress -the page while delivering it. - - A random outburst ~~~~~~~~~~~~~~~~~ From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] d781eb9 fix a badly worded sentence, make choice of tense more consistent Message-ID: commit d781eb9c672d400c4a05dc6fa4b4cf1b8eb66a7b Author: Nils Goroll Date: Thu Aug 28 11:18:51 2014 +0200 fix a badly worded sentence, make choice of tense more consistent Thanks to fgs for reviewing diff --git a/doc/sphinx/users-guide/compression.rst b/doc/sphinx/users-guide/compression.rst index e7cfda2..2c3699e 100644 --- a/doc/sphinx/users-guide/compression.rst +++ b/doc/sphinx/users-guide/compression.rst @@ -21,10 +21,12 @@ The default behaviour is active when the `http_gzip_support` parameter is set to "on" and neither `beresp.do_gzip` nor `beresp.do_gunzip` are used in VCL. -Unless returning from `vcl_recv` with `pipe` or `pass`, varnish sets -`req.http.Accept-Encoding` to "gzip". It removes the header otherwise. +Unless returning from `vcl_recv` with `pipe` or `pass`, varnish +modifies `req.http.Accept-Encoding`: If the client supports gzip, +`req.http.Accept-Encoding` is set to "gzip". Otherwise, the header is +removed. -Unless the request is a `pass`, Varnish will set +Unless the request is a `pass`, Varnish sets `bereq.http.Accept-Encoding` to "gzip" before `vcl_backend_fetch` runs, so the header can be changed in VCL. From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 9fd6ff8 fix errors and clarify that do_g* are ignored when gzip support is off Message-ID: commit 9fd6ff853b9ee27633163618f788260b7e9d2424 Author: Nils Goroll Date: Thu Aug 28 12:28:39 2014 +0200 fix errors and clarify that do_g* are ignored when gzip support is off diff --git a/doc/sphinx/users-guide/compression.rst b/doc/sphinx/users-guide/compression.rst index 2c3699e..bea7176 100644 --- a/doc/sphinx/users-guide/compression.rst +++ b/doc/sphinx/users-guide/compression.rst @@ -43,18 +43,11 @@ gets removed and any `Etag` gets weakened (by prepending "W/"). For Vary Lookups, `Accept-Encoding` is ignored. -Turning off gzip support -~~~~~~~~~~~~~~~~~~~~~~~~ - -When the `http_gzip_support` parameter is set to "off", Varnish does -not do any of the header alterations documented above and handles -`Vary: Accept-Encoding` like it would for any other `Vary` value. - Compressing content if backends don't ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can make Varnish compress content before storing it in cache in -`vcl_backend_response` by setting 'do_gzip' to true, like this:: +`vcl_backend_response` by setting `beresp.do_gzip` to true, like this:: sub vcl_backend_response { if (beresp.http.content-type ~ "text") { @@ -62,9 +55,9 @@ You can make Varnish compress content before storing it in cache in } } -With `do_gzip` Varnish will make the following alterations to the -headers of the resulting object which cannot be modified in a backend -VCL (but in `vcl_deliver`): +With `beresp.do_gzip` set to "true", Varnish will make the following +alterations to the headers of the resulting object which cannot be +modified in a backend VCL (but in `vcl_deliver`): * set `obj.http.Content-Encoding` to "gzip" * add "Accept-Encoding" to `obj.http.Content-Encoding`, unless already @@ -83,16 +76,16 @@ Uncompressing content before entering the cache ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can also uncompress objects before storing it in memory by setting -`do_gunzip` to true. One use case for this feature is to work around -badly configured backends uselessly compressing already compressed -content like JPG images (but fixing the misbehaving backend is always -the better option). +`beresp.do_gunzip` to true. One use case for this feature is to work +around badly configured backends uselessly compressing already +compressed content like JPG images (but fixing the misbehaving backend +is always the better option). -With `do_gzip` Varnish will make the following alterations to the -headers of the resulting object which cannot be modified in a backend -VCL (but in `vcl_deliver`): +With `beresp.do_gunzip` Varnish will make the following alterations to +the headers of the resulting object which cannot be modified in a +backend VCL (but in `vcl_deliver`): -* remove `obj.http.Content-Encoding` to "gzip" +* remove `obj.http.Content-Encoding` * remove any "Accept-Encoding" from `obj.http.Content-Encoding` (XXX review when closing #940) * weaken any `Etag` (by prepending "W/") @@ -103,7 +96,15 @@ GZIP and ESI If you are using Edge Side Includes (ESIs) you'll be happy to note that ESI and GZIP work together really well. Varnish will magically decompress the content to do the ESI-processing, then recompress it for efficient -storage and delivery. +storage and delivery. + +Turning off gzip support +~~~~~~~~~~~~~~~~~~~~~~~~ + +When the `http_gzip_support` parameter is set to "off", Varnish does +not do any of the header alterations documented above, handles `Vary: +Accept-Encoding` like it would for any other `Vary` value and ignores +`beresp.do_gzip` and `beresp.do_gunzip`. A random outburst ~~~~~~~~~~~~~~~~~ From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 3056ec6 varnishd: streamline exit codes, fix exit codes for vsubs Message-ID: commit 3056ec64ca397ecb79c1b8b9f7c56b8dcbd78552 Author: Nils Goroll Date: Thu Aug 28 12:32:08 2014 +0200 varnishd: streamline exit codes, fix exit codes for vsubs This should bring us closer to the exit codes now documented in varnishd.rst Fixes #1572 for varnishd diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index ecdc6d8..4f4a8f6 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -117,7 +117,7 @@ void STV_Config_Transient(void); /* mgt_vcc.c */ void mgt_vcc_init(void); -int mgt_vcc_default(const char *bflag, const char *f_arg, char *vcl, int Cflag); +unsigned mgt_vcc_default(const char *bflag, const char *f_arg, char *vcl, int Cflag); int mgt_push_vcls_and_start(unsigned *status, char **p); int mgt_has_vcl(void); extern char *mgt_cc_cmd; diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 1ae0c0e..1fe8473 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -425,7 +425,7 @@ mgt_launch_child(struct cli *cli) child_main(); - exit(1); + exit(0); } assert(pid > 1); REPORT(LOG_NOTICE, "child (%jd) Started", (intmax_t)pid); @@ -683,7 +683,7 @@ mgt_sigint(const struct vev *e, int what) (void)fflush(stdout); if (child_pid >= 0) mgt_stop_child(); - exit (2); + exit(0); } /*--------------------------------------------------------------------*/ @@ -753,6 +753,7 @@ MGT_Run(void) else if (!d_flag) { mgt_launch_child(NULL); if (child_state == CH_STOPPED) { + // XXX correct? or 0? exit_status = 2; return; } diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c index e7d7aa2..21fecfb 100644 --- a/bin/varnishd/mgt/mgt_cli.c +++ b/bin/varnishd/mgt/mgt_cli.c @@ -499,17 +499,17 @@ mgt_cli_secret(const char *S_arg) fd = open(S_arg, O_RDONLY); if (fd < 0) { fprintf(stderr, "Can not open secret-file \"%s\"\n", S_arg); - exit (2); + exit(2); } mgt_got_fd(fd); i = read(fd, buf, sizeof buf); if (i == 0) { fprintf(stderr, "Empty secret-file \"%s\"\n", S_arg); - exit (2); + exit(2); } if (i < 0) { fprintf(stderr, "Can not read secret-file \"%s\"\n", S_arg); - exit (2); + exit(2); } AZ(close(fd)); secret_file = S_arg; @@ -644,7 +644,7 @@ mgt_cli_master(const char *M_arg) M_nta = VSS_resolve(M_arg, NULL, &M_ta); if (M_nta <= 0) { fprintf(stderr, "Could resolve -M argument to address\n"); - exit (1); + exit(2); } M_nxt = 0; AZ(M_poker); diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 76b702b..ea7f950 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -199,7 +199,7 @@ cli_check(const struct cli *cli) } AZ(VSB_finish(cli->sb)); fprintf(stderr, "Error:\n%s\n", VSB_data(cli->sb)); - exit (2); + exit(2); } /*-------------------------------------------------------------------- @@ -559,11 +559,11 @@ main(int argc, char * const *argv) case 'x': if (!strcmp(optarg, "dumprstparam")) { MCF_DumpRstParam(); - exit (0); + exit(0); } if (!strcmp(optarg, "dumprstvsl")) { mgt_DumpRstVsl(); - exit (0); + exit(0); } usage(); break; @@ -651,11 +651,11 @@ main(int argc, char * const *argv) P_arg, strerror(errno)); if (b_arg != NULL || f_arg != NULL) - if (mgt_vcc_default(b_arg, f_arg, vcl, C_flag)) - exit (2); + if ((o = mgt_vcc_default(b_arg, f_arg, vcl, C_flag)) != 0) + exit(o); if (C_flag) - exit (0); + exit(0); if (!d_flag) { if (MGT_open_sockets()) diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 500916e..abfd86a 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -175,7 +175,7 @@ mcf_wrap(struct cli *cli, const char *text) if (r == NULL) { fprintf(stderr, "LINE with just one TAB: <%s>\n", text); - exit(2); + exit(4); } if (r - q > tw) tw = r - q; @@ -385,11 +385,11 @@ MCF_AddParams(struct parspec *ps) if (isspace(s[-1])) { fprintf(stderr, "Param->descr has trailing space: %s\n", pp->name); - exit(2); + exit(4); } if (mcf_findpar(pp->name) != NULL) { fprintf(stderr, "Duplicate param: %s\n", pp->name); - exit(2); + exit(4); } if (strlen(pp->name) + 1 > margin2) margin2 = strlen(pp->name) + 1; diff --git a/bin/varnishd/mgt/mgt_sandbox_solaris.c b/bin/varnishd/mgt/mgt_sandbox_solaris.c index 255b99c..2d92968 100644 --- a/bin/varnishd/mgt/mgt_sandbox_solaris.c +++ b/bin/varnishd/mgt/mgt_sandbox_solaris.c @@ -236,7 +236,7 @@ mgt_sandbox_solaris_add_inheritable(priv_set_t *pset, enum sandbox_e who) break; default: REPORT(LOG_ERR, "INCOMPLETE AT: %s(%d)\n", __func__, __LINE__); - exit(1); + exit(4); } } @@ -263,7 +263,7 @@ mgt_sandbox_solaris_add_effective(priv_set_t *pset, enum sandbox_e who) break; default: REPORT(LOG_ERR, "INCOMPLETE AT: %s(%d)\n", __func__, __LINE__); - exit(1); + exit(4); } } @@ -286,7 +286,7 @@ mgt_sandbox_solaris_add_permitted(priv_set_t *pset, enum sandbox_e who) break; default: REPORT(LOG_ERR, "INCOMPLETE AT: %s(%d)\n", __func__, __LINE__); - exit(1); + exit(4); } } diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c index 130b48f..ed020f7 100644 --- a/bin/varnishd/mgt/mgt_shmem.c +++ b/bin/varnishd/mgt/mgt_shmem.c @@ -100,7 +100,7 @@ vsm_n_check(void) struct stat st; pid_t pid; struct VSM_head vsmh; - int retval = 2; + int retval = 1; fd = open(VSM_FILENAME, O_RDWR, 0644); if (fd < 0) @@ -217,7 +217,7 @@ mgt_SHM_Create(void) if (p == MAP_FAILED) { fprintf(stderr, "Mmap error %s: %s\n", fnbuf, strerror(errno)); - exit (-1); + exit(1); } mgt_vsm_p = p; @@ -250,7 +250,7 @@ mgt_SHM_Create(void) fprintf(stderr, "Rename failed %s -> %s: %s\n", fnbuf, VSM_FILENAME, strerror(errno)); (void)unlink(fnbuf); - exit (-1); + exit(1); } #ifdef __OpenBSD__ @@ -325,7 +325,7 @@ mgt_SHM_Init(void) /* Collision check with already running varnishd */ i = vsm_n_check(); if (i) - exit(i); + exit(2); /* Create our static VSM instance */ static_vsm = VSM_common_new(static_vsm_buf, sizeof static_vsm_buf); diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 4d97de6..b76b016 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -151,22 +151,22 @@ run_vcc(void *priv) printf("%s", VSB_data(sb)); VSB_delete(sb); if (csrc == NULL) - exit (1); + exit(2); fd = open(vp->sf, O_WRONLY); if (fd < 0) { fprintf(stderr, "Cannot open %s", vp->sf); - exit (1); + exit(2); } l = strlen(csrc); i = write(fd, csrc, l); if (i != l) { fprintf(stderr, "Cannot write %s", vp->sf); - exit (1); + exit(2); } AZ(close(fd)); free(csrc); - exit (0); + exit(0); } /*-------------------------------------------------------------------- @@ -228,7 +228,7 @@ run_dlopen(void *priv) */ static char * -mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag) +mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag, unsigned *status) { char *csrc; struct vsb *cmdsb; @@ -236,12 +236,16 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag) char of[sizeof sf + 1]; char *retval; int sfd, i; + unsigned subs; struct vcc_priv vp; + *status = 0; + /* Create temporary C source file */ sfd = VFIL_tmpfile(sf); if (sfd < 0) { VSB_printf(sb, "Failed to create %s: %s", sf, strerror(errno)); + *status = 2; return (NULL); } (void)fchown(sfd, mgt_param.uid, mgt_param.gid); @@ -253,8 +257,10 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag) vp.magic = VCC_PRIV_MAGIC; vp.sf = sf; vp.vcl = vcl; - if (VSUB_run(sb, run_vcc, &vp, "VCC-compiler", -1)) { + subs = VSUB_run(sb, run_vcc, &vp, "VCC-compiler", -1); + if (subs) { (void)unlink(sf); + *status = subs; return (NULL); } @@ -277,6 +283,7 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag) VSB_printf(sb, "Failed to create %s: %s", of, strerror(errno)); (void)unlink(sf); + *status = 2; return (NULL); } (void)fchown(i, mgt_param.uid, mgt_param.gid); @@ -286,24 +293,27 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag) cmdsb = mgt_make_cc_cmd(sf, of); /* Run the C-compiler in a sub-shell */ - i = VSUB_run(sb, run_cc, VSB_data(cmdsb), "C-compiler", 10); + subs = VSUB_run(sb, run_cc, VSB_data(cmdsb), "C-compiler", 10); (void)unlink(sf); VSB_delete(cmdsb); - if (!i) - i = VSUB_run(sb, run_dlopen, of, "dlopen", 10); + if (!subs) + subs = VSUB_run(sb, run_dlopen, of, "dlopen", 10); /* Ensure the file is readable to the unprivileged user */ - if (!i) { + if (!subs) { i = chmod(of, 0755); - if (i) + if (i) { VSB_printf(sb, "Failed to set permissions on %s: %s", of, strerror(errno)); + subs = 2; + } } - if (i) { + if (subs) { (void)unlink(of); + *status = subs; return (NULL); } @@ -315,13 +325,13 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag) /*--------------------------------------------------------------------*/ static char * -mgt_VccCompile(struct vsb **sb, const char *b, int C_flag) +mgt_VccCompile(struct vsb **sb, const char *b, int C_flag, unsigned *status) { char *vf; *sb = VSB_new_auto(); XXXAN(*sb); - vf = mgt_run_cc(b, *sb, C_flag); + vf = mgt_run_cc(b, *sb, C_flag, status); AZ(VSB_finish(*sb)); return (vf); } @@ -380,13 +390,14 @@ mgt_vcc_delbyname(const char *name) /*--------------------------------------------------------------------*/ -int +unsigned mgt_vcc_default(const char *b_arg, const char *f_arg, char *vcl, int C_flag) { char *vf; struct vsb *sb; struct vclprog *vp; char buf[BUFSIZ]; + unsigned status = 0; /* XXX: annotate vcl with -b/-f arg so people know where it came from */ (void)f_arg; @@ -411,7 +422,7 @@ mgt_vcc_default(const char *b_arg, const char *f_arg, char *vcl, int C_flag) } strcpy(buf, "boot"); - vf = mgt_VccCompile(&sb, vcl, C_flag); + vf = mgt_VccCompile(&sb, vcl, C_flag, &status); free(vcl); if (VSB_len(sb) > 0) fprintf(stderr, "%s", VSB_data(sb)); @@ -419,13 +430,13 @@ mgt_vcc_default(const char *b_arg, const char *f_arg, char *vcl, int C_flag) if (C_flag && vf != NULL) AZ(unlink(vf)); if (vf == NULL) { + assert(status != 0); fprintf(stderr, "\nVCL compilation failed\n"); - return (1); } else { vp = mgt_vcc_add(buf, vf); vp->active = 1; - return (0); } + return (status); } /*--------------------------------------------------------------------*/ @@ -511,11 +522,12 @@ mcf_config_inline(struct cli *cli, const char * const *av, void *priv) return; } - vf = mgt_VccCompile(&sb, av[3], 0); + vf = mgt_VccCompile(&sb, av[3], 0, &status); if (VSB_len(sb) > 0) VCLI_Out(cli, "%s\n", VSB_data(sb)); VSB_delete(sb); if (vf == NULL) { + assert(status != 0); VCLI_Out(cli, "VCL compilation failed"); VCLI_SetResult(cli, CLIS_PARAM); return; @@ -536,7 +548,7 @@ mcf_config_load(struct cli *cli, const char * const *av, void *priv) { char *vf, *vcl; struct vsb *sb; - unsigned status; + unsigned status = 0; char *p = NULL; struct vclprog *vp; @@ -555,13 +567,14 @@ mcf_config_load(struct cli *cli, const char * const *av, void *priv) return; } - vf = mgt_VccCompile(&sb, vcl, 0); + vf = mgt_VccCompile(&sb, vcl, 0, &status); free(vcl); if (VSB_len(sb) > 0) VCLI_Out(cli, "%s", VSB_data(sb)); VSB_delete(sb); if (vf == NULL) { + assert(status != 0); VCLI_Out(cli, "VCL compilation failed"); VCLI_SetResult(cli, CLIS_PARAM); return; diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c index c48200c..876e444 100644 --- a/bin/varnishd/storage/storage_file.c +++ b/bin/varnishd/storage/storage_file.c @@ -443,7 +443,7 @@ smf_open(const struct stevedore *st) /* XXX */ if (sum < MINPAGES * (off_t)getpagesize()) - exit (2); + exit(4); sc->stats->g_space += sc->filesize; } diff --git a/include/vsub.h b/include/vsub.h index 6705b11..274663a 100644 --- a/include/vsub.h +++ b/include/vsub.h @@ -31,5 +31,5 @@ /* from libvarnish/subproc.c */ typedef void vsub_func_f(void*); -int VSUB_run(struct vsb *sb, vsub_func_f *func, void *priv, const char *name, +unsigned VSUB_run(struct vsb *sb, vsub_func_f *func, void *priv, const char *name, int maxlines); diff --git a/lib/libvarnish/vsub.c b/lib/libvarnish/vsub.c index 497119c..ffe93ab 100644 --- a/lib/libvarnish/vsub.c +++ b/lib/libvarnish/vsub.c @@ -64,7 +64,8 @@ vsub_vlu(void *priv, const char *str) return (0); } -int +/* returns an exit code */ +unsigned VSUB_run(struct vsb *sb, vsub_func_f *func, void *priv, const char *name, int maxlines) { @@ -81,7 +82,7 @@ VSUB_run(struct vsb *sb, vsub_func_f *func, void *priv, const char *name, if (pipe(p) < 0) { VSB_printf(sb, "Starting %s: pipe() failed: %s", name, strerror(errno)); - return (-1); + return (1); } assert(p[0] > STDERR_FILENO); assert(p[1] > STDERR_FILENO); @@ -90,7 +91,7 @@ VSUB_run(struct vsb *sb, vsub_func_f *func, void *priv, const char *name, name, strerror(errno)); AZ(close(p[0])); AZ(close(p[1])); - return (-1); + return (1); } if (pid == 0) { AZ(close(STDIN_FILENO)); @@ -101,7 +102,12 @@ VSUB_run(struct vsb *sb, vsub_func_f *func, void *priv, const char *name, for (sfd = STDERR_FILENO + 1; sfd < 100; sfd++) (void)close(sfd); func(priv); - _exit(1); + /* + * func should either exec or exit, so getting here should be + * treated like an assertion failure - except that we don't know + * if it's save to trigger an acutal assertion + */ + _exit(4); } AZ(close(p[1])); vlu = VLU_New(&sp, vsub_vlu, 0); @@ -117,19 +123,19 @@ VSUB_run(struct vsb *sb, vsub_func_f *func, void *priv, const char *name, if (rv < 0 && errno != EINTR) { VSB_printf(sb, "Running %s: waitpid() failed: %s\n", name, strerror(errno)); - return (-1); + return (1); } } while (rv < 0); if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { VSB_printf(sb, "Running %s failed", name); if (WIFEXITED(status)) - VSB_printf(sb, ", exit %d", WEXITSTATUS(status)); + VSB_printf(sb, ", exited with %d", WEXITSTATUS(status)); if (WIFSIGNALED(status)) VSB_printf(sb, ", signal %d", WTERMSIG(status)); if (WCOREDUMP(status)) VSB_printf(sb, ", core dumped"); VSB_printf(sb, "\n"); - return (-1); + return (WEXITSTATUS(status)); } return (0); } diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index c0efdbd..50033c2 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -224,7 +224,7 @@ tst(const char *s, time_t good) if (t != good) { printf("Parse error! Got: %jd should have %jd diff %jd\n", (intmax_t)t, (intmax_t)good, (intmax_t)(t - good)); - exit (2); + exit(4); } } @@ -267,7 +267,7 @@ tst_delta() if (err) { printf("%d time delta test errrors\n", err); - exit (2); + exit(4); } } From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] bda7856 tools: streamline exit codes, fix exit codes for vsubs Message-ID: commit bda7856dee92b42e86735c4c6888190a7ea4c696 Author: Nils Goroll Date: Thu Aug 28 12:35:33 2014 +0200 tools: streamline exit codes, fix exit codes for vsubs This should bring us closer to the exit codes now documented in varnishd.rst Fixes #1572 for tools diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c index 259094e..fbb2f4d 100644 --- a/bin/varnishlog/varnishlog.c +++ b/bin/varnishlog/varnishlog.c @@ -82,7 +82,7 @@ openout(int append) else LOG.fo = fopen(LOG.w_arg, append ? "a" : "w"); if (LOG.fo == NULL) - VUT_Error(1, "Can't open output file (%s)", + VUT_Error(2, "Can't open output file (%s)", LOG.B_opt ? VSL_Error(VUT.vsl) : strerror(errno)); VUT.dispatch_priv = LOG.fo; } diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c index d943d8f..50a9f27 100644 --- a/lib/libvarnishtools/vut.c +++ b/lib/libvarnishtools/vut.c @@ -159,7 +159,7 @@ VUT_Arg(int opt, const char *arg) case 'V': /* Print version number and exit */ VCS_Message(VUT.progname); - exit(1); + exit(0); default: AN(VUT.vsl); i = VSL_Arg(VUT.vsl, opt, arg); From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] bb29cda Fix a header name confusion Message-ID: commit bb29cda310616384ef44ff5fe8716887b10f335d Author: Nils Goroll Date: Thu Aug 28 13:01:18 2014 +0200 Fix a header name confusion Thanks to dridi.boukelmoune at zenika.com for reporting. diff --git a/doc/sphinx/users-guide/compression.rst b/doc/sphinx/users-guide/compression.rst index bea7176..a046311 100644 --- a/doc/sphinx/users-guide/compression.rst +++ b/doc/sphinx/users-guide/compression.rst @@ -60,8 +60,7 @@ alterations to the headers of the resulting object which cannot be modified in a backend VCL (but in `vcl_deliver`): * set `obj.http.Content-Encoding` to "gzip" -* add "Accept-Encoding" to `obj.http.Content-Encoding`, unless already - present +* add "Accept-Encoding" to `obj.http.Vary`, unless already present * weaken any `Etag` (by prepending "W/") Generally, Varnish doesn't use much CPU so it might make more sense to @@ -86,7 +85,7 @@ the headers of the resulting object which cannot be modified in a backend VCL (but in `vcl_deliver`): * remove `obj.http.Content-Encoding` -* remove any "Accept-Encoding" from `obj.http.Content-Encoding` +* remove any "Accept-Encoding" from `obj.http.Vary` (XXX review when closing #940) * weaken any `Etag` (by prepending "W/") From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 274df85 Update names and examples for V4 Message-ID: commit 274df859373d435cf50548f668c86f5addb70fd5 Author: Federico G. Schwindt Date: Thu Aug 28 18:04:30 2014 +0100 Update names and examples for V4 diff --git a/doc/sphinx/phk/gzip.rst b/doc/sphinx/phk/gzip.rst index 4200db8..ae244d9 100644 --- a/doc/sphinx/phk/gzip.rst +++ b/doc/sphinx/phk/gzip.rst @@ -55,9 +55,9 @@ Varnish will not gzip any content on its own (but see below), we trust the backend to know what content can be sensibly gzip'ed (html) and what can not (jpeg) -If in vcl_fetch{} we find out that we are trying to deliver a gzip'ed object -to a client that has not indicated willingness to receive gzip, we will -ungzip the object during deliver. +If in vcl_backend_response{} we find out that we are trying to deliver a +gzip'ed object to a client that has not indicated willingness to receive +gzip, we will ungzip the object during deliver. Tuning, tweaking and frobbing ----------------------------- @@ -74,8 +74,8 @@ gunzip the object before delivering to the client. In vcl_miss{} you can remove the "Accept-Encoding: gzip" header, if you do not want the backend to gzip this object. -In vcl_fetch{} two new variables allow you to modify the gzip-ness of -objects during fetch: +In vcl_backend_response{} two new variables allow you to modify the +gzip-ness of objects during fetch: set beresp.do_gunzip = true; @@ -91,8 +91,8 @@ Remember that a lot of content types cannot sensibly be gziped, most notably compressed image formats like jpeg, png and similar, so a typical use would be:: - sub vcl_fetch { - if (req.url ~ "html$") { + sub vcl_backend_response { + if (bereq.url ~ "html$") { set beresp.do_gzip = true; } } @@ -102,7 +102,7 @@ GZIP and ESI First, note the new syntax for activating ESI:: - sub vcl_fetch { + sub vcl_backend_response { set beresp.do_esi = true; } @@ -155,7 +155,7 @@ compression efficiency, you should:: } } - sub vcl_fetch { + sub vcl_backend_response { if (object needs ESI processing) { set beresp.do_esi = true; set beresp.do_gzip = true; From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 4ab15f7 Wording and whatnot Message-ID: commit 4ab15f7481d9840d8c2555387fe822e829f16df0 Author: Federico G. Schwindt Date: Thu Aug 28 18:06:34 2014 +0100 Wording and whatnot Also hide a might-be change until it's really implemented. diff --git a/doc/sphinx/users-guide/compression.rst b/doc/sphinx/users-guide/compression.rst index a046311..50c5c86 100644 --- a/doc/sphinx/users-guide/compression.rst +++ b/doc/sphinx/users-guide/compression.rst @@ -13,7 +13,6 @@ If you don't want Varnish tampering with the encoding you can disable compression all together by setting the parameter `http_gzip_support` to false. Please see man :ref:`ref-varnishd` for details. - Default behaviour ~~~~~~~~~~~~~~~~~ @@ -21,24 +20,23 @@ The default behaviour is active when the `http_gzip_support` parameter is set to "on" and neither `beresp.do_gzip` nor `beresp.do_gunzip` are used in VCL. -Unless returning from `vcl_recv` with `pipe` or `pass`, varnish -modifies `req.http.Accept-Encoding`: If the client supports gzip, -`req.http.Accept-Encoding` is set to "gzip". Otherwise, the header is +Unless returning from `vcl_recv` with `pipe` or `pass`, Varnish +modifies `req.http.Accept-Encoding`: if the client supports gzip +`req.http.Accept-Encoding` is set to "gzip", otherwise the header is removed. -Unless the request is a `pass`, Varnish sets -`bereq.http.Accept-Encoding` to "gzip" before `vcl_backend_fetch` -runs, so the header can be changed in VCL. +Unless the request is a `pass`, Varnish sets `bereq.http.Accept-Encoding` +to "gzip" before `vcl_backend_fetch` runs, so the header can be changed +in VCL. -If the server responds with gzip'ed content it will be stored in -memory in its compressed form and `Accept-Encoding` will be added to -the `Vary` header. +If the server responds with gzip'ed content it will be stored in memory +in its compressed form and `Accept-Encoding` will be added to the +`Vary` header. -To clients supporting gzip, compressed content is delivered -unmodified. +To clients supporting gzip, compressed content is delivered unmodified. For clients not supporting gzip, compressed content gets decompressed -on the fly while delivering. The `Content-Encoding` response header +on the fly while delivering it. The `Content-Encoding` response header gets removed and any `Etag` gets weakened (by prepending "W/"). For Vary Lookups, `Accept-Encoding` is ignored. @@ -46,54 +44,53 @@ For Vary Lookups, `Accept-Encoding` is ignored. Compressing content if backends don't ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You can make Varnish compress content before storing it in cache in -`vcl_backend_response` by setting `beresp.do_gzip` to true, like this:: +You can tell Varnish to compress content before storing it in cache in +`vcl_backend_response` by setting `beresp.do_gzip` to "true", like this:: - sub vcl_backend_response { + sub vcl_backend_response { if (beresp.http.content-type ~ "text") { - set beresp.do_gzip = true; + set beresp.do_gzip = true; } - } + } With `beresp.do_gzip` set to "true", Varnish will make the following -alterations to the headers of the resulting object which cannot be -modified in a backend VCL (but in `vcl_deliver`): +changes to the headers of the resulting object before inserting it in +the cache: * set `obj.http.Content-Encoding` to "gzip" * add "Accept-Encoding" to `obj.http.Vary`, unless already present * weaken any `Etag` (by prepending "W/") Generally, Varnish doesn't use much CPU so it might make more sense to -have Varnish spend CPU cycles compressing content than doing it in -your web- or application servers, which are more likely to be -CPU-bound. +have Varnish spend CPU cycles compressing content than doing it in your +web- or application servers, which are more likely to be CPU-bound. Please make sure that you don't try to compress content that is -uncompressable, like jpgs, gifs and mp3. You'll only waste CPU cycles. +uncompressable, like JPG, GIF and MP3 files. You'll only waste CPU cycles. Uncompressing content before entering the cache ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You can also uncompress objects before storing it in memory by setting -`beresp.do_gunzip` to true. One use case for this feature is to work -around badly configured backends uselessly compressing already -compressed content like JPG images (but fixing the misbehaving backend -is always the better option). +You can also uncompress content before storing it in cache by setting +`beresp.do_gunzip` to "true". One use case for this feature is to work +around badly configured backends uselessly compressing already compressed +content like JPG images (but fixing the misbehaving backend is always +the better option). -With `beresp.do_gunzip` Varnish will make the following alterations to -the headers of the resulting object which cannot be modified in a -backend VCL (but in `vcl_deliver`): +With `beresp.do_gunzip` set to "true", Varnish will make the following +changes to the headers of the resulting object before inserting it in +the cache: * remove `obj.http.Content-Encoding` -* remove any "Accept-Encoding" from `obj.http.Vary` - (XXX review when closing #940) * weaken any `Etag` (by prepending "W/") +.. XXX pending closing #940: remove any "Accept-Encoding" from `obj.http.Vary` + GZIP and ESI ~~~~~~~~~~~~ -If you are using Edge Side Includes (ESIs) you'll be happy to note that ESI -and GZIP work together really well. Varnish will magically decompress +If you are using Edge Side Includes (ESI) you'll be happy to note that +ESI and GZIP work together really well. Varnish will magically decompress the content to do the ESI-processing, then recompress it for efficient storage and delivery. @@ -108,5 +105,5 @@ Accept-Encoding` like it would for any other `Vary` value and ignores A random outburst ~~~~~~~~~~~~~~~~~ -Poul-Henning Kamp has written :ref:`phk_gzip` which talks a bit more about how the -implementation works. +Poul-Henning Kamp has written :ref:`phk_gzip` which talks a bit more +about how the implementation works. From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 21a1ae5 fix a typo Message-ID: commit 21a1ae5850339a6405e44b390d7d102774c3ed79 Author: Nils Goroll Date: Fri Aug 29 14:32:19 2014 +0200 fix a typo diff --git a/lib/libvarnish/vas.c b/lib/libvarnish/vas.c index d63763d..9380b6c 100644 --- a/lib/libvarnish/vas.c +++ b/lib/libvarnish/vas.c @@ -49,7 +49,7 @@ VAS_Fail_default(const char *func, const char *file, int line, func, file, line, cond); } else if (kind == VAS_INCOMPLETE) { fprintf(stderr, - "Incompelte code in %s(), %s line %d:\n", + "Incomplete code in %s(), %s line %d:\n", func, file, line); } else if (kind == VAS_WRONG) { fprintf(stderr, From lkarsten at varnish-software.com Mon Sep 22 14:38:24 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:24 +0200 Subject: [4.0] 5c54bd5 Return 2 from VSUB_run() when the sub process dies with a signal. Message-ID: commit 5c54bd5d7c162f08d8543cf7d3043bec7c7869d4 Author: Nils Goroll Date: Fri Aug 29 14:44:36 2014 +0200 Return 2 from VSUB_run() when the sub process dies with a signal. Fixes a regression from 33e6fe71743059b912d2b00cdbc08196b003e440 Fixes #1585 diff --git a/lib/libvarnish/vsub.c b/lib/libvarnish/vsub.c index ffe93ab..767ec5a 100644 --- a/lib/libvarnish/vsub.c +++ b/lib/libvarnish/vsub.c @@ -127,15 +127,21 @@ VSUB_run(struct vsb *sb, vsub_func_f *func, void *priv, const char *name, } } while (rv < 0); if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + rv = -1; VSB_printf(sb, "Running %s failed", name); - if (WIFEXITED(status)) - VSB_printf(sb, ", exited with %d", WEXITSTATUS(status)); - if (WIFSIGNALED(status)) + if (WIFEXITED(status)) { + rv = WEXITSTATUS(status); + VSB_printf(sb, ", exited with %d", rv); + } + if (WIFSIGNALED(status)) { + rv = 2; VSB_printf(sb, ", signal %d", WTERMSIG(status)); + } if (WCOREDUMP(status)) VSB_printf(sb, ", core dumped"); VSB_printf(sb, "\n"); - return (WEXITSTATUS(status)); + assert(rv != -1); + return (rv); } return (0); } From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 3d6268f panic with INCOMPL() rather than exit(4) Message-ID: commit 3d6268f839994a8a568129487e1a137ce0edc3ba Author: Nils Goroll Date: Fri Aug 29 14:50:31 2014 +0200 panic with INCOMPL() rather than exit(4) Fixes #1583 diff --git a/bin/varnishd/mgt/mgt_sandbox_solaris.c b/bin/varnishd/mgt/mgt_sandbox_solaris.c index 2d92968..1cc277a 100644 --- a/bin/varnishd/mgt/mgt_sandbox_solaris.c +++ b/bin/varnishd/mgt/mgt_sandbox_solaris.c @@ -235,8 +235,7 @@ mgt_sandbox_solaris_add_inheritable(priv_set_t *pset, enum sandbox_e who) case SANDBOX_WORKER: break; default: - REPORT(LOG_ERR, "INCOMPLETE AT: %s(%d)\n", __func__, __LINE__); - exit(4); + INCOMPL(); } } @@ -262,8 +261,7 @@ mgt_sandbox_solaris_add_effective(priv_set_t *pset, enum sandbox_e who) priv_setop_assert(priv_addset(pset, "file_write")); break; default: - REPORT(LOG_ERR, "INCOMPLETE AT: %s(%d)\n", __func__, __LINE__); - exit(4); + INCOMPL(); } } @@ -285,8 +283,7 @@ mgt_sandbox_solaris_add_permitted(priv_set_t *pset, enum sandbox_e who) AZ(priv_addset(pset, PRIV_SYS_RESOURCE)); break; default: - REPORT(LOG_ERR, "INCOMPLETE AT: %s(%d)\n", __func__, __LINE__); - exit(4); + INCOMPL(); } } From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 765124a make the update interval a double to allow values < 1 Message-ID: commit 765124a841b5249a6a1b83b0f27cbdad71064702 Author: Nils Goroll Date: Fri Aug 29 18:04:56 2014 +0200 make the update interval a double to allow values < 1 diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c index 6fa7cc7..2f41daa 100644 --- a/bin/varnishstat/varnishstat.c +++ b/bin/varnishstat/varnishstat.c @@ -273,7 +273,8 @@ main(int argc, char * const *argv) { int c; struct VSM_data *vd; - int delay = 1, once = 0, xml = 0, json = 0, do_repeat = 0, f_list = 0; + double delay = 1.0; + int once = 0, xml = 0, json = 0, do_repeat = 0, f_list = 0; vd = VSM_New(); @@ -290,7 +291,7 @@ main(int argc, char * const *argv) exit(0); case 'w': do_repeat = 1; - delay = atoi(optarg); + delay = atof(optarg); break; case 'x': xml = 1; @@ -332,7 +333,7 @@ main(int argc, char * const *argv) // end of output block marker. printf("\n"); - sleep(delay); + usleep(delay * 1e6); } exit(0); } diff --git a/bin/varnishstat/varnishstat.h b/bin/varnishstat/varnishstat.h index 89dee92..18d3def 100644 --- a/bin/varnishstat/varnishstat.h +++ b/bin/varnishstat/varnishstat.h @@ -35,4 +35,4 @@ #include "vas.h" #include "vcs.h" -void do_curses(struct VSM_data *vd, int delay); +void do_curses(struct VSM_data *vd, double delay); diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index 8ce35f1..4d63e71 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -908,7 +908,7 @@ handle_keypress(int ch) } void -do_curses(struct VSM_data *vd, int delay) +do_curses(struct VSM_data *vd, double delay) { struct pollfd pollfd; long t; From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 31ebc99 should ignore return value of usleep() Message-ID: commit 31ebc99ef9eb7c1b27b776dc5f379ef25909c684 Author: Nils Goroll Date: Fri Aug 29 18:09:09 2014 +0200 should ignore return value of usleep() diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c index 2f41daa..f5153b4 100644 --- a/bin/varnishstat/varnishstat.c +++ b/bin/varnishstat/varnishstat.c @@ -333,7 +333,7 @@ main(int argc, char * const *argv) // end of output block marker. printf("\n"); - usleep(delay * 1e6); + (void)usleep(delay * 1e6); } exit(0); } From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] fce798b Add another request to this test-case, hoping to catch info about #1520 Message-ID: commit fce798b2bd1f6dd518b5e15bbee63f67be1712b8 Author: Poul-Henning Kamp Date: Mon Sep 1 07:16:31 2014 +0000 Add another request to this test-case, hoping to catch info about #1520 diff --git a/bin/varnishtest/tests/m00011.vtc b/bin/varnishtest/tests/m00011.vtc index 9f17dfe..bd17059 100644 --- a/bin/varnishtest/tests/m00011.vtc +++ b/bin/varnishtest/tests/m00011.vtc @@ -4,6 +4,8 @@ varnishtest "Test std.ip" server s1 { rxreq txresp -body "1" + rxreq + txresp -body "22" } -start @@ -21,8 +23,12 @@ varnish v1 -vcl+backend { client c1 { txreq -url "/foo1" rxresp + expect resp.bodylen == 1 expect resp.http.foo0 == "127.0.0.1" expect resp.http.foo1 == "127.0.0.1" expect resp.http.foo2 == "127.0.0.2" expect resp.http.foo3 == "1.2.3.5" + txreq -url "/foo2" + rxresp + expect resp.bodylen == 2 } -run From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] f0994a0 Don't leak struct req if we cannot restart it from a waiting list. Message-ID: commit f0994a0bf0bc165fafbd8fc128b4aa1d0d6fe541 Author: Poul-Henning Kamp Date: Mon Sep 1 07:35:16 2014 +0000 Don't leak struct req if we cannot restart it from a waiting list. Fixes: #1466 diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 73acf01..76ad5c3 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -537,9 +537,9 @@ hsh_rush(struct dstat *ds, struct objhead *oh) DSL(DBG_WAITINGLIST, req->vsl->wid, "off waiting list"); if (SES_ScheduleReq(req)) { /* - * We could not schedule the session, leave the - * rest on the busy list. + * We could not schedule the session, put it back. */ + VTAILQ_INSERT_HEAD(&wl->list, req, w_list); break; } } From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] abe6218 Reinstate this testcase, but make the servers failure to deliver non-fatal. Message-ID: commit abe621845787c100e90358b138e8600e48aaaaf5 Author: Poul-Henning Kamp Date: Mon Sep 1 08:09:46 2014 +0000 Reinstate this testcase, but make the servers failure to deliver non-fatal. Fixes: #1414 diff --git a/bin/varnishtest/tests.disabled/r01391.vtc b/bin/varnishtest/tests.disabled/r01391.vtc deleted file mode 100644 index 4b98fda..0000000 --- a/bin/varnishtest/tests.disabled/r01391.vtc +++ /dev/null @@ -1,39 +0,0 @@ -varnishtest "client abandoning hit-for-pass" - - -server s1 { - rxreq - txresp -nolen -hdr "Transfer-Encoding: chunked" -hdr "Set-Cookie: foo=bar" - chunked "foo" - sema r1 sync 2 - chunked "bar" - delay .1 - chunkedlen 64 - delay .1 - chunkedlen 64 - chunkedlen 0 -} -start - -varnish v1 -vcl+backend { -} -start - - -client c1 { - txreq - rxhdrs - rxchunk - sema r1 sync 2 -} -run - -delay 2 - -server s1 { - rxreq - txresp -} -start - -client c1 { - txreq - rxresp - expect resp.status == 200 -} -run diff --git a/bin/varnishtest/tests/r01391.vtc b/bin/varnishtest/tests/r01391.vtc new file mode 100644 index 0000000..3858558 --- /dev/null +++ b/bin/varnishtest/tests/r01391.vtc @@ -0,0 +1,40 @@ +varnishtest "client abandoning hit-for-pass" + + +server s1 { + non-fatal + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" -hdr "Set-Cookie: foo=bar" + chunked "foo" + sema r1 sync 2 + chunked "bar" + delay .1 + chunkedlen 64 + delay .1 + chunkedlen 64 + chunkedlen 0 +} -start + +varnish v1 -vcl+backend { +} -start + + +client c1 { + txreq + rxresphdrs + rxchunk + sema r1 sync 2 +} -run + +delay 2 + +server s1 { + rxreq + txresp +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 8916322 Don't expect the workspace length to be aligned. Message-ID: commit 8916322619ffb5cbb6af59661d602d706ed0674e Author: Poul-Henning Kamp Date: Mon Sep 1 08:48:47 2014 +0000 Don't expect the workspace length to be aligned. diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index 3930c4d..91613b5 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -75,8 +75,8 @@ WS_Init(struct ws *ws, const char *id, void *space, unsigned len) ws->magic = WS_MAGIC; ws->s = space; assert(PAOK(space)); + len = PRNDDN(len); ws->e = ws->s + len; - assert(PAOK(len)); ws->f = ws->s; assert(id[0] & 0x40); assert(strlen(id) < sizeof ws->id); From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 49900b7 Bump the minimum workspace_client to 9k. Message-ID: commit 49900b779035d2f2f1e0e3f677a40972017ab833 Author: Poul-Henning Kamp Date: Mon Sep 1 08:57:24 2014 +0000 Bump the minimum workspace_client to 9k. diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index 9135d8f..62aff53 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -92,7 +92,7 @@ struct parspec mgt_parspec[] = { "384", "bytes" }, { "workspace_client", tweak_bytes_u, &mgt_param.workspace_client, - "3072", NULL, + "9k", NULL, "Bytes of HTTP protocol workspace for clients HTTP req/resp." " If larger than 4k, use a multiple of 4k for VM efficiency.", DELAYED_EFFECT, From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 30f2d7f These two disabled testcases don't make sense any more Message-ID: commit 30f2d7ff0d96145837f3dc12590f801121cc0b2e Author: Poul-Henning Kamp Date: Mon Sep 1 09:17:02 2014 +0000 These two disabled testcases don't make sense any more diff --git a/bin/varnishtest/tests.disabled/r00929.vtc b/bin/varnishtest/tests.disabled/r00929.vtc deleted file mode 100644 index 6f7729d..0000000 --- a/bin/varnishtest/tests.disabled/r00929.vtc +++ /dev/null @@ -1,26 +0,0 @@ -varnishtest "stream test" - - -server s1 { - rxreq - txresp -bodylen 400 -} -start - -varnish v1 -vcl+backend { - - sub vcl_backend_response { - set beresp.do_stream = true; - } - -} -start - -client c1 { - txreq - rxresp - expect resp.bodylen == 400 - txreq - rxresp - expect resp.bodylen == 400 -} -run - - diff --git a/bin/varnishtest/tests.disabled/r00978.vtc b/bin/varnishtest/tests.disabled/r00978.vtc deleted file mode 100644 index e02be57..0000000 --- a/bin/varnishtest/tests.disabled/r00978.vtc +++ /dev/null @@ -1,40 +0,0 @@ -varnishtest "r00978.vtc Test esi_level > 0 and do_stream" - -server s1 { - rxreq - expect req.url == "/" - txresp -body { - - Before include - - After include - } - rxreq - expect req.url == "/body1" - txresp -body { - Included file - } -} -start - -varnish v1 -vcl+backend { - sub vcl_recv { - if (req.esi_level > 0) { - set req.url = req.url + req.esi_level; - } - } - sub vcl_backend_response { - if (bereq.url == "/") { - set beresp.do_esi = true; - } - set beresp.do_stream = true; - } -} -start - -client c1 { - txreq - rxresp - expect resp.bodylen == 65 - expect resp.status == 200 -} -run - -varnish v1 -expect esi_errors == 0 From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 6c0eb3d Resurrect this test-case Message-ID: commit 6c0eb3d59736ca036bf664698b8d33b532291bcd Author: Poul-Henning Kamp Date: Mon Sep 1 09:23:35 2014 +0000 Resurrect this test-case diff --git a/bin/varnishtest/tests/r00979.vtc b/bin/varnishtest/tests/r00979.vtc new file mode 100644 index 0000000..46d7fa9 --- /dev/null +++ b/bin/varnishtest/tests/r00979.vtc @@ -0,0 +1,31 @@ +varnishtest "r00979.vtc Test restart when do_stream in vcl_deliver" + +server s1 { + rxreq + txresp -status 200 -gzipbody "1" + + accept + rxreq + txresp -status 200 -body "11" +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_response { + set beresp.do_stream = true; + set beresp.uncacheable = true; + } + sub vcl_deliver { + if (req.restarts == 0) { + return (restart); + } + } +} -start + +varnish v1 -cliok "param.set debug +syncvsl" + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.bodylen == 2 +} -run From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] ab54a9e Don't log SLT_ReqAcct when we don't have assigned a VXID yet. Message-ID: commit ab54a9e571b5368b7a050ae4602c3805f42d5e38 Author: Martin Blix Grydeland Date: Mon Sep 1 14:22:34 2014 +0200 Don't log SLT_ReqAcct when we don't have assigned a VXID yet. This prevents SLT_ReqAcct log records with 0 as VXID. diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index d57de9a..73a36c0 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -956,7 +956,7 @@ CNT_AcctLogCharge(struct dstat *ds, struct req *req) a = &req->acct; - if (!(req->res_mode & RES_PIPE)) { + if (req->vsl->wid && !(req->res_mode & RES_PIPE)) { VSLb(req->vsl, SLT_ReqAcct, "%ju %ju %ju %ju %ju %ju", (uintmax_t)a->req_hdrbytes, (uintmax_t)a->req_bodybytes, From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 234c2ef Resurrect another test-case Message-ID: commit 234c2ef5946319a54ee21bff895301b5384065d1 Author: Poul-Henning Kamp Date: Mon Sep 1 09:30:30 2014 +0000 Resurrect another test-case diff --git a/bin/varnishtest/tests.disabled/r00979.vtc b/bin/varnishtest/tests.disabled/r00979.vtc deleted file mode 100644 index 432aea5..0000000 --- a/bin/varnishtest/tests.disabled/r00979.vtc +++ /dev/null @@ -1,29 +0,0 @@ -varnishtest "r00979.vtc Test restart when do_stream in vcl_deliver" - -server s1 { - rxreq - txresp -status 200 -gzipbody "1" - expect_close - - accept - rxreq - txresp -status 200 -body "11" -} -start - -varnish v1 -vcl+backend { - sub vcl_backend_response { - set beresp.do_stream = true; - } - sub vcl_deliver { - if (req.restarts == 0) { - return (restart); - } - } -} -start - -client c1 { - txreq - rxresp - expect resp.status == 200 - expect resp.bodylen == 2 -} -run diff --git a/bin/varnishtest/tests.disabled/r00980.vtc b/bin/varnishtest/tests.disabled/r00980.vtc deleted file mode 100644 index 006de3e..0000000 --- a/bin/varnishtest/tests.disabled/r00980.vtc +++ /dev/null @@ -1,29 +0,0 @@ -varnishtest "r00980 test gzip on fetch with content_length and do_stream" - -server s1 { - rxreq - expect req.url == "/foobar" - expect req.http.accept-encoding == "gzip" - txresp -bodylen 43 -} -start - -varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend { - - sub vcl_backend_response { - set beresp.do_gzip = true; - set beresp.do_stream = true; - } -} -start - -client c1 { - txreq -url /foobar -hdr "Accept-Encoding: gzip" - rxresp - expect resp.http.content-encoding == "gzip" - gunzip - expect resp.bodylen == 43 - - txreq -url /foobar - rxresp - expect resp.http.content-encoding == - expect resp.bodylen == 43 -} -run diff --git a/bin/varnishtest/tests/r00980.vtc b/bin/varnishtest/tests/r00980.vtc new file mode 100644 index 0000000..006de3e --- /dev/null +++ b/bin/varnishtest/tests/r00980.vtc @@ -0,0 +1,29 @@ +varnishtest "r00980 test gzip on fetch with content_length and do_stream" + +server s1 { + rxreq + expect req.url == "/foobar" + expect req.http.accept-encoding == "gzip" + txresp -bodylen 43 +} -start + +varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend { + + sub vcl_backend_response { + set beresp.do_gzip = true; + set beresp.do_stream = true; + } +} -start + +client c1 { + txreq -url /foobar -hdr "Accept-Encoding: gzip" + rxresp + expect resp.http.content-encoding == "gzip" + gunzip + expect resp.bodylen == 43 + + txreq -url /foobar + rxresp + expect resp.http.content-encoding == + expect resp.bodylen == 43 +} -run From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 3826496 Don't rename the VSM file into place until we have started the child process during startup. Message-ID: commit 3826496b3a00e769ce580396a4f40efea423e491 Author: Poul-Henning Kamp Date: Mon Sep 1 14:46:33 2014 +0000 Don't rename the VSM file into place until we have started the child process during startup. Fixes #1407 diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 4f4a8f6..c57ac27 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -107,6 +107,7 @@ void mgt_SHM_Init(void); void mgt_SHM_static_alloc(const void *, ssize_t size, const char *class, const char *type, const char *ident); void mgt_SHM_Create(void); +void mgt_SHM_Commit(void); void mgt_SHM_Destroy(int keep); void mgt_SHM_Size_Adjust(void); diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 1fe8473..7c777de 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -570,6 +570,7 @@ mgt_reap_child(void) mgt_SHM_Destroy(0); } mgt_SHM_Create(); + mgt_SHM_Commit(); if (child_state == CH_RUNNING) child_state = CH_DIED; @@ -752,13 +753,15 @@ MGT_Run(void) REPORT0(LOG_ERR, "No VCL loaded yet"); else if (!d_flag) { mgt_launch_child(NULL); - if (child_state == CH_STOPPED) { + if (child_state != CH_RUNNING) { // XXX correct? or 0? exit_status = 2; return; } } + mgt_SHM_Commit(); + i = vev_schedule(mgt_evb); if (i != 0) REPORT(LOG_ERR, "vev_schedule() = %d", i); diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c index ed020f7..dc1fa5c 100644 --- a/bin/varnishd/mgt/mgt_shmem.c +++ b/bin/varnishd/mgt/mgt_shmem.c @@ -246,17 +246,28 @@ mgt_SHM_Create(void) AN(VSC_C_mgt); *VSC_C_mgt = static_VSC_C_mgt; +#ifdef __OpenBSD__ + /* Commit changes, for OS's without coherent VM/buf */ + AZ(msync(p, getpagesize(), MS_SYNC)); +#endif +} + +/*-------------------------------------------------------------------- + * Commit the VSM + */ + +void +mgt_SHM_Commit(void) +{ + char fnbuf[64]; + + bprintf(fnbuf, "%s.%jd", VSM_FILENAME, (intmax_t)getpid()); if (rename(fnbuf, VSM_FILENAME)) { fprintf(stderr, "Rename failed %s -> %s: %s\n", fnbuf, VSM_FILENAME, strerror(errno)); (void)unlink(fnbuf); exit(1); } - -#ifdef __OpenBSD__ - /* Commit changes, for OS's without coherent VM/buf */ - AZ(msync(p, getpagesize(), MS_SYNC)); -#endif } /*-------------------------------------------------------------------- From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] fb58aeb Don't add a trailing dash to URL if missing path. Message-ID: commit fb58aeb74a736f0a4be93238aa4bade43d3c4f13 Author: Lasse Karstensen Date: Tue Sep 2 12:29:47 2014 +0200 Don't add a trailing dash to URL if missing path. Adding a dash if missing the path component does not make sense. Related bug: #1584 diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 685d869..59c1918 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -300,7 +300,7 @@ format_requestline(const struct format *format) AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_host], NULL)); } else AZ(VSB_cat(CTX.vsb, "http://localhost")); - AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_U], "-")); + AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_U], NULL)); AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_q], "")); AZ(VSB_putc(CTX.vsb, ' ')); AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_H], "HTTP/1.0")); From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 1eb0a36 Do not log garbage requests. Message-ID: commit 1eb0a36786e2d653daa14f28af514f516d30514d Author: Lasse Karstensen Date: Tue Sep 2 12:33:16 2014 +0200 Do not log garbage requests. Requests that end up in the hard "400 Bad Request" handling used to be logged with incomplete data. (no method, maybe no path, possibly no proto, and no response status) Port scans or anything sending a byte and a linefeed would be logged. Since this is used for logging access to a web site, it makes more sense to skip these garbage requests. Fixes: #1584 diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 59c1918..a6658e3 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -760,6 +760,9 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], e--; switch (tag) { + case SLT_HttpGarbage: + skip = 1; + break; case SLT_PipeAcct: frag_fields(b, e, 3, &CTX.frag[F_I], From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 5e92b13 Fix assertion error introduced in bb9d7879. Message-ID: commit 5e92b134ffbee6739bf47c3d3882c7238153672d Author: Lasse Karstensen Date: Tue Sep 2 13:45:28 2014 +0200 Fix assertion error introduced in bb9d7879. diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index a6658e3..162829d 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -300,7 +300,7 @@ format_requestline(const struct format *format) AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_host], NULL)); } else AZ(VSB_cat(CTX.vsb, "http://localhost")); - AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_U], NULL)); + AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_U], "")); AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_q], "")); AZ(VSB_putc(CTX.vsb, ' ')); AZ(vsb_fcat(CTX.vsb, &CTX.frag[F_H], "HTTP/1.0")); From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 1b14357 Remove noisy debugging Message-ID: commit 1b14357f89016a9f0eb5d98fe30fc304d32430e1 Author: Poul-Henning Kamp Date: Tue Sep 2 20:40:53 2014 +0000 Remove noisy debugging diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 754438a..f4de5f3 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -334,7 +334,6 @@ pool_stat_summ(struct worker *wrk, void *priv) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(wrk->pool, POOL_MAGIC); - VSL(SLT_Debug, 0, "STATSUMM"); AN(priv); src = priv; Lck_Lock(&wstat_mtx); From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] a613f49 Protect this test-case against a streaming related race. Message-ID: commit a613f494e4153c7201e21ce565e4ca22ffb91141 Author: Poul-Henning Kamp Date: Wed Sep 3 10:02:47 2014 +0000 Protect this test-case against a streaming related race. diff --git a/bin/varnishtest/tests/r01399.vtc b/bin/varnishtest/tests/r01399.vtc index 48a0bc0..5d7171a 100644 --- a/bin/varnishtest/tests/r01399.vtc +++ b/bin/varnishtest/tests/r01399.vtc @@ -23,6 +23,7 @@ server s1 { varnish v1 -vcl+backend { sub vcl_backend_response { + set beresp.do_stream = false; set beresp.ttl = 2s; set beresp.grace = 1800s; } From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] d6755ff Improve examples and reduce duplicated information Message-ID: commit d6755fffe99d491b7a552331263ac833f0a13ac9 Author: Federico G. Schwindt Date: Wed Sep 3 14:52:37 2014 +0100 Improve examples and reduce duplicated information diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 6828f99..90ac019 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -213,6 +213,8 @@ are available: Backends can be used with *directors*. Please see the vmod_directors(3) man page for more information. +.. _reference-vcl_probes: + Probes ------ diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst index 5d31e77..1a1f556 100644 --- a/doc/sphinx/users-guide/vcl-backends.rst +++ b/doc/sphinx/users-guide/vcl-backends.rst @@ -156,14 +156,14 @@ Health checks ------------- Lets set up a director with two backends and health checks. First let -us define the backends.:: +us define the backends:: backend server1 { .host = "server1.example.com"; .probe = { .url = "/"; + .timeout = 1s; .interval = 5s; - .timeout = 1 s; .window = 5; .threshold = 3; } @@ -173,40 +173,23 @@ us define the backends.:: .host = "server2.example.com"; .probe = { .url = "/"; + .timeout = 1s; .interval = 5s; - .timeout = 1 s; .window = 5; .threshold = 3; } } -What is new here is the ``probe``. Varnish will check the health of each -backend with a probe. The options are: - -url - The URL Varnish will use to send a probe request. - -interval - How often should we poll. - -timeout - What is the timeout of the probe. - -window - Varnish will retain up to this many probes when considering backend health. - -threshold - How many of the `.window` last polls must be good for the backend to be - declared healthy. - -initial - How many of the probes that needs to be succesful when Varnish starts. - Defaults to the same amount as the threshold. +What is new here is the ``probe``. In this example Varnish will check the +health of each backend every 5 seconds, timing out after 1 second. Each +poll will send a GET request to /. If 3 out of the last 5 polls succeeded +the backend is considered healthy, otherwise it will be marked as sick. +Refer to the :ref:`reference-vcl_probes` section in the +:ref:`reference-vcl` documentation for more information. Now we define the 'director':: - import directors; sub vcl_init { @@ -215,17 +198,16 @@ Now we define the 'director':: vdir.add_backend(server2); } +You use this `vdir` director as a backend_hint for requests, just like +you would with a simple backend. Varnish will not send traffic to hosts +that are marked as unhealthy. -You use this `vdir` director as a backend_hint for requests, just like you would -with a simple backend. Varnish will not send traffic to hosts that are marked as -unhealthy. - -Varnish can also serve stale content if all the backends are -down. See :ref:`users-guide-handling_misbehaving_servers` for more -information on how to enable this. - -Please note that Varnish will keep health probes running for all loaded VCLs. Varnish -will coalesce probes that seem identical - so be careful not to change the -probe config if you do a lot of VCL loading. Unloading the VCL will discard the -probes. For more information on how to do this please see ref:`reference-vcl-director`. +Varnish can also serve stale content if all the backends are down. See +:ref:`users-guide-handling_misbehaving_servers` for more information on +how to enable this. +Please note that Varnish will keep health probes running for all loaded +VCLs. Varnish will coalesce probes that seem identical - so be careful +not to change the probe config if you do a lot of VCL loading. Unloading +the VCL will discard the probes. For more information on how to do this +please see ref:`reference-vcl-director`. From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] b853437 Double allocated varnishlog buffer space. Message-ID: commit b85343734ce803d97eef60d1ef3a1f25a7f83acf Author: Lasse Karstensen Date: Wed Sep 3 17:18:08 2014 +0200 Double allocated varnishlog buffer space. Extensive test cases often lead to a lot of log output, which at times tripped the following assert: Assert error in vtc_log_emit(), vtc_log.c line 122: Condition(vtclog_left > l) not true. Double the amount of buffer space allowed to avoid this. Another 256KB memory use per concurrent running test should be fine. diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 823a0a2..6fdfb6f 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -260,7 +260,7 @@ start_test(void) ALLOC_OBJ(jp, JOB_MAGIC); AN(jp); - jp->bufsiz = 256*1024; /* XXX */ + jp->bufsiz = 512*1024; /* XXX */ jp->buf = mmap(NULL, jp->bufsiz, PROT_READ|PROT_WRITE, MAP_ANON | MAP_SHARED, -1, 0); From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] cb07d86 Squash warnings for warn_unused_result. Message-ID: commit cb07d86cbb0b99f941aeead613d564a40b8e1c50 Author: Lasse Karstensen Date: Thu Sep 4 12:56:47 2014 +0200 Squash warnings for warn_unused_result. diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c index 610ca8b..5498f11 100644 --- a/lib/libvarnish/vlu.c +++ b/lib/libvarnish/vlu.c @@ -96,6 +96,7 @@ vlu_dotelnet(struct vlu *l, char *p) char *e; char tno[3]; int i; + ssize_t r = 0; e = l->buf + l->bufp; assert(p >= l->buf && p < e); @@ -120,7 +121,9 @@ vlu_dotelnet(struct vlu *l, char *p) /* Return WONT for these */ memcpy(tno, p, 3); tno[1] = (char)252; - (void)write(l->telnet, tno, 3); + r = write(l->telnet, tno, 3); + if (r != 3) + return (1); i = 3; break; default: diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 6e0f20a..7319253 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -408,6 +408,7 @@ VSL_WriteOpen(struct VSL_data *vsl, const char *name, int append, int unbuf) { const char head[] = VSL_FILE_ID; FILE* f; + size_t r = 0; f = fopen(name, append ? "a" : "w"); if (f == NULL) { @@ -417,7 +418,12 @@ VSL_WriteOpen(struct VSL_data *vsl, const char *name, int append, int unbuf) if (unbuf) setbuf(f, NULL); if (0 == ftell(f)) - fwrite(head, 1, sizeof head, f); + r = fwrite(head, 1, sizeof head, f); + if (r != sizeof head) { + vsl_diag(vsl, "%s", strerror(errno)); + fclose(f); + return (NULL); + } return (f); } From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 1c4e35f Add an informative warning when only starting the master process in daemon mode Message-ID: commit 1c4e35f838f2e969dfc186c3b0fc9f3f8fbdd782 Author: Nils Goroll Date: Fri Aug 29 15:47:47 2014 +0200 Add an informative warning when only starting the master process in daemon mode Fixes #1580 diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index ea7f950..28b4823 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -661,6 +661,12 @@ main(int argc, char * const *argv) if (MGT_open_sockets()) ARGV_ERR("Failed to open (any) accept sockets.\n"); MGT_close_sockets(); + + if (b_arg == NULL && f_arg == NULL) { + fprintf(stderr, + "Warning: Neither -b nor -f given, won't start a worker child.\n" + " Master process started, use varnishadm to control it.\n"); + } } /* If no -s argument specified, process default -s argument */ From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 488f16a Clean up mistakes from d63c53e. Message-ID: commit 488f16a039f824d90fa20e62d2575390b7ffc9ad Author: Lasse Karstensen Date: Thu Sep 4 14:04:34 2014 +0200 Clean up mistakes from d63c53e. Add missing braces, avoid unnecessary variables. Noticed by: fgs diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c index 5498f11..c983442 100644 --- a/lib/libvarnish/vlu.c +++ b/lib/libvarnish/vlu.c @@ -96,7 +96,6 @@ vlu_dotelnet(struct vlu *l, char *p) char *e; char tno[3]; int i; - ssize_t r = 0; e = l->buf + l->bufp; assert(p >= l->buf && p < e); @@ -121,8 +120,7 @@ vlu_dotelnet(struct vlu *l, char *p) /* Return WONT for these */ memcpy(tno, p, 3); tno[1] = (char)252; - r = write(l->telnet, tno, 3); - if (r != 3) + if (write(l->telnet, tno, 3) != 3) return (1); i = 3; break; diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 7319253..949028f 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -408,8 +408,6 @@ VSL_WriteOpen(struct VSL_data *vsl, const char *name, int append, int unbuf) { const char head[] = VSL_FILE_ID; FILE* f; - size_t r = 0; - f = fopen(name, append ? "a" : "w"); if (f == NULL) { vsl_diag(vsl, "%s", strerror(errno)); @@ -417,13 +415,13 @@ VSL_WriteOpen(struct VSL_data *vsl, const char *name, int append, int unbuf) } if (unbuf) setbuf(f, NULL); - if (0 == ftell(f)) - r = fwrite(head, 1, sizeof head, f); - if (r != sizeof head) { + if (0 == ftell(f)) { + if (fwrite(head, 1, sizeof head, f) != sizeof head) { vsl_diag(vsl, "%s", strerror(errno)); fclose(f); return (NULL); } + } return (f); } From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] ffa893e Squash further warn_unused_result warnings. Message-ID: commit ffa893e3c69df0ddf6b3f3cffdc35154e82b5ab4 Author: Lasse Karstensen Date: Thu Sep 4 14:45:11 2014 +0200 Squash further warn_unused_result warnings. Not being able to fchown() is normal when running varnishd unprivileged, so don't report failure unless running as root. diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index b76b016..35e3806 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -248,7 +248,10 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag, unsigned *status) *status = 2; return (NULL); } - (void)fchown(sfd, mgt_param.uid, mgt_param.gid); + if (fchown(sfd, mgt_param.uid, mgt_param.gid) != 0) + if (geteuid() == 0) + VSB_printf(sb, "Failed to change owner on %s: %s\n", + sf, strerror(errno)); AZ(close(sfd)); @@ -286,7 +289,10 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag, unsigned *status) *status = 2; return (NULL); } - (void)fchown(i, mgt_param.uid, mgt_param.gid); + if (fchown(i, mgt_param.uid, mgt_param.gid) != 0) + if (geteuid() == 0) + VSB_printf(sb, "Failed to change owner on %s: %s\n", + of, strerror(errno)); AZ(close(i)); /* Build the C-compiler command line */ From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 3efca0b Relax the vmod abi check if we are not in master Message-ID: commit 3efca0bee53c39b9bcc4cf4ce62d31b0b671985c Author: Federico G. Schwindt Date: Thu Sep 4 19:14:22 2014 +0100 Relax the vmod abi check if we are not in master For master we retain the more strict check in case we forget to bump the API version as discussed on irc. Implementation idea from scoof. Fixes #1538 diff --git a/include/Makefile.am b/include/Makefile.am index 50fed78..96d9663 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -96,6 +96,7 @@ MAINTAINERCLEANFILES = vcs_version.h vcs_version.h: FORCE @if [ -d "$(top_srcdir)/.git" ]; then \ V="$$(git show -s --pretty=format:%h)" \ + B="$$(git rev-parse --abbrev-ref HEAD)" \ H="$$(head -n 1 vcs_version.h 2>/dev/null || true)"; \ if [ "/* $$V */" != "$$H" ]; then \ ( \ @@ -108,7 +109,8 @@ vcs_version.h: FORCE echo ' */' ;\ echo "/* $$V */" ;\ echo '' ;\ - echo "#define VCS_Version \"$$V\"" \ + echo "#define VCS_Version \"$$V\"" ; \ + echo "#define VCS_Branch \"$$B\"" \ ) > vcs_version.h ; \ fi \ else \ @@ -116,7 +118,8 @@ vcs_version.h: FORCE ( \ echo "/* NOGIT */" ; \ echo '/* No git commit ID available, see include/Makefile.am for explanation */' ; \ - echo '#define VCS_Version "NOGIT"' \ + echo '#define VCS_Version "NOGIT"' ; \ + echo '#define VCS_Branch "NOGIT"' \ ) > vcs_version.h ; \ fi \ fi diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 4f139de..7644621 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -34,6 +34,8 @@ #include "vcc_compile.h" +#include "vcs_version.h" + #include "vmod_abi.h" #include "vrt.h" @@ -123,6 +125,15 @@ vcc_ParseImport(struct vcc *tl) vcc_ErrWhere(tl, mod); return; } + if (strcmp(VCS_Branch, "master") == 0 && + strcmp(vmd->abi, VMOD_ABI_Version) != 0) { + VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); + VSB_printf(tl->sb, "\tFile name: %s\n", fn); + VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n", + VMOD_ABI_Version, vmd->abi); + vcc_ErrWhere(tl, mod); + return; + } if (vmd->vrt_major != VRT_MAJOR_VERSION || vmd->vrt_minor > VRT_MINOR_VERSION) { VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); @@ -154,15 +165,6 @@ vcc_ParseImport(struct vcc *tl) return; } - if (strcmp(vmd->abi, VMOD_ABI_Version) != 0) { - VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", fn); - VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n", - VMOD_ABI_Version, vmd->abi); - vcc_ErrWhere(tl, mod); - return; - } - ifp = New_IniFin(tl); VSB_printf(ifp->ini, "\tif (VRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod)); From lkarsten at varnish-software.com Mon Sep 22 14:38:25 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:25 +0200 Subject: [4.0] 56bac2c Set X-F-F early enough that it's present if we head directly to synth{} Message-ID: commit 56bac2c4bacfe5b906f6cc840397cab4879df201 Author: Poul-Henning Kamp Date: Mon Sep 8 19:52:31 2014 +0000 Set X-F-F early enough that it's present if we head directly to synth{} diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 73a36c0..56762e9 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -719,6 +719,21 @@ cnt_recv(struct worker *wrk, struct req *req) http_VSL_log(req->http); + if (req->restarts == 0) { + /* + * This really should be done earlier, but we want to capture + * it in the VSL log. + */ + if (http_GetHdr(req->http, H_X_Forwarded_For, &xff)) { + http_Unset(req->http, H_X_Forwarded_For); + http_PrintfHeader(req->http, "X-Forwarded-For: %s, %s", + xff, req->sp->client_addr_str); + } else { + http_PrintfHeader(req->http, "X-Forwarded-For: %s", + req->sp->client_addr_str); + } + } + if (req->err_code) { req->req_step = R_STP_SYNTH; return (REQ_FSM_MORE); @@ -734,25 +749,15 @@ cnt_recv(struct worker *wrk, struct req *req) req->hash_always_miss = 0; req->hash_ignore_busy = 0; req->client_identity = NULL; - if (req->restarts == 0) { - if (http_GetHdr(req->http, H_X_Forwarded_For, &xff)) { - http_Unset(req->http, H_X_Forwarded_For); - http_PrintfHeader(req->http, "X-Forwarded-For: %s, %s", xff, - req->sp->client_addr_str); - } else { - http_PrintfHeader(req->http, "X-Forwarded-For: %s", - req->sp->client_addr_str); - } - } http_CollectHdr(req->http, H_Cache_Control); VCL_recv_method(req->vcl, wrk, req, NULL, req->http->ws); /* Attempts to cache req.body may fail */ - if (req->req_body_status == REQ_BODY_FAIL) { + if (req->req_body_status == REQ_BODY_FAIL) return (REQ_FSM_DONE); - } + recv_handling = wrk->handling; /* We wash the A-E header here for the sake of VRY */ From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] 607c46d Update references to vcl_error Message-ID: commit 607c46dfabb108a62e865c44564016f3c4b730d4 Author: Federico G. Schwindt Date: Tue Sep 9 18:26:35 2014 +0100 Update references to vcl_error Use vcl_synth or vcl_backend_error where appropriate. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index ddc071c..584a6c6 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -732,7 +732,7 @@ struct req { /* Temporary accounting */ struct acct_req acct; - /* Synth content in vcl_error */ + /* Synth content in vcl_synth */ struct vsb *synth_body; }; diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst index 36d51bd..deb8e42 100644 --- a/doc/sphinx/reference/vsl.rst +++ b/doc/sphinx/reference/vsl.rst @@ -95,7 +95,7 @@ Retry Backend request is being retried. Error - Backend request failed to vcl_error. + Backend request failed to vcl_backend_error. HISTORY From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] 86ec68a Spelling Message-ID: commit 86ec68a929a5edc84666ed1e7fced55e8086dae1 Author: Federico G. Schwindt Date: Tue Sep 9 18:28:24 2014 +0100 Spelling diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index a74e6a4..1aaec4c 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -225,7 +225,7 @@ vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label) CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); VSLb_ts_busyobj(ctx->bo, label, VTIM_real()); } else if (ctx->req != NULL) { - /* Called from request vcl methdos */ + /* Called from request vcl methods */ CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); VSLb_ts_req(ctx->req, label, VTIM_real()); } From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] b84a901 Remove duplicated line Message-ID: commit b84a901bb563b846251fd3a81b9010e709f6f4c1 Author: Federico G. Schwindt Date: Tue Sep 9 18:28:39 2014 +0100 Remove duplicated line diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 92168e8..699d9a8 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -713,7 +713,6 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) bo->synth_body = NULL; if (bo->retries++ < cache_param->max_retries) return (F_STP_RETRY); - bo->synth_body = NULL; return (F_STP_FAIL); } From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] 49bbb95 Fix reference Message-ID: commit 49bbb954131c8acea7c631dbae711d39d03b0eb5 Author: Federico G. Schwindt Date: Thu Sep 11 13:24:24 2014 +0100 Fix reference Spotted by: Dridi Boukelmoune diff --git a/doc/sphinx/tutorial/now_what.rst b/doc/sphinx/tutorial/now_what.rst index 1e23e10..90e28f4 100644 --- a/doc/sphinx/tutorial/now_what.rst +++ b/doc/sphinx/tutorial/now_what.rst @@ -7,5 +7,5 @@ Now what? You've read through the tutorial. You should have Varnish up and running. You should know about the logs and you should have a rough idea of what VCL is. Next, you might want to have a look at -:ref:``users-guide-index``, where we go through the features of +:ref:`users-guide-index`, where we go through the features of Varnish in more detail. From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] c4d966d Start ignoring lower case ctags file. Message-ID: commit c4d966dbe6bf342257351a3a6741dc0ca98b7cae Author: Lasse Karstensen Date: Thu Sep 11 14:54:32 2014 +0200 Start ignoring lower case ctags file. diff --git a/.gitignore b/.gitignore index b4ae99b..460e76a 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ Makefile.in /varnishapi.pc /build-aux TAGS +tags cscope.*out # /varnishapi-uninstalled.pc From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] 1a59d8e clarify that we will not see piped connections ever again Message-ID: commit 1a59d8e7222f29628a300d6ad05b2a1c0651349c Author: Nils Goroll Date: Sun Sep 14 13:31:35 2014 +0200 clarify that we will not see piped connections ever again diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst index 5280eb4..9c9c690 100644 --- a/doc/sphinx/users-guide/vcl-built-in-subs.rst +++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst @@ -43,11 +43,12 @@ of the following keywords: vcl_pipe ~~~~~~~~ -Called upon entering pipe mode. In this mode, the request is passed -on to the backend, and any further data from either client or backend +Called upon entering pipe mode. In this mode, the request is passed on +to the backend, and any further data from both the client and backend is passed on unaltered until either end closes the connection. Basically, Varnish will degrade into a simple TCP proxy, -shuffling bytes back and forth. +shuffling bytes back and forth. For a connection in pipe mode, no +other VCL subroutine will ever get called after `vcl_pipe`. The `vcl_pipe` subroutine may terminate with calling ``return()`` with one of the following keywords: From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] 6f9eb5a Harden vtcp_sa_to_ascii Message-ID: commit 6f9eb5a126c3e367b3d6e7400852442c41d8b79a Author: Martin Blix Grydeland Date: Mon Sep 15 15:43:25 2014 +0200 Harden vtcp_sa_to_ascii Require that buffer arguments have greater than zero length to allow for zero termination. Make sure that NULL buffers are not touched. diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 26191fe..624dbb8 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -61,6 +61,8 @@ vtcp_sa_to_ascii(const void *sa, socklen_t l, char *abuf, unsigned alen, { int i; + assert(abuf == NULL || alen > 0); + assert(pbuf == NULL || plen > 0); i = getnameinfo(sa, l, abuf, alen, pbuf, plen, NI_NUMERICHOST | NI_NUMERICSERV); if (i) { @@ -69,12 +71,14 @@ vtcp_sa_to_ascii(const void *sa, socklen_t l, char *abuf, unsigned alen, * for the gai_strerror in the bufffer :-( */ printf("getnameinfo = %d %s\n", i, gai_strerror(i)); - (void)snprintf(abuf, alen, "Conversion"); - (void)snprintf(pbuf, plen, "Failed"); + if (abuf != NULL) + (void)snprintf(abuf, alen, "Conversion"); + if (pbuf != NULL) + (void)snprintf(pbuf, plen, "Failed"); return; } /* XXX dirty hack for v4-to-v6 mapped addresses */ - if (strncmp(abuf, "::ffff:", 7) == 0) { + if (abuf != NULL && strncmp(abuf, "::ffff:", 7) == 0) { for (i = 0; abuf[i + 7]; ++i) abuf[i] = abuf[i + 7]; abuf[i] = '\0'; From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] de67fd6 Delay allocating log line data until it's known to be needed in varnishtop Message-ID: commit de67fd626b9bd8ee7933d8f0feab4a0260a7c93c Author: Martin Blix Grydeland Date: Mon Sep 15 16:49:19 2014 +0200 Delay allocating log line data until it's known to be needed in varnishtop diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c index 2c195ef..d29aeb8 100644 --- a/bin/varnishtop/varnishtop.c +++ b/bin/varnishtop/varnishtop.c @@ -117,7 +117,6 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], void *priv) { struct top *tp, t; - char *rd; unsigned int u; unsigned tag; const char *b, *e, *p; @@ -148,12 +147,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], t.hash = u; t.tag = tag; t.clen = len; - t.count = 0; - rd = calloc(len+1, 1); - AN(rd); - memcpy(rd, VSL_CDATA(tr->c->rec.ptr), len); - rd[len] = '\0'; - t.rec_data = rd; + t.rec_data = (char *)VSL_CDATA(tr->c->rec.ptr); AZ(pthread_mutex_lock(&mtx)); tp = VRB_FIND(top_tree, &top_tree_head, &t); @@ -162,7 +156,6 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], tp->count += 1.0; /* Reinsert to rebalance */ VRB_INSERT(top_tree, &top_tree_head, tp); - free(rd); } else { ntop++; tp = calloc(sizeof *tp, 1); @@ -171,7 +164,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], tp->count = 1.0; tp->clen = len; tp->tag = tag; - tp->rec_data = rd; + tp->rec_data = strdup(t.rec_data); + AN(tp->rec_data); VRB_INSERT(top_tree, &top_tree_head, tp); } AZ(pthread_mutex_unlock(&mtx)); From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] 29867c1 Bail out on workspace exhaustion in VRT_IP_string Message-ID: commit 29867c1eae2be7d51e61711a7b6aecd579c1065c Author: Martin Blix Grydeland Date: Mon Sep 15 15:47:26 2014 +0200 Bail out on workspace exhaustion in VRT_IP_string Fixes: #1592 diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index edfa2a5..4ffb51a 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -304,6 +304,10 @@ VRT_IP_string(const struct vrt_ctx *ctx, VCL_IP ip) if (ip == NULL) return (NULL); len = WS_Reserve(ctx->ws, 0); + if (len == 0) { + WS_Release(ctx->ws, 0); + return (NULL); + } p = ctx->ws->f; VTCP_name(ip, p, len, NULL, 0); WS_Release(ctx->ws, strlen(p) + 1); From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] 0ce5d09 Keep two trees in varnishtop, one for order and one for uniqueness. Message-ID: commit 0ce5d09115b9ea65651a0b3045bc2c52109e7ba2 Author: Martin Blix Grydeland Date: Mon Sep 15 16:49:53 2014 +0200 Keep two trees in varnishtop, one for order and one for uniqueness. Fixes: #1591 diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c index d29aeb8..45e8205 100644 --- a/bin/varnishtop/varnishtop.c +++ b/bin/varnishtop/varnishtop.c @@ -69,19 +69,37 @@ struct top { char *rec_data; int clen; unsigned hash; - VRB_ENTRY(top) entry; + VRB_ENTRY(top) e_order; + VRB_ENTRY(top) e_key; double count; }; -static int -top_cmp(const struct top *tp, const struct top *tp2); +static VRB_HEAD(t_order, top) h_order = VRB_INITIALIZER(&h_order); +static VRB_HEAD(t_key, top) h_key = VRB_INITIALIZER(&h_key); -static VRB_HEAD(top_tree, top) top_tree_head = VRB_INITIALIZER(&top_tree_head); -VRB_PROTOTYPE(top_tree, top, entry, top_cmp); +static inline int +cmp_key(const struct top *a, const struct top *b) +{ + if (a->hash != b->hash) + return (a->hash - b->hash); + if (a->tag != b->tag) + return (a->tag - b->tag); + if (a->clen != b->clen) + return (a->clen - b->clen); + return (memcmp(a->rec_data, b->rec_data, a->clen)); +} -static unsigned ntop; +static inline int +cmp_order(const struct top *a, const struct top *b) +{ + if (a->count > b->count) + return (-1); + else if (a->count < b->count) + return (1); + return (cmp_key(a, b)); +} -/*--------------------------------------------------------------------*/ +static unsigned ntop; static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; @@ -89,28 +107,10 @@ static int f_flag = 0; static unsigned maxfieldlen = 0; -VRB_GENERATE(top_tree, top, entry, top_cmp); - -static int -top_cmp(const struct top *tp, const struct top *tp2) -{ - if (tp->count == tp2->count || tp->count == 0.0) { - if (tp->hash != tp2->hash) - return (tp->hash - tp2->hash); - if (tp->tag != tp2->tag) - return (tp->tag - tp2->tag); - if (tp->clen != tp2->clen) - return (tp->clen - tp2->clen); - else - return (memcmp(tp->rec_data, tp2->rec_data, tp->clen)); - } else { - if (tp->count > tp2->count) - return -1; - else - return 1; - } -} - +VRB_PROTOTYPE(t_order, top, e_order, cmp_order); +VRB_GENERATE(t_order, top, e_order, cmp_order); +VRB_PROTOTYPE(t_key, top, e_key, cmp_key); +VRB_GENERATE(t_key, top, e_key, cmp_key); static int __match_proto__(VSLQ_dispatch_f) accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], @@ -150,12 +150,12 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], t.rec_data = (char *)VSL_CDATA(tr->c->rec.ptr); AZ(pthread_mutex_lock(&mtx)); - tp = VRB_FIND(top_tree, &top_tree_head, &t); + tp = VRB_FIND(t_key, &h_key, &t); if (tp) { - VRB_REMOVE(top_tree, &top_tree_head, tp); + VRB_REMOVE(t_order, &h_order, tp); tp->count += 1.0; /* Reinsert to rebalance */ - VRB_INSERT(top_tree, &top_tree_head, tp); + VRB_INSERT(t_order, &h_order, tp); } else { ntop++; tp = calloc(sizeof *tp, 1); @@ -166,7 +166,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], tp->tag = tag; tp->rec_data = strdup(t.rec_data); AN(tp->rec_data); - VRB_INSERT(top_tree, &top_tree_head, tp); + VRB_INSERT(t_key, &h_key, tp); + VRB_INSERT(t_order, &h_order, tp); } AZ(pthread_mutex_unlock(&mtx)); @@ -201,8 +202,8 @@ update(int p) else AC(mvprintw(0, COLS - 1 - strlen(VUT.name), "%s", VUT.name)); AC(mvprintw(0, 0, "list length %u", ntop)); - for (tp = VRB_MIN(top_tree, &top_tree_head); tp != NULL; tp = tp2) { - tp2 = VRB_NEXT(top_tree, &top_tree_head, tp); + for (tp = VRB_MIN(t_order, &h_order); tp != NULL; tp = tp2) { + tp2 = VRB_NEXT(t_order, &h_order, tp); if (++l < LINES) { len = tp->clen; @@ -218,7 +219,8 @@ update(int p) continue; tp->count += (1.0/3.0 - tp->count) / (double)n; if (tp->count * 10 < t || l > LINES * 10) { - VRB_REMOVE(top_tree, &top_tree_head, tp); + VRB_REMOVE(t_key, &h_key, tp); + VRB_REMOVE(t_order, &h_order, tp); free(tp->rec_data); free(tp); ntop--; @@ -290,8 +292,8 @@ static void dump(void) { struct top *tp, *tp2; - for (tp = VRB_MIN(top_tree, &top_tree_head); tp != NULL; tp = tp2) { - tp2 = VRB_NEXT(top_tree, &top_tree_head, tp); + for (tp = VRB_MIN(t_order, &h_order); tp != NULL; tp = tp2) { + tp2 = VRB_NEXT(t_order, &h_order, tp); if (tp->count <= 1.0) break; printf("%9.2f %s %*.*s\n", From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] cebdf05 remove some unused declarations Message-ID: commit cebdf0524ef36d6d7c9190315f40725030138fc9 Author: Nils Goroll Date: Wed Sep 17 17:02:21 2014 +0200 remove some unused declarations Fixes #1546 diff --git a/include/vrt.h b/include/vrt.h index e49926d..1c5eea2 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -170,38 +170,6 @@ struct vrt_backend { }; /* - * A director with an unpredictable reply - */ - -struct vrt_dir_random_entry { - int host; - double weight; -}; - -struct vrt_dir_random { - const char *name; - unsigned retries; - unsigned nmember; - const struct vrt_dir_random_entry *members; -}; - -/* - * A director with dns-based selection - */ - -struct vrt_dir_dns_entry { - int host; -}; - -struct vrt_dir_dns { - const char *name; - const char *suffix; - const double ttl; - unsigned nmember; - const struct vrt_dir_dns_entry *members; -}; - -/* * other stuff. * XXX: document when bored */ From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] 6bc0250 add VCL_BYTES for vmods Message-ID: commit 6bc02500271837b8ce59c5f9fc2bf6cecb0709d3 Author: Nils Goroll Date: Fri Sep 19 11:38:37 2014 +0200 add VCL_BYTES for vmods diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 0553015..2861b40 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -49,6 +49,7 @@ ctypes = { 'BACKEND': "VCL_BACKEND", 'BLOB': "VCL_BLOB", 'BOOL': "VCL_BOOL", + 'BYTES': "VCL_BYTES", 'DURATION': "VCL_DURATION", 'ENUM': "VCL_ENUM", 'HEADER': "VCL_HEADER", From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] 158fc1c Oops, forgot this bit of previous commit Message-ID: commit 158fc1cf17866cd96d84f073fbdf943cc2ff016c Author: Poul-Henning Kamp Date: Fri Sep 19 10:48:32 2014 +0000 Oops, forgot this bit of previous commit diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 6e3c91f..9af3420 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -182,14 +182,14 @@ sp_variables = [ ), ('server.hostname', 'STRING', - ( 'client',), + ( 'all',), ( ), """ The host name of the server. """ ), ('server.identity', 'STRING', - ( 'client',), + ( 'all',), ( ), """ The identity of the server, as set by the -i parameter. If the -i parameter is not passed to varnishd, From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] c67aa08 Try again to make server.hostname and server.identity available in all VCL_methods. Message-ID: commit c67aa085f0650d4fce443e71d80d6dff10fab6ee Author: Poul-Henning Kamp Date: Fri Sep 19 16:11:24 2014 +0000 Try again to make server.hostname and server.identity available in all VCL_methods. Apologies for the previous breakage. diff --git a/bin/varnishtest/tests/v00001.vtc b/bin/varnishtest/tests/v00001.vtc index 1798ec9..822c847 100644 --- a/bin/varnishtest/tests/v00001.vtc +++ b/bin/varnishtest/tests/v00001.vtc @@ -32,6 +32,8 @@ varnish v1 -vcl+backend { set beresp.proto = "HTTP/1.2"; set beresp.reason = "For circular files"; set beresp.status = 903; + set beresp.http.y-served-by-hostname = server.hostname; + set beresp.http.y-served-by-identity = server.identity; } sub vcl_deliver { set resp.proto = "HTTP/1.2"; From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] efc0340 add std.strstr Message-ID: commit efc034099cbe1e34a092089dd58a8cb1e12f6b8b Author: Per Buer Date: Sat Sep 20 17:57:18 2014 +0200 add std.strstr Conflicts: lib/libvmod_std/vmod_std.c diff --git a/bin/varnishtest/tests/m00018.vtc b/bin/varnishtest/tests/m00018.vtc new file mode 100644 index 0000000..b51ed18 --- /dev/null +++ b/bin/varnishtest/tests/m00018.vtc @@ -0,0 +1,34 @@ +varnishtest "Test substring matching in std" + +server s1 { + rxreq + txresp + + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + import ${vmod_std}; + + sub vcl_deliver { + if (std.strstr(req.url, "foo")) { + set resp.http.sub = "found"; + } else { + set resp.http.sub = "not found"; + } + + } +} -start + +client c1 { + txreq -url "/foobar" + rxresp + expect resp.http.sub == "found" + + + txreq -url "/quux" + rxresp + expect resp.http.sub == "not found" +} -run + diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index 5bcc403..01d9274 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -195,11 +195,21 @@ $Function STRING querysort(STRING) Description Sorts the querystring for cache normalization purposes. - Example set req.url = std.querysort(req.url); +$Function STRING strstr(STRING, STRING) + +Description + Returns true if the second string is a substring of the first + string. Note that the comparison is case sensitive. You can + use the tolower function on both strings if you want case + insensitivity. +Example + if (std.strstr(req.url, req.http.x-restrict)) + + SEE ALSO ======== diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 1aaec4c..e08229f 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -230,3 +230,14 @@ vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label) VSLb_ts_req(ctx->req, label, VTIM_real()); } } + + +VCL_STRING __match_proto__(td_std_strstr) +vmod_strstr(const struct vrt_ctx *ctx, VCL_STRING mstr, VCL_STRING msubstr) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + char *match = strstr(mstr, msubstr); + + return(match); +} From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] a7927cb Nitpicks Message-ID: commit a7927cb856a0228eae512aad547d2f83368b21ef Author: Per Buer Date: Mon Sep 22 11:00:51 2014 +0200 Nitpicks Conflicts: lib/libvmod_std/vmod_std.c diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index 01d9274..b15a603 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -202,10 +202,14 @@ Example $Function STRING strstr(STRING, STRING) Description - Returns true if the second string is a substring of the first + Returns the substring if the second string is a substring of the first string. Note that the comparison is case sensitive. You can use the tolower function on both strings if you want case insensitivity. + + If there is no match a NULL pointer is returned which would + evaluate to false in an if-test. + Example if (std.strstr(req.url, req.http.x-restrict)) diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index e08229f..fd2bbbc 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -235,9 +235,10 @@ vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label) VCL_STRING __match_proto__(td_std_strstr) vmod_strstr(const struct vrt_ctx *ctx, VCL_STRING mstr, VCL_STRING msubstr) { - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - char *match = strstr(mstr, msubstr); - return(match); + if ((mstr == NULL) || (msubstr == NULL)) + return (NULL); + else + return(strstr(mstr, msubstr)); } From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] 20ada6a Use a bigger buffer for the stdout/stderr to avoid annoying linebreaks in panic messages. Message-ID: commit 20ada6ae15240f2a873d61973f837b04e48a1ae0 Author: Poul-Henning Kamp Date: Mon Sep 22 09:35:55 2014 +0000 Use a bigger buffer for the stdout/stderr to avoid annoying linebreaks in panic messages. diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 469effd..92027ba 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -325,7 +325,7 @@ static void * varnish_thread(void *priv) { struct varnish *v; - char buf[BUFSIZ]; + char buf[65536]; struct pollfd *fds, fd; int i; From lkarsten at varnish-software.com Mon Sep 22 14:38:26 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 22 Sep 2014 16:38:26 +0200 Subject: [4.0] d38ea8a Revert "Tweak size-sensitive test case for 32bit systems" Message-ID: commit d38ea8a79a29018aa10e9343633903eb87b767bf Author: Lasse Karstensen Date: Mon Sep 22 16:31:24 2014 +0200 Revert "Tweak size-sensitive test case for 32bit systems" This reverts commit f54e14f64007f3fbeda2ce155620e1c065c3ada7. diff --git a/bin/varnishtest/tests/r01140.vtc b/bin/varnishtest/tests/r01140.vtc index 098a653..11e2a72 100644 --- a/bin/varnishtest/tests/r01140.vtc +++ b/bin/varnishtest/tests/r01140.vtc @@ -4,7 +4,7 @@ server s1 { # This response should almost completely fill the storage rxreq expect req.url == /url1 - txresp -bodylen 1048408 + txresp -bodylen 1048208 # The next one should not fit in the storage, ending up in transient # with zero ttl (=shortlived) @@ -31,7 +31,7 @@ client c1 { txreq -url /url1 rxresp expect resp.status == 200 - expect resp.bodylen == 1048408 + expect resp.bodylen == 1048208 } -run delay .1 From phk at FreeBSD.org Tue Sep 23 07:48:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Sep 2014 09:48:53 +0200 Subject: [master] 95c9cf0 Use proper HTTP/1 header syntax Message-ID: commit 95c9cf065dbabc9207b0266dac57dbccbcca7155 Author: Poul-Henning Kamp Date: Tue Sep 23 07:48:01 2014 +0000 Use proper HTTP/1 header syntax diff --git a/bin/varnishtest/tests/c00022.vtc b/bin/varnishtest/tests/c00022.vtc index 39ce5dc..e2fa3e7 100644 --- a/bin/varnishtest/tests/c00022.vtc +++ b/bin/varnishtest/tests/c00022.vtc @@ -76,7 +76,7 @@ client c1 { # Ban: everything else client c1 { - txreq -req PURGESTR -hdr "ban=req.url != /foo" + txreq -req PURGESTR -hdr "ban: req.url != /foo" rxresp expect resp.status == 410 } -run From phk at FreeBSD.org Tue Sep 23 07:48:53 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Sep 2014 09:48:53 +0200 Subject: [master] 3eae7f2 Issue 400 code if header lines lack a ':' Message-ID: commit 3eae7f22a8e842f47c169cfb1f4fd335b0af65da Author: Poul-Henning Kamp Date: Tue Sep 23 07:48:29 2014 +0000 Issue 400 code if header lines lack a ':' Fixes #1598 diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c index 79288c6..49047d3 100644 --- a/bin/varnishd/cache/cache_http1_proto.c +++ b/bin/varnishd/cache/cache_http1_proto.c @@ -282,6 +282,12 @@ http1_dissect_hdrs(struct http *hp, char *p, const struct http_conn *htc) q--; *q = '\0'; + if (strchr(p, ':') == NULL) { + VSLb(hp->vsl, SLT_BogoHeader, "Header without ':' %.*s", + (int)(q - p > 20 ? 20 : q - p), p); + return (400); + } + if (hp->nhd < hp->shd) { hp->hdf[hp->nhd] = 0; hp->hd[hp->nhd].b = p; diff --git a/bin/varnishtest/tests/r01598.vtc b/bin/varnishtest/tests/r01598.vtc new file mode 100644 index 0000000..39b3ae8 --- /dev/null +++ b/bin/varnishtest/tests/r01598.vtc @@ -0,0 +1,34 @@ +varnishtest "#1598 - Missing ':' in server response headers" + +server s1 { + rxreq + txresp -hdr "ETag: \"tag\"" -hdr "foo" + accept + + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_response { + set beresp.ttl = 1s; + set beresp.grace = 0s; + set beresp.keep = 60s; + } +} -start + +varnish v1 -cliok "param.set debug +syncvsl" + +client c1 { + txreq + rxresp + expect resp.status == 503 +} -run + +delay .1 + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run From phk at FreeBSD.org Tue Sep 23 07:49:24 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Sep 2014 09:49:24 +0200 Subject: [master] 7d7b413 White-space OCD Message-ID: commit 7d7b41374b36b7b305e6deb3b8c57eb9854bac14 Author: Poul-Henning Kamp Date: Tue Sep 23 07:49:16 2014 +0000 White-space OCD diff --git a/bin/varnishtest/tests/m00018.vtc b/bin/varnishtest/tests/m00018.vtc index b51ed18..e2b501d 100644 --- a/bin/varnishtest/tests/m00018.vtc +++ b/bin/varnishtest/tests/m00018.vtc @@ -13,11 +13,11 @@ varnish v1 -vcl+backend { sub vcl_deliver { if (std.strstr(req.url, "foo")) { - set resp.http.sub = "found"; + set resp.http.sub = "found"; } else { - set resp.http.sub = "not found"; + set resp.http.sub = "not found"; } - + } } -start From phk at FreeBSD.org Tue Sep 23 08:30:57 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Sep 2014 10:30:57 +0200 Subject: [master] 6b388a2 Give busyobj a copy of the hash-key so directors can access it. Message-ID: commit 6b388a2191bb5a39041d6000d56c5433e6b60724 Author: Poul-Henning Kamp Date: Tue Sep 23 08:30:24 2014 +0000 Give busyobj a copy of the hash-key so directors can access it. (VDD14q3 item) diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 289ea28..316e9bc 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -517,6 +517,8 @@ struct busyobj { struct vsl_log vsl[1]; struct vsb *synth_body; + + uint8_t digest[DIGEST_LEN]; }; diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 97b6645..5195688 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -151,6 +151,8 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req) bo->t_first = bo->t_prev = NAN; bo->doclose = SC_NULL; + memcpy(bo->digest, req->digest, sizeof bo->digest); + return (bo); } From phk at FreeBSD.org Tue Sep 23 08:41:04 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Sep 2014 10:41:04 +0200 Subject: [master] 4b85700 Name some arguments just to show that we (already) can Message-ID: commit 4b857004e15b05dd61ff470d7f4be91b163b87b5 Author: Poul-Henning Kamp Date: Tue Sep 23 08:40:40 2014 +0000 Name some arguments just to show that we (already) can (VDD14q3 item) diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index ca1b307..74ec02f 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -39,7 +39,7 @@ $Function VOID panic(STRING_LIST) Don't. -$Function STRING author(ENUM { phk, des, kristian, mithrandir }) +$Function STRING author(ENUM { phk, des, kristian, mithrandir } person) Test function for ENUM arguments @@ -51,11 +51,11 @@ $Function VOID test_priv_vcl(PRIV_VCL) Test function for VCL private pointers -$Function BLOB str2blob(STRING) +$Function BLOB str2blob(STRING src) Turn a string into a blob -$Function STRING blob2hex(BLOB) +$Function STRING blob2hex(BLOB src) Hexdump a blob @@ -63,7 +63,7 @@ $Function BACKEND no_backend() Fails at backend selection -$Object obj(STRING) +$Object obj(STRING) Test object From phk at FreeBSD.org Tue Sep 23 08:44:24 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Sep 2014 10:44:24 +0200 Subject: [master] 1473576 Whitespace OCD Message-ID: commit 1473576fea26089fccb50a886e84cf0ac5cd39a7 Author: Poul-Henning Kamp Date: Tue Sep 23 08:44:08 2014 +0000 Whitespace OCD diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index bbfa0eb..fe8f998 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -47,9 +47,9 @@ want to emulate the previous behavior of Varnish 3.0 you can just initialize the directors in vcl_init, like this::: sub vcl_init { - new vdir = directors.round_robin(); - vdir.add_backend(backend1); - vdir.add_backend(backend2); + new vdir = directors.round_robin(); + vdir.add_backend(backend1); + vdir.add_backend(backend2); } As you can see there is nothing keeping you from manipulating the @@ -61,7 +61,7 @@ Note that directors can use other directors as backends. $Object round_robin() Description - Create a round robin director. + Create a round robin director. This director will pick backends in a round robin fashion. Example @@ -86,18 +86,18 @@ Example $Object fallback() Description - Create a fallback director. + Create a fallback director. - A fallback director will try each of the added backends in turn, - and return the first one that is healthy. + A fallback director will try each of the added backends in turn, + and return the first one that is healthy. Example - new vdir = directors.fallback(); + new vdir = directors.fallback(); $Method VOID .add_backend(BACKEND) Description - Add a backend to the director. + Add a backend to the director. Note that the order in which this is done matters for the fallback director. diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index ad832d1..61c539f 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -194,9 +194,9 @@ Example $Function STRING querysort(STRING) Description - Sorts the querystring for cache normalization purposes. + Sorts the querystring for cache normalization purposes. Example - set req.url = std.querysort(req.url); + set req.url = std.querysort(req.url); $Function VOID cache_req_body(BYTES) @@ -214,12 +214,12 @@ Description string. Note that the comparison is case sensitive. You can use the tolower function on both strings if you want case insensitivity. - - If there is no match a NULL pointer is returned which would - evaluate to false in an if-test. + + If there is no match a NULL pointer is returned which would + evaluate to false in an if-test. Example - if (std.strstr(req.url, req.http.x-restrict)) + if (std.strstr(req.url, req.http.x-restrict)) diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index a3635e9..ad0066a 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -242,12 +242,11 @@ vmod_cache_req_body(const struct vrt_ctx *ctx, VCL_BYTES size) VCL_STRING __match_proto__(td_std_strstr) vmod_strstr(const struct vrt_ctx *ctx, VCL_STRING mstr, VCL_STRING msubstr) -{ - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - if ((mstr == NULL) || (msubstr == NULL)) + if ((mstr == NULL) || (msubstr == NULL)) return (NULL); - else - return(strstr(mstr, msubstr)); + return(strstr(mstr, msubstr)); } From daghf at varnish-software.com Tue Sep 23 09:21:44 2014 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Tue, 23 Sep 2014 11:21:44 +0200 Subject: [master] 6728ac7 Get the name token out of the way prior to looking for a ', ' or ')'. Message-ID: commit 6728ac74a058e8accb948fdcefde21c45f2b6d77 Author: Dag Haavi Finstad Date: Tue Sep 23 11:21:42 2014 +0200 Get the name token out of the way prior to looking for a ',' or ')'. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 2861b40..96fa347 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -647,8 +647,8 @@ def parse_func(tl, rt_type=None, pobj=None): t = tl.get_token() if is_c_name(t.str): al[-1].nam = t.str - t = None - elif t.str == ",": + t = tl.get_token() + if t.str == ",": t = None elif t.str == ")": break From phk at FreeBSD.org Tue Sep 23 09:47:59 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Sep 2014 11:47:59 +0200 Subject: [master] 97014d3 Eliminate a surplus argument Message-ID: commit 97014d355a59bd34558cb3322075d40a49873276 Author: Poul-Henning Kamp Date: Tue Sep 23 09:47:44 2014 +0000 Eliminate a surplus argument diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 316e9bc..f12eb0c 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -740,7 +740,7 @@ void VBO_waitstate(struct busyobj *bo, enum busyobj_state_e want); /* cache_http1_fetch.c [V1F] */ -int V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req); +int V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo); void V1F_Setup_Fetch(struct vfp_ctx *vfc, struct http_conn *htc); /* cache_http1_fsm.c [HTTP1] */ diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 8de0923..c28d1cb 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -265,7 +265,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) assert(bo->state <= BOS_REQ_DONE); - i = V1F_fetch_hdr(wrk, bo, bo->req); + i = V1F_fetch_hdr(wrk, bo); /* * If we recycle a backend connection, there is a finite chance * that the backend closed it before we get a request to it. @@ -274,7 +274,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) if (i == 1) { VSLb_ts_busyobj(bo, "Beresp", W_TIM_real(wrk)); VSC_C_main->backend_retry++; - i = V1F_fetch_hdr(wrk, bo, bo->req); + i = V1F_fetch_hdr(wrk, bo); } now = W_TIM_real(wrk); VSLb_ts_busyobj(bo, "Beresp", now); diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index 3874fb5..a4c4ec5 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -73,7 +73,7 @@ vbf_iter_req_body(struct req *req, void *priv, void *ptr, size_t l) */ int -V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req) +V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo) { struct vbc *vc; struct http *hp; @@ -85,8 +85,8 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req) int do_chunked = 0; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_ORNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + CHECK_OBJ_ORNULL(bo->req, REQ_MAGIC); htc = bo->htc; if (bo->director == NULL) { @@ -114,7 +114,8 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req) if (!http_GetHdr(bo->bereq, H_Host, NULL)) VDI_AddHostHeader(bo->bereq, vc); - if (req != NULL && req->req_body_status == REQ_BODY_WITHOUT_LEN) { + if (bo->req != NULL && + bo->req->req_body_status == REQ_BODY_WITHOUT_LEN) { http_PrintfHeader(hp, "Transfer-Encoding: chunked"); do_chunked = 1; } @@ -126,18 +127,18 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req) /* Deal with any message-body the request might (still) have */ i = 0; - if (req != NULL) { + if (bo->req != NULL) { if (do_chunked) WRW_Chunked(wrk); - i = VRB_Iterate(req, vbf_iter_req_body, wrk); + i = VRB_Iterate(bo->req, vbf_iter_req_body, wrk); - if (req->req_body_status == REQ_BODY_TAKEN) { + if (bo->req->req_body_status == REQ_BODY_TAKEN) { retry = -1; - } else if (req->req_body_status == REQ_BODY_FAIL) { + } else if (bo->req->req_body_status == REQ_BODY_FAIL) { VSLb(bo->vsl, SLT_FetchError, "req.body read error: %d (%s)", errno, strerror(errno)); - req->doclose = SC_RX_BODY; + bo->req->doclose = SC_RX_BODY; retry = -1; } if (do_chunked) From phk at FreeBSD.org Tue Sep 23 10:36:57 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Sep 2014 12:36:57 +0200 Subject: [master] 016531b Shuffling the backend code around a bit in preparation for multi-protocol backends. Message-ID: commit 016531bbd66dcbb49630e11b0c1e7b7e143de531 Author: Poul-Henning Kamp Date: Tue Sep 23 10:36:28 2014 +0000 Shuffling the backend code around a bit in preparation for multi-protocol backends. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index f12eb0c..a3b0cd7 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -695,6 +695,7 @@ void VBE_UseHealth(const struct director *vdi); void VBE_DiscardHealth(const struct director *vdi); +int VDI_GetHdr(struct worker *wrk, struct busyobj *bo); struct vbc *VDI_GetFd(struct busyobj *); int VDI_Healthy(const struct director *); void VDI_CloseFd(struct vbc **vbp, const struct acct_bereq *); diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 9854b41..3dc8c6d 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -42,6 +42,7 @@ #include "cache_backend.h" #include "vrt.h" #include "vtcp.h" +#include "vtim.h" static struct mempool *vbcpool; @@ -356,6 +357,35 @@ VBE_DiscardHealth(const struct director *vdi) } /*-------------------------------------------------------------------- + */ + +int +VDI_GetHdr(struct worker *wrk, struct busyobj *bo) +{ + int i; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + + if (bo->director == NULL) { + VSLb(bo->vsl, SLT_FetchError, "No backend"); + return (-1); + } + + i = V1F_fetch_hdr(wrk, bo); + /* + * If we recycle a backend connection, there is a finite chance + * that the backend closed it before we get a request to it. + * Do a single retry in that case. + */ + if (i == 1) { + VSC_C_main->backend_retry++; + i = VDI_GetHdr(wrk, bo); + } + return (i); +} + +/*-------------------------------------------------------------------- * */ @@ -378,7 +408,7 @@ vdi_simple_getfd(const struct director *d, struct busyobj *bo) return (vc); } -static unsigned +static unsigned __match_proto__(vdi_healthy_f) vdi_simple_healthy(const struct director *d, double *changed) { struct vdi_simple *vs; @@ -391,6 +421,17 @@ vdi_simple_healthy(const struct director *d, double *changed) return (VBE_Healthy(be, changed)); } +static int __match_proto__(vdi_gethdrs_f) +vdi_simple_gethdrs(const struct director *d, struct worker *wrk, + struct busyobj *bo) +{ + + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + return (-1); +} + /*--------------------------------------------------------------------*/ void @@ -428,6 +469,7 @@ VRT_init_dir(struct cli *cli, struct director **bp, int idx, const void *priv) REPLACE(vs->dir.vcl_name, t->vcl_name); vs->dir.getfd = vdi_simple_getfd; vs->dir.healthy = vdi_simple_healthy; + vs->dir.gethdrs = vdi_simple_gethdrs; vs->vrt = t; diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index c9ce112..fe4bb15 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -76,7 +76,9 @@ struct vrt_backend_probe; */ typedef struct vbc *vdi_getfd_f(const struct director *, struct busyobj *); -typedef unsigned vdi_healthy(const struct director *, double *changed); +typedef unsigned vdi_healthy_f(const struct director *, double *changed); +typedef int vdi_gethdrs_f(const struct director *, struct worker *, + struct busyobj *); struct director { unsigned magic; @@ -84,7 +86,8 @@ struct director { const char *name; char *vcl_name; vdi_getfd_f *getfd; - vdi_healthy *healthy; + vdi_healthy_f *healthy; + vdi_gethdrs_f *gethdrs; void *priv; }; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index c28d1cb..38e72f0 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -265,17 +265,8 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) assert(bo->state <= BOS_REQ_DONE); - i = V1F_fetch_hdr(wrk, bo); - /* - * If we recycle a backend connection, there is a finite chance - * that the backend closed it before we get a request to it. - * Do a single retry in that case. - */ - if (i == 1) { - VSLb_ts_busyobj(bo, "Beresp", W_TIM_real(wrk)); - VSC_C_main->backend_retry++; - i = V1F_fetch_hdr(wrk, bo); - } + i = VDI_GetHdr(wrk, bo); + now = W_TIM_real(wrk); VSLb_ts_busyobj(bo, "Beresp", now); diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index a4c4ec5..7e38a34 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -86,15 +86,10 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(bo->director, DIRECTOR_MAGIC); CHECK_OBJ_ORNULL(bo->req, REQ_MAGIC); htc = bo->htc; - if (bo->director == NULL) { - VSLb(bo->vsl, SLT_FetchError, "No backend"); - return (-1); - } - AN(bo->director); - hp = bo->bereq; bo->vbc = VDI_GetFd(bo); diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index aae7a14..75ea080 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -51,7 +51,7 @@ vdir_expand(struct vdir *vd, unsigned n) } void -vdir_new(struct vdir **vdp, const char *vcl_name, vdi_healthy *healthy, +vdir_new(struct vdir **vdp, const char *vcl_name, vdi_healthy_f *healthy, vdi_getfd_f *getfd, void *priv) { struct vdir *vd; diff --git a/lib/libvmod_directors/vdir.h b/lib/libvmod_directors/vdir.h index d237b2d..583d6e1 100644 --- a/lib/libvmod_directors/vdir.h +++ b/lib/libvmod_directors/vdir.h @@ -41,7 +41,7 @@ struct vdir { struct vbitmap *vbm; }; -void vdir_new(struct vdir **vdp, const char *vcl_name, vdi_healthy *healthy, +void vdir_new(struct vdir **vdp, const char *vcl_name, vdi_healthy_f *healthy, vdi_getfd_f *getfd, void *priv); void vdir_delete(struct vdir **vdp); void vdir_lock(struct vdir *vd); From martin at varnish-software.com Tue Sep 23 11:33:24 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 23 Sep 2014 13:33:24 +0200 Subject: [master] 44aca24 Round up vary length Message-ID: commit 44aca24dc521e6f952b19ab5a548cbc71d15b418 Author: Martin Blix Grydeland Date: Mon Sep 22 14:02:55 2014 +0200 Round up vary length Round up vary length to make sure that the object workspace can take the rounded up Vary string without overflowing. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 38e72f0..a4aa167 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -106,7 +106,7 @@ vbf_beresp2obj(struct busyobj *bo) if (varyl > 0) { AN(vary); assert(varyl == VSB_len(vary)); - l += varyl; + l += PRNDUP(varyl); } else if (varyl < 0) { /* * Vary parse error From phk at FreeBSD.org Tue Sep 23 11:41:32 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Sep 2014 13:41:32 +0200 Subject: [master] 9618aa1 Use dir->gethdrs() as far down as we can, then switch to getfd(). Message-ID: commit 9618aa15add263e01887a78ffd5c85df3864178e Author: Poul-Henning Kamp Date: Tue Sep 23 11:41:03 2014 +0000 Use dir->gethdrs() as far down as we can, then switch to getfd(). diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 3dc8c6d..fe25fbe 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -42,7 +42,6 @@ #include "cache_backend.h" #include "vrt.h" #include "vtcp.h" -#include "vtim.h" static struct mempool *vbcpool; @@ -357,35 +356,6 @@ VBE_DiscardHealth(const struct director *vdi) } /*-------------------------------------------------------------------- - */ - -int -VDI_GetHdr(struct worker *wrk, struct busyobj *bo) -{ - int i; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - - if (bo->director == NULL) { - VSLb(bo->vsl, SLT_FetchError, "No backend"); - return (-1); - } - - i = V1F_fetch_hdr(wrk, bo); - /* - * If we recycle a backend connection, there is a finite chance - * that the backend closed it before we get a request to it. - * Do a single retry in that case. - */ - if (i == 1) { - VSC_C_main->backend_retry++; - i = VDI_GetHdr(wrk, bo); - } - return (i); -} - -/*-------------------------------------------------------------------- * */ @@ -425,11 +395,57 @@ static int __match_proto__(vdi_gethdrs_f) vdi_simple_gethdrs(const struct director *d, struct worker *wrk, struct busyobj *bo) { + int i; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - return (-1); + + bo->vbc = VDI_GetFd(bo); + if (bo->vbc == NULL) { + VSLb(bo->vsl, SLT_FetchError, "no backend connection"); + return (-1); + } + + i = V1F_fetch_hdr(wrk, bo); + /* + * If we recycle a backend connection, there is a finite chance + * that the backend closed it before we get a request to it. + * Do a single retry in that case. + */ + if (i == 1) { + VSC_C_main->backend_retry++; + bo->vbc = VDI_GetFd(bo); + if (bo->vbc == NULL) { + VSLb(bo->vsl, SLT_FetchError, "no backend connection"); + return (-1); + } + i = V1F_fetch_hdr(wrk, bo); + } + return (i); +} + +/*-------------------------------------------------------------------- + */ + +int +VDI_GetHdr(struct worker *wrk, struct busyobj *bo) +{ + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + + if (bo->director == NULL) { + VSLb(bo->vsl, SLT_FetchError, "No backend"); + return (-1); + } + + CHECK_OBJ_NOTNULL(bo->director, DIRECTOR_MAGIC); + + if (bo->director->gethdrs != NULL) + return (bo->director->gethdrs(bo->director, wrk, bo)); + else + return (vdi_simple_gethdrs(bo->director, wrk, bo)); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index 7e38a34..bb58f1b 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -87,16 +87,11 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->director, DIRECTOR_MAGIC); + CHECK_OBJ_NOTNULL(bo->vbc, VBC_MAGIC); CHECK_OBJ_ORNULL(bo->req, REQ_MAGIC); - htc = bo->htc; + htc = bo->htc; hp = bo->bereq; - - bo->vbc = VDI_GetFd(bo); - if (bo->vbc == NULL) { - VSLb(bo->vsl, SLT_FetchError, "no backend connection"); - return (-1); - } vc = bo->vbc; if (vc->recycled) retry = 1; From phk at FreeBSD.org Tue Sep 23 12:30:30 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Sep 2014 14:30:30 +0200 Subject: [master] 18be02e Add a new director method "->resolve" which returns another backend to use. Message-ID: commit 18be02e3419c665b9d5b0a455abb56b66a1cfd10 Author: Poul-Henning Kamp Date: Tue Sep 23 12:28:27 2014 +0000 Add a new director method "->resolve" which returns another backend to use. This is called recursively until we get to the bottom of the pile. Doing it this way means that "pick one" directors dont need to know about other methods on directors and can concentrate on just picking. It also prevents us from eating up a lot of stack. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index fe25fbe..ccd4cf4 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -441,11 +441,16 @@ VDI_GetHdr(struct worker *wrk, struct busyobj *bo) } CHECK_OBJ_NOTNULL(bo->director, DIRECTOR_MAGIC); + while (bo->director != NULL && bo->director->resolve != NULL) + bo->director = bo->director->resolve(bo->director, wrk, bo); - if (bo->director->gethdrs != NULL) - return (bo->director->gethdrs(bo->director, wrk, bo)); - else - return (vdi_simple_gethdrs(bo->director, wrk, bo)); + if (bo->director == NULL) { + VSLb(bo->vsl, SLT_FetchError, "Backend selection failed"); + return (-1); + } + + AN(bo->director->gethdrs); + return (bo->director->gethdrs(bo->director, wrk, bo)); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index fe4bb15..5fa79bd 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -77,6 +77,8 @@ struct vrt_backend_probe; typedef struct vbc *vdi_getfd_f(const struct director *, struct busyobj *); typedef unsigned vdi_healthy_f(const struct director *, double *changed); +typedef struct director *vdi_resolve_f(const struct director *, + struct worker *, struct busyobj *); typedef int vdi_gethdrs_f(const struct director *, struct worker *, struct busyobj *); @@ -87,6 +89,7 @@ struct director { char *vcl_name; vdi_getfd_f *getfd; vdi_healthy_f *healthy; + vdi_resolve_f *resolve; vdi_gethdrs_f *gethdrs; void *priv; }; diff --git a/lib/libvmod_directors/fall_back.c b/lib/libvmod_directors/fall_back.c index edec2af..272864e 100644 --- a/lib/libvmod_directors/fall_back.c +++ b/lib/libvmod_directors/fall_back.c @@ -45,7 +45,7 @@ struct vmod_directors_fallback { }; static unsigned __match_proto__(vdi_healthy) -vmod_rr_healthy(const struct director *dir, double *changed) +vmod_fallback_healthy(const struct director *dir, double *changed) { struct vmod_directors_fallback *rr; @@ -53,13 +53,17 @@ vmod_rr_healthy(const struct director *dir, double *changed) return (vdir_any_healthy(rr->vd, changed)); } -static struct vbc * __match_proto__(vdi_getfd_f) -vmod_rr_getfd(const struct director *dir, struct busyobj *bo) +static struct director * __match_proto__(vdi_resolve_f) +vmod_fallback_resolve(const struct director *dir, struct worker *wrk, + struct busyobj *bo) { struct vmod_directors_fallback *rr; unsigned u; VCL_BACKEND be = NULL; + CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_FALLBACK_MAGIC); vdir_lock(rr->vd); for (u = 0; u < rr->vd->n_backend; u++) { @@ -69,9 +73,9 @@ vmod_rr_getfd(const struct director *dir, struct busyobj *bo) break; } vdir_unlock(rr->vd); - if (u == rr->vd->n_backend || be == NULL) - return (NULL); - return (be->getfd(be, bo)); + if (u == rr->vd->n_backend) + be = NULL; + return (be); } VCL_VOID __match_proto__() @@ -86,7 +90,8 @@ vmod_fallback__init(const struct vrt_ctx *ctx, ALLOC_OBJ(rr, VMOD_DIRECTORS_FALLBACK_MAGIC); AN(rr); *rrp = rr; - vdir_new(&rr->vd, vcl_name, vmod_rr_healthy, vmod_rr_getfd, rr); + vdir_new(&rr->vd, vcl_name, vmod_fallback_healthy, + vmod_fallback_resolve, rr); } VCL_VOID __match_proto__() diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c index 8ae36a7..a7e26d8 100644 --- a/lib/libvmod_directors/random.c +++ b/lib/libvmod_directors/random.c @@ -50,7 +50,7 @@ struct vmod_directors_random { }; static unsigned __match_proto__(vdi_healthy) -vmod_rr_healthy(const struct director *dir, double *changed) +vmod_random_healthy(const struct director *dir, double *changed) { struct vmod_directors_random *rr; @@ -58,20 +58,22 @@ vmod_rr_healthy(const struct director *dir, double *changed) return (vdir_any_healthy(rr->vd, changed)); } -static struct vbc * __match_proto__(vdi_getfd_f) -vmod_rr_getfd(const struct director *dir, struct busyobj *bo) +static struct director * __match_proto__(vdi_resolve_f) +vmod_random_resolve(const struct director *dir, struct worker *wrk, + struct busyobj *bo) { struct vmod_directors_random *rr; VCL_BACKEND be; double r; + CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_RANDOM_MAGIC); r = scalbn(random(), -31); assert(r >= 0 && r < 1.0); be = vdir_pick_be(rr->vd, r, rr->n_backend); - if (be == NULL) - return (NULL); - return (be->getfd(be, bo)); + return (be); } VCL_VOID __match_proto__() @@ -88,7 +90,8 @@ vmod_random__init(const struct vrt_ctx *ctx, struct vmod_directors_random **rrp, rr->vbm = vbit_init(8); AN(rr->vbm); *rrp = rr; - vdir_new(&rr->vd, vcl_name, vmod_rr_healthy, vmod_rr_getfd, rr); + vdir_new(&rr->vd, vcl_name, vmod_random_healthy, vmod_random_resolve, + rr); } VCL_VOID __match_proto__() diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c index 629bbbf..581eef3 100644 --- a/lib/libvmod_directors/round_robin.c +++ b/lib/libvmod_directors/round_robin.c @@ -54,13 +54,17 @@ vmod_rr_healthy(const struct director *dir, double *changed) return (vdir_any_healthy(rr->vd, changed)); } -static struct vbc * __match_proto__(vdi_getfd_f) -vmod_rr_getfd(const struct director *dir, struct busyobj *bo) +static struct director * __match_proto__(vdi_resolve_f) +vmod_rr_resolve(const struct director *dir, struct worker *wrk, + struct busyobj *bo) { struct vmod_directors_round_robin *rr; unsigned u; VCL_BACKEND be = NULL; + CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC); vdir_lock(rr->vd); for (u = 0; u < rr->vd->n_backend; u++) { @@ -72,9 +76,9 @@ vmod_rr_getfd(const struct director *dir, struct busyobj *bo) break; } vdir_unlock(rr->vd); - if (u == rr->vd->n_backend || be == NULL) - return (NULL); - return (be->getfd(be, bo)); + if (u == rr->vd->n_backend) + be = NULL; + return (be); } VCL_VOID __match_proto__() @@ -89,7 +93,7 @@ vmod_round_robin__init(const struct vrt_ctx *ctx, ALLOC_OBJ(rr, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC); AN(rr); *rrp = rr; - vdir_new(&rr->vd, vcl_name, vmod_rr_healthy, vmod_rr_getfd, rr); + vdir_new(&rr->vd, vcl_name, vmod_rr_healthy, vmod_rr_resolve, rr); } VCL_VOID __match_proto__() diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index 75ea080..52cf4c4 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -52,7 +52,7 @@ vdir_expand(struct vdir *vd, unsigned n) void vdir_new(struct vdir **vdp, const char *vcl_name, vdi_healthy_f *healthy, - vdi_getfd_f *getfd, void *priv) + vdi_resolve_f *resolve, void *priv) { struct vdir *vd; @@ -69,7 +69,7 @@ vdir_new(struct vdir **vdp, const char *vcl_name, vdi_healthy_f *healthy, REPLACE(vd->dir->vcl_name, vcl_name); vd->dir->priv = priv; vd->dir->healthy = healthy; - vd->dir->getfd = getfd; + vd->dir->resolve = resolve; vd->vbm = vbit_init(8); AN(vd->vbm); } diff --git a/lib/libvmod_directors/vdir.h b/lib/libvmod_directors/vdir.h index 583d6e1..5afb754 100644 --- a/lib/libvmod_directors/vdir.h +++ b/lib/libvmod_directors/vdir.h @@ -42,7 +42,7 @@ struct vdir { }; void vdir_new(struct vdir **vdp, const char *vcl_name, vdi_healthy_f *healthy, - vdi_getfd_f *getfd, void *priv); + vdi_resolve_f *resolve, void *priv); void vdir_delete(struct vdir **vdp); void vdir_lock(struct vdir *vd); void vdir_unlock(struct vdir *vd); From lkarsten at varnish-software.com Tue Sep 23 13:09:29 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Tue, 23 Sep 2014 15:09:29 +0200 Subject: [4.0] d6daf2b Don't mix const and non-const pointers. This should make jenkins happy again. Message-ID: commit d6daf2bf2322054c09a8bd9930666595ec9098c0 Author: Martin Blix Grydeland Date: Tue Sep 16 10:48:52 2014 +0200 Don't mix const and non-const pointers. This should make jenkins happy again. diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c index 45e8205..15835cb 100644 --- a/bin/varnishtop/varnishtop.c +++ b/bin/varnishtop/varnishtop.c @@ -66,7 +66,8 @@ static int end_of_file = 0; struct top { uint8_t tag; - char *rec_data; + const char *rec_data; + char *rec_buf; int clen; unsigned hash; VRB_ENTRY(top) e_order; @@ -147,7 +148,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], t.hash = u; t.tag = tag; t.clen = len; - t.rec_data = (char *)VSL_CDATA(tr->c->rec.ptr); + t.rec_data = VSL_CDATA(tr->c->rec.ptr); AZ(pthread_mutex_lock(&mtx)); tp = VRB_FIND(t_key, &h_key, &t); @@ -164,7 +165,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], tp->count = 1.0; tp->clen = len; tp->tag = tag; - tp->rec_data = strdup(t.rec_data); + tp->rec_buf = strdup(t.rec_data); + tp->rec_data = tp->rec_buf; AN(tp->rec_data); VRB_INSERT(t_key, &h_key, tp); VRB_INSERT(t_order, &h_order, tp); @@ -221,7 +223,7 @@ update(int p) if (tp->count * 10 < t || l > LINES * 10) { VRB_REMOVE(t_key, &h_key, tp); VRB_REMOVE(t_order, &h_order, tp); - free(tp->rec_data); + free(tp->rec_buf); free(tp); ntop--; } From lkarsten at varnish-software.com Tue Sep 23 13:09:29 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Tue, 23 Sep 2014 15:09:29 +0200 Subject: [4.0] dbaba70 If workspace_thread is not be big enough to hold all the objcore pointers that need to be purged we iterate until done. Message-ID: commit dbaba7024a9775175b588260287bf6588ed6da9e Author: Poul-Henning Kamp Date: Mon Jul 28 10:00:15 2014 +0000 If workspace_thread is not be big enough to hold all the objcore pointers that need to be purged we iterate until done. Fixes #1551 Conflicts: bin/varnishd/cache/cache_hash.c diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 76ad5c3..0a94a3d 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -559,49 +559,61 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace, double keep) { struct objcore *oc, **ocp; - unsigned spc, nobj, n; + unsigned spc, ospc, nobj, n; struct object *o; + int more = 0; + double now; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); - spc = WS_Reserve(wrk->aws, 0); - ocp = (void*)wrk->aws->f; - Lck_Lock(&oh->mtx); - assert(oh->refcnt > 0); - nobj = 0; - now = VTIM_real(); - VTAILQ_FOREACH(oc, &oh->objcs, list) { - CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - assert(oc->objhead == oh); - if (oc->flags & OC_F_BUSY) { - /* - * We cannot purge busy objects here, because their - * owners have special rights to them, and may nuke - * them without concern for the refcount, which by - * definition always must be one, so they don't check. - */ - continue; + ospc = WS_Reserve(wrk->aws, 0); + assert(ospc >= sizeof *ocp); + do { + more = 0; + spc = ospc; + nobj = 0; + ocp = (void*)wrk->aws->f; + Lck_Lock(&oh->mtx); + assert(oh->refcnt > 0); + now = VTIM_real(); + VTAILQ_FOREACH(oc, &oh->objcs, list) { + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + assert(oc->objhead == oh); + if (oc->flags & OC_F_BUSY) { + /* + * We cannot purge busy objects here, because + * their owners have special rights to them, + * and may nuke them without concern for the + * refcount, which by definition always must + * be one, so they don't check. + */ + continue; + } + if (oc->exp_flags & OC_EF_DYING) + continue; + if (spc < sizeof *ocp) { + /* Iterate if aws is not big enough */ + more = 1; + break; + } + oc->refcnt++; + spc -= sizeof *ocp; + ocp[nobj++] = oc; } - if (oc->exp_flags & OC_EF_DYING) - continue; - xxxassert(spc >= sizeof *ocp); - oc->refcnt++; - spc -= sizeof *ocp; - ocp[nobj++] = oc; - } - Lck_Unlock(&oh->mtx); + Lck_Unlock(&oh->mtx); - for (n = 0; n < nobj; n++) { - oc = ocp[n]; - CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - o = oc_getobj(&wrk->stats, oc); - if (o == NULL) - continue; - CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); - EXP_Rearm(o, now, ttl, grace, keep); - (void)HSH_DerefObj(&wrk->stats, &o); - } + for (n = 0; n < nobj; n++) { + oc = ocp[n]; + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + o = oc_getobj(&wrk->stats, oc); + if (o == NULL) + continue; + CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); + EXP_Rearm(o, now, ttl, grace, keep); + (void)HSH_DerefObj(&wrk->stats, &o); + } + } while (more); WS_Release(wrk->aws, 0); Pool_PurgeStat(nobj); } From lkarsten at varnish-software.com Tue Sep 23 13:09:29 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Tue, 23 Sep 2014 15:09:29 +0200 Subject: [4.0] 752a95f Ensure that we never call a VDP with a zero length unless we are done. Message-ID: commit 752a95f1fc5bb60cf7e592dac8f34489623dad7a Author: Poul-Henning Kamp Date: Tue Jul 29 18:44:17 2014 +0000 Ensure that we never call a VDP with a zero length unless we are done. Fixes #1561 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 584a6c6..414ec87 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -883,6 +883,7 @@ VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + assert(act > VDP_NULL || len > 0); /* Call the present layer, while pointing to the next layer down */ i = req->vdp_nxt--; assert(i >= 0 && i < N_VDPS); diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index a92d29f..5d70534 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -76,9 +76,12 @@ ObjIter(struct objiter *oi, void **p, ssize_t *l) oi->st = VTAILQ_FIRST(&oi->obj->store); else oi->st = VTAILQ_NEXT(oi->st, list); + while(oi->st != NULL && oi->st->len == 0) + oi->st = VTAILQ_NEXT(oi->st, list); if (oi->st != NULL) { *p = oi->st->ptr; *l = oi->st->len; + assert(*l > 0); return (OIS_DATA); } return (OIS_DONE); From lkarsten at varnish-software.com Tue Sep 23 13:09:29 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Tue, 23 Sep 2014 15:09:29 +0200 Subject: [4.0] e388600 Make a dedicated cleanup function for req->vary_? to match the dedicated setup function we have. Message-ID: commit e38860038b92e4e2e0b27882fd9af5ab487665a3 Author: Poul-Henning Kamp Date: Tue Jul 29 19:09:09 2014 +0000 Make a dedicated cleanup function for req->vary_? to match the dedicated setup function we have. Fixes #1553 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 414ec87..04debc8 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -1187,6 +1187,7 @@ int VRY_Create(struct busyobj *bo, struct vsb **psb); int VRY_Match(struct req *, const uint8_t *vary); unsigned VRY_Validate(const uint8_t *vary); void VRY_Prep(struct req *); +void VRY_Clear(struct req *); enum vry_finish_flag { KEEP, DISCARD }; void VRY_Finish(struct req *req, enum vry_finish_flag); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 56762e9..073e9fb 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -472,8 +472,7 @@ cnt_lookup(struct worker *wrk, struct req *req) if (boc != NULL) { (void)HSH_DerefObjCore(&wrk->stats, &boc); - free(req->vary_b); - req->vary_b = NULL; + VRY_Clear(req); } return (REQ_FSM_MORE); @@ -527,7 +526,7 @@ cnt_miss(struct worker *wrk, struct req *req) default: WRONG("Illegal return from vcl_miss{}"); } - free(req->vary_b); + VRY_Clear(req); if (o != NULL) (void)HSH_DerefObj(&wrk->stats, &o); AZ(HSH_DerefObjCore(&wrk->stats, &req->objcore)); diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c index 4baa65a..e4a7628 100644 --- a/bin/varnishd/cache/cache_vary.c +++ b/bin/varnishd/cache/cache_vary.c @@ -238,6 +238,18 @@ VRY_Prep(struct req *req) req->vary_b[2] = '\0'; } +void +VRY_Clear(struct req *req) +{ + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + if (req->vary_b != NULL) + free(req->vary_b); + req->vary_b = NULL; + AZ(req->vary_e); + AZ(req->vary_l); +} + /********************************************************************** * Finish predictive vary processing */ From lkarsten at varnish-software.com Tue Sep 23 13:09:30 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Tue, 23 Sep 2014 15:09:30 +0200 Subject: [4.0] 074d3c8 Prepare for 4.0.2-rc1 release. Message-ID: commit 074d3c8f80d94a51e5f62462e424a8bbd2354f46 Author: Lasse Karstensen Date: Tue Sep 23 15:09:08 2014 +0200 Prepare for 4.0.2-rc1 release. diff --git a/configure.ac b/configure.ac index d4f5c5b..4fab5be 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS Copyright (c) 2006-2014 Varnish Software AS]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [4.0.1], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [4.0.2-rc1], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/redhat/varnish.spec b/redhat/varnish.spec index ad2a914..1a66a1e 100644 --- a/redhat/varnish.spec +++ b/redhat/varnish.spec @@ -1,10 +1,10 @@ -%define XXXv_rc rc1 +%define v_rc rc1 %define vd_rc %{?v_rc:-%{?v_rc}} %define _use_internal_dependency_generator 0 %define __find_provides %{_builddir}/varnish-%{version}%{?v_rc:-%{?v_rc}}/redhat/find-provides Summary: High-performance HTTP accelerator Name: varnish -Version: 4.0.1 +Version: 4.0.2 #Release: 0.20140328%{?v_rc}%{?dist} Release: 1%{?v_rc}%{?dist} License: BSD From phk at FreeBSD.org Tue Sep 23 15:32:18 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Sep 2014 17:32:18 +0200 Subject: [master] 9ba9a8b Introduce beresp.backend. Message-ID: commit 9ba9a8bb1b40d0007e903ccfd948178fdf467176 Author: Poul-Henning Kamp Date: Tue Sep 23 15:29:58 2014 +0000 Introduce beresp.backend. Now bereq.backend is the backend or director we chose, whereas beresp.backend is the actual backend, after directors have done their picking and choosing, which sent us the reply. Constify the VCL_BACKEND datatype. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index a3b0cd7..92a4f71 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -511,7 +511,8 @@ struct busyobj { struct acct_bereq acct; const char *storage_hint; - struct director *director; + const struct director *director_req; + const struct director *director_resp; struct VCL_conf *vcl; struct vsl_log vsl[1]; @@ -564,7 +565,7 @@ struct req { uint16_t err_code; const char *err_reason; - struct director *director_hint; + const struct director *director_hint; struct VCL_conf *vcl; char *ws_req; /* WS above request data */ @@ -696,7 +697,8 @@ void VBE_DiscardHealth(const struct director *vdi); int VDI_GetHdr(struct worker *wrk, struct busyobj *bo); -struct vbc *VDI_GetFd(struct busyobj *); +struct vbc *VDI_GetFd(const struct director *d, struct worker *wrk, + struct busyobj *); int VDI_Healthy(const struct director *); void VDI_CloseFd(struct vbc **vbp, const struct acct_bereq *); void VDI_RecycleFd(struct vbc **vbp, const struct acct_bereq *); diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index ccd4cf4..4c00753 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -273,7 +273,7 @@ vbe_GetVbe(struct busyobj *bo, struct vdi_simple *vs) /* XXX locking of stats */ VSC_C_main->backend_reuse += 1; VSLb(bo->vsl, SLT_Backend, "%d %s %s", - vc->fd, bo->director->vcl_name, + vc->fd, bo->director_resp->vcl_name, bp->display_name); vc->vdis = vs; vc->recycled = 1; @@ -312,7 +312,7 @@ vbe_GetVbe(struct busyobj *bo, struct vdi_simple *vs) vc->backend = bp; VSC_C_main->backend_conn++; VSLb(bo->vsl, SLT_Backend, "%d %s %s", - vc->fd, bo->director->vcl_name, bp->display_name); + vc->fd, bo->director_resp->vcl_name, bp->display_name); vc->vdis = vs; return (vc); } @@ -401,7 +401,7 @@ vdi_simple_gethdrs(const struct director *d, struct worker *wrk, CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - bo->vbc = VDI_GetFd(bo); + bo->vbc = VDI_GetFd(d, wrk, bo); if (bo->vbc == NULL) { VSLb(bo->vsl, SLT_FetchError, "no backend connection"); return (-1); @@ -415,7 +415,7 @@ vdi_simple_gethdrs(const struct director *d, struct worker *wrk, */ if (i == 1) { VSC_C_main->backend_retry++; - bo->vbc = VDI_GetFd(bo); + bo->vbc = VDI_GetFd(d, wrk, bo); if (bo->vbc == NULL) { VSLb(bo->vsl, SLT_FetchError, "no backend connection"); return (-1); @@ -425,34 +425,6 @@ vdi_simple_gethdrs(const struct director *d, struct worker *wrk, return (i); } -/*-------------------------------------------------------------------- - */ - -int -VDI_GetHdr(struct worker *wrk, struct busyobj *bo) -{ - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - - if (bo->director == NULL) { - VSLb(bo->vsl, SLT_FetchError, "No backend"); - return (-1); - } - - CHECK_OBJ_NOTNULL(bo->director, DIRECTOR_MAGIC); - while (bo->director != NULL && bo->director->resolve != NULL) - bo->director = bo->director->resolve(bo->director, wrk, bo); - - if (bo->director == NULL) { - VSLb(bo->vsl, SLT_FetchError, "Backend selection failed"); - return (-1); - } - - AN(bo->director->gethdrs); - return (bo->director->gethdrs(bo->director, wrk, bo)); -} - /*--------------------------------------------------------------------*/ void diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index 5fa79bd..36c31c0 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -77,7 +77,7 @@ struct vrt_backend_probe; typedef struct vbc *vdi_getfd_f(const struct director *, struct busyobj *); typedef unsigned vdi_healthy_f(const struct director *, double *changed); -typedef struct director *vdi_resolve_f(const struct director *, +typedef const struct director *vdi_resolve_f(const struct director *, struct worker *, struct busyobj *); typedef int vdi_gethdrs_f(const struct director *, struct worker *, struct busyobj *); diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 5195688..9c47154 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -144,7 +144,7 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req) bo->do_stream = 1; - bo->director = req->director_hint; + bo->director_req = req->director_hint; bo->vcl = req->vcl; VCL_Ref(bo->vcl); diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c index 9687366..b8cc79c 100644 --- a/bin/varnishd/cache/cache_dir.c +++ b/bin/varnishd/cache/cache_dir.c @@ -90,17 +90,63 @@ VDI_RecycleFd(struct vbc **vbp, const struct acct_bereq *acct_bereq) VBE_DropRefLocked(bp, acct_bereq); } +/* Resolve director --------------------------------------------------*/ + +static const struct director * +vdi_resolve(struct worker *wrk, struct busyobj *bo, const struct director *d) +{ + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + + if (d == NULL) { + VSLb(bo->vsl, SLT_FetchError, "No backend"); + return (NULL); + } + + while (d != NULL && d->resolve != NULL) { + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + d = d->resolve(d, wrk, bo); + } + CHECK_OBJ_ORNULL(d, DIRECTOR_MAGIC); + if (d == NULL) + VSLb(bo->vsl, SLT_FetchError, "Backend selection failed"); + bo->director_resp = d; + return (d); +} + +/* Get a set of response headers -------------------------------------*/ + +int +VDI_GetHdr(struct worker *wrk, struct busyobj *bo) +{ + + const struct director *d; + + d = vdi_resolve(wrk, bo, bo->director_req); + if (d == NULL) + return (-1); + + AN(d->gethdrs); + return (d->gethdrs(d, wrk, bo)); +} + /* Get a connection --------------------------------------------------*/ struct vbc * -VDI_GetFd(struct busyobj *bo) +VDI_GetFd(const struct director *d, struct worker *wrk, struct busyobj *bo) { struct vbc *vc; - struct director *d; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - d = bo->director; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + + d = vdi_resolve(wrk, bo, d); + if (d == NULL) + return (NULL); + + AN(d->getfd); vc = d->getfd(d, bo); if (vc != NULL) vc->vsl = bo->vsl; diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index bb58f1b..7884d5e 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -86,7 +86,6 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - CHECK_OBJ_NOTNULL(bo->director, DIRECTOR_MAGIC); CHECK_OBJ_NOTNULL(bo->vbc, VBC_MAGIC); CHECK_OBJ_ORNULL(bo->req, REQ_MAGIC); diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c index 36ef425..71cf3e6 100644 --- a/bin/varnishd/cache/cache_pipe.c +++ b/bin/varnishd/cache/cache_pipe.c @@ -113,7 +113,7 @@ PipeRequest(struct req *req, struct busyobj *bo) acct_pipe.req = req->acct.req_hdrbytes; req->acct.req_hdrbytes = 0; - vc = VDI_GetFd(bo); + vc = VDI_GetFd(bo->director_req, wrk, bo); if (vc == NULL) { VSLb(bo->vsl, SLT_FetchError, "no backend connection"); pipecharge(req, &acct_pipe, NULL); diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 439ef46..78d1ea8 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -313,12 +313,12 @@ VRT_r_beresp_backend_name(const struct vrt_ctx *ctx) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + if (ctx->bo->director_resp != NULL) + return (ctx->bo->director_resp->vcl_name); if (ctx->bo->vbc != NULL) { CHECK_OBJ_NOTNULL(ctx->bo->vbc, VBC_MAGIC); return (ctx->bo->vbc->backend->vcl_name); } - if (ctx->bo->director != NULL) - return (ctx->bo->director->vcl_name); return (NULL); } @@ -390,29 +390,38 @@ VRT_r_req_##nm(const struct vrt_ctx *ctx) \ return(ctx->req->elem); \ } -REQ_VAR_L(backend_hint, director_hint, struct director *,) -REQ_VAR_R(backend_hint, director_hint, struct director *) +REQ_VAR_L(backend_hint, director_hint, const struct director *,) +REQ_VAR_R(backend_hint, director_hint, const struct director *) REQ_VAR_L(ttl, d_ttl, double, if (!(arg>0.0)) arg = 0;) REQ_VAR_R(ttl, d_ttl, double) /*--------------------------------------------------------------------*/ void -VRT_l_bereq_backend(const struct vrt_ctx *ctx, struct director *be) +VRT_l_bereq_backend(const struct vrt_ctx *ctx, const struct director *be) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); - ctx->bo->director = be; + ctx->bo->director_req = be; } -struct director * +const struct director * VRT_r_bereq_backend(const struct vrt_ctx *ctx) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); - return (ctx->bo->director); + return (ctx->bo->director_req); +} + +const struct director * +VRT_r_beresp_backend(const struct vrt_ctx *ctx) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + return (ctx->bo->director_resp); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishtest/tests/m00009.vtc b/bin/varnishtest/tests/m00009.vtc index 7fbeec1..01b3e0a 100644 --- a/bin/varnishtest/tests/m00009.vtc +++ b/bin/varnishtest/tests/m00009.vtc @@ -37,6 +37,9 @@ varnish v1 -vcl+backend { sub vcl_backend_fetch { set bereq.backend = rr.backend(); } + sub vcl_backend_response { + set beresp.http.where = bereq.backend + "-->" + beresp.backend; + } } -start client c1 { diff --git a/bin/varnishtest/tests/v00007.vtc b/bin/varnishtest/tests/v00007.vtc index 196f92e..7f9507f 100644 --- a/bin/varnishtest/tests/v00007.vtc +++ b/bin/varnishtest/tests/v00007.vtc @@ -19,12 +19,16 @@ varnish v1 -vcl+backend { sub vcl_backend_fetch { set bereq.backend = foo.backend(); } + sub vcl_backend_response { + set beresp.http.where = bereq.backend + "-->" + beresp.backend; + } } -start client c1 { - timeout 10 txreq -url "/foo" rxresp + expect resp.http.where == "foo-->s1" txreq -url "/bar" rxresp + expect resp.http.where == "foo-->s1" } -run diff --git a/bin/varnishtest/tests/v00026.vtc b/bin/varnishtest/tests/v00026.vtc index 2b52238..11475dc 100644 --- a/bin/varnishtest/tests/v00026.vtc +++ b/bin/varnishtest/tests/v00026.vtc @@ -42,6 +42,9 @@ varnish v1 -vcl+backend { set bereq.backend = h1.backend(bereq.url); } } + sub vcl_backend_response { + set beresp.http.where = bereq.backend + "-->" + beresp.backend; + } } -start diff --git a/include/vrt.h b/include/vrt.h index 1bb0b66..e8c322c 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -62,7 +62,7 @@ struct suckaddr; * (alphabetic order) */ -typedef struct director * VCL_BACKEND; +typedef const struct director * VCL_BACKEND; typedef const struct vmod_priv * VCL_BLOB; typedef unsigned VCL_BOOL; typedef double VCL_BYTES; diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 4fffa57..2bf7133 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -326,6 +326,7 @@ sp_variables = [ 'BACKEND', ( 'pipe', 'backend', ), ( 'pipe', 'backend', ), """ + This is the backend or director we attempt to fetch from. """ ), ('bereq.method', @@ -490,6 +491,15 @@ sp_variables = [ ( 'backend_response', 'backend_error'), """ """ ), + ('beresp.backend', + 'BACKEND', + ( 'backend_response', 'backend_error'), + ( ), """ + This is the backend we fetched from. If bereq.backend + was set to a director, this will be the backend selected + by the director. + """ + ), ('beresp.backend.name', 'STRING', ( 'backend_response', 'backend_error'), diff --git a/lib/libvmod_directors/fall_back.c b/lib/libvmod_directors/fall_back.c index 272864e..a721617 100644 --- a/lib/libvmod_directors/fall_back.c +++ b/lib/libvmod_directors/fall_back.c @@ -53,7 +53,7 @@ vmod_fallback_healthy(const struct director *dir, double *changed) return (vdir_any_healthy(rr->vd, changed)); } -static struct director * __match_proto__(vdi_resolve_f) +static const struct director * __match_proto__(vdi_resolve_f) vmod_fallback_resolve(const struct director *dir, struct worker *wrk, struct busyobj *bo) { diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c index a7e26d8..0e7c0a4 100644 --- a/lib/libvmod_directors/random.c +++ b/lib/libvmod_directors/random.c @@ -58,7 +58,7 @@ vmod_random_healthy(const struct director *dir, double *changed) return (vdir_any_healthy(rr->vd, changed)); } -static struct director * __match_proto__(vdi_resolve_f) +static const struct director * __match_proto__(vdi_resolve_f) vmod_random_resolve(const struct director *dir, struct worker *wrk, struct busyobj *bo) { diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c index 581eef3..a8fba65 100644 --- a/lib/libvmod_directors/round_robin.c +++ b/lib/libvmod_directors/round_robin.c @@ -54,7 +54,7 @@ vmod_rr_healthy(const struct director *dir, double *changed) return (vdir_any_healthy(rr->vd, changed)); } -static struct director * __match_proto__(vdi_resolve_f) +static const struct director * __match_proto__(vdi_resolve_f) vmod_rr_resolve(const struct director *dir, struct worker *wrk, struct busyobj *bo) { From phk at FreeBSD.org Wed Sep 24 08:56:58 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 24 Sep 2014 10:56:58 +0200 Subject: [master] e88f87b Polishing Message-ID: commit e88f87b46f9851fac89a4900603de2d559f1543a Author: Poul-Henning Kamp Date: Wed Sep 24 08:56:50 2014 +0000 Polishing diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 92a4f71..dd3c62a 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -695,7 +695,7 @@ void VCA_FailSess(struct worker *w); void VBE_UseHealth(const struct director *vdi); void VBE_DiscardHealth(const struct director *vdi); - +/* cache_dir.c */ int VDI_GetHdr(struct worker *wrk, struct busyobj *bo); struct vbc *VDI_GetFd(const struct director *d, struct worker *wrk, struct busyobj *); diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index 36c31c0..5888180 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -26,43 +26,21 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * This is the central switch-board for backend connections and it is - * slightly complicated by a number of optimizations. + * Backend and Director APIs * - * The data structures: + * A director ("VDI") is an abstract entity which can either satisfy a + * backend fetch request or select another director for the job. * - * A vrt_backend is a definition of a backend in a VCL program. + * In theory a director does not have to talk HTTP over TCP, it can satisfy + * the backend request using any means it wants, although this is presently + * not implemented. * - * A backend is a TCP destination, possibly multi-homed and it has a - * number of associated properties and statistics. + * A backend ("VBE") is a director which talks HTTP over TCP. * - * A vbc is an open TCP connection to a backend. - * - * A bereq is a memory carrier for handling a HTTP transaction with - * a backend over a vbc. - * - * A director is a piece of code that selects which backend to use, - * by whatever method or metric it chooses. - * - * The relationships: - * - * Backends and directors get instantiated when VCL's are loaded, - * and this always happen in the CLI thread. - * - * When a VCL tries to instantiate a backend, any existing backend - * with the same identity (== definition in VCL) will be used instead - * so that vbc's can be reused across VCL changes. - * - * Directors disapper with the VCL that created them. - * - * Backends disappear when their reference count drop to zero. - * - * Backends have their host/port name looked up to addrinfo structures - * when they are instantiated, and we just cache that result and cycle - * through the entries (for multihomed backends) on failure only. - * XXX: add cli command to redo lookup. - * - * bereq is sort of a step-child here, we just manage the pool of them. + * As you'll notice the terminology is a bit muddled here, but we try to + * keep it clean on the user-facing side, where a "director" is always + * a "pick a backend/director" functionality, and a "backend" is whatever + * satisfies the actual request in the end. * */ diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c index b8cc79c..639c4b4 100644 --- a/bin/varnishd/cache/cache_dir.c +++ b/bin/varnishd/cache/cache_dir.c @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * Handle backend connections and backend request structures. + * Abstract backend API * */ @@ -161,5 +161,6 @@ VDI_Healthy(const struct director *d) { CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + AN(d->healthy); return (d->healthy(d, NULL)); } diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index a4aa167..5e27e76 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -133,7 +133,8 @@ vbf_beresp2obj(struct busyobj *bo) return (-1); if (vary != NULL) { - b = ObjSetattr(bo->wrk, bo->fetch_objcore, OA_VARY, varyl, VSB_data(vary)); + b = ObjSetattr(bo->wrk, bo->fetch_objcore, OA_VARY, varyl, + VSB_data(vary)); VSB_delete(vary); } @@ -149,7 +150,8 @@ vbf_beresp2obj(struct busyobj *bo) bo->uncacheable ? HTTPH_R_PASS : HTTPH_A_INS); if (http_GetHdr(bo->beresp, H_Last_Modified, &b)) - AZ(ObjSetDouble(bo->wrk, bo->fetch_objcore, OA_LASTMODIFIED, VTIM_parse(b))); + AZ(ObjSetDouble(bo->wrk, bo->fetch_objcore, OA_LASTMODIFIED, + VTIM_parse(b))); else AZ(ObjSetDouble(bo->wrk, bo->fetch_objcore, OA_LASTMODIFIED, floor(bo->fetch_objcore->exp.t_origin))); From phk at FreeBSD.org Wed Sep 24 09:07:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 24 Sep 2014 11:07:50 +0200 Subject: [master] c3a4194 Concentrate director and backend related prototypes in cache_backend.h Message-ID: commit c3a41949e1956627256d488117b1099f362aeb48 Author: Poul-Henning Kamp Date: Wed Sep 24 09:07:33 2014 +0000 Concentrate director and backend related prototypes in cache_backend.h diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index dd3c62a..142fcef 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -691,28 +691,6 @@ int VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa); const char *VCA_SetupSess(struct worker *w, struct sess *sp); void VCA_FailSess(struct worker *w); -/* cache_backend.c */ -void VBE_UseHealth(const struct director *vdi); -void VBE_DiscardHealth(const struct director *vdi); - -/* cache_dir.c */ -int VDI_GetHdr(struct worker *wrk, struct busyobj *bo); -struct vbc *VDI_GetFd(const struct director *d, struct worker *wrk, - struct busyobj *); -int VDI_Healthy(const struct director *); -void VDI_CloseFd(struct vbc **vbp, const struct acct_bereq *); -void VDI_RecycleFd(struct vbc **vbp, const struct acct_bereq *); -void VDI_AddHostHeader(struct http *to, const struct vbc *vbc); -void VBE_Poll(void); -void VDI_Init(void); - -/* cache_backend_cfg.c */ -void VBE_InitCfg(void); -struct backend *VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb); - -/* cache_backend_poll.c */ -void VBP_Init(void); - /* cache_ban.c */ struct ban *BAN_New(void); int BAN_AddTest(struct ban *, const char *, const char *, const char *); diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index 5888180..7bec22d 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -134,12 +134,16 @@ struct vbc { /* cache_backend.c */ void VBE_ReleaseConn(struct vbc *vc); +void VBE_UseHealth(const struct director *vdi); +void VBE_DiscardHealth(const struct director *vdi); /* cache_backend_cfg.c */ void VBE_DropRefConn(struct backend *, const struct acct_bereq *); void VBE_DropRefVcl(struct backend *); void VBE_DropRefLocked(struct backend *b, const struct acct_bereq *); unsigned VBE_Healthy(const struct backend *b, double *changed); +void VBE_InitCfg(void); +struct backend *VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb); /* cache_backend_poll.c */ void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, @@ -147,3 +151,18 @@ void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, void VBP_Remove(struct backend *b, struct vrt_backend_probe const *p); void VBP_Use(const struct backend *b, const struct vrt_backend_probe *p); void VBP_Summary(struct cli *cli, const struct vbp_target *vt); + +/* cache_dir.c */ +int VDI_GetHdr(struct worker *wrk, struct busyobj *bo); +struct vbc *VDI_GetFd(const struct director *d, struct worker *wrk, + struct busyobj *); +int VDI_Healthy(const struct director *); +void VDI_CloseFd(struct vbc **vbp, const struct acct_bereq *); +void VDI_RecycleFd(struct vbc **vbp, const struct acct_bereq *); +void VDI_AddHostHeader(struct http *to, const struct vbc *vbc); +void VBE_Poll(void); +void VDI_Init(void); + +/* cache_backend_poll.c */ +void VBP_Init(void); + diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 5e27e76..cc581cf 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -35,6 +35,7 @@ #include #include "cache.h" +#include "cache_backend.h" #include "vend.h" #include "hash/hash_slinger.h" #include "vcl.h" diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index 665a024..15b8daa 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -33,6 +33,7 @@ #include #include "cache.h" +#include "cache_backend.h" #include "common/heritage.h" #include "vcli_priv.h" diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 521f291..c383e76 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -37,6 +37,7 @@ #include #include "cache.h" +#include "cache_backend.h" #include "vcl.h" #include "vrt.h" From phk at FreeBSD.org Wed Sep 24 09:23:05 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 24 Sep 2014 11:23:05 +0200 Subject: [master] b103cc1 Abstract the body setup so that directors can use other means than HTTP/1 Message-ID: commit b103cc1d47529e0c40166388b096513f33833e47 Author: Poul-Henning Kamp Date: Wed Sep 24 09:22:40 2014 +0000 Abstract the body setup so that directors can use other means than HTTP/1 diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 4c00753..ba692fb 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -425,6 +425,19 @@ vdi_simple_gethdrs(const struct director *d, struct worker *wrk, return (i); } +static int __match_proto__(vdi_getbody_f) +vdi_simple_getbody(const struct director *d, struct worker *wrk, + struct busyobj *bo) +{ + + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + + V1F_Setup_Fetch(bo->vfc, bo->htc); + return (0); +} + /*--------------------------------------------------------------------*/ void @@ -463,6 +476,7 @@ VRT_init_dir(struct cli *cli, struct director **bp, int idx, const void *priv) vs->dir.getfd = vdi_simple_getfd; vs->dir.healthy = vdi_simple_healthy; vs->dir.gethdrs = vdi_simple_gethdrs; + vs->dir.getbody = vdi_simple_getbody; vs->vrt = t; diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index 7bec22d..6ff1351 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -59,6 +59,8 @@ typedef const struct director *vdi_resolve_f(const struct director *, struct worker *, struct busyobj *); typedef int vdi_gethdrs_f(const struct director *, struct worker *, struct busyobj *); +typedef int vdi_getbody_f(const struct director *, struct worker *, + struct busyobj *); struct director { unsigned magic; @@ -69,6 +71,7 @@ struct director { vdi_healthy_f *healthy; vdi_resolve_f *resolve; vdi_gethdrs_f *gethdrs; + vdi_getbody_f *getbody; void *priv; }; @@ -154,6 +157,8 @@ void VBP_Summary(struct cli *cli, const struct vbp_target *vt); /* cache_dir.c */ int VDI_GetHdr(struct worker *wrk, struct busyobj *bo); +int VDI_GetBody(const struct director *d, struct worker *wrk, + struct busyobj *bo); struct vbc *VDI_GetFd(const struct director *d, struct worker *wrk, struct busyobj *); int VDI_Healthy(const struct director *); diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c index 639c4b4..a9ad94e 100644 --- a/bin/varnishd/cache/cache_dir.c +++ b/bin/varnishd/cache/cache_dir.c @@ -131,6 +131,22 @@ VDI_GetHdr(struct worker *wrk, struct busyobj *bo) return (d->gethdrs(d, wrk, bo)); } +/* Setup body fetch --------------------------------------------------*/ + +int +VDI_GetBody(const struct director *d, struct worker *wrk, struct busyobj *bo) +{ + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + + AZ(d->resolve); + AN(d->getbody); + + return (d->getbody(d, wrk, bo)); +} + /* Get a connection --------------------------------------------------*/ struct vbc * diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index cc581cf..7ba5852 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -107,7 +107,7 @@ vbf_beresp2obj(struct busyobj *bo) if (varyl > 0) { AN(vary); assert(varyl == VSB_len(vary)); - l += PRNDUP(varyl); + l += PRNDUP((intptr_t)varyl); } else if (varyl < 0) { /* * Vary parse error @@ -593,7 +593,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 1); if (bo->htc->body_status != BS_NONE) - V1F_Setup_Fetch(bo->vfc, bo->htc); + AZ(VDI_GetBody(bo->director_resp, bo->wrk, bo)); /* * Ready to fetch the body From phk at FreeBSD.org Wed Sep 24 12:21:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 24 Sep 2014 14:21:52 +0200 Subject: [master] fb046f6 Sort backend fetches (more) into protocol-agnostic bits, and protocol- implementation bits. Still not there, but getting closer... Message-ID: commit fb046f6ead21ec0ec5044e9223e0c41a7b4de5b1 Author: Poul-Henning Kamp Date: Wed Sep 24 12:21:03 2014 +0000 Sort backend fetches (more) into protocol-agnostic bits, and protocol- implementation bits. Still not there, but getting closer... diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 142fcef..238baa1 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -457,6 +457,12 @@ enum busyobj_state_e { BOS_FAILED, /* something went wrong */ }; +enum director_state_e { + DIR_S_NULL = 0, + DIR_S_HDRS = 1, + DIR_S_BODY = 2, +}; + struct busyobj { unsigned magic; #define BUSYOBJ_MAGIC 0x23b95567 @@ -513,6 +519,7 @@ struct busyobj { const char *storage_hint; const struct director *director_req; const struct director *director_resp; + enum director_state_e director_state; struct VCL_conf *vcl; struct vsl_log vsl[1]; diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index ba692fb..b8a2a79 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -414,6 +414,7 @@ vdi_simple_gethdrs(const struct director *d, struct worker *wrk, * Do a single retry in that case. */ if (i == 1) { + AZ(bo->vbc); VSC_C_main->backend_retry++; bo->vbc = VDI_GetFd(d, wrk, bo); if (bo->vbc == NULL) { @@ -422,6 +423,10 @@ vdi_simple_gethdrs(const struct director *d, struct worker *wrk, } i = V1F_fetch_hdr(wrk, bo); } + if (i != 0) + AZ(bo->vbc); + else + AN(bo->vbc); return (i); } @@ -438,6 +443,23 @@ vdi_simple_getbody(const struct director *d, struct worker *wrk, return (0); } +static void __match_proto__(vdi_finish_f) +vdi_simple_finish(const struct director *d, struct worker *wrk, + struct busyobj *bo) +{ + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + + if (bo->vbc != NULL) { + if (bo->doclose != SC_NULL) + VDI_CloseFd(&bo->vbc, &bo->acct); + else + VDI_RecycleFd(&bo->vbc, &bo->acct); + } +} + + /*--------------------------------------------------------------------*/ void @@ -477,6 +499,7 @@ VRT_init_dir(struct cli *cli, struct director **bp, int idx, const void *priv) vs->dir.healthy = vdi_simple_healthy; vs->dir.gethdrs = vdi_simple_gethdrs; vs->dir.getbody = vdi_simple_getbody; + vs->dir.finish = vdi_simple_finish; vs->vrt = t; diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index 6ff1351..c51d62f 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -61,6 +61,8 @@ typedef int vdi_gethdrs_f(const struct director *, struct worker *, struct busyobj *); typedef int vdi_getbody_f(const struct director *, struct worker *, struct busyobj *); +typedef void vdi_finish_f(const struct director *, struct worker *, + struct busyobj *); struct director { unsigned magic; @@ -72,6 +74,7 @@ struct director { vdi_resolve_f *resolve; vdi_gethdrs_f *gethdrs; vdi_getbody_f *getbody; + vdi_finish_f *finish; void *priv; }; @@ -159,6 +162,8 @@ void VBP_Summary(struct cli *cli, const struct vbp_target *vt); int VDI_GetHdr(struct worker *wrk, struct busyobj *bo); int VDI_GetBody(const struct director *d, struct worker *wrk, struct busyobj *bo); +void VDI_Finish(const struct director *d, struct worker *wrk, + struct busyobj *bo); struct vbc *VDI_GetFd(const struct director *d, struct worker *wrk, struct busyobj *); int VDI_Healthy(const struct director *); diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 9c47154..4256400 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -197,6 +197,8 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo) VSL_End(bo->vsl); + AZ(bo->vbc); + if (bo->fetch_objcore != NULL) { AN(wrk); (void)HSH_DerefObjCore(wrk, &bo->fetch_objcore); diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c index a9ad94e..0dd65b5 100644 --- a/bin/varnishd/cache/cache_dir.c +++ b/bin/varnishd/cache/cache_dir.c @@ -120,15 +120,21 @@ vdi_resolve(struct worker *wrk, struct busyobj *bo, const struct director *d) int VDI_GetHdr(struct worker *wrk, struct busyobj *bo) { - const struct director *d; + int i = -1; - d = vdi_resolve(wrk, bo, bo->director_req); - if (d == NULL) - return (-1); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - AN(d->gethdrs); - return (d->gethdrs(d, wrk, bo)); + d = vdi_resolve(wrk, bo, bo->director_req); + if (d != NULL) { + AN(d->gethdrs); + bo->director_state = DIR_S_HDRS; + i = d->gethdrs(d, wrk, bo); + } + if (i) + bo->director_state = DIR_S_NULL; + return (i); } /* Setup body fetch --------------------------------------------------*/ @@ -137,16 +143,36 @@ int VDI_GetBody(const struct director *d, struct worker *wrk, struct busyobj *bo) { + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); AZ(d->resolve); AN(d->getbody); + bo->director_state = DIR_S_BODY; return (d->getbody(d, wrk, bo)); } +/* Finish fetch ------------------------------------------------------*/ + +void +VDI_Finish(const struct director *d, struct worker *wrk, struct busyobj *bo) +{ + + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + + AZ(d->resolve); + AN(d->finish); + + assert(bo->director_state != DIR_S_NULL); + bo->director_state = DIR_S_NULL; + + d->finish(d, wrk, bo); +} + /* Get a connection --------------------------------------------------*/ struct vbc * diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 7ba5852..f680f02 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -174,7 +174,6 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo) CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC); assert(bo->state == BOS_INVALID); - AZ(bo->vbc); assert(bo->doclose == SC_NULL); AZ(bo->storage_hint); @@ -245,7 +244,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - AZ(bo->vbc); assert(bo->doclose == SC_NULL); AZ(bo->storage_hint); @@ -274,11 +272,10 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) VSLb_ts_busyobj(bo, "Beresp", now); if (i) { - AZ(bo->vbc); + assert(bo->director_state == DIR_S_NULL); return (F_STP_ERROR); } - AN(bo->vbc); http_VSL_log(bo->beresp); if (!http_GetHdr(bo->beresp, H_Date, NULL)) { @@ -355,9 +352,10 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) } if (bo->htc->body_status == BS_ERROR) { - AN (bo->vbc); - VDI_CloseFd(&bo->vbc, &bo->acct); + bo->doclose = SC_RX_BODY; + VDI_Finish(bo->director_resp, bo->wrk, bo); VSLb(bo->vsl, SLT_Error, "Body cannot be fetched"); + assert(bo->director_state == DIR_S_NULL); return (F_STP_ERROR); } @@ -398,18 +396,22 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws); - if (wrk->handling == VCL_RET_ABANDON) + if (wrk->handling == VCL_RET_ABANDON) { + bo->doclose = SC_RESP_CLOSE; + VDI_Finish(bo->director_resp, bo->wrk, bo); return (F_STP_FAIL); + } if (wrk->handling == VCL_RET_RETRY) { - AN (bo->vbc); - VDI_CloseFd(&bo->vbc, &bo->acct); + bo->doclose = SC_RESP_CLOSE; + VDI_Finish(bo->director_resp, bo->wrk, bo); bo->doclose = SC_NULL; bo->retries++; if (bo->retries <= cache_param->max_retries) return (F_STP_RETRY); VSLb(bo->vsl, SLT_VCL_Error, "Too many retries, delivering 503"); + assert(bo->director_state == DIR_S_NULL); return (F_STP_ERROR); } @@ -481,14 +483,14 @@ vbf_fetch_body_helper(struct busyobj *bo) } } while (vfps == VFP_OK); + VFP_Close(vfc); + if (vfps == VFP_ERROR) { AN(vfc->failed); (void)VFP_Error(vfc, "Fetch Pipeline failed to process"); bo->doclose = SC_RX_BODY; } - VFP_Close(vfc); - if (!bo->do_stream) ObjTrimStore(bo->wrk, vfc->oc); } @@ -539,8 +541,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) if (bo->do_gzip && !bo->is_gunzip) bo->do_gzip = 0; - AN(bo->vbc); - /* But we can't do both at the same time */ assert(bo->do_gzip == 0 || bo->do_gunzip == 0); @@ -569,12 +569,14 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) if (VFP_Open(bo->vfc)) { (void)VFP_Error(bo->vfc, "Fetch Pipeline failed to open"); bo->doclose = SC_RX_BODY; + VDI_Finish(bo->director_resp, bo->wrk, bo); return (F_STP_ERROR); } if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo->vfc, "Could not get storage"); bo->doclose = SC_RX_BODY; + VDI_Finish(bo->director_resp, bo->wrk, bo); return (F_STP_ERROR); } @@ -620,11 +622,15 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) if (bo->vfc->failed && !bo->do_stream) { assert(bo->state < BOS_STREAM); ObjFreeObj(bo->wrk, bo->fetch_objcore); + // XXX: doclose = ? + VDI_Finish(bo->director_resp, bo->wrk, bo); return (F_STP_ERROR); } - if (bo->vfc->failed) + if (bo->vfc->failed) { + VDI_Finish(bo->director_resp, bo->wrk, bo); return (F_STP_FAIL); + } if (bo->do_stream) assert(bo->state == BOS_STREAM); @@ -635,10 +641,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ - if (bo->vbc != NULL && bo->doclose == SC_NULL) { - VDI_RecycleFd(&bo->vbc, &bo->acct); - AZ(bo->vbc); - } + VDI_Finish(bo->director_resp, bo->wrk, bo); VBO_setstate(bo, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); @@ -708,10 +711,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ - if (bo->vbc != NULL && bo->doclose == SC_NULL) { - VDI_RecycleFd(&bo->vbc, &bo->acct); - AZ(bo->vbc); - } + VDI_Finish(bo->director_resp, bo->wrk, bo); VBO_setstate(bo, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); @@ -732,6 +732,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + assert(bo->director_state == DIR_S_NULL); now = W_TIM_real(wrk); VSLb_ts_busyobj(bo, "Error", now); @@ -868,13 +869,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv) } assert(WRW_IsReleased(wrk)); - if (bo->vbc != NULL) { - if (bo->doclose != SC_NULL) - VDI_CloseFd(&bo->vbc, &bo->acct); - else - VDI_RecycleFd(&bo->vbc, &bo->acct); - AZ(bo->vbc); - } + assert(bo->director_state == DIR_S_NULL); http_Teardown(bo->bereq); http_Teardown(bo->beresp); diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 78d1ea8..0679086 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -315,10 +315,6 @@ VRT_r_beresp_backend_name(const struct vrt_ctx *ctx) CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); if (ctx->bo->director_resp != NULL) return (ctx->bo->director_resp->vcl_name); - if (ctx->bo->vbc != NULL) { - CHECK_OBJ_NOTNULL(ctx->bo->vbc, VBC_MAGIC); - return (ctx->bo->vbc->backend->vcl_name); - } return (NULL); } From phk at FreeBSD.org Wed Sep 24 12:36:04 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 24 Sep 2014 14:36:04 +0200 Subject: [master] e386255 Fix printf format for 32 bit archs Message-ID: commit e3862554a6662d6ccb09877dc2d2f16444109e5e Author: Poul-Henning Kamp Date: Wed Sep 24 12:35:50 2014 +0000 Fix printf format for 32 bit archs diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 625f314..ca169d9 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -278,7 +278,7 @@ pan_wrk(const struct worker *wrk) VSB_printf(pan_vsp, " worker = %p {\n", wrk); VSB_printf(pan_vsp, " stack = {0x%jx -> 0x%jx}\n", - wrk->stack_start, wrk->stack_end); + (uintmax_t)wrk->stack_start, (uintmax_t)wrk->stack_end); pan_ws(wrk->aws, 4); m = wrk->cur_method; From phk at FreeBSD.org Thu Sep 25 06:42:34 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 25 Sep 2014 08:42:34 +0200 Subject: [master] c7611af Move the last bits of file descriptor awareness out of the "abstract" director layer. Message-ID: commit c7611af55556de29f6bff5beda78b83473d9f264 Author: Poul-Henning Kamp Date: Thu Sep 25 06:41:57 2014 +0000 Move the last bits of file descriptor awareness out of the "abstract" director layer. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index b8a2a79..f40c5d1 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -355,6 +355,60 @@ VBE_DiscardHealth(const struct director *vdi) VBP_Remove(vs->backend, vs->vrt->probe); } +/* Close a connection ------------------------------------------------*/ + +void +VBE_CloseFd(struct vbc **vbp, const struct acct_bereq *acct_bereq) +{ + struct backend *bp; + struct vbc *vc; + + AN(vbp); + vc = *vbp; + *vbp = NULL; + CHECK_OBJ_NOTNULL(vc, VBC_MAGIC); + CHECK_OBJ_NOTNULL(vc->backend, BACKEND_MAGIC); + assert(vc->fd >= 0); + + bp = vc->backend; + + VSLb(vc->vsl, SLT_BackendClose, "%d %s", vc->fd, bp->display_name); + + vc->vsl = NULL; + VTCP_close(&vc->fd); + VBE_DropRefConn(bp, acct_bereq); + vc->backend = NULL; + VBE_ReleaseConn(vc); +} + +/* Recycle a connection ----------------------------------------------*/ + +static void +vbe_RecycleFd(struct vbc **vbp, const struct acct_bereq *acct_bereq) +{ + struct backend *bp; + struct vbc *vc; + + AN(vbp); + vc = *vbp; + *vbp = NULL; + CHECK_OBJ_NOTNULL(vc, VBC_MAGIC); + CHECK_OBJ_NOTNULL(vc->backend, BACKEND_MAGIC); + assert(vc->fd >= 0); + + bp = vc->backend; + + VSLb(vc->vsl, SLT_BackendReuse, "%d %s", vc->fd, bp->display_name); + + vc->vsl = NULL; + + Lck_Lock(&bp->mtx); + VSC_C_main->backend_recycle++; + VTAILQ_INSERT_HEAD(&bp->connlist, vc, list); + VBE_DropRefLocked(bp, acct_bereq); +} + + /*-------------------------------------------------------------------- * */ @@ -453,9 +507,9 @@ vdi_simple_finish(const struct director *d, struct worker *wrk, if (bo->vbc != NULL) { if (bo->doclose != SC_NULL) - VDI_CloseFd(&bo->vbc, &bo->acct); + VBE_CloseFd(&bo->vbc, &bo->acct); else - VDI_RecycleFd(&bo->vbc, &bo->acct); + vbe_RecycleFd(&bo->vbc, &bo->acct); } } diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index c51d62f..5c18107 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -142,6 +142,7 @@ struct vbc { void VBE_ReleaseConn(struct vbc *vc); void VBE_UseHealth(const struct director *vdi); void VBE_DiscardHealth(const struct director *vdi); +void VBE_CloseFd(struct vbc **vbp, const struct acct_bereq *); /* cache_backend_cfg.c */ void VBE_DropRefConn(struct backend *, const struct acct_bereq *); @@ -167,8 +168,6 @@ void VDI_Finish(const struct director *d, struct worker *wrk, struct vbc *VDI_GetFd(const struct director *d, struct worker *wrk, struct busyobj *); int VDI_Healthy(const struct director *); -void VDI_CloseFd(struct vbc **vbp, const struct acct_bereq *); -void VDI_RecycleFd(struct vbc **vbp, const struct acct_bereq *); void VDI_AddHostHeader(struct http *to, const struct vbc *vbc); void VBE_Poll(void); void VDI_Init(void); diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c index 0dd65b5..9f7ae43 100644 --- a/bin/varnishd/cache/cache_dir.c +++ b/bin/varnishd/cache/cache_dir.c @@ -26,7 +26,10 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * Abstract backend API + * Abstract director API + * + * The abstract director API does not know how we talk to the backend or + * if there even is one in the usual meaning of the word. * */ @@ -35,60 +38,6 @@ #include "cache.h" #include "cache_backend.h" -#include "vtcp.h" - -/* Close a connection ------------------------------------------------*/ - -void -VDI_CloseFd(struct vbc **vbp, const struct acct_bereq *acct_bereq) -{ - struct backend *bp; - struct vbc *vc; - - AN(vbp); - vc = *vbp; - *vbp = NULL; - CHECK_OBJ_NOTNULL(vc, VBC_MAGIC); - CHECK_OBJ_NOTNULL(vc->backend, BACKEND_MAGIC); - assert(vc->fd >= 0); - - bp = vc->backend; - - VSLb(vc->vsl, SLT_BackendClose, "%d %s", vc->fd, bp->display_name); - - vc->vsl = NULL; - VTCP_close(&vc->fd); - VBE_DropRefConn(bp, acct_bereq); - vc->backend = NULL; - VBE_ReleaseConn(vc); -} - -/* Recycle a connection ----------------------------------------------*/ - -void -VDI_RecycleFd(struct vbc **vbp, const struct acct_bereq *acct_bereq) -{ - struct backend *bp; - struct vbc *vc; - - AN(vbp); - vc = *vbp; - *vbp = NULL; - CHECK_OBJ_NOTNULL(vc, VBC_MAGIC); - CHECK_OBJ_NOTNULL(vc->backend, BACKEND_MAGIC); - assert(vc->fd >= 0); - - bp = vc->backend; - - VSLb(vc->vsl, SLT_BackendReuse, "%d %s", vc->fd, bp->display_name); - - vc->vsl = NULL; - - Lck_Lock(&bp->mtx); - VSC_C_main->backend_recycle++; - VTAILQ_INSERT_HEAD(&bp->connlist, vc, list); - VBE_DropRefLocked(bp, acct_bereq); -} /* Resolve director --------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index 7884d5e..de12e88 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -143,7 +143,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo) VSLb(bo->vsl, SLT_FetchError, "backend write error: %d (%s)", errno, strerror(errno)); VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk)); - VDI_CloseFd(&bo->vbc, &bo->acct); + VBE_CloseFd(&bo->vbc, &bo->acct); /* XXX: other cleanup ? */ return (retry); } @@ -169,7 +169,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo) VSLb(bo->vsl, SLT_FetchError, "http %sread error: overflow", first ? "first " : ""); - VDI_CloseFd(&bo->vbc, &bo->acct); + VBE_CloseFd(&bo->vbc, &bo->acct); /* XXX: other cleanup ? */ return (-1); } @@ -177,7 +177,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo) bo->acct.beresp_hdrbytes += Tlen(htc->rxbuf); VSLb(bo->vsl, SLT_FetchError, "http %sread error: EOF", first ? "first " : ""); - VDI_CloseFd(&bo->vbc, &bo->acct); + VBE_CloseFd(&bo->vbc, &bo->acct); /* XXX: other cleanup ? */ return (retry); } @@ -194,7 +194,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo) if (HTTP1_DissectResponse(hp, htc)) { VSLb(bo->vsl, SLT_FetchError, "http format error"); - VDI_CloseFd(&bo->vbc, &bo->acct); + VBE_CloseFd(&bo->vbc, &bo->acct); /* XXX: other cleanup ? */ return (-1); } diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c index 71cf3e6..9d43be0 100644 --- a/bin/varnishd/cache/cache_pipe.c +++ b/bin/varnishd/cache/cache_pipe.c @@ -141,7 +141,7 @@ PipeRequest(struct req *req, struct busyobj *bo) if (i) { pipecharge(req, &acct_pipe, vc->backend->vsc); SES_Close(req->sp, SC_TX_PIPE); - VDI_CloseFd(&vc, NULL); + VBE_CloseFd(&vc, NULL); return; } @@ -183,7 +183,7 @@ PipeRequest(struct req *req, struct busyobj *bo) VSLb_ts_req(req, "PipeSess", W_TIM_real(wrk)); pipecharge(req, &acct_pipe, vc->backend->vsc); SES_Close(req->sp, SC_TX_PIPE); - VDI_CloseFd(&vc, NULL); + VBE_CloseFd(&vc, NULL); bo->vbc = NULL; } From phk at FreeBSD.org Thu Sep 25 07:43:01 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 25 Sep 2014 09:43:01 +0200 Subject: [master] a239c03 Allow vcl_init{} to return(fail) in which case vcl.load or vcl.inline fails too. Message-ID: commit a239c03cdc7bfdba540eff0ba71878b33285156c Author: Poul-Henning Kamp Date: Thu Sep 25 07:41:59 2014 +0000 Allow vcl_init{} to return(fail) in which case vcl.load or vcl.inline fails too. diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index c383e76..245a1c1 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -220,13 +220,19 @@ VCL_Load(const char *fn, const char *name, struct cli *cli) FREE_OBJ(vcl); return (1); } - REPLACE(vcl->name, name); - VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name); - VTAILQ_INSERT_TAIL(&vcl_head, vcl, list); ctx.method = VCL_MET_INIT; ctx.handling = &hand; (void)vcl->conf->init_func(&ctx); + if (hand == VCL_RET_FAIL) { + VCLI_Out(cli, "VCL \"%s\" vcl_init{} failed", name); + (void)dlclose(vcl->dlh); + FREE_OBJ(vcl); + return (1); + } assert(hand == VCL_RET_OK); + REPLACE(vcl->name, name); + VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name); + VTAILQ_INSERT_TAIL(&vcl_head, vcl, list); Lck_Lock(&vcl_mtx); if (vcl_active == NULL) vcl_active = vcl; diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 35e3806..7866f8a 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -538,7 +538,7 @@ mcf_config_inline(struct cli *cli, const char * const *av, void *priv) VCLI_SetResult(cli, CLIS_PARAM); return; } - VCLI_Out(cli, "VCL compiled."); + VCLI_Out(cli, "VCL compiled.\n"); if (child_pid >= 0 && mgt_cli_askchild(&status, &p, "vcl.load %s %s\n", av[2], vf)) { VCLI_SetResult(cli, status); diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 2bf7133..ac62cfc 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -140,7 +140,7 @@ returns =( ('init', "", - ('ok',) + ('ok', 'fail') ), ('fini', "", From phk at FreeBSD.org Thu Sep 25 07:43:01 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 25 Sep 2014 09:43:01 +0200 Subject: [master] 6c29a9c Test case for vcl_init -> return(fail) Message-ID: commit 6c29a9c8e13fe952f04fa51f95faa9cce9460c5c Author: Poul-Henning Kamp Date: Thu Sep 25 07:42:37 2014 +0000 Test case for vcl_init -> return(fail) diff --git a/bin/varnishtest/tests/v00040.vtc b/bin/varnishtest/tests/v00040.vtc new file mode 100644 index 0000000..a21ca4b --- /dev/null +++ b/bin/varnishtest/tests/v00040.vtc @@ -0,0 +1,23 @@ +varnishtest "test failing in vcl_init{}" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend {} -start + +client c1 { + txreq + rxresp +} -run + +varnish v1 -errvcl {VCL "vcl2" vcl_init{} failed} { + sub vcl_init { + return (fail); + } + + backend b1 { .host = "${s1_addr}"; } +} + +varnish v1 -cliok vcl.list From phk at FreeBSD.org Thu Sep 25 08:32:17 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 25 Sep 2014 10:32:17 +0200 Subject: [master] 04c4043 Use vrt_ctx also for init/fini Message-ID: commit 04c40433c0f05f4605a62aa2005f58f2306bd00d Author: Poul-Henning Kamp Date: Thu Sep 25 08:32:00 2014 +0000 Use vrt_ctx also for init/fini diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index f40c5d1..7c25563 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -517,12 +517,12 @@ vdi_simple_finish(const struct director *d, struct worker *wrk, /*--------------------------------------------------------------------*/ void -VRT_fini_dir(struct cli *cli, struct director *d) +VRT_fini_dir(const struct vrt_ctx *ctx, struct director *d) { struct vdi_simple *vs; - (void)cli; ASSERT_CLI(); + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC); @@ -534,13 +534,14 @@ VRT_fini_dir(struct cli *cli, struct director *d) } void -VRT_init_dir(struct cli *cli, struct director **bp, int idx, const void *priv) +VRT_init_dir(const struct vrt_ctx *ctx, struct director **bp, int idx, + const void *priv) { const struct vrt_backend *t; struct vdi_simple *vs; ASSERT_CLI(); - (void)cli; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); t = priv; ALLOC_OBJ(vs, VDI_SIMPLE_MAGIC); @@ -557,7 +558,7 @@ VRT_init_dir(struct cli *cli, struct director **bp, int idx, const void *priv) vs->vrt = t; - vs->backend = VBE_AddBackend(cli, t); + vs->backend = VBE_AddBackend(NULL, t); if (vs->vrt->probe != NULL) VBP_Insert(vs->backend, vs->vrt->probe, vs->vrt->hosthdr); diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 245a1c1..0aa40fe 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -180,9 +180,6 @@ VCL_Load(const char *fn, const char *name, struct cli *cli) ASSERT_CLI(); - memset(&ctx, 0, sizeof ctx); - ctx.magic = VRT_CTX_MAGIC; - vcl = vcl_find(name); if (vcl != NULL) { VCLI_Out(cli, "Config '%s' already loaded", name); @@ -214,17 +211,26 @@ VCL_Load(const char *fn, const char *name, struct cli *cli) FREE_OBJ(vcl); return (1); } - if (vcl->conf->init_vcl(cli)) { + + memset(&ctx, 0, sizeof ctx); + ctx.magic = VRT_CTX_MAGIC; + ctx.method = VCL_MET_INIT; + ctx.handling = &hand; + ctx.cli = cli; + + if (vcl->conf->init_vcl(&ctx)) { VCLI_Out(cli, "VCL \"%s\" Failed to initialize", name); + vcl->conf->fini_vcl(&ctx); (void)dlclose(vcl->dlh); FREE_OBJ(vcl); return (1); } - ctx.method = VCL_MET_INIT; - ctx.handling = &hand; (void)vcl->conf->init_func(&ctx); if (hand == VCL_RET_FAIL) { VCLI_Out(cli, "VCL \"%s\" vcl_init{} failed", name); + ctx.method = VCL_MET_FINI; + (void)vcl->conf->fini_func(&ctx); + vcl->conf->fini_vcl(&ctx); (void)dlclose(vcl->dlh); FREE_OBJ(vcl); return (1); @@ -264,7 +270,7 @@ VCL_Nuke(struct vcls *vcl) ctx.handling = &hand; (void)vcl->conf->fini_func(&ctx); assert(hand == VCL_RET_OK); - vcl->conf->fini_vcl(NULL); + vcl->conf->fini_vcl(&ctx); free(vcl->name); (void)dlclose(vcl->dlh); FREE_OBJ(vcl); diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c index a2dc7c0..b1aef3e 100644 --- a/bin/varnishd/cache/cache_vrt_vmod.c +++ b/bin/varnishd/cache/cache_vrt_vmod.c @@ -63,7 +63,7 @@ static VTAILQ_HEAD(,vmod) vmods = VTAILQ_HEAD_INITIALIZER(vmods); int VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, - const char *path, const char *file_id, struct cli *cli) + const char *path, const char *file_id, const struct vrt_ctx *ctx) { struct vmod *v; const struct vmod_data *d; @@ -71,12 +71,14 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, void *dlhdl; ASSERT_CLI(); + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(ctx->cli); dlhdl = dlopen(path, RTLD_NOW | RTLD_LOCAL); if (dlhdl == NULL) { - VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path); - VCLI_Out(cli, "dlopen() failed: %s\n", dlerror()); - VCLI_Out(cli, "Check child process permissions.\n"); + VCLI_Out(ctx->cli, "Loading VMOD %s from %s:\n", nm, path); + VCLI_Out(ctx->cli, "dlopen() failed: %s\n", dlerror()); + VCLI_Out(ctx->cli, "Check child process permissions.\n"); return (1); } @@ -94,8 +96,9 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, if (d == NULL || d->file_id == NULL || strcmp(d->file_id, file_id)) { - VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path); - VCLI_Out(cli, + VCLI_Out(ctx->cli, + "Loading VMOD %s from %s:\n", nm, path); + VCLI_Out(ctx->cli, "This is no longer the same file seen by" " the VCL-compiler.\n"); (void)dlclose(v->hdl); @@ -111,8 +114,9 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, d->proto == NULL || d->spec == NULL || d->abi == NULL) { - VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path); - VCLI_Out(cli, "VMOD data is mangled.\n"); + VCLI_Out(ctx->cli, + "Loading VMOD %s from %s:\n", nm, path); + VCLI_Out(ctx->cli, "VMOD data is mangled.\n"); (void)dlclose(v->hdl); FREE_OBJ(v); return (1); diff --git a/include/vrt.h b/include/vrt.h index e8c322c..2d9a6be 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -89,6 +89,7 @@ struct vrt_ctx { unsigned method; unsigned *handling; + struct cli *cli; // Only in ...init() struct vsl_log *vsl; struct VCL_conf *vcl; struct ws *ws; @@ -223,15 +224,16 @@ void VRT_Rollback(const struct vrt_ctx *, const struct http *); void VRT_synth_page(const struct vrt_ctx *, const char *, ...); /* Backend related */ -void VRT_init_dir(struct cli *, struct director **, int idx, const void *priv); -void VRT_fini_dir(struct cli *, struct director *); +void VRT_init_dir(const struct vrt_ctx*, struct director **, int idx, + const void *priv); +void VRT_fini_dir(const struct vrt_ctx*, struct director *); /* Suckaddr related */ int VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst); /* VMOD/Modules related */ int VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, - const char *path, const char *file_id, struct cli *cli); + const char *path, const char *file_id, const struct vrt_ctx *ctx); void VRT_Vmod_Fini(void **hdl); struct vmod_priv; diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index ac62cfc..962f89c 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -929,8 +929,8 @@ struct ws; struct cli; struct worker; -typedef int vcl_init_f(struct cli *); -typedef void vcl_fini_f(struct cli *); +typedef int vcl_init_f(const struct vrt_ctx *ctx); +typedef void vcl_fini_f(const struct vrt_ctx *ctx); typedef int vcl_func_f(const struct vrt_ctx *ctx); """) diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index 7c079a9..b9114f6 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -423,9 +423,9 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be) ifp = New_IniFin(tl); VSB_printf(ifp->ini, - "\tVRT_init_dir(cli, VCL_conf.director,\n" + "\tVRT_init_dir(ctx, VCL_conf.director,\n" "\t VGC_backend_%s, &vgc_dir_priv_%s);", vgcname, vgcname); - VSB_printf(ifp->fin, "\tVRT_fini_dir(cli, VGCDIR(%s));", vgcname); + VSB_printf(ifp->fin, "\tVRT_fini_dir(ctx, VGCDIR(%s));", vgcname); tl->ndirector++; } diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 934943d..9085e2c 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -302,7 +302,7 @@ EmitInitFunc(const struct vcc *tl) { struct inifin *p; - Fc(tl, 0, "\nstatic int\nVGC_Init(struct cli *cli)\n{\n\n"); + Fc(tl, 0, "\nstatic int\nVGC_Init(const struct vrt_ctx *ctx)\n{\n\n"); VTAILQ_FOREACH(p, &tl->inifin, list) { AZ(VSB_finish(p->ini)); if (VSB_len(p->ini)) @@ -319,7 +319,7 @@ EmitFiniFunc(const struct vcc *tl) { struct inifin *p; - Fc(tl, 0, "\nstatic void\nVGC_Fini(struct cli *cli)\n{\n\n"); + Fc(tl, 0, "\nstatic void\nVGC_Fini(const struct vrt_ctx *ctx)\n{\n\n"); VTAILQ_FOREACH_REVERSE(p, &tl->inifin, inifinhead, list) { AZ(VSB_finish(p->fin)); diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 7644621..9564ca9 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -177,7 +177,7 @@ vcc_ParseImport(struct vcc *tl) AN(vmd); AN(vmd->file_id); VSB_printf(ifp->ini, "\t \"%s\",\n", vmd->file_id); - VSB_printf(ifp->ini, "\t cli))\n"); + VSB_printf(ifp->ini, "\t ctx))\n"); VSB_printf(ifp->ini, "\t\treturn(1);"); /* XXX: zero the function pointer structure ?*/ From phk at FreeBSD.org Thu Sep 25 08:51:26 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 25 Sep 2014 10:51:26 +0200 Subject: [master] f584202 Introduce "VRT_CTX" as a shorthand macro for "const struct vrt_ctx *ctx" Message-ID: commit f584202c8a497a597be82954ac0f93a112a5b4b7 Author: Poul-Henning Kamp Date: Thu Sep 25 08:50:50 2014 +0000 Introduce "VRT_CTX" as a shorthand macro for "const struct vrt_ctx *ctx" diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 7c25563..845da6a 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -517,7 +517,7 @@ vdi_simple_finish(const struct director *d, struct worker *wrk, /*--------------------------------------------------------------------*/ void -VRT_fini_dir(const struct vrt_ctx *ctx, struct director *d) +VRT_fini_dir(VRT_CTX, struct director *d) { struct vdi_simple *vs; @@ -534,7 +534,7 @@ VRT_fini_dir(const struct vrt_ctx *ctx, struct director *d) } void -VRT_init_dir(const struct vrt_ctx *ctx, struct director **bp, int idx, +VRT_init_dir(VRT_CTX, struct director **bp, int idx, const void *priv) { const struct vrt_backend *t; diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 8422f79..b45cdf9 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -52,7 +52,7 @@ const void * const vrt_magic_string_unset = &vrt_magic_string_unset; /*--------------------------------------------------------------------*/ void -VRT_error(const struct vrt_ctx *ctx, unsigned code, const char *reason) +VRT_error(VRT_CTX, unsigned code, const char *reason) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -69,7 +69,7 @@ VRT_error(const struct vrt_ctx *ctx, unsigned code, const char *reason) /*--------------------------------------------------------------------*/ void -VRT_count(const struct vrt_ctx *ctx, unsigned u) +VRT_count(VRT_CTX, unsigned u) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -81,7 +81,7 @@ VRT_count(const struct vrt_ctx *ctx, unsigned u) /*--------------------------------------------------------------------*/ void -VRT_acl_log(const struct vrt_ctx *ctx, const char *msg) +VRT_acl_log(VRT_CTX, const char *msg) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -91,7 +91,7 @@ VRT_acl_log(const struct vrt_ctx *ctx, const char *msg) /*--------------------------------------------------------------------*/ static struct http * -vrt_selecthttp(const struct vrt_ctx *ctx, enum gethdr_e where) +vrt_selecthttp(VRT_CTX, enum gethdr_e where) { struct http *hp; @@ -118,7 +118,7 @@ vrt_selecthttp(const struct vrt_ctx *ctx, enum gethdr_e where) /*--------------------------------------------------------------------*/ const char * -VRT_GetHdr(const struct vrt_ctx *ctx, const struct gethdr_s *hs) +VRT_GetHdr(VRT_CTX, const struct gethdr_s *hs) { char *p; struct http *hp; @@ -202,7 +202,7 @@ VRT_String(struct ws *ws, const char *h, const char *p, va_list ap) */ const char * -VRT_CollectString(const struct vrt_ctx *ctx, const char *p, ...) +VRT_CollectString(VRT_CTX, const char *p, ...) { va_list ap; const char *b; @@ -218,7 +218,7 @@ VRT_CollectString(const struct vrt_ctx *ctx, const char *p, ...) /*--------------------------------------------------------------------*/ void -VRT_SetHdr(const struct vrt_ctx *ctx , const struct gethdr_s *hs, +VRT_SetHdr(VRT_CTX , const struct gethdr_s *hs, const char *p, ...) { struct http *hp; @@ -248,7 +248,7 @@ VRT_SetHdr(const struct vrt_ctx *ctx , const struct gethdr_s *hs, /*--------------------------------------------------------------------*/ void -VRT_handling(const struct vrt_ctx *ctx, unsigned hand) +VRT_handling(VRT_CTX, unsigned hand) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -261,7 +261,7 @@ VRT_handling(const struct vrt_ctx *ctx, unsigned hand) */ void -VRT_hashdata(const struct vrt_ctx *ctx, const char *str, ...) +VRT_hashdata(VRT_CTX, const char *str, ...) { va_list ap; const char *p; @@ -288,7 +288,7 @@ VRT_hashdata(const struct vrt_ctx *ctx, const char *str, ...) /*--------------------------------------------------------------------*/ double -VRT_r_now(const struct vrt_ctx *ctx) +VRT_r_now(VRT_CTX) { (void)ctx; @@ -298,7 +298,7 @@ VRT_r_now(const struct vrt_ctx *ctx) /*--------------------------------------------------------------------*/ char * -VRT_IP_string(const struct vrt_ctx *ctx, VCL_IP ip) +VRT_IP_string(VRT_CTX, VCL_IP ip) { char *p; unsigned len; @@ -318,7 +318,7 @@ VRT_IP_string(const struct vrt_ctx *ctx, VCL_IP ip) } char * -VRT_INT_string(const struct vrt_ctx *ctx, long num) +VRT_INT_string(VRT_CTX, long num) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -326,7 +326,7 @@ VRT_INT_string(const struct vrt_ctx *ctx, long num) } char * -VRT_REAL_string(const struct vrt_ctx *ctx, double num) +VRT_REAL_string(VRT_CTX, double num) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -334,7 +334,7 @@ VRT_REAL_string(const struct vrt_ctx *ctx, double num) } char * -VRT_TIME_string(const struct vrt_ctx *ctx, double t) +VRT_TIME_string(VRT_CTX, double t) { char *p; @@ -364,7 +364,7 @@ VRT_BOOL_string(unsigned val) /*--------------------------------------------------------------------*/ void -VRT_Rollback(const struct vrt_ctx *ctx, const struct http *hp) +VRT_Rollback(VRT_CTX, const struct http *hp) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -385,7 +385,7 @@ VRT_Rollback(const struct vrt_ctx *ctx, const struct http *hp) /*--------------------------------------------------------------------*/ void -VRT_synth_page(const struct vrt_ctx *ctx, const char *str, ...) +VRT_synth_page(VRT_CTX, const char *str, ...) { va_list ap; const char *p; @@ -415,7 +415,7 @@ VRT_synth_page(const struct vrt_ctx *ctx, const char *str, ...) /*--------------------------------------------------------------------*/ void -VRT_ban_string(const struct vrt_ctx *ctx, const char *str) +VRT_ban_string(VRT_CTX, const char *str) { char *a1, *a2, *a3; char **av; @@ -482,7 +482,7 @@ VRT_ban_string(const struct vrt_ctx *ctx, const char *str) */ int -VRT_CacheReqBody(const struct vrt_ctx *ctx, long long maxsize) +VRT_CacheReqBody(VRT_CTX, long long maxsize) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -500,7 +500,7 @@ VRT_CacheReqBody(const struct vrt_ctx *ctx, long long maxsize) */ void -VRT_purge(const struct vrt_ctx *ctx, double ttl, double grace, double keep) +VRT_purge(VRT_CTX, double ttl, double grace, double keep) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c index d0ecb78..d198d1f 100644 --- a/bin/varnishd/cache/cache_vrt_re.c +++ b/bin/varnishd/cache/cache_vrt_re.c @@ -62,7 +62,7 @@ VRT_re_fini(void *rep) } int -VRT_re_match(const struct vrt_ctx *ctx, const char *s, void *re) +VRT_re_match(VRT_CTX, const char *s, void *re) { vre_t *t; int i; @@ -81,7 +81,7 @@ VRT_re_match(const struct vrt_ctx *ctx, const char *s, void *re) } const char * -VRT_regsub(const struct vrt_ctx *ctx, int all, const char *str, void *re, +VRT_regsub(VRT_CTX, int all, const char *str, void *re, const char *sub) { int ovector[30]; diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 0679086..ff49d3a 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -68,7 +68,7 @@ vrt_do_string(const struct http *hp, int fld, #define VRT_HDR_L(obj, hdr, fld) \ void \ -VRT_l_##obj##_##hdr(const struct vrt_ctx *ctx, const char *p, ...) \ +VRT_l_##obj##_##hdr(VRT_CTX, const char *p, ...) \ { \ va_list ap; \ \ @@ -80,7 +80,7 @@ VRT_l_##obj##_##hdr(const struct vrt_ctx *ctx, const char *p, ...) \ #define VRT_HDR_R(obj, hdr, fld) \ const char * \ -VRT_r_##obj##_##hdr(const struct vrt_ctx *ctx) \ +VRT_r_##obj##_##hdr(VRT_CTX) \ { \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ CHECK_OBJ_NOTNULL(ctx->http_##obj, HTTP_MAGIC); \ @@ -93,7 +93,7 @@ VRT_r_##obj##_##hdr(const struct vrt_ctx *ctx) \ #define VRT_STATUS_L(obj) \ void \ -VRT_l_##obj##_status(const struct vrt_ctx *ctx, long num) \ +VRT_l_##obj##_status(VRT_CTX, long num) \ { \ \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ @@ -112,7 +112,7 @@ VRT_l_##obj##_status(const struct vrt_ctx *ctx, long num) \ #define VRT_STATUS_R(obj) \ long \ -VRT_r_##obj##_status(const struct vrt_ctx *ctx) \ +VRT_r_##obj##_status(VRT_CTX) \ { \ \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ @@ -142,7 +142,7 @@ VRT_STATUS_R(beresp) */ long -VRT_r_obj_status(const struct vrt_ctx *ctx) +VRT_r_obj_status(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); @@ -152,7 +152,7 @@ VRT_r_obj_status(const struct vrt_ctx *ctx) } const char * -VRT_r_obj_proto(const struct vrt_ctx *ctx) +VRT_r_obj_proto(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); @@ -162,7 +162,7 @@ VRT_r_obj_proto(const struct vrt_ctx *ctx) } const char * -VRT_r_obj_reason(const struct vrt_ctx *ctx) +VRT_r_obj_reason(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); @@ -178,7 +178,7 @@ VRT_r_obj_reason(const struct vrt_ctx *ctx) #define VBERESPW0(field) #define VBERESPW1(field) \ void \ -VRT_l_beresp_##field(const struct vrt_ctx *ctx, unsigned a) \ +VRT_l_beresp_##field(VRT_CTX, unsigned a) \ { \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); \ @@ -188,7 +188,7 @@ VRT_l_beresp_##field(const struct vrt_ctx *ctx, unsigned a) \ #define VBERESPR0(field) #define VBERESPR1(field) \ unsigned \ -VRT_r_beresp_##field(const struct vrt_ctx *ctx) \ +VRT_r_beresp_##field(VRT_CTX) \ { \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); \ @@ -204,7 +204,7 @@ VRT_r_beresp_##field(const struct vrt_ctx *ctx) \ /*--------------------------------------------------------------------*/ void -VRT_l_bereq_uncacheable(const struct vrt_ctx *ctx, unsigned a) +VRT_l_bereq_uncacheable(VRT_CTX, unsigned a) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); @@ -218,7 +218,7 @@ VRT_l_bereq_uncacheable(const struct vrt_ctx *ctx, unsigned a) } unsigned -VRT_r_bereq_uncacheable(const struct vrt_ctx *ctx) +VRT_r_bereq_uncacheable(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); @@ -226,7 +226,7 @@ VRT_r_bereq_uncacheable(const struct vrt_ctx *ctx) } void -VRT_l_beresp_uncacheable(const struct vrt_ctx *ctx, unsigned a) +VRT_l_beresp_uncacheable(VRT_CTX, unsigned a) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); @@ -240,7 +240,7 @@ VRT_l_beresp_uncacheable(const struct vrt_ctx *ctx, unsigned a) } unsigned -VRT_r_beresp_uncacheable(const struct vrt_ctx *ctx) +VRT_r_beresp_uncacheable(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); @@ -250,7 +250,7 @@ VRT_r_beresp_uncacheable(const struct vrt_ctx *ctx) /*--------------------------------------------------------------------*/ const char * -VRT_r_client_identity(const struct vrt_ctx *ctx) +VRT_r_client_identity(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -262,7 +262,7 @@ VRT_r_client_identity(const struct vrt_ctx *ctx) } void -VRT_l_client_identity(const struct vrt_ctx *ctx, const char *str, ...) +VRT_l_client_identity(VRT_CTX, const char *str, ...) { va_list ap; const char *b; @@ -284,7 +284,7 @@ VRT_l_client_identity(const struct vrt_ctx *ctx, const char *str, ...) #define BEREQ_TIMEOUT(which) \ void \ -VRT_l_bereq_##which(const struct vrt_ctx *ctx, double num) \ +VRT_l_bereq_##which(VRT_CTX, double num) \ { \ \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ @@ -293,7 +293,7 @@ VRT_l_bereq_##which(const struct vrt_ctx *ctx, double num) \ } \ \ double \ -VRT_r_bereq_##which(const struct vrt_ctx *ctx) \ +VRT_r_bereq_##which(VRT_CTX) \ { \ \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ @@ -308,7 +308,7 @@ BEREQ_TIMEOUT(between_bytes_timeout) /*--------------------------------------------------------------------*/ const char * -VRT_r_beresp_backend_name(const struct vrt_ctx *ctx) +VRT_r_beresp_backend_name(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -319,7 +319,7 @@ VRT_r_beresp_backend_name(const struct vrt_ctx *ctx) } VCL_IP -VRT_r_beresp_backend_ip(const struct vrt_ctx *ctx) +VRT_r_beresp_backend_ip(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -334,7 +334,7 @@ VRT_r_beresp_backend_ip(const struct vrt_ctx *ctx) /*--------------------------------------------------------------------*/ const char * -VRT_r_beresp_storage_hint(const struct vrt_ctx *ctx) +VRT_r_beresp_storage_hint(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); @@ -345,7 +345,7 @@ VRT_r_beresp_storage_hint(const struct vrt_ctx *ctx) } void -VRT_l_beresp_storage_hint(const struct vrt_ctx *ctx, const char *str, ...) +VRT_l_beresp_storage_hint(VRT_CTX, const char *str, ...) { va_list ap; const char *b; @@ -368,7 +368,7 @@ VRT_l_beresp_storage_hint(const struct vrt_ctx *ctx, const char *str, ...) #define REQ_VAR_L(nm, elem, type,extra) \ \ void \ -VRT_l_req_##nm(const struct vrt_ctx *ctx, type arg) \ +VRT_l_req_##nm(VRT_CTX, type arg) \ { \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); \ @@ -379,7 +379,7 @@ VRT_l_req_##nm(const struct vrt_ctx *ctx, type arg) \ #define REQ_VAR_R(nm, elem, type) \ \ type \ -VRT_r_req_##nm(const struct vrt_ctx *ctx) \ +VRT_r_req_##nm(VRT_CTX) \ { \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); \ @@ -394,7 +394,7 @@ REQ_VAR_R(ttl, d_ttl, double) /*--------------------------------------------------------------------*/ void -VRT_l_bereq_backend(const struct vrt_ctx *ctx, const struct director *be) +VRT_l_bereq_backend(VRT_CTX, const struct director *be) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -403,7 +403,7 @@ VRT_l_bereq_backend(const struct vrt_ctx *ctx, const struct director *be) } const struct director * -VRT_r_bereq_backend(const struct vrt_ctx *ctx) +VRT_r_bereq_backend(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -412,7 +412,7 @@ VRT_r_bereq_backend(const struct vrt_ctx *ctx) } const struct director * -VRT_r_beresp_backend(const struct vrt_ctx *ctx) +VRT_r_beresp_backend(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -423,7 +423,7 @@ VRT_r_beresp_backend(const struct vrt_ctx *ctx) /*--------------------------------------------------------------------*/ void -VRT_l_req_esi(const struct vrt_ctx *ctx, unsigned process_esi) +VRT_l_req_esi(VRT_CTX, unsigned process_esi) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -437,7 +437,7 @@ VRT_l_req_esi(const struct vrt_ctx *ctx, unsigned process_esi) } unsigned -VRT_r_req_esi(const struct vrt_ctx *ctx) +VRT_r_req_esi(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -446,7 +446,7 @@ VRT_r_req_esi(const struct vrt_ctx *ctx) } long -VRT_r_req_esi_level(const struct vrt_ctx *ctx) +VRT_r_req_esi_level(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -457,7 +457,7 @@ VRT_r_req_esi_level(const struct vrt_ctx *ctx) /*--------------------------------------------------------------------*/ unsigned -VRT_r_req_can_gzip(const struct vrt_ctx *ctx) +VRT_r_req_can_gzip(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -468,7 +468,7 @@ VRT_r_req_can_gzip(const struct vrt_ctx *ctx) /*--------------------------------------------------------------------*/ long -VRT_r_req_restarts(const struct vrt_ctx *ctx) +VRT_r_req_restarts(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -477,7 +477,7 @@ VRT_r_req_restarts(const struct vrt_ctx *ctx) } long -VRT_r_bereq_retries(const struct vrt_ctx *ctx) +VRT_r_bereq_retries(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -498,7 +498,7 @@ VRT_r_bereq_retries(const struct vrt_ctx *ctx) #define VRT_DO_EXP_L(which, sexp, fld, offset) \ \ void \ -VRT_l_##which##_##fld(const struct vrt_ctx *ctx, double a) \ +VRT_l_##which##_##fld(VRT_CTX, double a) \ { \ \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ @@ -513,7 +513,7 @@ VRT_l_##which##_##fld(const struct vrt_ctx *ctx, double a) \ #define VRT_DO_EXP_R(which, sexp, fld, offset) \ \ double \ -VRT_r_##which##_##fld(const struct vrt_ctx *ctx) \ +VRT_r_##which##_##fld(VRT_CTX) \ { \ double d; \ \ @@ -545,7 +545,7 @@ VRT_DO_EXP_R(beresp, ctx->bo->fetch_objcore->exp, keep, 0) #define VRT_DO_AGE_R(which, sexp) \ \ double \ -VRT_r_##which##_##age(const struct vrt_ctx *ctx) \ +VRT_r_##which##_##age(VRT_CTX) \ { \ \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ @@ -560,7 +560,7 @@ VRT_DO_AGE_R(beresp, ctx->bo->fetch_objcore->exp) */ const char * -VRT_r_req_xid(const struct vrt_ctx *ctx) +VRT_r_req_xid(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -570,7 +570,7 @@ VRT_r_req_xid(const struct vrt_ctx *ctx) } const char * -VRT_r_bereq_xid(const struct vrt_ctx *ctx) +VRT_r_bereq_xid(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -583,7 +583,7 @@ VRT_r_bereq_xid(const struct vrt_ctx *ctx) #define REQ_BOOL(hash_var) \ void \ -VRT_l_req_##hash_var(const struct vrt_ctx *ctx, unsigned val) \ +VRT_l_req_##hash_var(VRT_CTX, unsigned val) \ { \ \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ @@ -592,7 +592,7 @@ VRT_l_req_##hash_var(const struct vrt_ctx *ctx, unsigned val) \ } \ \ unsigned \ -VRT_r_req_##hash_var(const struct vrt_ctx *ctx) \ +VRT_r_req_##hash_var(VRT_CTX) \ { \ \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ @@ -606,7 +606,7 @@ REQ_BOOL(hash_always_miss) /*--------------------------------------------------------------------*/ VCL_IP -VRT_r_client_ip(const struct vrt_ctx *ctx) +VRT_r_client_ip(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -616,7 +616,7 @@ VRT_r_client_ip(const struct vrt_ctx *ctx) } VCL_IP -VRT_r_server_ip(const struct vrt_ctx *ctx) +VRT_r_server_ip(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -626,7 +626,7 @@ VRT_r_server_ip(const struct vrt_ctx *ctx) } const char* -VRT_r_server_identity(const struct vrt_ctx *ctx) +VRT_r_server_identity(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -637,7 +637,7 @@ VRT_r_server_identity(const struct vrt_ctx *ctx) } const char* -VRT_r_server_hostname(const struct vrt_ctx *ctx) +VRT_r_server_hostname(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -649,7 +649,7 @@ VRT_r_server_hostname(const struct vrt_ctx *ctx) /*--------------------------------------------------------------------*/ long -VRT_r_obj_hits(const struct vrt_ctx *ctx) +VRT_r_obj_hits(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); @@ -659,7 +659,7 @@ VRT_r_obj_hits(const struct vrt_ctx *ctx) } unsigned -VRT_r_obj_uncacheable(const struct vrt_ctx *ctx) +VRT_r_obj_uncacheable(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -672,7 +672,7 @@ VRT_r_obj_uncacheable(const struct vrt_ctx *ctx) #define HTTP_VAR(x) \ struct http * \ -VRT_r_##x(const struct vrt_ctx *ctx) \ +VRT_r_##x(VRT_CTX) \ { \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ CHECK_OBJ_NOTNULL(ctx->http_##x, HTTP_MAGIC); \ diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c index b1aef3e..2c393b7 100644 --- a/bin/varnishd/cache/cache_vrt_vmod.c +++ b/bin/varnishd/cache/cache_vrt_vmod.c @@ -63,7 +63,7 @@ static VTAILQ_HEAD(,vmod) vmods = VTAILQ_HEAD_INITIALIZER(vmods); int VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, - const char *path, const char *file_id, const struct vrt_ctx *ctx) + const char *path, const char *file_id, VRT_CTX) { struct vmod *v; const struct vmod_data *d; diff --git a/include/vrt.h b/include/vrt.h index 2d9a6be..3e21a5f 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -105,6 +105,8 @@ struct vrt_ctx { double now; }; +#define VRT_CTX const struct vrt_ctx *ctx + /***********************************************************************/ struct vmod_data { @@ -186,54 +188,51 @@ struct vrt_ref { /* ACL related */ #define VRT_ACL_MAXADDR 16 /* max(IPv4, IPv6) */ -void VRT_acl_log(const struct vrt_ctx *, const char *msg); +void VRT_acl_log(VRT_CTX, const char *msg); /* req related */ -int VRT_CacheReqBody(const struct vrt_ctx *, long long maxsize); +int VRT_CacheReqBody(VRT_CTX, long long maxsize); /* Regexp related */ void VRT_re_init(void **, const char *); void VRT_re_fini(void *); -int VRT_re_match(const struct vrt_ctx *, const char *, void *re); -const char *VRT_regsub(const struct vrt_ctx *, int all, const char *, - void *, const char *); +int VRT_re_match(VRT_CTX, const char *, void *re); +const char *VRT_regsub(VRT_CTX, int all, const char *, void *, const char *); -void VRT_ban_string(const struct vrt_ctx *, const char *); -void VRT_purge(const struct vrt_ctx *, double ttl, double grace, double keep); +void VRT_ban_string(VRT_CTX, const char *); +void VRT_purge(VRT_CTX, double ttl, double grace, double keep); -void VRT_count(const struct vrt_ctx *, unsigned); +void VRT_count(VRT_CTX, unsigned); int VRT_rewrite(const char *, const char *); -void VRT_error(const struct vrt_ctx *, unsigned, const char *); +void VRT_error(VRT_CTX, unsigned, const char *); int VRT_switch_config(const char *); -const char *VRT_GetHdr(const struct vrt_ctx *, const struct gethdr_s *); -void VRT_SetHdr(const struct vrt_ctx *, const struct gethdr_s *, - const char *, ...); -void VRT_handling(const struct vrt_ctx *, unsigned hand); +const char *VRT_GetHdr(VRT_CTX, const struct gethdr_s *); +void VRT_SetHdr(VRT_CTX, const struct gethdr_s *, const char *, ...); +void VRT_handling(VRT_CTX, unsigned hand); -void VRT_hashdata(const struct vrt_ctx *, const char *str, ...); +void VRT_hashdata(VRT_CTX, const char *str, ...); /* Simple stuff */ int VRT_strcmp(const char *s1, const char *s2); void VRT_memmove(void *dst, const void *src, unsigned len); -void VRT_Rollback(const struct vrt_ctx *, const struct http *); +void VRT_Rollback(VRT_CTX, const struct http *); /* Synthetic pages */ -void VRT_synth_page(const struct vrt_ctx *, const char *, ...); +void VRT_synth_page(VRT_CTX, const char *, ...); /* Backend related */ -void VRT_init_dir(const struct vrt_ctx*, struct director **, int idx, - const void *priv); -void VRT_fini_dir(const struct vrt_ctx*, struct director *); +void VRT_init_dir(VRT_CTX, struct director **, int idx, const void *priv); +void VRT_fini_dir(VRT_CTX, struct director *); /* Suckaddr related */ int VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst); /* VMOD/Modules related */ int VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, - const char *path, const char *file_id, const struct vrt_ctx *ctx); + const char *path, const char *file_id, VRT_CTX); void VRT_Vmod_Fini(void **hdl); struct vmod_priv; @@ -253,10 +252,10 @@ int VRT_Stv(const char *nm); /* Convert things to string */ -char *VRT_IP_string(const struct vrt_ctx *, VCL_IP); -char *VRT_INT_string(const struct vrt_ctx *, VCL_INT); -char *VRT_REAL_string(const struct vrt_ctx *, VCL_REAL); -char *VRT_TIME_string(const struct vrt_ctx *, VCL_TIME); +char *VRT_IP_string(VRT_CTX, VCL_IP); +char *VRT_INT_string(VRT_CTX, VCL_INT); +char *VRT_REAL_string(VRT_CTX, VCL_REAL); +char *VRT_TIME_string(VRT_CTX, VCL_TIME); const char *VRT_BOOL_string(VCL_BOOL); const char *VRT_BACKEND_string(VCL_BACKEND); -const char *VRT_CollectString(const struct vrt_ctx *, const char *p, ...); +const char *VRT_CollectString(VRT_CTX, const char *p, ...); diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 962f89c..c82ae0e 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -923,15 +923,16 @@ file_header(fo) fo.write(""" struct vrt_ctx; +#define VRT_CTX const struct vrt_ctx *ctx struct req; struct busyobj; struct ws; struct cli; struct worker; -typedef int vcl_init_f(const struct vrt_ctx *ctx); -typedef void vcl_fini_f(const struct vrt_ctx *ctx); -typedef int vcl_func_f(const struct vrt_ctx *ctx); +typedef int vcl_init_f(VRT_CTX); +typedef void vcl_fini_f(VRT_CTX); +typedef int vcl_func_f(VRT_CTX); """) def tbl40(a, b): @@ -1065,7 +1066,7 @@ def one_var(nm, spec): fo.write('\t "VRT_r_%s(ctx)",\n' % cnam) if nm == i[0]: fh.write("VCL_" + typ + - " VRT_r_%s(const struct vrt_ctx *);\n" % cnam ) + " VRT_r_%s(VRT_CTX);\n" % cnam ) restrict(fo, spec[2]) if len(spec[3]) == 0: @@ -1078,7 +1079,7 @@ def one_var(nm, spec): fo.write('\t "VRT_l_%s(ctx, ",\n' % cnam) if nm == i[0]: fh.write( - "void VRT_l_%s(const struct vrt_ctx *, " % cnam) + "void VRT_l_%s(VRT_CTX, " % cnam) if typ != "STRING": fh.write("VCL_" + typ + ");\n") else: diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 997ae3c..72dd304 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -354,7 +354,7 @@ vcc_acl_emit(struct vcc *tl, const char *acln, int anon) Fh(tl, 0, "\nstatic int\n"); Fh(tl, 0, - "match_acl_%s_%s(const struct vrt_ctx *ctx, const VCL_IP p)\n", + "match_acl_%s_%s(VRT_CTX, const VCL_IP p)\n", anon ? "anon" : "named", acln); Fh(tl, 0, "{\n"); Fh(tl, 0, "\tconst unsigned char *a;\n"); diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 9085e2c..237a07c 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -302,7 +302,7 @@ EmitInitFunc(const struct vcc *tl) { struct inifin *p; - Fc(tl, 0, "\nstatic int\nVGC_Init(const struct vrt_ctx *ctx)\n{\n\n"); + Fc(tl, 0, "\nstatic int\nVGC_Init(VRT_CTX)\n{\n\n"); VTAILQ_FOREACH(p, &tl->inifin, list) { AZ(VSB_finish(p->ini)); if (VSB_len(p->ini)) @@ -319,7 +319,7 @@ EmitFiniFunc(const struct vcc *tl) { struct inifin *p; - Fc(tl, 0, "\nstatic void\nVGC_Fini(const struct vrt_ctx *ctx)\n{\n\n"); + Fc(tl, 0, "\nstatic void\nVGC_Fini(VRT_CTX)\n{\n\n"); VTAILQ_FOREACH_REVERSE(p, &tl->inifin, inifinhead, list) { AZ(VSB_finish(p->fin)); @@ -671,11 +671,11 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp) for (i = 1; i < VCL_MET_MAX; i++) { Fh(tl, 1, "\nint __match_proto__(vcl_func_f)\n"); Fh(tl, 1, - "VGC_function_%s(const struct vrt_ctx *ctx);\n", + "VGC_function_%s(VRT_CTX);\n", method_tab[i].name); Fc(tl, 1, "\nint __match_proto__(vcl_func_f)\n"); Fc(tl, 1, - "VGC_function_%s(const struct vrt_ctx *ctx)\n", + "VGC_function_%s(VRT_CTX)\n", method_tab[i].name); AZ(VSB_finish(tl->fm[i])); Fc(tl, 1, "{\n"); diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c index dab1131..9412a58 100644 --- a/lib/libvcc/vcc_parse.c +++ b/lib/libvcc/vcc_parse.c @@ -251,9 +251,9 @@ vcc_ParseFunction(struct vcc *tl) } tl->curproc = vcc_AddProc(tl, tl->t); Fh(tl, 0, "int VGC_function_%.*s " - "(const struct vrt_ctx *ctx);\n", PF(tl->t)); + "(VRT_CTX);\n", PF(tl->t)); Fc(tl, 1, "\nint __match_proto__(vcl_func_t)\n"); - Fc(tl, 1, "VGC_function_%.*s(const struct vrt_ctx *ctx)\n", + Fc(tl, 1, "VGC_function_%.*s(VRT_CTX)\n", PF(tl->t)); } vcc_NextToken(tl); diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 96fa347..6ad9504 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -358,7 +358,7 @@ class Func(object): s = ctypes[self.retval] + " vmod_" + self.cnam + "(" p = "" if not fini: - s += "const struct vrt_ctx *" + s += "VRT_CTX" p = ", " if self.pfx != None: s += p + self.pfx @@ -377,7 +377,7 @@ class Func(object): s += " td_" + modname + "_" + self.cnam + "(" p = "" if not fini: - s += "const struct vrt_ctx *" + s += "VRT_CTX" p = ", " if self.pfx != None: s += p + self.pfx @@ -861,7 +861,6 @@ def runmain(inputvcc, outputname="vcc_if"): write_c_file_warning(fc) write_c_file_warning(fh) - fh.write('struct vrt_ctx;\n') fh.write('struct VCL_conf;\n') fh.write('struct vmod_priv;\n') fh.write("\n") diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index b952f64..76e5a5f 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -37,7 +37,7 @@ #include "vcc_if.h" VCL_VOID __match_proto__(td_debug_panic) -vmod_panic(const struct vrt_ctx *ctx, const char *str, ...) +vmod_panic(VRT_CTX, const char *str, ...) { va_list ap; const char *b; @@ -50,7 +50,7 @@ vmod_panic(const struct vrt_ctx *ctx, const char *str, ...) } VCL_STRING __match_proto__(td_debug_author) -vmod_author(const struct vrt_ctx *ctx, VCL_ENUM id) +vmod_author(VRT_CTX, VCL_ENUM id) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -76,7 +76,7 @@ init_function(struct vmod_priv *priv, const struct VCL_conf *cfg) } VCL_VOID __match_proto__(td_debug_test_priv_call) -vmod_test_priv_call(const struct vrt_ctx *ctx, struct vmod_priv *priv) +vmod_test_priv_call(VRT_CTX, struct vmod_priv *priv) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -89,7 +89,7 @@ vmod_test_priv_call(const struct vrt_ctx *ctx, struct vmod_priv *priv) } VCL_VOID __match_proto__(td_debug_test_priv_vcl) -vmod_test_priv_vcl(const struct vrt_ctx *ctx, struct vmod_priv *priv) +vmod_test_priv_vcl(VRT_CTX, struct vmod_priv *priv) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -97,7 +97,7 @@ vmod_test_priv_vcl(const struct vrt_ctx *ctx, struct vmod_priv *priv) } VCL_BLOB -vmod_str2blob(const struct vrt_ctx *ctx, VCL_STRING s) +vmod_str2blob(VRT_CTX, VCL_STRING s) { struct vmod_priv *p; @@ -111,7 +111,7 @@ vmod_str2blob(const struct vrt_ctx *ctx, VCL_STRING s) } VCL_STRING -vmod_blob2hex(const struct vrt_ctx *ctx, VCL_BLOB b) +vmod_blob2hex(VRT_CTX, VCL_BLOB b) { char *s, *p; uint8_t *q; @@ -131,7 +131,7 @@ vmod_blob2hex(const struct vrt_ctx *ctx, VCL_BLOB b) } VCL_BACKEND -vmod_no_backend(const struct vrt_ctx *ctx) +vmod_no_backend(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -139,7 +139,7 @@ vmod_no_backend(const struct vrt_ctx *ctx) } VCL_VOID __match_proto__(td_debug_rot52) -vmod_rot52(const struct vrt_ctx *ctx, VCL_HTTP hp) +vmod_rot52(VRT_CTX, VCL_HTTP hp) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/lib/libvmod_debug/vmod_debug_obj.c index ad24578..91dfa0d 100644 --- a/lib/libvmod_debug/vmod_debug_obj.c +++ b/lib/libvmod_debug/vmod_debug_obj.c @@ -42,7 +42,7 @@ struct vmod_debug_obj { }; VCL_VOID -vmod_obj__init(const struct vrt_ctx *ctx, struct vmod_debug_obj **op, +vmod_obj__init(VRT_CTX, struct vmod_debug_obj **op, const char *vcl_name, VCL_STRING s) { struct vmod_debug_obj *o; @@ -70,7 +70,7 @@ vmod_obj__fini(struct vmod_debug_obj **op) } VCL_VOID __match_proto__() -vmod_obj_enum(const struct vrt_ctx *ctx, struct vmod_debug_obj *o, VCL_ENUM e) +vmod_obj_enum(VRT_CTX, struct vmod_debug_obj *o, VCL_ENUM e) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -79,7 +79,7 @@ vmod_obj_enum(const struct vrt_ctx *ctx, struct vmod_debug_obj *o, VCL_ENUM e) } VCL_STRING __match_proto__() -vmod_obj_foo(const struct vrt_ctx *ctx, struct vmod_debug_obj *o, VCL_STRING s) +vmod_obj_foo(VRT_CTX, struct vmod_debug_obj *o, VCL_STRING s) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -90,7 +90,7 @@ vmod_obj_foo(const struct vrt_ctx *ctx, struct vmod_debug_obj *o, VCL_STRING s) } VCL_TIME __match_proto__() -vmod_obj_date(const struct vrt_ctx *ctx, struct vmod_debug_obj *o) +vmod_obj_date(VRT_CTX, struct vmod_debug_obj *o) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); diff --git a/lib/libvmod_directors/fall_back.c b/lib/libvmod_directors/fall_back.c index a721617..a317e46 100644 --- a/lib/libvmod_directors/fall_back.c +++ b/lib/libvmod_directors/fall_back.c @@ -79,7 +79,7 @@ vmod_fallback_resolve(const struct director *dir, struct worker *wrk, } VCL_VOID __match_proto__() -vmod_fallback__init(const struct vrt_ctx *ctx, +vmod_fallback__init(VRT_CTX, struct vmod_directors_fallback **rrp, const char *vcl_name) { struct vmod_directors_fallback *rr; @@ -107,7 +107,7 @@ vmod_fallback__fini(struct vmod_directors_fallback **rrp) } VCL_VOID __match_proto__() -vmod_fallback_add_backend(const struct vrt_ctx *ctx, +vmod_fallback_add_backend(VRT_CTX, struct vmod_directors_fallback *rr, VCL_BACKEND be) { @@ -117,7 +117,7 @@ vmod_fallback_add_backend(const struct vrt_ctx *ctx, } VCL_BACKEND __match_proto__() -vmod_fallback_backend(const struct vrt_ctx *ctx, +vmod_fallback_backend(VRT_CTX, struct vmod_directors_fallback *rr) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c index 090039f..6ff2a77 100644 --- a/lib/libvmod_directors/hash.c +++ b/lib/libvmod_directors/hash.c @@ -52,7 +52,7 @@ struct vmod_directors_hash { }; VCL_VOID __match_proto__() -vmod_hash__init(const struct vrt_ctx *ctx, struct vmod_directors_hash **rrp, +vmod_hash__init(VRT_CTX, struct vmod_directors_hash **rrp, const char *vcl_name) { struct vmod_directors_hash *rr; @@ -82,7 +82,7 @@ vmod_hash__fini(struct vmod_directors_hash **rrp) } VCL_VOID __match_proto__() -vmod_hash_add_backend(const struct vrt_ctx *ctx, +vmod_hash_add_backend(VRT_CTX, struct vmod_directors_hash *rr, VCL_BACKEND be, double w) { @@ -93,7 +93,7 @@ vmod_hash_add_backend(const struct vrt_ctx *ctx, } VCL_BACKEND __match_proto__() -vmod_hash_backend(const struct vrt_ctx *ctx, struct vmod_directors_hash *rr, +vmod_hash_backend(VRT_CTX, struct vmod_directors_hash *rr, const char *arg, ...) { struct SHA256Context sha_ctx; diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c index 0e7c0a4..d83b840 100644 --- a/lib/libvmod_directors/random.c +++ b/lib/libvmod_directors/random.c @@ -77,7 +77,7 @@ vmod_random_resolve(const struct director *dir, struct worker *wrk, } VCL_VOID __match_proto__() -vmod_random__init(const struct vrt_ctx *ctx, struct vmod_directors_random **rrp, +vmod_random__init(VRT_CTX, struct vmod_directors_random **rrp, const char *vcl_name) { struct vmod_directors_random *rr; @@ -108,7 +108,7 @@ vmod_random__fini(struct vmod_directors_random **rrp) } VCL_VOID __match_proto__() -vmod_random_add_backend(const struct vrt_ctx *ctx, +vmod_random_add_backend(VRT_CTX, struct vmod_directors_random *rr, VCL_BACKEND be, double w) { @@ -119,7 +119,7 @@ vmod_random_add_backend(const struct vrt_ctx *ctx, } VCL_BACKEND __match_proto__() -vmod_random_backend(const struct vrt_ctx *ctx, struct vmod_directors_random *rr) +vmod_random_backend(VRT_CTX, struct vmod_directors_random *rr) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_RANDOM_MAGIC); diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c index a8fba65..1f7d9e8 100644 --- a/lib/libvmod_directors/round_robin.c +++ b/lib/libvmod_directors/round_robin.c @@ -82,7 +82,7 @@ vmod_rr_resolve(const struct director *dir, struct worker *wrk, } VCL_VOID __match_proto__() -vmod_round_robin__init(const struct vrt_ctx *ctx, +vmod_round_robin__init(VRT_CTX, struct vmod_directors_round_robin **rrp, const char *vcl_name) { struct vmod_directors_round_robin *rr; @@ -109,7 +109,7 @@ vmod_round_robin__fini(struct vmod_directors_round_robin **rrp) } VCL_VOID __match_proto__() -vmod_round_robin_add_backend(const struct vrt_ctx *ctx, +vmod_round_robin_add_backend(VRT_CTX, struct vmod_directors_round_robin *rr, VCL_BACKEND be) { @@ -119,7 +119,7 @@ vmod_round_robin_add_backend(const struct vrt_ctx *ctx, } VCL_BACKEND __match_proto__() -vmod_round_robin_backend(const struct vrt_ctx *ctx, +vmod_round_robin_backend(VRT_CTX, struct vmod_directors_round_robin *rr) { diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index ad0066a..a5aec9d 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -49,7 +49,7 @@ #include "vcc_if.h" VCL_VOID __match_proto__(td_std_set_ip_tos) -vmod_set_ip_tos(const struct vrt_ctx *ctx, VCL_INT tos) +vmod_set_ip_tos(VRT_CTX, VCL_INT tos) { int itos = tos; @@ -59,7 +59,7 @@ vmod_set_ip_tos(const struct vrt_ctx *ctx, VCL_INT tos) } static const char * -vmod_updown(const struct vrt_ctx *ctx, int up, const char *s, va_list ap) +vmod_updown(VRT_CTX, int up, const char *s, va_list ap) { unsigned u; char *b, *e; @@ -95,7 +95,7 @@ vmod_updown(const struct vrt_ctx *ctx, int up, const char *s, va_list ap) } VCL_STRING __match_proto__(td_std_toupper) -vmod_toupper(const struct vrt_ctx *ctx, const char *s, ...) +vmod_toupper(VRT_CTX, const char *s, ...) { const char *p; va_list ap; @@ -108,7 +108,7 @@ vmod_toupper(const struct vrt_ctx *ctx, const char *s, ...) } VCL_STRING __match_proto__(td_std_tolower) -vmod_tolower(const struct vrt_ctx *ctx, const char *s, ...) +vmod_tolower(VRT_CTX, const char *s, ...) { const char *p; va_list ap; @@ -121,7 +121,7 @@ vmod_tolower(const struct vrt_ctx *ctx, const char *s, ...) } VCL_REAL __match_proto__(td_std_random) -vmod_random(const struct vrt_ctx *ctx, VCL_REAL lo, VCL_REAL hi) +vmod_random(VRT_CTX, VCL_REAL lo, VCL_REAL hi) { double a; @@ -133,7 +133,7 @@ vmod_random(const struct vrt_ctx *ctx, VCL_REAL lo, VCL_REAL hi) } VCL_VOID __match_proto__(td_std_log) -vmod_log(const struct vrt_ctx *ctx, const char *fmt, ...) +vmod_log(VRT_CTX, const char *fmt, ...) { unsigned u; va_list ap; @@ -154,7 +154,7 @@ vmod_log(const struct vrt_ctx *ctx, const char *fmt, ...) } VCL_VOID __match_proto__(td_std_syslog) -vmod_syslog(const struct vrt_ctx *ctx, VCL_INT fac, const char *fmt, ...) +vmod_syslog(VRT_CTX, VCL_INT fac, const char *fmt, ...) { unsigned u; va_list ap; @@ -172,7 +172,7 @@ vmod_syslog(const struct vrt_ctx *ctx, VCL_INT fac, const char *fmt, ...) } VCL_VOID __match_proto__(td_std_collect) -vmod_collect(const struct vrt_ctx *ctx, VCL_HEADER hdr) +vmod_collect(VRT_CTX, VCL_HEADER hdr) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -187,7 +187,7 @@ vmod_collect(const struct vrt_ctx *ctx, VCL_HEADER hdr) } VCL_BOOL __match_proto__(td_std_healthy) -vmod_healthy(const struct vrt_ctx *ctx, VCL_BACKEND be) +vmod_healthy(VRT_CTX, VCL_BACKEND be) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (be == NULL) @@ -197,7 +197,7 @@ vmod_healthy(const struct vrt_ctx *ctx, VCL_BACKEND be) } VCL_INT __match_proto__(td_std_port) -vmod_port(const struct vrt_ctx *ctx, VCL_IP ip) +vmod_port(VRT_CTX, VCL_IP ip) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (ip == NULL) @@ -206,13 +206,13 @@ vmod_port(const struct vrt_ctx *ctx, VCL_IP ip) } VCL_VOID __match_proto__(td_std_rollback) -vmod_rollback(const struct vrt_ctx *ctx, VCL_HTTP hp) +vmod_rollback(VRT_CTX, VCL_HTTP hp) { VRT_Rollback(ctx, hp); } VCL_VOID __match_proto__(td_std_timestamp) -vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label) +vmod_timestamp(VRT_CTX, VCL_STRING label) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -232,7 +232,7 @@ vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label) } VCL_VOID __match_proto__(td_std_cache_req_body) -vmod_cache_req_body(const struct vrt_ctx *ctx, VCL_BYTES size) +vmod_cache_req_body(VRT_CTX, VCL_BYTES size) { int result; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -241,7 +241,7 @@ vmod_cache_req_body(const struct vrt_ctx *ctx, VCL_BYTES size) } VCL_STRING __match_proto__(td_std_strstr) -vmod_strstr(const struct vrt_ctx *ctx, VCL_STRING mstr, VCL_STRING msubstr) +vmod_strstr(VRT_CTX, VCL_STRING mstr, VCL_STRING msubstr) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index cdcd3cb..d050f43 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -44,7 +44,7 @@ #include "vcc_if.h" VCL_DURATION __match_proto__(td_std_duration) -vmod_duration(const struct vrt_ctx *ctx, VCL_STRING p, VCL_DURATION d) +vmod_duration(VRT_CTX, VCL_STRING p, VCL_DURATION d) { char *e; double r; @@ -101,7 +101,7 @@ vmod_duration(const struct vrt_ctx *ctx, VCL_STRING p, VCL_DURATION d) } VCL_INT __match_proto__(td_std_integer) -vmod_integer(const struct vrt_ctx *ctx, VCL_STRING p, VCL_INT i) +vmod_integer(VRT_CTX, VCL_STRING p, VCL_INT i) { char *e; long r; @@ -128,7 +128,7 @@ vmod_integer(const struct vrt_ctx *ctx, VCL_STRING p, VCL_INT i) } VCL_IP -vmod_ip(const struct vrt_ctx *ctx, VCL_STRING s, VCL_IP d) +vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d) { struct addrinfo hints, *res0 = NULL; const struct addrinfo *res; @@ -167,7 +167,7 @@ vmod_ip(const struct vrt_ctx *ctx, VCL_STRING s, VCL_IP d) } VCL_REAL __match_proto__(td_std_real) -vmod_real(const struct vrt_ctx *ctx, VCL_STRING p, VCL_REAL d) +vmod_real(VRT_CTX, VCL_STRING p, VCL_REAL d) { char *e; double r; @@ -197,7 +197,7 @@ vmod_real(const struct vrt_ctx *ctx, VCL_STRING p, VCL_REAL d) } VCL_TIME __match_proto__(td_std_real2time) -vmod_real2time(const struct vrt_ctx *ctx, VCL_REAL r) +vmod_real2time(VRT_CTX, VCL_REAL r) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -205,7 +205,7 @@ vmod_real2time(const struct vrt_ctx *ctx, VCL_REAL r) } VCL_INT __match_proto__(td_std_time2integer) -vmod_time2integer(const struct vrt_ctx *ctx, VCL_TIME t) +vmod_time2integer(VRT_CTX, VCL_TIME t) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -213,7 +213,7 @@ vmod_time2integer(const struct vrt_ctx *ctx, VCL_TIME t) } VCL_REAL __match_proto__(td_std_time2real) -vmod_time2real(const struct vrt_ctx *ctx, VCL_TIME t) +vmod_time2real(VRT_CTX, VCL_TIME t) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); diff --git a/lib/libvmod_std/vmod_std_fileread.c b/lib/libvmod_std/vmod_std_fileread.c index 4fce9a2..0e37268 100644 --- a/lib/libvmod_std/vmod_std_fileread.c +++ b/lib/libvmod_std/vmod_std_fileread.c @@ -83,7 +83,7 @@ free_frfile(void *ptr) } VCL_STRING __match_proto__(td_std_fileread) -vmod_fileread(const struct vrt_ctx *ctx, struct vmod_priv *priv, +vmod_fileread(VRT_CTX, struct vmod_priv *priv, VCL_STRING file_name) { struct frfile *frf = NULL; diff --git a/lib/libvmod_std/vmod_std_querysort.c b/lib/libvmod_std/vmod_std_querysort.c index 8f7f93b..6f6a90b 100644 --- a/lib/libvmod_std/vmod_std_querysort.c +++ b/lib/libvmod_std/vmod_std_querysort.c @@ -50,7 +50,7 @@ compa(const void *a, const void *b) } VCL_STRING __match_proto__(td_std_querysort) -vmod_querysort(const struct vrt_ctx *ctx, VCL_STRING url) +vmod_querysort(VRT_CTX, VCL_STRING url) { const char *cq, *cu; char *p, *r; From martin at varnish-software.com Thu Sep 25 13:02:28 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 25 Sep 2014 15:02:28 +0200 Subject: [master] 429966c Cast y argument to uintmax_t in RDN2/RUP2 Message-ID: commit 429966cd9a7b9f36068589d87cef013b539740ad Author: Martin Blix Grydeland Date: Wed Sep 24 17:07:07 2014 +0200 Cast y argument to uintmax_t in RDN2/RUP2 If the type of the y argument to RDN2/RUP2 is smaller than the type of the x argument, these macros would fail to round correctly. Cast to uintmax_t to avoid this problem. diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h index 17b68b1..8295394 100644 --- a/bin/varnishd/common/common.h +++ b/bin/varnishd/common/common.h @@ -128,8 +128,8 @@ void VSM_common_ageupdate(struct vsm_sc *sc); */ #define PWR2(x) ((((x)-1)&(x))==0) /* Is a power of two */ -#define RDN2(x, y) ((x)&(~((y)-1))) /* if y is powers of two */ -#define RUP2(x, y) (((x)+((y)-1))&(~((y)-1))) /* if y is powers of two */ +#define RDN2(x, y) ((x)&(~((uintmax_t)(y)-1))) /* PWR2(y) true */ +#define RUP2(x, y) (((x)+((y)-1))&(~((uintmax_t)(y)-1))) /* PWR2(y) true */ /*-------------------------------------------------------------------- * Pointer aligment magic From martin at varnish-software.com Fri Sep 26 15:05:55 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Fri, 26 Sep 2014 17:05:55 +0200 Subject: [master] 181fed0 Prevent a segfault on varnish reconnect in the varnishstat hitrate implementation Message-ID: commit 181fed08c7878e606e666c556211f061d53ed2d6 Author: Martin Blix Grydeland Date: Fri Sep 26 17:05:24 2014 +0200 Prevent a segfault on varnish reconnect in the varnishstat hitrate implementation diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index 4d63e71..00c4dac 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -414,7 +414,7 @@ sample_hitrate(void) double hr, mr, ratio; uint64_t hit, miss; - if (VSC_C_mgt == NULL) + if (VSC_C_main == NULL) return; tv = VTIM_mono(); @@ -548,17 +548,8 @@ draw_status(void) werase(w_status); - if (VSC_C_mgt != NULL) { + if (VSC_C_mgt != NULL) up_mgt = VSC_C_mgt->uptime; - if( COLS > 70) { - mvwprintw(w_status, 0, (getmaxx (w_status) - 37), - "Hitrate n: %8u %8u %8u", hitrate.hr_10.n, hitrate.hr_100.n, - hitrate.hr_1000.n); - mvwprintw(w_status, 1, (getmaxx (w_status) - 37), - " avg(n): %8.4f %8.4f %8.4f", hitrate.hr_10.acc, - hitrate.hr_100.acc, hitrate.hr_1000.acc); - } - } if (VSC_C_main != NULL) up_chld = VSC_C_main->uptime; @@ -571,6 +562,14 @@ draw_status(void) if (VSC_C_mgt == NULL) mvwprintw(w_status, 0, COLS - strlen(discon), discon); + else if (COLS > 70) { + mvwprintw(w_status, 0, getmaxx(w_status) - 37, + "Hitrate n: %8u %8u %8u", hitrate.hr_10.n, hitrate.hr_100.n, + hitrate.hr_1000.n); + mvwprintw(w_status, 1, getmaxx(w_status) - 37, + " avg(n): %8.4f %8.4f %8.4f", hitrate.hr_10.acc, + hitrate.hr_100.acc, hitrate.hr_1000.acc); + } wnoutrefresh(w_status); } From martin at varnish-software.com Fri Sep 26 15:10:16 2014 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Fri, 26 Sep 2014 17:10:16 +0200 Subject: [4.0] 7bebd3f Prevent a segfault on varnish reconnect in the varnishstat hitrate implementation Message-ID: commit 7bebd3f7813603b5a48c1a44662c6c9deaaed478 Author: Martin Blix Grydeland Date: Fri Sep 26 17:05:24 2014 +0200 Prevent a segfault on varnish reconnect in the varnishstat hitrate implementation diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index 4d63e71..00c4dac 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -414,7 +414,7 @@ sample_hitrate(void) double hr, mr, ratio; uint64_t hit, miss; - if (VSC_C_mgt == NULL) + if (VSC_C_main == NULL) return; tv = VTIM_mono(); @@ -548,17 +548,8 @@ draw_status(void) werase(w_status); - if (VSC_C_mgt != NULL) { + if (VSC_C_mgt != NULL) up_mgt = VSC_C_mgt->uptime; - if( COLS > 70) { - mvwprintw(w_status, 0, (getmaxx (w_status) - 37), - "Hitrate n: %8u %8u %8u", hitrate.hr_10.n, hitrate.hr_100.n, - hitrate.hr_1000.n); - mvwprintw(w_status, 1, (getmaxx (w_status) - 37), - " avg(n): %8.4f %8.4f %8.4f", hitrate.hr_10.acc, - hitrate.hr_100.acc, hitrate.hr_1000.acc); - } - } if (VSC_C_main != NULL) up_chld = VSC_C_main->uptime; @@ -571,6 +562,14 @@ draw_status(void) if (VSC_C_mgt == NULL) mvwprintw(w_status, 0, COLS - strlen(discon), discon); + else if (COLS > 70) { + mvwprintw(w_status, 0, getmaxx(w_status) - 37, + "Hitrate n: %8u %8u %8u", hitrate.hr_10.n, hitrate.hr_100.n, + hitrate.hr_1000.n); + mvwprintw(w_status, 1, getmaxx(w_status) - 37, + " avg(n): %8.4f %8.4f %8.4f", hitrate.hr_10.acc, + hitrate.hr_100.acc, hitrate.hr_1000.acc); + } wnoutrefresh(w_status); } From nils.goroll at uplex.de Sun Sep 28 15:53:12 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 28 Sep 2014 17:53:12 +0200 Subject: [master] 04a8293 Fix boolean logic in http_GetHdrToken Message-ID: commit 04a8293c8c51e5518965f8a52e2acf2e721d542c Author: Nils Goroll Date: Sun Sep 28 17:48:36 2014 +0200 Fix boolean logic in http_GetHdrToken For quoted tokens, http_GetHdrToken should not try a case insensitive match if the sensitive match fails diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 031b7dd..15605e1 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -443,8 +443,9 @@ http_GetHdrToken(const struct http *hp, const char *hdr, } /* Check for substrings before memcmp() */ if ((h + fl == e || vct_issepctl(h[fl])) && - ((quoted && !memcmp(h, token, fl)) || - !strncasecmp(h, token, fl))) { + (quoted + ? !memcmp(h, token, fl) + : !strncasecmp(h, token, fl))) { if (ptr != NULL) { h += fl; while (vct_islws(*h)) From phk at FreeBSD.org Sun Sep 28 17:53:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 28 Sep 2014 19:53:46 +0200 Subject: [master] 2fd2a78 Move Tadd() to the only source file which uses it Message-ID: commit 2fd2a78dd21c303602e7d01a5f8d6763bb2aa5bf Author: Poul-Henning Kamp Date: Sun Sep 28 16:25:52 2014 +0000 Move Tadd() to the only source file which uses it diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 238baa1..f3a05be 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -1206,20 +1206,6 @@ Tlen(const txt t) return ((unsigned)(t.e - t.b)); } -static inline void -Tadd(txt *t, const char *p, int l) -{ - Tcheck(*t); - - if (l <= 0) { - } if (t->b + l < t->e) { - memcpy(t->b, p, l); - t->b += l; - } else { - t->b = t->e; - } -} - /* * We want to cache the most recent timestamp in wrk->lastused to avoid * extra timestamps in cache_pool.c. Hide this detail with a macro diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c index d198d1f..2727ba7 100644 --- a/bin/varnishd/cache/cache_vrt_re.c +++ b/bin/varnishd/cache/cache_vrt_re.c @@ -38,6 +38,20 @@ #include "vre.h" #include "vrt.h" +static void +Tadd(txt *t, const char *p, int l) +{ + Tcheck(*t); + + if (l <= 0) { + } if (t->b + l < t->e) { + memcpy(t->b, p, l); + t->b += l; + } else { + t->b = t->e; + } +} + void VRT_re_init(void **rep, const char *re) { From phk at FreeBSD.org Sun Sep 28 17:53:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 28 Sep 2014 19:53:46 +0200 Subject: [master] d53f0f7 Convert htc->pipeline from txt to two char* pointers Message-ID: commit d53f0f7febdd7534cec775149fe081b3cad5fafb Author: Poul-Henning Kamp Date: Sun Sep 28 16:42:00 2014 +0000 Convert htc->pipeline from txt to two char* pointers diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index f3a05be..ccbf45b 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -248,7 +248,8 @@ struct http_conn { unsigned maxhdr; struct ws *ws; txt rxbuf; - txt pipeline; + char *pipeline_b; + char *pipeline_e; ssize_t content_length; enum body_status body_status; struct vfp_ctx vfc[1]; diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c index 49047d3..04f1125 100644 --- a/bin/varnishd/cache/cache_http1_proto.c +++ b/bin/varnishd/cache/cache_http1_proto.c @@ -75,8 +75,8 @@ HTTP1_Init(struct http_conn *htc, struct ws *ws, int fd, struct vsl_log *vsl, htc->rxbuf.b = ws->f; htc->rxbuf.e = ws->f; *htc->rxbuf.e = '\0'; - htc->pipeline.b = NULL; - htc->pipeline.e = NULL; + htc->pipeline_b = NULL; + htc->pipeline_e = NULL; } /*-------------------------------------------------------------------- @@ -88,18 +88,19 @@ HTTP1_Init(struct http_conn *htc, struct ws *ws, int fd, struct vsl_log *vsl, enum http1_status_e HTTP1_Reinit(struct http_conn *htc) { - unsigned l; + ssize_t l; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); (void)WS_Reserve(htc->ws, htc->maxbytes); htc->rxbuf.b = htc->ws->f; htc->rxbuf.e = htc->ws->f; - if (htc->pipeline.b != NULL) { - l = Tlen(htc->pipeline); - memmove(htc->rxbuf.b, htc->pipeline.b, l); + if (htc->pipeline_b != NULL) { + l = htc->pipeline_e - htc->pipeline_b; + assert(l > 0); + memmove(htc->rxbuf.b, htc->pipeline_b, l); htc->rxbuf.e += l; - htc->pipeline.b = NULL; - htc->pipeline.e = NULL; + htc->pipeline_b = NULL; + htc->pipeline_e = NULL; } *htc->rxbuf.e = '\0'; return (HTTP1_Complete(htc)); @@ -116,8 +117,8 @@ HTTP1_Complete(struct http_conn *htc) txt *t; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); - AZ(htc->pipeline.b); - AZ(htc->pipeline.e); + AZ(htc->pipeline_b); + AZ(htc->pipeline_e); t = &htc->rxbuf; Tcheck(*t); @@ -145,8 +146,8 @@ HTTP1_Complete(struct http_conn *htc) p++; WS_ReleaseP(htc->ws, t->e); if (p < t->e) { - htc->pipeline.b = p; - htc->pipeline.e = t->e; + htc->pipeline_b = p; + htc->pipeline_e = t->e; t->e = p; } return (HTTP1_COMPLETE); @@ -163,8 +164,8 @@ HTTP1_Rx(struct http_conn *htc) CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); AN(htc->ws->r); - AZ(htc->pipeline.b); - AZ(htc->pipeline.e); + AZ(htc->pipeline_b); + AZ(htc->pipeline_e); i = (htc->ws->r - htc->rxbuf.e) - 1; /* space for NUL */ if (i <= 0) { WS_ReleaseP(htc->ws, htc->rxbuf.b); @@ -191,23 +192,24 @@ HTTP1_Rx(struct http_conn *htc) ssize_t HTTP1_Read(struct http_conn *htc, void *d, size_t len) { - size_t l; + ssize_t l; unsigned char *p; ssize_t i = 0; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); l = 0; p = d; - if (htc->pipeline.b) { - l = Tlen(htc->pipeline); + if (htc->pipeline_b) { + l = htc->pipeline_e - htc->pipeline_b; + assert(l > 0); if (l > len) l = len; - memcpy(p, htc->pipeline.b, l); + memcpy(p, htc->pipeline_b, l); p += l; len -= l; - htc->pipeline.b += l; - if (htc->pipeline.b == htc->pipeline.e) - htc->pipeline.b = htc->pipeline.e = NULL; + htc->pipeline_b += l; + if (htc->pipeline_b == htc->pipeline_e) + htc->pipeline_b = htc->pipeline_e = NULL; } if (len > 0) { i = read(htc->fd, p, len); diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c index 9d43be0..2e2d57d 100644 --- a/bin/varnishd/cache/cache_pipe.c +++ b/bin/varnishd/cache/cache_pipe.c @@ -126,9 +126,9 @@ PipeRequest(struct req *req, struct busyobj *bo) WRW_Reserve(wrk, &vc->fd, bo->vsl, req->t_req); hdrbytes = HTTP1_Write(wrk, bo->bereq, HTTP1_Req); - if (req->htc->pipeline.b != NULL) - (void)WRW_Write(wrk, req->htc->pipeline.b, - Tlen(req->htc->pipeline)); + if (req->htc->pipeline_b != NULL) + (void)WRW_Write(wrk, req->htc->pipeline_b, + req->htc->pipeline_e - req->htc->pipeline_b); i = WRW_FlushRelease(wrk, &acct_pipe.bereq); if (acct_pipe.bereq > hdrbytes) { From phk at FreeBSD.org Sun Sep 28 17:53:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 28 Sep 2014 19:53:46 +0200 Subject: [master] 49af0c1 Convert htc->rxbuf from txt to dual char * Message-ID: commit 49af0c1eafd47484fd6e6e9828ebab08dfb7b5ce Author: Poul-Henning Kamp Date: Sun Sep 28 17:05:51 2014 +0000 Convert htc->rxbuf from txt to dual char * diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index ccbf45b..fda22a1 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -247,7 +247,8 @@ struct http_conn { unsigned maxbytes; unsigned maxhdr; struct ws *ws; - txt rxbuf; + char *rxbuf_b; + char *rxbuf_e; char *pipeline_b; char *pipeline_e; ssize_t content_length; diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c index de12e88..ca369d2 100644 --- a/bin/varnishd/cache/cache_http1_fetch.c +++ b/bin/varnishd/cache/cache_http1_fetch.c @@ -165,7 +165,8 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo) do { hs = HTTP1_Rx(htc); if (hs == HTTP1_OVERFLOW) { - bo->acct.beresp_hdrbytes += Tlen(htc->rxbuf); + bo->acct.beresp_hdrbytes += + htc->rxbuf_e - htc->rxbuf_b; VSLb(bo->vsl, SLT_FetchError, "http %sread error: overflow", first ? "first " : ""); @@ -174,7 +175,8 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo) return (-1); } if (hs == HTTP1_ERROR_EOF) { - bo->acct.beresp_hdrbytes += Tlen(htc->rxbuf); + bo->acct.beresp_hdrbytes += + htc->rxbuf_e - htc->rxbuf_b; VSLb(bo->vsl, SLT_FetchError, "http %sread error: EOF", first ? "first " : ""); VBE_CloseFd(&bo->vbc, &bo->acct); @@ -188,7 +190,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo) vc->between_bytes_timeout); } } while (hs != HTTP1_COMPLETE); - bo->acct.beresp_hdrbytes += Tlen(htc->rxbuf); + bo->acct.beresp_hdrbytes += htc->rxbuf_e - htc->rxbuf_b; hp = bo->beresp; diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 0f68cd1..3bbe277 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -124,7 +124,8 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req) req->t_first = now; if (isnan(req->t_req)) req->t_req = now; - req->acct.req_hdrbytes += Tlen(req->htc->rxbuf); + req->acct.req_hdrbytes += + req->htc->rxbuf_e - req->htc->rxbuf_b; return (REQ_FSM_MORE); } else if (hs == HTTP1_ERROR_EOF) { why = SC_REM_CLOSE; @@ -160,7 +161,7 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req) } } } - req->acct.req_hdrbytes += Tlen(req->htc->rxbuf); + req->acct.req_hdrbytes += req->htc->rxbuf_e - req->htc->rxbuf_b; CNT_AcctLogCharge(wrk->stats, req); SES_ReleaseReq(req); assert(why != SC_NULL); @@ -240,10 +241,11 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req) AZ(req->vsl->wid); req->t_first = req->t_req = sp->t_idle; wrk->stats->sess_pipeline++; - req->acct.req_hdrbytes += Tlen(req->htc->rxbuf); + req->acct.req_hdrbytes += + req->htc->rxbuf_e - req->htc->rxbuf_b; return (SESS_DONE_RET_START); } else { - if (Tlen(req->htc->rxbuf)) + if (req->htc->rxbuf_e != req->htc->rxbuf_b) wrk->stats->sess_readahead++; return (SESS_DONE_RET_WAIT); } @@ -286,7 +288,9 @@ http1_dissect(struct worker *wrk, struct req *req) /* If we could not even parse the request, just close */ if (req->err_code != 0) { - VSLbt(req->vsl, SLT_HttpGarbage, req->htc->rxbuf); + VSLb(req->vsl, SLT_HttpGarbage, "%.*s", + (int)(req->htc->rxbuf_e - req->htc->rxbuf_b), + req->htc->rxbuf_b); wrk->stats->client_req_400++; r = write(req->sp->fd, r_400, strlen(r_400)); if (r > 0) diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c index 04f1125..ded7702 100644 --- a/bin/varnishd/cache/cache_http1_proto.c +++ b/bin/varnishd/cache/cache_http1_proto.c @@ -72,9 +72,9 @@ HTTP1_Init(struct http_conn *htc, struct ws *ws, int fd, struct vsl_log *vsl, htc->maxhdr = maxhdr; (void)WS_Reserve(htc->ws, htc->maxbytes); - htc->rxbuf.b = ws->f; - htc->rxbuf.e = ws->f; - *htc->rxbuf.e = '\0'; + htc->rxbuf_b = ws->f; + htc->rxbuf_e = ws->f; + *htc->rxbuf_e = '\0'; htc->pipeline_b = NULL; htc->pipeline_e = NULL; } @@ -92,17 +92,17 @@ HTTP1_Reinit(struct http_conn *htc) CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); (void)WS_Reserve(htc->ws, htc->maxbytes); - htc->rxbuf.b = htc->ws->f; - htc->rxbuf.e = htc->ws->f; + htc->rxbuf_b = htc->ws->f; + htc->rxbuf_e = htc->ws->f; if (htc->pipeline_b != NULL) { l = htc->pipeline_e - htc->pipeline_b; assert(l > 0); - memmove(htc->rxbuf.b, htc->pipeline_b, l); - htc->rxbuf.e += l; + memmove(htc->rxbuf_b, htc->pipeline_b, l); + htc->rxbuf_e += l; htc->pipeline_b = NULL; htc->pipeline_e = NULL; } - *htc->rxbuf.e = '\0'; + *htc->rxbuf_e = '\0'; return (HTTP1_Complete(htc)); } @@ -114,23 +114,21 @@ enum http1_status_e HTTP1_Complete(struct http_conn *htc) { char *p; - txt *t; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); AZ(htc->pipeline_b); AZ(htc->pipeline_e); - t = &htc->rxbuf; - Tcheck(*t); - assert(*t->e == '\0'); + assert(htc->rxbuf_e >= htc->rxbuf_b); + assert(*htc->rxbuf_e == '\0'); /* Skip any leading white space */ - for (p = t->b ; vct_islws(*p); p++) + for (p = htc->rxbuf_b ; vct_islws(*p); p++) continue; - if (p == t->e) { + if (p == htc->rxbuf_e) { /* All white space */ - t->e = t->b; - *t->e = '\0'; + htc->rxbuf_e = htc->rxbuf_b; + *htc->rxbuf_e = '\0'; return (HTTP1_ALL_WHITESPACE); } while (1) { @@ -144,11 +142,11 @@ HTTP1_Complete(struct http_conn *htc) break; } p++; - WS_ReleaseP(htc->ws, t->e); - if (p < t->e) { + WS_ReleaseP(htc->ws, htc->rxbuf_e); + if (p < htc->rxbuf_e) { htc->pipeline_b = p; - htc->pipeline_e = t->e; - t->e = p; + htc->pipeline_e = htc->rxbuf_e; + htc->rxbuf_e = p; } return (HTTP1_COMPLETE); } @@ -166,22 +164,22 @@ HTTP1_Rx(struct http_conn *htc) AN(htc->ws->r); AZ(htc->pipeline_b); AZ(htc->pipeline_e); - i = (htc->ws->r - htc->rxbuf.e) - 1; /* space for NUL */ + i = (htc->ws->r - htc->rxbuf_e) - 1; /* space for NUL */ if (i <= 0) { - WS_ReleaseP(htc->ws, htc->rxbuf.b); + WS_ReleaseP(htc->ws, htc->rxbuf_b); return (HTTP1_OVERFLOW); } - i = read(htc->fd, htc->rxbuf.e, i); + i = read(htc->fd, htc->rxbuf_e, i); if (i <= 0) { /* * We wouldn't come here if we had a complete HTTP header * so consequently an EOF can not be OK */ - WS_ReleaseP(htc->ws, htc->rxbuf.b); + WS_ReleaseP(htc->ws, htc->rxbuf_b); return (HTTP1_ERROR_EOF); } - htc->rxbuf.e += i; - *htc->rxbuf.e = '\0'; + htc->rxbuf_e += i; + *htc->rxbuf_e = '\0'; return (HTTP1_Complete(htc)); } @@ -230,24 +228,25 @@ static uint16_t http1_dissect_hdrs(struct http *hp, char *p, const struct http_conn *htc) { char *q, *r; - txt t = htc->rxbuf; + assert(p > htc->rxbuf_b); + assert(p < htc->rxbuf_e); hp->nhd = HTTP_HDR_FIRST; hp->conds = 0; r = NULL; /* For FlexeLint */ - for (; p < t.e; p = r) { + for (; p < htc->rxbuf_e; p = r) { /* Find end of next header */ q = r = p; - while (r < t.e) { + while (r < htc->rxbuf_e) { if (!vct_iscrlf(r)) { r++; continue; } q = r; - assert(r < t.e); + assert(r < htc->rxbuf_e); r += vct_skipcrlf(r); - if (r >= t.e) + if (r >= htc->rxbuf_e) break; /* If line does not continue: got it. */ if (!vct_issp(*r)) @@ -316,14 +315,14 @@ http1_splitline(struct http *hp, const struct http_conn *htc, const int *hf) assert(hf == HTTP1_Req || hf == HTTP1_Resp); CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); - Tcheck(htc->rxbuf); + assert(htc->rxbuf_e >= htc->rxbuf_b); AZ(hp->hd[hf[0]].b); AZ(hp->hd[hf[1]].b); AZ(hp->hd[hf[2]].b); /* Skip leading LWS */ - for (p = htc->rxbuf.b ; vct_islws(*p); p++) + for (p = htc->rxbuf_b ; vct_islws(*p); p++) continue; hp->hd[hf[0]].b = p; @@ -536,7 +535,8 @@ HTTP1_DissectResponse(struct http *hp, struct http_conn *htc) } if (retval != 0) { - VSLbt(hp->vsl, SLT_HttpGarbage, htc->rxbuf); + VSLb(hp->vsl, SLT_HttpGarbage, "%.*s", + (int)(htc->rxbuf_e - htc->rxbuf_b), htc->rxbuf_b); assert(retval >= 100 && retval <= 999); assert(retval == 503); hp->status = retval; From phk at FreeBSD.org Sun Sep 28 17:53:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 28 Sep 2014 19:53:46 +0200 Subject: [master] 2b6d5a5 Make the various http_Get* functions return const char * to what they find. Message-ID: commit 2b6d5a5f5e7e2064e14aa54162909ae5bffd8416 Author: Poul-Henning Kamp Date: Sun Sep 28 17:22:51 2014 +0000 Make the various http_Get* functions return const char * to what they find. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index fda22a1..a92b011 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -876,11 +876,11 @@ void http_SetH(const struct http *to, unsigned n, const char *fm); void http_ForceField(const struct http *to, unsigned n, const char *t); void HTTP_Setup(struct http *, struct ws *, struct vsl_log *, enum VSL_tag_e); void http_Teardown(struct http *ht); -int http_GetHdr(const struct http *hp, const char *hdr, char **ptr); +int http_GetHdr(const struct http *hp, const char *hdr, const char **ptr); int http_GetHdrToken(const struct http *hp, const char *hdr, - const char *token, char **ptr); + const char *token, const char **ptr); int http_GetHdrField(const struct http *hp, const char *hdr, - const char *field, char **ptr); + const char *field, const char **ptr); double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field); ssize_t http_GetContentLength(const struct http *hp); uint16_t http_GetStatus(const struct http *hp); diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index daa040f..e0a765f 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -832,7 +832,7 @@ ban_evaluate(struct worker *wrk, const uint8_t *bs, struct objcore *oc, { struct ban_test bt; const uint8_t *be; - char *p; + const char *p; const char *arg1; be = bs + ban_len(bs); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index f680f02..8ac532a 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -94,7 +94,7 @@ static int vbf_beresp2obj(struct busyobj *bo) { unsigned l, l2; - char *b; + const char *b; uint8_t *bp; struct vsb *vary = NULL; int varyl = 0; @@ -501,7 +501,7 @@ vbf_fetch_body_helper(struct busyobj *bo) static enum fetch_step vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { - char *p; + const char *p; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 15605e1..8462ceb 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -377,7 +377,7 @@ http_CollectHdr(struct http *hp, const char *hdr) /*--------------------------------------------------------------------*/ int -http_GetHdr(const struct http *hp, const char *hdr, char **ptr) +http_GetHdr(const struct http *hp, const char *hdr, const char **ptr) { unsigned u, l; char *p; @@ -420,9 +420,9 @@ http_GetHdr(const struct http *hp, const char *hdr, char **ptr) int http_GetHdrToken(const struct http *hp, const char *hdr, - const char *token, char **ptr) + const char *token, const char **ptr) { - char *h, *e; + const char *h, *e; unsigned fl; int quoted; @@ -468,7 +468,7 @@ http_GetHdrToken(const struct http *hp, const char *hdr, double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field) { - char *h; + const char *h; int i; double a, b; @@ -517,9 +517,9 @@ http_GetHdrQ(const struct http *hp, const char *hdr, const char *field) int http_GetHdrField(const struct http *hp, const char *hdr, - const char *field, char **ptr) + const char *field, const char **ptr) { - char *h; + const char *h; int i; if (ptr != NULL) @@ -550,7 +550,7 @@ ssize_t http_GetContentLength(const struct http *hp) { ssize_t cl, cll; - char *b; + const char *b; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); @@ -579,7 +579,7 @@ http_GetContentLength(const struct http *hp) enum sess_close http_DoConnection(struct http *hp) { - char *p, *q; + const char *p, *q; enum sess_close retval; unsigned u; @@ -618,7 +618,7 @@ http_DoConnection(struct http *hp) int http_HdrIs(const struct http *hp, const char *hdr, const char *val) { - char *p; + const char *p; if (!http_GetHdr(hp, hdr, &p)) return (0); diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c index 3eb10d6..766e493 100644 --- a/bin/varnishd/cache/cache_http1_deliver.c +++ b/bin/varnishd/cache/cache_http1_deliver.c @@ -230,7 +230,7 @@ V1D_FlushReleaseAcct(struct req *req) void V1D_Deliver(struct req *req, struct busyobj *bo) { - char *r; + const char *r; enum objiter_status ois; ssize_t l; @@ -369,7 +369,7 @@ V1D_Deliver(struct req *req, struct busyobj *bo) void V1D_Deliver_Synth(struct req *req) { - char *r; + const char *r; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AN(req->synth_body); diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 3bbe277..8e4508a 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -260,7 +260,7 @@ http1_dissect(struct worker *wrk, struct req *req) const char *r_100 = "HTTP/1.1 100 Continue\r\n\r\n"; const char *r_400 = "HTTP/1.1 400 Bad Request\r\n\r\n"; const char *r_417 = "HTTP/1.1 417 Expectation Failed\r\n\r\n"; - char *p; + const char *p; ssize_t r; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c index ded7702..6bfa53b 100644 --- a/bin/varnishd/cache/cache_http1_proto.c +++ b/bin/varnishd/cache/cache_http1_proto.c @@ -382,7 +382,7 @@ static enum body_status http1_body_status(const struct http *hp, struct http_conn *htc) { ssize_t cl; - char *b; + const char *b; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 3d20e43..bba2dea 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -680,7 +680,7 @@ cnt_recv(struct worker *wrk, struct req *req) { unsigned recv_handling; struct SHA256Context sha256ctx; - char *xff; + const char *xff; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 99de23f..7c086b9 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -67,7 +67,7 @@ RFC2616_Ttl(struct busyobj *bo, double now) { unsigned max_age, age; double h_date, h_expires; - char *p; + const char *p; const struct http *hp; struct exp *expp; @@ -214,7 +214,7 @@ RFC2616_Req_Gzip(const struct http *hp) int RFC2616_Do_Cond(const struct req *req) { - char *p, *e; + const char *p, *e; double ims, lm; int do_cond = 0; @@ -246,7 +246,7 @@ RFC2616_Do_Cond(const struct req *req) void RFC2616_Weaken_Etag(struct http *hp) { - char *p; + const char *p; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); @@ -264,7 +264,7 @@ RFC2616_Weaken_Etag(struct http *hp) void RFC2616_Vary_AE(struct http *hp) { - char *vary; + const char *vary; if (http_GetHdrToken(hp, H_Vary, "Accept-Encoding", NULL)) return; diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c index 7c783af..492af99 100644 --- a/bin/varnishd/cache/cache_vary.c +++ b/bin/varnishd/cache/cache_vary.c @@ -77,7 +77,7 @@ static unsigned VRY_Validate(const uint8_t *vary); int VRY_Create(struct busyobj *bo, struct vsb **psb) { - char *v, *p, *q, *h, *e; + const char *v, *p, *q, *h, *e; struct vsb *sb, *sbh; unsigned l; int error = 0; @@ -286,7 +286,7 @@ int VRY_Match(struct req *req, const uint8_t *vary) { uint8_t *vsp = req->vary_b; - char *h, *e; + const char *h, *e; unsigned lh, ln; int i, oflo = 0; diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index b45cdf9..c41bcaa 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -120,7 +120,7 @@ vrt_selecthttp(VRT_CTX, enum gethdr_e where) const char * VRT_GetHdr(VRT_CTX, const struct gethdr_s *hs) { - char *p; + const char *p; struct http *hp; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); From phk at FreeBSD.org Sun Sep 28 17:53:46 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 28 Sep 2014 19:53:46 +0200 Subject: [master] d5bc442 Constify txt. Message-ID: commit d5bc442e0511c19f9ce2db3301a2e66e8aa71545 Author: Poul-Henning Kamp Date: Sun Sep 28 17:53:27 2014 +0000 Constify txt. This is all Slink's fault. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index a92b011..39359b4 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -130,8 +130,8 @@ struct wrw; /*--------------------------------------------------------------------*/ typedef struct { - char *b; - char *e; + const char *b; + const char *e; } txt; /*--------------------------------------------------------------------*/ @@ -867,7 +867,7 @@ void http_PutResponse(struct http *to, const char *proto, uint16_t status, const char *response); void http_FilterReq(struct http *to, const struct http *fm, unsigned how); void HTTP_Encode(const struct http *fm, uint8_t *, unsigned len, unsigned how); -int HTTP_Decode(struct http *to, uint8_t *fm); +int HTTP_Decode(struct http *to, const uint8_t *fm); void http_ForceHeader(struct http *to, const char *hdr, const char *val); void http_PrintfHeader(struct http *to, const char *fmt, ...) __printflike(2, 3); diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 8462ceb..d38519e 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -380,7 +380,7 @@ int http_GetHdr(const struct http *hp, const char *hdr, const char **ptr) { unsigned u, l; - char *p; + const char *p; l = hdr[0]; assert(l == strlen(hdr + 1)); @@ -791,7 +791,7 @@ HTTP_Encode(const struct http *fm, uint8_t *p0, unsigned l, unsigned how) */ int -HTTP_Decode(struct http *to, uint8_t *fm) +HTTP_Decode(struct http *to, const uint8_t *fm) { CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c index 6bfa53b..ca0a4d1 100644 --- a/bin/varnishd/cache/cache_http1_proto.c +++ b/bin/varnishd/cache/cache_http1_proto.c @@ -230,7 +230,7 @@ http1_dissect_hdrs(struct http *hp, char *p, const struct http_conn *htc) char *q, *r; assert(p > htc->rxbuf_b); - assert(p < htc->rxbuf_e); + assert(p <= htc->rxbuf_e); hp->nhd = HTTP_HDR_FIRST; hp->conds = 0; r = NULL; /* For FlexeLint */ @@ -311,6 +311,7 @@ static uint16_t http1_splitline(struct http *hp, const struct http_conn *htc, const int *hf) { char *p; + int i; assert(hf == HTTP1_Req || hf == HTTP1_Resp); CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); @@ -333,6 +334,7 @@ http1_splitline(struct http *hp, const struct http_conn *htc, const int *hf) } hp->hd[hf[0]].e = p; assert(Tlen(hp->hd[hf[0]])); + *p++ = '\0'; /* Skip SP */ for (; vct_issp(*p); p++) { @@ -349,6 +351,7 @@ http1_splitline(struct http *hp, const struct http_conn *htc, const int *hf) hp->hd[hf[1]].e = p; if (!Tlen(hp->hd[hf[1]])) return (400); + *p++ = '\0'; /* Skip SP */ for (; vct_issp(*p); p++) { @@ -367,11 +370,9 @@ http1_splitline(struct http *hp, const struct http_conn *htc, const int *hf) hp->hd[hf[2]].e = p; /* Skip CRLF */ - p += vct_skipcrlf(p); - - *hp->hd[hf[0]].e = '\0'; - *hp->hd[hf[1]].e = '\0'; - *hp->hd[hf[2]].e = '\0'; + i = vct_skipcrlf(p); + *p = '\0'; + p += i; return (http1_dissect_hdrs(hp, p, htc)); } @@ -451,7 +452,7 @@ HTTP1_DissectRequest(struct http_conn *htc, struct http *hp) { uint16_t retval; const char *p; - char *b, *e; + const char *b, *e; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); @@ -504,7 +505,7 @@ uint16_t HTTP1_DissectResponse(struct http *hp, struct http_conn *htc) { uint16_t retval = 0; - char *p; + const char *p; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c index 2727ba7..822f373 100644 --- a/bin/varnishd/cache/cache_vrt_re.c +++ b/bin/varnishd/cache/cache_vrt_re.c @@ -39,16 +39,16 @@ #include "vrt.h" static void -Tadd(txt *t, const char *p, int l) +Tadd(char **b, char *e, const char *p, int l) { - Tcheck(*t); + assert((*b) <= e); if (l <= 0) { - } if (t->b + l < t->e) { - memcpy(t->b, p, l); - t->b += l; + } if ((*b) + l < e) { + memcpy((*b), p, l); + (*b) += l; } else { - t->b = t->e; + (*b) = e; } } @@ -101,7 +101,8 @@ VRT_regsub(VRT_CTX, int all, const char *str, void *re, int ovector[30]; vre_t *t; int i, l; - txt res; + char *res_b; + char *res_e; char *b0; const char *s; unsigned u, x; @@ -129,27 +130,27 @@ VRT_regsub(VRT_CTX, int all, const char *str, void *re, } u = WS_Reserve(ctx->ws, 0); - res.e = res.b = b0 = ctx->ws->f; - res.e += u; + res_e = res_b = b0 = ctx->ws->f; + res_e += u; do { /* Copy prefix to match */ - Tadd(&res, str, ovector[0]); + Tadd(&res_b, res_e, str, ovector[0]); for (s = sub ; *s != '\0'; s++ ) { if (*s != '\\' || s[1] == '\0') { - if (res.b < res.e) - *res.b++ = *s; + if (res_b < res_e) + *res_b++ = *s; continue; } s++; if (isdigit(*s)) { x = *s - '0'; l = ovector[2*x+1] - ovector[2*x]; - Tadd(&res, str + ovector[2*x], l); + Tadd(&res_b, res_e, str + ovector[2*x], l); continue; } else { - if (res.b < res.e) - *res.b++ = *s; + if (res_b < res_e) + *res_b++ = *s; } } str += ovector[1]; @@ -169,12 +170,12 @@ VRT_regsub(VRT_CTX, int all, const char *str, void *re, } while (i != VRE_ERROR_NOMATCH); /* Copy suffix to match */ - Tadd(&res, str, len+1); - if (res.b >= res.e) { + Tadd(&res_b, res_e, str, len+1); + if (res_b >= res_e) { WS_Release(ctx->ws, 0); return (str); } - Tcheck(res); - WS_ReleaseP(ctx->ws, res.b); + assert(res_b <= res_e); + WS_ReleaseP(ctx->ws, res_b); return (b0); } diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index a5aec9d..0778637 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -143,7 +143,7 @@ vmod_log(VRT_CTX, const char *fmt, ...) u = WS_Reserve(ctx->ws, 0); t.b = ctx->ws->f; va_start(ap, fmt); - t.e = VRT_StringList(t.b, u, fmt, ap); + t.e = VRT_StringList(ctx->ws->f, u, fmt, ap); va_end(ap); if (t.e != NULL) { assert(t.e > t.b); @@ -164,7 +164,7 @@ vmod_syslog(VRT_CTX, VCL_INT fac, const char *fmt, ...) u = WS_Reserve(ctx->ws, 0); t.b = ctx->ws->f; va_start(ap, fmt); - t.e = VRT_StringList(t.b, u, fmt, ap); + t.e = VRT_StringList(ctx->ws->f, u, fmt, ap); va_end(ap); if (t.e != NULL) syslog((int)fac, "%s", t.b); From nils.goroll at uplex.de Sun Sep 28 18:04:35 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 28 Sep 2014 20:04:35 +0200 Subject: [master] e40edce now that txt is const, fm can be, too Message-ID: commit e40edceffff77d0578237a85cf5109cd0d069599 Author: Nils Goroll Date: Sun Sep 28 20:04:31 2014 +0200 now that txt is const, fm can be, too diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index d38519e..05de39f 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -786,8 +786,6 @@ HTTP_Encode(const struct http *fm, uint8_t *p0, unsigned l, unsigned how) /*-------------------------------------------------------------------- * Decode byte string into http struct - * - * XXX: cannot make fm const because to->hd isn't. */ int @@ -808,9 +806,9 @@ HTTP_Decode(struct http *to, const uint8_t *fm) } if (*fm == '\0') return (0); - to->hd[to->nhd].b = (void*)fm; - fm = (void*)strchr((void*)fm, '\0'); - to->hd[to->nhd].e = (void*)fm; + to->hd[to->nhd].b = fm; + fm = strchr(fm, '\0'); + to->hd[to->nhd].e = fm; fm++; if (to->vsl != NULL) http_VSLH(to, to->nhd); From phk at FreeBSD.org Sun Sep 28 19:38:08 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 28 Sep 2014 21:38:08 +0200 Subject: [master] 335c440 Fix casts Message-ID: commit 335c440c541054e4e42d20c1be02cff7a3d139da Author: Poul-Henning Kamp Date: Sun Sep 28 18:04:36 2014 +0000 Fix casts diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 05de39f..c74aa38 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -806,9 +806,9 @@ HTTP_Decode(struct http *to, const uint8_t *fm) } if (*fm == '\0') return (0); - to->hd[to->nhd].b = fm; - fm = strchr(fm, '\0'); - to->hd[to->nhd].e = fm; + to->hd[to->nhd].b = (const void*)fm; + fm = (const void*)strchr((const void*)fm, '\0'); + to->hd[to->nhd].e = (const void*)fm; fm++; if (to->vsl != NULL) http_VSLH(to, to->nhd); From phk at FreeBSD.org Mon Sep 29 05:42:35 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Sep 2014 07:42:35 +0200 Subject: [master] f96d286 Explicit cast back to pointer Message-ID: commit f96d286a98110b32cd4b2a0a34115ee983118419 Author: Poul-Henning Kamp Date: Mon Sep 29 05:42:24 2014 +0000 Explicit cast back to pointer diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index 411fc9a..3bf3353 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -491,7 +491,7 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size, /* Fill the storage structure */ memset(ss, 0, sizeof *ss); ss->magic = STORAGE_MAGIC; - ss->ptr = PRNUP(sc, ss + 1); + ss->ptr = (void*)PRNUP(sc, ss + 1); ss->space = max_size; ss->priv = sc; ss->stevedore = st; From phk at FreeBSD.org Mon Sep 29 06:06:32 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Sep 2014 08:06:32 +0200 Subject: [master] 77e0b2c Take a more competent stab a the casts which breaks builds in jenkins. Message-ID: commit 77e0b2c205d1dc7bb7cd91e62bac74fc6f88cbc6 Author: Poul-Henning Kamp Date: Mon Sep 29 06:06:09 2014 +0000 Take a more competent stab a the casts which breaks builds in jenkins. diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h index 8295394..006df7a 100644 --- a/bin/varnishd/common/common.h +++ b/bin/varnishd/common/common.h @@ -127,9 +127,9 @@ void VSM_common_ageupdate(struct vsm_sc *sc); * Generic power-2 rounding macros */ -#define PWR2(x) ((((x)-1)&(x))==0) /* Is a power of two */ -#define RDN2(x, y) ((x)&(~((uintmax_t)(y)-1))) /* PWR2(y) true */ -#define RUP2(x, y) (((x)+((y)-1))&(~((uintmax_t)(y)-1))) /* PWR2(y) true */ +#define PWR2(x) ((((x)-1UL)&(x))==0) /* Is a power of two */ +#define RDN2(x, y) ((x)&(~((uintptr_t)(y)-1UL))) /* PWR2(y) true */ +#define RUP2(x, y) (((x)+((y)-1))&(~((uintptr_t)(y)-1UL))) /* PWR2(y) true */ /*-------------------------------------------------------------------- * Pointer aligment magic diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index 3bf3353..411fc9a 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -491,7 +491,7 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size, /* Fill the storage structure */ memset(ss, 0, sizeof *ss); ss->magic = STORAGE_MAGIC; - ss->ptr = (void*)PRNUP(sc, ss + 1); + ss->ptr = PRNUP(sc, ss + 1); ss->space = max_size; ss->priv = sc; ss->stevedore = st; From phk at FreeBSD.org Mon Sep 29 08:04:55 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Sep 2014 10:04:55 +0200 Subject: [master] 5aca8b4 Move handling of PRIV_ variables to separate source file before it gets too complicated. Message-ID: commit 5aca8b44ce317901c3be45e918e406ffc1f0930f Author: Poul-Henning Kamp Date: Mon Sep 29 08:04:32 2014 +0000 Move handling of PRIV_ variables to separate source file before it gets too complicated. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index edb0e31..26ad745 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -46,6 +46,7 @@ varnishd_SOURCES = \ cache/cache_vary.c \ cache/cache_vcl.c \ cache/cache_vrt.c \ + cache/cache_vrt_priv.c \ cache/cache_vrt_re.c \ cache/cache_vrt_var.c \ cache/cache_vrt_vmod.c \ diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index c41bcaa..faae807 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -515,16 +515,6 @@ VRT_purge(VRT_CTX, double ttl, double grace, double keep) } /*-------------------------------------------------------------------- - */ - -void -VRT_priv_fini(const struct vmod_priv *p) -{ - if (p->priv != (void*)0 && p->free != (void*)0) - p->free(p->priv); -} - -/*-------------------------------------------------------------------- * Simple stuff */ diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c new file mode 100644 index 0000000..aab904a --- /dev/null +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -0,0 +1,57 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2014 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Runtime support for compiled VCL programs: private variables + */ + +#include "config.h" + + +#include +#include + +#include "cache.h" + +#include "cache_backend.h" +#include "hash/hash_slinger.h" +#include "vav.h" +#include "vcl.h" +#include "vrt.h" +#include "vrt_obj.h" +#include "vtcp.h" +#include "vtim.h" +/*-------------------------------------------------------------------- + */ + +void +VRT_priv_fini(const struct vmod_priv *p) +{ + if (p->priv != (void*)0 && p->free != (void*)0) + p->free(p->priv); +} + From phk at FreeBSD.org Mon Sep 29 09:05:52 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Sep 2014 11:05:52 +0200 Subject: [master] 3fd1663 Now that things are properly prepared: Implement PRIV_SESS and PRIV_REQ Message-ID: commit 3fd16639cc7774578a3ec5fd4894cac9baec6242 Author: Poul-Henning Kamp Date: Mon Sep 29 09:03:43 2014 +0000 Now that things are properly prepared: Implement PRIV_SESS and PRIV_REQ Note that each VMOD only gets one of each, so that the same 'vmod_priv' will appear on all VMOD functions which use PRIV_REQ and similarly only a single vmod_priv appear for all PRIV_SESS uses in that VMOD. Feedback welcome. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 39359b4..61298a3 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -120,6 +120,7 @@ struct sess; struct sesspool; struct vbc; struct vrt_backend; +struct vrt_privs; struct vsb; struct waitinglist; struct worker; @@ -681,11 +682,12 @@ struct sess { char *client_addr_str; char *client_port_str; - /* Timestamps, all on TIM_real() timescale */ double t_open; /* fd accepted */ double t_idle; /* fd accepted or resp sent */ + VTAILQ_HEAD(,vrt_privs) privs; + #if defined(HAVE_EPOLL_CTL) struct epoll_event ev; #endif @@ -1116,6 +1118,7 @@ const char *VCL_Method_Name(unsigned); */ const char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap); char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap); +void VRTPRIV_dynamic_kill(struct sess *sp, uintptr_t id); void ESI_Deliver(struct req *); void ESI_DeliverChild(struct req *); diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index 8e4508a..b85b099 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -201,6 +201,7 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req) req->vcl = NULL; } + VRTPRIV_dynamic_kill(sp, (uintptr_t)req); /* Charge and log byte counters */ AN(req->vsl->wid); CNT_AcctLogCharge(wrk->stats, req); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index ca4574f..001dad7 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -90,6 +90,7 @@ ses_new(struct sesspool *pp) sp->t_open = NAN; sp->t_idle = NAN; + VTAILQ_INIT(&sp->privs); Lck_New(&sp->mtx, lck_sess); CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); return (sp); @@ -302,6 +303,7 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now) now = VTIM_real(); AZ(isnan(sp->t_open)); + VRTPRIV_dynamic_kill(sp, 0); VSL(SLT_SessClose, sp->vxid, "%s %.3f", sess_close_2str(sp->reason, 0), now - sp->t_open); VSL(SLT_End, sp->vxid, "%s", ""); diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index aab904a..0893b4c 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -37,20 +37,78 @@ #include "cache.h" -#include "cache_backend.h" -#include "hash/hash_slinger.h" -#include "vav.h" #include "vcl.h" #include "vrt.h" -#include "vrt_obj.h" -#include "vtcp.h" -#include "vtim.h" + +struct vrt_privs { + unsigned magic; +#define VRT_PRIVS_MAGIC 0x24157a52 + VTAILQ_ENTRY(vrt_privs) list; + struct vmod_priv priv[1]; + const struct VCL_conf *vcl; + uintptr_t id; +}; + +/*-------------------------------------------------------------------- + */ + +static struct vmod_priv * +VRT_priv_dynamic(VRT_CTX, uintptr_t id) +{ + struct vrt_privs *vps; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + VTAILQ_FOREACH(vps, &ctx->req->sp->privs, list) { + CHECK_OBJ_NOTNULL(vps, VRT_PRIVS_MAGIC); + if (vps->vcl == ctx->vcl && vps->id == id) + return (vps->priv); + } + ALLOC_OBJ(vps, VRT_PRIVS_MAGIC); + AN(vps); + vps->vcl = ctx->vcl; + vps->id = id; + VTAILQ_INSERT_TAIL(&ctx->req->sp->privs, vps, list); + return (vps->priv); +} + +void +VRTPRIV_dynamic_kill(struct sess *sp, uintptr_t id) +{ + struct vrt_privs *vps, *vps1; + + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + + VTAILQ_FOREACH_SAFE(vps, &sp->privs, list, vps1) { + CHECK_OBJ_NOTNULL(vps, VRT_PRIVS_MAGIC); + if (id == vps->id) { + VTAILQ_REMOVE(&sp->privs, vps, list); + VRT_priv_fini(vps->priv); + FREE_OBJ(vps); + } + } + if (id == 0) + assert(VTAILQ_EMPTY(&sp->privs)); +} + +struct vmod_priv * +VRT_priv_req(VRT_CTX) +{ + return (VRT_priv_dynamic(ctx, (uintptr_t)ctx->req)); +} + +struct vmod_priv * +VRT_priv_sess(VRT_CTX) +{ + return (VRT_priv_dynamic(ctx, (uintptr_t)NULL)); +} + /*-------------------------------------------------------------------- */ void VRT_priv_fini(const struct vmod_priv *p) { + if (p->priv != (void*)0 && p->free != (void*)0) p->free(p->priv); } diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc new file mode 100644 index 0000000..af951f9 --- /dev/null +++ b/bin/varnishtest/tests/v00041.vtc @@ -0,0 +1,41 @@ +varnishtest "Test priv_sess and priv_req" + +server s1 { + rxreq + txresp + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + import ${vmod_debug}; + + sub vcl_recv { + set req.http.x0 = debug.test_priv_req(req.url); + set req.http.y0 = debug.test_priv_sess(req.url); + } + + sub vcl_deliver { + set resp.http.x0 = req.http.x0; + set resp.http.x1 = debug.test_priv_req(""); + set resp.http.y0 = req.http.y0; + set resp.http.y1 = debug.test_priv_sess(""); + } +} -start + + +client c1 { + txreq -url /foobar + rxresp + expect resp.http.x0 == /foobar + expect resp.http.x1 == /foobar + expect resp.http.y0 == /foobar + expect resp.http.y1 == /foobar + + txreq -url /snafu + rxresp + expect resp.http.x0 == /snafu + expect resp.http.x1 == /snafu + expect resp.http.y0 == /foobar + expect resp.http.y1 == /foobar +} -run diff --git a/include/vrt.h b/include/vrt.h index 3e21a5f..5b7ab7a 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -246,6 +246,8 @@ struct vmod_priv { typedef int vmod_init_f(struct vmod_priv *, const struct VCL_conf *); void VRT_priv_fini(const struct vmod_priv *p); +struct vmod_priv *VRT_priv_sess(VRT_CTX); +struct vmod_priv *VRT_priv_req(VRT_CTX); /* Stevedore related functions */ int VRT_Stv(const char *nm); diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 3557a4b..acc7b51 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -566,6 +566,12 @@ vcc_func(struct vcc *tl, struct expr **e, const char *cfunc, VSB_printf(ifp->fin, "\tVRT_priv_fini(&%s);", buf); e2 = vcc_mk_expr(VOID, "&%s", buf); p += strlen(p) + 1; + } else if (fmt == VOID && !strcmp(p, "PRIV_REQ")) { + e2 = vcc_mk_expr(VOID, "VRT_priv_req(ctx)"); + p += strlen(p) + 1; + } else if (fmt == VOID && !strcmp(p, "PRIV_SESS")) { + e2 = vcc_mk_expr(VOID, "VRT_priv_sess(ctx)"); + p += strlen(p) + 1; } else if (fmt == ENUM) { ExpectErr(tl, ID); ERRCHK(tl); diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 6ad9504..14d9c95 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -58,6 +58,8 @@ ctypes = { 'IP': "VCL_IP", 'PRIV_CALL': "struct vmod_priv *", 'PRIV_VCL': "struct vmod_priv *", + 'PRIV_SESS': "struct vmod_priv *", + 'PRIV_REQ': "struct vmod_priv *", 'REAL': "VCL_REAL", 'STRING': "VCL_STRING", 'STRING_LIST': "const char *, ...", diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 74ec02f..468fc51 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -51,6 +51,14 @@ $Function VOID test_priv_vcl(PRIV_VCL) Test function for VCL private pointers +$Function STRING test_priv_req(PRIV_REQ, STRING) + +Test function for REQ private pointers + +$Function STRING test_priv_sess(PRIV_SESS, STRING) + +Test function for SESS private pointers + $Function BLOB str2blob(STRING src) Turn a string into a blob diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 76e5a5f..08b5aa8 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -88,6 +88,30 @@ vmod_test_priv_call(VRT_CTX, struct vmod_priv *priv) } } +VCL_STRING __match_proto__(td_debug_test_priv_sess) +vmod_test_priv_sess(VRT_CTX, struct vmod_priv *priv, VCL_STRING s) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + if (priv->priv == NULL) { + priv->priv = strdup(s); + priv->free = free; + } + return (priv->priv); +} + +VCL_STRING __match_proto__(td_debug_test_priv_req) +vmod_test_priv_req(VRT_CTX, struct vmod_priv *priv, VCL_STRING s) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + if (priv->priv == NULL) { + priv->priv = strdup(s); + priv->free = free; + } + return (priv->priv); +} + VCL_VOID __match_proto__(td_debug_test_priv_vcl) vmod_test_priv_vcl(VRT_CTX, struct vmod_priv *priv) { From phk at FreeBSD.org Mon Sep 29 12:36:00 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Sep 2014 14:36:00 +0200 Subject: [master] 73b829f Add vct_tchar() (ref: rfc7230#section-3.2.6) Message-ID: commit 73b829fb7a7b15e2d214810a1fdd9a30a7181890 Author: Poul-Henning Kamp Date: Mon Sep 29 12:11:11 2014 +0000 Add vct_tchar() (ref: rfc7230#section-3.2.6) diff --git a/include/vct.h b/include/vct.h index 3c41dd7..5e1af85 100644 --- a/include/vct.h +++ b/include/vct.h @@ -40,6 +40,7 @@ #define VCT_HEX (1<<6) #define VCT_XMLNAMESTART (1<<7) #define VCT_XMLNAME (1<<8) +#define VCT_TCHAR (1<<9) extern const uint16_t vct_typtab[256]; @@ -61,6 +62,7 @@ vct_is(int x, uint16_t y) #define vct_issepctl(x) vct_is(x, VCT_SEPARATOR | VCT_CTL) #define vct_isxmlnamestart(x) vct_is(x, VCT_XMLNAMESTART) #define vct_isxmlname(x) vct_is(x, VCT_XMLNAMESTART | VCT_XMLNAME) +#define vct_istchar(x) vct_is(x, VCT_ALPHA | VCT_DIGIT | VCT_TCHAR) #define vct_iscrlf(p) (((p)[0] == '\r' && (p)[1] == '\n') || (p)[0] == '\n') diff --git a/lib/libvarnish/vct.c b/lib/libvarnish/vct.c index c82e972..3ba4046 100644 --- a/lib/libvarnish/vct.c +++ b/lib/libvarnish/vct.c @@ -73,12 +73,20 @@ const uint16_t vct_typtab[256] = { [0x1e] = VCT_CTL, [0x1f] = VCT_CTL, [0x20] = VCT_SP | VCT_SEPARATOR, + [0x21] = VCT_TCHAR, [0x22] = VCT_SEPARATOR, + [0x23] = VCT_TCHAR, + [0x24] = VCT_TCHAR, + [0x25] = VCT_TCHAR, + [0x26] = VCT_TCHAR, + [0x27] = VCT_TCHAR, [0x28] = VCT_SEPARATOR, [0x29] = VCT_SEPARATOR, + [0x2a] = VCT_TCHAR, + [0x2b] = VCT_TCHAR, [0x2c] = VCT_SEPARATOR, - [0x2d] = VCT_XMLNAME, - [0x2e] = VCT_XMLNAME, + [0x2d] = VCT_XMLNAME | VCT_TCHAR, + [0x2e] = VCT_XMLNAME | VCT_TCHAR, [0x2f] = VCT_SEPARATOR, [0x30] = VCT_DIGIT | VCT_HEX | VCT_XMLNAME, [0x31] = VCT_DIGIT | VCT_HEX | VCT_XMLNAME, @@ -126,7 +134,9 @@ const uint16_t vct_typtab[256] = { [0x5b] = VCT_SEPARATOR, [0x5c] = VCT_SEPARATOR, [0x5d] = VCT_SEPARATOR, - [0x5f] = VCT_XMLNAMESTART, + [0x5e] = VCT_TCHAR, + [0x5f] = VCT_XMLNAMESTART | VCT_TCHAR, + [0x60] = VCT_TCHAR, [0x61] = VCT_LOALPHA | VCT_HEX | VCT_XMLNAMESTART, [0x62] = VCT_LOALPHA | VCT_HEX | VCT_XMLNAMESTART, [0x63] = VCT_LOALPHA | VCT_HEX | VCT_XMLNAMESTART, @@ -154,7 +164,9 @@ const uint16_t vct_typtab[256] = { [0x79] = VCT_LOALPHA | VCT_XMLNAMESTART, [0x7a] = VCT_LOALPHA | VCT_XMLNAMESTART, [0x7b] = VCT_SEPARATOR, + [0x7c] = VCT_TCHAR, [0x7d] = VCT_SEPARATOR, + [0x7e] = VCT_TCHAR, [0x7f] = VCT_CTL, [0xb7] = VCT_XMLNAME, [0xc0] = VCT_XMLNAMESTART, From lkarsten at varnish-software.com Mon Sep 29 13:10:10 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 29 Sep 2014 15:10:10 +0200 Subject: [4.0] e4394b8 Add list of changes. Message-ID: commit e4394b860464f2f5ea18fc6036d09590835358d2 Author: Lasse Karstensen Date: Tue Sep 23 17:50:13 2014 +0200 Add list of changes. diff --git a/doc/changes.rst b/doc/changes.rst index d66e7a0..e424a20 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,4 +1,100 @@ +============================================ +Changes from 4.0.1 to 4.0.2-rc1 (2014-09-23) +============================================ + +New since 4.0.1: + +- [libvmod-std] New function strstr() for matching substrings. +- server.(hostname|identity) is now available in all VCL functions. +- VCL variable type BYTES was added. +- X-Forwarded-For is now set on return(synth) from vcl_recv. +- `workspace_client` default is now 9k. +- [varnishstat] Update interval can now be subsecond. +- Document that reloading VCL does not reload a VMOD. +- Guru meditation page is now valid HTML5. +- [varnishstat] hitrate calculation is back. +- New parameter `group_cc` adds a GID to the grouplist of + VCL compiler sandbox. +- Parameter shm_reclen is now an alias for vsl_reclen. +- Workspace overflows are now handled with a 500 client response. +- VCL variable type added: HTTP, representing a HTTP header set. +- It is now possible to return(synth) from vcl_deliver. +- [varnishadm] vcl.show now has a -v option that output the + complete set of VCL and included VCL files. +- RHEL7 packaging (systemd) was added. +- [libvmod-std] querysort() fixed parameter limit has been lifted. +- Fix small memory leak in ESI parser. +- Fix unreported race/assert in V1D_Deliver(). + +Bugs fixed +---------- + +* 1553_ - Fully reset workspace (incl. Vary state) before reusing it. +* 1551_ - Handle workspace exhaustion during purge. +* 1591_ - Group entries correctly in varnishtop. +* 1592_ - Bail out on workspace exhaustion in VRT_IP_string. +* 1538_ - Relax VMOD ABI check for release branches. +* 1584_ - Don't log garbage/non-HTTP requests. [varnishncsa] +* 1407_ - Don't rename VSM file until child has started. +* 1466_ - Don't leak request structs on restart after waitinglist. +* 1580_ - Output warning if started without -b and -f. [varnishd] +* 1583_ - Abort on fatal sandbox errors on Solaris. (Related: 1572_) +* 1585_ - Handle fatal sandbox errors. +* 1572_ - Exit codes have been cleaned up. +* 1569_ - Order of symbols should not influence compilation result. +* 1579_ - Clean up type inference in VCL. +* 1578_ - Don't count Age twice when computing new object TTL. +* 1574_ - std.syslog() logged empty strings. +* 1555_ - autoconf editline/readline build issue. +* 1568_ - Skip NULL arguments when hashing. +* 1567_ - Compile on systems without SO_SNDTIMEO/SO_RCVTIMEO. +* 1512_ - Changes to bereq are lost between v_b_r and v_b_f. +* 1563_ - Increase varnishtest read timeout. +* 1562_ - Fail correctly when rereading a failed client request body. +* 1521_ - VCL compilation fails on OSX x86_64. +* 1547_ - Panic when increasing shm_reclen. +* 1503_ - Document return(retry). +* 1581_ - Don't log duplicate Begin records to shmlog. +* 1588_ - Correct timestamps on pipelined requests. +* 1575_ - Use all director backends when looking for a healthy one. +* 1577_ - Read the full request body if shunted to synth. +* 1532_ - Use correct VCL representation of reals. +* 1531_ - Work around libedit bug in varnishadm. + +.. _1553: https://www.varnish-cache.org/trac/ticket/1553 +.. _1551: https://www.varnish-cache.org/trac/ticket/1551 +.. _1591: https://www.varnish-cache.org/trac/ticket/1591 +.. _1592: https://www.varnish-cache.org/trac/ticket/1592 +.. _1538: https://www.varnish-cache.org/trac/ticket/1538 +.. _1584: https://www.varnish-cache.org/trac/ticket/1584 +.. _1407: https://www.varnish-cache.org/trac/ticket/1407 +.. _1466: https://www.varnish-cache.org/trac/ticket/1466 +.. _1580: https://www.varnish-cache.org/trac/ticket/1580 +.. _1583: https://www.varnish-cache.org/trac/ticket/1583 +.. _1585: https://www.varnish-cache.org/trac/ticket/1585 +.. _1572: https://www.varnish-cache.org/trac/ticket/1572 +.. _1569: https://www.varnish-cache.org/trac/ticket/1569 +.. _1579: https://www.varnish-cache.org/trac/ticket/1579 +.. _1578: https://www.varnish-cache.org/trac/ticket/1578 +.. _1574: https://www.varnish-cache.org/trac/ticket/1574 +.. _1555: https://www.varnish-cache.org/trac/ticket/1555 +.. _1568: https://www.varnish-cache.org/trac/ticket/1568 +.. _1567: https://www.varnish-cache.org/trac/ticket/1567 +.. _1512: https://www.varnish-cache.org/trac/ticket/1512 +.. _1563: https://www.varnish-cache.org/trac/ticket/1563 +.. _1562: https://www.varnish-cache.org/trac/ticket/1562 +.. _1521: https://www.varnish-cache.org/trac/ticket/1521 +.. _1547: https://www.varnish-cache.org/trac/ticket/1547 +.. _1503: https://www.varnish-cache.org/trac/ticket/1503 +.. _1581: https://www.varnish-cache.org/trac/ticket/1581 +.. _1588: https://www.varnish-cache.org/trac/ticket/1588 +.. _1575: https://www.varnish-cache.org/trac/ticket/1575 +.. _1577: https://www.varnish-cache.org/trac/ticket/1577 +.. _1532: https://www.varnish-cache.org/trac/ticket/1532 +.. _1531: https://www.varnish-cache.org/trac/ticket/1531 + + ======================================== Changes from 4.0.0 to 4.0.1 (2014-06-24) ======================================== From lkarsten at varnish-software.com Mon Sep 29 13:25:51 2014 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Mon, 29 Sep 2014 15:25:51 +0200 Subject: [master] 9cbce71 Update copyright statement. Message-ID: commit 9cbce710247c383f865f031ba06adfee8b29f8c3 Author: Lasse Karstensen Date: Mon Sep 29 15:25:17 2014 +0200 Update copyright statement. diff --git a/LICENSE b/LICENSE index a6cf987..67dc00a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,5 @@ Copyright (c) 2006 Verdens Gang AS -Copyright (c) 2006-2011 Varnish Software AS +Copyright (c) 2006-2014 Varnish Software AS All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/lib/libvarnish/version.c b/lib/libvarnish/version.c index b7b2edc..284c661 100644 --- a/lib/libvarnish/version.c +++ b/lib/libvarnish/version.c @@ -44,5 +44,5 @@ VCS_Message(const char *progname) { fprintf(stderr, "%s (%s)\n", progname, VCS_version); fprintf(stderr, "Copyright (c) 2006 Verdens Gang AS\n"); - fprintf(stderr, "Copyright (c) 2006-2011 Varnish Software AS\n"); + fprintf(stderr, "Copyright (c) 2006-2014 Varnish Software AS\n"); } From nils.goroll at uplex.de Mon Sep 29 14:34:00 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 29 Sep 2014 16:34:00 +0200 Subject: [master] 9d8b62f Fix out-of-tree builds Message-ID: commit 9d8b62f42276aa80540f22eb1bd1c12836954066 Author: Nils Goroll Date: Mon Sep 29 16:33:32 2014 +0200 Fix out-of-tree builds Fixes #1479 diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index 0d681e8..c872ddd 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -10,7 +10,7 @@ BUILDDIR = build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(srcdir) +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(distdir) .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest @@ -95,7 +95,6 @@ EXTRA_DIST = \ glossary \ installation \ phk \ - reference \ tutorial \ users-guide \ whats-new @@ -144,11 +143,16 @@ include/varnishhist_synopsis.rst: $(top_builddir)/bin/varnishhist/varnishhist_op BUILT_SOURCES += include/varnishhist_options.rst \ include/varnishhist_synopsis.rst -reference/vmod_std.generated.rst: $(top_srcdir)/lib/libvmod_std/vmod_std.rst +.PHONY: reference +reference: + test -d $@ || mkdir $@ +BUILT_SOURCES += reference + +reference/vmod_std.generated.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst cp $? $@ BUILT_SOURCES += reference/vmod_std.generated.rst -reference/vmod_directors.generated.rst: $(top_srcdir)/lib/libvmod_directors/vmod_directors.rst +reference/vmod_directors.generated.rst: $(top_builddir)/lib/libvmod_directors/vmod_directors.rst cp $? $@ BUILT_SOURCES += reference/vmod_directors.generated.rst diff --git a/man/Makefile.am b/man/Makefile.am index c7dd1bf..d791d1f 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -86,8 +86,8 @@ varnishhist.1: \ $(top_srcdir)/doc/sphinx/include/varnishhist_synopsis.rst ${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishhist.rst $@ -vmod_std.3: $(top_srcdir)/lib/libvmod_std/vmod_std.man.rst +vmod_std.3: $(top_builddir)/lib/libvmod_std/vmod_std.man.rst ${RST2MAN} $(RST2ANY_FLAGS) $? $@ -vmod_directors.3: $(top_srcdir)/lib/libvmod_directors/vmod_directors.man.rst +vmod_directors.3: $(top_builddir)/lib/libvmod_directors/vmod_directors.man.rst ${RST2MAN} $(RST2ANY_FLAGS) $? $@ From nils.goroll at uplex.de Mon Sep 29 14:57:03 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 29 Sep 2014 16:57:03 +0200 Subject: [master] f9e1dd9 revert ALLSPHINXOPTS from the previous commit Message-ID: commit f9e1dd9455eb6ce4e54074b5616cdeed22e4da1e Author: Nils Goroll Date: Mon Sep 29 16:56:56 2014 +0200 revert ALLSPHINXOPTS from the previous commit diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index c872ddd..ba098ca 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -10,7 +10,7 @@ BUILDDIR = build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(distdir) +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(srcdir) .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest From nils.goroll at uplex.de Mon Sep 29 16:33:08 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 29 Sep 2014 18:33:08 +0200 Subject: [master] 7a51ed9 need explicit reference to subdir to be created Message-ID: commit 7a51ed991348063571c1667fea53ca88829281b9 Author: Nils Goroll Date: Mon Sep 29 17:46:40 2014 +0200 need explicit reference to subdir to be created diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index ba098ca..c82dec8 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -148,12 +148,12 @@ reference: test -d $@ || mkdir $@ BUILT_SOURCES += reference -reference/vmod_std.generated.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst - cp $? $@ +reference/vmod_std.generated.rst: reference $(top_builddir)/lib/libvmod_std/vmod_std.rst + cp $(top_builddir)/lib/libvmod_std/vmod_std.rst $@ BUILT_SOURCES += reference/vmod_std.generated.rst -reference/vmod_directors.generated.rst: $(top_builddir)/lib/libvmod_directors/vmod_directors.rst - cp $? $@ +reference/vmod_directors.generated.rst: reference $(top_builddir)/lib/libvmod_directors/vmod_directors.rst + cp $(top_builddir)/lib/libvmod_directors/vmod_directors.rst $@ BUILT_SOURCES += reference/vmod_directors.generated.rst EXTRA_DIST += $(BUILT_SOURCES) From nils.goroll at uplex.de Mon Sep 29 16:33:08 2014 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 29 Sep 2014 18:33:08 +0200 Subject: [master] 5822d0c Get the out-of-tree build of docs right - finally? Message-ID: commit 5822d0ce45b99e03ee2e611c5ef0d74bfc6a585d Author: Nils Goroll Date: Mon Sep 29 18:21:46 2014 +0200 Get the out-of-tree build of docs right - finally? Trouble is that sphinx only takes one source dir, but when we build out-of-tree, we need to both use sources from $(srcdir) as well as $(builddir). Fix this by hard-linking from $(srcdir) to $(builddir) and wait for someone to come up with an out-of-tree build on different file systems. Fixes #1479 diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index c82dec8..176c07d 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -10,7 +10,7 @@ BUILDDIR = build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(srcdir) +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(builddir) .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest @@ -30,33 +30,43 @@ help: clean: -rm -rf $(BUILDDIR)/* -html: +# use index.rst as an indicator if we have copied already +.PHONY: link_srcdir +link_srcdir: + if test "x$(srcdir)" != "x$(builddir)" && test ! -f index.rst ; then \ + d=`pwd`/$(builddir) ; \ + cd $(srcdir) && find . -name \*.rst -type f | cpio -ldmp $${d} ; \ + fi + +sphinx_prereq: link_srcdir conf.py + +html: sphinx_prereq $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." -dirhtml: +dirhtml: sphinx_prereq $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." -pickle: +pickle: sphinx_prereq $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." -json: +json: sphinx_prereq $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." -htmlhelp: +htmlhelp: sphinx_prereq $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." -qthelp: +qthelp: sphinx_prereq $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ @@ -65,25 +75,25 @@ qthelp: @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Varnish.qhc" -latex: +latex: sphinx_prereq $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." -changes: +changes: sphinx_prereq $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." -linkcheck: +linkcheck: sphinx_prereq $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." -doctest: +doctest: sphinx_prereq $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." From phk at FreeBSD.org Mon Sep 29 20:21:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Sep 2014 22:21:50 +0200 Subject: [master] f92e560 Write a general http_split() function to use when finding tokens and what's not in http headers. Message-ID: commit f92e560e4ed21b20e21e9b039be9b189b7d11071 Author: Poul-Henning Kamp Date: Mon Sep 29 20:20:23 2014 +0000 Write a general http_split() function to use when finding tokens and what's not in http headers. Make http_GetHdrToken() return both start and end of tokendata so we can chew on it without thinking about NUL vs. anything else. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 61298a3..4c9568f 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -880,7 +880,7 @@ void HTTP_Setup(struct http *, struct ws *, struct vsl_log *, enum VSL_tag_e); void http_Teardown(struct http *ht); int http_GetHdr(const struct http *hp, const char *hdr, const char **ptr); int http_GetHdrToken(const struct http *hp, const char *hdr, - const char *token, const char **ptr); + const char *token, const char **pb, const char **pe); int http_GetHdrField(const struct http *hp, const char *hdr, const char *field, const char **ptr); double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field); diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index c74aa38..4735000 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -402,6 +402,54 @@ http_GetHdr(const struct http *hp, const char *hdr, const char **ptr) } /*----------------------------------------------------------------------------- + * Split source string at any of the separators, return pointer to first + * and last+1 char of substrings, with whitespace trimed at both ends. + * If sep being an empty string is shorthand for VCT::SP + * If stop is NULL, src is NUL terminated. + */ + +static int +http_split(const char **src, const char *stop, const char *sep, + const char **b, const char **e) +{ + const char *p, *q; + + AN(src); + AN(*src); + AN(sep); + AN(b); + AN(e); + + if (stop == NULL) + stop = strchr(*src, '\0'); + + for (p = *src; p < stop && (vct_issp(*p) || strchr(sep, *p)); p++) + continue; + + if (p >= stop) { + *b = NULL; + *e = NULL; + return (0); + } + + *b = p; + if (*sep == '\0') { + for (q = p + 1; q < stop && !vct_issp(*q); q++) + continue; + *e = q; + *src = q; + return (1); + } + for (q = p + 1; q < stop && !strchr(sep, *q); q++) + continue; + *src = q; + while (q > p && vct_issp(q[-1])) + q--; + *e = q; + return (1); +} + +/*----------------------------------------------------------------------------- * Find a given data element (token) in a header according to RFC2616's #rule * (section 2.1, p15) * @@ -416,49 +464,47 @@ http_GetHdr(const struct http *hp, const char *hdr, const char **ptr) * But all examples and specific statements regarding tokens follow the rule * that unquoted tokens are to be matched case-insensitively and quoted tokens * case-sensitively. + * + * The optional pb and pe arguments will point to the token content start and + * end+1, white space trimmed on both sides. */ int http_GetHdrToken(const struct http *hp, const char *hdr, - const char *token, const char **ptr) + const char *token, const char **pb, const char **pe) { - const char *h, *e; + const char *h, *b, *e; unsigned fl; - int quoted; - if (ptr != NULL) - *ptr = NULL; + if (pb != NULL) + *pb = NULL; + if (pe != NULL) + *pe = NULL; if (!http_GetHdr(hp, hdr, &h)) return (0); AN(h); - e = strchr(h, '\0'); fl = strlen(token); - quoted = token[0] == '"' && token[fl] == '"'; - while (h + fl <= e) { - /* Skip leading whitespace and commas */ - if (vct_islws(*h) || *h == ',') { - h++; + while(http_split(&h, NULL, ",", &b, &e)) { + if (*b == '"' && !memcmp(b + 1, token, fl) && b[fl + 1] == '"') + break; + if (!strncasecmp(b, token, fl) && !vct_istchar(b[fl])) + break; + } + if (b == NULL) + return (0); + if (pb != NULL) { + for (b += fl; vct_islws(*b); b++) continue; + if (b == e) { + b = NULL; + e = NULL; } - /* Check for substrings before memcmp() */ - if ((h + fl == e || vct_issepctl(h[fl])) && - (quoted - ? !memcmp(h, token, fl) - : !strncasecmp(h, token, fl))) { - if (ptr != NULL) { - h += fl; - while (vct_islws(*h)) - h++; - *ptr = h; - } - return (1); - } - /* Skip until end of header or comma */ - while (*h && *h != ',') - h++; + *pb = b; + if (pe != NULL) + *pe = e; } - return (0); + return (1); } /*-------------------------------------------------------------------- @@ -468,45 +514,44 @@ http_GetHdrToken(const struct http *hp, const char *hdr, double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field) { - const char *h; + const char *hb, *he, *b, *e; int i; - double a, b; + double a, f; - h = NULL; - i = http_GetHdrToken(hp, hdr, field, &h); + i = http_GetHdrToken(hp, hdr, field, &hb, &he); if (!i) return (0.); - if (h == NULL) + if (hb == NULL) return (1.); - /* Skip whitespace, looking for '=' */ - while (*h && vct_issp(*h)) - h++; - if (*h++ != ';') - return (1.); - while (*h && vct_issp(*h)) - h++; - if (*h++ != 'q') - return (1.); - while (*h && vct_issp(*h)) - h++; - if (*h++ != '=') + while(http_split(&hb, he, ";", &b, &e)) { + if (*b != 'q') + continue; + for (b++; b < e && vct_issp(*b); b++) + continue; + if (b == e || *b != '=') + continue; + break; + } + if (b == NULL) return (1.); - while (*h && vct_issp(*h)) - h++; - a = 0.; - while (vct_isdigit(*h)) { + for (b++; b < e && vct_issp(*b); b++) + continue; + if (b == e || (*b != '.' && !vct_isdigit(*b))) + return (0.); + a = 0; + while (b < e && vct_isdigit(*b)) { a *= 10.; - a += *h - '0'; - h++; + a += *b - '0'; + b++; } - if (*h++ != '.') + if (b == e || *b++ != '.') return (a); - b = .1; - while (vct_isdigit(*h)) { - a += b * (*h - '0'); - b *= .1; - h++; + f = .1; + while (b < e && vct_isdigit(*b)) { + a += f * (*b - '0'); + f *= .1; + b++; } return (a); } @@ -526,7 +571,7 @@ http_GetHdrField(const struct http *hp, const char *hdr, *ptr = NULL; h = NULL; - i = http_GetHdrToken(hp, hdr, field, &h); + i = http_GetHdrToken(hp, hdr, field, &h, NULL); if (!i) return (i); @@ -579,7 +624,7 @@ http_GetContentLength(const struct http *hp) enum sess_close http_DoConnection(struct http *hp) { - const char *p, *q; + const char *h, *b, *e; enum sess_close retval; unsigned u; @@ -589,26 +634,16 @@ http_DoConnection(struct http *hp) retval = SC_NULL; http_CollectHdr(hp, H_Connection); - if (!http_GetHdr(hp, H_Connection, &p)) + if (!http_GetHdr(hp, H_Connection, &h)) return (retval); - AN(p); - for (; *p; p++) { - if (vct_issp(*p)) - continue; - if (*p == ',') - continue; - for (q = p + 1; *q; q++) - if (*q == ',' || vct_issp(*q)) - break; - u = pdiff(p, q); - if (u == 5 && !strncasecmp(p, "close", u)) + AN(h); + while (http_split(&h, NULL, ",", &b, &e)) { + u = pdiff(b, e); + if (u == 5 && !strncasecmp(b, "close", u)) retval = SC_REQ_CLOSE; - if (u == 10 && !strncasecmp(p, "keep-alive", u)) + if (u == 10 && !strncasecmp(b, "keep-alive", u)) retval = SC_NULL; - http_MarkHeader(hp, p, u, HDF_FILTER); - if (!*q) - break; - p = q; + http_MarkHeader(hp, b, u, HDF_FILTER); } return (retval); } diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 7c086b9..20b366b 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -194,7 +194,7 @@ RFC2616_Req_Gzip(const struct http *hp) * p104 says to not do q values for x-gzip, so we just test * for its existence. */ - if (http_GetHdrToken(hp, H_Accept_Encoding, "x-gzip", NULL)) + if (http_GetHdrToken(hp, H_Accept_Encoding, "x-gzip", NULL, NULL)) return (1); /* @@ -266,7 +266,7 @@ RFC2616_Vary_AE(struct http *hp) { const char *vary; - if (http_GetHdrToken(hp, H_Vary, "Accept-Encoding", NULL)) + if (http_GetHdrToken(hp, H_Vary, "Accept-Encoding", NULL, NULL)) return; if (http_GetHdr(hp, H_Vary, &vary)) { http_Unset(hp, H_Vary); From phk at FreeBSD.org Mon Sep 29 20:21:50 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Sep 2014 22:21:50 +0200 Subject: [master] 48509ec Merge branch 'master' of git.varnish-cache.org:varnish-cache Message-ID: commit 48509ec868826b2f669d28bf9505b40b9973d5d4 Merge: f92e560 5822d0c Author: Poul-Henning Kamp Date: Mon Sep 29 20:21:46 2014 +0000 Merge branch 'master' of git.varnish-cache.org:varnish-cache From phk at FreeBSD.org Tue Sep 30 06:17:33 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 30 Sep 2014 08:17:33 +0200 Subject: [master] 2e765f3 Make token name comparison a robust and reusable function Message-ID: commit 2e765f310a24a7b66bd7452d626e0c9fd99ef89c Author: Poul-Henning Kamp Date: Tue Sep 30 06:17:16 2014 +0000 Make token name comparison a robust and reusable function diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 4735000..e655fa0 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -450,6 +450,40 @@ http_split(const char **src, const char *stop, const char *sep, } /*----------------------------------------------------------------------------- + * Comparison rule for tokens: + * if target string starts with '"', we use memcmp() and expect closing + * double quote as well + * otherwise we use strncascmp() + * + * On match we increment *bp past the token name. + */ + +static int +http_istoken(const char **bp, const char *e, const char *token) +{ + int fl = strlen(token); + const char *b; + + AN(bp); + AN(e); + AN(token); + + b = *bp; + + if (b + fl + 2 <= e && *b == '"' && + !memcmp(b + 1, token, fl) && b[fl + 1] == '"') { + *bp += fl + 2; + return (1); + } + if (b + fl <= e && !strncasecmp(b, token, fl) && + (b + fl == e || !vct_istchar(b[fl]))) { + *bp += fl; + return (1); + } + return (0); +} + +/*----------------------------------------------------------------------------- * Find a given data element (token) in a header according to RFC2616's #rule * (section 2.1, p15) * @@ -474,7 +508,6 @@ http_GetHdrToken(const struct http *hp, const char *hdr, const char *token, const char **pb, const char **pe) { const char *h, *b, *e; - unsigned fl; if (pb != NULL) *pb = NULL; @@ -483,18 +516,14 @@ http_GetHdrToken(const struct http *hp, const char *hdr, if (!http_GetHdr(hp, hdr, &h)) return (0); AN(h); - fl = strlen(token); - while(http_split(&h, NULL, ",", &b, &e)) { - if (*b == '"' && !memcmp(b + 1, token, fl) && b[fl + 1] == '"') + while(http_split(&h, NULL, ",", &b, &e)) + if (http_istoken(&b, e, token)) break; - if (!strncasecmp(b, token, fl) && !vct_istchar(b[fl])) - break; - } if (b == NULL) return (0); if (pb != NULL) { - for (b += fl; vct_islws(*b); b++) + for (; vct_islws(*b); b++) continue; if (b == e) { b = NULL; From phk at FreeBSD.org Tue Sep 30 09:09:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 30 Sep 2014 11:09:41 +0200 Subject: [master] 5aec889 Don't bother with any of the beresp.body transformations (esi/gzip/gunzip) if we know the length to be zero. Message-ID: commit 5aec8898c71b7cb8b18093af99733e4199d18e10 Author: Poul-Henning Kamp Date: Tue Sep 30 09:07:57 2014 +0000 Don't bother with any of the beresp.body transformations (esi/gzip/gunzip) if we know the length to be zero. This is inspired by ticket #1590, even though I can't seem to see what Varnish does wrong in that case to begin with. Fixes #1590 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 8ac532a..030c1bb 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -547,16 +547,18 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) (void)VFP_Push(bo->vfc, &vfp_gunzip, 1); - if (bo->do_esi && bo->do_gzip) { - (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); - } else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) { - (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); - } else if (bo->do_esi) { - (void)VFP_Push(bo->vfc, &vfp_esi, 1); - } else if (bo->do_gzip) { - (void)VFP_Push(bo->vfc, &vfp_gzip, 1); - } else if (bo->is_gzip && !bo->do_gunzip) { - (void)VFP_Push(bo->vfc, &vfp_testgunzip, 1); + if (bo->htc->content_length != 0) { + if (bo->do_esi && bo->do_gzip) { + (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); + } else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) { + (void)VFP_Push(bo->vfc, &vfp_esi_gzip, 1); + } else if (bo->do_esi) { + (void)VFP_Push(bo->vfc, &vfp_esi, 1); + } else if (bo->do_gzip) { + (void)VFP_Push(bo->vfc, &vfp_gzip, 1); + } else if (bo->is_gzip && !bo->do_gunzip) { + (void)VFP_Push(bo->vfc, &vfp_testgunzip, 1); + } } if (bo->fetch_objcore->flags & OC_F_PRIVATE) From phk at FreeBSD.org Tue Sep 30 09:20:41 2014 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 30 Sep 2014 11:20:41 +0200 Subject: [master] 4ad0d5b Remove the outdated and increasingly misleading dot(1) state graphs from the source code. We now have a working documentation system and these state engines should be documented there, using appropriate tools. Message-ID: commit 4ad0d5bdfe45eb31111fb1d0b1a4aa72942b477d Author: Poul-Henning Kamp Date: Tue Sep 30 09:19:45 2014 +0000 Remove the outdated and increasingly misleading dot(1) state graphs from the source code. We now have a working documentation system and these state engines should be documented there, using appropriate tools. Fixes #1487 diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index b85b099..d3438f8 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -29,41 +29,6 @@ * This file contains the two central state machine for pushing HTTP1 * sessions through their states. * - * The following dot-graph shows the big picture, and the two major - * complicating features: - * - * - The blue path is where a request disembarks its worker thread while - * waiting for a busy object to become available: - * - * - The green path is where we time out waiting for the next request to - * arrive, release the worker thread and hand the session to the waiter. - * - * Render the graph with: - * sed -n '/^..DOT/s///p' % | dot -Tps > /tmp/_.ps - * - *DOT digraph vcl_center { - *DOT size="7.2,10.5" - *DOT margin="0.5" - *DOT center="1" - *DOT - *DOT acceptor -> http1_wait [label=S_STP_NEWREQ, align=center] - *DOT hash -> CNT_Request [label="Busy object\nS_STP_WORKING\nR_STP_LOOKUP" - *DOT color=blue] - *DOT disembark -> hash [style=dotted, color=blue] - *DOT http1_wait -> CNT_Request [label="S_STP_WORKING\nR_STP_RECV"] - *DOT http1_wait -> disembark [label="Session close"] - *DOT http1_wait -> disembark [label="Timeout" color=green] - *DOT disembark -> waiter [style=dotted, color=green] - *DOT waiter -> http1_wait [color=green] - *DOT CNT_Request -> disembark - *DOT [label="Busy object\nS_STP_WORKING\nR_STP_LOOKUP" color=blue] - *DOT CNT_Request -> http1_cleanup - *DOT http1_cleanup -> disembark [label="Session close"] - *DOT http1_cleanup -> CNT_Request [label="S_STP_WORKING\nR_STP_RECV"] - *DOT http1_cleanup -> http1_wait [label="S_STP_NEWREQ"] - *DOT - *DOT } - * */ #include "config.h" diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index bba2dea..c9bdb35 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -34,28 +34,6 @@ * A special complication is the fact that we can suspend processing of * a request when hash-lookup finds a busy objhdr. * - * Since the states are rather nasty in detail, I have decided to embedd - * a dot(1) graph in the source code comments. So to see the big picture, - * extract the DOT lines and run though dot(1), for instance with the - * command: - * sed -n '/^DOT/s///p' cache/cache_req_fsm.c | dot -Tps > /tmp/_.ps - */ - -/* -DOT digraph vcl_center { -xDOT page="8.2,11.5" -DOT size="7.2,10.5" -DOT margin="0.5" -DOT center="1" -DOT acceptor [ -DOT shape=hexagon -DOT label="Request received" -DOT ] -DOT ESI_REQ [ shape=hexagon ] -DOT ESI_REQ -> recv -DOT SYNTH [shape=plaintext] -DOT RESTART [shape=plaintext] -DOT acceptor -> recv [style=bold,color=green] */ #include "config.h" @@ -72,16 +50,7 @@ DOT acceptor -> recv [style=bold,color=green] #include "vtim.h" /*-------------------------------------------------------------------- - * Deliver an already stored object - * -DOT deliver [ -DOT shape=record -DOT label="{cnt_deliver:|Filter obj.-\>resp.|{vcl_deliver\{\}|{req.|resp.}}|{restart?|deliver?}}" -DOT ] -DOT deliver:deliver:s -> DONE [style=bold,color=green] -DOT deliver:deliver:s -> DONE [style=bold,color=red] -DOT deliver:deliver:s -> DONE [style=bold,color=blue] - * + * Deliver an object to client */ static enum req_fsm_nxt @@ -199,15 +168,6 @@ cnt_deliver(struct worker *wrk, struct req *req) /*-------------------------------------------------------------------- * Emit a synthetic response - * -DOT subgraph xcluster_synth { -DOT synth [ -DOT shape=record -DOT label="{cnt_synth:|{vcl_synth\{\}|resp.}|{deliver?|restart?}}" -DOT ] -DOT SYNTH -> synth -DOT synth:del:s -> deliver [label=deliver] -DOT } */ static enum req_fsm_nxt @@ -275,16 +235,6 @@ cnt_synth(struct worker *wrk, struct req *req) /*-------------------------------------------------------------------- * Initiated a fetch (pass/miss) which we intend to deliver - * -DOT subgraph xcluster_body { -DOT fetch [ -DOT shape=record -DOT label="{cnt_fetch:|wait for fetch|{OK|Failed}}" -DOT ] -DOT } -DOT fetch:ok:s -> deliver [style=bold,color=red] -DOT fetch:ok:s -> deliver [style=bold,color=blue] -DOT fetch:err:s -> vcl_error */ static enum req_fsm_nxt @@ -311,31 +261,8 @@ cnt_fetch(struct worker *wrk, struct req *req) } /*-------------------------------------------------------------------- - * LOOKUP - * Hash things together and look object up in hash-table. - * - * LOOKUP consists of two substates so that we can reenter if we - * encounter a busy object. - * -DOT subgraph xcluster_lookup { -DOT lookup [ -DOT shape=record -DOT label="{cnt_lookup:|hash lookup|{busy?|exp?|exp+busy?|hit?|miss?|hit-for-pass?}}" -DOT ] -DOT lookup2 [ -DOT shape=record -DOT label="{cnt_lookup:|{vcl_hit\{\}|req.*, obj.*}|{deliver?|synth?|restart?|fetch?|pass?}}" -DOT ] -DOT } -DOT lookup:busy:w -> lookup:top:w [label="(waitinglist)"] -DOT lookup:miss:s -> miss [style=bold,color=blue] -DOT lookup:hfp:s -> pass [style=bold,color=red] -DOT lookup:e:s -> lookup2 [style=bold,color=green] -DOT lookup:eb:s -> lookup2 [style=bold,color=green] -DOT lookup:h:s -> lookup2 [style=bold,color=green] -DOT lookup2:pass:s -> pass [style=bold,color=red] -DOT lookup2:fetch:s -> miss [style=bold,color=blue] -DOT lookup2:deliver:s -> deliver:n [style=bold,color=green] + * Attempt to lookup objhdr from hash. We disembark and reenter + * this state if we get suspended on a busy objhdr. */ static enum req_fsm_nxt @@ -467,17 +394,7 @@ cnt_lookup(struct worker *wrk, struct req *req) } /*-------------------------------------------------------------------- - * We had a miss, ask VCL, proceed as instructed - * -DOT subgraph xcluster_miss { -DOT miss [ -DOT shape=record -DOT label="{cnt_miss:|{vcl_miss\{\}|req.*}|{fetch?|synth?|restart?|pass?}}" -DOT ] -DOT } -DOT miss:fetch:s -> fetch [label="fetch",style=bold,color=blue] -DOT miss:pass:s -> pass [label="pass",style=bold,color=red] -DOT + * Cache miss. */ static enum req_fsm_nxt @@ -518,20 +435,7 @@ cnt_miss(struct worker *wrk, struct req *req) } /*-------------------------------------------------------------------- - * Start pass processing by getting headers from backend, then - * continue in passbody. - * -DOT subgraph xcluster_pass { -DOT pass [ -DOT shape=record -DOT label="{cnt_pass:|{vcl_pass\{\}|req.*}|{fetch?|synth?|restart?}}" -DOT ] -DOT } -DOT pass:fetch:s -> fetch:n [style=bold, color=red] -XDOT pass:rst -> rst_pass [label="restart",color=purple] -XDOT rst_pass [label="RESTART",shape=plaintext] -XDOT pass:err -> err_pass [label="error"] -XDOT err_pass [label="ERROR",shape=plaintext] + * Pass processing */ static enum req_fsm_nxt @@ -565,21 +469,7 @@ cnt_pass(struct worker *wrk, struct req *req) } /*-------------------------------------------------------------------- - * Ship the request header to the backend unchanged, then pipe - * until one of the ends close the connection. - * -DOT subgraph xcluster_pipe { -DOT pipe [ -DOT shape=record -DOT label="{cnt_pipe:|filter req.*-\>bereq.*|{vcl_pipe()|req.*, bereq\.*}|{pipe?|synth?}}" -DOT ] -DOT pipe_do [ -DOT shape=ellipse -DOT label="send bereq.\npipe until close" -DOT ] -DOT pipe:pipe -> pipe_do [label="pipe",style=bold,color=orange] -DOT } -DOT pipe_do -> DONE [style=bold,color=orange] + * Pipe mode */ static enum req_fsm_nxt @@ -618,17 +508,7 @@ cnt_pipe(struct worker *wrk, struct req *req) } /*-------------------------------------------------------------------- - * -DOT subgraph xcluster_restart { -DOT restart [ -DOT shape=record -DOT label="{cnt_restart}" -DOT ] -DOT } -DOT RESTART -> restart [color=purple] -DOT restart -> recv [color=purple] -DOT restart -> err_restart -DOT err_restart [label="SYNTH",shape=plaintext] + * Handle restart events */ static enum req_fsm_nxt @@ -656,23 +536,9 @@ cnt_restart(struct worker *wrk, struct req *req) } /*-------------------------------------------------------------------- - * RECV * We have a complete request, set everything up and start it. * We can come here both with a request from the client and with * a interior request during ESI delivery. - * -DOT subgraph xcluster_recv { -DOT recv [ -DOT shape=record -DOT label="{cnt_recv:|{vcl_recv\{\}|req.*}|{vcl_hash\{\}|req.*}|{lookup?|pass?|pipe?|synth?|purge?}}" -DOT ] -DOT } -DOT recv:pipe -> pipe [style=bold,color=orange] -DOT recv:pass -> pass [style=bold,color=red] -DOT recv:lookup:s -> lookup [style=bold,color=green] -DOT recv:purge:s -> purge [style=bold,color=purple] -#DOT recv:error -> err_recv -#DOT err_recv [label="ERROR",shape=plaintext] */ static enum req_fsm_nxt @@ -791,17 +657,9 @@ cnt_recv(struct worker *wrk, struct req *req) } /*-------------------------------------------------------------------- - * PURGE - * Find the objhead, purge it and ask VCL if we should fetch or - * just return. - * XXX: fetching not implemented yet. + * Find the objhead, purge it. * -DOT subgraph xcluster_purge { -DOT purge [ -DOT shape=record -DOT label="{cnt_purge:|{vcl_purge\{\}|req.*}|{synth?}}" -DOT ] -DOT } + * XXX: We should ask VCL if we should fetch a new copy of the object. */ static enum req_fsm_nxt @@ -953,7 +811,3 @@ CNT_AcctLogCharge(struct dstat *ds, struct req *req) #include "tbl/acct_fields_req.h" #undef ACCT } - -/* -DOT } -*/