From phk at FreeBSD.org Thu Sep 1 07:34:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 01 Sep 2016 09:34:11 +0200 Subject: [master] f9eec3b Respect remote MAX_FRAME_SIZE Message-ID: commit f9eec3b3cb52ba6a31ba13f95cd1fb7f1c4f0dd5 Author: Poul-Henning Kamp Date: Thu Sep 1 07:33:17 2016 +0000 Respect remote MAX_FRAME_SIZE diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 1378ef8..3bf1fdc 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -50,7 +50,12 @@ enum h2_stream_e { #define H2_FRAME_FLAGS(l,u,v) extern const uint8_t H2FF_##u; #include "tbl/h2_frames.h" -#define H2_SETTINGS_N 7 +enum h2setting { +#define H2_SETTINGS(n,v,d) H2S_##n = v, +#include "tbl/h2_settings.h" +#undef H2_SETTINGS + H2_SETTINGS_N +}; struct h2_req { unsigned magic; diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 14b3d8a..f205e23 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -44,7 +44,6 @@ #include "../cache/cache_filter.h" #include "../cache/cache_transport.h" -#include "../http1/cache_http1.h" #include "../http2/cache_http2.h" #include "vct.h" diff --git a/bin/varnishd/http2/cache_http2_panic.c b/bin/varnishd/http2/cache_http2_panic.c index 085c5c4..0605862 100644 --- a/bin/varnishd/http2/cache_http2_panic.c +++ b/bin/varnishd/http2/cache_http2_panic.c @@ -47,8 +47,6 @@ #include "vend.h" #include "vsb.h" -#include "vtcp.h" -#include "vtim.h" void h2_sess_panic(struct vsb *vsb, const struct sess *sp) diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index e013b6d..4175367 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -55,12 +55,6 @@ enum h2frame { #include "tbl/h2_frames.h" }; -enum h2setting { -#define H2_SETTINGS(n,v,d) H2S_##n = v, -#include "tbl/h2_settings.h" -#undef H2_SETTINGS -}; - static const char * h2_framename(enum h2frame h2f) { diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index 70f48bf..0ea2690 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -100,15 +100,36 @@ H2_Send(struct worker *wrk, struct h2_req *r2, int flush, { int retval; struct h2_sess *h2; + uint32_t mfs, tf; + const char *p; (void)flush; + AN(ptr); CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); h2 = r2->h2sess; CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); Lck_Lock(&h2->sess->mtx); - retval = H2_Send_Frame(wrk, h2, type, flags, len, r2->stream, ptr); + mfs = h2->their_settings[H2S_MAX_FRAME_SIZE]; + if (len < mfs) { + retval = H2_Send_Frame(wrk, h2, + type, flags, len, r2->stream, ptr); + } else if (type == H2_FRAME_DATA) { + p = ptr; + do { + tf = mfs; + if (tf > len) + tf = len; + retval = H2_Send_Frame(wrk, h2, type, + tf == len ? flags : 0, + tf, r2->stream, p); + p += tf; + len -= tf; + } while (len > 0); + } else { + INCOMPL(); + } Lck_Unlock(&h2->sess->mtx); return (retval); } From phk at FreeBSD.org Thu Sep 1 10:05:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 01 Sep 2016 12:05:11 +0200 Subject: [master] b2eb0a4 Flush our own random generator, not libcs. Message-ID: commit b2eb0a4d36de1d0b345c559b03a20c331082a8e0 Author: Poul-Henning Kamp Date: Thu Sep 1 10:04:24 2016 +0000 Flush our own random generator, not libcs. diff --git a/bin/varnishtest/tests/d00004.vtc b/bin/varnishtest/tests/d00004.vtc index 32506c4..5b9037d 100644 --- a/bin/varnishtest/tests/d00004.vtc +++ b/bin/varnishtest/tests/d00004.vtc @@ -3,6 +3,8 @@ varnishtest "Deeper test of random director" server s1 { rxreq txresp -body "1" + rxreq + txresp -body "1" } -start server s2 { @@ -10,10 +12,6 @@ server s2 { txresp -body "22" rxreq txresp -body "22" - rxreq - txresp -body "22" - rxreq - txresp -body "22" } -start server s3 { @@ -30,6 +28,8 @@ server s3 { server s4 { rxreq txresp -body "4444" + rxreq + txresp -body "4444" } -start varnish v1 -vcl+backend { @@ -57,19 +57,19 @@ varnish v1 -cliok "debug.srandom" client c1 { txreq rxresp - expect resp.bodylen == 2 + expect resp.bodylen == 3 txreq rxresp - expect resp.bodylen == 2 + expect resp.bodylen == 4 txreq rxresp expect resp.bodylen == 1 txreq rxresp - expect resp.bodylen == 2 + expect resp.bodylen == 1 txreq rxresp - expect resp.bodylen == 3 + expect resp.bodylen == 2 txreq rxresp expect resp.bodylen == 2 diff --git a/bin/varnishtest/tests/m00002.vtc b/bin/varnishtest/tests/m00002.vtc index 0f5ade5..e0bc560 100644 --- a/bin/varnishtest/tests/m00002.vtc +++ b/bin/varnishtest/tests/m00002.vtc @@ -21,8 +21,8 @@ varnish v1 -cliok "debug.srandom" client c1 { txreq rxresp - expect resp.http.rnd1 == 0.256 - expect resp.http.rnd2 == 1.643 - expect resp.http.rnd3 == 9.011 - expect resp.http.rnd4 == 99.654 + expect resp.http.rnd1 == 0.656 + expect resp.http.rnd2 == 0.390 + expect resp.http.rnd3 == 8.585 + expect resp.http.rnd4 == 99.636 } -run diff --git a/lib/libvarnish/vrnd.c b/lib/libvarnish/vrnd.c index c55d73e..d586b08 100644 --- a/lib/libvarnish/vrnd.c +++ b/lib/libvarnish/vrnd.c @@ -106,7 +106,7 @@ VRND_SeedTestable(unsigned int x) rptr = &state[0]; lim = 10 * rand_deg; for (i = 0; i < lim; i++) - (void)random(); + (void)VRND_RandomTestable(); } long From document at varnish-cache.org Thu Sep 1 11:53:51 2016 From: document at varnish-cache.org (document at varnish-cache.org) Date: Thu, 01 Sep 2016 17:23:51 +0530 Subject: Please find attached invoice no: 83081085 Message-ID: <7120-526ee8-9b7844-78ac-42D23@varnish-cache.org> Attached is a Print Manager form. Format = Portable Document Format File (PDF) ________________________________ Disclaimer This email/fax transmission is confidential and intended solely for the person or organisation to whom it is addressed. If you are not the intended recipient, you must not copy, distribute or disseminate the information, or take any action in reliance of it. Any views expressed in this message are those of the individual sender, except where the sender specifically states them to be the views of any organisation or employer. If you have received this message in error, do not open any attachment but please notify the sender (above) deleting this message from your system. For email transmissions please rely on your own virus check no responsibility is taken by the sender for any damage rising out of any bug or virus infection. -------------- next part -------------- A non-text attachment was scrubbed... Name: 5f11.zip Type: application/zip Size: 8368 bytes Desc: not available URL: From dridi.boukelmoune at gmail.com Thu Sep 1 17:04:10 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 01 Sep 2016 19:04:10 +0200 Subject: [master] 2d98cc5 Add VCL labels to the request state machine Message-ID: commit 2d98cc5f29a82d7651b3680f4608dbf8e4747307 Author: Dridi Boukelmoune Date: Thu Sep 1 18:54:31 2016 +0200 Add VCL labels to the request state machine diff --git a/doc/graphviz/cache_req_fsm.dot b/doc/graphviz/cache_req_fsm.dot index 183d603..a9bc854 100644 --- a/doc/graphviz/cache_req_fsm.dot +++ b/doc/graphviz/cache_req_fsm.dot @@ -47,16 +47,16 @@ digraph cache_req_fsm { fini:ok -> ok */ - acceptor [ - shape=hexagon - label="Request received" - ] - ESI_REQ [ shape=hexagon ] + acceptor [shape=hexagon label="Request received"] + label_select [shape=hexagon label="Label selected"] + ESI_REQ [shape=hexagon label="ESI request"] RESTART [shape=plaintext] ESI_REQ -> recv SYNTH [shape=plaintext] + LABEL [shape=plaintext] acceptor -> recv [style=bold] + label_select -> recv [style=bold] subgraph xcluster_deliver { /* cnt_deliver() */ @@ -187,7 +187,7 @@ digraph cache_req_fsm { subgraph xcluster_recv { recv [ shape=record - label="{cnt_recv:|{vcl_recv\{\}|req.*}|{hash|purge|pass|pipe|synth}}" + label="{cnt_recv:|{vcl_recv\{\}|req.*}|{hash|purge|pass|pipe|synth|vcl}}" ] recv:hash -> hash [style=bold,color=green] hash [ @@ -204,6 +204,9 @@ digraph cache_req_fsm { recv:purge:s -> hash [style=bold,color=purple] + recv:vcl:s -> vcl_label + vcl_label [label="LABEL",shape=plaintext] + /* cnt_purge */ subgraph xcluster_purge { purge [ @@ -211,4 +214,4 @@ digraph cache_req_fsm { label="{cnt_purge:|{vcl_purge\{\}|req.*}|{synth|restart}}" ] } -} \ No newline at end of file +} diff --git a/doc/graphviz/cache_req_fsm.svg b/doc/graphviz/cache_req_fsm.svg index 21828c8..26d4564 100644 --- a/doc/graphviz/cache_req_fsm.svg +++ b/doc/graphviz/cache_req_fsm.svg @@ -4,445 +4,470 @@ - + cache_req_fsm - + cluster_backend - + acceptor - -Request received + +Request received -recv - -cnt_recv: - -vcl_recv{} - -req.* - -hash - -purge - -pass - -pipe - -synth +recv + +cnt_recv: + +vcl_recv{} + +req.* + +hash + +purge + +pass + +pipe + +synth + +vcl acceptor->recv - - + + + + +label_select + +Label selected + + +label_select->recv + + -ESI_REQ - -ESI_REQ +ESI_REQ + +ESI request ESI_REQ->recv - - + + -RESTART -RESTART +RESTART +RESTART -restart - -cnt_restart: - -ok? - -max_restarts? +restart + +cnt_restart: + +ok? + +max_restarts? -RESTART->restart - - +RESTART->restart + + -hash - -cnt_recv: - -vcl_hash{} - -req.* - -lookup +hash + +cnt_recv: + +vcl_hash{} + +req.* + +lookup -recv:hash->hash - - +recv:hash->hash + + -recv:pipe->hash - - +recv:pipe->hash + + -recv:pass->hash - - +recv:pass->hash + + -recv:purge:s->hash - - +recv:purge:s->hash + + + + +vcl_label +LABEL + + +recv:vcl:s->vcl_label + + -SYNTH -SYNTH +SYNTH +SYNTH -synth - -cnt_synth: - -vcl_synth{} - -req.* - -resp.* - -deliver - -restart +synth + +cnt_synth: + +vcl_synth{} + +req.* + +resp.* + +deliver + +restart -SYNTH->synth - - +SYNTH->synth + + + + +LABEL +LABEL -deliver - -cnt_deliver: - -Filter obj.->resp. - -vcl_deliver{} - -req.* - -resp.* - -restart - -deliver - -synth +deliver + +cnt_deliver: + +Filter obj.->resp. + +vcl_deliver{} + +req.* + +resp.* + +restart + +deliver + +synth -V1D_Deliver +V1D_Deliver V1D_Deliver -deliver:deliver:s->V1D_Deliver - - - - deliver:deliver:s->V1D_Deliver - - + + deliver:deliver:s->V1D_Deliver - - + + + + +deliver:deliver:s->V1D_Deliver + + -DONE - -DONE +DONE + +DONE -V1D_Deliver->DONE - - +V1D_Deliver->DONE + + -stream - -stream? -body +stream + +stream? +body -stream->V1D_Deliver - - +stream->V1D_Deliver + + -synth:del:s->V1D_Deliver - - +synth:del:s->V1D_Deliver + + -see backend graph -see backend graph +see backend graph +see backend graph -BGFETCH - -BGFETCH +BGFETCH + +BGFETCH -FETCH - -FETCH +FETCH + +FETCH -FETCH_DONE - -FETCH_DONE +FETCH_DONE + +FETCH_DONE -FETCH->FETCH_DONE - - +FETCH->FETCH_DONE + + -FETCH_FAIL - -FETCH_FAIL +FETCH_FAIL + +FETCH_FAIL -FETCH->FETCH_FAIL - - +FETCH->FETCH_FAIL + + -FETCH_DONE->deliver - - +FETCH_DONE->deliver + + -FETCH_DONE->deliver - - +FETCH_DONE->deliver + + -FETCH_FAIL->synth - - +FETCH_FAIL->synth + + -lookup2 - -cnt_lookup: - -vcl_hit{} - -req.* - -obj.* - -deliver - -miss - -restart - -synth - -pass +lookup2 + +cnt_lookup: + +vcl_hit{} + +req.* + +obj.* + +deliver + +miss + +restart + +synth + +pass -lookup2:deliver:s->deliver:n - - +lookup2:deliver:s->deliver:n + + -lookup2:deliver:s->BGFETCH - - -parallel -if obj expired +lookup2:deliver:s->BGFETCH + + +parallel +if obj expired -miss - -cnt_miss: - -vcl_miss{} - -req.* - -fetch - -synth - -restart - -pass +miss + +cnt_miss: + +vcl_miss{} + +req.* + +fetch + +synth + +restart + +pass -lookup2:miss:s->miss - - +lookup2:miss:s->miss + + -pass - -cnt_pass: - -vcl_pass{} - -req.* - -fetch - -synth - -restart +pass + +cnt_pass: + +vcl_pass{} + +req.* + +fetch + +synth + +restart -lookup2:pass:s->pass - - +lookup2:pass:s->pass + + -lookup - -cnt_lookup: - -hash lookup - -hit? - -miss? - -hit-for-pass? - -busy? +lookup + +cnt_lookup: + +hash lookup + +hit? + +miss? + +hit-for-pass? + +busy? -lookup:h:s->lookup2 - - +lookup:h:s->lookup2 + + -lookup:busy:e->lookup:top:e - - -(waitinglist) +lookup:busy:e->lookup:top:e + + +(waitinglist) -lookup:miss:s->miss - - +lookup:miss:s->miss + + -lookup:hfp:s->pass - - +lookup:hfp:s->pass + + -miss:fetch:s->FETCH - - +miss:fetch:s->FETCH + + -miss:pass:s->pass - - +miss:pass:s->pass + + -pass:fetch:s->FETCH - - +pass:fetch:s->FETCH + + -pipe - -cnt_pipe: - -filter req.*->bereq.* - -vcl_pipe{} - -req.* - -bereq.* - -pipe - -synth +pipe + +cnt_pipe: + +filter req.*->bereq.* + +vcl_pipe{} + +req.* + +bereq.* + +pipe + +synth -pipe_do - -send bereq, -copy bytes until close +pipe_do + +send bereq, +copy bytes until close -pipe:pipe->pipe_do - - +pipe:pipe->pipe_do + + -pipe_do->DONE - - +pipe_do->DONE + + -restart:ok:s->recv - - +restart:ok:s->recv + + -err_restart -SYNTH +err_restart +SYNTH -restart:max:s->err_restart - - +restart:max:s->err_restart + + -hash:lookup:w->lookup - - +hash:lookup:w->lookup + + -hash:lookup:s->pass - - +hash:lookup:s->pass + + -hash:lookup:e->pipe - - +hash:lookup:e->pipe + + -purge - -cnt_purge: - -vcl_purge{} - -req.* - -synth - -restart +purge + +cnt_purge: + +vcl_purge{} + +req.* + +synth + +restart -hash:lookup:s->purge:top:n - - +hash:lookup:s->purge:top:n + + From office at varnish-cache.org Fri Sep 2 01:26:57 2016 From: office at varnish-cache.org (office at varnish-cache.org) Date: Fri, 02 Sep 2016 08:26:57 +0700 Subject: Scanned image from MX2310U@varnish-cache.org Message-ID: <20160902082657.2B00.OFFICE@varnish-cache.org> Reply to: office at varnish-cache.org Device Name: MX2310U at varnish-cache.org Device Model: MX-2310U Location: Reception File Format: PDF MMR(G4) Resolution: 200dpi x 200dpi Attached file is scanned image in PDF format. Use Acrobat(R)Reader(R) or Adobe(R)Reader(R) of Adobe Systems Incorporated to view the document. Adobe(R)Reader(R) can be downloaded from the following URL: Adobe, the Adobe logo, Acrobat, the Adobe PDF logo, and Reader are registered trademarks or trademarks of Adobe Systems Incorporated in the United States and other countries. http://www.adobe.com/ -------------- next part -------------- A non-text attachment was scrubbed... Name: office at varnish-cache.org_20160902_082657.docm Type: application/vnd.ms-word.document.macroenabled.12 Size: 37477 bytes Desc: not available URL: From dridi.boukelmoune at gmail.com Fri Sep 2 10:31:10 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 02 Sep 2016 12:31:10 +0200 Subject: [master] f02d470 Typo Message-ID: commit f02d4700c4fd0183e67ddebc1fb24785af754407 Author: Dridi Boukelmoune Date: Fri Sep 2 12:30:27 2016 +0200 Typo diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index b4727d9..94d05b7 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -88,7 +88,7 @@ SLTM(SessOpen, 0, "Client connection opened", */ SLTM(SessClose, 0, "Client connection closed", - "SessionClose is the last record for any client connection.\n\n" + "SessClose is the last record for any client connection.\n\n" "The format is::\n\n" "\t%s %f\n" "\t| |\n" From dridi.boukelmoune at gmail.com Sat Sep 3 06:18:08 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Sat, 03 Sep 2016 08:18:08 +0200 Subject: [master] c99382d Fix build for 32bit systems Message-ID: commit c99382d6519364715a36cbd000cbfa0dcb576fb7 Author: Dridi Boukelmoune Date: Sat Sep 3 08:08:42 2016 +0200 Fix build for 32bit systems diff --git a/bin/varnishd/hpack/vhp_decode.c b/bin/varnishd/hpack/vhp_decode.c index 334db2d..49a0e74 100644 --- a/bin/varnishd/hpack/vhp_decode.c +++ b/bin/varnishd/hpack/vhp_decode.c @@ -1190,15 +1190,15 @@ main(int argc, char **argv) } if (verbose) { - printf("sizeof (struct vhd_int)=%ju\n", + printf("sizeof (struct vhd_int)=%zu\n", sizeof (struct vhd_int)); - printf("sizeof (struct vhd_lookup)=%ju\n", + printf("sizeof (struct vhd_lookup)=%zu\n", sizeof (struct vhd_lookup)); - printf("sizeof (struct vhd_raw)=%ju\n", + printf("sizeof (struct vhd_raw)=%zu\n", sizeof (struct vhd_raw)); - printf("sizeof (struct vhd_huffman)=%ju\n", + printf("sizeof (struct vhd_huffman)=%zu\n", sizeof (struct vhd_huffman)); - printf("sizeof (struct vhd_decode)=%ju\n", + printf("sizeof (struct vhd_decode)=%zu\n", sizeof (struct vhd_decode)); } diff --git a/bin/varnishd/hpack/vhp_table.c b/bin/varnishd/hpack/vhp_table.c index 26a046b..547a446 100644 --- a/bin/varnishd/hpack/vhp_table.c +++ b/bin/varnishd/hpack/vhp_table.c @@ -767,9 +767,9 @@ main(int argc, char **argv) } if (verbose) { - printf("sizeof (struct vht_table) == %ju\n", + printf("sizeof (struct vht_table) == %zu\n", sizeof (struct vht_table)); - printf("sizeof (struct vht_entry) == %ju\n", + printf("sizeof (struct vht_entry) == %zu\n", sizeof (struct vht_entry)); printf("\n"); } diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 4a90d24..224b308 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -1084,7 +1085,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) * Id of the stream this one depends on. */ else if (!strcmp(spec, "stream.window")) { - snprintf(buf, 20, "%ld", s->id ? s->ws : s->hp->ws); + snprintf(buf, 20, "%" PRIu64 "d", s->id ? s->ws : s->hp->ws); return (buf); } else if (!strcmp(spec, "stream.weight")) { @@ -2735,7 +2736,7 @@ b64_settings(const struct http *hp, const char *s) HPK_ResizeTbl(hp->decctx, v); } - vtc_log(hp->vl, 4, "Upgrade: %s (%d): %ld", buf, i, v); + vtc_log(hp->vl, 4, "Upgrade: %s (%d): %" PRIu64 "d", buf, i, v); } } From phk at FreeBSD.org Mon Sep 5 06:09:08 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 05 Sep 2016 08:09:08 +0200 Subject: [master] 3a4532b Add gquintar's HTTP2 "internal" varnishtest cases. Message-ID: commit 3a4532bfa68ac7f90a5c390d811346fc62d700c7 Author: Poul-Henning Kamp Date: Mon Sep 5 06:07:29 2016 +0000 Add gquintar's HTTP2 "internal" varnishtest cases. White-space polishing and fix extraneous 'd' appended to expect values in previous commit. diff --git a/bin/varnishtest/tests/README b/bin/varnishtest/tests/README index 971ee3d..80075ea 100644 --- a/bin/varnishtest/tests/README +++ b/bin/varnishtest/tests/README @@ -13,19 +13,18 @@ Naming scheme [id]%05d.vtc - id ~ [a] --> varnishtest(1) tests - id ~ [b] --> Basic functionality tests - id ~ [c] --> Complex functionality tests - id ~ [d] --> Director VMOD tests - id ~ [e] --> ESI tests - id ~ [g] --> GZIP tests - id ~ [j] --> JAIL tests - id ~ [l] --> VSL tests - id ~ [m] --> VMOD tests excluding director - id ~ [o] --> prOxy protocol - id ~ [p] --> Persistent tests - id ~ [r] --> Regression tests, same number as ticket - id ~ [s] --> Slow tests, expiry, grace etc. - id ~ [t] --> sTreaming tests - id ~ [u] --> Unusual background processes - id ~ [v] --> VCL tests: execute VRT functions + id ~ ^a --> varnishtest(1) tests + id ~ ^b --> Basic functionality tests + id ~ ^c --> Complex functionality tests + id ~ ^d --> Director VMOD tests + id ~ ^e --> ESI tests + id ~ ^g --> GZIP tests + id ~ ^j --> JAIL tests + id ~ ^l --> VSL tests + id ~ ^m --> VMOD tests excluding director + id ~ ^o --> prOxy protocol + id ~ ^p --> Persistent tests + id ~ ^r --> Regression tests, same number as ticket + id ~ ^s --> Slow tests, expiry, grace etc. + id ~ ^u --> Unusual background processes + id ~ ^v --> VCL tests: execute VRT functions diff --git a/bin/varnishtest/tests/a02000.vtc b/bin/varnishtest/tests/a02000.vtc new file mode 100644 index 0000000..c272dac --- /dev/null +++ b/bin/varnishtest/tests/a02000.vtc @@ -0,0 +1,50 @@ +varnishtest "Close/accept after H/2 upgrade" + +server s1 { + stream 1 { + rxreq + txresp + } -run + + expect_close + accept + + rxreq + txresp + + close + accept + + rxreq + txresp + + stream 1 { + rxreq + txresp + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + txreq + rxresp + } -run +} -run + +client c1 -connect ${s1_sock} { + txreq + rxresp + +} -run + +client c1 -connect ${s1_sock} { + txreq + rxresp + + stream 1 { + txreq + rxresp + } -run +} -run + +server s1 -wait diff --git a/bin/varnishtest/tests/a02001.vtc b/bin/varnishtest/tests/a02001.vtc new file mode 100644 index 0000000..3329579 --- /dev/null +++ b/bin/varnishtest/tests/a02001.vtc @@ -0,0 +1,96 @@ +varnishtest "Quickly test all frames" + +server s1 { + rxpri + stream 0 { +# PRIO + txprio -stream 23456 -weight 123 + +# RST + txrst -err 2 + +# SETTINGS + txsettings -push true -hdrtbl 11111111 -maxstreams 222222 -winsize 333333 -framesize 444444 -hdrsize 555555 + txsettings -ack + +# PING + txping -data "01234567" + txping -data "abcdefgh" -ack + +# GOAWAY + txgoaway -laststream 17432423 -err 12 -debug "kangaroo" + +# WINUP + txwinup -size 500 + + +# FRAME + txresp -body "floubidou" + + +# FRAME + txresp -body "tata" + } -run +} -start + +client c1 -connect ${s1_sock} { + txpri + stream 0 { +# PRIO + rxprio + expect prio.stream == 23456 + expect prio.weight == 123 + +# RST + rxrst + expect rst.err >= 2 + expect rst.err < 3 + +# SETTINGS + rxsettings + expect settings.hdrtbl == 11111111 + expect settings.maxstreams == 222222 + expect settings.winsize == 333333 + expect settings.framesize == 444444 + expect settings.hdrsize == 555555 + + rxsettings + expect settings.ack == true + expect settings.hdrtbl == + expect settings.maxstreams == + expect settings.winsize == + expect settings.framesize == + expect settings.hdrsize == + +# PING + rxping + expect ping.ack == "false" + expect ping.data == "01234567" + expect ping.data != "O1234567" + rxping + expect ping.ack == "true" + expect ping.data == "abcdefgh" + expect ping.data != "abcdefgt" + +# GOAWAY + rxgoaway + expect goaway.err == 12 + expect goaway.laststream == 17432423 + expect goaway.debug == "kangaroo" + +# WINUP + rxwinup + expect winup.size == 500 + +# FRAME + rxhdrs + rxdata + expect frame.data == "floubidou" + expect frame.type == 0 + expect frame.size == 9 + expect frame.stream == 0 + + rxresp + expect resp.body == "floubidoutata" + } -run +} -run diff --git a/bin/varnishtest/tests/a02002.vtc b/bin/varnishtest/tests/a02002.vtc new file mode 100644 index 0000000..5f11fbf --- /dev/null +++ b/bin/varnishtest/tests/a02002.vtc @@ -0,0 +1,20 @@ +varnishtest "Trigger a compression error via bad index" + +server s1 { + non-fatal + stream 1 { + rxreq + expect req.http.foo == + txgoaway -laststream 0 -err 9 -debug "COMPRESSION_ERROR" + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + txreq -idxHdr 100 -litHdr inc plain "foo" plain "bar" + rxgoaway + expect goaway.err == 9 + expect goaway.laststream == 0 + expect goaway.debug == "COMPRESSION_ERROR" + } -run +} -run diff --git a/bin/varnishtest/tests/a02003.vtc b/bin/varnishtest/tests/a02003.vtc new file mode 100644 index 0000000..6e0bb50 --- /dev/null +++ b/bin/varnishtest/tests/a02003.vtc @@ -0,0 +1,17 @@ +varnishtest "Check bodylen" + +server s1 { + stream 1 { + rxreq + expect req.bodylen == 3 + txresp -bodylen 7 + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + txreq -bodylen 3 + rxresp + expect resp.bodylen == 7 + } -run +} -run diff --git a/bin/varnishtest/tests/a02004.vtc b/bin/varnishtest/tests/a02004.vtc new file mode 100644 index 0000000..992b8a1 --- /dev/null +++ b/bin/varnishtest/tests/a02004.vtc @@ -0,0 +1,16 @@ +varnishtest "Simple request with body" + +server s1 { + stream 1 { + rxreq + txresp -body "bob" + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + txreq + rxresp + expect resp.bodylen == 3 + } -run +} -run diff --git a/bin/varnishtest/tests/a02005.vtc b/bin/varnishtest/tests/a02005.vtc new file mode 100644 index 0000000..986a072 --- /dev/null +++ b/bin/varnishtest/tests/a02005.vtc @@ -0,0 +1,36 @@ +varnishtest "Continuation frames" + +server s1 { + stream 1 { + rxreq + txresp -nohdrend + txcont -nohdrend -hdr "foo" "bar" + txcont -hdr "baz" "qux" + } -run + stream 3 { + rxreq + txresp -nohdrend + txcont -nohdrend -hdr "foo2" "bar2" + txcont -hdr "baz2" "qux2" + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + txreq + rxhdrs -all + expect resp.http.foo == "bar" + expect resp.http.baz == "qux" + } -run + stream 3 { + txreq + rxhdrs -some 2 + expect resp.http.foo2 == + expect resp.http.baz2 == + rxcont + expect resp.http.foo2 == "bar2" + expect resp.http.baz2 == "qux2" + } -run +} -run + +server s1 -wait diff --git a/bin/varnishtest/tests/a02006.vtc b/bin/varnishtest/tests/a02006.vtc new file mode 100644 index 0000000..e6ee7d4 --- /dev/null +++ b/bin/varnishtest/tests/a02006.vtc @@ -0,0 +1,40 @@ +varnishtest "Keep track of window credits" + +server s1 { + stream 2 { + rxreq + txresp -nohdrend + txcont -nohdrend -hdr "foo" "bar" + txcont -hdr "baz" "qux" + txdata -data "foo" + txdata -data "bar" + } -run + +} -start + +client c1 -connect ${s1_sock} { + stream 0 { + expect stream.window == 65535 + } -run + stream 2 { + expect stream.window == 65535 + txreq + rxhdrs + rxcont + rxcont + expect resp.http.:status == "200" + expect resp.http.foo == "bar" + expect stream.window == 65535 + rxdata + expect stream.window == 65532 + rxdata + expect stream.window == 65529 + expect resp.body == "foobar" + expect resp.http.baz == "qux" + } -run + stream 0 { + expect stream.window == 65529 + } -run +} -run + +server s1 -wait diff --git a/bin/varnishtest/tests/a02007.vtc b/bin/varnishtest/tests/a02007.vtc new file mode 100644 index 0000000..7d1be92 --- /dev/null +++ b/bin/varnishtest/tests/a02007.vtc @@ -0,0 +1,79 @@ +varnishtest "HPACK test" + +server s1 { + stream 1 { + rxreq + expect tbl.dec.size == 57 + expect tbl.dec[1].key == ":authority" + expect tbl.dec[1].value == "www.example.com" + txresp + } -run + + stream 3 { + rxreq + expect tbl.dec[1].key == "cache-control" + expect tbl.dec[1].value == "no-cache" + expect tbl.dec[2].key == ":authority" + expect tbl.dec[2].value == "www.example.com" + expect tbl.dec.size == 110 + txresp + } -run + + stream 5 { + rxreq + expect tbl.dec[1].key == "custom-key" + expect tbl.dec[1].value == "custom-value" + expect tbl.dec[2].key == "cache-control" + expect tbl.dec[2].value == "no-cache" + expect tbl.dec[3].key == ":authority" + expect tbl.dec[3].value == "www.example.com" + expect tbl.dec.size == 164 + txresp + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + + txreq -idxHdr 2 \ + -idxHdr 6 \ + -idxHdr 4 \ + -litIdxHdr inc 1 huf "www.example.com" + expect tbl.enc[1].key == ":authority" + expect tbl.enc[1].value == "www.example.com" + rxresp + } -run + + stream 3 { + txreq -idxHdr 2 \ + -idxHdr 6 \ + -idxHdr 4 \ + -idxHdr 62 \ + -litIdxHdr inc 24 huf no-cache + expect tbl.enc[1].key == "cache-control" + expect tbl.enc[1].value == "no-cache" + expect tbl.enc[2].key == ":authority" + expect tbl.enc[2].value == "www.example.com" + expect tbl.enc.size == 110 + rxresp + } -run + + stream 5 { + txreq -idxHdr 2 \ + -idxHdr 7 \ + -idxHdr 5 \ + -idxHdr 63 \ + -litHdr inc huf "custom-key" huf "custom-value" + expect tbl.enc[1].key == "custom-key" + expect tbl.enc[1].value == "custom-value" + expect tbl.enc[2].key == "cache-control" + expect tbl.enc[2].value == "no-cache" + expect tbl.enc[3].key == ":authority" + expect tbl.enc[3].value == "www.example.com" + expect tbl.enc.size == 164 + rxresp + } -run + +} -run + +server s1 -wait diff --git a/bin/varnishtest/tests/a02008.vtc b/bin/varnishtest/tests/a02008.vtc new file mode 100644 index 0000000..0c0c402 --- /dev/null +++ b/bin/varnishtest/tests/a02008.vtc @@ -0,0 +1,118 @@ +varnishtest "HPACK test" + +server s1 { + stream 0 { + txsettings -hdrtbl 256 + rxsettings + txsettings -ack + rxsettings + expect settings.ack == true + } -run + + stream 1 { + rxreq + expect tbl.dec[1].key == "location" + expect tbl.dec[1].value == "https://www.example.com" + expect tbl.dec[2].key == "date" + expect tbl.dec[2].value == "Mon, 21 Oct 2013 20:13:21 GMT" + expect tbl.dec[3].key == "cache-control" + expect tbl.dec[3].value == "private" + expect tbl.dec[4].key == ":status" + expect tbl.dec[4].value == "302" + expect tbl.dec.size == 222 + txresp + } -run + + stream 3 { + rxreq + expect tbl.dec[1].key == ":status" + expect tbl.dec[1].value == "307" + expect tbl.dec[2].key == "location" + expect tbl.dec[2].value == "https://www.example.com" + expect tbl.dec[3].key == "date" + expect tbl.dec[3].value == "Mon, 21 Oct 2013 20:13:21 GMT" + expect tbl.dec[4].key == "cache-control" + expect tbl.dec[4].value == "private" + expect tbl.dec.size == 222 + txresp + } -run + + stream 5 { + rxreq + expect tbl.dec[1].key == "set-cookie" + expect tbl.dec[1].value == "foo=ASDJKHQKBZXOQWEOPIUAXQWEOIU; max-age=3600; version=1" + expect tbl.dec[2].key == "content-encoding" + expect tbl.dec[2].value == "gzip" + expect tbl.dec[3].key == "date" + expect tbl.dec[3].value == "Mon, 21 Oct 2013 20:13:22 GMT" + expect tbl.dec.size == 215 + txresp + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 0 { + txsettings -hdrtbl 256 + rxsettings + txsettings -ack + rxsettings + expect settings.ack == true + } -run + + stream 1 { + + txreq \ + -litIdxHdr inc 8 plain "302" \ + -litIdxHdr inc 24 plain "private" \ + -litIdxHdr inc 33 plain "Mon, 21 Oct 2013 20:13:21 GMT" \ + -litIdxHdr inc 46 plain "https://www.example.com" + expect tbl.enc[1].key == "location" + expect tbl.enc[1].value == "https://www.example.com" + expect tbl.enc[2].key == "date" + expect tbl.enc[2].value == "Mon, 21 Oct 2013 20:13:21 GMT" + expect tbl.enc[3].key == "cache-control" + expect tbl.enc[3].value == "private" + expect tbl.enc[4].key == ":status" + expect tbl.enc[4].value == "302" + expect tbl.enc.size == 222 + rxresp + } -run + + stream 3 { + txreq \ + -litIdxHdr inc 8 huf "307" \ + -idxHdr 65 \ + -idxHdr 64 \ + -idxHdr 63 + expect tbl.enc[1].key == ":status" + expect tbl.enc[1].value == "307" + expect tbl.enc[2].key == "location" + expect tbl.enc[2].value == "https://www.example.com" + expect tbl.enc[3].key == "date" + expect tbl.enc[3].value == "Mon, 21 Oct 2013 20:13:21 GMT" + expect tbl.enc[4].key == "cache-control" + expect tbl.enc[4].value == "private" + expect tbl.enc.size == 222 + rxresp + } -run + + stream 5 { + txreq -idxHdr 8 \ + -idxHdr 65 \ + -litIdxHdr inc 33 plain "Mon, 21 Oct 2013 20:13:22 GMT" \ + -idxHdr 64 \ + -litIdxHdr inc 26 plain "gzip" \ + -litIdxHdr inc 55 plain "foo=ASDJKHQKBZXOQWEOPIUAXQWEOIU; max-age=3600; version=1" + expect tbl.enc[1].key == "set-cookie" + expect tbl.enc[1].value == "foo=ASDJKHQKBZXOQWEOPIUAXQWEOIU; max-age=3600; version=1" + expect tbl.enc[2].key == "content-encoding" + expect tbl.enc[2].value == "gzip" + expect tbl.enc[3].key == "date" + expect tbl.enc[3].value == "Mon, 21 Oct 2013 20:13:22 GMT" + expect tbl.enc.size == 215 + rxresp + } -run + +} -run + +server s1 -wait diff --git a/bin/varnishtest/tests/a02009.vtc b/bin/varnishtest/tests/a02009.vtc new file mode 100644 index 0000000..8668cb9 --- /dev/null +++ b/bin/varnishtest/tests/a02009.vtc @@ -0,0 +1,78 @@ +varnishtest "More HPACK tests" +server s1 { + stream 1 { + rxreq + expect tbl.dec.size == 57 + expect tbl.dec[1].key == ":authority" + expect tbl.dec[1].value == "www.example.com" + txresp + } -run + + stream 3 { + rxreq + expect tbl.dec[1].key == "cache-control" + expect tbl.dec[1].value == "no-cache" + expect tbl.dec[2].key == ":authority" + expect tbl.dec[2].value == "www.example.com" + expect tbl.dec.size == 110 + txresp + } -run + + stream 5 { + rxreq + expect tbl.dec[1].key == "custom-key" + expect tbl.dec[1].value == "custom-value" + expect tbl.dec[2].key == "cache-control" + expect tbl.dec[2].value == "no-cache" + expect tbl.dec[3].key == ":authority" + expect tbl.dec[3].value == "www.example.com" + expect tbl.dec.size == 164 + txresp + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + + txreq -idxHdr 2 \ + -idxHdr 6 \ + -idxHdr 4 \ + -litIdxHdr inc 1 huf "www.example.com" + expect tbl.enc[1].key == ":authority" + expect tbl.enc[1].value == "www.example.com" + rxresp + } -run + + stream 3 { + txreq -idxHdr 2 \ + -idxHdr 6 \ + -idxHdr 4 \ + -idxHdr 62 \ + -litIdxHdr inc 24 huf no-cache + expect tbl.enc[1].key == "cache-control" + expect tbl.enc[1].value == "no-cache" + expect tbl.enc[2].key == ":authority" + expect tbl.enc[2].value == "www.example.com" + expect tbl.enc.size == 110 + rxresp + } -run + + stream 5 { + txreq -idxHdr 2 \ + -idxHdr 7 \ + -idxHdr 5 \ + -idxHdr 63 \ + -litHdr inc huf "custom-key" huf "custom-value" + expect tbl.enc[1].key == "custom-key" + expect tbl.enc[1].value == "custom-value" + expect tbl.enc[2].key == "cache-control" + expect tbl.enc[2].value == "no-cache" + expect tbl.enc[3].key == ":authority" + expect tbl.enc[3].value == "www.example.com" + expect tbl.enc.size == 164 + rxresp + } -run + +} -run + +server s1 -wait diff --git a/bin/varnishtest/tests/a02010.vtc b/bin/varnishtest/tests/a02010.vtc new file mode 100644 index 0000000..adf12d3 --- /dev/null +++ b/bin/varnishtest/tests/a02010.vtc @@ -0,0 +1,40 @@ +varnishtest "Verify the initial window size" + +server s1 { + stream 0 { + rxsettings + txsettings -ack + } -run + stream 1 { + rxreq + txresp -bodylen 100 + } -run + stream 0 { + rxsettings + txsettings -ack + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 0 { + txsettings -winsize 128 + rxsettings + } -run + stream 1 { + txreq + rxresp + expect resp.bodylen == 100 + expect stream.window == 28 + } -run + stream 0 { + txsettings -winsize 64 + rxsettings + + expect stream.window == 65435 + } -run + stream 1 { + expect stream.window == -36 + } -run +} -run + +server s1 -wait diff --git a/bin/varnishtest/tests/a02011.vtc b/bin/varnishtest/tests/a02011.vtc new file mode 100644 index 0000000..031bc40 --- /dev/null +++ b/bin/varnishtest/tests/a02011.vtc @@ -0,0 +1,22 @@ +varnishtest "overflow" + +server s1 { + stream 1 { + rxreq + txresp -hdr long-header-original1 original1 \ + -hdr long-header-original2 original2 \ + -hdr long-header-original3 original3 \ + -hdr long-header-original4 original4 + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + txreq -req GET \ + -url / \ + -hdr :scheme http \ + -hdr :authority localhost + rxresp + expect resp.http.:status == 200 + } -run +} -run diff --git a/bin/varnishtest/tests/a02012.vtc b/bin/varnishtest/tests/a02012.vtc new file mode 100644 index 0000000..76ad72d --- /dev/null +++ b/bin/varnishtest/tests/a02012.vtc @@ -0,0 +1,56 @@ +varnishtest "padded DATA frames" + +server s1 { + stream 1 { + rxreq + # HDR indexed ":status: 200" + 2 padding bytes + sendhex "00 00 04 01 0c 00 00 00 01 02 88 12 34" + # DATA "foo" + 4 padding bytes + sendhex "00 00 08 00 09 00 00 00 01 04 66 6f 6f 6e 6e 6e 6e" + + } -run + + stream 3 { + rxreq + txresp -nostrend + txdata -data "bull" -pad "frog" -nostrend + txdata -data "terrier" -padlen 17 + txdata -datalen 4 -padlen 2 + } -run + + stream 5 { + rxreq + txresp -pad "pwepew" + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + txreq + rxresp + expect resp.bodylen == 3 + expect resp.body == "foo" + } -run + + stream 3 { + txreq + rxhdrs + + rxdata + expect frame.size == 9 + expect resp.body == "bull" + + rxdata + expect frame.size == 25 + expect resp.body == "bullterrier" + + rxdata + expect frame.size == 7 + } -run + + stream 5 { + txreq + rxresp + expect frame.padding == 6 + } -run +} -run diff --git a/bin/varnishtest/tests/a02013.vtc b/bin/varnishtest/tests/a02013.vtc new file mode 100644 index 0000000..51d3a80 --- /dev/null +++ b/bin/varnishtest/tests/a02013.vtc @@ -0,0 +1,15 @@ +varnishtest "H/2 state after sending/receiving preface" + +server s1 { + expect h2.state == false + rxpri + expect h2.state == true +} -start + +client c1 -connect ${s1_sock} { + expect h2.state == false + txpri + expect h2.state == true +} -start + +server s1 -wait diff --git a/bin/varnishtest/tests/a02014.vtc b/bin/varnishtest/tests/a02014.vtc new file mode 100644 index 0000000..d11af49 --- /dev/null +++ b/bin/varnishtest/tests/a02014.vtc @@ -0,0 +1,52 @@ +varnishtest "priority" + +server s1 { + stream 1 { + rxreq + txresp + expect stream.weight == 16 + expect stream.dependency == 0 + } -run + stream 3 { + rxreq + txresp + expect stream.weight == 123 + expect stream.dependency == 5 + + rxprio + expect prio.weight == 10 + expect prio.stream == 7 + expect stream.weight == 10 + expect stream.dependency == 7 + } -run +} -start +client c1 -connect ${s1_sock} { + stream 1 { + txreq -req GET -url /1 \ + -hdr :scheme http -hdr :authority localhost + rxresp + expect stream.weight == 16 + expect stream.dependency == 0 + } -run + stream 3 { + txreq -req GET -url /3 \ + -hdr :scheme http -hdr :authority localhost \ + -weight 123 -ex -dep 5 + rxresp + expect stream.weight == 123 + expect stream.dependency == 5 + + txprio -weight 10 -stream 7 + expect stream.weight == 10 + expect stream.dependency == 7 + } -run + stream 5 { + expect stream.weight == 16 + expect stream.dependency == 0 + } -run + stream 0 { + expect stream.weight == + expect stream.dependency == + } -run +} -run +server s1 -wait diff --git a/bin/varnishtest/tests/a02015.vtc b/bin/varnishtest/tests/a02015.vtc new file mode 100644 index 0000000..9af3a7a --- /dev/null +++ b/bin/varnishtest/tests/a02015.vtc @@ -0,0 +1,63 @@ +varnishtest "exclusive dependency" + +server s1 { + stream 1 { + rxreq + txresp + } -run + stream 3 { + rxreq + txresp + } -run + stream 5 { + rxreq + txresp + expect stream.dependency == 0 + } -run + + stream 1 { + expect stream.dependency == 5 + } -run + stream 3 { + expect stream.dependency == 5 + } -run + + stream 1 { + rxprio + } -run + stream 5 { + expect stream.dependency == 1 + } -run + +} -start +client c1 -connect ${s1_sock} { + stream 1 { + txreq + rxresp + } -run + stream 3 { + txreq + rxresp + } -run + stream 5 { + txreq -req GET -ex + expect stream.dependency == 0 + rxresp + } -run + + stream 1 { + expect stream.dependency == 5 + } -run + stream 3 { + expect stream.dependency == 5 + } -run + + stream 1 { + txprio -stream 0 -ex + } -run + stream 5 { + expect stream.dependency == 1 + } -run +} -run + +server s1 -wait diff --git a/bin/varnishtest/tests/a02016.vtc b/bin/varnishtest/tests/a02016.vtc new file mode 100644 index 0000000..ea06e33 --- /dev/null +++ b/bin/varnishtest/tests/a02016.vtc @@ -0,0 +1,37 @@ +varnishtest "Test pseudo-headers inspection" + +server s1 { + stream 1 { + rxreq + + expect req.url == "/foo" + expect req.http.:path == "/foo" + + expect req.method == "NOTGET" + expect req.http.:method == "NOTGET" + + expect req.authority == "bar" + expect req.http.:authority == "bar" + + expect req.scheme == "baz" + expect req.http.:scheme == "baz" + + txresp -status 123 + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + txreq -url "/foo" \ + -req "NOTGET" \ + -hdr ":authority" "bar" \ + -scheme "baz" + + rxresp + + expect resp.status == 123 + expect resp.http.:status == 123 + } -run +} -start + +server s1 -wait diff --git a/bin/varnishtest/tests/a02017.vtc b/bin/varnishtest/tests/a02017.vtc new file mode 100644 index 0000000..c0374f7 --- /dev/null +++ b/bin/varnishtest/tests/a02017.vtc @@ -0,0 +1,22 @@ +varnishtest "Push promise" + +server s1 { + stream 1 { + rxreq + txpush -promised 2 -url "/hereyougo" + txresp + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + txreq + rxpush + expect push.id == 2 + expect req.url == "/hereyougo" + expect req.method == "GET" + rxresp + } -run +} -run + +server s1 -wait diff --git a/bin/varnishtest/tests/a02018.vtc b/bin/varnishtest/tests/a02018.vtc new file mode 100644 index 0000000..7c281df --- /dev/null +++ b/bin/varnishtest/tests/a02018.vtc @@ -0,0 +1,21 @@ +varnishtest "H/2 state after sending/receiving preface" + +server s1 { + expect h2.state == "false" + stream 1 { + rxreq + txresp + } -run + expect h2.state == "true" +} -start + +client c1 -connect ${s1_sock} { + expect h2.state == "false" + stream 1 { + txreq + rxresp + } -run + expect h2.state == "true" +} -start + +server s1 -wait diff --git a/bin/varnishtest/tests/a02019.vtc b/bin/varnishtest/tests/a02019.vtc new file mode 100644 index 0000000..712d93e --- /dev/null +++ b/bin/varnishtest/tests/a02019.vtc @@ -0,0 +1,23 @@ +varnishtest "Static table encoding" +server s1 { + stream 1 { + rxreq + expect req.http.:path == "/index.html" + txresp + } -run + +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + + txreq -idxHdr 2 \ + -idxHdr 6 \ + -idxHdr 5 + rxresp + } -run + + +} -run + +server s1 -wait diff --git a/bin/varnishtest/tests/a02020.vtc b/bin/varnishtest/tests/a02020.vtc new file mode 100644 index 0000000..244abb6 --- /dev/null +++ b/bin/varnishtest/tests/a02020.vtc @@ -0,0 +1,41 @@ +varnishtest "Reduce dynamic table while incoming headers are flying" + +server s1 { + stream 1 { + rxreq + txresp -litHdr inc plain hoge plain fuga + expect tbl.enc[1].key == "hoge" + expect tbl.enc[1].value == "fuga" + expect tbl.enc.size == 40 + + } -run + + + stream 3 { + rxreq + txresp -idxHdr 62 -litHdr inc plain "foo" plain "bar" + } -run + + stream 0 { rxsettings } -run +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + txreq + rxresp + expect tbl.dec[1].key == "hoge" + expect tbl.dec[1].value == "fuga" + expect tbl.dec.size == 40 + expect tbl.dec.length == 1 + } -run + + stream 3 { txreq } -run + stream 0 { txsettings -hdrtbl 0 } -run + + non-fatal + stream 3 { + rxresp + expect resp.http.foo == + } -run + +} -run diff --git a/bin/varnishtest/tests/a02021.vtc b/bin/varnishtest/tests/a02021.vtc new file mode 100644 index 0000000..a04265f --- /dev/null +++ b/bin/varnishtest/tests/a02021.vtc @@ -0,0 +1,52 @@ +varnishtest "Reduce dynamic table size" + +server s1 { + stream 1 { + rxreq + txresp -litHdr inc plain hoge plain fuga + expect tbl.dec.size == 57 + expect tbl.enc.size == 40 + } -run + + stream 0 { + rxsettings + txsettings -ack + } -run + + stream 3 { + rxreq + expect tbl.dec.size == 110 + expect tbl.enc.size == 0 + txresp + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + txreq -idxHdr 2 \ + -idxHdr 6 \ + -idxHdr 4 \ + -litIdxHdr inc 1 huf "www.example.com" + rxresp + expect tbl.dec.size == 40 + expect tbl.enc.size == 57 + } -run + + stream 0 { + txsettings -hdrtbl 0 + rxsettings + } -run + + stream 3 { + txreq -idxHdr 2 \ + -idxHdr 6 \ + -idxHdr 4 \ + -idxHdr 62 \ + -litIdxHdr inc 24 huf no-cache + expect tbl.enc.size == 110 + expect tbl.dec.size == 0 + rxresp + } -run +} -run + +server s1 -wait diff --git a/bin/varnishtest/tests/a02022.vtc b/bin/varnishtest/tests/a02022.vtc new file mode 100644 index 0000000..0c34071 --- /dev/null +++ b/bin/varnishtest/tests/a02022.vtc @@ -0,0 +1,21 @@ +varnishtest "H/1 -> H/2 upgrade" + +server s1 { + rxreq + upgrade + + stream 3 { rxprio } -run + stream 5 { rxprio } -run + stream 7 { rxprio } -run + stream 9 { rxprio } -run + stream 11 { rxprio } -run + + stream 1 { + rxprio + txresp + } -start + +} -start + + +shell { nghttp http://${s1_addr}:${s1_port} -nu } diff --git a/bin/varnishtest/tests/a02023.vtc b/bin/varnishtest/tests/a02023.vtc new file mode 100644 index 0000000..3cbc94d --- /dev/null +++ b/bin/varnishtest/tests/a02023.vtc @@ -0,0 +1,44 @@ +varnishtest "Window update" + +server s1 { + stream 1 { + rxreq + txresp -body "bob" + } -run + stream 3 { + rxreq + txresp -nohdrend + txcont -nohdrend -hdr "foo" "bar" + txcont -hdr "baz" "qux" + txdata -data "foo" + txdata -data "bar" + } -run + +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + txreq + rxresp + expect resp.bodylen == 3 + } -run + stream 3 { + txreq + rxhdrs + rxcont + rxcont + expect resp.http.:status == "200" + expect resp.http.foo == "bar" + expect stream.window == 65535 + + rxdata + expect stream.window == 65532 + + rxdata + expect stream.window == 65529 + expect resp.body == "foobar" + expect resp.http.baz == "qux" + } -run +} -run + +server s1 -wait diff --git a/bin/varnishtest/vtc_h2_hpack.c b/bin/varnishtest/vtc_h2_hpack.c index 63034fd..a378966 100644 --- a/bin/varnishtest/vtc_h2_hpack.c +++ b/bin/varnishtest/vtc_h2_hpack.c @@ -111,8 +111,8 @@ huff_encode(struct hpk_iter *iter, const char *str, int len) int pl = 0; /* pack length*/ uint32_t v; uint8_t s; - - assert(iter->buf < iter->end); + + assert(iter->buf < iter->end); while (len--) { v = coding_table[(uint8_t)*str].val; @@ -279,7 +279,7 @@ str_decode(struct hpk_iter *iter, struct txt *t) t->ptr[num] = '\0'; t->len = num; - + return (ITER_DONE(iter)); } diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 3c85f50..1dad7a5 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1047,7 +1047,7 @@ cmd_http_upgrade(CMD_ARGS) h = http_find_header(hp->req, "Connection"); if (!h || strcmp(h, "Upgrade, HTTP2-Settings")) - vtc_log(vl, 0, "Req misses \"Connection: " + vtc_log(vl, 0, "Req misses \"Connection: " "Upgrade, HTTP2-Settings\" header"); h = http_find_header(hp->req, "HTTP2-Settings"); diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 224b308..b8a0349 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -99,7 +99,7 @@ struct stream { char *name; VTAILQ_ENTRY(stream) list; unsigned running; - pthread_cond_t cond; + pthread_cond_t cond; struct frame *frame; pthread_t tp; unsigned reading; @@ -210,10 +210,10 @@ VTAILQ_HEAD(fq_head, frame); struct frame { unsigned magic; #define FRAME_MAGIC 0x5dd3ec4 - uint32_t size; + uint32_t size; uint32_t stid; - uint8_t type; - uint8_t flags; + uint8_t type; + uint8_t flags; char *data; VTAILQ_ENTRY(frame) list; @@ -347,10 +347,10 @@ exclusive_stream_dependency(const struct stream *s) { struct stream *target = NULL; struct http *hp = s->hp; - + if (s->id == 0) return; - + VTAILQ_FOREACH(target, &hp->streams, list) { if (target->id != s->id && target->dependency == s->dependency) target->dependency = s->id; @@ -845,7 +845,7 @@ do { \ STRTOU32(n, *sp, p, v, c); \ if (l && n >= (1 << l)) \ vtc_log(v, 0, c " must be a %d-bits integer (found %s)", \ - l, *sp); \ + l, *sp); \ } while (0) #define CHECK_LAST_FRAME(TYPE) \ @@ -908,9 +908,9 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) n = 0; /* SECTION: stream.spec.zexpect.ping PING specific * ping.data - * The 8-bytes string of the PING frame payload. + * The 8-bytes string of the PING frame payload. * ping.ack (PING) - * "true" if the ACK flag was set, "false" otherwise. + * "true" if the ACK flag was set, "false" otherwise. */ if (!strcmp(spec, "ping.data")) { CHECK_LAST_FRAME(PING); @@ -923,7 +923,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) } /* SECTION: stream.spec.zexpect.winup WINDOW_UPDATE specific * winup.size - * The size of the upgrade given by the WINDOW_UPDATE frame. + * The size of the upgrade given by the WINDOW_UPDATE frame. */ else if (!strcmp(spec, "winup.size")) { CHECK_LAST_FRAME(WINDOW_UPDATE); @@ -931,13 +931,13 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) } /* SECTION: stream.spec.zexpect.prio PRIORITY specific * prio.stream - * The stream ID announced. + * The stream ID announced. * * prio.exclusive - * "true" if the priority is exclusive, else "false". + * "true" if the priority is exclusive, else "false". * * prio.weight - * The dependency weight. + * The dependency weight. */ else if (!strcmp(spec, "prio.stream")) { CHECK_LAST_FRAME(PRIORITY); @@ -954,7 +954,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) } /* SECTION: stream.spec.zexpect.rst RESET_STREAM specific * rst.err - * The error code (as integer) of the RESET_STREAM frame. + * The error code (as integer) of the RESET_STREAM frame. */ else if (!strcmp(spec, "rst.err")) { CHECK_LAST_FRAME(RST_STREAM); @@ -963,26 +963,26 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) /* SECTION: stream.spec.zexpect.settings SETTINGS specific * * settings.ack - * "true" if the ACK flag was set, else ""false. + * "true" if the ACK flag was set, else ""false. * * settings.push - * "true" if the push settings was set to yes, "false" if set to - * no, and if not present. + * "true" if the push settings was set to yes, "false" if set to + * no, and if not present. * * settings.hdrtbl - * Value of HEADER_TABLE_SIZE if set, otherwise. + * Value of HEADER_TABLE_SIZE if set, otherwise. * * settings.maxstreams - * Value of MAX_CONCURRENT_STREAMS if set, otherwise. + * Value of MAX_CONCURRENT_STREAMS if set, otherwise. * * settings.winsize - * Value of INITIAL_WINDOW_SIZE if set, otherwise. + * Value of INITIAL_WINDOW_SIZE if set, otherwise. * * setting.framesize - * Value of MAX_FRAME_SIZE if set, otherwise. + * Value of MAX_FRAME_SIZE if set, otherwise. * * settings.hdrsize - * Value of MAX_HEADER_LIST_SIZE if set, otherwise. + * Value of MAX_HEADER_LIST_SIZE if set, otherwise. */ else if (!strncmp(spec, "settings.", 9)) { CHECK_LAST_FRAME(SETTINGS); @@ -1008,7 +1008,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) } /* SECTION: stream.spec.zexpect.push PUSH_PROMISE specific * push.id - * The id of the promised stream. + * The id of the promised stream. */ else if (!strcmp(spec, "push.id")) { CHECK_LAST_FRAME(PUSH_PROMISE); @@ -1016,13 +1016,13 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) } /* SECTION: stream.spec.zexpect.goaway GOAWAY specific * goaway.err - * The error code (as integer) of the GOAWAY frame. + * The error code (as integer) of the GOAWAY frame. * * goaway.laststream - * Last-Stream-ID + * Last-Stream-ID * * goaway.debug - * Debug data, if any. + * Debug data, if any. */ else if (!strncmp(spec, "goaway.", 7)) { spec += 7; @@ -1040,20 +1040,20 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) } /* SECTION: stream.spec.zexpect.zframe Generic frame * frame.data - * Payload of the last frame + * Payload of the last frame * * frame.type - * Type of the frame, as integer. + * Type of the frame, as integer. * * frame.size - * Size of the frame. + * Size of the frame. * * frame.stream - * Stream of the frame (correspond to the one you are executing - * this from, obviously). + * Stream of the frame (correspond to the one you are executing + * this from, obviously). * * frame.padding (for DATA, HEADERS, PUSH_PROMISE frames) - * Number of padded bytes. + * Number of padded bytes. */ else if (!strncmp(spec, "frame.", 6)) { spec += 6; @@ -1075,17 +1075,18 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) } /* SECTION: stream.spec.zexpect.zstream Stream * stream.window - * The current window size of the stream, or, if on stream 0, - * of the connection. + * The current window size of the stream, or, if on stream 0, + * of the connection. * * stream.weight - * Weight of the stream + * Weight of the stream * * stream.dependency - * Id of the stream this one depends on. + * Id of the stream this one depends on. */ else if (!strcmp(spec, "stream.window")) { - snprintf(buf, 20, "%" PRIu64 "d", s->id ? s->ws : s->hp->ws); + snprintf(buf, 20, "%jd", + (intmax_t)(s->id ? s->ws : s->hp->ws)); return (buf); } else if (!strcmp(spec, "stream.weight")) { @@ -1106,21 +1107,21 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) } /* SECTION: stream.spec.zexpect.ztable Index tables * tbl.dec.size / tbl.enc.size - * Size (bytes) of the decoding/encoding table. + * Size (bytes) of the decoding/encoding table. * * tbl.dec.size / tbl.enc.maxsize - * Maximum size (bytes) of the decoding/encoding table. + * Maximum size (bytes) of the decoding/encoding table. * * tbl.dec.length / tbl.enc.length - * Number of headers in decoding/encoding table. + * Number of headers in decoding/encoding table. * * tbl.dec[INT].key / tbl.enc[INT].key - * Name of the header at index INT of the decoding/encoding - * table. + * Name of the header at index INT of the decoding/encoding + * table. * * tbl.dec[INT].value / tbl.enc[INT].value - * Value of the header at index INT of the decoding/encoding - * table. + * Value of the header at index INT of the decoding/encoding + * table. */ else if (!strncmp(spec, "tbl.dec", 7) || !strncmp(spec, "tbl.enc", 7)) { @@ -1156,28 +1157,28 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) * still being construct (in-between two frames for example). * * req.bodylen / resp.bodylen - * Length in bytes of the request/response so far. + * Length in bytes of the request/response so far. * * req.body / resp.body - * Body of the request/response so far. + * Body of the request/response so far. * * req.http.STRING / resp.http.STRING - * Value of the header STRING in the request/response. + * Value of the header STRING in the request/response. * * req.status / resp.status - * :status pseudo-header's value. + * :status pseudo-header's value. * * req.url / resp.url - * :path pseudo-header's value. + * :path pseudo-header's value. * * req.method / resp.method - * :method pseudo-header's value. + * :method pseudo-header's value. * * req.authority / resp.authority - * :method pseudo-header's value. + * :method pseudo-header's value. * * req.scheme / resp.scheme - * :method pseudo-header's value. + * :method pseudo-header's value. */ else if (!strncmp(spec, "req.", 4) || !strncmp(spec, "resp.", 5)) { if (spec[2] == 'q') { @@ -1217,7 +1218,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) * describing the bytes, in hex notation, will possible whitespaces between * them. Here's an example:: * - * sendhex "00 00 08 00 0900 8d" + * sendhex "00 00 08 00 0900 8d" */ static void cmd_sendhex(CMD_ARGS) @@ -1298,80 +1299,80 @@ cmd_sendhex(CMD_ARGS) * each of them. * * \-noadd - * Do not add default headers. Useful to avoid duplicates when sending - * default headers using ``-hdr``, ``-idxHdr`` and ``-litIdxHdr``. + * Do not add default headers. Useful to avoid duplicates when sending + * default headers using ``-hdr``, ``-idxHdr`` and ``-litIdxHdr``. * * \-status INT (txresp) - * Set the :status pseudo-header. + * Set the :status pseudo-header. * * \-url STRING (txreq, txpush) - * Set the :path pseudo-header. + * Set the :path pseudo-header. * * \-req STRING (txreq, txpush) - * Set the :method pseudo-header. + * Set the :method pseudo-header. * * \-scheme STRING (txreq, txpush) - * Set the :scheme pseudo-header. + * Set the :scheme pseudo-header. * * \-hdr STRING1 STRING2 - * Insert a header, STRING1 being the name, and STRING2 the value. + * Insert a header, STRING1 being the name, and STRING2 the value. * * \-idxHdr INT - * Insert an indexed header, using INT as index. + * Insert an indexed header, using INT as index. * * \-litIdxHdr inc|not|never INT huf|plain STRING - * Insert an literal, indexed header. The first argument specify if the - * header should be added to the table, shouldn't, or mustn't be - * compressed if/when retransmitted. + * Insert an literal, indexed header. The first argument specify if the + * header should be added to the table, shouldn't, or mustn't be + * compressed if/when retransmitted. * - * INT is the idex of the header name to use. + * INT is the idex of the header name to use. * - * The third argument informs about the Huffman encoding: yes (huf) or - * no (plain). + * The third argument informs about the Huffman encoding: yes (huf) or + * no (plain). * - * The last term is the literal value of the header. + * The last term is the literal value of the header. * * \-litHdr inc|not|never huf|plain STRING1 huf|plain STRING2 - * Insert a literal header, with the same first argument as - * ``-litIdxHdr``. + * Insert a literal header, with the same first argument as + * ``-litIdxHdr``. * - * The second and third terms tell what the name of the header is and if - * it should be Huffman-encoded, while the last two do the same - * regarding the value. + * The second and third terms tell what the name of the header is and if + * it should be Huffman-encoded, while the last two do the same + * regarding the value. * * \-body STRING (txreq, txresp) - * Specify a body, effectively putting STRING into a DATA frame after - * the HEADER frame is sent. + * Specify a body, effectively putting STRING into a DATA frame after + * the HEADER frame is sent. * * \-bodylen INT (txreq, txresp) - * Do the same thing as ``-body`` but generate an string of INT length - * for you. + * Do the same thing as ``-body`` but generate an string of INT length + * for you. * * \-nostrend (txreq, txresp) - * Don't set the END_STREAM flag automatically, making the peer expect - * a body after the headers. + * Don't set the END_STREAM flag automatically, making the peer expect + * a body after the headers. * * \-nohdrend - * Don't set the END_HEADERS flag automatically, making the peer expect - * more HEADER frames. + * Don't set the END_HEADERS flag automatically, making the peer expect + * more HEADER frames. * * \-dep INT (txreq, txresp) - * Tell the peer that this content depends on the stream with the INT - * id. + * Tell the peer that this content depends on the stream with the INT + * id. * * \-ex (txreq, txresp) - * Make the dependency exclusive (``-dep`` is still needed). + * Make the dependency exclusive (``-dep`` is still needed). * * \-weight (txreq, txresp) - * Set the weight for the dependency. + * Set the weight for the dependency. * * \-promised INT (txpush) - * The id of the promised stream. + * The id of the promised stream. * * \-pad STRING / -padlen INT (txreq, txresp, txpush) - * Add string as padding to the frame, either the one you provided with - * \-pad, or one that is generated for you, of length INT is -padlen - * case. + * Add string as padding to the frame, either the one you provided with + * \-pad, or one that is generated for you, of length INT is -padlen + * case. */ static void cmd_tx11obj(CMD_ARGS) @@ -1567,7 +1568,7 @@ cmd_tx11obj(CMD_ARGS) s->dependency = stid; assert(f.size + 5 < BUF_SIZE); - memmove(buf + 5, buf, f.size); + memmove(buf + 5, buf, f.size); vbe32enc(buf, (stid | exclusive)); buf[4] = s->weight; f.size += 5; @@ -1591,7 +1592,7 @@ cmd_tx11obj(CMD_ARGS) } if (f.type == TYPE_PUSH_PROMISE) f.size += 4; - f.data = buf; + f.data = buf; HPK_FreeIter(iter); write_frame(s->hp, &f, 1); free(buf); @@ -1613,19 +1614,19 @@ cmd_tx11obj(CMD_ARGS) * and txdata automatically set it. * * \-data STRING - * Data to be embedded into the frame. + * Data to be embedded into the frame. * * \-datalen INT - * Generate and INT-bytes long string to be sent in the frame. + * Generate and INT-bytes long string to be sent in the frame. * * \-pad STRING / -padlen INT - * Add string as padding to the frame, either the one you provided with - * \-pad, or one that is generated for you, of length INT is -padlen - * case. + * Add string as padding to the frame, either the one you provided with + * \-pad, or one that is generated for you, of length INT is -padlen + * case. * * \-nostrend - * Don't set the END_STREAM flag, allowing to send more data on this - * stream. + * Don't set the END_STREAM flag, allowing to send more data on this + * stream. */ static void cmd_txdata(CMD_ARGS) @@ -1699,9 +1700,9 @@ cmd_txdata(CMD_ARGS) * (NO_ERROR). * * \-err STRING|INT - * Sets the error code to be sent. The argument can be an integer or a - * string describing the error, such as NO_ERROR, or CANCEL (see - * rfc7540#11.4 for more strings). + * Sets the error code to be sent. The argument can be an integer or a + * string describing the error, such as NO_ERROR, or CANCEL (see + * rfc7540#11.4 for more strings). */ static void cmd_txrst(CMD_ARGS) @@ -1728,7 +1729,7 @@ cmd_txrst(CMD_ARGS) STRTOU32(err, *av, p, vl, "-err"); } else - break; + break; } if (*av != NULL) vtc_log(vl, 0, "Unknown txrst spec: %s\n", *av); @@ -1743,15 +1744,15 @@ cmd_txrst(CMD_ARGS) * Send a PRIORITY frame * * \-stream INT - * indicate the id of the stream the sender stream depends on. + * indicate the id of the stream the sender stream depends on. * * \-ex - * the dependency should be made exclusive (only this streams depends on - * the parent stream). + * the dependency should be made exclusive (only this streams depends on + * the parent stream). * * \-weight INT - * an 8-bits integer is used to balance priority between streams - * depending on the same streams. + * an 8-bits integer is used to balance priority between streams + * depending on the same streams. */ static void cmd_txprio(CMD_ARGS) @@ -1778,7 +1779,7 @@ cmd_txprio(CMD_ARGS) } else if (!strcmp(*av, "-weight")) { STRTOU32_CHECK(weight, av, p, vl, "-weight", 8); } else - break; + break; } if (*av != NULL) vtc_log(vl, 0, "Unknown txprio spec: %s\n", *av); @@ -1809,25 +1810,25 @@ cmd_txprio(CMD_ARGS) * are from rfc7540#6.5.2): * * \-hdrtbl INT - * headers table size + * headers table size * * \-push BOOL - * whether push frames are accepted or not + * whether push frames are accepted or not * * \-maxstreams INT - * maximum concurrent streams allowed + * maximum concurrent streams allowed * * \-winsize INT - * sender's initial window size + * sender's initial window size * * \-framesize INT - * largest frame size authorized + * largest frame size authorized * * \-hdrsize INT - * maximum size of the header list authorized + * maximum size of the header list authorized * * \-ack - * set the ack bit + * set the ack bit */ static void cmd_txsettings(CMD_ARGS) @@ -1902,10 +1903,10 @@ cmd_txsettings(CMD_ARGS) * Send PING frame. * * \-data STRING - * specify the payload of the frame, with STRING being an 8-char string. + * specify the payload of the frame, with STRING being an 8-char string. * * \-ack - * set the ACK flag. + * set the ACK flag. */ static void cmd_txping(CMD_ARGS) @@ -1945,17 +1946,17 @@ cmd_txping(CMD_ARGS) * Possible options include: * * \-err STRING|INT - * set the error code to eplain the termination. The second argument - * can be a integer or the string version of the error code as found - * in rfc7540#7. + * set the error code to eplain the termination. The second argument + * can be a integer or the string version of the error code as found + * in rfc7540#7. * * \-laststream INT - * the id of the "highest-numbered stream identifier for which the - * sender of the GOAWAY frame might have taken some action on or might - * yet take action on". + * the id of the "highest-numbered stream identifier for which the + * sender of the GOAWAY frame might have taken some action on or might + * yet take action on". * * \-debug - * specify the debug data, if any to append to the frame. + * specify the debug data, if any to append to the frame. */ static void cmd_txgoaway(CMD_ARGS) @@ -2014,7 +2015,7 @@ cmd_txgoaway(CMD_ARGS) * connection (from stream 0) or of the stream (any other stream). * * \-size INT - * give INT credits to the peer. + * give INT credits to the peer. */ static void cmd_txwinup(CMD_ARGS) @@ -2024,7 +2025,7 @@ cmd_txwinup(CMD_ARGS) char *p; struct frame f; char buf[8]; - uint32_t size = 0; + uint32_t size = 0; (void)cmd; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); @@ -2041,7 +2042,7 @@ cmd_txwinup(CMD_ARGS) if (!strcmp(*av, "-size")) { STRTOU32_CHECK(size, av, p, vl, "-size", 0); } else - break; + break; } if (*av != NULL) vtc_log(vl, 0, "Unknown txwinup spec: %s\n", *av); @@ -2089,7 +2090,7 @@ rxstuff(struct stream *s) { "instead of %s (%d)", \ rcv, func, \ rt < TYPE_MAX ? h2_types[rt] : "?", rt, \ - wt < TYPE_MAX ? h2_types[wt] : "?", wt); \ + wt < TYPE_MAX ? h2_types[wt] : "?", wt); \ } while (0); /* SECTION: stream.spec.data_12 rxhdrs @@ -2098,10 +2099,10 @@ rxstuff(struct stream *s) { * zero or more CONTINUATION frame. * * \-all - * Keep waiting for CONTINUATION frames until END_HEADERS flag is seen. + * Keep waiting for CONTINUATION frames until END_HEADERS flag is seen. * * \-some INT - * Retrieve INT - 1 CONTINUATION frames after the HEADER frame. + * Retrieve INT - 1 CONTINUATION frames after the HEADER frame. * */ static void @@ -2125,7 +2126,7 @@ cmd_rxhdrs(CMD_ARGS) } else if (!strcmp(*av, "-all")) { loop = 1; } else - break; + break; } if (*av != NULL) vtc_log(vl, 0, "Unknown rxhdrs spec: %s\n", *av); @@ -2160,7 +2161,7 @@ cmd_rxcont(CMD_ARGS) } else if (!strcmp(*av, "-all")) { loop = 1; } else - break; + break; } if (*av != NULL) vtc_log(vl, 0, "Unknown rxcont spec: %s\n", *av); @@ -2182,10 +2183,10 @@ cmd_rxcont(CMD_ARGS) * arguments: * * \-all - * keep waiting for DATA frame until one sets the END_STREAM flag + * keep waiting for DATA frame until one sets the END_STREAM flag * * \-some INT - * retrieve INT DATA frames. + * retrieve INT DATA frames. * */ static void @@ -2208,7 +2209,7 @@ cmd_rxdata(CMD_ARGS) } else if (!strcmp(*av, "-all")) { loop = 1; } else - break; + break; } if (*av != NULL) vtc_log(vl, 0, "Unknown rxdata spec: %s\n", *av); @@ -2276,10 +2277,10 @@ cmd_rxreqsp(CMD_ARGS) * CONTINUATION frames. * * \-all - * Keep waiting for CONTINUATION frames until END_HEADERS flag is seen. + * Keep waiting for CONTINUATION frames until END_HEADERS flag is seen. * * \-some INT - * Retrieve INT - 1 CONTINUATION frames after the PUSH frame. + * Retrieve INT - 1 CONTINUATION frames after the PUSH frame. * */ static void @@ -2618,7 +2619,7 @@ stream_run(struct stream *s) * * Stream syntax follow the client/server one:: * - * stream ID [SPEC] [ACTION] + * stream ID [SPEC] [ACTION] * * ID is the H/2 stream number, while SPEC describes what will be done in that * stream. @@ -2626,27 +2627,27 @@ stream_run(struct stream *s) * Note that, when parsing a stream action, if the entity isn't operating in H/2 * mode, these spec is ran before:: * - * txpri/rxpri # client/server - * stream 0 { - * txsettings - * rxsettings - * txsettings -ack - * rxsettings - * expect settings.ack == true - * } -run + * txpri/rxpri # client/server + * stream 0 { + * txsettings + * rxsettings + * txsettings -ack + * rxsettings + * expect settings.ack == true + * } -run * * And H/2 mode is then activated before parsing the specification. * * SECTION: stream.actions Actions * * \-start - * Run the specification in a thread, giving back control immediately. + * Run the specification in a thread, giving back control immediately. * * \-wait - * Wait for the started thread to finish running the spec. + * Wait for the started thread to finish running the spec. * * \-run - * equivalent to calling ``-start`` then ``-wait``. + * equivalent to calling ``-start`` then ``-wait``. */ void @@ -2722,7 +2723,7 @@ b64_settings(const struct http *hp, const char *s) s++; } i = v >> 32; - v &= 0xffff; + v &= 0xffff; if (i <= SETTINGS_MAX) { buf = h2_settings[i]; @@ -2736,7 +2737,8 @@ b64_settings(const struct http *hp, const char *s) HPK_ResizeTbl(hp->decctx, v); } - vtc_log(hp->vl, 4, "Upgrade: %s (%d): %" PRIu64 "d", buf, i, v); + vtc_log(hp->vl, 4, "Upgrade: %s (%d): %ju", + buf, i, (intmax_t)v); } } From phk at FreeBSD.org Mon Sep 5 06:47:08 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 05 Sep 2016 08:47:08 +0200 Subject: [master] 26fcf62 Minor flexelint nit-picking, while trying to make sure there was nothing major lurking. Message-ID: commit 26fcf62958b33569d4570621a4869f7ef8a0456b Author: Poul-Henning Kamp Date: Mon Sep 5 06:45:54 2016 +0000 Minor flexelint nit-picking, while trying to make sure there was nothing major lurking. diff --git a/bin/varnishtest/flint.lnt b/bin/varnishtest/flint.lnt index a02b3be..a527ec3 100644 --- a/bin/varnishtest/flint.lnt +++ b/bin/varnishtest/flint.lnt @@ -5,6 +5,8 @@ -esym(850, av) -esym(528, iter_call) // Flexelint bug +-esym(534, snprintf) // Only for varnishtest, and not really nice + -e712 // 14 Info 712 Loss of precision (___) (___ to ___) -e747 // 16 Info 747 Significant prototype coercion (___) ___ to ___ diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index a614cd0..49dff08 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -113,7 +113,6 @@ void extmacro_def(const char *name, const char *fmt, ...) __v_printflike(2, 3); struct http; -extern const struct cmds http_cmds[]; void cmd_stream(CMD_ARGS); void start_h2(struct http *hp); void stop_h2(struct http *hp); diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index 67c86d1..d3c3f8b 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -78,7 +78,7 @@ proxy_cb(void *priv, const struct suckaddr *sa) } static void -client_proxy(struct vtclog *vl, int fd, int version, char *spec) +client_proxy(struct vtclog *vl, int fd, int version, const char *spec) { struct suckaddr *sac, *sas; const char *err; diff --git a/bin/varnishtest/vtc_h2_tbl.c b/bin/varnishtest/vtc_h2_tbl.c index 0a50d1c..21bcbd9 100644 --- a/bin/varnishtest/vtc_h2_tbl.c +++ b/bin/varnishtest/vtc_h2_tbl.c @@ -44,7 +44,7 @@ #undef STAT_HDRS /*lint -save -e778 */ -struct hpk_hdr sttbl[] = { +const struct hpk_hdr sttbl[] = { {{NULL, 0, 0}, {NULL, 0, 0}, hpk_idx, 0}, #define STAT_HDRS(j, k, v) \ { \ diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 1dad7a5..15e48f5 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -49,6 +49,8 @@ #include "vtcp.h" #include "hpack.h" +static const struct cmds http_cmds[]; + /* SECTION: client-server client/server * * Client and server threads are fake HTTP entities used to test your Varnish @@ -738,11 +740,8 @@ cmd_http_rxresphdrs(CMD_ARGS) * Ungzip rx'ed body */ -#define TRUST_ME(ptr) ((void*)(uintptr_t)(ptr)) - #define OVERHEAD 64L - static void cmd_http_gunzip_body(CMD_ARGS) { @@ -1679,7 +1678,7 @@ cmd_http_fatal(CMD_ARGS) * Same as for the top-level barrier */ -char PREFACE[] = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"; +const char PREFACE[] = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"; /* SECTION: client-server.spec.txpri txpri (client) * @@ -1785,7 +1784,7 @@ cmd_http_stream(CMD_ARGS) * Execute HTTP specifications */ -const struct cmds http_cmds[] = { +static const struct cmds http_cmds[] = { { "timeout", cmd_http_timeout }, { "txreq", cmd_http_txreq }, @@ -1856,7 +1855,8 @@ http_process(struct vtclog *vl, const char *spec, int sock, int *sfd) hp->gziplevel = 0; hp->gzipresidual = -1; - VTCP_hisname(sock, hp->rem_ip, VTCP_ADDRBUFSIZE, hp->rem_port, VTCP_PORTBUFSIZE); + VTCP_hisname(sock, + hp->rem_ip, VTCP_ADDRBUFSIZE, hp->rem_port, VTCP_PORTBUFSIZE); parse_string(spec, http_cmds, hp, vl); if (hp->h2) stop_h2(hp); diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index b8a0349..6f7cc04 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -52,7 +52,6 @@ #include "hpack.h" #include "vend.h" -#define MAX_HDR 50 #define ERR_MAX 13 #define BUF_SIZE (1024*2048) @@ -102,7 +101,6 @@ struct stream { pthread_cond_t cond; struct frame *frame; pthread_t tp; - unsigned reading; struct http *hp; int64_t ws; @@ -509,7 +507,7 @@ parse_hdr(struct stream *s, struct frame *f, struct vsb *vsb) if (f->type == TYPE_HEADERS && f->flags & PRIORITY){ shift += 5; n = vbe32dec(f->data); - s->dependency = n & ~(1 << 31); + s->dependency = n & ~(1U << 31); exclusive = n >> 31; s->weight = f->data[4]; @@ -521,7 +519,7 @@ parse_hdr(struct stream *s, struct frame *f, struct vsb *vsb) } else if (f->type == TYPE_PUSH_PROMISE){ shift += 4; n = vbe32dec(f->data); - f->md.promised = n & ~(1 << 31); + f->md.promised = n & ~(1U << 31); } AZ(VSB_bcat(vsb, data + shift, size - shift)); @@ -545,7 +543,7 @@ parse_prio(struct stream *s, struct frame *f) AN(buf); n = vbe32dec(f->data); - f->md.prio.stream = n & ~(1 << 31); + f->md.prio.stream = n & ~(1U << 31); s->dependency = f->md.prio.stream; if (n >> 31){ @@ -838,14 +836,14 @@ receive_frame(void *priv) } \ } while (0) -#define STRTOU32_CHECK(n, sp, p, v, c, l) \ -do { \ - sp++; \ - AN(*sp); \ - STRTOU32(n, *sp, p, v, c); \ - if (l && n >= (1 << l)) \ - vtc_log(v, 0, c " must be a %d-bits integer (found %s)", \ - l, *sp); \ +#define STRTOU32_CHECK(n, sp, p, v, c, l) \ +do { \ + sp++; \ + AN(*sp); \ + STRTOU32(n, *sp, p, v, c); \ + if (l && n >= (1U << l)) \ + vtc_log(v, 0, \ + c " must be a %d-bits integer (found %s)", l, *sp); \ } while (0) #define CHECK_LAST_FRAME(TYPE) \ @@ -1471,7 +1469,7 @@ cmd_tx11obj(CMD_ARGS) } else if (AV_IS("-litIdxHdr")) { av++; - if (AV_IS("inc")) { hdr.t = hpk_inc; } + if (AV_IS("inc")) { hdr.t = hpk_inc; } else if (AV_IS("not")) { hdr.t = hpk_not; } else if (AV_IS("never")) { hdr.t = hpk_never; } else @@ -1489,7 +1487,7 @@ cmd_tx11obj(CMD_ARGS) } else if (AV_IS("-litHdr")) { av++; - if (AV_IS("inc")) { hdr.t = hpk_inc; } + if (AV_IS("inc")) { hdr.t = hpk_inc; } else if (AV_IS("not")) { hdr.t = hpk_not; } else if (AV_IS("never")) { hdr.t = hpk_never; } else @@ -1538,7 +1536,7 @@ cmd_tx11obj(CMD_ARGS) f.flags |= PRIORITY; } else if (AV_IS("-ex")) { - exclusive = 1 << 31; + exclusive = 1U << 31; f.flags |= PRIORITY; } else if (AV_IS("-weight")) { @@ -2527,7 +2525,7 @@ stream_new(const char *name, struct http *h) s->dependency = 0; STRTOU32(s->id, name, p, h->vl, "stream"); - if (s->id & (1 << 31)) + if (s->id & (1U << 31)) vtc_log(h->vl, 0, "Stream id must be a 31-bits integer " "(found %s)", name); From phk at FreeBSD.org Mon Sep 5 07:09:09 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 05 Sep 2016 09:09:09 +0200 Subject: [master] a9af555 Fix the pseudo header lengths Message-ID: commit a9af55506fbac9d17ea2428f26f22b71fcfc4022 Author: Poul-Henning Kamp Date: Mon Sep 5 07:00:03 2016 +0000 Fix the pseudo header lengths diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index a1470d7..3626327 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -115,9 +115,11 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) "http" or "https"). For now set it as a normal header, stripping the first ':'. */ b++; + len-=1; n = hp->nhd; } else if (!strncmp(b, ":authority: ", namelen)) { b+=6; + len-=6; memcpy(b, "host", 4); n = hp->nhd; } else { From phk at FreeBSD.org Mon Sep 5 07:09:09 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 05 Sep 2016 09:09:09 +0200 Subject: [master] 26d615e Fix assert to be more useful Message-ID: commit 26d615ec4d3a815cf288f18f166a9744fcf10f6c Author: Poul-Henning Kamp Date: Mon Sep 5 07:00:32 2016 +0000 Fix assert to be more useful diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index 0ea2690..644b276 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -105,10 +105,10 @@ H2_Send(struct worker *wrk, struct h2_req *r2, int flush, (void)flush; - AN(ptr); CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); h2 = r2->h2sess; CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); + assert(len == 0 || ptr != NULL); Lck_Lock(&h2->sess->mtx); mfs = h2->their_settings[H2S_MAX_FRAME_SIZE]; From phk at FreeBSD.org Mon Sep 5 07:09:09 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 05 Sep 2016 09:09:09 +0200 Subject: [master] 3ad994a Add our first H2 varnishd test Message-ID: commit 3ad994a810ac95cb077e732ea6b78b638e651962 Author: Poul-Henning Kamp Date: Mon Sep 5 07:08:09 2016 +0000 Add our first H2 varnishd test diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc new file mode 100644 index 0000000..0833ca9 --- /dev/null +++ b/bin/varnishtest/tests/t02000.vtc @@ -0,0 +1,17 @@ +varnishtest "simple H2 get" + +server s1 { + rxreq + txresp -bodylen 10 +} -start + +varnish v1 -vcl+backend {} -start + +varnish v1 -cliok "param.set debug +syncvsl" + +client c1 { + stream 1 { + txreq + rxresp + } -run +} -run From phk at FreeBSD.org Mon Sep 5 07:37:09 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 05 Sep 2016 09:37:09 +0200 Subject: [master] db6276d Update the naming scheme for tests Message-ID: commit db6276da3f75e996a21308a1ade7cc82da1cc927 Author: Poul-Henning Kamp Date: Mon Sep 5 07:36:31 2016 +0000 Update the naming scheme for tests diff --git a/bin/varnishtest/tests/README b/bin/varnishtest/tests/README index 80075ea..3fefab8 100644 --- a/bin/varnishtest/tests/README +++ b/bin/varnishtest/tests/README @@ -13,18 +13,21 @@ Naming scheme [id]%05d.vtc - id ~ ^a --> varnishtest(1) tests - id ~ ^b --> Basic functionality tests - id ~ ^c --> Complex functionality tests - id ~ ^d --> Director VMOD tests - id ~ ^e --> ESI tests - id ~ ^g --> GZIP tests - id ~ ^j --> JAIL tests - id ~ ^l --> VSL tests - id ~ ^m --> VMOD tests excluding director - id ~ ^o --> prOxy protocol - id ~ ^p --> Persistent tests - id ~ ^r --> Regression tests, same number as ticket - id ~ ^s --> Slow tests, expiry, grace etc. - id ~ ^u --> Unusual background processes - id ~ ^v --> VCL tests: execute VRT functions + id ~ ^a --> varnishtest(1) tests + id ~ ^a02 --> HTTP2 + id ~ ^b --> Basic functionality tests + id ~ ^c --> Complex functionality tests + id ~ ^d --> Director VMOD tests + id ~ ^e --> ESI tests + id ~ ^g --> GZIP tests + id ~ ^j --> JAIL tests + id ~ ^l --> VSL tests + id ~ ^m --> VMOD tests excluding director + id ~ ^o --> prOxy protocol + id ~ ^p --> Persistent tests + id ~ ^r --> Regression tests, same number as ticket + id ~ ^s --> Slow tests, expiry, grace etc. + id ~ ^t --> Transport protocol tests + id ~ ^t02 --> HTTP2 + id ~ ^u --> Unusual background processes + id ~ ^v --> VCL tests: execute VRT functions From dridi.boukelmoune at gmail.com Mon Sep 5 07:49:10 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 05 Sep 2016 09:49:10 +0200 Subject: [master] 1d56bc0 Remove unused import Message-ID: commit 1d56bc0ccbdb562e17197dad3e58b8ff93d81624 Author: Dridi Boukelmoune Date: Mon Sep 5 09:48:05 2016 +0200 Remove unused import diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 6f7cc04..3fb0dca 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -32,7 +32,6 @@ #include #include -#include #include #include #include From phk at FreeBSD.org Mon Sep 5 08:01:10 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 05 Sep 2016 10:01:10 +0200 Subject: [master] dc8177b Don't skip all tests depending on features Message-ID: commit dc8177b2aeb336fa1e280472ac9a3f1ab7948852 Author: Poul-Henning Kamp Date: Mon Sep 5 07:41:35 2016 +0000 Don't skip all tests depending on features diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index f3bf538..fbedb85 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -574,7 +574,7 @@ cmd_feature(CMD_ARGS) if (av == NULL) return; - for (; *av != NULL; av++) { + for (av++; *av != NULL; av++) { #ifdef SO_RCVTIMEO_WORKS if (!strcmp(*av, "SO_RCVTIMEO_WORKS")) continue; From phk at FreeBSD.org Mon Sep 5 08:01:10 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 05 Sep 2016 10:01:10 +0200 Subject: [master] d602c96 Fail tests which ask for features we have not heard about. Message-ID: commit d602c96722b358182639c1c3b2013741668a9fa6 Author: Poul-Henning Kamp Date: Mon Sep 5 07:59:27 2016 +0000 Fail tests which ask for features we have not heard about. diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index fbedb85..60c6ae9 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -567,6 +567,7 @@ static void cmd_feature(CMD_ARGS) { int r; + int good; (void)priv; (void)cmd; @@ -574,54 +575,65 @@ cmd_feature(CMD_ARGS) if (av == NULL) return; +#define FEATURE(nm, tst) \ + do { \ + if (!strcmp(*av, nm)) { \ + if (tst) { \ + good = 1; \ + } else { \ + vtc_stop = 1; \ + } \ + } \ + } while (0) + for (av++; *av != NULL; av++) { + good = 0; + if (!strcmp(*av, "SO_RCVTIMEO_WORKS")) { #ifdef SO_RCVTIMEO_WORKS - if (!strcmp(*av, "SO_RCVTIMEO_WORKS")) - continue; + good = 1; +#else + vtc_stop = 1; #endif - if (sizeof(void*) == 8 && !strcmp(*av, "64bit")) - continue; - - if (!strcmp(*av, "pcre_jit") && VRE_has_jit) - continue; + } if (!strcmp(*av, "!OSX")) { #if !defined(__APPLE__) || !defined(__MACH__) - continue; + good = 1; +#else + vtc_stop = 1; #endif } - if (!strcmp(*av, "dns") && feature_dns) - continue; - - if (!strcmp(*av, "topbuild") && iflg) - continue; - - if (!strcmp(*av, "root") && !geteuid()) - continue; - - if (!strcmp(*av, "user_varnish") && - getpwnam("varnish") != NULL) - continue; - - if (!strcmp(*av, "user_vcache") && - getpwnam("vcache") != NULL) - continue; - - if (!strcmp(*av, "group_varnish") && - getgrnam("varnish") != NULL) - continue; + FEATURE("pcre_jit", VRE_has_jit); + FEATURE("64bit", sizeof(void*) == 8); + FEATURE("dns", feature_dns); + FEATURE("topbuild", iflg); + FEATURE("root", !geteuid()); + FEATURE("user_varnish", getpwnam("varnish") != NULL); + FEATURE("user_vcache", getpwnam("vcache") != NULL); + FEATURE("group_varnish", getgrnam("varnish") != NULL); if (!strcmp(*av, "cmd")) { av++; - if (*av == NULL) + if (*av == NULL) { vtc_log(vl, 0, "Missing the command-line"); + return; + } r = system(*av); if (WEXITSTATUS(r) == 0) - continue; + good = 1; + else + vtc_stop = 1; } + if (good) + continue; - vtc_log(vl, 1, "SKIPPING test, missing feature: %s", *av); - vtc_stop = 1; + if (!vtc_stop) { + vtc_log(vl, 0, + "FAIL test, unknown feature: %s", *av); + } else { + vtc_log(vl, 1, + "SKIPPING test, lacking feature: %s", *av); + } return; } } From phk at FreeBSD.org Mon Sep 5 08:01:10 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 05 Sep 2016 10:01:10 +0200 Subject: [master] 1727141 feature test that nghttp can be executed Message-ID: commit 17271417eea8b9596a190e175f372ca1c61489da Author: Poul-Henning Kamp Date: Mon Sep 5 07:59:40 2016 +0000 feature test that nghttp can be executed diff --git a/bin/varnishtest/tests/a02022.vtc b/bin/varnishtest/tests/a02022.vtc index 0c34071..aeb1ff6 100644 --- a/bin/varnishtest/tests/a02022.vtc +++ b/bin/varnishtest/tests/a02022.vtc @@ -1,5 +1,7 @@ varnishtest "H/1 -> H/2 upgrade" +feature cmd "nghttp --help > /dev/null 2>&1" + server s1 { rxreq upgrade From phk at FreeBSD.org Mon Sep 5 08:22:12 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 05 Sep 2016 10:22:12 +0200 Subject: [master] f330d35 Until our HTTP2 implementation is more comprehensive it is a feature you need to explicitly enable (params.set feature +http2) Message-ID: commit f330d354ea176292384b647318b3cde0b91b630e Author: Poul-Henning Kamp Date: Mon Sep 5 08:17:33 2016 +0000 Until our HTTP2 implementation is more comprehensive it is a feature you need to explicitly enable (params.set feature +http2) diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index 2a4a11a..4c44102 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -393,6 +393,13 @@ HTTP1_Session(struct worker *wrk, struct req *req) WRONG("htc_status (nonbad)"); if (H2_prism_complete(req->htc) == HTC_S_COMPLETE) { + if (!FEATURE(FEATURE_HTTP2)) { + VSLb(req->vsl, SLT_Debug, + "H2 attempt"); + SES_Close(req->sp, req->doclose); + http1_setstate(sp, H1CLEANUP); + continue; + } VSLb(req->vsl, SLT_Debug, "H2 Prior Knowledge Upgrade"); http1_setstate(sp, NULL); @@ -409,10 +416,17 @@ HTTP1_Session(struct worker *wrk, struct req *req) http1_setstate(sp, H1CLEANUP); continue; } - if (req->htc->body_status == BS_NONE && + if (req->htc->body_status == BS_NONE && /* XXX */ http_HdrIs(req->http, H_Upgrade, "h2c")) { + if (!FEATURE(FEATURE_HTTP2)) { + VSLb(req->vsl, SLT_Debug, + "H2 upgrade attempt"); + SES_Close(req->sp, req->doclose); + http1_setstate(sp, H1CLEANUP); + continue; + } VSLb(req->vsl, SLT_Debug, - "H2 Optimistic Upgrade"); + "H2 Upgrade"); http1_setstate(sp, NULL); req->err_code = 2; SES_SetTransport(wrk, sp, req, &H2_transport); diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index 0833ca9..74e9f3b 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -7,6 +7,7 @@ server s1 { varnish v1 -vcl+backend {} -start +varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.set debug +syncvsl" client c1 { diff --git a/include/tbl/feature_bits.h b/include/tbl/feature_bits.h index 7bd675f..a178b56 100644 --- a/include/tbl/feature_bits.h +++ b/include/tbl/feature_bits.h @@ -63,4 +63,8 @@ FEATURE_BIT(HTTPS_SCHEME, https_scheme, "Also split https URIs", "Extract host from full URI in the request line if the scheme is https." ) +FEATURE_BIT(HTTP2, http2, + "Support HTTP/2 protocol", + "Enable HTTP/2 protocol support." +) /*lint -restore */ From phk at FreeBSD.org Mon Sep 5 09:07:12 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 05 Sep 2016 11:07:12 +0200 Subject: [master] 782e07a Implement PING frames Message-ID: commit 782e07a19ee5be7b1a7d1f10e02033f42bb75ff2 Author: Poul-Henning Kamp Date: Mon Sep 5 09:05:53 2016 +0000 Implement PING frames diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 4175367..6c69558 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -111,7 +111,6 @@ static const uint8_t H2_settings[] = { DUMMY_FRAME(data) DUMMY_FRAME(rst_stream) -DUMMY_FRAME(ping) DUMMY_FRAME(push_promise) DUMMY_FRAME(continuation) @@ -289,6 +288,20 @@ h2_vsl_frame(const struct h2_sess *h2, const void *ptr, size_t len) */ void __match_proto__(h2_frame_f) +h2_rx_ping(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) +{ + (void)r2; + xxxassert(h2->rxf_len == 8); + xxxassert(h2->rxf_flags == 0); + xxxassert(h2->rxf_stream == 0); + H2_Send_Frame(wrk, h2, + H2_FRAME_PING, H2FF_PING_ACK, 8, 0, h2->rxf_data); +} + +/********************************************************************** + */ + +void __match_proto__(h2_frame_f) h2_rx_goaway(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) { uint32_t error; diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index 74e9f3b..f1343ec 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -15,4 +15,10 @@ client c1 { txreq rxresp } -run + stream 0 { + txping -data "_-__-_-_" + rxping + expect ping.ack == "true" + expect ping.data == "_-__-_-_" + } -run } -run From dridi.boukelmoune at gmail.com Mon Sep 5 11:45:13 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 05 Sep 2016 13:45:13 +0200 Subject: [4.1] 370ba20 Don't skip all tests depending on features Message-ID: commit 370ba2073be09ba8c7a0dec5e6d15b3d3ea59d40 Author: Poul-Henning Kamp Date: Mon Sep 5 07:41:35 2016 +0000 Don't skip all tests depending on features diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 952adac..cd49c9f 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -577,7 +577,7 @@ cmd_feature(CMD_ARGS) if (av == NULL) return; - for (; *av != NULL; av++) { + for (av++; *av != NULL; av++) { #ifdef SO_RCVTIMEO_WORKS if (!strcmp(*av, "SO_RCVTIMEO_WORKS")) continue; From dridi.boukelmoune at gmail.com Mon Sep 5 11:45:13 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 05 Sep 2016 13:45:13 +0200 Subject: [4.1] 20d4cfd Fail tests which ask for features we have not heard about. Message-ID: commit 20d4cfda55dc87d470ab5864d16195bcdf7b0320 Author: Poul-Henning Kamp Date: Mon Sep 5 07:59:27 2016 +0000 Fail tests which ask for features we have not heard about. diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index cd49c9f..5532e33 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -570,6 +570,7 @@ static void cmd_feature(CMD_ARGS) { int r; + int good; (void)priv; (void)cmd; @@ -577,54 +578,65 @@ cmd_feature(CMD_ARGS) if (av == NULL) return; +#define FEATURE(nm, tst) \ + do { \ + if (!strcmp(*av, nm)) { \ + if (tst) { \ + good = 1; \ + } else { \ + vtc_stop = 1; \ + } \ + } \ + } while (0) + for (av++; *av != NULL; av++) { + good = 0; + if (!strcmp(*av, "SO_RCVTIMEO_WORKS")) { #ifdef SO_RCVTIMEO_WORKS - if (!strcmp(*av, "SO_RCVTIMEO_WORKS")) - continue; + good = 1; +#else + vtc_stop = 1; #endif - if (sizeof(void*) == 8 && !strcmp(*av, "64bit")) - continue; - - if (!strcmp(*av, "pcre_jit") && VRE_has_jit) - continue; + } if (!strcmp(*av, "!OSX")) { #if !defined(__APPLE__) || !defined(__MACH__) - continue; + good = 1; +#else + vtc_stop = 1; #endif } - if (!strcmp(*av, "dns") && feature_dns) - continue; - - if (!strcmp(*av, "topbuild") && iflg) - continue; - - if (!strcmp(*av, "root") && !geteuid()) - continue; - - if (!strcmp(*av, "user_varnish") && - getpwnam("varnish") != NULL) - continue; - - if (!strcmp(*av, "user_vcache") && - getpwnam("vcache") != NULL) - continue; - - if (!strcmp(*av, "group_varnish") && - getgrnam("varnish") != NULL) - continue; + FEATURE("pcre_jit", VRE_has_jit); + FEATURE("64bit", sizeof(void*) == 8); + FEATURE("dns", feature_dns); + FEATURE("topbuild", iflg); + FEATURE("root", !geteuid()); + FEATURE("user_varnish", getpwnam("varnish") != NULL); + FEATURE("user_vcache", getpwnam("vcache") != NULL); + FEATURE("group_varnish", getgrnam("varnish") != NULL); if (!strcmp(*av, "cmd")) { av++; - if (*av == NULL) + if (*av == NULL) { vtc_log(vl, 0, "Missing the command-line"); + return; + } r = system(*av); if (WEXITSTATUS(r) == 0) - continue; + good = 1; + else + vtc_stop = 1; } + if (good) + continue; - vtc_log(vl, 1, "SKIPPING test, missing feature: %s", *av); - vtc_stop = 1; + if (!vtc_stop) { + vtc_log(vl, 0, + "FAIL test, unknown feature: %s", *av); + } else { + vtc_log(vl, 1, + "SKIPPING test, lacking feature: %s", *av); + } return; } } From nils.goroll at uplex.de Mon Sep 5 14:57:10 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 05 Sep 2016 16:57:10 +0200 Subject: [master] b96a983 move vrnd.h to vmod api Message-ID: commit b96a9834b8e2aa8232bc9e44b87bc14bbdd844d0 Author: Nils Goroll Date: Wed Aug 31 18:35:31 2016 +0200 move vrnd.h to vmod api diff --git a/include/Makefile.am b/include/Makefile.am index 5ea662a..f3dfe09 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -72,7 +72,8 @@ nobase_pkginclude_HEADERS += \ vsb.h \ vsha256.h \ vtcp.h \ - vtim.h + vtim.h \ + vrnd.h # Private headers nobase_noinst_HEADERS = \ @@ -93,7 +94,6 @@ nobase_noinst_HEADERS = \ vmb.h \ vnum.h \ vpf.h \ - vrnd.h \ vsl_priv.h \ vsm_priv.h \ vsub.h \ From dridi.boukelmoune at gmail.com Mon Sep 5 20:16:08 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 05 Sep 2016 22:16:08 +0200 Subject: [4.1] e607a6b Make life easier for vim/syntastic users Message-ID: commit e607a6be91ef20fda4665eaa995a07c240aa5a97 Author: Dridi Boukelmoune Date: Mon Sep 5 22:11:52 2016 +0200 Make life easier for vim/syntastic users diff --git a/.syntastic_c_config b/.syntastic_c_config new file mode 100644 index 0000000..4238ab4 --- /dev/null +++ b/.syntastic_c_config @@ -0,0 +1 @@ +let g:syntastic_c_include_dirs = [".", "include"] From phk at FreeBSD.org Mon Sep 5 21:14:08 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 05 Sep 2016 23:14:08 +0200 Subject: [master] 9f52d60 Insist that VCL events are only sent from CLI thread Message-ID: commit 9f52d6033262c2647f7ec719c83fec1a63402fd0 Author: Poul-Henning Kamp Date: Mon Sep 5 21:13:07 2016 +0000 Insist that VCL events are only sent from CLI thread diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 5afc65a..627b8c9 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -518,6 +518,7 @@ static int vcl_setup_event(VRT_CTX, enum vcl_event_e ev) { + ASSERT_CLI(); CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); @@ -533,6 +534,7 @@ static void vcl_failsafe_event(VRT_CTX, enum vcl_event_e ev) { + ASSERT_CLI(); CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); From phk at FreeBSD.org Tue Sep 6 06:28:09 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 06 Sep 2016 08:28:09 +0200 Subject: [master] eb8b4ee More bells & whistles Message-ID: commit eb8b4ee11724f42cec06bad5f21401a4be8b8f5d Author: Poul-Henning Kamp Date: Tue Sep 6 06:27:10 2016 +0000 More bells & whistles diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index f1343ec..f4d660c 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -2,6 +2,7 @@ varnishtest "simple H2 get" server s1 { rxreq + expect req.http.host == foo.bar txresp -bodylen 10 } -start @@ -12,8 +13,18 @@ varnish v1 -cliok "param.set debug +syncvsl" client c1 { stream 1 { - txreq + txprio -weight 10 -stream 0 + } -start + stream 3 { + txprio -weight 10 -stream 0 + } -start + stream 5 { + txprio -weight 10 -stream 2 + } -start + stream 7 { + txreq -hdr :authority foo.bar rxresp + expect resp.status == 200 } -run stream 0 { txping -data "_-__-_-_" From hermunn at varnish-software.com Tue Sep 6 08:35:14 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Tue, 06 Sep 2016 10:35:14 +0200 Subject: [4.1] c3374b1 Revert to wrlock to avoid vd->vbm trampling Message-ID: commit c3374b185e7db43db673590fdefc4424ca8de610 Author: Guillaume Quintard Date: Tue Aug 2 11:34:21 2016 +0200 Revert to wrlock to avoid vd->vbm trampling fix #2027 diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index e5a52a3..cdaa09b 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -217,7 +217,7 @@ vdir_pick_be(struct vdir *vd, double w, const struct busyobj *bo) double tw = 0.0; VCL_BACKEND be = NULL; - vdir_rdlock(vd); + vdir_wrlock(vd); for (u = 0; u < vd->n_backend; u++) { if (vd->backend[u]->healthy(vd->backend[u], bo, NULL)) { vbit_clr(vd->vbm, u); From scanner at varnish-cache.org Tue Sep 6 16:50:29 2016 From: scanner at varnish-cache.org (scanner at varnish-cache.org) Date: Tue, 06 Sep 2016 11:50:29 -0500 Subject: Message from "CUKPR0014729" Message-ID: <20160906115029KX.DCSML-S000990000.70B64BD0A8E11@varnish-cache.org> This E-mail was sent from "CUKPR0014729" (Aficio MP C305). Scan Date: Tue, 06 Sep 2016 11:50:29 -0500 Queries to: From phk at FreeBSD.org Wed Sep 7 07:26:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 07 Sep 2016 09:26:11 +0200 Subject: [master] 88f326a Add a testcase for H1->H2 upgrade Message-ID: commit 88f326a799c05b8f3d4256ede612c6b08abbb720 Author: Poul-Henning Kamp Date: Wed Sep 7 07:02:33 2016 +0000 Add a testcase for H1->H2 upgrade diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index f4d660c..89211f9 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -1,4 +1,4 @@ -varnishtest "simple H2 get" +varnishtest "Direct H2 start" server s1 { rxreq diff --git a/bin/varnishtest/tests/t02001.vtc b/bin/varnishtest/tests/t02001.vtc new file mode 100644 index 0000000..2fc8e54 --- /dev/null +++ b/bin/varnishtest/tests/t02001.vtc @@ -0,0 +1,29 @@ +varnishtest "H1->H2 Upgrade" + +server s1 { + rxreq + expect req.http.host == foo.bar + txresp -status 400 -bodylen 10 +} -start + +varnish v1 -vcl+backend {} -start + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set debug +syncvsl" + +client c1 { + send "GET / HTTP/1.1\r\n" + send "Host: foo.bar\r\n" + send "Upgrade: h2c\r\n" + send "HTTP2-Settings: AAMAAABkAAQAAP__\r\n" + send "\r\n" + rxresp + expect resp.status == 101 + expect resp.http.upgrade == h2c + expect resp.http.connection == Upgrade + stream 1 { + rxresp + expect resp.status == 400 + expect resp.bodylen == 10 + } -run +} -run From phk at FreeBSD.org Wed Sep 7 07:41:10 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 07 Sep 2016 09:41:10 +0200 Subject: [master] 45de3ad VSL the frame before holding the mtx Message-ID: commit 45de3ad225f28f2dfeef5c901aff33a62d79fb73 Author: Poul-Henning Kamp Date: Wed Sep 7 07:40:02 2016 +0000 VSL the frame before holding the mtx diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 6c69558..f76263f 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -513,6 +513,8 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2) /* XXX: later full DATA will not be rx'ed yet. */ HTC_RxPipeline(h2->htc, h2->htc->rxbuf_b + h2->rxf_len + 9); + h2_vsl_frame(h2, h2->htc->rxbuf_b, 9L + h2->rxf_len); + Lck_Lock(&h2->sess->mtx); VTAILQ_FOREACH(r2, &h2->streams, list) if (r2->stream == h2->rxf_stream) @@ -523,8 +525,6 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2) r2 = h2_new_req(wrk, h2, h2->rxf_stream, NULL); } - h2_vsl_frame(h2, h2->htc->rxbuf_b, 9L + h2->rxf_len); - ft = (enum h2frame)h2->htc->rxbuf_b[3]; switch (ft) { #define H2_FRAME(l,u,t,f) \ From phk at FreeBSD.org Wed Sep 7 08:05:14 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 07 Sep 2016 10:05:14 +0200 Subject: [master] 111bca9 Make sure stream numbers don't become unordered Message-ID: commit 111bca97469052d7c061cb4302d4764c7b69552b Author: Poul-Henning Kamp Date: Wed Sep 7 08:04:35 2016 +0000 Make sure stream numbers don't become unordered diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index 89211f9..da3d6b2 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -14,18 +14,18 @@ varnish v1 -cliok "param.set debug +syncvsl" client c1 { stream 1 { txprio -weight 10 -stream 0 - } -start + } -run stream 3 { txprio -weight 10 -stream 0 - } -start + } -run stream 5 { txprio -weight 10 -stream 2 - } -start + } -run stream 7 { txreq -hdr :authority foo.bar rxresp expect resp.status == 200 - } -run + } -start stream 0 { txping -data "_-__-_-_" rxping From dridi.boukelmoune at gmail.com Wed Sep 7 08:49:11 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 07 Sep 2016 10:49:11 +0200 Subject: [master] e8f4171 Implicit backend cast to boolean Message-ID: commit e8f41715b95d45c0b9088b64dd2d01d6e9b8b733 Author: Dridi Boukelmoune Date: Mon Aug 29 16:33:18 2016 +0200 Implicit backend cast to boolean This introduces a way to check whether a director returned no backend and act upon it. Closes #2060 diff --git a/bin/varnishtest/tests/d00014.vtc b/bin/varnishtest/tests/d00014.vtc new file mode 100644 index 0000000..3053db0 --- /dev/null +++ b/bin/varnishtest/tests/d00014.vtc @@ -0,0 +1,20 @@ +varnishtest "Backend as a boolean expression" + +server s1 -start + +varnish v1 -vcl+backend { + import debug; + + sub vcl_recv { + set req.backend_hint = debug.no_backend(); + if (!req.backend_hint) { + return (synth(404)); + } + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 404 +} -run diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index b68fdc6..789b20d 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -87,7 +87,11 @@ characters except for the NUL (0x00) character. Booleans ~~~~~~~~ -Booleans can be either *true* or *false*. +Booleans can be either *true* or *false*. Some values may be inferred as +truthy or falthy. When cast to booleans, strings will evaluate to *false* +if they are empty or NULL. This allows broad checks on headers existence +for instance. Backend expressions can also be tested as booleans and can +help find when a director like the hash director returns a NULL backend. Time ---- diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 4838450..d91c8bd 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1191,7 +1191,7 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, vcc_type_t fmt) default: break; } - if (fmt == BOOL && (*e)->fmt == STRING) { + if (fmt == BOOL && ((*e)->fmt == STRING || (*e)->fmt == BACKEND)) { *e = vcc_expr_edit(BOOL, "(\v1 != 0)", *e, NULL); return; } From nils.goroll at uplex.de Wed Sep 7 08:52:13 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 07 Sep 2016 10:52:13 +0200 Subject: [master] c4c444c Add a workspace for vcl events Message-ID: commit c4c444ca3221f8b94eb9294cbd59385a7826af68 Author: Nils Goroll Date: Wed Sep 7 09:52:09 2016 +0200 Add a workspace for vcl events diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 627b8c9..e213010 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -514,9 +514,31 @@ VRT_rel_vcl(VRT_CTX, struct vclref **refp) /*--------------------------------------------------------------------*/ +static struct ws * +vcl_event_ws(void) +{ + static struct ws *ws = NULL; + static char *ws_snap; + + ASSERT_CLI(); + + if (ws == NULL) { + ws = malloc(sizeof(*ws)); + AN(ws); + WS_Init(ws, "cli", malloc(cache_param->workspace_client), + cache_param->workspace_client); + ws_snap = WS_Snapshot(ws); + } else { + WS_Reset(ws, ws_snap); + WS_Assert(ws); + } + return (ws); +} + static int -vcl_setup_event(VRT_CTX, enum vcl_event_e ev) +vcl_setup_event(struct vrt_ctx *ctx, enum vcl_event_e ev) { + int r; ASSERT_CLI(); CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -526,12 +548,16 @@ vcl_setup_event(VRT_CTX, enum vcl_event_e ev) AN(ctx->vcl); AN(ctx->msg); assert(ev == VCL_EVENT_LOAD || ev == VCL_EVENT_WARM); + AZ(ctx->ws); + ctx->ws = vcl_event_ws(); + r = ctx->vcl->conf->event_vcl(ctx, ev); + ctx->ws = NULL; - return (ctx->vcl->conf->event_vcl(ctx, ev)); + return (r); } static void -vcl_failsafe_event(VRT_CTX, enum vcl_event_e ev) +vcl_failsafe_event(struct vrt_ctx *ctx, enum vcl_event_e ev) { ASSERT_CLI(); @@ -541,9 +567,13 @@ vcl_failsafe_event(VRT_CTX, enum vcl_event_e ev) AN(ctx->handling); AN(ctx->vcl); assert(ev == VCL_EVENT_COLD || ev == VCL_EVENT_DISCARD); + AZ(ctx->ws); + ctx->ws = vcl_event_ws(); if (ctx->vcl->conf->event_vcl(ctx, ev) != 0) WRONG("A VMOD cannot fail COLD or DISCARD events"); + + ctx->ws = NULL; } static void @@ -564,7 +594,7 @@ vcl_print_refs(VRT_CTX) } static int -vcl_set_state(VRT_CTX, const char *state) +vcl_set_state(struct vrt_ctx *ctx, const char *state) { struct vcl *vcl; int i = 0; From nils.goroll at uplex.de Wed Sep 7 08:52:13 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 07 Sep 2016 10:52:13 +0200 Subject: [master] edd955f fix std.log() to behave the same in ini/fini and the rest, now that we got a ws Message-ID: commit edd955fcafbd1bfa6a64dee6c98d61cf12bb89d8 Author: Nils Goroll Date: Tue Sep 6 12:33:03 2016 +0200 fix std.log() to behave the same in ini/fini and the rest, now that we got a ws Previously, std.log had printf-semantics for the case that there is no ctx->vsl. Now, for all cases, it just prints the given STRING_LIST Fixes the example given in #2063 diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index b4d29c8..6028b0b 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -20,7 +20,7 @@ varnish v1 -vcl+backend { set resp.http.who = debug.author(phk); debug.test_priv_call(); debug.test_priv_vcl(); - std.log("VCL initiated log"); + std.log("VCL" + " initiated " + "log"); std.syslog(8 + 7, "Somebody runs varnishtest"); debug.rot52(resp); } @@ -36,6 +36,20 @@ client c1 { expect resp.http.encrypted == "ROT52" } -run +logexpect l1 -v v1 -g raw -d 1 { + expect * 1001 VCL_call {^DELIVER} + expect 0 = RespUnset {^foo: bAr} + expect 0 = RespHeader {^foo: BAR} + expect 0 = RespUnset {^bar: fOo} + expect 0 = RespHeader {^bar: foo} + expect 0 = RespHeader {^who: Poul-Henning} + expect 0 = VCL_Log {^VCL initiated log} + expect 0 = RespHeader {^Encrypted: ROT52} + expect 0 = VCL_return {^deliver} +} -start + +logexpect l1 -wait + varnish v1 -errvcl {Wrong enum value. Expected one of:} { import debug; sub vcl_deliver { diff --git a/bin/varnishtest/tests/r01924.vtc b/bin/varnishtest/tests/r01924.vtc index d553124..3ea78eb 100644 --- a/bin/varnishtest/tests/r01924.vtc +++ b/bin/varnishtest/tests/r01924.vtc @@ -9,16 +9,26 @@ varnish v1 -vcl+backend { import std; sub vcl_init { - std.log("init"); + std.log("init" + " one " + "two"); std.syslog(8 + 7, "init"); } sub vcl_fini { - std.log("fini"); + std.log("fini" + " one " + "two"); std.syslog(8 + 7, "fini"); } } -start +logexpect l1 -v v1 -g raw -d 1 { + expect 0 0 CLI {^Rd vcl.load} + expect 0 = VCL_Log {^init one two} + + expect * 0 CLI {^Rd vcl.discard} + expect 0 = VCL_Log {^fini one two} +} -start + varnish v1 -vcl+backend { } varnish v1 -cliok "vcl.discard vcl1" + +logexpect l1 -wait diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 7f0b35d..a76fa78 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -133,27 +133,31 @@ vmod_random(VRT_CTX, VCL_REAL lo, VCL_REAL hi) } VCL_VOID __match_proto__(td_std_log) -vmod_log(VRT_CTX, const char *fmt, ...) +vmod_log(VRT_CTX, const char *s, ...) { + txt t; unsigned u; va_list ap; - txt t; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - va_start(ap, fmt); - if (ctx->vsl != NULL) { - u = WS_Reserve(ctx->ws, 0); - t.b = ctx->ws->f; - t.e = VRT_StringList(ctx->ws->f, u, fmt, ap); - if (t.e != NULL) { - assert(t.e > t.b); - t.e--; - VSLbt(ctx->vsl, SLT_VCL_Log, t); - } - WS_Release(ctx->ws, 0); - } else - VSLv(SLT_VCL_Log, 0, fmt, ap); + WS_Assert(ctx->ws); + + + u = WS_Reserve(ctx->ws, 0); + t.b = ctx->ws->f; + va_start(ap, s); + t.e = VRT_StringList(ctx->ws->f, u, s, ap); va_end(ap); + + if (t.e != NULL) { + assert(t.e > t.b); + t.e--; + if (ctx->vsl != NULL) + VSLbt(ctx->vsl, SLT_VCL_Log, t); + else + VSL(SLT_VCL_Log, 0, "%s", t.b); + } + WS_Release(ctx->ws, 0); } VCL_VOID __match_proto__(td_std_syslog) From nils.goroll at uplex.de Wed Sep 7 09:34:14 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 07 Sep 2016 11:34:14 +0200 Subject: [master] d035171 document it before the next person stumbles over it Message-ID: commit d0351710d38fc78dde7ef16022dc14dc19c5ba99 Author: Nils Goroll Date: Wed Sep 7 11:33:30 2016 +0200 document it before the next person stumbles over it diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index e213010..d174e75 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -514,6 +514,10 @@ VRT_rel_vcl(VRT_CTX, struct vclref **refp) /*--------------------------------------------------------------------*/ +/* + * the workspace allocated here exists for the lifetime of the varnishd process + * and is never explicitly freed + */ static struct ws * vcl_event_ws(void) { From hermunn at varnish-software.com Wed Sep 7 09:52:12 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Wed, 07 Sep 2016 11:52:12 +0200 Subject: [4.1] 564250a Use "none" to disable authentication Message-ID: commit 564250aec47047375c4bbbe4ab983f5723e04ebe Author: P?l Hermunn Johansen Date: Wed Sep 7 11:50:17 2016 +0200 Use "none" to disable authentication This is an adaptation of b2c18706f2d4b100 by Federico G. Schwindt. Previously you needed an empty argument (''), which didn't play very well along sysv init scripts and was not documented. Now both "none" and the empty argument work, but an extra warning is echoed if the empty argument is used. The documentation only mentions "none" for disabling the secret, since this is the only way of doing it in the next version. Fixes #2007. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 7440302..ae4ce15 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -717,6 +717,11 @@ main(int argc, char * const *argv) if (S_arg != NULL && *S_arg == '\0') { fprintf(stderr, "Warning: Empty -S argument, no CLI authentication.\n"); + fprintf(stderr, "Warning: For forward compatibility," + " please switch to using -S none\n"); + } else if (S_arg != NULL && !strcmp(S_arg, "none")) { + fprintf(stderr, + "Warning: CLI authentication disabled.\n"); } else if (S_arg != NULL) { VJ_master(JAIL_MASTER_FILE); o = open(S_arg, O_RDONLY, 0); @@ -817,7 +822,7 @@ main(int argc, char * const *argv) if (d_flag) mgt_cli_setup(0, 1, 1, "debug", cli_stdin_close, NULL); - if (*S_arg != '\0') + if (strcmp(S_arg, "none")) mgt_cli_secret(S_arg); if (M_arg != NULL) diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 351ba15..78a285b 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -123,11 +123,11 @@ OPTIONS *vmod_dir* read only as these can potentially be used to escalate privileges from the CLI. --S file +-S secret-file Path to a file containing a secret used for authorizing access to the management port. If not provided a new secret will be drawn - from the system PRNG. + from the system PRNG. To disable authentication use ``none``. -s <[name=]type[,options]> From varnish-commit at varnish-cache.org Wed Sep 7 19:14:47 2016 From: varnish-commit at varnish-cache.org (ufh) Date: Thu, 8 Sep 2016 03:14:47 +0800 Subject: =?utf-8?B?NjU0MDUg5Lit6auY5bGC566h55CG6ICF55qE6IGM5Lia5YyW5L+u54K8ICA=?= =?utf-8?B?dmFybmlzaC1jb21taXQ=?= Message-ID: <20160908031501118780@iyu.com> ???????????? ????? 2016?9?28-29????10?20-21????10?27-28???? ?????????????????????????? ?????3800?/?(??????????????) ?????021-31006787?0755-6128 0006 18917870808 ??? QQ/??? 320588808 ????: ??????????????????????????????????????????????????????????????????????????????????????????????????????????? ???????????????????????????????????????????????????????? ????????????????????????????????????????????????????? ????????????????????????????????????????????????????????????????????????? ?????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? ???????????????????????????????????????? ????: ???????????????????? ??????????? ???????????????????? ???????????????????????? ????????????????????? ????: ?????????????? A:?????? 1. ???????????? 2. ??????????? B:???????? 1. ????????????????????????????????????? 2. ????????????????????????????????? C????? 1. ????????? 2. ??????? 3. ???????????? 4. ???????????????? D??????? 1. ??????80%???????20%????? 2. ???????? 3. ?????????????????????? 4. ??????????7/21?? 5. ?????????? 6. ????????? 7. ?????????????? ? ?????????????????????????????????????????????????????????????????????????? ? ??????????????????????????????????????????????????????????????? ? ???????????????????????????????????????????????????????????????????????????????? ????????? A:???? 1. ????????? 2. ????????? 3. ???????????????? B:????????????? 1. ?????????????????????????????? 2. ??????????????? 3. ??????????? 4. ??????????? 5. ??????????? C:????? 1. ????????????? 2. ???? 3. ?????????????????????????????????? 4. ???????????? 5. ??????????? 6. ?????????? 7. ??????????????????????????????????? D????? 1. ?????????? 2. ????????????????? 3. ?????????????? 4. ??????????????????? ? ??????????????????????????????????????????????????????????????????????????? ? ????????????????????????????????????????????????????????????????????????? ???????????? A:????? 1. ??????????????????????????? 2. ???????????????????????? 3. ??????????????? 4. ???????? B:??????? 1. ???????????? 2. ????????? 3. ????????????? 4. ????????????????? C:??????? 1. ??????? 2. ?????????????????? 3. ?????????????????? 4. ?????????????????? ? ????????????????????????????????????????? ? ??????????????????????????????????????????????????????????????????? ????????? A:???????? 1. ??????????? 2. ????????????? 3. ????? 4. ???????????????? 5. ????????? B:???? 1. ??????? 2. ???????? 3. PDCA????? ???????????????????????? ?????????? A:???? 1. ??VS???? 2. ????????????????? ???? 3. ????????????????? B:???? 1. ???????????????????????? 2. ????? 3. ??????? 4. ???????? C:????? 1. ????? 2. ?????????? 3. ?????? 4. ???????????????? 5. ???????????? 6. ?????? 7. ??????? D:????? 1. ???????????????????????????????????? 2. ??????? 3. ???? 4. ?????????? ? ??????????????????????????????????????????????????????????????????????????????????????? ? ?????????????????????????????????????????????????? ????????? A????? 1. ????????????????????????? 2. ???? 3. ?????8??? B???????? 1. ????????? 2. ?????? 3. ????????? 4. ??????????????? C??????? 1. ????????????? 2. ???? 3. ?????One of my days in SH? 4. ?????????????Love what U do? 5. ?????????????? 6. ??????????????????? ? (?????????????????????????????????????????????????? ? ????????????????????????????????????????????????????) ???????????? A??????? 1. ???????????????????? 2. ?????? B????? 1. ????? 2. ????? 3. ????? 4. ????? C????????? 1. ??????? 2. ??????? 3. ?????SMARTER?? 4. ??????? 5. ???????????MECE 6. MECE????? 7. MECE????? 8. ?????????????? 9. ??????????????? D????????? 1. Mindmanager?????????????? 2. Visio???????? 3. AdobeAcrobat??????????? E??????? 1. ?????SIPOC?? 2. ????????????? 3. ????????? 4. ?????? ? (??????????????????????????????????????????????????????????????????????????? ? ????????????????????MECE??????Mindmanager?Visio?Adobe Acrobat?????????????????????????????) ?????????????? A?????? 1. ??????????????????? 2. ???? 3. ?????????? B????? 1. ???????????????????????? 2. ??????? 3. ???????????????????? 4. ???????????????? 5. ??????????? 6. ??????????????? C????????? 1. ???????????? 2. ???????? 3. ??????? ? (?????????????????????????????????????? ? ????????????????????????????????????????????????????????????????????????????????? ? ????????????????????????????????????????????????????) ????: ????? ??????-???? ?Delphi???????? - ????? ?Liteon????? - VQE??????? ?Schneider???????? - SQE&SQE(?????)?? ?Schneider??????????? - ????????? ?Schneider??????????? - ???? ???????? - ????? ???? ????????????????????????????????500????????CEO????????????????????????????????????????????????????TRW??????(Honeywell)????????? ???? ? ???????????????????????????????????? ? ????????????????????????? ? ???????????????????????????? ???? ???????????????????????????????????????????????????????????????????????????????????????????????????????? ???? ???????????????????????????????&??????? ????????????????????????????&????&????????????? ????????? ? ??500?????????????????????????????????????? ? ?????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????2013??????????????????????????????????????????????&??????????????????????????????&???????????????????????????????????? ???????????5~6?? ?? ???? ????/?? ???? ??500? ??????? ??????&???? 2011.07 ???? ???? 2011.08?2012.03 ??? ??????&???? 2012.03?2012.12 ???? ???? 2012.11?2012.12 ???? ????????? 2013.10? ???? ????? ??????&???? 2008.08?2009.12 ???? ?????&????&???? 2011.09?2012.09 ???? ??????&????&???? 2010.06?2013.12 ???? ??????&????&?????? 2012.09? ??????? ???? 2012.02?2013.03 ???? ??????&????&???? 2012.07? ?????? ???? 2013.12? ???? ???? 2014.02? ???? ???? 2014.02? ????? ???? 2014.09? -------------- next part -------------- An HTML attachment was scrubbed... URL: From phk at FreeBSD.org Wed Sep 7 19:38:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 07 Sep 2016 21:38:11 +0200 Subject: [master] 0516feb We cannot trust settings-ack to arrive before any other frames Message-ID: commit 0516feba93201d07d87009d93e02e404c2c8e698 Author: Poul-Henning Kamp Date: Wed Sep 7 08:16:11 2016 +0000 We cannot trust settings-ack to arrive before any other frames diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 15e48f5..aa78975 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1775,7 +1775,7 @@ cmd_http_stream(CMD_ARGS) "txsettings -ack\n" "rxsettings\n" "expect settings.ack == true" - "} -run\n", http_cmds, hp, vl); + "} -start\n", http_cmds, hp, vl); } cmd_stream(av, hp, cmd, vl); } From phk at FreeBSD.org Wed Sep 7 19:38:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 07 Sep 2016 21:38:11 +0200 Subject: [master] c672889 Revert "We cannot trust settings-ack to arrive before any other frames" Message-ID: commit c67288945e87cbeff6c9f2e384bcc859514a6802 Author: Poul-Henning Kamp Date: Wed Sep 7 08:41:33 2016 +0000 Revert "We cannot trust settings-ack to arrive before any other frames" This reverts commit 7e3e34a4251536919c51648a7354d521c4e002b7. diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index aa78975..15e48f5 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1775,7 +1775,7 @@ cmd_http_stream(CMD_ARGS) "txsettings -ack\n" "rxsettings\n" "expect settings.ack == true" - "} -start\n", http_cmds, hp, vl); + "} -run\n", http_cmds, hp, vl); } cmd_stream(av, hp, cmd, vl); } From phk at FreeBSD.org Wed Sep 7 19:38:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 07 Sep 2016 21:38:11 +0200 Subject: [master] 16ed166 Try to stabilize this test with a barrier. Message-ID: commit 16ed166092c4c6cdf76ceb9109a2f3ffb8262c4a Author: Poul-Henning Kamp Date: Wed Sep 7 19:33:02 2016 +0000 Try to stabilize this test with a barrier. diff --git a/bin/varnishtest/tests/t02001.vtc b/bin/varnishtest/tests/t02001.vtc index 2fc8e54..af32f27 100644 --- a/bin/varnishtest/tests/t02001.vtc +++ b/bin/varnishtest/tests/t02001.vtc @@ -1,8 +1,11 @@ varnishtest "H1->H2 Upgrade" +barrier b1 cond 2 + server s1 { rxreq expect req.http.host == foo.bar + barrier b1 sync txresp -status 400 -bodylen 10 } -start @@ -21,6 +24,15 @@ client c1 { expect resp.status == 101 expect resp.http.upgrade == h2c expect resp.http.connection == Upgrade + txpri + stream 0 { + rxsettings + txsettings + txsettings -ack + rxsettings + expect settings.ack == true + } -run + barrier b1 sync stream 1 { rxresp expect resp.status == 400 From fgsch at lodoss.net Thu Sep 8 08:30:30 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 08 Sep 2016 10:30:30 +0200 Subject: [master] c936b7a Allow to use regex matches in expressions Message-ID: commit c936b7a446e94d4634c01199f61e1c00149b168a Author: Federico G. Schwindt Date: Thu Sep 8 01:16:37 2016 +0100 Allow to use regex matches in expressions E.g. set req.http.foo = req.http.bar ~ "foobar" Related to #2075. diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index e041356..b800411 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -115,6 +115,11 @@ varnish v1 -vcl { set req.http.foo = 3.6 * 1.4; set req.http.foo = 3.6 / 1.4; + set req.http.foo = req.http.foo ~ "bar"; + set req.http.foo = req.http.foo !~ "bar"; + set req.http.foo = "foo" ~ "bar"; + set req.http.foo = "foo" !~ "bar"; + set req.ttl = 1s; } } diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index d91c8bd..6259934 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1136,7 +1136,7 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, vcc_type_t fmt) *e = vcc_expr_edit(BOOL, cp->emit, *e, e2); return; } - if ((*e)->fmt == STRING && + if (((*e)->fmt == STRING || (*e)->fmt == STRING_LIST) && (tl->t->tok == '~' || tl->t->tok == T_NOMATCH)) { not = tl->t->tok == '~' ? "" : "!"; vcc_NextToken(tl); From phk at FreeBSD.org Thu Sep 8 08:32:16 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 08 Sep 2016 10:32:16 +0200 Subject: [master] 3028aa3 Add the beginnings of a HPACK coverage test Message-ID: commit 3028aa365bfa6d20e836cf1e423f1e5ddeddc6aa Author: Poul-Henning Kamp Date: Thu Sep 8 08:31:14 2016 +0000 Add the beginnings of a HPACK coverage test diff --git a/bin/varnishtest/tests/t02002.vtc b/bin/varnishtest/tests/t02002.vtc new file mode 100644 index 0000000..ebb0146 --- /dev/null +++ b/bin/varnishtest/tests/t02002.vtc @@ -0,0 +1,37 @@ +varnishtest "HPACK coverage" + +server s1 { + rxreq + expect req.http.host == www.example.com + expect req.http.foohdr == FOOcont + txresp -status 500 -bodylen 10 + rxreq + expect req.http.host == www.example.com + expect req.http.foohdr == FOOcont2 + txresp -status 404 -bodylen 20 +} -start + +varnish v1 -vcl+backend {} -start + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set debug +syncvsl" + +client c1 { + stream 7 { + txreq \ + -litIdxHdr inc 1 huf "www.example.com" \ + -idxHdr 16 \ + -litHdr inc huf "foohdr" huf "FOOcont" + rxresp + expect resp.status == 500 + expect resp.bodylen == 10 + } -run + stream 9 { + txreq \ + -idxHdr 63 \ + -litIdxHdr never 62 plain "FOOcont2" + rxresp + expect resp.status == 404 + expect resp.bodylen == 20 + } -run +} -run From guillaume at varnish-software.com Thu Sep 8 10:22:12 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 08 Sep 2016 12:22:12 +0200 Subject: [master] 7865dd7 Fix unchecked return Message-ID: commit 7865dd7e19d7800df1128dd9ca78a76cb8c322d6 Author: Guillaume Quintard Date: Thu Sep 8 11:27:28 2016 +0200 Fix unchecked return diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index d3c3f8b..9e10019 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -143,7 +143,8 @@ client_thread(void *priv) vtc_log(c->vl, 0, "Failed to open %s: %s", VSB_data(vsb), err); assert(fd >= 0); - VTCP_blocking(fd); + /* VTCP_blocking does its own checks, trust it */ + (void)VTCP_blocking(fd); VTCP_myname(fd, mabuf, sizeof mabuf, mpbuf, sizeof mpbuf); vtc_log(vl, 3, "connected fd %d from %s %s to %s", fd, mabuf, mpbuf, VSB_data(vsb)); From guillaume at varnish-software.com Thu Sep 8 10:22:12 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 08 Sep 2016 12:22:12 +0200 Subject: [master] d708fe0 Please flexelint, pl can be unsigned Message-ID: commit d708fe048c85351bec1ba42a71388b45e4896e85 Author: Guillaume Quintard Date: Thu Sep 8 11:47:35 2016 +0200 Please flexelint, pl can be unsigned diff --git a/bin/varnishtest/vtc_h2_hpack.c b/bin/varnishtest/vtc_h2_hpack.c index a378966..1239816 100644 --- a/bin/varnishtest/vtc_h2_hpack.c +++ b/bin/varnishtest/vtc_h2_hpack.c @@ -57,13 +57,12 @@ huff_decode(char *str, int nm, struct hpk_iter *iter, int ilen) { int l = 0; uint64_t pack = 0; - int pl = 0; /* pack length*/ + unsigned pl = 0; /* pack length*/ struct stbl *tbl = &byte0; struct ssym *sym; (void)nm; while (ilen > 0 || pl != 0) { - assert(pl >= 0); /* make sure we have enough data*/ if (pl < tbl->msk) { if (ilen == 0) { @@ -92,6 +91,7 @@ huff_decode(char *str, int nm, struct hpk_iter *iter, int ilen) return (0); pack <<= sym->csm; + assert(sym->csm <= pl); pl -= sym->csm; if (sym->nxt) { tbl = sym->nxt; From guillaume at varnish-software.com Thu Sep 8 10:22:12 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 08 Sep 2016 12:22:12 +0200 Subject: [master] 60eaab2 Fix bad enum to int conversion Message-ID: commit 60eaab28c7e21223417505669b9b848ba47894b5 Author: Guillaume Quintard Date: Thu Sep 8 11:57:46 2016 +0200 Fix bad enum to int conversion diff --git a/bin/varnishtest/vtc_h2_hpack.c b/bin/varnishtest/vtc_h2_hpack.c index 1239816..f6378f7 100644 --- a/bin/varnishtest/vtc_h2_hpack.c +++ b/bin/varnishtest/vtc_h2_hpack.c @@ -104,7 +104,7 @@ huff_decode(char *str, int nm, struct hpk_iter *iter, int ilen) } /* inspired from Dridi Boukelmoune's cashpack. */ -static int +static enum hpk_result huff_encode(struct hpk_iter *iter, const char *str, int len) { uint64_t pack = 0; @@ -123,7 +123,7 @@ huff_encode(struct hpk_iter *iter, const char *str, int len) while (pl >= 8) { if (iter->buf == iter->end) - return (1); + return (hpk_done); *iter->buf = (char)(pack >> 56); iter->buf++; pack <<= 8; @@ -143,7 +143,7 @@ huff_encode(struct hpk_iter *iter, const char *str, int len) iter->buf++; } - return (0); + return (hpk_more); } static int From guillaume at varnish-software.com Thu Sep 8 10:22:12 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 08 Sep 2016 12:22:12 +0200 Subject: [master] 3fc1e51 Remove useless semi-colon Message-ID: commit 3fc1e5123a7264a6ceac4b5e715e8c40b91c8d47 Author: Guillaume Quintard Date: Thu Sep 8 12:03:31 2016 +0200 Remove useless semi-colon diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 3fb0dca..e6ae7de 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -256,7 +256,7 @@ readFrameHeader(struct frame *f, const char *buf) f->stid += (0xff & (unsigned char)buf[6]) << 16; f->stid += (0xff & (unsigned char)buf[7]) << 8; f->stid += (0xff & (unsigned char)buf[8]); -}; +} static void writeFrameHeader(char *buf, const struct frame *f) From guillaume at varnish-software.com Thu Sep 8 10:22:12 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 08 Sep 2016 12:22:12 +0200 Subject: [master] 791d44d No need for initialization Message-ID: commit 791d44da6501976691cb78de84ecf37c286b7055 Author: Guillaume Quintard Date: Thu Sep 8 12:06:40 2016 +0200 No need for initialization diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index e6ae7de..62a1243 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -342,7 +342,7 @@ write_frame(struct http *hp, const struct frame *f, const unsigned lock) static void exclusive_stream_dependency(const struct stream *s) { - struct stream *target = NULL; + struct stream *target; struct http *hp = s->hp; if (s->id == 0) From guillaume at varnish-software.com Thu Sep 8 10:22:12 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 08 Sep 2016 12:22:12 +0200 Subject: [master] 4830ec8 Check HPK_ResizeTbl return Message-ID: commit 4830ec8abec12e0b1357080e650b55550d553d34 Author: Guillaume Quintard Date: Thu Sep 8 12:10:45 2016 +0200 Check HPK_ResizeTbl return diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 62a1243..4b1f549 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -588,6 +588,7 @@ parse_settings(const struct stream *s, struct frame *f) struct http *hp; int i, t, v; const char *buf; + enum hpk_result r; CHECK_OBJ_NOTNULL(f, FRAME_MAGIC); CHECK_OBJ_NOTNULL(s, STREAM_MAGIC); CAST_OBJ_NOTNULL(hp, s->hp, HTTP_MAGIC);; @@ -609,8 +610,10 @@ parse_settings(const struct stream *s, struct frame *f) buf = "unknown"; i += 4; - if (t == 1 ) - HPK_ResizeTbl(s->hp->encctx, v); + if (t == 1 ) { + r = HPK_ResizeTbl(s->hp->encctx, v); + assert(r == hpk_done); + } vtc_log(hp->vl, 4, "settings->%s (%d): %d", buf, t, v); } From guillaume at varnish-software.com Thu Sep 8 10:22:12 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 08 Sep 2016 12:22:12 +0200 Subject: [master] 475dbed Un-confuse flexelint Message-ID: commit 475dbed976d2ccb6bfaecc3f21f3702bcc147100 Author: Guillaume Quintard Date: Thu Sep 8 12:11:41 2016 +0200 Un-confuse flexelint diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 4b1f549..c7ca7ee 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -663,7 +663,7 @@ parse_goaway(const struct stream *s, struct frame *f) err_buf = "unknown"; if (f->size > 8) { - f->md.goaway.debug = malloc(f->size - 8 + 1L); + f->md.goaway.debug = malloc((f->size - 8) + 1L); AN(f->md.goaway.debug); f->md.goaway.debug[f->size - 8] = '\0'; From guillaume at varnish-software.com Thu Sep 8 10:22:12 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 08 Sep 2016 12:22:12 +0200 Subject: [master] e0810f2 Check get_bytes return Message-ID: commit e0810f250b9c4344eda16105b2c9a537ac440798 Author: Guillaume Quintard Date: Thu Sep 8 12:13:13 2016 +0200 Check get_bytes return diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index c7ca7ee..962eccd 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -739,7 +739,7 @@ receive_frame(void *priv) f->data = malloc(f->size + 1L); AN(f->data); f->data[f->size] = '\0'; - get_bytes(hp, f->data, f->size); + AN(get_bytes(hp, f->data, f->size)); } /* is the corresponding stream waiting? */ From guillaume at varnish-software.com Thu Sep 8 10:22:12 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 08 Sep 2016 12:22:12 +0200 Subject: [master] 4ff7beb Check VSB_finish return Message-ID: commit 4ff7bebc6dcd3f65e89b2da40e7b9fefbf0d5f22 Author: Guillaume Quintard Date: Thu Sep 8 12:14:15 2016 +0200 Check VSB_finish return diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 962eccd..0ffaf7f 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -789,7 +789,7 @@ receive_frame(void *priv) parse_hdr(s, f, vsb); if (f->flags & END_HEADERS) { expect_cont = 0; - VSB_finish(vsb); + AZ(VSB_finish(vsb)); decode_hdr(hp, hdrs, vsb); VSB_destroy(&vsb); hdrs = NULL; From guillaume at varnish-software.com Thu Sep 8 10:22:12 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 08 Sep 2016 12:22:12 +0200 Subject: [master] 62753c4 Remove useless semi-colon Message-ID: commit 62753c4c62a84dc896ad750e0f76e0f9bbeb8148 Author: Guillaume Quintard Date: Thu Sep 8 12:15:43 2016 +0200 Remove useless semi-colon diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 0ffaf7f..64d4271 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -860,7 +860,7 @@ do { \ } \ snprintf(buf, 20, "%.0f", f->md.settings[idx]); \ return (buf); \ -} while (0); +} while (0) #define RETURN_BUFFED(val) \ do { \ From dridi.boukelmoune at gmail.com Thu Sep 8 10:38:10 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 08 Sep 2016 12:38:10 +0200 Subject: [master] d231a15 Whitespace OCD Message-ID: commit d231a1587f52cbf109b142f4c070f3c19ef0c728 Author: Dridi Boukelmoune Date: Thu Sep 8 12:37:00 2016 +0200 Whitespace OCD diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 64d4271..b4cb96f 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -610,7 +610,7 @@ parse_settings(const struct stream *s, struct frame *f) buf = "unknown"; i += 4; - if (t == 1 ) { + if (t == 1) { r = HPK_ResizeTbl(s->hp->encctx, v); assert(r == hpk_done); } @@ -2730,7 +2730,7 @@ b64_settings(const struct http *hp, const char *s) } else buf = "unknown"; - if (v == 1 ) { + if (v == 1) { if (hp->sfd) HPK_ResizeTbl(hp->encctx, v); else From guillaume at varnish-software.com Thu Sep 8 11:05:12 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 08 Sep 2016 13:05:12 +0200 Subject: [master] 8f7ac0f Align varnishtest naming with vcl Message-ID: commit 8f7ac0fc4ca080db8ee38f5e0bdd1e1cff9032a2 Author: Guillaume Quintard Date: Thu Sep 8 13:01:19 2016 +0200 Align varnishtest naming with vcl change resp.msg to resp.reason Spotted by: Nils Goroll diff --git a/bin/varnishtest/tests/a00001.vtc b/bin/varnishtest/tests/a00001.vtc index d842287..fd2ea9c 100644 --- a/bin/varnishtest/tests/a00001.vtc +++ b/bin/varnishtest/tests/a00001.vtc @@ -15,7 +15,7 @@ client c1 -connect ${s1_sock} { rxresp expect resp.proto == HTTP/1.1 expect resp.status == 200 - expect resp.msg == OK + expect resp.reason == OK } client c1 -run diff --git a/bin/varnishtest/tests/a00002.vtc b/bin/varnishtest/tests/a00002.vtc index 39b8334..79c5517 100644 --- a/bin/varnishtest/tests/a00002.vtc +++ b/bin/varnishtest/tests/a00002.vtc @@ -15,7 +15,7 @@ client c1 -connect ${s1_sock} { rxresp expect resp.proto == HTTP/1.2 expect resp.status == 201 - expect resp.msg == Foo + expect resp.reason == Foo } client c1 -run diff --git a/bin/varnishtest/tests/a00003.vtc b/bin/varnishtest/tests/a00003.vtc index 810da48..b924c76 100644 --- a/bin/varnishtest/tests/a00003.vtc +++ b/bin/varnishtest/tests/a00003.vtc @@ -24,7 +24,7 @@ client c1 -connect ${s1_sock} { rxresp expect resp.proto == HTTP/1.2 expect resp.status == 201 - expect resp.msg == Foo + expect resp.reason == Foo } client c2 -connect ${s2_sock} { @@ -32,7 +32,7 @@ client c2 -connect ${s2_sock} { rxresp expect resp.proto == HTTP/1.1 expect resp.status == 200 - expect resp.msg == OK + expect resp.reason == OK } client c1 -start diff --git a/bin/varnishtest/tests/a00004.vtc b/bin/varnishtest/tests/a00004.vtc index 85659ac..ce3a3b5 100644 --- a/bin/varnishtest/tests/a00004.vtc +++ b/bin/varnishtest/tests/a00004.vtc @@ -15,7 +15,7 @@ client c1 -connect ${s1_sock} { rxresp expect resp.proto == HTTP/1.2 expect resp.status == 201 - expect resp.msg == Foo + expect resp.reason == Foo } client c2 -connect ${s1_sock} { @@ -23,7 +23,7 @@ client c2 -connect ${s1_sock} { rxresp expect resp.proto == HTTP/1.2 expect resp.status == 201 - expect resp.msg == Foo + expect resp.reason == Foo } client c1 -start diff --git a/bin/varnishtest/tests/a00005.vtc b/bin/varnishtest/tests/a00005.vtc index 6c0d634..3d8aa49 100644 --- a/bin/varnishtest/tests/a00005.vtc +++ b/bin/varnishtest/tests/a00005.vtc @@ -24,7 +24,7 @@ client c1 -connect ${s1_sock} { rxresp expect resp.proto == HTTP/1.2 expect resp.status == 201 - expect resp.msg == Foo + expect resp.reason == Foo } client c1 -run @@ -34,7 +34,7 @@ client c1 -connect ${s2_sock} { rxresp expect resp.proto == HTTP/1.1 expect resp.status == 200 - expect resp.msg == OK + expect resp.reason == OK } client c1 -run diff --git a/bin/varnishtest/tests/a00006.vtc b/bin/varnishtest/tests/a00006.vtc index f8a928e..160c883 100644 --- a/bin/varnishtest/tests/a00006.vtc +++ b/bin/varnishtest/tests/a00006.vtc @@ -17,7 +17,7 @@ client c1 -connect ${s1_sock} { rxresp expect resp.proto == HTTP/1.2 expect resp.status == 201 - expect resp.msg == Foo + expect resp.reason == Foo } client c1 -run diff --git a/bin/varnishtest/tests/b00030.vtc b/bin/varnishtest/tests/b00030.vtc index 1115613..a3fb9cc 100644 --- a/bin/varnishtest/tests/b00030.vtc +++ b/bin/varnishtest/tests/b00030.vtc @@ -20,5 +20,5 @@ varnish v1 -vcl+backend { client c1 { txreq rxresp - expect resp.msg == "-1.000" + expect resp.reason == "-1.000" } -run diff --git a/bin/varnishtest/tests/b00036.vtc b/bin/varnishtest/tests/b00036.vtc index 4cd6e17..c07fe86 100644 --- a/bin/varnishtest/tests/b00036.vtc +++ b/bin/varnishtest/tests/b00036.vtc @@ -26,7 +26,7 @@ client c1 { txreq -req PURGE rxresp - expect resp.msg == "Purged" + expect resp.reason == "Purged" } -run client c1 { diff --git a/bin/varnishtest/tests/c00066.vtc b/bin/varnishtest/tests/c00066.vtc index 14d4126..7d8f972 100644 --- a/bin/varnishtest/tests/c00066.vtc +++ b/bin/varnishtest/tests/c00066.vtc @@ -43,13 +43,13 @@ client c1 { txreq -url "/" rxresp expect resp.status == 200 - expect resp.msg == "OK" + expect resp.reason == "OK" delay .1 txreq -url "/synth" rxresp expect resp.status == 200 - expect resp.msg == "Synth test" + expect resp.reason == "Synth test" delay .1 txreq -url "/error" diff --git a/bin/varnishtest/tests/r00325.vtc b/bin/varnishtest/tests/r00325.vtc index b9c7a56..e361479 100644 --- a/bin/varnishtest/tests/r00325.vtc +++ b/bin/varnishtest/tests/r00325.vtc @@ -15,7 +15,7 @@ client c1 { txreq -url /bar rxresp expect resp.status == 200 - expect resp.msg == OK + expect resp.reason == OK } -run server s1 { @@ -31,5 +31,5 @@ client c1 { txreq -url /foo rxresp expect resp.status == 200 - expect resp.msg == OK + expect resp.reason == OK } -run diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc index cc5168c..c66dc84 100644 --- a/bin/varnishtest/tests/r00962.vtc +++ b/bin/varnishtest/tests/r00962.vtc @@ -51,7 +51,7 @@ varnish v2 \ client c1 -connect ${v2_sock} { txreq -url "/" rxresp - expect resp.msg != "Persistent Object Not Found" + expect resp.reason != "Persistent Object Not Found" expect resp.status == 200 expect resp.http.X-Varnish == "1001 1002" } -run diff --git a/bin/varnishtest/tests/r01485.vtc b/bin/varnishtest/tests/r01485.vtc index 14aba70..302f177 100644 --- a/bin/varnishtest/tests/r01485.vtc +++ b/bin/varnishtest/tests/r01485.vtc @@ -22,7 +22,7 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == 1 - expect resp.msg == "OK" + expect resp.reason == "OK" delay 0.1 @@ -30,5 +30,5 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == 1 - expect resp.msg == "OK" + expect resp.reason == "OK" } -run diff --git a/bin/varnishtest/tests/r01990.vtc b/bin/varnishtest/tests/r01990.vtc index a430d63..e9f5a19 100644 --- a/bin/varnishtest/tests/r01990.vtc +++ b/bin/varnishtest/tests/r01990.vtc @@ -38,7 +38,7 @@ client c1 { txreq -url "/" rxresp expect resp.status == 503 - expect resp.msg == "Unknown HTTP Status" + expect resp.reason == "Unknown HTTP Status" } -run logexpect l1 -wait diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 15e48f5..b2a021f 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -283,7 +283,7 @@ http_count_header(char * const *hh, const char *hdr) * - req.proto * - resp.proto * - resp.status - * - resp.msg + * - resp.reason * - resp.chunklen * - req.bodylen * - req.body @@ -311,7 +311,7 @@ cmd_var_resolve(struct http *hp, char *spec) return(hp->resp[0]); if (!strcmp(spec, "resp.status")) return(hp->resp[1]); - if (!strcmp(spec, "resp.msg")) + if (!strcmp(spec, "resp.reason")) return(hp->resp[2]); if (!strcmp(spec, "resp.chunklen")) return(hp->chunklen); From phk at FreeBSD.org Thu Sep 8 11:21:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 08 Sep 2016 13:21:11 +0200 Subject: [master] 169e162 Enforce that VCL names must be C-language identifiers ([A-Za-z][A-Za-z0-9_]*) Message-ID: commit 169e162c7506614090a33faef7a0df38c6ffac7a Author: Poul-Henning Kamp Date: Thu Sep 8 11:19:39 2016 +0000 Enforce that VCL names must be C-language identifiers ([A-Za-z][A-Za-z0-9_]*) diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 3fb2132..56c89b7 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -42,6 +42,7 @@ #include "libvcc.h" #include "vcli_serve.h" +#include "vct.h" #include "vev.h" #include "vtim.h" @@ -81,6 +82,74 @@ static struct vev *e_poker; /*--------------------------------------------------------------------*/ +static struct vclprog * +mcf_vcl_byname(const char *name) +{ + struct vclprog *vp; + + VTAILQ_FOREACH(vp, &vclhead, list) + if (!strcmp(name, vp->name)) + return (vp); + return (NULL); +} + +static int +mcf_bad_vclname(struct cli *cli, const char *name) +{ + const char *p; + int bad = 0; + + AN(name); + p = name; + if (!vct_isalpha(*p)) + bad = *p; + for (p++; bad == 0 && *p != '\0'; p++) + if (!vct_isalpha(*p) && !vct_isdigit(*p) && *p != '_') + bad = *p; + if (bad) { + VCLI_SetResult(cli, CLIS_PARAM); + VCLI_Out(cli, "Illegal character in VCL name "); + if (bad > 0x20 && bad < 0x7f) + VCLI_Out(cli, "('%c')", bad); + else + VCLI_Out(cli, "(0x%02x)", bad & 0xff); + } + return (bad); +} + +static struct vclprog * +mcf_find_vcl(struct cli *cli, const char *name) +{ + struct vclprog *vp; + + if (mcf_bad_vclname(cli, name)) + return (NULL); + + vp = mcf_vcl_byname(name); + if (vp == NULL) { + VCLI_SetResult(cli, CLIS_PARAM); + VCLI_Out(cli, "No VCL named %s known.", name); + } + return (vp); +} + +static int +mcf_find_no_vcl(struct cli *cli, const char *name) +{ + + if (mcf_bad_vclname(cli, name)) + return (0); + + if (mcf_vcl_byname(name) != NULL) { + VCLI_SetResult(cli, CLIS_PARAM); + VCLI_Out(cli, "Already a VCL named %s", name); + return (0); + } + return (1); +} + +/*--------------------------------------------------------------------*/ + static void mgt_vcl_dep_add(struct vclprog *vp_from, struct vclprog *vp_to) { @@ -163,17 +232,6 @@ mgt_vcl_del(struct vclprog *vp) FREE_OBJ(vp); } -static struct vclprog * -mgt_vcl_byname(const char *name) -{ - struct vclprog *vp; - - VTAILQ_FOREACH(vp, &vclhead, list) - if (!strcmp(name, vp->name)) - return (vp); - return (NULL); -} - void mgt_vcl_depends(struct vclprog *vp1, const char *name) { @@ -181,7 +239,7 @@ mgt_vcl_depends(struct vclprog *vp1, const char *name) CHECK_OBJ_NOTNULL(vp1, VCLPROG_MAGIC); - vp2 = mgt_vcl_byname(name); + vp2 = mcf_vcl_byname(name); CHECK_OBJ_NOTNULL(vp2, VCLPROG_MAGIC); mgt_vcl_dep_add(vp1, vp2); } @@ -390,16 +448,11 @@ mgt_push_vcls_and_start(struct cli *cli, unsigned *status, char **p) static void __match_proto__(cli_func_t) mcf_vcl_inline(struct cli *cli, const char * const *av, void *priv) { - struct vclprog *vp; (void)priv; - vp = mgt_vcl_byname(av[2]); - if (vp != NULL) { - VCLI_Out(cli, "Already a VCL program named %s", av[2]); - VCLI_SetResult(cli, CLIS_PARAM); + if (!mcf_find_no_vcl(cli, av[2])) return; - } mgt_new_vcl(cli, av[2], av[3], "", av[4], 0); } @@ -407,31 +460,14 @@ mcf_vcl_inline(struct cli *cli, const char * const *av, void *priv) static void __match_proto__(cli_func_t) mcf_vcl_load(struct cli *cli, const char * const *av, void *priv) { - struct vclprog *vp; (void)priv; - vp = mgt_vcl_byname(av[2]); - if (vp != NULL) { - VCLI_Out(cli, "Already a VCL program named %s", av[2]); - VCLI_SetResult(cli, CLIS_PARAM); + if (!mcf_find_no_vcl(cli, av[2])) return; - } mgt_new_vcl(cli, av[2], NULL, av[3], av[4], 0); } -static struct vclprog * -mcf_find_vcl(struct cli *cli, const char *name) -{ - struct vclprog *vp; - - vp = mgt_vcl_byname(name); - if (vp == NULL) { - VCLI_SetResult(cli, CLIS_PARAM); - VCLI_Out(cli, "No configuration named %s known.", name); - } - return (vp); -} static void __match_proto__(cli_func_t) mcf_vcl_state(struct cli *cli, const char * const *av, void *priv) @@ -614,6 +650,10 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) int i; (void)priv; + if (mcf_bad_vclname(cli, av[2])) + return; + if (mcf_bad_vclname(cli, av[3])) + return; vpt = mcf_find_vcl(cli, av[3]); if (vpt == NULL) return; @@ -628,7 +668,7 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) vpt->label->name); return; } - vpl = mgt_vcl_byname(av[2]); + vpl = mcf_vcl_byname(av[2]); if (vpl != NULL) { if (strcmp(vpl->state, VCL_STATE_LABEL)) { VCLI_SetResult(cli, CLIS_PARAM); @@ -643,12 +683,6 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) mgt_vcl_dep_del(VTAILQ_FIRST(&vpl->dfrom)); AN(VTAILQ_EMPTY(&vpl->dfrom)); } else { - /* XXX should check for C-syntax */ - if (strchr(av[2], '.')) { - VCLI_SetResult(cli, CLIS_PARAM); - VCLI_Out(cli, "VCL labels cannot contain '.'"); - return; - } vpl = mgt_vcl_add(av[2], VCL_STATE_LABEL); } AN(vpl); diff --git a/bin/varnishtest/tests/v00048.vtc b/bin/varnishtest/tests/v00048.vtc index bef46c1..35ba584 100644 --- a/bin/varnishtest/tests/v00048.vtc +++ b/bin/varnishtest/tests/v00048.vtc @@ -7,6 +7,11 @@ server s1 { varnish v1 -vcl+backend {} +# VCL name must be C-names +varnish v1 -clierr 106 {vcl.inline 0000 "vcl 4.0; backend b { .host = \"localhost\";} "} +varnish v1 -clierr 106 {vcl.inline a00/ "vcl 4.0; backend b { .host = \"localhost\";} "} +varnish v1 -clierr 106 {vcl.inline a00? "vcl 4.0; backend b { .host = \"localhost\";} "} + varnish v1 -vcl+backend { sub vcl_recv { return (synth(400)); From phk at FreeBSD.org Thu Sep 8 11:35:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 08 Sep 2016 13:35:11 +0200 Subject: [master] 2ec362a Make a convenience function to test if a VCL is a label Message-ID: commit 2ec362a8dfa696c3ca258dd161dee9a4c4cc3337 Author: Poul-Henning Kamp Date: Thu Sep 8 11:33:49 2016 +0000 Make a convenience function to test if a VCL is a label diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 56c89b7..f69ea1d 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -148,6 +148,12 @@ mcf_find_no_vcl(struct cli *cli, const char *name) return (1); } +static int +mcf_is_label(struct vclprog *vp) +{ + return (!strcmp(vp->state, VCL_STATE_LABEL)); +} + /*--------------------------------------------------------------------*/ static void @@ -383,7 +389,7 @@ mgt_vcl_export_labels(struct vcc *vcc) { struct vclprog *vp; VTAILQ_FOREACH(vp, &vclhead, list) { - if (!strcmp(vp->state, VCL_STATE_LABEL)) + if (mcf_is_label(vp)) VCC_Predef(vcc, "VCL_VCL", vp->name); } } @@ -403,7 +409,7 @@ mgt_push_vcls_and_start(struct cli *cli, unsigned *status, char **p) VTAILQ_FOREACH(vp, &vclhead, list) { if (!VTAILQ_EMPTY(&vp->dfrom)) continue; - if (!strcmp(vp->state, VCL_STATE_LABEL)) + if (mcf_is_label(vp)) continue; if (mgt_cli_askchild(status, p, "vcl.load \"%s\" %s %d%s\n", vp->name, vp->fname, vp->warm, vp->state)) @@ -412,7 +418,7 @@ mgt_push_vcls_and_start(struct cli *cli, unsigned *status, char **p) *p = NULL; } VTAILQ_FOREACH(vp, &vclhead, list) { - if (strcmp(vp->state, VCL_STATE_LABEL)) + if (!mcf_is_label(vp)) continue; if (mgt_cli_askchild(status, p, "vcl.label %s %s\n", vp->name, vp->label->name)) @@ -423,7 +429,7 @@ mgt_push_vcls_and_start(struct cli *cli, unsigned *status, char **p) VTAILQ_FOREACH(vp, &vclhead, list) { if (VTAILQ_EMPTY(&vp->dfrom)) continue; - if (!strcmp(vp->state, VCL_STATE_LABEL)) + if (mcf_is_label(vp)) continue; if (mgt_cli_askchild(status, p, "vcl.load \"%s\" %s %d%s\n", vp->name, vp->fname, vp->warm, vp->state)) @@ -479,7 +485,7 @@ mcf_vcl_state(struct cli *cli, const char * const *av, void *priv) if (vp == NULL) return; - if (!strcmp(vp->state, VCL_STATE_LABEL)) { + if (mcf_is_label(vp)) { VCLI_Out(cli, "Labels are always warm"); VCLI_SetResult(cli, CLIS_PARAM); return; @@ -571,7 +577,7 @@ mcf_vcl_discard(struct cli *cli, const char * const *av, void *priv) return; } if (!VTAILQ_EMPTY(&vp->dto)) { - if (vp->label != NULL && strcmp(vp->state, VCL_STATE_LABEL)) { + if (vp->label != NULL && !mcf_is_label(vp)) { AN(vp->warm); VCLI_SetResult(cli, CLIS_PARAM); VCLI_Out(cli, @@ -593,7 +599,7 @@ mcf_vcl_discard(struct cli *cli, const char * const *av, void *priv) } return; } - if (!strcmp(vp->state, VCL_STATE_LABEL)) { + if (mcf_is_label(vp)) { AN(vp->warm); vp->label->label = NULL; vp->label = NULL; @@ -633,8 +639,8 @@ mcf_vcl_list(struct cli *cli, const char * const *av, void *priv) VCLI_Out(cli, " %6s %s", "", vp->name); if (vp->label != NULL) VCLI_Out(cli, " %s %s", - strcmp(vp->state, VCL_STATE_LABEL) ? - "<-" : "->", vp->label->name); + mcf_is_label(vp) ? + "->" : "<-", vp->label->name); VCLI_Out(cli, "\n"); } } @@ -657,7 +663,7 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) vpt = mcf_find_vcl(cli, av[3]); if (vpt == NULL) return; - if (!strcmp(vpt->state, VCL_STATE_LABEL)) { + if (mcf_is_label(vpt)) { VCLI_SetResult(cli, CLIS_PARAM); VCLI_Out(cli, "VCL labels cannot point to labels"); return; @@ -670,7 +676,7 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) } vpl = mcf_vcl_byname(av[2]); if (vpl != NULL) { - if (strcmp(vpl->state, VCL_STATE_LABEL)) { + if (!mcf_is_label(vpl)) { VCLI_SetResult(cli, CLIS_PARAM); VCLI_Out(cli, "%s is not a label", vpl->name); return; From voicemail at varnish-cache.org Thu Sep 8 11:51:30 2016 From: voicemail at varnish-cache.org (voicemail at varnish-cache.org) Date: Thu, 08 Sep 2016 11:51:30 -0000 Subject: [Vigor2820 Series] New voice mail message from 01456672968 on 2016/09/08 17:21:30 Message-ID: Dear varnish-commit : There is a message for you from 01456672968, on 2016/09/08 17:21:30 . You might want to check it when you get a chance.Thanks! -------------- next part -------------- A non-text attachment was scrubbed... Name: Message_from_01456672968.wav.zip Type: application/zip Size: 8435 bytes Desc: Voicemail sound attachment. URL: From fgsch at lodoss.net Thu Sep 8 12:11:23 2016 From: fgsch at lodoss.net (Federico Schwindt) Date: Thu, 8 Sep 2016 13:11:23 +0100 Subject: [master] 169e162 Enforce that VCL names must be C-language identifiers ([A-Za-z][A-Za-z0-9_]*) In-Reply-To: References: Message-ID: Isn't dash (-) usage quite widespread? I'd expect it to be the most common character outside digits and numbers. On 8 Sep 2016 12:21 pm, "Poul-Henning Kamp" wrote: > > commit 169e162c7506614090a33faef7a0df38c6ffac7a > Author: Poul-Henning Kamp > Date: Thu Sep 8 11:19:39 2016 +0000 > > Enforce that VCL names must be C-language identifiers > ([A-Za-z][A-Za-z0-9_]*) > > diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c > index 3fb2132..56c89b7 100644 > --- a/bin/varnishd/mgt/mgt_vcl.c > +++ b/bin/varnishd/mgt/mgt_vcl.c > @@ -42,6 +42,7 @@ > > #include "libvcc.h" > #include "vcli_serve.h" > +#include "vct.h" > #include "vev.h" > #include "vtim.h" > > @@ -81,6 +82,74 @@ static struct vev *e_poker; > > /*--------------------------------------------------------------------*/ > > +static struct vclprog * > +mcf_vcl_byname(const char *name) > +{ > + struct vclprog *vp; > + > + VTAILQ_FOREACH(vp, &vclhead, list) > + if (!strcmp(name, vp->name)) > + return (vp); > + return (NULL); > +} > + > +static int > +mcf_bad_vclname(struct cli *cli, const char *name) > +{ > + const char *p; > + int bad = 0; > + > + AN(name); > + p = name; > + if (!vct_isalpha(*p)) > + bad = *p; > + for (p++; bad == 0 && *p != '\0'; p++) > + if (!vct_isalpha(*p) && !vct_isdigit(*p) && *p != '_') > + bad = *p; > + if (bad) { > + VCLI_SetResult(cli, CLIS_PARAM); > + VCLI_Out(cli, "Illegal character in VCL name "); > + if (bad > 0x20 && bad < 0x7f) > + VCLI_Out(cli, "('%c')", bad); > + else > + VCLI_Out(cli, "(0x%02x)", bad & 0xff); > + } > + return (bad); > +} > + > +static struct vclprog * > +mcf_find_vcl(struct cli *cli, const char *name) > +{ > + struct vclprog *vp; > + > + if (mcf_bad_vclname(cli, name)) > + return (NULL); > + > + vp = mcf_vcl_byname(name); > + if (vp == NULL) { > + VCLI_SetResult(cli, CLIS_PARAM); > + VCLI_Out(cli, "No VCL named %s known.", name); > + } > + return (vp); > +} > + > +static int > +mcf_find_no_vcl(struct cli *cli, const char *name) > +{ > + > + if (mcf_bad_vclname(cli, name)) > + return (0); > + > + if (mcf_vcl_byname(name) != NULL) { > + VCLI_SetResult(cli, CLIS_PARAM); > + VCLI_Out(cli, "Already a VCL named %s", name); > + return (0); > + } > + return (1); > +} > + > +/*--------------------------------------------------------------------*/ > + > static void > mgt_vcl_dep_add(struct vclprog *vp_from, struct vclprog *vp_to) > { > @@ -163,17 +232,6 @@ mgt_vcl_del(struct vclprog *vp) > FREE_OBJ(vp); > } > > -static struct vclprog * > -mgt_vcl_byname(const char *name) > -{ > - struct vclprog *vp; > - > - VTAILQ_FOREACH(vp, &vclhead, list) > - if (!strcmp(name, vp->name)) > - return (vp); > - return (NULL); > -} > - > void > mgt_vcl_depends(struct vclprog *vp1, const char *name) > { > @@ -181,7 +239,7 @@ mgt_vcl_depends(struct vclprog *vp1, const char *name) > > CHECK_OBJ_NOTNULL(vp1, VCLPROG_MAGIC); > > - vp2 = mgt_vcl_byname(name); > + vp2 = mcf_vcl_byname(name); > CHECK_OBJ_NOTNULL(vp2, VCLPROG_MAGIC); > mgt_vcl_dep_add(vp1, vp2); > } > @@ -390,16 +448,11 @@ mgt_push_vcls_and_start(struct cli *cli, unsigned > *status, char **p) > static void __match_proto__(cli_func_t) > mcf_vcl_inline(struct cli *cli, const char * const *av, void *priv) > { > - struct vclprog *vp; > > (void)priv; > > - vp = mgt_vcl_byname(av[2]); > - if (vp != NULL) { > - VCLI_Out(cli, "Already a VCL program named %s", av[2]); > - VCLI_SetResult(cli, CLIS_PARAM); > + if (!mcf_find_no_vcl(cli, av[2])) > return; > - } > > mgt_new_vcl(cli, av[2], av[3], "", av[4], 0); > } > @@ -407,31 +460,14 @@ mcf_vcl_inline(struct cli *cli, const char * const > *av, void *priv) > static void __match_proto__(cli_func_t) > mcf_vcl_load(struct cli *cli, const char * const *av, void *priv) > { > - struct vclprog *vp; > > (void)priv; > - vp = mgt_vcl_byname(av[2]); > - if (vp != NULL) { > - VCLI_Out(cli, "Already a VCL program named %s", av[2]); > - VCLI_SetResult(cli, CLIS_PARAM); > + if (!mcf_find_no_vcl(cli, av[2])) > return; > - } > > mgt_new_vcl(cli, av[2], NULL, av[3], av[4], 0); > } > > -static struct vclprog * > -mcf_find_vcl(struct cli *cli, const char *name) > -{ > - struct vclprog *vp; > - > - vp = mgt_vcl_byname(name); > - if (vp == NULL) { > - VCLI_SetResult(cli, CLIS_PARAM); > - VCLI_Out(cli, "No configuration named %s known.", name); > - } > - return (vp); > -} > > static void __match_proto__(cli_func_t) > mcf_vcl_state(struct cli *cli, const char * const *av, void *priv) > @@ -614,6 +650,10 @@ mcf_vcl_label(struct cli *cli, const char * const > *av, void *priv) > int i; > > (void)priv; > + if (mcf_bad_vclname(cli, av[2])) > + return; > + if (mcf_bad_vclname(cli, av[3])) > + return; > vpt = mcf_find_vcl(cli, av[3]); > if (vpt == NULL) > return; > @@ -628,7 +668,7 @@ mcf_vcl_label(struct cli *cli, const char * const *av, > void *priv) > vpt->label->name); > return; > } > - vpl = mgt_vcl_byname(av[2]); > + vpl = mcf_vcl_byname(av[2]); > if (vpl != NULL) { > if (strcmp(vpl->state, VCL_STATE_LABEL)) { > VCLI_SetResult(cli, CLIS_PARAM); > @@ -643,12 +683,6 @@ mcf_vcl_label(struct cli *cli, const char * const > *av, void *priv) > mgt_vcl_dep_del(VTAILQ_FIRST(&vpl->dfrom)); > AN(VTAILQ_EMPTY(&vpl->dfrom)); > } else { > - /* XXX should check for C-syntax */ > - if (strchr(av[2], '.')) { > - VCLI_SetResult(cli, CLIS_PARAM); > - VCLI_Out(cli, "VCL labels cannot contain '.'"); > - return; > - } > vpl = mgt_vcl_add(av[2], VCL_STATE_LABEL); > } > AN(vpl); > diff --git a/bin/varnishtest/tests/v00048.vtc b/bin/varnishtest/tests/ > v00048.vtc > index bef46c1..35ba584 100644 > --- a/bin/varnishtest/tests/v00048.vtc > +++ b/bin/varnishtest/tests/v00048.vtc > @@ -7,6 +7,11 @@ server s1 { > > varnish v1 -vcl+backend {} > > +# VCL name must be C-names > +varnish v1 -clierr 106 {vcl.inline 0000 "vcl 4.0; backend b { .host = > \"localhost\";} "} > +varnish v1 -clierr 106 {vcl.inline a00/ "vcl 4.0; backend b { .host = > \"localhost\";} "} > +varnish v1 -clierr 106 {vcl.inline a00? "vcl 4.0; backend b { .host = > \"localhost\";} "} > + > varnish v1 -vcl+backend { > sub vcl_recv { > return (synth(400)); > > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit -------------- next part -------------- An HTML attachment was scrubbed... URL: From guillaume at varnish-software.com Thu Sep 8 12:14:12 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 08 Sep 2016 14:14:12 +0200 Subject: [master] 5d0852c Use a switch-case for the fini sequence Message-ID: commit 5d0852c180c7c278b4429b9925368656c7b282b1 Author: Guillaume Quintard Date: Mon Aug 29 17:40:48 2016 +0200 Use a switch-case for the fini sequence This allows to test vgc_inistep only once and make it easier to work for the compiler. diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 7c155fb..86e69e5 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -321,16 +321,18 @@ EmitInitFini(const struct vcc *tl) Fc(tl, 0, "\nstatic int\nVGC_Discard(VRT_CTX)\n{\n\n"); Fc(tl, 0, "\t(void)VGC_function_vcl_fini(ctx);\n\n"); + + Fc(tl, 0, "\tswitch (vgc_inistep) {\n\n"); VTAILQ_FOREACH_REVERSE(p, &tl->inifin, inifinhead, list) { AZ(VSB_finish(p->fin)); if (VSB_len(p->fin)) { - Fc(tl, 0, "\t/* %u */\n", p->n); - Fc(tl, 0, "\tif (vgc_inistep >= %u) {\n", p->n); - Fc(tl, 0, "%s\n", VSB_data(p->fin)); - Fc(tl, 0, "\t}\n\n"); + Fc(tl, 0, "\t\tcase %u :\n", p->n); + Fc(tl, 0, "\t%s\n", VSB_data(p->fin)); + Fc(tl, 0, "\t\t\t/* FALLTHROUGH */\n"); } VSB_destroy(&p->fin); } + Fc(tl, 0, "\t}\n\n"); Fc(tl, 0, "\treturn (0);\n"); Fc(tl, 0, "}\n"); From guillaume at varnish-software.com Thu Sep 8 12:14:12 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 08 Sep 2016 14:14:12 +0200 Subject: [master] f70bcbc Make sure the step numbes are decreasing Message-ID: commit f70bcbc490121e21b227330ebb2a19cd477bb10c Author: Guillaume Quintard Date: Thu Sep 8 12:57:00 2016 +0200 Make sure the step numbes are decreasing Thanks @nigoroll for the suggestion. diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 86e69e5..3ebc651 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -286,7 +286,7 @@ EmitCoordinates(const struct vcc *tl, struct vsb *vsb) static void EmitInitFini(const struct vcc *tl) { - struct inifin *p; + struct inifin *p, *q = NULL; unsigned has_event = 0; Fh(tl, 0, "\n"); @@ -325,6 +325,8 @@ EmitInitFini(const struct vcc *tl) Fc(tl, 0, "\tswitch (vgc_inistep) {\n\n"); VTAILQ_FOREACH_REVERSE(p, &tl->inifin, inifinhead, list) { AZ(VSB_finish(p->fin)); + if (q) + assert(q->n > p->n); if (VSB_len(p->fin)) { Fc(tl, 0, "\t\tcase %u :\n", p->n); Fc(tl, 0, "\t%s\n", VSB_data(p->fin)); From phk at phk.freebsd.dk Thu Sep 8 12:28:23 2016 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Thu, 08 Sep 2016 12:28:23 +0000 Subject: [master] 169e162 Enforce that VCL names must be C-language identifiers ([A-Za-z][A-Za-z0-9_]*) In-Reply-To: References: Message-ID: <15940.1473337703@critter.freebsd.dk> -------- In message , Federico Schwindt writes: >Isn't dash (-) usage quite widespread? > >I'd expect it to be the most common character outside digits and numbers. The problem is that VCL label names need to be C-idents, and I'm not terribly keen on having different namespace for VCL labels and "real" VCLs. The alternative would be to encode a freer form into C-idents, but that turns really ugly really fast... -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi at varni.sh Thu Sep 8 12:35:33 2016 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 8 Sep 2016 14:35:33 +0200 Subject: [master] 169e162 Enforce that VCL names must be C-language identifiers ([A-Za-z][A-Za-z0-9_]*) In-Reply-To: <15940.1473337703@critter.freebsd.dk> References: <15940.1473337703@critter.freebsd.dk> Message-ID: On Thu, Sep 8, 2016 at 2:28 PM, Poul-Henning Kamp wrote: > -------- > In message > , Federico Schwindt writes: > >>Isn't dash (-) usage quite widespread? >> >>I'd expect it to be the most common character outside digits and numbers. > > The problem is that VCL label names need to be C-idents, and I'm > not terribly keen on having different namespace for VCL labels and > "real" VCLs. > > The alternative would be to encode a freer form into C-idents, but that > turns really ugly really fast... I can fix packaging scripts once we reach consensus, but this probably deserve a proper entry in upgrading.rst since this change may break things downstream (like the varnish agent for instance). Dridi From phk at phk.freebsd.dk Thu Sep 8 12:38:59 2016 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Thu, 08 Sep 2016 12:38:59 +0000 Subject: [master] 169e162 Enforce that VCL names must be C-language identifiers ([A-Za-z][A-Za-z0-9_]*) In-Reply-To: References: <15940.1473337703@critter.freebsd.dk> Message-ID: <17299.1473338339@critter.freebsd.dk> -------- In message , Dridi Boukelmoune writes: >> The alternative would be to encode a freer form into C-idents, but that >> turns really ugly really fast... > >I can fix packaging scripts once we reach consensus, but this probably >deserve a proper entry in upgrading.rst since this change may break >things downstream (like the varnish agent for instance). If '-' is in widespread use, we should not break it. I asked on IRC earlier, but nobody reacted... -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi at varni.sh Thu Sep 8 12:46:52 2016 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 8 Sep 2016 14:46:52 +0200 Subject: [master] 169e162 Enforce that VCL names must be C-language identifiers ([A-Za-z][A-Za-z0-9_]*) In-Reply-To: <17299.1473338339@critter.freebsd.dk> References: <15940.1473337703@critter.freebsd.dk> <17299.1473338339@critter.freebsd.dk> Message-ID: > If '-' is in widespread use, we should not break it. > > I asked on IRC earlier, but nobody reacted... I don't really care, I merely pointed out that it's a breaking change and we should not forget packaging before the release. Downstream or in-house tools like the varnish agent may catch up later, but people should be aware when reading the upgrade notes that VCL names may break. After all, this is 5.0 :) I'm also for having C-idents to match VCL labels naming rules, and migrating from '-' to '_' should be enough for users (for this character at least). Cheers From guillaume at varnish-software.com Thu Sep 8 12:59:12 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 08 Sep 2016 14:59:12 +0200 Subject: [master] 205ce3f Assign q, otherwise the whole check is useless Message-ID: commit 205ce3f5d90fd809d4c9073fa0e1c72704dd9bee Author: Guillaume Quintard Date: Thu Sep 8 14:57:22 2016 +0200 Assign q, otherwise the whole check is useless Spotted by: Dridi Boukelmoune diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 3ebc651..461ac56 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -327,6 +327,7 @@ EmitInitFini(const struct vcc *tl) AZ(VSB_finish(p->fin)); if (q) assert(q->n > p->n); + q = p; if (VSB_len(p->fin)) { Fc(tl, 0, "\t\tcase %u :\n", p->n); Fc(tl, 0, "\t%s\n", VSB_data(p->fin)); From phk at FreeBSD.org Thu Sep 8 13:07:12 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 08 Sep 2016 15:07:12 +0200 Subject: [master] b38feaa Track labels using the VCL dependencies (in MGR) Message-ID: commit b38feaa8199da3b4ab60f4e187483621c6a4eab6 Author: Poul-Henning Kamp Date: Thu Sep 8 13:05:04 2016 +0000 Track labels using the VCL dependencies (in MGR) Also try to clarify CLIS_PARAM (= nonsensical parameter) from CLIS_CANT (= params make sense, but you can't do that.) diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index f69ea1d..2a4f938 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -71,7 +71,6 @@ struct vclprog { unsigned warm; const char * state; double go_cold; - struct vclprog *label; VTAILQ_HEAD(, vcldep) dfrom; VTAILQ_HEAD(, vcldep) dto; }; @@ -149,7 +148,7 @@ mcf_find_no_vcl(struct cli *cli, const char *name) } static int -mcf_is_label(struct vclprog *vp) +mcf_is_label(const struct vclprog *vp) { return (!strcmp(vp->state, VCL_STATE_LABEL)); } @@ -265,7 +264,7 @@ mgt_vcl_setstate(struct cli *cli, struct vclprog *vp, const char *vs) char *p; int i; - if (vp == active_vcl || vp->label != NULL) { + if (vp == active_vcl || mcf_is_label(vp)) { AN(vp->warm); return (0); } @@ -400,6 +399,7 @@ int mgt_push_vcls_and_start(struct cli *cli, unsigned *status, char **p) { struct vclprog *vp; + struct vcldep *vd; AN(active_vcl); @@ -420,8 +420,10 @@ mgt_push_vcls_and_start(struct cli *cli, unsigned *status, char **p) VTAILQ_FOREACH(vp, &vclhead, list) { if (!mcf_is_label(vp)) continue; + vd = VTAILQ_FIRST(&vp->dfrom); + AN(vd); if (mgt_cli_askchild(status, p, "vcl.label %s %s\n", - vp->name, vp->label->name)) + vp->name, vd->to->name)) return (1); free(*p); *p = NULL; @@ -490,8 +492,8 @@ mcf_vcl_state(struct cli *cli, const char * const *av, void *priv) VCLI_SetResult(cli, CLIS_PARAM); return; } - if (vp->label != NULL) { - AZ(!strcmp(vp->state, "cold")); + if (!VTAILQ_EMPTY(&vp->dto)) { + AN(strcmp(vp->state, "cold")); if (!strcmp(av[3], "cold")) { VCLI_Out(cli, "A labeled VCL cannot be set cold"); VCLI_SetResult(cli, CLIS_CANT); @@ -511,7 +513,7 @@ mcf_vcl_state(struct cli *cli, const char * const *av, void *priv) } else if (!strcmp(av[3], VCL_STATE_COLD)) { if (vp == active_vcl) { VCLI_Out(cli, "Cannot set the active VCL cold."); - VCLI_SetResult(cli, CLIS_PARAM); + VCLI_SetResult(cli, CLIS_CANT); return; } vp->state = VCL_STATE_AUTO; @@ -572,23 +574,19 @@ mcf_vcl_discard(struct cli *cli, const char * const *av, void *priv) if (vp == NULL) return; if (vp == active_vcl) { - VCLI_SetResult(cli, CLIS_PARAM); + VCLI_SetResult(cli, CLIS_CANT); VCLI_Out(cli, "Cannot discard active VCL program\n"); return; } if (!VTAILQ_EMPTY(&vp->dto)) { - if (vp->label != NULL && !mcf_is_label(vp)) { - AN(vp->warm); - VCLI_SetResult(cli, CLIS_PARAM); + VCLI_SetResult(cli, CLIS_CANT); + AN(vp->warm); + if (!mcf_is_label(vp)) + VCLI_Out(cli, "Cannot discard labeled VCL program.\n"); + else VCLI_Out(cli, - "Cannot discard labeled (\"%s\") VCL program.\n", - vp->label->name); - return; - } - VCLI_SetResult(cli, CLIS_PARAM); - VCLI_Out(cli, - "Cannot discard \"%s\" VCL label, " - "other VCLs depend on it.\n", vp->name); + "Cannot discard this VCL label, " + "other VCLs depend on it.\n"); n = 0; VTAILQ_FOREACH(vd, &vp->dto, lto) { if (n++ == 5) { @@ -599,13 +597,10 @@ mcf_vcl_discard(struct cli *cli, const char * const *av, void *priv) } return; } - if (mcf_is_label(vp)) { + if (mcf_is_label(vp)) AN(vp->warm); - vp->label->label = NULL; - vp->label = NULL; - } else { + else (void)mgt_vcl_setstate(cli, vp, VCL_STATE_COLD); - } if (child_pid >= 0) { /* XXX If this fails the child is crashing, figure that later */ (void)mgt_cli_askchild(&status, &p, "vcl.discard %s\n", av[2]); @@ -614,6 +609,21 @@ mcf_vcl_discard(struct cli *cli, const char * const *av, void *priv) mgt_vcl_del(vp); } +static void +mcf_list_labels(struct cli *cli, const struct vclprog *vp) +{ + int n = 0; + struct vcldep *vd; + + VTAILQ_FOREACH(vd, &vp->dto, lto) { + if (n++ == 5) { + VCLI_Out(cli, " [...]"); + break; + } + VCLI_Out(cli, " %s", vd->from->name); + } +} + static void __match_proto__(cli_func_t) mcf_vcl_list(struct cli *cli, const char * const *av, void *priv) { @@ -637,10 +647,7 @@ mcf_vcl_list(struct cli *cli, const char * const *av, void *priv) VCLI_Out(cli, "/%-8s", vp->warm ? VCL_STATE_WARM : VCL_STATE_COLD); VCLI_Out(cli, " %6s %s", "", vp->name); - if (vp->label != NULL) - VCLI_Out(cli, " %s %s", - mcf_is_label(vp) ? - "->" : "<-", vp->label->name); + mcf_list_labels(cli, vp); VCLI_Out(cli, "\n"); } } @@ -664,14 +671,15 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) if (vpt == NULL) return; if (mcf_is_label(vpt)) { - VCLI_SetResult(cli, CLIS_PARAM); + VCLI_SetResult(cli, CLIS_CANT); VCLI_Out(cli, "VCL labels cannot point to labels"); return; } - if (vpt->label != NULL) { - VCLI_SetResult(cli, CLIS_PARAM); - VCLI_Out(cli, "VCL already labeled (\"%s\")", - vpt->label->name); + if (!VTAILQ_EMPTY(&vpt->dto)) { + VCLI_SetResult(cli, CLIS_CANT); + VCLI_Out(cli, "VCL already labeled with"); + mcf_list_labels(cli, vpt); + VCLI_Out(cli, "\n"); return; } vpl = mcf_vcl_byname(av[2]); @@ -681,11 +689,6 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) VCLI_Out(cli, "%s is not a label", vpl->name); return; } - AN(vpl->label); - assert(vpl->label->label == vpl); - /* XXX SET vp->label AUTO */ - vpl->label->label = NULL; - vpl->label = NULL; mgt_vcl_dep_del(VTAILQ_FIRST(&vpl->dfrom)); AN(VTAILQ_EMPTY(&vpl->dfrom)); } else { @@ -694,8 +697,6 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) AN(vpl); mgt_vcl_dep_add(vpl, vpt); vpl->warm = 1; - vpl->label = vpt; - vpt->label = vpl; if (vpt->state == VCL_STATE_COLD) vpt->state = VCL_STATE_AUTO; (void)mgt_vcl_setstate(cli, vpt, VCL_STATE_WARM); diff --git a/bin/varnishtest/tests/b00032.vtc b/bin/varnishtest/tests/b00032.vtc index 48aa465..89cb26f 100644 --- a/bin/varnishtest/tests/b00032.vtc +++ b/bin/varnishtest/tests/b00032.vtc @@ -16,7 +16,7 @@ varnish v1 -cliok start varnish v1 -cliok "vcl.use vcl1" -varnish v1 -clierr 106 "vcl.discard vcl1" +varnish v1 -clierr 300 "vcl.discard vcl1" varnish v1 -clierr 106 "vcl.discard vcl0" diff --git a/bin/varnishtest/tests/c00015.vtc b/bin/varnishtest/tests/c00015.vtc index 9725ead..1acbbd2 100644 --- a/bin/varnishtest/tests/c00015.vtc +++ b/bin/varnishtest/tests/c00015.vtc @@ -58,4 +58,4 @@ varnish v1 -clierr 106 "vcl.show -x nowhere" varnish v1 -clierr 106 "vcl.show nothere" varnish v1 -clierr 106 "vcl.use nothere" varnish v1 -clierr 106 "vcl.discard nowhere" -varnish v1 -clierr 106 "vcl.discard vcl1" +varnish v1 -clierr 300 "vcl.discard vcl1" diff --git a/bin/varnishtest/tests/c00077.vtc b/bin/varnishtest/tests/c00077.vtc index 005a17d..49ed767 100644 --- a/bin/varnishtest/tests/c00077.vtc +++ b/bin/varnishtest/tests/c00077.vtc @@ -37,7 +37,7 @@ client c1 { expect resp.http.vcl == vclA } -run -varnish v1 -clierr 106 "vcl.discard vcl1" +varnish v1 -clierr 300 "vcl.discard vcl1" varnish v1 -vcl+backend { sub vcl_recv { return (vcl(vclB)); } } @@ -54,7 +54,7 @@ varnish v1 -vcl+backend { sub vcl_recv { return (vcl(vclA)); } } varnish v1 -vcl+backend { sub vcl_recv { return (vcl(vclA)); } } varnish v1 -vcl+backend { sub vcl_recv { return (vcl(vclA)); } } -varnish v1 -clierr 106 "vcl.discard vclA" +varnish v1 -clierr 300 "vcl.discard vclA" varnish v1 -vcl+backend { } varnish v1 -cliok "vcl.discard vcl3" diff --git a/bin/varnishtest/tests/v00044.vtc b/bin/varnishtest/tests/v00044.vtc index d09501b..26feae3 100644 --- a/bin/varnishtest/tests/v00044.vtc +++ b/bin/varnishtest/tests/v00044.vtc @@ -63,7 +63,7 @@ varnish v1 -expect VBE.vcl1.default.happy >= 0 varnish v1 -expect VBE.vcl2.default.happy >= 0 # You can't freeze the active VCL -varnish v1 -clierr 106 "vcl.state vcl2 cold" +varnish v1 -clierr 300 "vcl.state vcl2 cold" # However a warm event is guaranteed... logexpect l1 -v v1 -g raw { diff --git a/bin/varnishtest/tests/v00048.vtc b/bin/varnishtest/tests/v00048.vtc index 35ba584..25cf629 100644 --- a/bin/varnishtest/tests/v00048.vtc +++ b/bin/varnishtest/tests/v00048.vtc @@ -37,16 +37,16 @@ client c1 { varnish v1 -cliok "vcl.list" varnish v1 -clierr 106 "vcl.label foo vcl0" varnish v1 -cliok "vcl.label foo vcl2" -varnish v1 -clierr 106 "vcl.label bar vcl2" -varnish v1 -clierr 106 "vcl.discard vcl2" +varnish v1 -clierr 300 "vcl.label bar vcl2" +varnish v1 -clierr 300 "vcl.discard vcl2" varnish v1 -cliok "vcl.label foo vcl1" varnish v1 -clierr 106 "vcl.label vcl1 vcl2" varnish v1 -clierr 106 "vcl.state foo cold" -varnish v1 -clierr 106 "vcl.label bar foo" -varnish v1 -clierr 106 "vcl.discard vcl1" +varnish v1 -clierr 300 "vcl.label bar foo" +varnish v1 -clierr 300 "vcl.discard vcl1" varnish v1 -cliok "vcl.list" varnish v1 -cliok "vcl.use foo" -varnish v1 -clierr 106 "vcl.discard foo" +varnish v1 -clierr 300 "vcl.discard foo" varnish v1 -cliok "vcl.list" client c1 -run @@ -75,14 +75,14 @@ varnish v1 -stop varnish v1 -cliok "vcl.list" varnish v1 -clierr 106 "vcl.label foo vcl0" varnish v1 -cliok "vcl.label foo vcl1" -varnish v1 -clierr 106 "vcl.label bar foo" +varnish v1 -clierr 300 "vcl.label bar foo" varnish v1 -clierr 200 "vcl.state vcl1 warm" varnish v1 -clierr 200 "vcl.state vcl1 auto" varnish v1 -clierr 300 "vcl.state vcl1 cold" -varnish v1 -clierr 106 "vcl.discard vcl1" +varnish v1 -clierr 300 "vcl.discard vcl1" varnish v1 -cliok "vcl.list" varnish v1 -cliok "vcl.use foo" -varnish v1 -clierr 106 "vcl.discard foo" +varnish v1 -clierr 300 "vcl.discard foo" varnish v1 -cliok "vcl.list" server s1 -start From nils.goroll at uplex.de Thu Sep 8 15:08:16 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 08 Sep 2016 17:08:16 +0200 Subject: [master] 45a4a2f tiny progress on the changelog Message-ID: commit 45a4a2f6eb0363eafd1a453bbadb86b40a319097 Author: Nils Goroll Date: Thu Sep 8 17:07:10 2016 +0200 tiny progress on the changelog diff --git a/doc/changes.rst b/doc/changes.rst index 90ba66c..057982f 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -8,6 +8,10 @@ Master * time bend option: control the log processing speed * y axis scale +* varnishd no longer seeds the libc random number generator, but + rather its own. This can affect vmods using random() with varnishd + debug.srandom + ====================================== Varnish Cache 4.1.3-beta1 (2016-06-15) ====================================== From fgsch at lodoss.net Thu Sep 8 18:07:50 2016 From: fgsch at lodoss.net (Federico Schwindt) Date: Thu, 8 Sep 2016 19:07:50 +0100 Subject: [master] 169e162 Enforce that VCL names must be C-language identifiers ([A-Za-z][A-Za-z0-9_]*) In-Reply-To: <17299.1473338339@critter.freebsd.dk> References: <15940.1473337703@critter.freebsd.dk> <17299.1473338339@critter.freebsd.dk> Message-ID: I reckon we should support the dash, at the very least. On 8 Sep 2016 1:39 pm, "Poul-Henning Kamp" wrote: > -------- > In message gmail.com> > , Dridi Boukelmoune writes: > > >> The alternative would be to encode a freer form into C-idents, but that > >> turns really ugly really fast... > > > >I can fix packaging scripts once we reach consensus, but this probably > >deserve a proper entry in upgrading.rst since this change may break > >things downstream (like the varnish agent for instance). > > If '-' is in widespread use, we should not break it. > > I asked on IRC earlier, but nobody reacted... > > -- > Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 > phk at FreeBSD.ORG | TCP/IP since RFC 956 > FreeBSD committer | BSD since 4.3-tahoe > Never attribute to malice what can adequately be explained by incompetence. > -------------- next part -------------- An HTML attachment was scrubbed... URL: From dridi.boukelmoune at gmail.com Thu Sep 8 21:26:11 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 08 Sep 2016 23:26:11 +0200 Subject: [master] 1b4395b Whitespace OCD Message-ID: commit 1b4395b417f78d786402138880863b0345aa571f Author: Dridi Boukelmoune Date: Thu Sep 8 23:20:15 2016 +0200 Whitespace OCD diff --git a/bin/varnishtest/tests/README b/bin/varnishtest/tests/README index 3fefab8..dd21d38 100644 --- a/bin/varnishtest/tests/README +++ b/bin/varnishtest/tests/README @@ -14,7 +14,7 @@ Naming scheme [id]%05d.vtc id ~ ^a --> varnishtest(1) tests - id ~ ^a02 --> HTTP2 + id ~ ^a02 --> HTTP2 id ~ ^b --> Basic functionality tests id ~ ^c --> Complex functionality tests id ~ ^d --> Director VMOD tests From dridi.boukelmoune at gmail.com Thu Sep 8 21:26:11 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 08 Sep 2016 23:26:11 +0200 Subject: [master] a063045 Fix the symbol pattern for probe definitions Message-ID: commit a063045dc337f0252e47fc6b7701df1fa8d2ce36 Author: Dridi Boukelmoune Date: Thu Sep 8 23:20:48 2016 +0200 Fix the symbol pattern for probe definitions To avoid another regression, a test case now passes a probe to the debug VMOD. This problem was discovered while working on #2068. diff --git a/bin/varnishtest/tests/m00025.vtc b/bin/varnishtest/tests/m00025.vtc new file mode 100644 index 0000000..cdd6a8e --- /dev/null +++ b/bin/varnishtest/tests/m00025.vtc @@ -0,0 +1,13 @@ +varnishtest "Pass probe definitions to VMODs" + +varnish v1 -vcl { + import debug; + + backend be { .host = "${bad_ip}"; } + probe pb { } + + sub vcl_init { + debug.test_probe(pb); + debug.test_probe(pb, pb); + } +} diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index 5847e72..a5b5b5e 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -258,7 +258,8 @@ vcc_ParseProbe(struct vcc *tl) t_probe = tl->t; vcc_NextToken(tl); - (void)VCC_HandleSymbol(tl, t_probe, PROBE, "%.s", PF(t_probe)); + (void)VCC_HandleSymbol(tl, t_probe, PROBE, "&vgc_probe_%.*s", + PF(t_probe)); ERRCHK(tl); vcc_ParseProbeSpec(tl, t_probe, &p); diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 973337d..cbe9de8 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -168,3 +168,7 @@ Perform an IP match against a named ACL. $Function BOOL barrier_sync(STRING) Synchronize with a varnishtest shared barrier. + +$Function VOID test_probe(PROBE probe, PROBE same = 0) + +Only here to make sure probe definitions are passed properly. diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index d58856a..dc9e347 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -515,3 +515,13 @@ vmod_barrier_sync(VRT_CTX, VCL_STRING addr) VSLb(ctx->vsl, SLT_Error, "Barrier unexpected data (%zdB)", sz); return (0); } + +VCL_VOID +vmod_test_probe(VRT_CTX, VCL_PROBE probe, VCL_PROBE same) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(probe, VRT_BACKEND_PROBE_MAGIC); + CHECK_OBJ_ORNULL(same, VRT_BACKEND_PROBE_MAGIC); + AZ(same == NULL || probe == same); +} From dridi.boukelmoune at gmail.com Thu Sep 8 21:27:09 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 08 Sep 2016 23:27:09 +0200 Subject: [4.1] 80a169e Backport m00025.vtc Message-ID: commit 80a169e80ba8ccb9982140d5344c094688a8d415 Author: Dridi Boukelmoune Date: Thu Sep 8 23:25:35 2016 +0200 Backport m00025.vtc diff --git a/bin/varnishtest/tests/m00025.vtc b/bin/varnishtest/tests/m00025.vtc new file mode 100644 index 0000000..cf78acd --- /dev/null +++ b/bin/varnishtest/tests/m00025.vtc @@ -0,0 +1,13 @@ +varnishtest "Pass probe definitions to VMODs" + +varnish v1 -vcl { + import ${vmod_debug}; + + backend be { .host = "${bad_ip}"; } + probe pb { } + + sub vcl_init { + debug.test_probe(pb); + debug.test_probe(pb, pb); + } +} diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 3ec9511..b905227 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -158,3 +158,7 @@ Perform an IP match against a named ACL. $Function BOOL barrier_sync(STRING) Synchronize with a varnishtest shared barrier. + +$Function VOID test_probe(PROBE probe, PROBE same = 0) + +Only here to make sure probe definitions are passed properly. diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index cca54e8..f2e0c38 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -513,3 +513,13 @@ vmod_barrier_sync(VRT_CTX, VCL_STRING addr) VSLb(ctx->vsl, SLT_Error, "Barrier unexpected data (%zdB)", sz); return (0); } + +VCL_VOID +vmod_test_probe(VRT_CTX, VCL_PROBE probe, VCL_PROBE same) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(probe, VRT_BACKEND_PROBE_MAGIC); + CHECK_OBJ_ORNULL(same, VRT_BACKEND_PROBE_MAGIC); + AZ(same == NULL || probe == same); +} From fgsch at lodoss.net Fri Sep 9 05:34:10 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 09 Sep 2016 07:34:10 +0200 Subject: [master] a630df0 Whitespaces, correct pasto and leftover from 4.0 Message-ID: commit a630df005ac8771c0f8bd29ca8f6d7e9d477b65b Author: Federico G. Schwindt Date: Fri Sep 9 06:23:59 2016 +0100 Whitespaces, correct pasto and leftover from 4.0 diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index d26d982..892b9c6 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -175,7 +175,7 @@ vhp_table_test_LDADD = \ $(top_builddir)/lib/libvarnish/libvarnish.la noinst_PROGRAMS += vhp_decode_test -vhp_decode_test_SOURCES = hpack/vhp_decode.c hpack/vhp_table.c +vhp_decode_test_SOURCES = hpack/vhp_decode.c hpack/vhp_table.c vhp_decode_test_CFLAGS = -DDECODE_TEST_DRIVER -include config.h vhp_decode_test_LDADD = \ $(top_builddir)/lib/libvarnish/libvarnish.la diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 53a15dd..360706e 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -36,7 +36,6 @@ -esym(765, BackSlash) -esym(759, BackSlashDecode) -esym(765, BackSlashDecode) --esym(714, VSS_open) -esym(757, base64*) diff --git a/bin/varnishtest/vtc_doc_hdr.rst b/bin/varnishtest/vtc_doc_hdr.rst index de8541f..bde6c56 100644 --- a/bin/varnishtest/vtc_doc_hdr.rst +++ b/bin/varnishtest/vtc_doc_hdr.rst @@ -18,7 +18,7 @@ Here's an example of test file:: txgoaway -laststream 0 -err 9 -debug "COMPRESSION_ERROR" } -run } -start - + client c1 -connect ${s1_sock} { stream 1 { txreq -idxHdr 100 -litHdr inc plain "foo" plain "bar" diff --git a/doc/sphinx/users-guide/vcl-separate.rst b/doc/sphinx/users-guide/vcl-separate.rst index 15873aa..f876ea9 100644 --- a/doc/sphinx/users-guide/vcl-separate.rst +++ b/doc/sphinx/users-guide/vcl-separate.rst @@ -71,7 +71,7 @@ If you want to change the top level VCL, do as you always did:: Details, details, details: -------------------------- -* All requests *always* start in the active VCL - the one from ``vcl.use`` +* All requests *always* start in the active VCL - the one from ``vcl.use`` * Only VCL labels can be used in ``return(vcl(name))``. Without this restriction the top level VCL would have to be reloaded every time diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 6259934..ad944b6 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -261,6 +261,7 @@ vcc_delete_expr(struct expr *e) VSB_destroy(&e->vsb); FREE_OBJ(e); } + /*-------------------------------------------------------------------- * We want to get the indentation right in the emitted C code so we have * to represent it symbolically until we are ready to render. @@ -273,7 +274,7 @@ vcc_delete_expr(struct expr *e) * \v1 insert subexpression 1 * \v2 insert subexpression 2 * \v+ increase indentation - * \v- increase indentation + * \v- decrease indentation * anything else is literal * * When editing, we check if any of the subexpressions contain a newline From phk at FreeBSD.org Fri Sep 9 08:59:12 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 09 Sep 2016 10:59:12 +0200 Subject: [master] f176f05 Allow automatic promotion from INT to REAL. Message-ID: commit f176f05c13abfb87a91ea938c2bcc09a7f74220a Author: Poul-Henning Kamp Date: Fri Sep 9 08:50:47 2016 +0000 Allow automatic promotion from INT to REAL. Fixes #2084 diff --git a/bin/varnishtest/tests/r02084.vtc b/bin/varnishtest/tests/r02084.vtc new file mode 100644 index 0000000..aa3bafb --- /dev/null +++ b/bin/varnishtest/tests/r02084.vtc @@ -0,0 +1,27 @@ +varnishtest "REAL - INT types" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + import std; + + sub vcl_deliver { + set resp.http.add = 10.0 + std.integer(req.http.foo, 0); + set resp.http.sub = 10.0 - std.integer(req.http.foo, 0); + set resp.http.mul = 10.0 * std.integer(req.http.foo, 0); + set resp.http.div = 10.0 / std.integer(req.http.foo, 0); + } + +} -start + +client c1 { + txreq -hdr "foo: 3" + rxresp + expect resp.http.add == 13.000 + expect resp.http.sub == 7.000 + expect resp.http.mul == 30.000 + expect resp.http.div == 3.333 +} -run diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index ad944b6..9624af2 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -910,14 +910,13 @@ vcc_expr_mul(struct vcc *tl, struct expr **e, vcc_type_t fmt) vcc_NextToken(tl); vcc_expr4(tl, &e2, f2); ERRCHK(tl); - if (e2->fmt->multype == NULL) { + if (e2->fmt != REAL && e2->fmt != INT) { VSB_printf(tl->sb, "%s %.*s %s not possible.\n", f2->name, PF(tk), e2->fmt->name); vcc_ErrWhere(tl, tk); return; } - assert(e2->fmt == f2); if (tk->tok == '*') *e = vcc_expr_edit(f3, "(\v1*\v2)", *e, e2); else @@ -1017,6 +1016,10 @@ vcc_expr_add(struct vcc *tl, struct expr **e, vcc_type_t fmt) ADD_OK("-", INT, INT, INT); ADD_OK("+", REAL, REAL, REAL); ADD_OK("-", REAL, REAL, REAL); + ADD_OK("+", REAL, INT, REAL); + ADD_OK("-", REAL, INT, REAL); + ADD_OK("+", INT, REAL, REAL); + ADD_OK("-", INT, REAL, REAL); #undef ADD_OK From phk at FreeBSD.org Fri Sep 9 09:17:10 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 09 Sep 2016 11:17:10 +0200 Subject: [master] e4ce8c0 In C enums are integertyped, but it is up to the compiler to decide if they are signed or unsigned. Message-ID: commit e4ce8c019442e7dc8db208ca94cd1fd771276223 Author: Poul-Henning Kamp Date: Fri Sep 9 09:13:57 2016 +0000 In C enums are integertyped, but it is up to the compiler to decide if they are signed or unsigned. Range-check enums is sound programming practice, but that concept seems to be beyond the imagination of certain compiler people: vhp_decode.c:96:2: error: comparison of unsigned expression >= 0 is always true [-Werror=type-limits] Add a dummy "MIN" value to the enum, give it value -1 to force the compiler to use signed ints for the enum, and then check that they're never negative. diff --git a/bin/varnishd/hpack/vhp_decode.c b/bin/varnishd/hpack/vhp_decode.c index 49a0e74..15e05fd 100644 --- a/bin/varnishd/hpack/vhp_decode.c +++ b/bin/varnishd/hpack/vhp_decode.c @@ -70,6 +70,7 @@ enum vhd_func_e { /* States */ enum vhd_state_e { + VHD_S__MIN = -1, #define VHD_FSM(STATE, FUNC, arg1, arg2) \ VHD_S_##STATE, #include "tbl/vhd_fsm.h" @@ -93,7 +94,7 @@ static void vhd_set_state(struct vhd_decode *d, enum vhd_state_e state) { AN(d); - assert(state >= 0 && state < VHD_S__MAX); + assert(state > VHD_S__MIN && state < VHD_S__MAX); d->state = state; d->first = 1; } From lkarsten at varnish-software.com Fri Sep 9 09:55:13 2016 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Fri, 09 Sep 2016 11:55:13 +0200 Subject: [master] 577dab5 Prepare for 5.0.0-beta1. Message-ID: commit 577dab5fb8b174b16448b4de1bd8708cb9b7cc89 Author: Lasse Karstensen Date: Fri Sep 9 10:42:06 2016 +0200 Prepare for 5.0.0-beta1. diff --git a/configure.ac b/configure.ac index fbbb0b8..086acf1 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS Copyright (c) 2006-2016 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [trunk], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [5.0.0-beta1], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/changes.rst b/doc/changes.rst index 057982f..41d92bc 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,16 +1,15 @@ -====== -Master -====== - -* varnishhist improvements: - * can now generate statistics of backend requests - * more predefined profiles - * time bend option: control the log processing speed - * y axis scale - -* varnishd no longer seeds the libc random number generator, but - rather its own. This can affect vmods using random() with varnishd - debug.srandom +====================================== +Varnish Cache 5.0.0-beta1 (2016-09-09) +====================================== + +This is the first beta release of the upcoming 5.0 release. + +The list of changes are numerous and will not be expanded on in detail. + +Major items: +* Experimental support for HTTP/2. +* VCL labels, allowing for per-vhost VCL. + ====================================== Varnish Cache 4.1.3-beta1 (2016-06-15) From nils.goroll at uplex.de Fri Sep 9 10:12:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 09 Sep 2016 12:12:12 +0200 Subject: [master] 6829b6b test real int Message-ID: commit 6829b6ba1e41aee72dda278ed7ef33793fca33ee Author: Nils Goroll Date: Thu Sep 8 19:00:34 2016 +0200 test real int diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index b800411..a932426 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -115,6 +115,11 @@ varnish v1 -vcl { set req.http.foo = 3.6 * 1.4; set req.http.foo = 3.6 / 1.4; + set req.http.foo = 1.0 + 1; + set req.http.foo = 1.0 - 1; + set req.http.foo = 3.0 * 2; + set req.http.foo = 3.0 / 2; + set req.http.foo = req.http.foo ~ "bar"; set req.http.foo = req.http.foo !~ "bar"; set req.http.foo = "foo" ~ "bar"; From hermunn at varnish-software.com Fri Sep 9 11:01:03 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:03 +0200 Subject: [4.1] 92a4bde vut clients: consistently mark all Message-ID: commit 92a4bde32376844f0c6f966d3825c2ffb9815033 Author: Nils Goroll Date: Mon Jul 25 14:24:21 2016 +0200 vut clients: consistently mark all In usage/documentation, mark those options with <> which are not a literal string, but rather a placeholder for some value. Conflicts: bin/varnishstat/varnishstat_options.h diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h index 7b23a6c..197b8c2 100644 --- a/bin/varnishlog/varnishlog_options.h +++ b/bin/varnishlog/varnishlog_options.h @@ -43,7 +43,7 @@ ) #define LOG_OPT_w \ - VOPT("w:", "[-w filename]", "Output filename", \ + VOPT("w:", "[-w ]", "Output filename", \ "Redirect output to file. The file will be overwritten" \ " unless the -a option was specified. If the application" \ " receives a SIGHUP in daemon mode the file will be " \ diff --git a/bin/varnishncsa/varnishncsa_options.h b/bin/varnishncsa/varnishncsa_options.h index 0f12e89..e47d696 100644 --- a/bin/varnishncsa/varnishncsa_options.h +++ b/bin/varnishncsa/varnishncsa_options.h @@ -36,12 +36,12 @@ ) #define NCSA_OPT_F \ - VOPT("F:", "[-F format]", "Set output format", \ + VOPT("F:", "[-F ]", "Set output format", \ "Set the output log format string." \ ) #define NCSA_OPT_f \ - VOPT("f:", "[-f formatfile]", "Read output format from file", \ + VOPT("f:", "[-f ]", "Read output format from file", \ "Read output format from a file. Will read a single line" \ " from the specified file, and use that line as the" \ " format." \ @@ -55,7 +55,7 @@ ) #define NCSA_OPT_w \ - VOPT("w:", "[-w filename]", "Output filename", \ + VOPT("w:", "[-w ]", "Output filename", \ "Redirect output to file. The file will be overwritten" \ " unless the -a option was specified. If the application" \ " receives a SIGHUP in daemon mode the file will be" \ diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c index c7746d6..d9f996d 100644 --- a/bin/varnishstat/varnishstat.c +++ b/bin/varnishstat/varnishstat.c @@ -254,7 +254,7 @@ usage(void) "[-1lV] [-f field] [-t seconds|] " VSC_n_USAGE "\n"); fprintf(stderr, FMT, "-1", "Print the statistics to stdout."); - fprintf(stderr, FMT, "-f field", "Field inclusion glob"); + fprintf(stderr, FMT, "-f ", "Field inclusion glob"); fprintf(stderr, FMT, "", "If it starts with '^' it is used as an exclusion list."); fprintf(stderr, FMT, "-l", diff --git a/bin/varnishtop/varnishtop_options.h b/bin/varnishtop/varnishtop_options.h index b2fc39a..f87a5f4 100644 --- a/bin/varnishtop/varnishtop_options.h +++ b/bin/varnishtop/varnishtop_options.h @@ -45,7 +45,7 @@ ) #define TOP_OPT_p \ - VOPT("p:", "[-p period]", "Sampling period", \ + VOPT("p:", "[-p ]", "Sampling period", \ "Specified the number of seconds to measure over, the" \ " default is 60 seconds. The first number in the list is" \ " the average number of requests seen over this time" \ diff --git a/include/vapi/vapi_options.h b/include/vapi/vapi_options.h index 46f826f..98b838b 100644 --- a/include/vapi/vapi_options.h +++ b/include/vapi/vapi_options.h @@ -51,7 +51,7 @@ ) #define VSL_OPT_i \ - VOPT("i:", "[-i taglist]", "Include tags", \ + VOPT("i:", "[-i ]", "Include tags", \ "Include log records of these tags in output. Taglist is" \ " a comma-separated list of tag globs. Multiple -i" \ " options may be given.\n" \ @@ -69,7 +69,7 @@ ) #define VSL_OPT_L \ - VOPT("L:", "[-L limit]", "Incomplete transaction limit", \ + VOPT("L:", "[-L ]", "Incomplete transaction limit", \ "Sets the upper limit of incomplete transactions kept" \ " before the oldest transaction is force completed. A" \ " warning record is synthesized when this happens. This" \ @@ -78,7 +78,7 @@ ) #define VSL_OPT_T \ - VOPT("T:", "[-T seconds]", "Transaction end timeout", \ + VOPT("T:", "[-T ]", "Transaction end timeout", \ "Sets the transaction timeout in seconds. This defines the" \ " maximum number of seconds elapsed between a Begin tag" \ " and the End tag. If the timeout expires, a warning" \ @@ -94,7 +94,7 @@ ) #define VSL_OPT_x \ - VOPT("x:", "[-x taglist]", "Exclude tags", \ + VOPT("x:", "[-x ]", "Exclude tags", \ "Exclude log records of these tags in output. Taglist is" \ " a comma-separated list of tag globs. Multiple -x" \ " options may be given.\n" \ diff --git a/include/vut_options.h b/include/vut_options.h index 16258b8..50881cd 100644 --- a/include/vut_options.h +++ b/include/vut_options.h @@ -51,35 +51,35 @@ ) #define VUT_OPT_k \ - VOPT("k:", "[-k num]", "Limit transactions", \ + VOPT("k:", "[-k ]", "Limit transactions", \ "Process this number of matching log transactions before" \ " exiting." \ ) #define VUT_OPT_n \ - VOPT("n:", "[-n name]", "Varnish instance name", \ + VOPT("n:", "[-n ]", "Varnish instance name", \ "Specify the name of the varnishd instance to get logs" \ " from. If -n is not specified, the host name is used." \ ) #define VUT_OPT_N \ - VOPT("N:", "[-N filename]", "VSM filename", \ + VOPT("N:", "[-N ]", "VSM filename", \ "Specify the filename of a stale VSM instance. When using" \ " this option the abandonment checking is disabled." \ ) #define VUT_OPT_P \ - VOPT("P:", "[-P file]", "PID file", \ + VOPT("P:", "[-P ]", "PID file", \ "Write the process' PID to the specified file." \ ) #define VUT_OPT_q \ - VOPT("q:", "[-q query]", "VSL query", \ + VOPT("q:", "[-q ]", "VSL query", \ "Specifies the VSL query to use." \ ) #define VUT_OPT_r \ - VOPT("r:", "[-r filename]", "Binary file input", \ + VOPT("r:", "[-r ]", "Binary file input", \ "Read log in binary file format from this file. The file" \ " can be created with ``varnishlog -w filename``." \ ) From hermunn at varnish-software.com Fri Sep 9 11:01:03 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:03 +0200 Subject: [4.1] 41f1673 Mark some more non-literal options Message-ID: commit 41f1673dd126dfaf93b3852fc3689a4c90a409b1 Author: P?l Hermunn Johansen Date: Wed Sep 7 14:25:20 2016 +0200 Mark some more non-literal options In the usage section of varnishstat, some non-literal options were not caught by the backport. diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c index d9f996d..1a690cf 100644 --- a/bin/varnishstat/varnishstat.c +++ b/bin/varnishstat/varnishstat.c @@ -251,7 +251,7 @@ usage(void) { #define FMT " %-28s # %s\n" fprintf(stderr, "usage: varnishstat " - "[-1lV] [-f field] [-t seconds|] " + "[-1lV] [-f ] [-t |] " VSC_n_USAGE "\n"); fprintf(stderr, FMT, "-1", "Print the statistics to stdout."); fprintf(stderr, FMT, "-f ", "Field inclusion glob"); @@ -259,11 +259,11 @@ usage(void) "If it starts with '^' it is used as an exclusion list."); fprintf(stderr, FMT, "-l", "Lists the available fields to use with the -f option."); - fprintf(stderr, FMT, "-n varnish_name", + fprintf(stderr, FMT, "-n ", "The varnishd instance to get logs from."); - fprintf(stderr, FMT, "-N filename", + fprintf(stderr, FMT, "-N ", "Filename of a stale VSM instance."); - fprintf(stderr, FMT, "-t seconds|", + fprintf(stderr, FMT, "-t |", "Timeout before returning error on initial VSM connection."); fprintf(stderr, FMT, "-V", "Display the version number and exit."); fprintf(stderr, FMT, "-x", From hermunn at varnish-software.com Fri Sep 9 11:01:03 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:03 +0200 Subject: [4.1] 9ca45ff Let varnishhist see the backend side and add some profiles ... Message-ID: commit 9ca45ff11d42be80652ec3fd5312ea7ae2d8b5d4 Author: Nils Goroll Date: Sat Jul 23 15:24:56 2016 +0200 Let varnishhist see the backend side and add some profiles ... ... for the standard backend timestamps and size. Conflicts: bin/varnishhist/varnishhist.c diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index 36a25e8..a8eec87 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -105,6 +105,7 @@ static int scales[] = { struct profile { const char *name; + char VSL_arg; enum VSL_tag_e tag; const char *prefix; int field; @@ -112,22 +113,64 @@ struct profile { int hist_high; } profiles[] = { + // client { .name = "responsetime", + .VSL_arg = 'c', .tag = SLT_Timestamp, .prefix = "Process:", .field = 3, .hist_low = -6, .hist_high = 3 - }, { + }, + { .name = "size", + .VSL_arg = 'c', .tag = SLT_ReqAcct, .prefix = NULL, .field = 5, .hist_low = 1, .hist_high = 8 - }, { - .name = 0, + }, + // backend + { + .name = "Bereqtime", + .VSL_arg = 'b', + .tag = SLT_Timestamp, + .prefix = "Bereq:", + .field = 3, + .hist_low = -6, + .hist_high = 3 + }, + { + .name = "Beresptime", + .VSL_arg = 'b', + .tag = SLT_Timestamp, + .prefix = "Beresp:", + .field = 3, + .hist_low = -6, + .hist_high = 3 + }, + { + .name = "BerespBodytime", + .VSL_arg = 'b', + .tag = SLT_Timestamp, + .prefix = "BerespBody:", + .field = 3, + .hist_low = -6, + .hist_high = 3 + }, + { + .name = "Besize", + .VSL_arg = 'b', + .tag = SLT_BereqAcct, + .prefix = NULL, + .field = 5, + .hist_low = 1, + .hist_high = 8 + }, + { + .name = NULL } }; @@ -203,9 +246,6 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], (void)priv; for (tr = pt[0]; tr != NULL; tr = *++pt) { - if (tr->type != VSL_t_req) - /* Only look at client requests */ - continue; if (tr->reason == VSL_r_esi) /* Skip ESI requests */ continue; @@ -223,7 +263,9 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], break; case SLT_VCL_return: if (!strcasecmp(VSL_CDATA(tr->c->rec.ptr), - "restart")) + "restart") || + !strcasecmp(VSL_CDATA(tr->c->rec.ptr), + "retry")) skip = 1; break; default: @@ -374,11 +416,19 @@ usage(int status) exit(status); } +static void +profile_error(const char *s) +{ + fprintf(stderr, "-P: '%s' is not a valid" + " profile name or definition\n", s); + exit(1); +} + int main(int argc, char **argv) { int i; - char *colon; + const char *colon, *ptag; const char *profile = "responsetime"; pthread_t thr; int fnum = -1; @@ -389,8 +439,6 @@ main(int argc, char **argv) if (0) (void)usage; - /* only client requests */ - assert(VUT_Arg('c', NULL)); while ((i = getopt(argc, argv, vopt_optstring)) != -1) { switch (i) { case 'h': @@ -404,14 +452,25 @@ main(int argc, char **argv) break; } /* else it's a definition, we hope */ + if (colon == optarg + 1 && + (*optarg == 'b' || *optarg == 'c')) { + cli_p.VSL_arg = *optarg; + ptag = colon + 1; + colon = strchr(colon + 1, ':'); + if (colon == NULL) + profile_error(optarg); + } else { + ptag = optarg; + cli_p.VSL_arg = 'c'; + } + + assert(colon); if (sscanf(colon+1, "%d:%d:%d", &cli_p.field, &cli_p.hist_low, &cli_p.hist_high) != 3) { - fprintf(stderr, "-P: '%s' is not a valid" - " profile name or definition\n", optarg); - exit(1); + profile_error(optarg); } - match_tag = VSL_Name2Tag(optarg, colon - optarg); + match_tag = VSL_Name2Tag(ptag, colon - ptag); if (match_tag < 0) { fprintf(stderr, "-P: '%s' is not a valid tag name\n", @@ -448,6 +507,8 @@ main(int argc, char **argv) fprintf(stderr, "-P: No such profile '%s'\n", profile); exit(1); } + + assert(VUT_Arg(active_profile->VSL_arg, NULL)); match_tag = active_profile->tag; fnum = active_profile->field; hist_low = active_profile->hist_low; diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index 5241539..16150a8 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -45,13 +45,18 @@ ) #define HIS_OPT_P \ - VOPT("P:", "[-P ]", \ + VOPT("P:", "[-P ]", \ "Profile definition", \ - "Either specify \"size\" or \"responsetime\" profile or" \ - " create a new one. Define the tag we'll look for, and the" \ - " field number of the value we are interested in. min and" \ - " max are the boundaries of the graph (these are power of" \ - " tens)." \ + "Either a predefined profile or a custom definition. " \ + " Predefined profiles for the client side are" \ + " \"responsetime \" and \"size\". For the backend side:" \ + " \"Bereqtime\", \"Beresptime\", \"BerespBodytime\"," \ + " and \"Besize\"." \ + " custom definitions include an optional (c)lient or" \ + " (b)backend filter (defaults to client), the tag we'll" \ + " look for, and the field number of the value we are" \ + " interested in. min and max are the boundaries of the" \ + " graph (these are power of ten)." \ ) VSL_OPT_C From hermunn at varnish-software.com Fri Sep 9 11:01:03 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:03 +0200 Subject: [4.1] 7e471b3 remove duplicate init Message-ID: commit 7e471b30d531db293c1fdb54bf62f25649098b70 Author: Nils Goroll Date: Sat Jul 23 19:09:14 2016 +0200 remove duplicate init diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index a8eec87..8d0eeee 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -189,8 +189,6 @@ update(void) erase(); /* Draw horizontal axis */ - w = COLS / hist_range; - n = w * hist_range; for (i = 0; i < n; ++i) (void)mvaddch(LINES - 2, i, '-'); for (k = 0, l = hist_low; k < hist_range; ++k, ++l) { From hermunn at varnish-software.com Fri Sep 9 11:01:03 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:03 +0200 Subject: [4.1] a63e2b0 draw a y axis scale Message-ID: commit a63e2b095837823bb150d83a041a8d4bafd655cf Author: Nils Goroll Date: Sat Jul 23 18:58:50 2016 +0200 draw a y axis scale diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index 8d0eeee..56dd246 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -220,6 +220,9 @@ update(void) mvprintw(0, 0, "1:%d, n = %d", scale, nhist); + for (j = 2; j < LINES - 3; j+=5) + mvprintw(j, 0, "%d_", (LINES - 3 - j) * scale); + /* show them */ for (i = 0; i < n; ++i) { for (j = 0; j < bm[i] / scale; ++j) From hermunn at varnish-software.com Fri Sep 9 11:01:03 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:03 +0200 Subject: [4.1] 17690d4 refresh period: missing option, sub-second, allow inc/dec Message-ID: commit 17690d40453dfeac2dae7fbffe0c15e5ee07c92b Author: Nils Goroll Date: Sat Jul 23 18:04:17 2016 +0200 refresh period: missing option, sub-second, allow inc/dec Actually implement the documented -p option Make the update delay a double to allow update frequencies of less than a second. Add + and - keys to the curses interface to update the refresh frequency while running. Show update frequency in the top line. diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index 56dd246..bd17652 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -68,7 +68,7 @@ static int hist_buckets; static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; static int end_of_file = 0; -static int delay = 1; +static double delay = 1; static unsigned rr_hist[HIST_N]; static unsigned nhist; static unsigned next_hist; @@ -218,7 +218,7 @@ update(void) /* nothing */ ; scale = scales[i]; - mvprintw(0, 0, "1:%d, n = %d", scale, nhist); + mvprintw(0, 0, "1:%d, n = %d, d = %g", scale, nhist, delay); for (j = 2; j < LINES - 3; j+=5) mvprintw(j, 0, "%d_", (LINES - 3 - j) * scale); @@ -395,6 +395,14 @@ do_curses(void *arg) case '9': delay = 1 << (ch - '0'); break; + case '+': + delay /= 2; + if (delay < 1e-3) + delay = 1e-3; + break; + case '-': + delay *= 2; + break; default: beep(); break; @@ -445,6 +453,13 @@ main(int argc, char **argv) case 'h': /* Usage help */ usage(0); + case 'p': + delay = strtod(optarg, NULL); + if (delay <= 0) { + fprintf(stderr, "-p: invalid '%s'\n", optarg); + exit(1); + } + break; case 'P': colon = strchr(optarg, ':'); /* no colon, take the profile as a name*/ diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index 16150a8..3846df3 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -41,7 +41,8 @@ VOPT("p:", "[-p period]", "Refresh period", \ "Specified the number of seconds between screen refreshes." \ " Default is 1 second, and can be changed at runtime by" \ - " pressing the [1-9] keys." \ + " pressing the [0-9] keys (powers of 2 in seconds" \ + " or + and - (double/halve the speed)" \ ) #define HIS_OPT_P \ From hermunn at varnish-software.com Fri Sep 9 11:01:03 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:03 +0200 Subject: [4.1] 97cd05c timebend option: allow to specify how fast time passes Message-ID: commit 97cd05c26e187360f28c0ae4e55ef326039d5f6b Author: Nils Goroll Date: Sat Jul 23 17:50:08 2016 +0200 timebend option: allow to specify how fast time passes This is particularly useful when replaying vsl files (-r option). The new -B option specifies a time factor limiting the speed at which the log is being read. It can also be halved/doubled using the < and > keys at runtime. Conflicts: bin/varnishhist/varnishhist.c diff --git a/bin/varnishhist/Makefile.am b/bin/varnishhist/Makefile.am index 5904430..e8eb866 100644 --- a/bin/varnishhist/Makefile.am +++ b/bin/varnishhist/Makefile.am @@ -19,6 +19,7 @@ varnishhist_SOURCES = varnishhist.c \ varnishhist_LDADD = \ $(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \ $(top_builddir)/lib/libvarnishapi/libvarnishapi.la \ + $(top_builddir)/lib/libvarnish/libvarnish.la \ -lm \ @CURSES_LIB@ ${RT_LIBS} ${PTHREAD_LIBS} diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index bd17652..c980a24 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -54,6 +54,7 @@ #include "vas.h" #include "vcs.h" #include "vut.h" +#include "vtim.h" #define HIST_N 2000 /* how far back we remember */ #define HIST_RES 100 /* bucket resolution */ @@ -76,7 +77,9 @@ static unsigned *bucket_miss; static unsigned *bucket_hit; static char *format; static int match_tag; - +double timebend = 0, t0; +double vsl_t0 = 0, vsl_to, vsl_ts = 0; +pthread_cond_t timebend_cv; static double log_ten; static int scales[] = { @@ -212,13 +215,21 @@ update(void) max = bm[l] + bh[l]; } - /* scale */ + /* scale,time */ assert(LINES - 3 >= 0); for (i = 0; max / scales[i] > (unsigned)(LINES - 3); ++i) /* nothing */ ; scale = scales[i]; - mvprintw(0, 0, "1:%d, n = %d, d = %g", scale, nhist, delay); + if (vsl_t0 > 0) { + char t[VTIM_FORMAT_SIZE]; + VTIM_format(vsl_ts, t); + + mvprintw(0, 0, "1:%d, n = %d, d = %g @ %s x %g", + scale, nhist, delay, t, timebend); + } else { + mvprintw(0, 0, "1:%d, n = %d, d = %g", scale, nhist, delay); + } for (j = 2; j < LINES - 3; j+=5) mvprintw(j, 0, "%d_", (LINES - 3 - j) * scale); @@ -234,6 +245,25 @@ update(void) refresh(); } +inline static void +upd_vsl_ts(const char *p) +{ + double t; + + if (timebend == 0) + return; + + p = strchr(p, ' '); + + if (p == NULL) + return; + + t = strtod(p + 1, NULL); + + if (t > vsl_ts) + vsl_ts = t; +} + static int /*__match_proto__ (VSLQ_dispatch_f)*/ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], void *priv) @@ -242,6 +272,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], unsigned u; double value; struct VSL_transaction *tr; + double t; + const char *tsp; (void)vsl; (void)priv; @@ -254,7 +286,20 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], hit = 0; skip = 0; match = 0; - while (skip == 0 && 1 == VSL_Next(tr->c)) { + tsp = NULL; + while (skip == 0) { + i = VSL_Next(tr->c); + if (i == -3) { + // overrun - need to skip forward + pthread_mutex_lock(&mtx); + vsl_to = vsl_t0 = vsl_ts = 0; + t0 = VTIM_mono(); + pthread_mutex_unlock(&mtx); + break; + } + if (i != 1) + break; + /* get the value we want, and register if it's a hit*/ tag = VSL_TAG(tr->c->rec.ptr); @@ -269,6 +314,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], "retry")) skip = 1; break; + case SLT_Timestamp: + tsp = VSL_CDATA(tr->c->rec.ptr); default: if (tag != match_tag) break; @@ -291,6 +338,12 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], if (skip || !match) continue; + /* + * only parse the last tsp seen in this transaction - it should + * be the latest + */ + upd_vsl_ts(tsp); + /* select bucket */ i = HIST_RES * (log(value) / log_ten); if (i < hist_low * HIST_RES) @@ -300,6 +353,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], i -= hist_low * HIST_RES; assert(i >= 0); assert(i < hist_buckets); + pthread_mutex_lock(&mtx); /* phase out old data */ @@ -331,6 +385,29 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], } pthread_mutex_unlock(&mtx); } + + if (vsl_ts < vsl_to) + return (0); + + t = VTIM_mono(); + + pthread_mutex_lock(&mtx); + if (vsl_t0 == 0) + vsl_to = vsl_t0 = vsl_ts; + + assert(t > t0); + vsl_to = vsl_t0 + (t - t0) * timebend; + + if (vsl_ts > vsl_to) { + double when = VTIM_real() + vsl_ts - vsl_to; + struct timespec ts; + // Lck_CondWait + ts.tv_nsec = (long)(modf(when, &t) * 1e9); + ts.tv_sec = (long)t; + (void) pthread_cond_timedwait(&timebend_cv, &mtx, &ts); + } + pthread_mutex_unlock(&mtx); + return (0); } @@ -403,10 +480,28 @@ do_curses(void *arg) case '-': delay *= 2; break; + case '>': + case '<': + // see below + break; default: beep(); break; } + + if (ch == '<' || ch == '>') { + pthread_mutex_lock(&mtx); + vsl_to = vsl_t0 = vsl_ts; + t0 = VTIM_mono(); + if (timebend == 0) + timebend = 1; + else if (ch == '<') + timebend /= 2; + else + timebend *= 2; + pthread_cond_broadcast(&timebend_cv); + pthread_mutex_unlock(&mtx); + } } pthread_exit(NULL); } @@ -445,8 +540,7 @@ main(int argc, char **argv) cli_p.name = 0; VUT_Init(progname); - if (0) - (void)usage; + AZ(pthread_cond_init(&timebend_cv, NULL)); while ((i = getopt(argc, argv, vopt_optstring)) != -1) { switch (i) { @@ -499,6 +593,23 @@ main(int argc, char **argv) active_profile = &cli_p; break; + case 'B': + timebend = strtod(optarg, NULL); + if (timebend == 0) { + fprintf(stderr, + "-B: being able to bend time does not" + " mean we can stop it" + " (invalid factor '%s')\n", optarg); + exit(1); + } + if (timebend < 0) { + fprintf(stderr, + "-B: being able to bend time does not" + " mean we can make it go backwards" + " (invalid factor '%s')\n", optarg); + exit(1); + } + break; default: if (!VUT_Arg(i, optarg)) usage(1); @@ -535,6 +646,9 @@ main(int argc, char **argv) bucket_hit = calloc(sizeof *bucket_hit, hist_buckets); bucket_miss = calloc(sizeof *bucket_miss, hist_buckets); + if (timebend > 0) + t0 = VTIM_mono(); + format = malloc(4 * fnum); for (i = 0; i < fnum-1; i++) { strcpy(format + 4*i, "%*s "); diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index 3846df3..4b99e5e 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -60,6 +60,16 @@ " graph (these are power of ten)." \ ) +#define HIS_OPT_B \ + VOPT("B:", "[-B ]", \ + "Time bending", \ + "Factor to bend time by. Particularly usedful when" \ + " [-r]eading from a vsl file. =1 process in near real" \ + " time, <1 slow-motion, >1 time-lapse (useless unless" \ + " reading from a file). At runtime, < halves and" \ + " > doubles." \ + ) + VSL_OPT_C VUT_OPT_d HIS_OPT_g @@ -74,3 +84,4 @@ VUT_OPT_r VUT_OPT_t VSL_OPT_T VUT_OPT_V +HIS_OPT_B From hermunn at varnish-software.com Fri Sep 9 11:01:03 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:03 +0200 Subject: [4.1] 1a51307 Move the profile definition into a tbl file and generate the docs from it Message-ID: commit 1a5130781e58db0a440c6aee1188be65c57c6765 Author: Nils Goroll Date: Mon Jul 25 15:01:24 2016 +0200 Move the profile definition into a tbl file and generate the docs from it Conflicts: bin/varnishhist/Makefile.am diff --git a/bin/varnishhist/Makefile.am b/bin/varnishhist/Makefile.am index e8eb866..8063442 100644 --- a/bin/varnishhist/Makefile.am +++ b/bin/varnishhist/Makefile.am @@ -8,6 +8,7 @@ bin_PROGRAMS = varnishhist varnishhist_SOURCES = varnishhist.c \ varnishhist_options.h \ + varnishhist_profiles.h \ varnishhist_options.c \ $(top_srcdir)/lib/libvarnish/vas.c \ $(top_srcdir)/lib/libvarnish/version.c \ diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index c980a24..d435e1d 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -115,67 +115,19 @@ struct profile { int hist_low; int hist_high; } +#define HIS_PROF(name,vsl_arg,tag,prefix,field,hist_low,high_high,doc) \ + {name,vsl_arg,tag,prefix,field,hist_low,high_high}, +#define HIS_NO_PREFIX NULL +#define HIS_CLIENT 'c' +#define HIS_BACKEND 'b' profiles[] = { - // client - { - .name = "responsetime", - .VSL_arg = 'c', - .tag = SLT_Timestamp, - .prefix = "Process:", - .field = 3, - .hist_low = -6, - .hist_high = 3 - }, - { - .name = "size", - .VSL_arg = 'c', - .tag = SLT_ReqAcct, - .prefix = NULL, - .field = 5, - .hist_low = 1, - .hist_high = 8 - }, - // backend - { - .name = "Bereqtime", - .VSL_arg = 'b', - .tag = SLT_Timestamp, - .prefix = "Bereq:", - .field = 3, - .hist_low = -6, - .hist_high = 3 - }, - { - .name = "Beresptime", - .VSL_arg = 'b', - .tag = SLT_Timestamp, - .prefix = "Beresp:", - .field = 3, - .hist_low = -6, - .hist_high = 3 - }, - { - .name = "BerespBodytime", - .VSL_arg = 'b', - .tag = SLT_Timestamp, - .prefix = "BerespBody:", - .field = 3, - .hist_low = -6, - .hist_high = 3 - }, - { - .name = "Besize", - .VSL_arg = 'b', - .tag = SLT_BereqAcct, - .prefix = NULL, - .field = 5, - .hist_low = 1, - .hist_high = 8 - }, - { - .name = NULL - } +#include "varnishhist_profiles.h" + { NULL } }; +#undef HIS_NO_PREFIX +#undef HIS_BACKEND +#undef HIS_CLIENT +#undef HIS_PROF static struct profile *active_profile; diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index 4b99e5e..912e6d3 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -38,7 +38,7 @@ ) #define HIS_OPT_p \ - VOPT("p:", "[-p period]", "Refresh period", \ + VOPT("p:", "[-p ]", "Refresh period", \ "Specified the number of seconds between screen refreshes." \ " Default is 1 second, and can be changed at runtime by" \ " pressing the [0-9] keys (powers of 2 in seconds" \ @@ -46,17 +46,12 @@ ) #define HIS_OPT_P \ - VOPT("P:", "[-P ]", \ - "Profile definition", \ - "Either a predefined profile or a custom definition. " \ - " Predefined profiles for the client side are" \ - " \"responsetime \" and \"size\". For the backend side:" \ - " \"Bereqtime\", \"Beresptime\", \"BerespBodytime\"," \ - " and \"Besize\"." \ - " custom definitions include an optional (c)lient or" \ - " (b)backend filter (defaults to client), the tag we'll" \ - " look for, and the field number of the value we are" \ - " interested in. min and max are the boundaries of the" \ + VOPT("P:", "[-P <[[cb:]tag:field_num:min:max>]", \ + "Custom profile definition", \ + " Graph the given custom definition defined as: an optional"\ + " (c)lient or (b)backend filter (defaults to client), the " \ + " tag we'll look for, and the field number of the value we" \ + " are interested in. min and max are the boundaries of the" \ " graph (these are power of ten)." \ ) @@ -78,6 +73,19 @@ VSL_OPT_L VUT_OPT_n VUT_OPT_N HIS_OPT_p +#define HIS_CLIENT "client" +#define HIS_BACKEND "backend" +#define HIS_NO_PREFIX "" +#define HIS_PROF(name,cb,tag,prefix,field,hist_low,high_high,doc) \ + VOPT("P:", "[-P " name "]", \ + "Predefined " cb " profile", \ + doc " (Field " #field " of " #tag " " prefix " log tags)" \ + ) +#include "varnishhist_profiles.h" +#undef HIS_NO_PREFIX +#undef HIS_BACKEND +#undef HIS_CLIENT +#undef HIS_PROF HIS_OPT_P VUT_OPT_q VUT_OPT_r diff --git a/bin/varnishhist/varnishhist_profiles.h b/bin/varnishhist/varnishhist_profiles.h new file mode 100644 index 0000000..fa72f9a --- /dev/null +++ b/bin/varnishhist/varnishhist_profiles.h @@ -0,0 +1,97 @@ +/*- + * Copyright 2016 UPLEX - Nils Goroll Systemoptimierung + * All rights reserved. + * + * Author: Nils Goroll + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Profile definitions for varnishhist + */ + +// client +HIS_PROF( + "responsetime", // name + HIS_CLIENT, // HIS_CLIENT | HIS_BACKEND + SLT_Timestamp, // tag + "Process:", // prefix + 3, // field + -6, // hist_low + 3, // hist_high + "Graph the total time from start of request processing" + " (first byte received) until ready to deliver the" + " client response." + ) +HIS_PROF( + "size", // name + HIS_CLIENT, // HIS_CLIENT | HIS_BACKEND + SLT_ReqAcct, // tag + HIS_NO_PREFIX, // prefix + 5, // field + 1, // hist_low + 8, // hist_high + "Graph the size of responses" + ) +// backend +HIS_PROF( + "Bereqtime", // name + HIS_BACKEND, // HIS_CLIENT | HIS_BACKEND + SLT_Timestamp, // tag + "Bereq:", // prefix + 3, // field + -6, // hist_low + 3, // hist_high + "Graph the time from beginning of backend processing" + " until a backend request is sent completely" + ) +HIS_PROF( + "Beresptime", // name + HIS_BACKEND, // HIS_CLIENT | HIS_BACKEND + SLT_Timestamp, // tag + "Beresp:", // prefix + 3, // field + -6, // hist_low + 3, // hist_high + "Graph the time from beginning of backend processing" + " until the response headers are being received completely" + ) +HIS_PROF( + "BerespBodytime", // name + HIS_BACKEND, // HIS_CLIENT | HIS_BACKEND + SLT_Timestamp, // tag + "BerespBody:", // prefix + 3, // field + -6, // hist_low + 3, // hist_high + "Graph the time from beginning of backend processing" + " until the response body has been received" + ) +HIS_PROF( + "Besize", // name + HIS_BACKEND, // HIS_CLIENT | HIS_BACKEND + SLT_BereqAcct, // tag + HIS_NO_PREFIX, // prefix + 5, // field + 1, // hist_low + 8, // hist_high + "Graph the backend response body size" + ) From hermunn at varnish-software.com Fri Sep 9 11:01:04 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:04 +0200 Subject: [4.1] 06fc521 Forgot to include client/backend profile info in the man page Message-ID: commit 06fc521eac6bb7eddb3d57e9a72f16b688ccf2cc Author: Nils Goroll Date: Mon Jul 25 15:20:51 2016 +0200 Forgot to include client/backend profile info in the man page diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index 912e6d3..04cc607 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -79,7 +79,8 @@ HIS_OPT_p #define HIS_PROF(name,cb,tag,prefix,field,hist_low,high_high,doc) \ VOPT("P:", "[-P " name "]", \ "Predefined " cb " profile", \ - doc " (Field " #field " of " #tag " " prefix " log tags)" \ + "Predefined " cb " profile: " doc \ + " (Field " #field " of " #tag " " prefix " log tags)" \ ) #include "varnishhist_profiles.h" #undef HIS_NO_PREFIX From hermunn at varnish-software.com Fri Sep 9 11:01:04 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:04 +0200 Subject: [4.1] 571cb5b try to help the release manager Message-ID: commit 571cb5b7a059594eb78a7d9ed2dbebfd159dd596 Author: Nils Goroll Date: Mon Jul 25 15:28:24 2016 +0200 try to help the release manager diff --git a/doc/changes.rst b/doc/changes.rst index 41649c6..fc21f89 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -10,6 +10,12 @@ Changes since 4.1.3: .. _2024: https://github.com/varnishcache/varnish-cache/issues/2024 .. _2011: https://github.com/varnishcache/varnish-cache/issues/2011 +* varnishhist improvements: + * can now generate statistics of backend reqeusts + * more predefined profiles + * time bend option: control the log processing speed + * y axis scale + ================================ Varnish Cache 4.1.3 (2016-07-06) ================================ From hermunn at varnish-software.com Fri Sep 9 11:01:04 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:04 +0200 Subject: [4.1] 7fd0a1e typo Message-ID: commit 7fd0a1ef91d7a176295c77ca4dda18c2da1086cc Author: Nils Goroll Date: Mon Jul 25 19:49:55 2016 +0200 typo diff --git a/doc/changes.rst b/doc/changes.rst index fc21f89..4fe1f39 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -11,7 +11,7 @@ Changes since 4.1.3: .. _2011: https://github.com/varnishcache/varnish-cache/issues/2011 * varnishhist improvements: - * can now generate statistics of backend reqeusts + * can now generate statistics of backend requests * more predefined profiles * time bend option: control the log processing speed * y axis scale From hermunn at varnish-software.com Fri Sep 9 11:01:04 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:04 +0200 Subject: [4.1] d7b7bcf put options in phonebook order again Message-ID: commit d7b7bcf0c7862eeccb297c5a6f4896e5e98e8c6a Author: Nils Goroll Date: Tue Jul 26 00:35:29 2016 +0200 put options in phonebook order again diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index 04cc607..bcebdfd 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -65,6 +65,7 @@ " > doubles." \ ) +HIS_OPT_B VSL_OPT_C VUT_OPT_d HIS_OPT_g @@ -93,4 +94,3 @@ VUT_OPT_r VUT_OPT_t VSL_OPT_T VUT_OPT_V -HIS_OPT_B From hermunn at varnish-software.com Fri Sep 9 11:01:04 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:04 +0200 Subject: [4.1] 0378d5e remove extra [ in the usage Message-ID: commit 0378d5ee15a68edb3312aee221d086480793510c Author: Nils Goroll Date: Tue Jul 26 00:40:20 2016 +0200 remove extra [ in the usage Thx @fgs diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index bcebdfd..682d1e1 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -46,7 +46,7 @@ ) #define HIS_OPT_P \ - VOPT("P:", "[-P <[[cb:]tag:field_num:min:max>]", \ + VOPT("P:", "[-P <[cb:]tag:field_num:min:max>]", \ "Custom profile definition", \ " Graph the given custom definition defined as: an optional"\ " (c)lient or (b)backend filter (defaults to client), the " \ From hermunn at varnish-software.com Fri Sep 9 11:01:04 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:04 +0200 Subject: [4.1] 5b424b8 more typos fixed Message-ID: commit 5b424b8aea71eee391f565710ef05aed7cab0f25 Author: Nils Goroll Date: Tue Jul 26 00:41:56 2016 +0200 more typos fixed Thx @fgs diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index 682d1e1..6b708b6 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -49,7 +49,7 @@ VOPT("P:", "[-P <[cb:]tag:field_num:min:max>]", \ "Custom profile definition", \ " Graph the given custom definition defined as: an optional"\ - " (c)lient or (b)backend filter (defaults to client), the " \ + " (c)lient or (b)ackend filter (defaults to client), the " \ " tag we'll look for, and the field number of the value we" \ " are interested in. min and max are the boundaries of the" \ " graph (these are power of ten)." \ @@ -58,7 +58,7 @@ #define HIS_OPT_B \ VOPT("B:", "[-B ]", \ "Time bending", \ - "Factor to bend time by. Particularly usedful when" \ + "Factor to bend time by. Particularly useful when" \ " [-r]eading from a vsl file. =1 process in near real" \ " time, <1 slow-motion, >1 time-lapse (useless unless" \ " reading from a file). At runtime, < halves and" \ From hermunn at varnish-software.com Fri Sep 9 11:01:04 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:04 +0200 Subject: [4.1] dec7588 Typo Message-ID: commit dec7588366b60c401bb2a68a911da533cb79a4b7 Author: Federico G. Schwindt Date: Mon Jul 25 23:36:43 2016 +0100 Typo diff --git a/bin/varnishncsa/varnishncsa_options.h b/bin/varnishncsa/varnishncsa_options.h index e47d696..98ab675 100644 --- a/bin/varnishncsa/varnishncsa_options.h +++ b/bin/varnishncsa/varnishncsa_options.h @@ -68,7 +68,7 @@ " backend requests will trigger log lines." \ ) #define NCSA_OPT_c \ - VOPT("c", "[-c]", "Cient mode", \ + VOPT("c", "[-c]", "Client mode", \ "Log client requests. This is the default. If -b is" \ " specified, then -c is needed to also log client requests" \ ) From hermunn at varnish-software.com Fri Sep 9 11:01:04 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:04 +0200 Subject: [4.1] f8d4c8e Polish Message-ID: commit f8d4c8e9cab2cec333c5bcd4857581b193428875 Author: Federico G. Schwindt Date: Tue Jul 26 00:00:42 2016 +0100 Polish diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index 6b708b6..055681b 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -32,7 +32,8 @@ #include "vut_options.h" #define HIS_OPT_g \ - VOPT("g:", "[-g ]", "Grouping mode (default: vxid)", \ + VOPT("g:", "[-g ]", \ + "Grouping mode (default: vxid)", \ "The grouping of the log records. The default is to group" \ " by vxid." \ ) @@ -42,14 +43,14 @@ "Specified the number of seconds between screen refreshes." \ " Default is 1 second, and can be changed at runtime by" \ " pressing the [0-9] keys (powers of 2 in seconds" \ - " or + and - (double/halve the speed)" \ + " or + and - (double/halve the speed)." \ ) #define HIS_OPT_P \ - VOPT("P:", "[-P <[cb:]tag:field_num:min:max>]", \ + VOPT("P:", "[-P <[cb:]tag:field_num:min:max>]", \ "Custom profile definition", \ - " Graph the given custom definition defined as: an optional"\ - " (c)lient or (b)ackend filter (defaults to client), the " \ + "Graph the given custom definition defined as: an optional" \ + " (c)lient or (b)ackend filter (defaults to client), the" \ " tag we'll look for, and the field number of the value we" \ " are interested in. min and max are the boundaries of the" \ " graph (these are power of ten)." \ @@ -81,7 +82,7 @@ HIS_OPT_p VOPT("P:", "[-P " name "]", \ "Predefined " cb " profile", \ "Predefined " cb " profile: " doc \ - " (Field " #field " of " #tag " " prefix " log tags)" \ + " (field " #field " of " #tag " " prefix " VSL tag)." \ ) #include "varnishhist_profiles.h" #undef HIS_NO_PREFIX diff --git a/bin/varnishhist/varnishhist_profiles.h b/bin/varnishhist/varnishhist_profiles.h index fa72f9a..68e109c 100644 --- a/bin/varnishhist/varnishhist_profiles.h +++ b/bin/varnishhist/varnishhist_profiles.h @@ -31,25 +31,25 @@ // client HIS_PROF( "responsetime", // name - HIS_CLIENT, // HIS_CLIENT | HIS_BACKEND + HIS_CLIENT, // HIS_CLIENT | HIS_BACKEND SLT_Timestamp, // tag - "Process:", // prefix + "Process:", // prefix 3, // field - -6, // hist_low + -6, // hist_low 3, // hist_high - "Graph the total time from start of request processing" + "graph the total time from start of request processing" " (first byte received) until ready to deliver the" - " client response." + " client response" ) HIS_PROF( "size", // name - HIS_CLIENT, // HIS_CLIENT | HIS_BACKEND + HIS_CLIENT, // HIS_CLIENT | HIS_BACKEND SLT_ReqAcct, // tag HIS_NO_PREFIX, // prefix 5, // field 1, // hist_low 8, // hist_high - "Graph the size of responses" + "graph the size of responses" ) // backend HIS_PROF( @@ -58,9 +58,9 @@ HIS_PROF( SLT_Timestamp, // tag "Bereq:", // prefix 3, // field - -6, // hist_low + -6, // hist_low 3, // hist_high - "Graph the time from beginning of backend processing" + "graph the time from beginning of backend processing" " until a backend request is sent completely" ) HIS_PROF( @@ -69,9 +69,9 @@ HIS_PROF( SLT_Timestamp, // tag "Beresp:", // prefix 3, // field - -6, // hist_low + -6, // hist_low 3, // hist_high - "Graph the time from beginning of backend processing" + "graph the time from beginning of backend processing" " until the response headers are being received completely" ) HIS_PROF( @@ -80,9 +80,9 @@ HIS_PROF( SLT_Timestamp, // tag "BerespBody:", // prefix 3, // field - -6, // hist_low + -6, // hist_low 3, // hist_high - "Graph the time from beginning of backend processing" + "graph the time from beginning of backend processing" " until the response body has been received" ) HIS_PROF( @@ -93,5 +93,5 @@ HIS_PROF( 5, // field 1, // hist_low 8, // hist_high - "Graph the backend response body size" + "graph the backend response body size" ) diff --git a/doc/sphinx/reference/varnishhist.rst b/doc/sphinx/reference/varnishhist.rst index 88043d3..aabb915 100644 --- a/doc/sphinx/reference/varnishhist.rst +++ b/doc/sphinx/reference/varnishhist.rst @@ -40,6 +40,7 @@ SEE ALSO * :ref:`varnishncsa(1)` * :ref:`varnishstat(1)` * :ref:`varnishtop(1)` +* :ref:`vsl(7)` HISTORY ======= From hermunn at varnish-software.com Fri Sep 9 11:01:04 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:04 +0200 Subject: [4.1] 8135bd0 Whitespace and format OCD Message-ID: commit 8135bd068da687c500306b9df5cb8917edfe4693 Author: Federico G. Schwindt Date: Thu Jul 28 15:05:05 2016 +0100 Whitespace and format OCD Also spring some VUT_Error() love. Conflicts: bin/varnishhist/varnishhist.c diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index d435e1d..f724bc7 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -56,8 +56,8 @@ #include "vut.h" #include "vtim.h" -#define HIST_N 2000 /* how far back we remember */ -#define HIST_RES 100 /* bucket resolution */ +#define HIST_N 2000 /* how far back we remember */ +#define HIST_RES 100 /* bucket resolution */ static const char progname[] = "varnishhist"; @@ -134,6 +134,7 @@ static struct profile *active_profile; static void update(void) { + char t[VTIM_FORMAT_SIZE]; unsigned w = COLS / hist_range; unsigned n = w * hist_range; unsigned bm[n], bh[n]; @@ -174,16 +175,15 @@ update(void) scale = scales[i]; if (vsl_t0 > 0) { - char t[VTIM_FORMAT_SIZE]; VTIM_format(vsl_ts, t); mvprintw(0, 0, "1:%d, n = %d, d = %g @ %s x %g", - scale, nhist, delay, t, timebend); - } else { - mvprintw(0, 0, "1:%d, n = %d, d = %g", scale, nhist, delay); - } + scale, nhist, delay, t, timebend); + } else + mvprintw(0, 0, "1:%d, n = %d, d = %g", + scale, nhist, delay); - for (j = 2; j < LINES - 3; j+=5) + for (j = 2; j < LINES - 3; j += 5) mvprintw(j, 0, "%d_", (LINES - 3 - j) * scale); /* show them */ @@ -218,7 +218,7 @@ upd_vsl_ts(const char *p) static int /*__match_proto__ (VSLQ_dispatch_f)*/ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], - void *priv) + void *priv) { int i, tag, skip, match, hit; unsigned u; @@ -242,7 +242,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], while (skip == 0) { i = VSL_Next(tr->c); if (i == -3) { - // overrun - need to skip forward + /* overrun - need to skip forward */ pthread_mutex_lock(&mtx); vsl_to = vsl_t0 = vsl_ts = 0; t0 = VTIM_mono(); @@ -252,7 +252,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], if (i != 1) break; - /* get the value we want, and register if it's a hit*/ + /* get the value we want and register if it's a hit */ tag = VSL_TAG(tr->c->rec.ptr); switch (tag) { @@ -261,9 +261,9 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], break; case SLT_VCL_return: if (!strcasecmp(VSL_CDATA(tr->c->rec.ptr), - "restart") || + "restart") || !strcasecmp(VSL_CDATA(tr->c->rec.ptr), - "retry")) + "retry")) skip = 1; break; case SLT_Timestamp: @@ -274,12 +274,12 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], if (active_profile->prefix && strncmp(VSL_CDATA(tr->c->rec.ptr), - active_profile->prefix, - strlen(active_profile->prefix)) != 0) + active_profile->prefix, + strlen(active_profile->prefix)) != 0) break; - i = sscanf(VSL_CDATA(tr->c->rec.ptr), format, - &value); + i = sscanf(VSL_CDATA(tr->c->rec.ptr), + format, &value); if (i != 1) break; match = 1; @@ -291,8 +291,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], continue; /* - * only parse the last tsp seen in this transaction - it should - * be the latest + * only parse the last tsp seen in this transaction - + * it should be the latest. */ upd_vsl_ts(tsp); @@ -356,7 +356,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], // Lck_CondWait ts.tv_nsec = (long)(modf(when, &t) * 1e9); ts.tv_sec = (long)t; - (void) pthread_cond_timedwait(&timebend_cv, &mtx, &ts); + (void)pthread_cond_timedwait(&timebend_cv, &mtx, &ts); } pthread_mutex_unlock(&mtx); @@ -396,17 +396,17 @@ do_curses(void *arg) erase(); break; #endif - case '\014': /* Ctrl-L */ - case '\024': /* Ctrl-T */ + case '\014': /* Ctrl-L */ + case '\024': /* Ctrl-T */ redrawwin(stdscr); refresh(); break; - case '\032': /* Ctrl-Z */ + case '\032': /* Ctrl-Z */ endwin(); raise(SIGTSTP); break; - case '\003': /* Ctrl-C */ - case '\021': /* Ctrl-Q */ + case '\003': /* Ctrl-C */ + case '\021': /* Ctrl-Q */ case 'Q': case 'q': raise(SIGINT); @@ -434,7 +434,7 @@ do_curses(void *arg) break; case '>': case '<': - // see below + /* see below */ break; default: beep(); @@ -476,7 +476,7 @@ static void profile_error(const char *s) { fprintf(stderr, "-P: '%s' is not a valid" - " profile name or definition\n", s); + " profile name or definition\n", s); exit(1); } @@ -501,14 +501,12 @@ main(int argc, char **argv) usage(0); case 'p': delay = strtod(optarg, NULL); - if (delay <= 0) { - fprintf(stderr, "-p: invalid '%s'\n", optarg); - exit(1); - } + if (delay <= 0) + VUT_Error(1, "-p: invalid '%s'", optarg); break; case 'P': colon = strchr(optarg, ':'); - /* no colon, take the profile as a name*/ + /* no colon, take the profile as a name */ if (colon == NULL) { profile = optarg; break; @@ -527,18 +525,15 @@ main(int argc, char **argv) } assert(colon); - if (sscanf(colon+1, "%d:%d:%d", &cli_p.field, - &cli_p.hist_low, &cli_p.hist_high) != 3) { + if (sscanf(colon + 1, "%d:%d:%d", &cli_p.field, + &cli_p.hist_low, &cli_p.hist_high) != 3) profile_error(optarg); - } match_tag = VSL_Name2Tag(ptag, colon - ptag); - if (match_tag < 0) { - fprintf(stderr, - "-P: '%s' is not a valid tag name\n", + if (match_tag < 0) + VUT_Error(1, + "-P: '%s' is not a valid tag name", optarg); - exit(1); - } cli_p.name = "custom"; cli_p.tag = match_tag; profile = NULL; @@ -547,20 +542,16 @@ main(int argc, char **argv) break; case 'B': timebend = strtod(optarg, NULL); - if (timebend == 0) { - fprintf(stderr, + if (timebend == 0) + VUT_Error(1, "-B: being able to bend time does not" " mean we can stop it" - " (invalid factor '%s')\n", optarg); - exit(1); - } - if (timebend < 0) { - fprintf(stderr, + " (invalid factor '%s')", optarg); + if (timebend < 0) + VUT_Error(1, "-B: being able to bend time does not" " mean we can make it go backwards" - " (invalid factor '%s')\n", optarg); - exit(1); - } + " (invalid factor '%s')", optarg); break; default: if (!VUT_Arg(i, optarg)) @@ -576,16 +567,13 @@ main(int argc, char **argv) if (profile) { for (active_profile = profiles; active_profile->name; - active_profile++) { - if (strcmp(active_profile->name, profile) == 0) { + active_profile++) { + if (strcmp(active_profile->name, profile) == 0) break; - } } } - if (! active_profile->name) { - fprintf(stderr, "-P: No such profile '%s'\n", profile); - exit(1); - } + if (!active_profile->name) + VUT_Error(1, "-P: No such profile '%s'", profile); assert(VUT_Arg(active_profile->VSL_arg, NULL)); match_tag = active_profile->tag; @@ -602,19 +590,15 @@ main(int argc, char **argv) t0 = VTIM_mono(); format = malloc(4 * fnum); - for (i = 0; i < fnum-1; i++) { - strcpy(format + 4*i, "%*s "); - } - strcpy(format + 4*(fnum-1), "%lf"); + for (i = 0; i < fnum - 1; i++) + strcpy(format + 4 * i, "%*s "); + strcpy(format + 4 * (fnum - 1), "%lf"); log_ten = log(10.0); VUT_Setup(); - if (pthread_create(&thr, NULL, do_curses, NULL) != 0) { - fprintf(stderr, "pthread_create(): %s\n", - strerror(errno)); - exit(1); - } + if (pthread_create(&thr, NULL, do_curses, NULL) != 0) + VUT_Error(1, "pthread_create(): %s", strerror(errno)); VUT.dispatch_f = &accumulate; VUT.dispatch_priv = NULL; VUT.sighup_f = sighup; From hermunn at varnish-software.com Fri Sep 9 11:01:04 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:04 +0200 Subject: [4.1] bf7fb97 Be consistent wrt vsl_ts locking Message-ID: commit bf7fb977c7df37c2ae7f4aac79afacc11b9c1124 Author: Martin Blix Grydeland Date: Mon Aug 8 14:37:04 2016 +0200 Be consistent wrt vsl_ts locking Fix an issue where the new timestamp was set without locking. Spotted by: Coverity diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index f724bc7..ea72614 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -290,12 +290,6 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], if (skip || !match) continue; - /* - * only parse the last tsp seen in this transaction - - * it should be the latest. - */ - upd_vsl_ts(tsp); - /* select bucket */ i = HIST_RES * (log(value) / log_ten); if (i < hist_low * HIST_RES) @@ -308,6 +302,12 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], pthread_mutex_lock(&mtx); + /* + * only parse the last tsp seen in this transaction - + * it should be the latest. + */ + upd_vsl_ts(tsp); + /* phase out old data */ if (nhist == HIST_N) { u = rr_hist[next_hist]; From hermunn at varnish-software.com Fri Sep 9 11:01:04 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:04 +0200 Subject: [4.1] 874294d Only update timestamp if we got a timestamp Message-ID: commit 874294d9414886eaeb795c549e0065966f41ac12 Author: Martin Blix Grydeland Date: Mon Aug 8 14:39:30 2016 +0200 Only update timestamp if we got a timestamp If a log transaction is parsed that don't contain timestamps (e.g. they are filtered in varnishd), then this would cause a NULL pointer dereference. diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index ea72614..f887c7a 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -306,7 +306,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], * only parse the last tsp seen in this transaction - * it should be the latest. */ - upd_vsl_ts(tsp); + if (tsp) + upd_vsl_ts(tsp); /* phase out old data */ if (nhist == HIST_N) { From hermunn at varnish-software.com Fri Sep 9 11:01:04 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:04 +0200 Subject: [4.1] bf7faed Revert "Add missing break to switch." Message-ID: commit bf7faedcbc66cac30da45212fc20ac0c2a16aa0b Author: Nils Goroll Date: Mon Aug 8 15:17:37 2016 +0200 Revert "Add missing break to switch." For match_tag == SLT_Timestamp, we'd miss a match otherwise. Add the missing /* FALLTHROUGH */ This reverts commit 97bfb00277c53e1197301fcfffd73479e5f72ccf. Conflicts: bin/varnishhist/varnishhist.c diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index f887c7a..f5b4f59 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -268,6 +268,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], break; case SLT_Timestamp: tsp = VSL_CDATA(tr->c->rec.ptr); + /* FALLTHROUGH */ default: if (tag != match_tag) break; From hermunn at varnish-software.com Fri Sep 9 11:01:05 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Fri, 09 Sep 2016 13:01:05 +0200 Subject: [4.1] b392d91 Check the pthread functions returns in varnishhist Message-ID: commit b392d910d4857db5ff5ef45e030b15ddfeaac686 Author: Guillaume Quintard Date: Wed Aug 31 14:00:49 2016 +0200 Check the pthread functions returns in varnishhist diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index f5b4f59..ec99a50 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -243,10 +243,10 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], i = VSL_Next(tr->c); if (i == -3) { /* overrun - need to skip forward */ - pthread_mutex_lock(&mtx); + AZ(pthread_mutex_lock(&mtx)); vsl_to = vsl_t0 = vsl_ts = 0; t0 = VTIM_mono(); - pthread_mutex_unlock(&mtx); + AZ(pthread_mutex_unlock(&mtx)); break; } if (i != 1) @@ -301,7 +301,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], assert(i >= 0); assert(i < hist_buckets); - pthread_mutex_lock(&mtx); + AZ(pthread_mutex_lock(&mtx)); /* * only parse the last tsp seen in this transaction - @@ -337,7 +337,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], if (++next_hist == HIST_N) { next_hist = 0; } - pthread_mutex_unlock(&mtx); + AZ(pthread_mutex_unlock(&mtx)); } if (vsl_ts < vsl_to) @@ -345,7 +345,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], t = VTIM_mono(); - pthread_mutex_lock(&mtx); + AZ(pthread_mutex_lock(&mtx)); if (vsl_t0 == 0) vsl_to = vsl_t0 = vsl_ts; @@ -360,7 +360,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], ts.tv_sec = (long)t; (void)pthread_cond_timedwait(&timebend_cv, &mtx, &ts); } - pthread_mutex_unlock(&mtx); + AZ(pthread_mutex_unlock(&mtx)); return (0); } @@ -385,9 +385,9 @@ do_curses(void *arg) curs_set(0); erase(); for (;;) { - pthread_mutex_lock(&mtx); + AZ(pthread_mutex_lock(&mtx)); update(); - pthread_mutex_unlock(&mtx); + AZ(pthread_mutex_unlock(&mtx)); timeout(delay * 1000); switch ((ch = getch())) { From nils.goroll at uplex.de Fri Sep 9 15:16:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 09 Sep 2016 17:16:12 +0200 Subject: [master] 2562b11 fix misnomer in comment Message-ID: commit 2562b110d8df44365b21af4600b3ffbd3b09ec40 Author: Nils Goroll Date: Fri Sep 9 16:11:11 2016 +0200 fix misnomer in comment diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index d1f001a..4b58fd7 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -380,7 +380,7 @@ vca_make_session(struct worker *wrk, void *arg) } /*-------------------------------------------------------------------- - * This function accepts on a single socket for a single session pool. + * This function accepts on a single socket for a single thread pool. * * As long as we can stick the accepted connection to another thread * we do so, otherwise we put the socket back on the "BACK" pool @@ -468,7 +468,7 @@ vca_accept_task(struct worker *wrk, void *arg) } /*-------------------------------------------------------------------- - * Called when a worker and attached session pool is created, to + * Called when a worker and attached thread pool is created, to * allocate the tasks which will listen to sockets for that pool. */ From nils.goroll at uplex.de Fri Sep 9 15:16:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 09 Sep 2016 17:16:12 +0200 Subject: [master] 080ce03 drop unimplemented parameter session_max Message-ID: commit 080ce03f238481e462622bcdb3f2dd30ecfae98f Author: Nils Goroll Date: Fri Sep 9 16:22:57 2016 +0200 drop unimplemented parameter session_max diff --git a/include/tbl/params.h b/include/tbl/params.h index 06ef16f..57bab3a 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -912,23 +912,6 @@ PARAM( ) PARAM( - /* name */ session_max, - /* typ */ uint, - /* min */ "1000", - /* max */ NULL, - /* default */ "100000", - /* units */ "sessions", - /* flags */ 0, - /* s-text */ - "Maximum number of sessions we will allocate from one pool before " - "just dropping connections.\n" - "This is mostly an anti-DoS measure, and setting it plenty high " - "should not hurt, as long as you have the memory for it.", - /* l-text */ "", - /* func */ NULL -) - -PARAM( /* name */ shm_reclen, /* typ */ vsl_reclen, /* min */ "16b", From nils.goroll at uplex.de Fri Sep 9 15:16:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 09 Sep 2016 17:16:12 +0200 Subject: [master] ca8bd53 improve thread_pools documentation Message-ID: commit ca8bd536e58589a208e9b855cdf726efd2a55675 Author: Nils Goroll Date: Fri Sep 9 16:31:03 2016 +0200 improve thread_pools documentation diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index 327a4c2..7f0e78f 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -84,11 +84,14 @@ struct parspec WRK_parspec[] = { "1", NULL, "Number of worker thread pools.\n" "\n" - "Increasing number of worker pools decreases lock " - "contention.\n" + "Increasing the number of worker pools decreases lock " + "contention. Each worker pool also has a thread accepting " + "new connections, so for very high rates of incoming new " + "connections on systems with many cores, increasing the " + "worker pools may be required.\n" "\n" - "Too many pools waste CPU and RAM resources, and more than " - "one pool for each CPU is probably detrimal to performance.\n" + "Too many pools waste CPU and RAM resources, and more than one " + "pool for each CPU is most likely detrimal to performance.\n" "\n" "Can be increased on the fly, but decreases require a " "restart to take effect.", diff --git a/include/tbl/params.h b/include/tbl/params.h index 57bab3a..475086c 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1170,10 +1170,14 @@ PARAM( /* s-text */ "Number of worker thread pools.\n" "\n" - "Increasing number of worker pools decreases lock contention.\n" + "Increasing the number of worker pools decreases lock " + "contention. Each worker pool also has a thread accepting " + "new connections, so for very high rates of incoming new " + "connections on systems with many cores, increasing the " + "worker pools may be required.\n" "\n" "Too many pools waste CPU and RAM resources, and more than one " - "pool for each CPU is probably detrimal to performance.\n" + "pool for each CPU is most likely detrimal to performance.\n" "\n" "Can be increased on the fly, but decreases require a restart to " "take effect.", From nils.goroll at uplex.de Fri Sep 9 15:16:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 09 Sep 2016 17:16:12 +0200 Subject: [master] 5e4377c leave some comments pointing to the right locations Message-ID: commit 5e4377cc2d5230fda03f57e8c955544559faedd0 Author: Nils Goroll Date: Fri Sep 9 17:06:52 2016 +0200 leave some comments pointing to the right locations Maybe it's just my brain that is incapable of considering the fact that a bit of macro-fillin may in fact be #if 0 - disabled here, so this should hopefully help me (and others?) to not spend more life time than absolutely necessary on feeling stupid. diff --git a/include/tbl/params.h b/include/tbl/params.h index 475086c..b2b3320 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -339,6 +339,8 @@ PARAM( ) #if 0 +/* actual location mgt_param_bits.c*/ +/* see tbl/debug_bits.h */ PARAM( /* name */ debug, /* typ */ debug, @@ -417,6 +419,8 @@ PARAM( ) #if 0 +/* actual location mgt_param_bits.c*/ +/* See tbl/feature_bits.h */ PARAM( /* name */ feature, /* typ */ feature, @@ -753,6 +757,7 @@ PARAM( ) #if 0 +/* actual location mgt_param_tbl.c */ PARAM( /* name */ pcre_match_limit, /* typ */ uint, @@ -768,6 +773,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tbl.c */ PARAM( /* name */ pcre_match_limit_recursion, /* typ */ uint, @@ -816,6 +822,7 @@ PARAM( ) #if 0 +/* actual location mgt_param_tbl.c */ PARAM( /* name */ pool_req, /* typ */ poolparam, @@ -831,6 +838,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tbl.c */ PARAM( /* name */ pool_sess, /* typ */ poolparam, @@ -846,6 +854,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tbl.c */ PARAM( /* name */ pool_vbo, /* typ */ poolparam, @@ -984,6 +993,7 @@ PARAM( ) #if 0 +/* actual location mgt_param_tcp.c */ PARAM( /* name */ tcp_keepalive_intvl, /* typ */ timeout, @@ -998,6 +1008,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tcp.c */ PARAM( /* name */ tcp_keepalive_probes, /* typ */ uint, @@ -1014,6 +1025,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tcp.c */ PARAM( /* name */ tcp_keepalive_time, /* typ */ timeout, @@ -1029,6 +1041,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_add_delay, /* typ */ timeout, @@ -1049,6 +1062,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_destroy_delay, /* typ */ timeout, @@ -1064,6 +1078,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_fail_delay, /* typ */ timeout, @@ -1090,6 +1105,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_max, /* typ */ thread_pool_max, @@ -1108,6 +1124,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_min, /* typ */ thread_pool_min, @@ -1126,6 +1143,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_stack, /* typ */ bytes, @@ -1142,6 +1160,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_timeout, /* typ */ timeout, @@ -1159,6 +1178,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pools, /* typ */ uint, @@ -1185,6 +1205,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_queue_limit, /* typ */ uint, @@ -1203,6 +1224,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_stats_rate, /* typ */ uint, @@ -1260,6 +1282,7 @@ PARAM( ) #if 0 +/* actual location mgt_param_tbl.c */ PARAM( /* name */ vcc_allow_inline_c, /* typ */ bool, @@ -1274,6 +1297,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tbl.c */ PARAM( /* name */ vcc_err_unref, /* typ */ bool, @@ -1288,6 +1312,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tbl.c */ PARAM( /* name */ vcc_unsafe_path, /* typ */ bool, @@ -1335,6 +1360,7 @@ PARAM( ) #if 0 +/* actual location mgt_param_tbl.c */ PARAM( /* name */ vcl_dir, /* typ */ string, @@ -1350,6 +1376,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tbl.c */ PARAM( /* name */ vmod_dir, /* typ */ string, @@ -1385,6 +1412,7 @@ PARAM( ) #if 0 +/* actual location mgt_param_bits.c*/ PARAM( /* name */ vsl_mask, /* typ */ vsl_mask, @@ -1453,6 +1481,7 @@ PARAM( ) #if 0 +/* see mgt_waiter.c */ PARAM( /* name */ waiter, /* typ */ waiter, From guillaume at varnish-software.com Fri Sep 9 15:57:52 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Fri, 9 Sep 2016 17:57:52 +0200 Subject: [master] ca8bd53 improve thread_pools documentation In-Reply-To: References: Message-ID: Typo : "is probably detrimMENTal" On Sep 9, 2016 17:17, "Nils Goroll" wrote: > > commit ca8bd536e58589a208e9b855cdf726efd2a55675 > Author: Nils Goroll > Date: Fri Sep 9 16:31:03 2016 +0200 > > improve thread_pools documentation > > diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c > index 327a4c2..7f0e78f 100644 > --- a/bin/varnishd/mgt/mgt_pool.c > +++ b/bin/varnishd/mgt/mgt_pool.c > @@ -84,11 +84,14 @@ struct parspec WRK_parspec[] = { > "1", NULL, > "Number of worker thread pools.\n" > "\n" > - "Increasing number of worker pools decreases lock " > - "contention.\n" > + "Increasing the number of worker pools decreases lock " > + "contention. Each worker pool also has a thread accepting " > + "new connections, so for very high rates of incoming new " > + "connections on systems with many cores, increasing the " > + "worker pools may be required.\n" > "\n" > - "Too many pools waste CPU and RAM resources, and more than > " > - "one pool for each CPU is probably detrimal to > performance.\n" > + "Too many pools waste CPU and RAM resources, and more than > one " > + "pool for each CPU is most likely detrimal to > performance.\n" > "\n" > "Can be increased on the fly, but decreases require a " > "restart to take effect.", > diff --git a/include/tbl/params.h b/include/tbl/params.h > index 57bab3a..475086c 100644 > --- a/include/tbl/params.h > +++ b/include/tbl/params.h > @@ -1170,10 +1170,14 @@ PARAM( > /* s-text */ > "Number of worker thread pools.\n" > "\n" > - "Increasing number of worker pools decreases lock contention.\n" > + "Increasing the number of worker pools decreases lock " > + "contention. Each worker pool also has a thread accepting " > + "new connections, so for very high rates of incoming new " > + "connections on systems with many cores, increasing the " > + "worker pools may be required.\n" > "\n" > "Too many pools waste CPU and RAM resources, and more than one " > - "pool for each CPU is probably detrimal to performance.\n" > + "pool for each CPU is most likely detrimal to performance.\n" > "\n" > "Can be increased on the fly, but decreases require a restart to " > "take effect.", > > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit > -------------- next part -------------- An HTML attachment was scrubbed... URL: From dridi at varni.sh Fri Sep 9 16:01:09 2016 From: dridi at varni.sh (Dridi Boukelmoune) Date: Fri, 9 Sep 2016 18:01:09 +0200 Subject: [master] ca8bd53 improve thread_pools documentation In-Reply-To: References: Message-ID: On Fri, Sep 9, 2016 at 5:57 PM, Guillaume Quintard wrote: > Typo : "is probably detrimMENTal" Isn't that one m too many? :p (good catch) From nils.goroll at uplex.de Fri Sep 9 16:03:13 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 09 Sep 2016 18:03:13 +0200 Subject: [master] 478c664 fix old typo Message-ID: commit 478c664fa33751df5011d80635d1ec359ff25507 Author: Nils Goroll Date: Fri Sep 9 18:01:36 2016 +0200 fix old typo thanks to @dridi for the dictionary lookup diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index 7f0e78f..1e75953 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -91,7 +91,7 @@ struct parspec WRK_parspec[] = { "worker pools may be required.\n" "\n" "Too many pools waste CPU and RAM resources, and more than one " - "pool for each CPU is most likely detrimal to performance.\n" + "pool for each CPU is most likely detrimental to performance.\n" "\n" "Can be increased on the fly, but decreases require a " "restart to take effect.", diff --git a/include/tbl/params.h b/include/tbl/params.h index b2b3320..71e2949 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1197,7 +1197,7 @@ PARAM( "worker pools may be required.\n" "\n" "Too many pools waste CPU and RAM resources, and more than one " - "pool for each CPU is most likely detrimal to performance.\n" + "pool for each CPU is most likely detrimental to performance.\n" "\n" "Can be increased on the fly, but decreases require a restart to " "take effect.", From phk at FreeBSD.org Fri Sep 9 16:05:13 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 09 Sep 2016 18:05:13 +0200 Subject: [master] 5972f08 Reallow '-' in VCL names by popular demand. Message-ID: commit 5972f08200989863bc0b062af215f75d621838fc Author: Poul-Henning Kamp Date: Fri Sep 9 16:03:31 2016 +0000 Reallow '-' in VCL names by popular demand. Pass childs status on failure Make atexit more robust diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 2a4f938..323f942 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -103,7 +103,10 @@ mcf_bad_vclname(struct cli *cli, const char *name) if (!vct_isalpha(*p)) bad = *p; for (p++; bad == 0 && *p != '\0'; p++) - if (!vct_isalpha(*p) && !vct_isdigit(*p) && *p != '_') + if (!vct_isalpha(*p) && + !vct_isdigit(*p) && + *p != '_' && + *p != '-') bad = *p; if (bad) { VCLI_SetResult(cli, CLIS_PARAM); @@ -213,6 +216,8 @@ mgt_vcl_del(struct vclprog *vp) char *p; CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); + while (!VTAILQ_EMPTY(&vp->dto)) + mgt_vcl_dep_del(VTAILQ_FIRST(&vp->dto)); while (!VTAILQ_EMPTY(&vp->dfrom)) mgt_vcl_dep_del(VTAILQ_FIRST(&vp->dfrom)); @@ -352,7 +357,7 @@ mgt_new_vcl(struct cli *cli, const char *vclname, const char *vclsrc, vp->name, vp->fname, vp->warm, vp->state)) { mgt_vcl_del(vp); VCLI_Out(cli, "%s", p); - VCLI_SetResult(cli, CLIS_PARAM); + VCLI_SetResult(cli, status); } free(p); } @@ -748,6 +753,7 @@ mgt_vcl_atexit(void) if (getpid() != mgt_pid) return; + active_vcl = NULL; do { vp = VTAILQ_FIRST(&vclhead); if (vp != NULL) diff --git a/bin/varnishtest/tests/v00048.vtc b/bin/varnishtest/tests/v00048.vtc index 25cf629..7f35402 100644 --- a/bin/varnishtest/tests/v00048.vtc +++ b/bin/varnishtest/tests/v00048.vtc @@ -73,16 +73,16 @@ varnish v1 -clierr 106 "vcl.discard foo" varnish v1 -stop varnish v1 -cliok "vcl.list" -varnish v1 -clierr 106 "vcl.label foo vcl0" -varnish v1 -cliok "vcl.label foo vcl1" -varnish v1 -clierr 300 "vcl.label bar foo" +varnish v1 -clierr 106 "vcl.label fo- vcl0" +varnish v1 -cliok "vcl.label fo- vcl1" +varnish v1 -clierr 300 "vcl.label bar fo-" varnish v1 -clierr 200 "vcl.state vcl1 warm" varnish v1 -clierr 200 "vcl.state vcl1 auto" varnish v1 -clierr 300 "vcl.state vcl1 cold" varnish v1 -clierr 300 "vcl.discard vcl1" varnish v1 -cliok "vcl.list" -varnish v1 -cliok "vcl.use foo" -varnish v1 -clierr 300 "vcl.discard foo" +varnish v1 -cliok "vcl.use fo-" +varnish v1 -clierr 300 "vcl.discard fo-" varnish v1 -cliok "vcl.list" server s1 -start @@ -91,8 +91,8 @@ varnish v1 -start client c1 -run varnish v1 -stop varnish v1 -cliok "vcl.use vcl1" -varnish v1 -cliok "vcl.discard foo" -varnish v1 -clierr 106 "vcl.discard foo" +varnish v1 -cliok "vcl.discard fo-" +varnish v1 -clierr 106 "vcl.discard fo-" varnish v1 -start varnish v1 -cliok "vcl.label snarf vcl1" From nils.goroll at uplex.de Fri Sep 9 16:05:31 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Sep 2016 18:05:31 +0200 Subject: [master] ca8bd53 improve thread_pools documentation In-Reply-To: References: Message-ID: sorry Guillaume, I mixed up the French and thanked Dridi in the last commit -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 473 bytes Desc: OpenPGP digital signature URL: From phk at FreeBSD.org Fri Sep 9 16:44:12 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 09 Sep 2016 18:44:12 +0200 Subject: [master] b1419a6 Don't insist on a particular status code from -errvcl Message-ID: commit b1419a63fee79f9cbb28fc7d602a42dce6a72a53 Author: Poul-Henning Kamp Date: Fri Sep 9 16:42:47 2016 +0000 Don't insist on a particular status code from -errvcl diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 47b7709..c0e9270 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -710,8 +710,7 @@ varnish_cli(struct varnish *v, const char *cli, unsigned exp) */ static void -varnish_vcl(struct varnish *v, const char *vcl, enum VCLI_status_e expect, - char **resp) +varnish_vcl(struct varnish *v, const char *vcl, int fail, char **resp) { struct vsb *vsb; enum VCLI_status_e u; @@ -728,14 +727,12 @@ varnish_vcl(struct varnish *v, const char *vcl, enum VCLI_status_e expect, AZ(VSB_finish(vsb)); u = varnish_ask_cli(v, VSB_data(vsb), resp); - if (u != expect) { + if (u == CLIS_OK && fail) { VSB_destroy(&vsb); vtc_log(v->vl, 0, - "VCL compilation got %u expected %u", - u, expect); + "VCL compilation successed expected failure"); return; - } - if (u == CLIS_OK) { + } else if (u == CLIS_OK) { VSB_clear(vsb); VSB_printf(vsb, "vcl.use vcl%d", v->vcl_nbr); AZ(VSB_finish(vsb)); @@ -1075,7 +1072,7 @@ cmd_varnish(CMD_ARGS) char *r = NULL; AN(av[1]); AN(av[2]); - varnish_vcl(v, av[2], CLIS_PARAM, &r); + varnish_vcl(v, av[2], 1, &r); if (strstr(r, av[1]) == NULL) vtc_log(v->vl, 0, "Did not find expected string: (\"%s\")", @@ -1124,7 +1121,7 @@ cmd_varnish(CMD_ARGS) } if (!strcmp(*av, "-vcl")) { AN(av[1]); - varnish_vcl(v, av[1], CLIS_OK, NULL); + varnish_vcl(v, av[1], 0, NULL); av++; continue; } From dridi.boukelmoune at gmail.com Fri Sep 9 16:50:16 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 09 Sep 2016 18:50:16 +0200 Subject: [master] 74c03d4 Clarify what makes a VCL warm or cold Message-ID: commit 74c03d48ffe8fc9b1cdf122ad7b13c15c53177c2 Author: Dridi Boukelmoune Date: Tue Aug 30 20:23:34 2016 +0200 Clarify what makes a VCL warm or cold diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index d174e75..e4daeea 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -53,6 +53,12 @@ static const char * const VCL_TEMP_BUSY = "busy"; static const char * const VCL_TEMP_COOLING = "cooling"; static const char * const VCL_TEMP_LABEL = "label"; +/* NB: The COOLING temperature is neither COLD nor WARM. And LABEL is not a + * temperature, it's a state. + */ +#define VCL_WARM(v) ((v)->temp == VCL_TEMP_WARM || (v)->temp == VCL_TEMP_BUSY) +#define VCL_COLD(v) ((v)->temp == VCL_TEMP_INIT || (v)->temp == VCL_TEMP_COLD) + struct vcl { unsigned magic; #define VCL_MAGIC 0x214188f2 @@ -176,7 +182,7 @@ vcl_get(struct vcl **vcc, struct vcl *vcl) { CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - assert(vcl->temp == VCL_TEMP_WARM); + assert(VCL_WARM(vcl)); Lck_Lock(&vcl_mtx); AN(vcl); if (vcl->label == NULL) @@ -204,7 +210,7 @@ void VCL_Refresh(struct vcl **vcc) { CHECK_OBJ_NOTNULL(vcl_active, VCL_MAGIC); - assert(vcl_active->temp == VCL_TEMP_WARM); + assert(VCL_WARM(vcl_active)); if (*vcc == vcl_active) return; if (*vcc != NULL) @@ -217,7 +223,7 @@ VCL_Ref(struct vcl *vcl) { CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - assert(vcl->temp != VCL_TEMP_INIT && vcl->temp != VCL_TEMP_COLD); + assert(!VCL_COLD(vcl)); Lck_Lock(&vcl_mtx); assert(vcl->busy > 0); vcl->busy++; @@ -260,7 +266,7 @@ VCL_AddBackend(struct vcl *vcl, struct backend *be) VTAILQ_INSERT_TAIL(&vcl->backend_list, be, vcl_list); Lck_Unlock(&vcl_mtx); - if (vcl->temp == VCL_TEMP_WARM || vcl->temp == VCL_TEMP_BUSY) + if (VCL_WARM(vcl)) /* Only when adding backend to already warm VCL */ VBE_Event(be, VCL_EVENT_WARM); else if (vcl->temp != VCL_TEMP_INIT) @@ -470,7 +476,7 @@ VRT_ref_vcl(VRT_CTX, const char *desc) vcl = ctx->vcl; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - xxxassert(vcl->temp == VCL_TEMP_WARM); + assert(VCL_WARM(vcl)); ALLOC_OBJ(ref, VCLREF_MAGIC); AN(ref); @@ -618,8 +624,7 @@ vcl_set_state(struct vrt_ctx *ctx, const char *state) switch(state[0]) { case '0': assert(vcl->temp != VCL_TEMP_COLD); - if (vcl->busy == 0 && (vcl->temp == VCL_TEMP_WARM || - vcl->temp == VCL_TEMP_BUSY)) { + if (vcl->busy == 0 && VCL_WARM(vcl)) { vcl->temp = VTAILQ_EMPTY(&vcl->ref_list) ? VCL_TEMP_COLD : VCL_TEMP_COOLING; From dridi.boukelmoune at gmail.com Fri Sep 9 16:50:16 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 09 Sep 2016 18:50:16 +0200 Subject: [master] 87ab209 Volatile read of the VCL temperature Message-ID: commit 87ab209018ac3b6f18d29ba27de78e34a47bf03b Author: Dridi Boukelmoune Date: Tue Aug 30 21:37:34 2016 +0200 Volatile read of the VCL temperature There is currently no locking around temperature changes despite the fact that it may be read outside of the CLI thread to make decisions. In the case of a panic, instead of locking we simply make sure not to perform an incomplete read of the pointer. diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index e4daeea..a7ad58b 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -127,7 +127,7 @@ VCL_Panic(struct vsb *vsb, const struct vcl *vcl) VSB_printf(vsb, "busy = %u\n", vcl->busy); VSB_printf(vsb, "discard = %u,\n", vcl->discard); VSB_printf(vsb, "state = %s,\n", vcl->state); - VSB_printf(vsb, "temp = %s,\n", vcl->temp); + VSB_printf(vsb, "temp = %s,\n", (const volatile char *)vcl->temp); VSB_printf(vsb, "conf = {\n"); VSB_indent(vsb, 2); if (vcl->conf == NULL) { From dridi.boukelmoune at gmail.com Fri Sep 9 16:50:16 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 09 Sep 2016 18:50:16 +0200 Subject: [master] c60f737 Add a read/write lock for VCL temperature Message-ID: commit c60f737975fbf11ed55ea893865a3595ef2bb2e8 Author: Dridi Boukelmoune Date: Tue Aug 30 20:31:34 2016 +0200 Add a read/write lock for VCL temperature diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index a7ad58b..298423f 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -34,6 +34,7 @@ #include "config.h" #include +#include #include #include @@ -70,6 +71,7 @@ struct vcl { unsigned busy; unsigned discard; const char *temp; + pthread_rwlock_t temp_rwl; VTAILQ_HEAD(,backend) backend_list; VTAILQ_HEAD(,vclref) ref_list; struct vcl *label; @@ -356,6 +358,7 @@ VCL_Open(const char *fn, struct vsb *msg) } ALLOC_OBJ(vcl, VCL_MAGIC); AN(vcl); + AZ(pthread_rwlock_init(&vcl->temp_rwl, NULL)); vcl->dlh = dlh; vcl->conf = cnf; return (vcl); @@ -370,6 +373,7 @@ VCL_Close(struct vcl **vclp) vcl = *vclp; *vclp = NULL; AZ(dlclose(vcl->dlh)); + AZ(pthread_rwlock_destroy(&vcl->temp_rwl)); FREE_OBJ(vcl); } From dridi.boukelmoune at gmail.com Fri Sep 9 16:50:16 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 09 Sep 2016 18:50:16 +0200 Subject: [master] 0ed12c4 Remove spurious assert on VCL release Message-ID: commit 0ed12c403912afa5b96571809e51f75ed0fa4a35 Author: Dridi Boukelmoune Date: Tue Aug 30 21:55:07 2016 +0200 Remove spurious assert on VCL release diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 298423f..e5bb70e 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -510,8 +510,12 @@ VRT_rel_vcl(VRT_CTX, struct vclref **refp) vcl = ctx->vcl; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); assert(vcl == ref->vcl); - assert(vcl->temp == VCL_TEMP_WARM || vcl->temp == VCL_TEMP_BUSY || - vcl->temp == VCL_TEMP_COOLING); + + /* NB: A VCL may be released by a VMOD at any time, but it must happen + * after a warmup and before the end of a cooldown. The release may or + * may not happen while the same thread holds the temperature lock, so + * instead we check that all references are gone in VCL_Nuke. + */ Lck_Lock(&vcl_mtx); assert(!VTAILQ_EMPTY(&vcl->ref_list)); From dridi.boukelmoune at gmail.com Fri Sep 9 16:50:16 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 09 Sep 2016 18:50:16 +0200 Subject: [master] 6f360c9 Non volatile reads/writes use the temperature lock Message-ID: commit 6f360c9f652a30a5964fef52ad9fe043498b5b05 Author: Dridi Boukelmoune Date: Tue Aug 30 22:02:46 2016 +0200 Non volatile reads/writes use the temperature lock Acquiring a read lock on the CLI thread is unnecessary since a write lock may only be acquired on that very thread. The locking order when both are required is: (struct vcl).temp_rwl => vcl_mtx Fixes #2008 diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index e5bb70e..85e3c5b 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -184,7 +184,9 @@ vcl_get(struct vcl **vcc, struct vcl *vcl) { CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + AZ(pthread_rwlock_rdlock(&vcl->temp_rwl)); assert(VCL_WARM(vcl)); + AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); Lck_Lock(&vcl_mtx); AN(vcl); if (vcl->label == NULL) @@ -212,7 +214,9 @@ void VCL_Refresh(struct vcl **vcc) { CHECK_OBJ_NOTNULL(vcl_active, VCL_MAGIC); + AZ(pthread_rwlock_rdlock(&vcl_active->temp_rwl)); assert(VCL_WARM(vcl_active)); + AZ(pthread_rwlock_unlock(&vcl_active->temp_rwl)); if (*vcc == vcl_active) return; if (*vcc != NULL) @@ -225,7 +229,9 @@ VCL_Ref(struct vcl *vcl) { CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + AZ(pthread_rwlock_rdlock(&vcl->temp_rwl)); assert(!VCL_COLD(vcl)); + AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); Lck_Lock(&vcl_mtx); assert(vcl->busy > 0); vcl->busy++; @@ -261,8 +267,11 @@ VCL_AddBackend(struct vcl *vcl, struct backend *be) CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); - if (vcl->temp == VCL_TEMP_COOLING) + AZ(pthread_rwlock_rdlock(&vcl->temp_rwl)); + if (vcl->temp == VCL_TEMP_COOLING) { + AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); return (1); + } Lck_Lock(&vcl_mtx); VTAILQ_INSERT_TAIL(&vcl->backend_list, be, vcl_list); @@ -273,6 +282,7 @@ VCL_AddBackend(struct vcl *vcl, struct backend *be) VBE_Event(be, VCL_EVENT_WARM); else if (vcl->temp != VCL_TEMP_INIT) WRONG("Dynamic Backends can only be added to warm VCLs"); + AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); return (0); } @@ -288,8 +298,11 @@ VCL_DelBackend(struct backend *be) Lck_Lock(&vcl_mtx); VTAILQ_REMOVE(&vcl->backend_list, be, vcl_list); Lck_Unlock(&vcl_mtx); - if (vcl->temp == VCL_TEMP_WARM) + + AZ(pthread_rwlock_rdlock(&vcl->temp_rwl)); + if (VCL_WARM(vcl)) VBE_Event(be, VCL_EVENT_COLD); + AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); } static void @@ -627,6 +640,7 @@ vcl_set_state(struct vrt_ctx *ctx, const char *state) assert(ctx->msg != NULL || *state == '0'); vcl = ctx->vcl; + AZ(pthread_rwlock_wrlock(&vcl->temp_rwl)); AN(vcl->temp); switch(state[0]) { @@ -668,6 +682,8 @@ vcl_set_state(struct vrt_ctx *ctx, const char *state) default: WRONG("Wrong enum state"); } + AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); + return (i); } From fgsch at lodoss.net Fri Sep 9 19:25:12 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 09 Sep 2016 21:25:12 +0200 Subject: [master] 86dcf63 Spelling Message-ID: commit 86dcf63bd9945446d92675f131124381d1e3a2f4 Author: Federico G. Schwindt Date: Fri Sep 9 20:20:20 2016 +0100 Spelling diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index c0e9270..ef01e96 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -730,7 +730,7 @@ varnish_vcl(struct varnish *v, const char *vcl, int fail, char **resp) if (u == CLIS_OK && fail) { VSB_destroy(&vsb); vtc_log(v->vl, 0, - "VCL compilation successed expected failure"); + "VCL compilation succeeded expected failure"); return; } else if (u == CLIS_OK) { VSB_clear(vsb); From fgsch at lodoss.net Fri Sep 9 19:25:12 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 09 Sep 2016 21:25:12 +0200 Subject: [master] 9f292e8 Minor cosmetic and more upcoming changes Message-ID: commit 9f292e81f4907bb72b6109534be2913c88d5b3d9 Author: Federico G. Schwindt Date: Fri Sep 9 20:20:33 2016 +0100 Minor cosmetic and more upcoming changes diff --git a/doc/changes.rst b/doc/changes.rst index 41d92bc..ce8a078 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -9,6 +9,8 @@ The list of changes are numerous and will not be expanded on in detail. Major items: * Experimental support for HTTP/2. * VCL labels, allowing for per-vhost VCL. +* Always send the request body to the backend, making possible to cache + e.g. POST. ====================================== @@ -733,7 +735,6 @@ Open issues .. _1381: https://www.varnish-cache.org/trac/ticket/1381 .. _1323: https://www.varnish-cache.org/trac/ticket/1323 .. _1268: https://www.varnish-cache.org/trac/ticket/1268 -.. _1268: https://www.varnish-cache.org/trac/ticket/1268 ============================================ @@ -989,7 +990,6 @@ varnishd .. _bug #897: https://www.varnish-cache.org/trac/ticket/897 .. _bug #1023: https://www.varnish-cache.org/trac/ticket/1023 .. _bug #1029: https://www.varnish-cache.org/trac/ticket/1029 -.. _bug #1023: https://www.varnish-cache.org/trac/ticket/1023 .. _bug #1035: https://www.varnish-cache.org/trac/ticket/1035 .. _bug #1037: https://www.varnish-cache.org/trac/ticket/1037 .. _bug #1038: https://www.varnish-cache.org/trac/ticket/1038 @@ -1176,7 +1176,7 @@ Changes from 3.0.0 to 3.0.1 rc 1 (2011-08-24) varnishd -------- -- Avoid sending an empty end-chunk when sending bodyless responsed. +- Avoid sending an empty end-chunk when sending bodyless responses. - `http_resp_hdr_len` and `http_req_hdr_len` were set to too low values leading to clients receiving `HTTP 400 Bad Request` errors. @@ -1253,7 +1253,7 @@ Changes from 3.0 beta 2 to 3.0.0 (2011-06-16) varnishd -------- -- Avoid sending an empty end-chunk when sending bodyless responsed. +- Avoid sending an empty end-chunk when sending bodyless responses. VCL --- @@ -1420,7 +1420,6 @@ varnishd .. _bug #663: https://www.varnish-cache.org/trac/ticket/663 .. _bug #880: https://www.varnish-cache.org/trac/ticket/880 .. _bug #411: https://www.varnish-cache.org/trac/ticket/411 -.. _bug #693: https://www.varnish-cache.org/trac/ticket/693 Tools ----- From phk at FreeBSD.org Fri Sep 9 20:58:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 09 Sep 2016 22:58:11 +0200 Subject: [master] ef9e5ff Go over the vcl-wrangling code and unspread the creation of the VRT_CTX used for VCL events. Message-ID: commit ef9e5fff328843529b7d065d573037a398550946 Author: Poul-Henning Kamp Date: Fri Sep 9 20:49:22 2016 +0000 Go over the vcl-wrangling code and unspread the creation of the VRT_CTX used for VCL events. diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 85e3c5b..0676943 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -95,6 +95,71 @@ static VTAILQ_HEAD(, vcl) vcl_head = static struct lock vcl_mtx; static struct vcl *vcl_active; /* protected by vcl_mtx */ +static struct vrt_ctx ctx_cli; +static unsigned handling_cli; +static struct ws ws_cli; +static char *ws_snapshot_cli; + +/*--------------------------------------------------------------------*/ + +static struct vrt_ctx * +vcl_get_ctx(unsigned method, int msg) +{ + AZ(ctx_cli.handling); + INIT_OBJ(&ctx_cli, VRT_CTX_MAGIC); + ctx_cli.handling = &handling_cli; + ctx_cli.method = method; + if (msg) { + ctx_cli.msg = VSB_new_auto(); + AN(ctx_cli.msg); + } + ctx_cli.ws = &ws_cli; + WS_Assert(ctx_cli.ws); + return (&ctx_cli); +} + +static void +vcl_rel_ctx(struct vrt_ctx **ctx) +{ + assert(*ctx == &ctx_cli); + AN((*ctx)->handling); + if (ctx_cli.msg) + VSB_destroy(&ctx_cli.msg); + WS_Assert(ctx_cli.ws); + WS_Reset(&ws_cli, ws_snapshot_cli); + INIT_OBJ(*ctx, VRT_CTX_MAGIC); + *ctx = NULL; +} + +/*--------------------------------------------------------------------*/ + +static int +vcl_send_event(VRT_CTX, enum vcl_event_e ev) +{ + int r; + + ASSERT_CLI(); + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); + CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); + assert(ev == VCL_EVENT_LOAD || + ev == VCL_EVENT_WARM || + ev == VCL_EVENT_COLD || + ev == VCL_EVENT_DISCARD); + AN(ctx->handling); + AN(ctx->ws); + + if (ev == VCL_EVENT_LOAD || ev == VCL_EVENT_WARM) + AN(ctx->msg); + + r = ctx->vcl->conf->event_vcl(ctx, ev); + + if (r && (ev == VCL_EVENT_COLD || ev == VCL_EVENT_DISCARD)) + WRONG("A VMOD cannot fail COLD or DISCARD events"); + + return (r); +} + /*--------------------------------------------------------------------*/ static struct vcl * @@ -540,73 +605,6 @@ VRT_rel_vcl(VRT_CTX, struct vclref **refp) } /*--------------------------------------------------------------------*/ - -/* - * the workspace allocated here exists for the lifetime of the varnishd process - * and is never explicitly freed - */ -static struct ws * -vcl_event_ws(void) -{ - static struct ws *ws = NULL; - static char *ws_snap; - - ASSERT_CLI(); - - if (ws == NULL) { - ws = malloc(sizeof(*ws)); - AN(ws); - WS_Init(ws, "cli", malloc(cache_param->workspace_client), - cache_param->workspace_client); - ws_snap = WS_Snapshot(ws); - } else { - WS_Reset(ws, ws_snap); - WS_Assert(ws); - } - return (ws); -} - -static int -vcl_setup_event(struct vrt_ctx *ctx, enum vcl_event_e ev) -{ - int r; - - ASSERT_CLI(); - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); - CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); - AN(ctx->handling); - AN(ctx->vcl); - AN(ctx->msg); - assert(ev == VCL_EVENT_LOAD || ev == VCL_EVENT_WARM); - AZ(ctx->ws); - ctx->ws = vcl_event_ws(); - r = ctx->vcl->conf->event_vcl(ctx, ev); - ctx->ws = NULL; - - return (r); -} - -static void -vcl_failsafe_event(struct vrt_ctx *ctx, enum vcl_event_e ev) -{ - - ASSERT_CLI(); - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); - CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); - AN(ctx->handling); - AN(ctx->vcl); - assert(ev == VCL_EVENT_COLD || ev == VCL_EVENT_DISCARD); - AZ(ctx->ws); - ctx->ws = vcl_event_ws(); - - if (ctx->vcl->conf->event_vcl(ctx, ev) != 0) - WRONG("A VMOD cannot fail COLD or DISCARD events"); - - ctx->ws = NULL; -} - static void vcl_print_refs(VRT_CTX) { @@ -625,7 +623,7 @@ vcl_print_refs(VRT_CTX) } static int -vcl_set_state(struct vrt_ctx *ctx, const char *state) +vcl_set_state(VRT_CTX, const char *state) { struct vcl *vcl; int i = 0; @@ -650,7 +648,7 @@ vcl_set_state(struct vrt_ctx *ctx, const char *state) vcl->temp = VTAILQ_EMPTY(&vcl->ref_list) ? VCL_TEMP_COLD : VCL_TEMP_COOLING; - vcl_failsafe_event(ctx, VCL_EVENT_COLD); + AZ(vcl_send_event(ctx, VCL_EVENT_COLD)); vcl_BackendEvent(vcl, VCL_EVENT_COLD); } else if (vcl->busy) @@ -672,7 +670,7 @@ vcl_set_state(struct vrt_ctx *ctx, const char *state) } else { vcl->temp = VCL_TEMP_WARM; - i = vcl_setup_event(ctx, VCL_EVENT_WARM); + i = vcl_send_event(ctx, VCL_EVENT_WARM); if (i == 0) vcl_BackendEvent(vcl, VCL_EVENT_WARM); else @@ -703,16 +701,13 @@ vcl_cancel_load(VRT_CTX, struct cli *cli, const char *name, const char *step) AZ(vcl->conf->event_vcl(ctx, VCL_EVENT_DISCARD)); vcl_KillBackends(vcl); VCL_Close(&vcl); - VSB_delete(ctx->msg); } static void -VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state) +VCL_Load(struct cli *cli, struct vrt_ctx *ctx, + const char *name, const char *fn, const char *state) { struct vcl *vcl; - struct vrt_ctx ctx; - unsigned hand = 0; - struct vsb *vsb; int i; ASSERT_CLI(); @@ -720,15 +715,11 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state) vcl = vcl_find(name); AZ(vcl); - vsb = VSB_new_auto(); - AN(vsb); - - vcl = VCL_Open(fn, vsb); + vcl = VCL_Open(fn, ctx->msg); if (vcl == NULL) { - AZ(VSB_finish(vsb)); + AZ(VSB_finish(ctx->msg)); VCLI_SetResult(cli, CLIS_PARAM); - VCLI_Out(cli, "%s", VSB_data(vsb)); - VSB_destroy(&vsb); + VCLI_Out(cli, "%s", VSB_data(ctx->msg)); return; } @@ -739,28 +730,23 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state) vcl->temp = VCL_TEMP_INIT; - INIT_OBJ(&ctx, VRT_CTX_MAGIC); - ctx.method = VCL_MET_INIT; - ctx.handling = &hand; - ctx.vcl = vcl; + ctx->vcl = vcl; - VSB_clear(vsb); - ctx.msg = vsb; - i = vcl_setup_event(&ctx, VCL_EVENT_LOAD); + VSB_clear(ctx->msg); + i = vcl_send_event(ctx, VCL_EVENT_LOAD); if (i) { - vcl_cancel_load(&ctx, cli, name, "initialization"); + vcl_cancel_load(ctx, cli, name, "initialization"); return; } - VSB_clear(vsb); - i = vcl_set_state(&ctx, state); + VSB_clear(ctx->msg); + i = vcl_set_state(ctx, state); if (i) { assert(*state == '1'); - vcl_cancel_load(&ctx, cli, name, "warmup"); + vcl_cancel_load(ctx, cli, name, "warmup"); return; } - VSB_destroy(&vsb); bprintf(vcl->state, "%s", state + 1); - assert(hand == VCL_RET_OK); + assert(*ctx->handling == VCL_RET_OK); VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name); VTAILQ_INSERT_TAIL(&vcl_head, vcl, list); Lck_Lock(&vcl_mtx); @@ -771,62 +757,45 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state) VSC_C_main->n_vcl_avail++; } -/*-------------------------------------------------------------------- - * This function is polled from the CLI thread to dispose of any non-busy - * VCLs which have been discarded. - */ - -static void -VCL_Nuke(struct vcl *vcl) -{ - struct vrt_ctx ctx; - unsigned hand = 0; - - INIT_OBJ(&ctx, VRT_CTX_MAGIC); - ASSERT_CLI(); - assert(vcl != vcl_active); - assert(vcl->discard); - AZ(vcl->busy); - assert(VTAILQ_EMPTY(&vcl->ref_list)); - VTAILQ_REMOVE(&vcl_head, vcl, list); - ctx.method = VCL_MET_FINI; - ctx.handling = &hand; - ctx.vcl = vcl; - vcl_failsafe_event(&ctx, VCL_EVENT_DISCARD); - vcl_KillBackends(vcl); - free(vcl->loaded_name); - VCL_Close(&vcl); - VSC_C_main->n_vcl--; - VSC_C_main->n_vcl_discard--; -} - /*--------------------------------------------------------------------*/ void VCL_Poll(void) { - struct vrt_ctx ctx; + struct vrt_ctx *ctx; struct vcl *vcl, *vcl2; - unsigned hand; ASSERT_CLI(); + ctx = vcl_get_ctx(0, 0); VTAILQ_FOREACH_SAFE(vcl, &vcl_head, list, vcl2) { if (vcl->temp == VCL_TEMP_BUSY || vcl->temp == VCL_TEMP_COOLING) { - INIT_OBJ(&ctx, VRT_CTX_MAGIC); - ctx.vcl = vcl; - ctx.handling = &hand; - (void)vcl_set_state(&ctx, "0"); + ctx->vcl = vcl; + ctx->method = 0; + (void)vcl_set_state(ctx, "0"); + } + if (vcl->discard && vcl->temp == VCL_TEMP_COLD) { + AZ(vcl->busy); + assert(vcl != vcl_active); + assert(VTAILQ_EMPTY(&vcl->ref_list)); + VTAILQ_REMOVE(&vcl_head, vcl, list); + ctx->method = VCL_MET_FINI; + ctx->vcl = vcl; + AZ(vcl_send_event(ctx, VCL_EVENT_DISCARD)); + vcl_KillBackends(vcl); + free(vcl->loaded_name); + VCL_Close(&vcl); + VSC_C_main->n_vcl--; + VSC_C_main->n_vcl_discard--; } - if (vcl->discard && vcl->temp == VCL_TEMP_COLD) - VCL_Nuke(vcl); } + vcl_rel_ctx(&ctx); } /*--------------------------------------------------------------------*/ static void __match_proto__(cli_func_t) -ccf_config_list(struct cli *cli, const char * const *av, void *priv) +vcl_cli_list(struct cli *cli, const char * const *av, void *priv) { struct vcl *vcl; const char *flg; @@ -853,48 +822,44 @@ ccf_config_list(struct cli *cli, const char * const *av, void *priv) } static void __match_proto__(cli_func_t) -ccf_config_load(struct cli *cli, const char * const *av, void *priv) +vcl_cli_load(struct cli *cli, const char * const *av, void *priv) { + struct vrt_ctx *ctx; AZ(priv); ASSERT_CLI(); - VCL_Load(cli, av[2], av[3], av[4]); + ctx = vcl_get_ctx(VCL_MET_INIT, 1); + VCL_Load(cli, ctx, av[2], av[3], av[4]); + vcl_rel_ctx(&ctx); } static void __match_proto__(cli_func_t) -ccf_config_state(struct cli *cli, const char * const *av, void *priv) +vcl_cli_state(struct cli *cli, const char * const *av, void *priv) { - struct vrt_ctx ctx; - unsigned hand; + struct vrt_ctx *ctx; - INIT_OBJ(&ctx, VRT_CTX_MAGIC); - ctx.msg = VSB_new_auto(); - AN(ctx.msg); - ctx.handling = &hand; - - (void)cli; AZ(priv); ASSERT_CLI(); AN(av[2]); AN(av[3]); - ctx.vcl = vcl_find(av[2]); - AN(ctx.vcl); // MGT ensures this - if (vcl_set_state(&ctx, av[3]) == 0) { - bprintf(ctx.vcl->state, "%s", av[3] + 1); - VSB_destroy(&ctx.msg); - return; + ctx = vcl_get_ctx(0, 1); + ctx->vcl = vcl_find(av[2]); + AN(ctx->vcl); // MGT ensures this + if (vcl_set_state(ctx, av[3]) == 0) { + bprintf(ctx->vcl->state, "%s", av[3] + 1); + } else { + AZ(VSB_finish(ctx->msg)); + VCLI_SetResult(cli, CLIS_CANT); + VCLI_Out(cli, "Failed ", ctx->vcl->loaded_name, + av[3] + 1); + if (VSB_len(ctx->msg)) + VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(ctx->msg)); } - AZ(VSB_finish(ctx.msg)); - VCLI_SetResult(cli, CLIS_CANT); - VCLI_Out(cli, "Failed ", ctx.vcl->loaded_name, - av[3] + 1); - if (VSB_len(ctx.msg)) - VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(ctx.msg)); - VSB_destroy(&ctx.msg); + vcl_rel_ctx(&ctx); } static void __match_proto__(cli_func_t) -ccf_config_discard(struct cli *cli, const char * const *av, void *priv) +vcl_cli_discard(struct cli *cli, const char * const *av, void *priv) { struct vcl *vcl; @@ -919,11 +884,11 @@ ccf_config_discard(struct cli *cli, const char * const *av, void *priv) VTAILQ_REMOVE(&vcl_head, vcl, list); free(vcl->loaded_name); } else if (vcl->temp == VCL_TEMP_COLD) - VCL_Nuke(vcl); + VCL_Poll(); } static void __match_proto__(cli_func_t) -ccf_config_label(struct cli *cli, const char * const *av, void *priv) +vcl_cli_label(struct cli *cli, const char * const *av, void *priv) { struct vcl *lbl; struct vcl *vcl; @@ -951,7 +916,7 @@ ccf_config_label(struct cli *cli, const char * const *av, void *priv) } static void __match_proto__(cli_func_t) -ccf_config_use(struct cli *cli, const char * const *av, void *priv) +vcl_cli_use(struct cli *cli, const char * const *av, void *priv) { struct vcl *vcl; @@ -967,7 +932,7 @@ ccf_config_use(struct cli *cli, const char * const *av, void *priv) } static void __match_proto__(cli_func_t) -ccf_config_show(struct cli *cli, const char * const *av, void *priv) +vcl_cli_show(struct cli *cli, const char * const *av, void *priv) { struct vcl *vcl; int verbose = 0; @@ -1093,13 +1058,13 @@ VCL_##func##_method(struct vcl *vcl, struct worker *wrk, \ /*--------------------------------------------------------------------*/ static struct cli_proto vcl_cmds[] = { - { CLICMD_VCL_LOAD, "", ccf_config_load }, - { CLICMD_VCL_LIST, "", ccf_config_list }, - { CLICMD_VCL_STATE, "", ccf_config_state }, - { CLICMD_VCL_DISCARD, "", ccf_config_discard }, - { CLICMD_VCL_USE, "", ccf_config_use }, - { CLICMD_VCL_SHOW, "", ccf_config_show }, - { CLICMD_VCL_LABEL, "", ccf_config_label }, + { CLICMD_VCL_LOAD, "", vcl_cli_load }, + { CLICMD_VCL_LIST, "", vcl_cli_list }, + { CLICMD_VCL_STATE, "", vcl_cli_state }, + { CLICMD_VCL_DISCARD, "", vcl_cli_discard }, + { CLICMD_VCL_USE, "", vcl_cli_use }, + { CLICMD_VCL_SHOW, "", vcl_cli_show }, + { CLICMD_VCL_LABEL, "", vcl_cli_label }, { NULL } }; @@ -1107,6 +1072,10 @@ void VCL_Init(void) { + assert(cache_param->workspace_client > 0); + WS_Init(&ws_cli, "cli", malloc(cache_param->workspace_client), + cache_param->workspace_client); + ws_snapshot_cli = WS_Snapshot(&ws_cli); CLI_AddFuncs(vcl_cmds); Lck_New(&vcl_mtx, lck_vcl); } diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 323f942..897fe1a 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -299,7 +299,7 @@ mgt_vcl_setstate(struct cli *cli, struct vclprog *vp, const char *vs) vp->name, vp->warm, vp->state); if (i) { AN(cli); - AN(vp->warm); + XXXAN(vp->warm); /* XXX: should restart child instead */ VCLI_SetResult(cli, status); VCLI_Out(cli, "%s", p); } diff --git a/include/vcli_serve.h b/include/vcli_serve.h index cfe4265..332432c 100644 --- a/include/vcli_serve.h +++ b/include/vcli_serve.h @@ -70,7 +70,7 @@ struct cli_proto { /* a CLI session */ struct cli { unsigned magic; -#define CLI_MAGIC 0x4038d570 +#define CLI_MAGIC 0x4038d570 struct vsb *sb; enum VCLI_status_e result; char *cmd; From phk at FreeBSD.org Sat Sep 10 08:21:10 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 10 Sep 2016 10:21:10 +0200 Subject: [master] 08675ee Fix sphinx syntax issue, slightly change emphasis Message-ID: commit 08675ee5d7dfd53e647460fc0e8d75af12bec969 Author: Poul-Henning Kamp Date: Sat Sep 10 08:20:21 2016 +0000 Fix sphinx syntax issue, slightly change emphasis diff --git a/doc/changes.rst b/doc/changes.rst index ce8a078..0fba41c 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -7,10 +7,13 @@ This is the first beta release of the upcoming 5.0 release. The list of changes are numerous and will not be expanded on in detail. Major items: -* Experimental support for HTTP/2. + * VCL labels, allowing for per-vhost VCL. + +* (Very!) experimental support for HTTP/2. + * Always send the request body to the backend, making possible to cache - e.g. POST. + POST, PUT, PATCH etc. ====================================== From phk at FreeBSD.org Sat Sep 10 08:41:08 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 10 Sep 2016 10:41:08 +0200 Subject: [master] 2679527 Polish Message-ID: commit 2679527ab59905049df1218133f7703376f79010 Author: Poul-Henning Kamp Date: Sat Sep 10 08:39:49 2016 +0000 Polish diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 0676943..57952cf 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2015 Varnish Software AS + * Copyright (c) 2006-2016 Varnish Software AS * All rights reserved. * * Author: Poul-Henning Kamp @@ -26,9 +26,6 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * Interface *to* compiled VCL code: Loading, unloading, calling into etc. - * - * The interface *from* the compiled VCL code is in cache_vrt.c. */ #include "config.h" @@ -54,8 +51,9 @@ static const char * const VCL_TEMP_BUSY = "busy"; static const char * const VCL_TEMP_COOLING = "cooling"; static const char * const VCL_TEMP_LABEL = "label"; -/* NB: The COOLING temperature is neither COLD nor WARM. And LABEL is not a - * temperature, it's a state. +/* + * NB: The COOLING temperature is neither COLD nor WARM. + * And LABEL is not a temperature, it's a state. */ #define VCL_WARM(v) ((v)->temp == VCL_TEMP_WARM || (v)->temp == VCL_TEMP_BUSY) #define VCL_COLD(v) ((v)->temp == VCL_TEMP_INIT || (v)->temp == VCL_TEMP_COLD) @@ -221,7 +219,9 @@ VCL_Return_Name(unsigned r) { switch (r) { -#define VCL_RET_MAC(l, U, B) case VCL_RET_##U: return(#l); +#define VCL_RET_MAC(l, U, B) \ + case VCL_RET_##U: \ + return(#l); #include "tbl/vcl_returns.h" #undef VCL_RET_MAC default: @@ -234,7 +234,9 @@ VCL_Method_Name(unsigned m) { switch (m) { -#define VCL_MET_MAC(func, upper, typ, bitmap) case VCL_MET_##upper: return (#upper); +#define VCL_MET_MAC(func, upper, typ, bitmap) \ + case VCL_MET_##upper: \ + return (#upper); #include "tbl/vcl_returns.h" #undef VCL_MET_MAC default: @@ -508,7 +510,9 @@ VCL_DefaultProbe(const struct vcl *vcl) return (vcl->conf->default_probe); } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * VRT apis relating to VCL's as VCLS. + */ void VRT_count(VRT_CTX, unsigned u) @@ -605,6 +609,7 @@ VRT_rel_vcl(VRT_CTX, struct vclref **refp) } /*--------------------------------------------------------------------*/ + static void vcl_print_refs(VRT_CTX) { @@ -704,7 +709,7 @@ vcl_cancel_load(VRT_CTX, struct cli *cli, const char *name, const char *step) } static void -VCL_Load(struct cli *cli, struct vrt_ctx *ctx, +vcl_load(struct cli *cli, struct vrt_ctx *ctx, const char *name, const char *fn, const char *state) { struct vcl *vcl; @@ -829,7 +834,7 @@ vcl_cli_load(struct cli *cli, const char * const *av, void *priv) AZ(priv); ASSERT_CLI(); ctx = vcl_get_ctx(VCL_MET_INIT, 1); - VCL_Load(cli, ctx, av[2], av[3], av[4]); + vcl_load(cli, ctx, av[2], av[3], av[4]); vcl_rel_ctx(&ctx); } From phk at FreeBSD.org Sat Sep 10 09:20:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 10 Sep 2016 11:20:11 +0200 Subject: [master] 6d4447c Add support for PRIV_TASK in CLI induced VCL events. Message-ID: commit 6d4447cedee482cfb6cb2019bb982250100007e4 Author: Poul-Henning Kamp Date: Sat Sep 10 09:18:58 2016 +0000 Add support for PRIV_TASK in CLI induced VCL events. See also: #2061 diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 57952cf..461931f 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -127,6 +127,7 @@ vcl_rel_ctx(struct vrt_ctx **ctx) WS_Reset(&ws_cli, ws_snapshot_cli); INIT_OBJ(*ctx, VRT_CTX_MAGIC); *ctx = NULL; + VRTPRIV_dynamic_kill(NULL, 0); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 1cfa281..44d95d3 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -48,23 +48,27 @@ struct vrt_priv { uintptr_t vmod_id; }; +struct vmod_priv cli_task_priv; + /*-------------------------------------------------------------------- */ void VRTPRIV_init(struct vrt_privs *privs) { - privs->magic = VRT_PRIVS_MAGIC; + + INIT_OBJ(privs, VRT_PRIVS_MAGIC); VTAILQ_INIT(&privs->privs); } static struct vmod_priv * -VRT_priv_dynamic(VRT_CTX, uintptr_t id, uintptr_t vmod_id) +vrt_priv_dynamic(VRT_CTX, uintptr_t id, uintptr_t vmod_id) { struct vrt_privs *vps; struct vrt_priv *vp; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(vmod_id); if (ctx->req) { CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(ctx->req->sp, SESS_MAGIC); @@ -94,7 +98,13 @@ VRTPRIV_dynamic_kill(struct vrt_privs *privs, uintptr_t id) { struct vrt_priv *vp, *vp1; + if (privs == NULL && id == 0) { + ASSERT_CLI(); + VRT_priv_fini(&cli_task_priv); + return; + } CHECK_OBJ_NOTNULL(privs, VRT_PRIVS_MAGIC); + AN(id); VTAILQ_FOREACH_SAFE(vp, &privs->privs, list, vp1) { CHECK_OBJ_NOTNULL(vp, VRT_PRIV_MAGIC); @@ -118,9 +128,11 @@ VRT_priv_task(VRT_CTX, void *vmod_id) } else if (ctx->bo) { CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); id = (uintptr_t)ctx->bo; - } else - WRONG("PRIV_TASK is only accessible in client or backend VCL contexts"); - return (VRT_priv_dynamic(ctx, id, (uintptr_t)vmod_id)); + } else { + ASSERT_CLI(); + return (&cli_task_priv); + } + return (vrt_priv_dynamic(ctx, id, (uintptr_t)vmod_id)); } struct vmod_priv * @@ -133,7 +145,7 @@ VRT_priv_top(VRT_CTX, void *vmod_id) CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(ctx->req->top, REQ_MAGIC); id = (uintptr_t)&ctx->req->top->top; - return (VRT_priv_dynamic(ctx, id, (uintptr_t)vmod_id)); + return (vrt_priv_dynamic(ctx, id, (uintptr_t)vmod_id)); } else WRONG("PRIV_TOP is only accessible in client VCL context"); NEEDLESS_RETURN(NULL); @@ -146,6 +158,6 @@ void VRT_priv_fini(const struct vmod_priv *p) { - if (p->priv != (void*)0 && p->free != (void*)0) + if (p->priv != NULL && p->free != NULL) p->free(p->priv); } From phk at FreeBSD.org Sat Sep 10 12:41:08 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 10 Sep 2016 14:41:08 +0200 Subject: [master] 80c5b90 Fix typo. Message-ID: commit 80c5b9087eee8b327edf63ca43538793a8a77934 Author: Poul-Henning Kamp Date: Sat Sep 10 12:40:10 2016 +0000 Fix typo. It's amazing how long time it can take to spot a bug like this :-( diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 897fe1a..612499f 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -179,7 +179,7 @@ mgt_vcl_dep_del(struct vcldep *vd) CHECK_OBJ_NOTNULL(vd, VCLDEP_MAGIC); VTAILQ_REMOVE(&vd->from->dfrom, vd, lfrom); - VTAILQ_REMOVE(&vd->to->dfrom, vd, lto); + VTAILQ_REMOVE(&vd->to->dto, vd, lto); FREE_OBJ(vd); } From phk at FreeBSD.org Sat Sep 10 13:15:10 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 10 Sep 2016 15:15:10 +0200 Subject: [master] a48962f Remove diagnostic check on active VCL, rather than increase cost by correctly holding the mtx while examining it. Message-ID: commit a48962fcf521d9e7587aaa6bc507f0d233512a07 Author: Poul-Henning Kamp Date: Sat Sep 10 13:14:15 2016 +0000 Remove diagnostic check on active VCL, rather than increase cost by correctly holding the mtx while examining it. diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 461931f..3cb2b69 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -282,9 +282,6 @@ void VCL_Refresh(struct vcl **vcc) { CHECK_OBJ_NOTNULL(vcl_active, VCL_MAGIC); - AZ(pthread_rwlock_rdlock(&vcl_active->temp_rwl)); - assert(VCL_WARM(vcl_active)); - AZ(pthread_rwlock_unlock(&vcl_active->temp_rwl)); if (*vcc == vcl_active) return; if (*vcc != NULL) From phk at FreeBSD.org Sat Sep 10 14:06:09 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 10 Sep 2016 16:06:09 +0200 Subject: [master] f367c79 Crude hack to try to get a clue to why rdlocks are unhappy on certain platforms Message-ID: commit f367c790f85402068ca02202732fb3ebc9f38f4d Author: Poul-Henning Kamp Date: Sat Sep 10 14:05:11 2016 +0000 Crude hack to try to get a clue to why rdlocks are unhappy on certain platforms diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 3cb2b69..621711c 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -30,6 +30,7 @@ #include "config.h" +#include #include #include #include @@ -252,9 +253,9 @@ vcl_get(struct vcl **vcc, struct vcl *vcl) { CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - AZ(pthread_rwlock_rdlock(&vcl->temp_rwl)); + AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); assert(VCL_WARM(vcl)); - AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); + AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); Lck_Lock(&vcl_mtx); AN(vcl); if (vcl->label == NULL) @@ -294,9 +295,9 @@ VCL_Ref(struct vcl *vcl) { CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - AZ(pthread_rwlock_rdlock(&vcl->temp_rwl)); + AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); assert(!VCL_COLD(vcl)); - AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); + AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); Lck_Lock(&vcl_mtx); assert(vcl->busy > 0); vcl->busy++; @@ -332,9 +333,9 @@ VCL_AddBackend(struct vcl *vcl, struct backend *be) CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); - AZ(pthread_rwlock_rdlock(&vcl->temp_rwl)); + AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); if (vcl->temp == VCL_TEMP_COOLING) { - AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); + AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); return (1); } @@ -347,7 +348,7 @@ VCL_AddBackend(struct vcl *vcl, struct backend *be) VBE_Event(be, VCL_EVENT_WARM); else if (vcl->temp != VCL_TEMP_INIT) WRONG("Dynamic Backends can only be added to warm VCLs"); - AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); + AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); return (0); } @@ -364,10 +365,10 @@ VCL_DelBackend(struct backend *be) VTAILQ_REMOVE(&vcl->backend_list, be, vcl_list); Lck_Unlock(&vcl_mtx); - AZ(pthread_rwlock_rdlock(&vcl->temp_rwl)); + AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl)); if (VCL_WARM(vcl)) VBE_Event(be, VCL_EVENT_COLD); - AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); + AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); } static void @@ -436,7 +437,7 @@ VCL_Open(const char *fn, struct vsb *msg) } ALLOC_OBJ(vcl, VCL_MAGIC); AN(vcl); - AZ(pthread_rwlock_init(&vcl->temp_rwl, NULL)); + AZ(errno=pthread_rwlock_init(&vcl->temp_rwl, NULL)); vcl->dlh = dlh; vcl->conf = cnf; return (vcl); @@ -451,7 +452,7 @@ VCL_Close(struct vcl **vclp) vcl = *vclp; *vclp = NULL; AZ(dlclose(vcl->dlh)); - AZ(pthread_rwlock_destroy(&vcl->temp_rwl)); + AZ(errno=pthread_rwlock_destroy(&vcl->temp_rwl)); FREE_OBJ(vcl); } @@ -641,7 +642,7 @@ vcl_set_state(VRT_CTX, const char *state) assert(ctx->msg != NULL || *state == '0'); vcl = ctx->vcl; - AZ(pthread_rwlock_wrlock(&vcl->temp_rwl)); + AZ(errno=pthread_rwlock_wrlock(&vcl->temp_rwl)); AN(vcl->temp); switch(state[0]) { @@ -683,7 +684,7 @@ vcl_set_state(VRT_CTX, const char *state) default: WRONG("Wrong enum state"); } - AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); + AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); return (i); } From fgsch at lodoss.net Sat Sep 10 19:11:08 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 10 Sep 2016 21:11:08 +0200 Subject: [master] 91363fe Make the includes relative to varnishd/ Message-ID: commit 91363fe94821c830837fecdd64e3e1b1cb241943 Author: Federico G. Schwindt Date: Sat Sep 10 19:31:28 2016 +0100 Make the includes relative to varnishd/ diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index f205e23..ee3d08f 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -40,11 +40,11 @@ #include #include -#include "../cache/cache.h" -#include "../cache/cache_filter.h" -#include "../cache/cache_transport.h" +#include "cache/cache.h" +#include "cache/cache_filter.h" +#include "cache/cache_transport.h" -#include "../http2/cache_http2.h" +#include "http2/cache_http2.h" #include "vct.h" #include "vend.h" diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 3626327..9717292 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -32,8 +32,8 @@ #include #include -#include "../cache/cache.h" -#include "../http2/cache_http2.h" +#include "cache/cache.h" +#include "http2/cache_http2.h" #include "vct.h" static int diff --git a/bin/varnishd/http2/cache_http2_panic.c b/bin/varnishd/http2/cache_http2_panic.c index 0605862..875f18a 100644 --- a/bin/varnishd/http2/cache_http2_panic.c +++ b/bin/varnishd/http2/cache_http2_panic.c @@ -40,10 +40,10 @@ #include #include -#include "../cache/cache.h" -#include "../cache/cache_filter.h" -#include "../cache/cache_transport.h" -#include "../http2/cache_http2.h" +#include "cache/cache.h" +#include "cache/cache_filter.h" +#include "cache/cache_transport.h" +#include "http2/cache_http2.h" #include "vend.h" #include "vsb.h" diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index f76263f..da64f9d 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -40,10 +40,10 @@ #include #include -#include "../cache/cache.h" -#include "../cache/cache_filter.h" -#include "../cache/cache_transport.h" -#include "../http2/cache_http2.h" +#include "cache/cache.h" +#include "cache/cache_filter.h" +#include "cache/cache_transport.h" +#include "http2/cache_http2.h" #include "vend.h" #include "vsb.h" diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index 644b276..0baec27 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -40,10 +40,10 @@ #include #include -#include "../cache/cache.h" -#include "../cache/cache_filter.h" -#include "../cache/cache_transport.h" -#include "../http2/cache_http2.h" +#include "cache/cache.h" +#include "cache/cache_filter.h" +#include "cache/cache_transport.h" +#include "http2/cache_http2.h" #include "vend.h" #include "vsb.h" diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index fafbd0a..264ad4c 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -38,8 +38,8 @@ #include #include -#include "../cache/cache.h" -#include "../cache/cache_transport.h" +#include "cache/cache.h" +#include "cache/cache_transport.h" #include "vend.h" #include "vsa.h" From fgsch at lodoss.net Sat Sep 10 19:11:08 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 10 Sep 2016 21:11:08 +0200 Subject: [master] 090bfa2 Skip tests with older nghttp versions Message-ID: commit 090bfa23a4ab99d92f569b6bf3142abeefc39d85 Author: Federico G. Schwindt Date: Sat Sep 10 19:53:30 2016 +0100 Skip tests with older nghttp versions These versions implement previous drafts that will fail the upgrade. diff --git a/bin/varnishtest/tests/a02022.vtc b/bin/varnishtest/tests/a02022.vtc index aeb1ff6..5e0e9b4 100644 --- a/bin/varnishtest/tests/a02022.vtc +++ b/bin/varnishtest/tests/a02022.vtc @@ -1,6 +1,6 @@ varnishtest "H/1 -> H/2 upgrade" -feature cmd "nghttp --help > /dev/null 2>&1" +feature cmd "nghttp --version | grep -q 'nghttp2/[1-9]'" server s1 { rxreq From fgsch at lodoss.net Sat Sep 10 19:11:08 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 10 Sep 2016 21:11:08 +0200 Subject: [master] c61d207 Ignore more files related to HTTP/2 Message-ID: commit c61d207aab7178d6cd07607a6c80961be5e26188 Author: Federico G. Schwindt Date: Sat Sep 10 19:55:28 2016 +0100 Ignore more files related to HTTP/2 diff --git a/.gitignore b/.gitignore index a4943d4..27b5ec7 100644 --- a/.gitignore +++ b/.gitignore @@ -105,6 +105,7 @@ cscope.*out /bin/varnishd/vhp_decode_test /bin/varnishd/vhp_gen_hufdec /bin/varnishd/vhp_table_test +/bin/varnishd/*.log /bin/varnishtest/tests/*.log /bin/varnishtest/tests/*.log-t /bin/varnishtest/test-suite.log From fgsch at lodoss.net Sat Sep 10 19:27:09 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 10 Sep 2016 21:27:09 +0200 Subject: [master] 2761c43 Let std.integer also do real to integer conversion Message-ID: commit 2761c4362cea5604ff3e27acb5cc1283e503269a Author: Federico G. Schwindt Date: Sat Sep 10 20:21:14 2016 +0100 Let std.integer also do real to integer conversion With input from nigoroll. diff --git a/bin/varnishtest/tests/m00016.vtc b/bin/varnishtest/tests/m00016.vtc index a216085..201682e 100644 --- a/bin/varnishtest/tests/m00016.vtc +++ b/bin/varnishtest/tests/m00016.vtc @@ -19,6 +19,9 @@ varnish v1 -vcl+backend { set resp.http.x-qux = std.real2integer( std.real(req.http.foo, 2.0), 2); + set resp.http.x-xyzzy = std.integer( + std.real(req.http.foo, 2.0), 2); + # Representation of 9e99, which is larger than what fits in the # 128bit integers on $exotic_platform. set resp.http.x-int-fallback = std.real2integer(9000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.000, 2); @@ -31,6 +34,7 @@ client c1 { expect resp.http.x-foo == resp.http.x-bar expect resp.http.x-baz == 1140618699.000 expect resp.http.x-qux == 1140618699 + expect resp.http.x-xyzzy == resp.http.x-qux expect resp.http.x-int-fallback == 2 } -run diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index 1506ec9..778fa6f 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -102,28 +102,23 @@ vmod_duration(VRT_CTX, VCL_STRING p, VCL_DURATION d) VCL_INT __match_proto__(td_std_integer) vmod_integer(VRT_CTX, VCL_STRING p, VCL_INT i) { - char *e; - long r; + const char *e; + double r; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (p == NULL) return (i); - while(isspace(*p)) - p++; - - if (*p != '+' && *p != '-' && !isdigit(*p)) + r = VNUMpfx(p, &e); + if (isnan(r) || e != NULL) return (i); - e = NULL; - - r = strtol(p, &e, 0); - - if (e == NULL || *e != '\0') + r = trunc(r); + if (r > LONG_MAX || r < LONG_MIN) return (i); - return (r); + return ((long)r); } VCL_IP From phk at FreeBSD.org Sat Sep 10 20:39:07 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 10 Sep 2016 22:39:07 +0200 Subject: [master] 1f80653 Allow multiple VCL labels to point to the same "real" VCL. Message-ID: commit 1f806539dc0b5c4780fe020355342c0266e50f3d Author: Poul-Henning Kamp Date: Sat Sep 10 20:07:55 2016 +0000 Allow multiple VCL labels to point to the same "real" VCL. diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 621711c..6995612 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -74,6 +74,7 @@ struct vcl { VTAILQ_HEAD(,backend) backend_list; VTAILQ_HEAD(,vclref) ref_list; struct vcl *label; + int nlabels; }; struct vclref { @@ -258,12 +259,13 @@ vcl_get(struct vcl **vcc, struct vcl *vcl) AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); Lck_Lock(&vcl_mtx); AN(vcl); - if (vcl->label == NULL) + if (vcl->label == NULL) { + AN(strcmp(vcl->state, VCL_TEMP_LABEL)); *vcc = vcl; - else if (strcmp(vcl->state, VCL_TEMP_LABEL)) - *vcc = vcl; - else + } else { + AZ(strcmp(vcl->state, VCL_TEMP_LABEL)); *vcc = vcl->label; + } AN(*vcc); AZ((*vcc)->discard); (*vcc)->busy++; @@ -804,6 +806,8 @@ vcl_cli_list(struct cli *cli, const char * const *av, void *priv) struct vcl *vcl; const char *flg; + /* NB: Shall generate same output as mcf_vcl_list() */ + (void)av; (void)priv; ASSERT_CLI(); @@ -816,11 +820,12 @@ vcl_cli_list(struct cli *cli, const char * const *av, void *priv) flg = "available"; VCLI_Out(cli, "%-10s %5s/%-8s %6u %s", flg, vcl->state, vcl->temp, vcl->busy, vcl->loaded_name); - if (vcl->label != NULL) { - VCLI_Out(cli, " %s %s", - strcmp(vcl->state, VCL_TEMP_LABEL) ? - "<-" : "->", vcl->label->loaded_name); - } + if (vcl->label != NULL) + VCLI_Out(cli, " -> %s", vcl->label->loaded_name); + else if (vcl->nlabels > 1) + VCLI_Out(cli, " (%d labels)", vcl->nlabels); + else if (vcl->nlabels > 0) + VCLI_Out(cli, " (%d label)", vcl->nlabels); VCLI_Out(cli, "\n"); } } @@ -874,12 +879,13 @@ vcl_cli_discard(struct cli *cli, const char * const *av, void *priv) AN(vcl); // MGT ensures this Lck_Lock(&vcl_mtx); assert (vcl != vcl_active); // MGT ensures this + AZ(vcl->nlabels); // MGT ensures this VSC_C_main->n_vcl_discard++; VSC_C_main->n_vcl_avail--; vcl->discard = 1; if (vcl->label != NULL) { AZ(strcmp(vcl->state, VCL_TEMP_LABEL)); - vcl->label->label = NULL; + vcl->label->nlabels--; vcl->label= NULL; } Lck_Unlock(&vcl_mtx); @@ -901,21 +907,20 @@ vcl_cli_label(struct cli *cli, const char * const *av, void *priv) (void)cli; (void)priv; vcl = vcl_find(av[3]); - AN(vcl); + AN(vcl); // MGT ensures this lbl = vcl_find(av[2]); if (lbl == NULL) { ALLOC_OBJ(lbl, VCL_MAGIC); AN(lbl); bprintf(lbl->state, "%s", VCL_TEMP_LABEL); lbl->temp = VCL_TEMP_WARM; - lbl->loaded_name = strdup(av[2]); - AN(lbl->loaded_name); + REPLACE(lbl->loaded_name, av[2]); VTAILQ_INSERT_TAIL(&vcl_head, lbl, list); } if (lbl->label != NULL) - lbl->label->label = NULL; + lbl->label->nlabels--; lbl->label = vcl; - vcl->label = lbl; + vcl->nlabels++; return; } diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 612499f..ea06216 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -73,6 +73,7 @@ struct vclprog { double go_cold; VTAILQ_HEAD(, vcldep) dfrom; VTAILQ_HEAD(, vcldep) dto; + int nto; }; static VTAILQ_HEAD(, vclprog) vclhead = VTAILQ_HEAD_INITIALIZER(vclhead); @@ -171,6 +172,7 @@ mgt_vcl_dep_add(struct vclprog *vp_from, struct vclprog *vp_to) VTAILQ_INSERT_TAIL(&vp_from->dfrom, vd, lfrom); vd->to = vp_to; VTAILQ_INSERT_TAIL(&vp_to->dto, vd, lto); + vp_to->nto++; } static void @@ -180,6 +182,7 @@ mgt_vcl_dep_del(struct vcldep *vd) CHECK_OBJ_NOTNULL(vd, VCLDEP_MAGIC); VTAILQ_REMOVE(&vd->from->dfrom, vd, lfrom); VTAILQ_REMOVE(&vd->to->dto, vd, lto); + vd->to->nto--; FREE_OBJ(vd); } @@ -614,27 +617,15 @@ mcf_vcl_discard(struct cli *cli, const char * const *av, void *priv) mgt_vcl_del(vp); } -static void -mcf_list_labels(struct cli *cli, const struct vclprog *vp) -{ - int n = 0; - struct vcldep *vd; - - VTAILQ_FOREACH(vd, &vp->dto, lto) { - if (n++ == 5) { - VCLI_Out(cli, " [...]"); - break; - } - VCLI_Out(cli, " %s", vd->from->name); - } -} - static void __match_proto__(cli_func_t) mcf_vcl_list(struct cli *cli, const char * const *av, void *priv) { unsigned status; char *p; struct vclprog *vp; + struct vcldep *vd; + + /* NB: Shall generate same output as vcl_cli_list() */ (void)av; (void)priv; @@ -652,7 +643,15 @@ mcf_vcl_list(struct cli *cli, const char * const *av, void *priv) VCLI_Out(cli, "/%-8s", vp->warm ? VCL_STATE_WARM : VCL_STATE_COLD); VCLI_Out(cli, " %6s %s", "", vp->name); - mcf_list_labels(cli, vp); + if (mcf_is_label(vp)) { + vd = VTAILQ_FIRST(&vp->dfrom); + AN(vd); + VCLI_Out(cli, " -> %s", vd->to->name); + } else if (vp->nto > 1) { + VCLI_Out(cli, " (%d labels)", vp->nto); + } else if (vp->nto > 0) { + VCLI_Out(cli, " (%d label)", vp->nto); + } VCLI_Out(cli, "\n"); } } @@ -680,13 +679,6 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) VCLI_Out(cli, "VCL labels cannot point to labels"); return; } - if (!VTAILQ_EMPTY(&vpt->dto)) { - VCLI_SetResult(cli, CLIS_CANT); - VCLI_Out(cli, "VCL already labeled with"); - mcf_list_labels(cli, vpt); - VCLI_Out(cli, "\n"); - return; - } vpl = mcf_vcl_byname(av[2]); if (vpl != NULL) { if (!mcf_is_label(vpl)) { From phk at FreeBSD.org Sat Sep 10 20:39:08 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 10 Sep 2016 22:39:08 +0200 Subject: [master] 1eed4bb Add multi-label tests Message-ID: commit 1eed4bb724e55943f7affaa295fb71f67343de9f Author: Poul-Henning Kamp Date: Sat Sep 10 20:31:22 2016 +0000 Add multi-label tests diff --git a/bin/varnishtest/tests/v00048.vtc b/bin/varnishtest/tests/v00048.vtc index 7f35402..68dedcf 100644 --- a/bin/varnishtest/tests/v00048.vtc +++ b/bin/varnishtest/tests/v00048.vtc @@ -37,8 +37,10 @@ client c1 { varnish v1 -cliok "vcl.list" varnish v1 -clierr 106 "vcl.label foo vcl0" varnish v1 -cliok "vcl.label foo vcl2" -varnish v1 -clierr 300 "vcl.label bar vcl2" +varnish v1 -cliok "vcl.label bar vcl2" +varnish v1 -cliok "vcl.list" varnish v1 -clierr 300 "vcl.discard vcl2" +varnish v1 -cliok "vcl.discard bar" varnish v1 -cliok "vcl.label foo vcl1" varnish v1 -clierr 106 "vcl.label vcl1 vcl2" varnish v1 -clierr 106 "vcl.state foo cold" @@ -94,7 +96,13 @@ varnish v1 -cliok "vcl.use vcl1" varnish v1 -cliok "vcl.discard fo-" varnish v1 -clierr 106 "vcl.discard fo-" +varnish v1 -cliok "vcl.label label1 vcl1" +varnish v1 -cliok "vcl.label label2 vcl1" +varnish v1 -cliok "vcl.label label3 vcl1" +varnish v1 -cliok "vcl.list" + varnish v1 -start +varnish v1 -cliok "vcl.list" varnish v1 -cliok "vcl.label snarf vcl1" server s1 -start client c1 -run From fgsch at lodoss.net Sun Sep 11 02:34:08 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 11 Sep 2016 04:34:08 +0200 Subject: [master] 85ad7ec Clarify doc regarding -p with -1 Message-ID: commit 85ad7ecf42c53ccfe36075ed7507465590b8d180 Author: Federico G. Schwindt Date: Sun Sep 11 03:30:11 2016 +0100 Clarify doc regarding -p with -1 Related to #2025. diff --git a/bin/varnishtop/varnishtop_options.h b/bin/varnishtop/varnishtop_options.h index f87a5f4..b65a961 100644 --- a/bin/varnishtop/varnishtop_options.h +++ b/bin/varnishtop/varnishtop_options.h @@ -50,6 +50,7 @@ " default is 60 seconds. The first number in the list is" \ " the average number of requests seen over this time" \ " period." \ + " This option has no effect if -1 option is also used." \ ) TOP_OPT_1 From fgsch at lodoss.net Sun Sep 11 22:06:09 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 12 Sep 2016 00:06:09 +0200 Subject: [master] 7427860 Avoid unneeded conversions Message-ID: commit 7427860ff11cd789b40ff39c0671ca4fbb28d19b Author: Federico G. Schwindt Date: Sun Sep 11 18:14:32 2016 +0100 Avoid unneeded conversions diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 9624af2..573d497 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -377,6 +377,7 @@ vcc_expr_tostring(struct vcc *tl, struct expr **e, vcc_type_t fmt) CHECK_OBJ_NOTNULL(*e, EXPR_MAGIC); AN(fmt == STRING || fmt == STRING_LIST); + AZ(fmt == (*e)->fmt); p = (*e)->fmt->tostring; if (p == NULL && (*e)->fmt == BLOB) { @@ -1300,7 +1301,7 @@ vcc_expr0(struct vcc *tl, struct expr **e, vcc_type_t fmt) } *e = vcc_expr_edit(BOOL, "\v1\v-\n)", *e, NULL); } - if (fmt == STRING || fmt == STRING_LIST) { + if (fmt != (*e)->fmt && (fmt == STRING || fmt == STRING_LIST)) { vcc_expr_tostring(tl, e, fmt); ERRCHK(tl); } From fgsch at lodoss.net Sun Sep 11 22:06:09 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 12 Sep 2016 00:06:09 +0200 Subject: [master] c9f721b Don't treat sub none specially Message-ID: commit c9f721bbbdd454b4ffbc3fa4c73b5ad3fd1e5e25 Author: Federico G. Schwindt Date: Sun Sep 11 20:30:09 2016 +0100 Don't treat sub none specially diff --git a/bin/varnishtest/tests/v00021.vtc b/bin/varnishtest/tests/v00021.vtc index 86ce100..664a836 100644 --- a/bin/varnishtest/tests/v00021.vtc +++ b/bin/varnishtest/tests/v00021.vtc @@ -40,7 +40,6 @@ varnish v1 -errvcl "Function recurses on" { sub vcl_recv { call foo; } } - varnish v1 -errvcl {Unused acl foo, defined:} { backend b { .host = "127.0.0.1"; } @@ -54,6 +53,14 @@ varnish v1 -errvcl {Unused sub foo, defined:} { sub foo { } } + +varnish v1 -errvcl {Unused sub none, defined:} { + backend b { .host = "127.0.0.1"; } + + sub none { + } +} + varnish v1 -errvcl {Invalid return "deliver"} { backend b { .host = "127.0.0.1"; } diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 461ac56..32f245a 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -133,12 +133,12 @@ New_IniFin(struct vcc *tl) int IsMethod(const struct token *t) { - struct method *m; + int i; assert(t->tok == ID); - for(m = method_tab; m->name != NULL; m++) { - if (vcc_IdIs(t, m->name)) - return (m - method_tab); + for (i = 1; method_tab[i].name != NULL; i++) { + if (vcc_IdIs(t, method_tab[i].name)) + return (i); } if ((t->b[0] == 'v'|| t->b[0] == 'V') && (t->b[1] == 'c'|| t->b[1] == 'C') && diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c index c5caa62..64ac0d9 100644 --- a/lib/libvcc/vcc_parse.c +++ b/lib/libvcc/vcc_parse.c @@ -220,7 +220,7 @@ vcc_ParseFunction(struct vcc *tl) "VCL sub's named 'vcl*' are reserved names.\n"); vcc_ErrWhere(tl, tl->t); VSB_printf(tl->sb, "Valid vcl_* methods are:\n"); - for (i = 0; method_tab[i].name != NULL; i++) + for (i = 1; method_tab[i].name != NULL; i++) VSB_printf(tl->sb, "\t%s\n", method_tab[i].name); return; } else if (m != -1) { From phk at FreeBSD.org Mon Sep 12 04:03:09 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Sep 2016 06:03:09 +0200 Subject: [master] 174ca62 Also initialize the temp_rwl for labels Message-ID: commit 174ca62a33a42d9d839a46233cae26d5c6c8c8ea Author: Poul-Henning Kamp Date: Mon Sep 12 03:54:46 2016 +0000 Also initialize the temp_rwl for labels diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 6995612..f0f6897 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -272,15 +272,6 @@ vcl_get(struct vcl **vcc, struct vcl *vcl) Lck_Unlock(&vcl_mtx); } -static void -vcl_get_active(struct vcl **vcc) -{ - while (vcl_active == NULL) - (void)usleep(100000); - - vcl_get(vcc, vcl_active); -} - void VCL_Refresh(struct vcl **vcc) { @@ -289,7 +280,11 @@ VCL_Refresh(struct vcl **vcc) return; if (*vcc != NULL) VCL_Rel(vcc); /* XXX: optimize locking */ - vcl_get_active(vcc); + + while (vcl_active == NULL) + (void)usleep(100000); + + vcl_get(vcc, vcl_active); } void @@ -915,6 +910,7 @@ vcl_cli_label(struct cli *cli, const char * const *av, void *priv) bprintf(lbl->state, "%s", VCL_TEMP_LABEL); lbl->temp = VCL_TEMP_WARM; REPLACE(lbl->loaded_name, av[2]); + AZ(errno=pthread_rwlock_init(&vcl->temp_rwl, NULL)); VTAILQ_INSERT_TAIL(&vcl_head, lbl, list); } if (lbl->label != NULL) From phk at FreeBSD.org Mon Sep 12 04:18:10 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Sep 2016 06:18:10 +0200 Subject: [master] 7b60fe7 Init the correct rwlock Message-ID: commit 7b60fe7dbff8f5cd0b52ea55e9dafbde51a331e1 Author: Poul-Henning Kamp Date: Mon Sep 12 04:17:38 2016 +0000 Init the correct rwlock diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index f0f6897..c214686 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -910,14 +910,13 @@ vcl_cli_label(struct cli *cli, const char * const *av, void *priv) bprintf(lbl->state, "%s", VCL_TEMP_LABEL); lbl->temp = VCL_TEMP_WARM; REPLACE(lbl->loaded_name, av[2]); - AZ(errno=pthread_rwlock_init(&vcl->temp_rwl, NULL)); + AZ(errno=pthread_rwlock_init(&lbl->temp_rwl, NULL)); VTAILQ_INSERT_TAIL(&vcl_head, lbl, list); } if (lbl->label != NULL) lbl->label->nlabels--; lbl->label = vcl; vcl->nlabels++; - return; } static void __match_proto__(cli_func_t) From phk at FreeBSD.org Mon Sep 12 05:03:09 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Sep 2016 07:03:09 +0200 Subject: [master] 873e75b Call it 'vhosts' instead of 'domains'. Message-ID: commit 873e75b23e1ec943da00fc5b64b5c1b8dfc83e01 Author: Poul-Henning Kamp Date: Mon Sep 12 05:02:19 2016 +0000 Call it 'vhosts' instead of 'domains'. diff --git a/doc/sphinx/users-guide/vcl-separate.rst b/doc/sphinx/users-guide/vcl-separate.rst index f876ea9..67c9727 100644 --- a/doc/sphinx/users-guide/vcl-separate.rst +++ b/doc/sphinx/users-guide/vcl-separate.rst @@ -3,9 +3,9 @@ Separate VCL files ================== -Having multiple different domains in the same Varnish is a very +Having multiple different vhosts in the same Varnish is a very typical use-case, and from Varnish 5.0 it is possible to have -a separate VCL files for separate domains or any other distinct +a separate VCL files for separate vhosts or any other distinct subset of requests. Assume that we want to handle ``varnish.org`` with one VCL file From phk at FreeBSD.org Mon Sep 12 06:17:09 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Sep 2016 08:17:09 +0200 Subject: [master] 3ed6c35 Minor polish while here anyway Message-ID: commit 3ed6c35b63b9684accce2ce5fe6d156bc9d679e4 Author: Poul-Henning Kamp Date: Mon Sep 12 06:16:27 2016 +0000 Minor polish while here anyway diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 1877424..590372d 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -161,7 +161,7 @@ void mgt_SHM_Init(void); void mgt_SHM_static_alloc(const void *, ssize_t size, const char *class, const char *type, const char *ident); void mgt_SHM_Create(void); -void mgt_SHM_Commit(void); +int mgt_SHM_Commit(void); void mgt_SHM_Destroy(int keep); void mgt_SHM_Size_Adjust(void); diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 4508117..077363d 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -764,7 +764,9 @@ MGT_Run(void) } } - mgt_SHM_Commit(); + i = mgt_SHM_Commit(); + if (i != 0) + MGT_complain(C_ERR, "Could not commit SHM file"); i = vev_schedule(mgt_evb); if (i != 0) diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c index fe6a638..d760066 100644 --- a/bin/varnishd/mgt/mgt_shmem.c +++ b/bin/varnishd/mgt/mgt_shmem.c @@ -251,20 +251,22 @@ mgt_SHM_Create(void) * Commit the VSM */ -void +int mgt_SHM_Commit(void) { char fnbuf[64]; + int retval = 0; bprintf(fnbuf, "%s.%jd", VSM_FILENAME, (intmax_t)getpid()); VJ_master(JAIL_MASTER_FILE); if (rename(fnbuf, VSM_FILENAME)) { - fprintf(stderr, "Rename failed %s -> %s: %s\n", + MGT_complain(C_ERR, "Rename failed %s -> %s: %s\n", fnbuf, VSM_FILENAME, strerror(errno)); (void)unlink(fnbuf); - exit(1); + retval = -1; } VJ_master(JAIL_MASTER_LOW); + return (retval); } /*-------------------------------------------------------------------- From phk at FreeBSD.org Mon Sep 12 06:18:09 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Sep 2016 08:18:09 +0200 Subject: [master] eb649cb Bail out on this error. Message-ID: commit eb649cb32d5f30c8eed9d9c91eac927c224ea8d8 Author: Poul-Henning Kamp Date: Mon Sep 12 06:16:57 2016 +0000 Bail out on this error. diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 077363d..58588db 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -765,8 +765,10 @@ MGT_Run(void) } i = mgt_SHM_Commit(); - if (i != 0) + if (i != 0) { MGT_complain(C_ERR, "Could not commit SHM file"); + return; + } i = vev_schedule(mgt_evb); if (i != 0) From phk at FreeBSD.org Mon Sep 12 09:13:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Sep 2016 11:13:11 +0200 Subject: [master] 4cd32f5 Only update the active_vcl on successful VCL compiles Message-ID: commit 4cd32f56edf61743d1aef144ca393d77e23fad32 Author: Poul-Henning Kamp Date: Mon Sep 12 09:12:41 2016 +0000 Only update the active_vcl on successful VCL compiles diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index ea06216..bbf590e 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -207,8 +207,6 @@ mgt_vcl_add(const char *name, const char *state) if (vp->state != VCL_STATE_COLD) vp->warm = 1; - if (active_vcl == NULL) - active_vcl = vp; VTAILQ_INSERT_TAIL(&vclhead, vp, list); return (vp); } @@ -353,6 +351,9 @@ mgt_new_vcl(struct cli *cli, const char *vclname, const char *vclsrc, AZ(C_flag); vp->fname = lib; + if (active_vcl == NULL) + active_vcl = vp; + if (child_pid < 0) return; From nils.goroll at uplex.de Mon Sep 12 09:14:14 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 11:14:14 +0200 Subject: [master] 811c0c5 Add test for support of PRIV_TASK in CLI induced VCL events. Message-ID: commit 811c0c5493831b24ed971fec6a15180774922b41 Author: Nils Goroll Date: Mon Sep 12 11:09:51 2016 +0200 Add test for support of PRIV_TASK in CLI induced VCL events. Part of #2061 diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index b5c053c..b111796 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -11,13 +11,21 @@ varnish v1 -vcl+backend { import debug; import std; + sub vcl_init { + debug.test_priv_task("something"); + debug.test_priv_task("to remember"); + std.log(debug.test_priv_task()); + } + sub vcl_recv { - set req.http.x0 = debug.test_priv_task(req.url); + debug.test_priv_task(req.url); + set req.http.x0 = debug.test_priv_task(); + debug.test_priv_task("bazz"); } sub vcl_deliver { set resp.http.x0 = req.http.x0; - set resp.http.x1 = debug.test_priv_task(""); + set resp.http.x1 = debug.test_priv_task(); } sub vcl_backend_fetch { @@ -32,19 +40,37 @@ varnish v1 -vcl+backend { } } -start +logexpect l1 -v v1 -g raw -d 1 { + expect 0 0 CLI {^Rd vcl.load} + expect 0 = VCL_Log {^something to remember} + + expect * 1002 Begin fetch$ + expect * = VCL_call ^BACKEND_FETCH + expect 0 = VCL_Log ^foo + expect 0 = BereqHeader {^bx0: /foobar} + expect 0 = VCL_Log ^bar + + expect * 1004 Begin fetch$ + expect * = VCL_call ^BACKEND_FETCH + expect 0 = VCL_Log ^foo + expect 0 = BereqHeader {^bx0: /snafu} + expect 0 = VCL_Log ^bar +} -start client c1 { txreq -url /foobar rxresp expect resp.http.x0 == /foobar - expect resp.http.x1 == /foobar + expect resp.http.x1 == "/foobar bazz" expect resp.http.bx0 == /foobar expect resp.http.bx1 == /foobar txreq -url /snafu rxresp expect resp.http.x0 == /snafu - expect resp.http.x1 == /snafu + expect resp.http.x1 == "/snafu bazz" expect resp.http.bx0 == /snafu expect resp.http.bx1 == /snafu } -run + +logexpect l1 -wait diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index cbe9de8..f20c127 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -51,7 +51,7 @@ $Function VOID test_priv_vcl(PRIV_VCL) Test function for VCL private pointers -$Function STRING test_priv_task(PRIV_TASK, STRING) +$Function STRING test_priv_task(PRIV_TASK, STRING s="") Test function for TASK private pointers diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index dc9e347..3322da7 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -101,9 +101,19 @@ vmod_test_priv_task(VRT_CTX, struct vmod_priv *priv, VCL_STRING s) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - if (priv->priv == NULL) { + if (s == NULL || *s == '\0') { + return priv->priv; + } else if (priv->priv == NULL) { priv->priv = strdup(s); priv->free = free; + } else { + char *n = realloc(priv->priv, + strlen(priv->priv) + strlen(s) + 2); + if (n == NULL) + return NULL; + strcat(n, " "); + strcat(n, s); + priv->priv = n; } return (priv->priv); } From nils.goroll at uplex.de Mon Sep 12 09:24:14 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 11:24:14 +0200 Subject: [master] a3ee09b fix the tiny bit necessary for PRIV_* to work with methods and add tests Message-ID: commit a3ee09b8cba977bb71c4f776d757c0d49c170527 Author: Nils Goroll Date: Tue Aug 30 21:49:47 2016 +0200 fix the tiny bit necessary for PRIV_* to work with methods and add tests ... demonstrating that the PRIV_* states are per vmod, no matter if accessed from a function or method Fixes #1800 part of #2061 diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index 6028b0b..999b39f 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -10,6 +10,10 @@ varnish v1 -vcl+backend { import std; import debug; + sub vcl_init { + new obj = debug.obj(); + } + sub vcl_recv { debug.rot52(req); } @@ -20,6 +24,8 @@ varnish v1 -vcl+backend { set resp.http.who = debug.author(phk); debug.test_priv_call(); debug.test_priv_vcl(); + obj.test_priv_call(); + obj.test_priv_vcl(); std.log("VCL" + " initiated " + "log"); std.syslog(8 + 7, "Somebody runs varnishtest"); debug.rot52(resp); diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index b111796..7678798 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -12,9 +12,14 @@ varnish v1 -vcl+backend { import std; sub vcl_init { + new obj = debug.obj(); + } + + sub vcl_init { debug.test_priv_task("something"); debug.test_priv_task("to remember"); - std.log(debug.test_priv_task()); + std.log("func " + debug.test_priv_task()); + std.log("obj " + obj.test_priv_task()); } sub vcl_recv { @@ -26,9 +31,11 @@ varnish v1 -vcl+backend { sub vcl_deliver { set resp.http.x0 = req.http.x0; set resp.http.x1 = debug.test_priv_task(); + set resp.http.o1 = obj.test_priv_task(); } sub vcl_backend_fetch { + obj.test_priv_task("b"); std.log("foo"); set bereq.http.bx0 = debug.test_priv_task(bereq.url); std.log("bar"); @@ -37,23 +44,25 @@ varnish v1 -vcl+backend { sub vcl_backend_response { set beresp.http.bx0 = bereq.http.bx0; set beresp.http.bx1 = debug.test_priv_task(""); + set beresp.http.bo1 = obj.test_priv_task(""); } } -start logexpect l1 -v v1 -g raw -d 1 { expect 0 0 CLI {^Rd vcl.load} - expect 0 = VCL_Log {^something to remember} + expect 0 = VCL_Log {^func something to remember} + expect 0 = VCL_Log {^obj something to remember} expect * 1002 Begin fetch$ expect * = VCL_call ^BACKEND_FETCH expect 0 = VCL_Log ^foo - expect 0 = BereqHeader {^bx0: /foobar} + expect 0 = BereqHeader {^bx0: b /foobar} expect 0 = VCL_Log ^bar expect * 1004 Begin fetch$ expect * = VCL_call ^BACKEND_FETCH expect 0 = VCL_Log ^foo - expect 0 = BereqHeader {^bx0: /snafu} + expect 0 = BereqHeader {^bx0: b /snafu} expect 0 = VCL_Log ^bar } -start @@ -62,15 +71,19 @@ client c1 { rxresp expect resp.http.x0 == /foobar expect resp.http.x1 == "/foobar bazz" - expect resp.http.bx0 == /foobar - expect resp.http.bx1 == /foobar + expect resp.http.o1 == "/foobar bazz" + expect resp.http.bx0 == "b /foobar" + expect resp.http.bx1 == "b /foobar" + expect resp.http.bo1 == "b /foobar" txreq -url /snafu rxresp expect resp.http.x0 == /snafu expect resp.http.x1 == "/snafu bazz" - expect resp.http.bx0 == /snafu - expect resp.http.bx1 == /snafu + expect resp.http.o1 == "/snafu bazz" + expect resp.http.bx0 == "b /snafu" + expect resp.http.bx1 == "b /snafu" + expect resp.http.bo1 == "b /snafu" } -run logexpect l1 -wait diff --git a/bin/varnishtest/tests/v00042.vtc b/bin/varnishtest/tests/v00042.vtc index 6b7ed9c..e7901b1 100644 --- a/bin/varnishtest/tests/v00042.vtc +++ b/bin/varnishtest/tests/v00042.vtc @@ -36,6 +36,10 @@ server s1 { varnish v1 -vcl+backend { import debug; + sub vcl_init { + new o = debug.obj(); + } + sub vcl_recv { set req.http.x0 = debug.test_priv_task(req.url + req.esi_level); } @@ -51,6 +55,7 @@ varnish v1 -vcl+backend { sub vcl_deliver { set resp.http.x1 = debug.test_priv_task(""); + set resp.http.o1 = o.test_priv_task(""); } } -start @@ -59,10 +64,12 @@ client c1 { txreq -url /a rxresp expect resp.http.x1 == "/a0" + expect resp.http.o1 == "/a0" txreq -url /b rxresp expect resp.http.x1 == "/b0" + expect resp.http.o1 == "/b0" } -run varnish v1 -expect s_req == 2 diff --git a/bin/varnishtest/tests/v00043.vtc b/bin/varnishtest/tests/v00043.vtc index 9bf7b47..7e7a4f9 100644 --- a/bin/varnishtest/tests/v00043.vtc +++ b/bin/varnishtest/tests/v00043.vtc @@ -38,6 +38,10 @@ server s1 { varnish v1 -cliok "param.set debug +syncvsl" -vcl+backend { import debug; + sub vcl_init { + new o = debug.obj(); + } + sub vcl_recv { set req.http.x0 = debug.test_priv_top(req.url + req.esi_level); } @@ -53,6 +57,7 @@ varnish v1 -cliok "param.set debug +syncvsl" -vcl+backend { sub vcl_deliver { set resp.http.x1 = debug.test_priv_top(""); + set resp.http.o1 = o.test_priv_top(""); } } -start @@ -61,10 +66,12 @@ client c1 { txreq -url /a rxresp expect resp.http.x1 == "/a0" + expect resp.http.o1 == "/a0" txreq -url /b rxresp expect resp.http.x1 == "/b0" + expect resp.http.o1 == "/b0" } -run varnish v1 -expect s_req == 2 diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index cf7fa91..efc3dcd 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -333,6 +333,7 @@ vcc_ParseNew(struct vcc *tl) sy3->eval_priv = p; sy3->fmt = VCC_Type(p); sy3->extra = TlDup(tl, buf1); + sy3->vmod = sy2->vmod; while (p[0] != '\0' || p[1] != '\0' || p[2] != '\0') p++; p += 3; diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index f20c127..82e264f 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -97,6 +97,29 @@ $Method TIME .date() You never know when you need a date. +$Method VOID .test_priv_call(PRIV_CALL) + +Test method for call private pointers + +Objects share the PRIV_* state with other objects and methods from the +same vmod - IOW the PRIV_* state is per vmod, not per object. + +$Method VOID .test_priv_vcl(PRIV_VCL) + +Test method for VCL private pointers + +Objects share the PRIV_* state with other objects and methods from the +same vmod - IOW the PRIV_* state is per vmod, not per object. + +$Method STRING .test_priv_task(PRIV_TASK, STRING s="") + +Test method for TASK private pointers + +Objects share the PRIV_* state with other objects and methods from the +same vmod - IOW the PRIV_* state is per vmod, not per object. + +$Method STRING .test_priv_top(PRIV_TOP, STRING) + $Function VOID rot52(HTTP hdr) Encrypt the HTTP header with quad-ROT13 encryption, diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/lib/libvmod_debug/vmod_debug_obj.c index 0028de0..f503943 100644 --- a/lib/libvmod_debug/vmod_debug_obj.c +++ b/lib/libvmod_debug/vmod_debug_obj.c @@ -120,3 +120,32 @@ vmod_obj_number(VRT_CTX, struct vmod_debug_obj *o) assert(o->foobar == 42); return (o->number); } + +VCL_VOID __match_proto__() +vmod_obj_test_priv_call(VRT_CTX, + struct vmod_debug_obj *o, struct vmod_priv *priv) +{ + (void) o; + vmod_test_priv_call(ctx, priv); +} +VCL_VOID __match_proto__() +vmod_obj_test_priv_vcl(VRT_CTX, + struct vmod_debug_obj *o, struct vmod_priv *priv) +{ + (void) o; + vmod_test_priv_vcl(ctx, priv); +} +VCL_STRING __match_proto__() +vmod_obj_test_priv_task(VRT_CTX, + struct vmod_debug_obj *o, struct vmod_priv *priv, VCL_STRING s) +{ + (void) o; + return (vmod_test_priv_task(ctx, priv, s)); +} +VCL_STRING __match_proto__() +vmod_obj_test_priv_top(VRT_CTX, + struct vmod_debug_obj *o, struct vmod_priv *priv, VCL_STRING s) +{ + (void) o; + return (vmod_test_priv_top(ctx, priv, s)); +} From phk at FreeBSD.org Mon Sep 12 10:04:14 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Sep 2016 12:04:14 +0200 Subject: [master] 1f746ae Zero the cli_task_priv after use. Message-ID: commit 1f746aeacb3b0ad463e30b035727da93e1486349 Author: Poul-Henning Kamp Date: Mon Sep 12 10:03:15 2016 +0000 Zero the cli_task_priv after use. diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 44d95d3..7bb19d7 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -101,6 +101,7 @@ VRTPRIV_dynamic_kill(struct vrt_privs *privs, uintptr_t id) if (privs == NULL && id == 0) { ASSERT_CLI(); VRT_priv_fini(&cli_task_priv); + memset(&cli_task_priv, 0, sizeof cli_task_priv); return; } CHECK_OBJ_NOTNULL(privs, VRT_PRIVS_MAGIC); From nils.goroll at uplex.de Mon Sep 12 10:13:10 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 12:13:10 +0200 Subject: [master] 724ce35 enable strict jemalloc checking for varnishtest Message-ID: commit 724ce352d0b19cd3d6721035e5746282e9906219 Author: Nils Goroll Date: Mon Sep 12 11:53:08 2016 +0200 enable strict jemalloc checking for varnishtest diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 90f5b4f..ce547ae 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -398,6 +398,11 @@ i_mode(void) AN(vmod_path); free(topbuild); VSB_destroy(&vsb); + + /* + * strict jemalloc checking + */ + AZ(putenv("MALLOC_CONF=abort:true,redzone:true,junk:true")); } /********************************************************************** From nils.goroll at uplex.de Mon Sep 12 10:18:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 12:18:12 +0200 Subject: [master] 9095c5c vcl_deliver read-only access to obj.ttl, obj.age, obj.grace and obj.keep Message-ID: commit 9095c5cfbac7fa3cb48257f6631f24239b30ee45 Author: Nils Goroll Date: Thu Aug 25 21:05:42 2016 +0200 vcl_deliver read-only access to obj.ttl, obj.age, obj.grace and obj.keep Noticed that I've been missing this since the beginning of time when porting some ancient inline-C. We already have obj.uncacheable und obj.hits in vcl_deliver, so not having these seems arbitrary. Copying out the values in vcl_hit just to have them available in vcl_deliver, as it was possible previously, implies unnecessary overhead. One use case is to extend object lifetime downstream when delivering a stale object, e.g. sub vcl_deliver { if (! obj.uncacheable && obj.ttl < 0s) { set resp.http.Expires = now + 1m; set resp.http.Cache-Control = "public, max-age=60"; unset req.http.Age; } } merges #2057 diff --git a/bin/varnishtest/tests/c00011.vtc b/bin/varnishtest/tests/c00011.vtc index be9aeb2..7372274 100644 --- a/bin/varnishtest/tests/c00011.vtc +++ b/bin/varnishtest/tests/c00011.vtc @@ -13,6 +13,13 @@ varnish v1 -vcl+backend { sub vcl_backend_response { set beresp.uncacheable = true; } + sub vcl_deliver { + set resp.http.o_uncacheable = obj.uncacheable; + set resp.http.o_age = obj.age; + set resp.http.o_ttl = obj.ttl; + set resp.http.o_grace = obj.grace; + set resp.http.o_keep = obj.keep; + } } -start client c1 { @@ -21,11 +28,20 @@ client c1 { expect resp.status == 200 expect resp.bodylen == 6 expect resp.http.x-varnish == "1001" + expect resp.http.o_age == "0.000" + expect resp.http.o_ttl == "120.000" + expect resp.http.o_grace == "10.000" + expect resp.http.o_keep == "0.000" + txreq -url "/foo" rxresp expect resp.status == 200 expect resp.bodylen == 7 expect resp.http.x-varnish == "1003" + expect resp.http.o_age == "0.000" + expect resp.http.o_ttl == "120.000" + expect resp.http.o_grace == "10.000" + expect resp.http.o_keep == "0.000" } client c1 -run diff --git a/bin/varnishtest/tests/c00012.vtc b/bin/varnishtest/tests/c00012.vtc index fda9f4d..77dafd8 100644 --- a/bin/varnishtest/tests/c00012.vtc +++ b/bin/varnishtest/tests/c00012.vtc @@ -13,6 +13,13 @@ varnish v1 -vcl+backend { sub vcl_miss { return(pass); } + sub vcl_deliver { + set resp.http.o_uncacheable = obj.uncacheable; + set resp.http.o_age = obj.age; + set resp.http.o_ttl = obj.ttl; + set resp.http.o_grace = obj.grace; + set resp.http.o_keep = obj.keep; + } } -start client c1 { @@ -21,11 +28,20 @@ client c1 { expect resp.status == 200 expect resp.bodylen == 6 expect resp.http.x-varnish == "1001" + expect resp.http.o_age == "0.000" + expect resp.http.o_ttl == "-0.000" + expect resp.http.o_grace == "0.000" + expect resp.http.o_keep == "0.000" + txreq -url "/foo" rxresp expect resp.status == 200 expect resp.bodylen == 7 expect resp.http.x-varnish == "1003" + expect resp.http.o_age == "0.000" + expect resp.http.o_ttl == "-0.000" + expect resp.http.o_grace == "0.000" + expect resp.http.o_keep == "0.000" } client c1 -run diff --git a/bin/varnishtest/tests/v00025.vtc b/bin/varnishtest/tests/v00025.vtc index 44426ad..af7b7eb 100644 --- a/bin/varnishtest/tests/v00025.vtc +++ b/bin/varnishtest/tests/v00025.vtc @@ -25,6 +25,10 @@ varnish v1 -arg "-i J.F.Nobody" -vcl+backend { set resp.http.l_ip = local.ip; set resp.http.r_ip = remote.ip; if (obj.uncacheable) { } + set resp.http.o_age = obj.age; + set resp.http.o_ttl = obj.ttl; + set resp.http.o_grace = obj.grace; + set resp.http.o_keep = obj.keep; } sub vcl_backend_response { diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 3104e00..fb167b4 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -654,28 +654,28 @@ sp_variables = [ ), ('obj.ttl', 'DURATION', - ('hit',), + ('hit', 'deliver'), (), """ The object's remaining time to live, in seconds. """ ), ('obj.age', 'DURATION', - ('hit',), + ('hit', 'deliver'), (), """ The age of the object. """ ), ('obj.grace', 'DURATION', - ('hit',), + ('hit', 'deliver'), (), """ The object's remaining grace period in seconds. """ ), ('obj.keep', 'DURATION', - ('hit',), + ('hit', 'deliver'), (), """ The object's remaining keep period in seconds. """ From fgsch at lodoss.net Mon Sep 12 10:35:14 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 12 Sep 2016 12:35:14 +0200 Subject: [master] 16977fc Teach vcc_expr_tostring how to handle STRING_LIST Message-ID: commit 16977fcc55298c9559afd478c7ad90a3508dbdff Author: Federico G. Schwindt Date: Mon Sep 12 09:51:54 2016 +0100 Teach vcc_expr_tostring how to handle STRING_LIST diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 573d497..415c584 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -379,6 +379,10 @@ vcc_expr_tostring(struct vcc *tl, struct expr **e, vcc_type_t fmt) AN(fmt == STRING || fmt == STRING_LIST); AZ(fmt == (*e)->fmt); + if ((*e)->fmt == STRING || ((*e)->fmt == STRING_LIST && vcc_isconst(*e))) { + (*e)->fmt = fmt; + return; + } p = (*e)->fmt->tostring; if (p == NULL && (*e)->fmt == BLOB) { VSB_printf(tl->sb, diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c index f6ab02e..b47e7f3 100644 --- a/lib/libvcc/vcc_types.c +++ b/lib/libvcc/vcc_types.c @@ -141,7 +141,7 @@ const struct type STRING[1] = {{ const struct type STRING_LIST[1] = {{ .magic = TYPE_MAGIC, .name = "STRING_LIST", - .tostring = "", + .tostring = "VRT_CollectString(ctx,\n\v1,\nvrt_magic_string_end)", }}; const struct type TIME[1] = {{ From fgsch at lodoss.net Mon Sep 12 10:35:14 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 12 Sep 2016 12:35:14 +0200 Subject: [master] 896aa8d Fix regex matches in some esoteric cases Message-ID: commit 896aa8dcbc163fe029b6827284891fe6d80a1ea2 Author: Federico G. Schwindt Date: Mon Sep 12 10:00:50 2016 +0100 Fix regex matches in some esoteric cases E.g. set req.http.foo = req.http.foo + req.http.bar ~ "foobar" diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index a932426..cd0ab28 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -120,10 +120,8 @@ varnish v1 -vcl { set req.http.foo = 3.0 * 2; set req.http.foo = 3.0 / 2; - set req.http.foo = req.http.foo ~ "bar"; - set req.http.foo = req.http.foo !~ "bar"; - set req.http.foo = "foo" ~ "bar"; - set req.http.foo = "foo" !~ "bar"; + set req.http.foo = req.http.foo + "bar" ~ "bar"; + set req.http.foo = req.http.foo + "bar" !~ "bar"; set req.ttl = 1s; } diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 415c584..3e0feef 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1147,6 +1147,8 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, vcc_type_t fmt) } if (((*e)->fmt == STRING || (*e)->fmt == STRING_LIST) && (tl->t->tok == '~' || tl->t->tok == T_NOMATCH)) { + if ((*e)->fmt == STRING_LIST) + vcc_expr_tostring(tl, e, STRING); not = tl->t->tok == '~' ? "" : "!"; vcc_NextToken(tl); ExpectErr(tl, CSTR); From fgsch at lodoss.net Mon Sep 12 10:35:14 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 12 Sep 2016 12:35:14 +0200 Subject: [master] 77f49d0 Use vcc_expr_tostring and polish Message-ID: commit 77f49d0db4752f17815c9642212895aa475e72ee Author: Federico G. Schwindt Date: Mon Sep 12 10:26:32 2016 +0100 Use vcc_expr_tostring and polish diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 3e0feef..4ced132 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1062,10 +1062,8 @@ vcc_expr_strfold(struct vcc *tl, struct expr **e, vcc_type_t fmt) ERRCHK(tl); if (fmt != STRING_LIST && (*e)->fmt == STRING_LIST) - *e = vcc_expr_edit(STRING, - "\v+VRT_CollectString(ctx,\n\v1,\nvrt_magic_string_end)\v-", - *e, NULL); - if (fmt == STRING_LIST && (*e)->fmt == STRING) + vcc_expr_tostring(tl, e, STRING); + else if (fmt == STRING_LIST && (*e)->fmt == STRING) (*e)->fmt = STRING_LIST; } From fgsch at lodoss.net Mon Sep 12 10:35:14 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 12 Sep 2016 12:35:14 +0200 Subject: [master] 3b40a01 Fix build Message-ID: commit 3b40a0163156c4e3ccac1b785da80713cb30ee95 Author: Federico G. Schwindt Date: Mon Sep 12 11:33:53 2016 +0100 Fix build diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index ce547ae..4492b36 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -402,7 +402,7 @@ i_mode(void) /* * strict jemalloc checking */ - AZ(putenv("MALLOC_CONF=abort:true,redzone:true,junk:true")); + AZ(putenv((char *)"MALLOC_CONF=abort:true,redzone:true,junk:true")); } /********************************************************************** From fgsch at lodoss.net Mon Sep 12 10:42:11 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 12 Sep 2016 12:42:11 +0200 Subject: [master] 8fb2900 Fix the fix Message-ID: commit 8fb2900a5260f9ba0441c46038ed90f75af6a7ba Author: Federico G. Schwindt Date: Mon Sep 12 11:41:08 2016 +0100 Fix the fix diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 4492b36..c6d2a99 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -402,7 +402,7 @@ i_mode(void) /* * strict jemalloc checking */ - AZ(putenv((char *)"MALLOC_CONF=abort:true,redzone:true,junk:true")); + AZ(putenv(strdup("MALLOC_CONF=abort:true,redzone:true,junk:true"))); } /********************************************************************** From nils.goroll at uplex.de Mon Sep 12 10:55:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 12:55:12 +0200 Subject: [master] ef39946 make newly introduced vtcs less timing-critical Message-ID: commit ef39946603d91e185875f3a1ff258687d35691bf Author: Nils Goroll Date: Mon Sep 12 12:53:53 2016 +0200 make newly introduced vtcs less timing-critical Follow-up 9095c5cfbac7fa3cb48257f6631f24239b30ee45 Belongs to #2057 diff --git a/bin/varnishtest/tests/c00011.vtc b/bin/varnishtest/tests/c00011.vtc index 7372274..758c35b 100644 --- a/bin/varnishtest/tests/c00011.vtc +++ b/bin/varnishtest/tests/c00011.vtc @@ -28,7 +28,7 @@ client c1 { expect resp.status == 200 expect resp.bodylen == 6 expect resp.http.x-varnish == "1001" - expect resp.http.o_age == "0.000" + expect resp.http.o_age ~ "^0" expect resp.http.o_ttl == "120.000" expect resp.http.o_grace == "10.000" expect resp.http.o_keep == "0.000" @@ -38,7 +38,7 @@ client c1 { expect resp.status == 200 expect resp.bodylen == 7 expect resp.http.x-varnish == "1003" - expect resp.http.o_age == "0.000" + expect resp.http.o_age ~ "^0" expect resp.http.o_ttl == "120.000" expect resp.http.o_grace == "10.000" expect resp.http.o_keep == "0.000" diff --git a/bin/varnishtest/tests/c00012.vtc b/bin/varnishtest/tests/c00012.vtc index 77dafd8..e1c5883 100644 --- a/bin/varnishtest/tests/c00012.vtc +++ b/bin/varnishtest/tests/c00012.vtc @@ -28,7 +28,7 @@ client c1 { expect resp.status == 200 expect resp.bodylen == 6 expect resp.http.x-varnish == "1001" - expect resp.http.o_age == "0.000" + expect resp.http.o_age ~ "^0" expect resp.http.o_ttl == "-0.000" expect resp.http.o_grace == "0.000" expect resp.http.o_keep == "0.000" @@ -38,7 +38,7 @@ client c1 { expect resp.status == 200 expect resp.bodylen == 7 expect resp.http.x-varnish == "1003" - expect resp.http.o_age == "0.000" + expect resp.http.o_age ~ "^0" expect resp.http.o_ttl == "-0.000" expect resp.http.o_grace == "0.000" expect resp.http.o_keep == "0.000" From phk at FreeBSD.org Mon Sep 12 10:57:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Sep 2016 12:57:11 +0200 Subject: [master] f3862b4 Start keeping track of H2 state Message-ID: commit f3862b4763ce27a08fbac6cd73a45f604aea9b3c Author: Poul-Henning Kamp Date: Mon Sep 12 10:56:24 2016 +0000 Start keeping track of H2 state diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 3bf1fdc..2fe1c23 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -32,17 +32,20 @@ struct h2_sess; #include "hpack/vhp.h" enum h2_error_e { + H2E__DUMMY = -1, #define H2_ERROR(NAME, val, desc) \ H2E_##NAME = val, #include "tbl/h2_error.h" }; enum h2_frame_e { + H2_FRAME__DUMMY = -1, #define H2_FRAME(l,u,t,f) H2_FRAME_##u = t, #include "tbl/h2_frames.h" }; enum h2_stream_e { + H2_STREAM__DUMMY = -1, #define H2_STREAM(U,s,d) H2_S_##U, #include "tbl/h2_stream.h" }; @@ -51,6 +54,7 @@ enum h2_stream_e { #include "tbl/h2_frames.h" enum h2setting { + H2_SETTINGS__DUMMY = -1, #define H2_SETTINGS(n,v,d) H2S_##n = v, #include "tbl/h2_settings.h" #undef H2_SETTINGS diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index da64f9d..62b5079 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -188,6 +188,7 @@ h2_new_req(const struct worker *wrk, struct h2_sess *h2, r2 = WS_Alloc(req->ws, sizeof *r2); AN(r2); INIT_OBJ(r2, H2_REQ_MAGIC); + r2->state = H2_S_IDLE; r2->h2sess = h2; r2->stream = stream; r2->req = req; @@ -384,6 +385,7 @@ h2_do_req(struct worker *wrk, void *priv) assert(CNT_Request(wrk, req) != REQ_FSM_DISEMBARK); VSL(SLT_Debug, 0, "H2REQ CNT done"); /* XXX clean up req */ + r2->state = H2_S_CLOSED; h2_del_req(wrk, r2, H2E_NO_ERROR); } @@ -400,7 +402,9 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) * read frames and proper error handling. */ - assert(h2->rxf_stream & 1); + xxxassert(h2->rxf_stream & 1); + xxxassert(r2->state == H2_S_IDLE); + r2->state = H2_S_OPEN; req = r2->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); From dridi.boukelmoune at gmail.com Mon Sep 12 10:59:12 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Sep 2016 12:59:12 +0200 Subject: [master] 1941f86 Append a default system-wide location to vmod_path Message-ID: commit 1941f8638814e30d879af6eb2e9e129c503512f1 Author: Dridi Boukelmoune Date: Thu Aug 25 23:02:01 2016 +0200 Append a default system-wide location to vmod_path By default: /usr/share/vcl This directory becomes a drop-in, and standard location for library-like VCL files. It streamlines third-party VCL packaging, offering a blessed for installation, leaving the canonical /etc/varnish directory entirely to the end-user. While it is common to see VMOD packages in the wild, there are some VCL libraries (sometimes backed by VMODs) that would benefit some form of standardization. It then becomes possible to install packages, and use relative includes that JustWork(tm). This is especially true for VCL libraries that are split into several files, they can put themselves in a subdirectory in /usr/share/vcl and use it as a namespace for includes between files of the library: # somewhere in my awesome "foo" library include "foo/some_feature.vcl" Choosing a default path of `/etc/varnish:/usr/share/vcl` prevents VCLs dropped in the system from shadowing end-users VCLs but at the same empowers said users to override a "standard" VCL in their `/etc` tree, with no additional parameters fiddling. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 892b9c6..d629a4a 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -145,7 +145,7 @@ varnishd_CFLAGS = \ -DVARNISHD_IS_NOT_A_VMOD \ -DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' \ -DVARNISH_VMOD_DIR='"${pkglibdir}/vmods"' \ - -DVARNISH_VCL_DIR='"${varnishconfdir}"' + -DVARNISH_VCL_DIR='"${varnishconfdir}:${datarootdir}/varnish/vcl"' varnishd_LDFLAGS = -export-dynamic From dridi.boukelmoune at gmail.com Mon Sep 12 10:59:12 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Sep 2016 12:59:12 +0200 Subject: [master] 45fd33b Install devicedetect.vcl system-wide Message-ID: commit 45fd33b01e3ab556854b045747c5c00e21709058 Author: Dridi Boukelmoune Date: Thu Aug 25 23:02:20 2016 +0200 Install devicedetect.vcl system-wide Downstream packages can always replace it with a different version. End-users can override it in their `/etc` tree. diff --git a/etc/Makefile.am b/etc/Makefile.am index d02d710..8fc49b9 100644 --- a/etc/Makefile.am +++ b/etc/Makefile.am @@ -10,3 +10,7 @@ builtin.vcl: $(top_srcdir)/bin/varnishd/builtin.vcl New users is recommended to use the example.vcl file as a starting point.\n\n";\ sed -n '/vcl_recv/,$$p' $(top_srcdir)/bin/varnishd/builtin.vcl ) | \ sed 's/^\(.*\)$$/# \1/' > builtin.vcl + +vcldir=$(datarootdir)/varnish/vcl + +dist_vcl_DATA = devicedetect.vcl From dridi.boukelmoune at gmail.com Mon Sep 12 10:59:12 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Sep 2016 12:59:12 +0200 Subject: [master] f9591f0 Update devicedetect.vcl to current upstream Message-ID: commit f9591f061e918d802f9adbe6556777ba28beaff9 Author: Dridi Boukelmoune Date: Mon Sep 12 12:47:16 2016 +0200 Update devicedetect.vcl to current upstream diff --git a/etc/devicedetect.vcl b/etc/devicedetect.vcl index 6c7e659..1910743 100644 --- a/etc/devicedetect.vcl +++ b/etc/devicedetect.vcl @@ -40,7 +40,8 @@ sub devicedetect { if (req.http.Cookie ~ "^ *$") { unset req.http.Cookie; } } else { if (req.http.User-Agent ~ "\(compatible; Googlebot-Mobile/2.1; \+http://www.google.com/bot.html\)" || - (req.http.User-Agent ~ "iPhone" && req.http.User-Agent ~ "\(compatible; Googlebot/2.1; \+http://www.google.com/bot.html")) { + (req.http.User-Agent ~ "(Android|iPhone)" && req.http.User-Agent ~ "\(compatible.?; Googlebot/2.1.?; \+http://www.google.com/bot.html") || + (req.http.User-Agent ~ "(iPhone|Windows Phone)" && req.http.User-Agent ~ "\(compatible; bingbot/2.0; \+http://www.bing.com/bingbot.htm")) { set req.http.X-UA-Device = "mobile-bot"; } elsif (req.http.User-Agent ~ "(?i)(ads|google|bing|msn|yandex|baidu|ro|career|seznam|)bot" || req.http.User-Agent ~ "(?i)(baidu|jike|symantec)spider" || From dridi.boukelmoune at gmail.com Mon Sep 12 10:59:12 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Sep 2016 12:59:12 +0200 Subject: [master] 989b39f Document the purpose of the default `vcl_path` Message-ID: commit 989b39fb2ee884f326477ad49212c4b033694013 Author: Dridi Boukelmoune Date: Mon Sep 12 12:57:08 2016 +0200 Document the purpose of the default `vcl_path` diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index 6ca1aeb..5d4d451 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -66,7 +66,11 @@ struct parspec mgt_parspec[] = { NULL, NULL, "Directory (or colon separated list of directories) " "from which relative VCL filenames (vcl.load and " - "include) are to be found.", + "include) are to be found. By default Varnish searches " + "VCL files in both the system configuration and shared " + "data directories to allow packages to drop their VCL " + "files in a standard location where relative includes " + "would work.", 0, VARNISH_VCL_DIR, NULL }, From nils.goroll at uplex.de Mon Sep 12 11:52:11 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 13:52:11 +0200 Subject: [master] 5d9f6b8 make newly introduced vtcs less timing-critical Message-ID: commit 5d9f6b8f393c72e65650ac6033a3cbb981b3407c Author: Nils Goroll Date: Mon Sep 12 13:50:04 2016 +0200 make newly introduced vtcs less timing-critical Follow-up 9095c5cfbac7fa3cb48257f6631f24239b30ee45 Belongs to #2057 diff --git a/bin/varnishtest/tests/c00012.vtc b/bin/varnishtest/tests/c00012.vtc index e1c5883..a45b291 100644 --- a/bin/varnishtest/tests/c00012.vtc +++ b/bin/varnishtest/tests/c00012.vtc @@ -29,7 +29,7 @@ client c1 { expect resp.bodylen == 6 expect resp.http.x-varnish == "1001" expect resp.http.o_age ~ "^0" - expect resp.http.o_ttl == "-0.000" + expect resp.http.o_ttl ~ "^-0" expect resp.http.o_grace == "0.000" expect resp.http.o_keep == "0.000" @@ -39,7 +39,7 @@ client c1 { expect resp.bodylen == 7 expect resp.http.x-varnish == "1003" expect resp.http.o_age ~ "^0" - expect resp.http.o_ttl == "-0.000" + expect resp.http.o_ttl ~ "^-0" expect resp.http.o_grace == "0.000" expect resp.http.o_keep == "0.000" } From nils.goroll at uplex.de Mon Sep 12 12:23:14 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 14:23:14 +0200 Subject: [master] fac13e9 now for real: make newly introduced vtcs less timing-critical Message-ID: commit fac13e90ff3b126ea7ddcba074dceb70cfa7e04b Author: Nils Goroll Date: Mon Sep 12 14:22:11 2016 +0200 now for real: make newly introduced vtcs less timing-critical Follow-up 9095c5cfbac7fa3cb48257f6631f24239b30ee45 Belongs to #2057 diff --git a/bin/varnishtest/tests/c00011.vtc b/bin/varnishtest/tests/c00011.vtc index 758c35b..02dd470 100644 --- a/bin/varnishtest/tests/c00011.vtc +++ b/bin/varnishtest/tests/c00011.vtc @@ -28,8 +28,10 @@ client c1 { expect resp.status == 200 expect resp.bodylen == 6 expect resp.http.x-varnish == "1001" - expect resp.http.o_age ~ "^0" - expect resp.http.o_ttl == "120.000" + expect resp.http.o_age >= 0 + expect resp.http.o_age < 0.5 + expect resp.http.o_ttl > 119.5 + expect resp.http.o_ttl <= 120 expect resp.http.o_grace == "10.000" expect resp.http.o_keep == "0.000" @@ -38,8 +40,10 @@ client c1 { expect resp.status == 200 expect resp.bodylen == 7 expect resp.http.x-varnish == "1003" - expect resp.http.o_age ~ "^0" - expect resp.http.o_ttl == "120.000" + expect resp.http.o_age >= 0 + expect resp.http.o_age < 0.5 + expect resp.http.o_ttl > 119.5 + expect resp.http.o_ttl <= 120 expect resp.http.o_grace == "10.000" expect resp.http.o_keep == "0.000" } diff --git a/bin/varnishtest/tests/c00012.vtc b/bin/varnishtest/tests/c00012.vtc index a45b291..9f64f68 100644 --- a/bin/varnishtest/tests/c00012.vtc +++ b/bin/varnishtest/tests/c00012.vtc @@ -28,8 +28,10 @@ client c1 { expect resp.status == 200 expect resp.bodylen == 6 expect resp.http.x-varnish == "1001" - expect resp.http.o_age ~ "^0" - expect resp.http.o_ttl ~ "^-0" + expect resp.http.o_age >= 0 + expect resp.http.o_age < 0.5 + expect resp.http.o_ttl <= -0 + expect resp.http.o_ttl > -0.5 expect resp.http.o_grace == "0.000" expect resp.http.o_keep == "0.000" @@ -38,8 +40,10 @@ client c1 { expect resp.status == 200 expect resp.bodylen == 7 expect resp.http.x-varnish == "1003" - expect resp.http.o_age ~ "^0" - expect resp.http.o_ttl ~ "^-0" + expect resp.http.o_age >= 0 + expect resp.http.o_age < 0.5 + expect resp.http.o_ttl <= -0 + expect resp.http.o_ttl > -0.5 expect resp.http.o_grace == "0.000" expect resp.http.o_keep == "0.000" } From nils.goroll at uplex.de Mon Sep 12 12:27:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 14:27:12 +0200 Subject: [master] 27aa521 take a note on the hit-for-pass to hit-for-miss change Message-ID: commit 27aa52117b4ac59cc0fe080a011eddfdf2f30d65 Author: Nils Goroll Date: Mon Sep 12 14:26:05 2016 +0200 take a note on the hit-for-pass to hit-for-miss change diff --git a/doc/changes.rst b/doc/changes.rst index 0fba41c..871875d 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -15,6 +15,14 @@ Major items: * Always send the request body to the backend, making possible to cache POST, PUT, PATCH etc. +* hit-for-pass is now actually hit-for-miss: Objects cached as + uncacheable now trigger misses rather than hits, so when a cacheable + object becomes available, it will be cached even before the ttl of + the uncacheable object expires. As a particular implication of this, + conditional request headers (If-Modified-Since, If-None-Match) are + now being removed for hits on uncacheable objects. + + Documentation and counters still refer to the mechanism as hit-for-pass ====================================== Varnish Cache 4.1.3-beta1 (2016-06-15) From nils.goroll at uplex.de Mon Sep 12 12:28:10 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 14:28:10 +0200 Subject: [master] 1bcc818 some little detail Message-ID: commit 1bcc8189b9d08991f15ccde939531c2dadfad157 Author: Nils Goroll Date: Mon Sep 12 14:27:44 2016 +0200 some little detail diff --git a/doc/changes.rst b/doc/changes.rst index 871875d..412cbb5 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -22,7 +22,8 @@ Major items: conditional request headers (If-Modified-Since, If-None-Match) are now being removed for hits on uncacheable objects. - Documentation and counters still refer to the mechanism as hit-for-pass + Documentation, counters and log entries still refer to the mechanism + as hit-for-pass and Hitpass, respectively. ====================================== Varnish Cache 4.1.3-beta1 (2016-06-15) From phk at FreeBSD.org Mon Sep 12 12:29:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Sep 2016 14:29:11 +0200 Subject: [master] c604ec1 Bump the VRT API version on a "we've probably overlooked something" basis. Message-ID: commit c604ec1ca7f9ec85d3a6977085487694532f6d4c Author: Poul-Henning Kamp Date: Mon Sep 12 12:25:02 2016 +0000 Bump the VRT API version on a "we've probably overlooked something" basis. diff --git a/include/vrt.h b/include/vrt.h index 629eb46..901e13f 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -39,6 +39,8 @@ * binary/load-time compatible, increment MAJOR version * * + * 5.0: + * Varnish 5.0 release "better safe than sorry" bump * 4.0: * VCL_BYTES changed to long long * 3.2: @@ -53,7 +55,7 @@ * */ -#define VRT_MAJOR_VERSION 4U +#define VRT_MAJOR_VERSION 5U #define VRT_MINOR_VERSION 0U From phk at FreeBSD.org Mon Sep 12 12:29:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Sep 2016 14:29:11 +0200 Subject: [master] a67944a Bump the Via header for 5.0 Message-ID: commit a67944a5342b6274e460461619977f83eebadb8b Author: Poul-Henning Kamp Date: Mon Sep 12 12:28:14 2016 +0000 Bump the Via header for 5.0 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index e7fec41..bab15ec 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -94,7 +94,7 @@ cnt_deliver(struct worker *wrk, struct req *req) http_PrintfHeader(req->resp, "Age: %.0f", fmax(0., req->t_prev - req->objcore->t_origin)); - http_SetHeader(req->resp, "Via: 1.1 varnish-v4"); + http_SetHeader(req->resp, "Via: 1.1 varnish (Varnish/5.0)"); if (cache_param->http_gzip_support && ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) && diff --git a/bin/varnishtest/tests/b00000.vtc b/bin/varnishtest/tests/b00000.vtc index 5b464a9..c33038a 100644 --- a/bin/varnishtest/tests/b00000.vtc +++ b/bin/varnishtest/tests/b00000.vtc @@ -32,4 +32,4 @@ varnish v1 -expect cache_miss == 1 varnish v1 -expect s_sess == 1 varnish v1 -expect s_req == 1 varnish v1 -expect s_resp_bodybytes == 7 -varnish v1 -expect s_resp_hdrbytes == 167 +varnish v1 -expect s_resp_hdrbytes == 178 diff --git a/bin/varnishtest/tests/e00003.vtc b/bin/varnishtest/tests/e00003.vtc index ba73c14..0ff09d8 100644 --- a/bin/varnishtest/tests/e00003.vtc +++ b/bin/varnishtest/tests/e00003.vtc @@ -30,7 +30,7 @@ varnish v1 -vcl+backend { logexpect l1 -v v1 -g request { expect 0 1001 Begin "^req .* rxreq" - expect * = ReqAcct "^18 0 18 176 75 251$" + expect * = ReqAcct "^18 0 18 187 75 262$" expect 0 = End } -start @@ -52,7 +52,7 @@ logexpect l4 -v v1 -g request { logexpect l5 -v v1 -g request { expect * 1005 Begin "^req .* rxreq" - expect * = ReqAcct "^18 0 18 181 75 256$" + expect * = ReqAcct "^18 0 18 192 75 267$" expect 0 = End } -start From hermunn at varnish-software.com Mon Sep 12 13:48:16 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:16 +0200 Subject: [4.1] 5df35e8 Stop incorrectly reporting overflow on zeroed ws. Message-ID: commit 5df35e87d3c0648d99f98bf45a35f4644c4ce42a Author: Lasse Karstensen Date: Thu Jul 7 15:36:07 2016 +0200 Stop incorrectly reporting overflow on zeroed ws. Ref: #2008 diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 08a4ad1..c475605 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -116,7 +116,7 @@ pan_ws(struct vsb *vsb, const struct ws *ws) VSB_printf(vsb, "ws = %p {\n", ws); VSB_indent(vsb, 2); PAN_CheckMagic(vsb, ws, WS_MAGIC); - if (!(ws->id[0] & 0x20)) + if (ws->id[0] != '\0' && (!(ws->id[0] & 0x20))) VSB_printf(vsb, "OVERFLOWED "); VSB_printf(vsb, "id = \"%s\",\n", ws->id); VSB_printf(vsb, "{s,f,r,e} = {%p", ws->s); diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index e8f9b7f..e4a0b0c 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -254,6 +254,7 @@ int WS_Overflowed(const struct ws *ws) { CHECK_OBJ_NOTNULL(ws, WS_MAGIC); + AN(ws->id[0]); if (ws->id[0] & 0x20) return (0); From hermunn at varnish-software.com Mon Sep 12 13:48:16 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:16 +0200 Subject: [4.1] 918e059 Go to v_b_e{} if we get a 304 without INM/IMS Message-ID: commit 918e059ab08b13def7ebfb8c54717aea2825ce95 Author: Federico G. Schwindt Date: Mon Jul 11 13:04:23 2016 +0100 Go to v_b_e{} if we get a 304 without INM/IMS Spotted by Foxik on #varnish @ irc.linpro.no. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 8a133dd..98336ed 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -416,7 +416,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) "304 response but not conditional fetch"); bo->htc->doclose = SC_RX_BAD; VDI_Finish(bo->wrk, bo); - return (F_STP_FAIL); + return (F_STP_ERROR); } } From hermunn at varnish-software.com Mon Sep 12 13:48:16 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:16 +0200 Subject: [4.1] 73b33d6 Lock when a backend is moved between lists Message-ID: commit 73b33d62de5bb843eb1018b2e791a052ffb84966 Author: Dridi Boukelmoune Date: Wed Jul 27 17:08:24 2016 +0200 Lock when a backend is moved between lists Refs #2008 diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c index ad96330..50e9e4d 100644 --- a/bin/varnishd/cache/cache_backend_cfg.c +++ b/bin/varnishd/cache/cache_backend_cfg.c @@ -157,9 +157,11 @@ VRT_delete_backend(VRT_CTX, struct director **dp) be->admin_health = vbe_ah_deleted; be->health_changed = VTIM_real(); be->cooled = VTIM_real() + 60.; + Lck_Unlock(&be->mtx); + Lck_Lock(&backends_mtx); VTAILQ_REMOVE(&backends, be, list); VTAILQ_INSERT_TAIL(&cool_backends, be, list); - Lck_Unlock(&be->mtx); + Lck_Unlock(&backends_mtx); // NB. The backend is still usable for the ongoing transactions, // this is why we don't bust the director's magic number. From hermunn at varnish-software.com Mon Sep 12 13:48:16 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:16 +0200 Subject: [4.1] 4f9d556 Be consistent wrt vsl_ts locking Message-ID: commit 4f9d5565fa738d1617159f437770c1e2a1619823 Author: Martin Blix Grydeland Date: Mon Aug 8 14:37:04 2016 +0200 Be consistent wrt vsl_ts locking Fix an issue where the new timestamp was set without locking. Spotted by: Coverity diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index ec99a50..4a82cef 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -310,6 +310,12 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], if (tsp) upd_vsl_ts(tsp); + /* + * only parse the last tsp seen in this transaction - + * it should be the latest. + */ + upd_vsl_ts(tsp); + /* phase out old data */ if (nhist == HIST_N) { u = rr_hist[next_hist]; From hermunn at varnish-software.com Mon Sep 12 13:48:16 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:16 +0200 Subject: [4.1] 41aa2e3 Fix redundant error message Message-ID: commit 41aa2e379bb242fe9ea7ec7295afc8b76511e9ec Author: Dridi Boukelmoune Date: Sat Aug 20 18:29:26 2016 +0200 Fix redundant error message diff --git a/bin/varnishd/mgt/mgt_jail.c b/bin/varnishd/mgt/mgt_jail.c index 4a71ea7..b0cd91d 100644 --- a/bin/varnishd/mgt/mgt_jail.c +++ b/bin/varnishd/mgt/mgt_jail.c @@ -154,7 +154,7 @@ VJ_make_workdir(const char *dname) fd = open("_.testfile", O_RDWR|O_CREAT|O_EXCL, 0600); if (fd < 0) - ARGV_ERR("Error: Cannot create test-file in %s (%s)\n" + ARGV_ERR("Cannot create test-file in %s (%s)\n" "Check permissions (or delete old directory)\n", dname, strerror(errno)); AZ(close(fd)); From hermunn at varnish-software.com Mon Sep 12 13:48:16 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:16 +0200 Subject: [4.1] 7419397 fix typo: actually save LIBS in save_LIBS Message-ID: commit 7419397b8fb8d507b890181496dcdd912357a097 Author: Nils Goroll Date: Mon Aug 22 14:56:58 2016 +0200 fix typo: actually save LIBS in save_LIBS diff --git a/configure.ac b/configure.ac index 0c90d69..7a0a7de 100644 --- a/configure.ac +++ b/configure.ac @@ -161,7 +161,7 @@ if test "$enable_pcre_jit" = yes; then AC_MSG_CHECKING(for PCRE JIT usability) save_CFLAGS="${CFLAGS}" CFLAGS="${PCRE_CFLAGS}" - save_LIBS="${CFLAGS}" + save_LIBS="${LIBS}" LIBS="${PCRE_LIBS}" AC_RUN_IFELSE( [AC_LANG_PROGRAM([[ From hermunn at varnish-software.com Mon Sep 12 13:48:16 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:16 +0200 Subject: [4.1] a913c08 When test-compiling to check for jit, also use CFLAGS and LIBS Message-ID: commit a913c08f9050ed270f8b67777ad7841a3b7153be Author: Nils Goroll Date: Mon Aug 22 14:59:56 2016 +0200 When test-compiling to check for jit, also use CFLAGS and LIBS rather than the values from pkg-config only. CLAGS and LIBS may contain important generic compiler / linker flags for this platform. diff --git a/configure.ac b/configure.ac index 7a0a7de..72554d3 100644 --- a/configure.ac +++ b/configure.ac @@ -160,9 +160,9 @@ AC_ARG_ENABLE(pcre-jit, if test "$enable_pcre_jit" = yes; then AC_MSG_CHECKING(for PCRE JIT usability) save_CFLAGS="${CFLAGS}" - CFLAGS="${PCRE_CFLAGS}" + CFLAGS="${CFLAGS} ${PCRE_CFLAGS}" save_LIBS="${LIBS}" - LIBS="${PCRE_LIBS}" + LIBS="${LIBS} ${PCRE_LIBS}" AC_RUN_IFELSE( [AC_LANG_PROGRAM([[ #include From hermunn at varnish-software.com Mon Sep 12 13:48:16 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:16 +0200 Subject: [4.1] 8c9f875 remove unused __VA_ARGS__ Message-ID: commit 8c9f8757288964756a256599a673e2b026e092b4 Author: Nils Goroll Date: Mon Aug 22 15:49:27 2016 +0200 remove unused __VA_ARGS__ The P macro is only ever called with one argument and Solaris cc rightly emits E_ARGUEMENT_MISMATCH (yes, with the typo) diff --git a/bin/varnishstat/vsc2rst.c b/bin/varnishstat/vsc2rst.c index 2aaa16d..f955b78 100644 --- a/bin/varnishstat/vsc2rst.c +++ b/bin/varnishstat/vsc2rst.c @@ -36,8 +36,8 @@ #include "tbl/vsc_levels.h" #undef VSC_LEVEL_F -#define P(x, ...) \ - printf(x "\n", ##__VA_ARGS__) +#define P(x) \ + printf(x "\n") #define VSC_LEVEL_F(v,l,e,d) \ printf("%s ? %s\n\t%s\n\n", l, e, d); #define VSC_F(n, t, l, s, f, v, d, e) \ From hermunn at varnish-software.com Mon Sep 12 13:48:16 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:16 +0200 Subject: [4.1] 6b85177 Improve wording of do_stream documentation Message-ID: commit 6b85177a230cb777efe18ae8c9a9eb238e5007e8 Author: Nils Goroll Date: Tue Aug 23 10:23:32 2016 +0200 Improve wording of do_stream documentation Addresses question in #2038 diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 7f2b50b..d8eacf6 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -487,11 +487,11 @@ sp_variables = [ ), ('beresp.do_stream', 'BOOL', - ( 'backend_response', 'backend_error'), - ( 'backend_response', 'backend_error'), """ - Deliver the object to the client directly without - fetching the whole object into varnish. If this - request is pass'ed it will not be stored in memory. + ('backend_response', 'backend_error'), + ('backend_response', 'backend_error'), """ + Deliver the object to the client while fetching the whole + object into varnish. If this request is pass'ed it will not be + stored in memory. """ ), ('beresp.do_gzip', From hermunn at varnish-software.com Mon Sep 12 13:48:17 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:17 +0200 Subject: [4.1] 6ddd00d Counter documentation for sess_herd Message-ID: commit 6ddd00d8a6ce1b555558c351dbcf119a69a497c4 Author: Dridi Boukelmoune Date: Tue Aug 23 15:20:02 2016 +0200 Counter documentation for sess_herd diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h index 31235e1..bb0d635 100644 --- a/include/tbl/vsc_f_main.h +++ b/include/tbl/vsc_f_main.h @@ -390,7 +390,7 @@ VSC_F(sess_readahead, uint64_t, 1, 'c', 'i', info, ) VSC_F(sess_herd, uint64_t, 1, 'c', 'i', diag, "Session herd", - "" + "Number of times the linger_timeout triggered" ) #define SESS_CLOSE_ERR0 "OK " From hermunn at varnish-software.com Mon Sep 12 13:48:17 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:17 +0200 Subject: [4.1] c231bda Further improve do_stream documentation Message-ID: commit c231bdafa942b1b3c079056341e75b008da088ea Author: Nils Goroll Date: Tue Aug 23 15:43:22 2016 +0200 Further improve do_stream documentation Thanks, Dridi Addresses question in #2038 diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index d8eacf6..7c09ba6 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -490,8 +490,9 @@ sp_variables = [ ('backend_response', 'backend_error'), ('backend_response', 'backend_error'), """ Deliver the object to the client while fetching the whole - object into varnish. If this request is pass'ed it will not be - stored in memory. + object into varnish. For uncacheable objects, storage for + parts of the body which have been sent to the client may + get freed early, depending on the storage engine used. """ ), ('beresp.do_gzip', From hermunn at varnish-software.com Mon Sep 12 13:48:17 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:17 +0200 Subject: [4.1] 5d0103a The parameter's name is timeout_linger Message-ID: commit 5d0103ab98f9379db01be4bffba8b5b5477e82be Author: Dridi Boukelmoune Date: Tue Aug 23 16:24:10 2016 +0200 The parameter's name is timeout_linger There are two timeout names where `timeout` is the prefix: - timeout_linger - timeout_idle If it were up to me, they'd be called the other way around and according to the documentation: > `session_linger` has been renamed to `timeout_linger If it were up to me, it would have been renamed to a self-describing `session_linger_timeout` (wink wink, nudge nudge). Typo spotted by scn. diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h index bb0d635..ab926d9 100644 --- a/include/tbl/vsc_f_main.h +++ b/include/tbl/vsc_f_main.h @@ -390,7 +390,7 @@ VSC_F(sess_readahead, uint64_t, 1, 'c', 'i', info, ) VSC_F(sess_herd, uint64_t, 1, 'c', 'i', diag, "Session herd", - "Number of times the linger_timeout triggered" + "Number of times the timeout_linger triggered" ) #define SESS_CLOSE_ERR0 "OK " From hermunn at varnish-software.com Mon Sep 12 13:48:17 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:17 +0200 Subject: [4.1] e6f46cf Add script to run Coverity. Message-ID: commit e6f46cf3c3d346a483b91b8dc8a7ccde27ddb973 Author: Lasse Karstensen Date: Thu Aug 25 11:40:08 2016 +0200 Add script to run Coverity. This will allow anyone with the Coverity scanner installed and our authtoken to upload their own feature branch if they want to. Main point is to get this out of the Jenkins job definition and into something that is more developer accessible. Doing my best to keep it /bin/sh compatible. diff --git a/.gitignore b/.gitignore index 0cfaa04..fb7114f 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,7 @@ cscope.*out /bin/varnishtest/tests/*.log /bin/varnishtest/tests/*.log-t /bin/varnishtest/test-suite.log +# +# Coverity output +/cov-int +/myproject.tgz diff --git a/devscripts/coverity-run b/devscripts/coverity-run new file mode 100755 index 0000000..a1d6263 --- /dev/null +++ b/devscripts/coverity-run @@ -0,0 +1,54 @@ +#!/bin/sh +# +# Build Varnish under Coverity, and upload the output to Coverity Scan. +# +# Requires the Coverity scanner in $PATH and the upload token set in $COVTOKEN. +# +# See https://github.com/varnishcache/varnish-cache/wiki/Coverity-scans for overview. +# + + +if [ -z "$COVTOKEN" ]; then + echo "ERROR: No COVTOKEN in environment" + exit 1 +fi + +if [ -z "`which cov-build`" ]; then + echo "ERROR: No Coverity (cov-build) in \$PATH. Download: https://scan.coverity.com/download?tab=cxx" + exit 1 +fi + +if [ -z "$EMAIL" ]; then + EMAIL="varnish-dev at varnish-cache.org" +fi + + +GITREF=`git rev-parse --short HEAD` +GITBRANCH=`git rev-parse --abbrev-ref HEAD` + +# Do a dirty check. +DIRT=`git status --porcelain 2>/dev/null | egrep '^(\ M|M)' | grep -v coverity-run` +if [ -n "$DIRT" ]; then + printf "ERROR: Refusing to analyse a dirty tree.\n$DIRT\n" + exit 2 +fi + +test "`basename $PWD`" = "devscripts" && cd .. + +make distclean || true +test -f configure || ./autogen.sh +./configure + +cov-build --dir cov-int make + +# the web ui seems to require the file to be called myproject.tgz. Very cute. +tar cvfz myproject.tgz cov-int + +curl --form token=$COVTOKEN \ + --form "email=$EMAIL" \ + --form "file=@myproject.tgz" \ + --form version="$GITREF" \ + --form description="description=${GITBRANCH}_branch" \ + 'https://scan.coverity.com/builds?project=varnish' + +rm myproject.tgz From hermunn at varnish-software.com Mon Sep 12 13:48:17 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:17 +0200 Subject: [4.1] 3a2ea1e Pointlessly fix a resource-leak in bail-out path, spotted by Coverity. Message-ID: commit 3a2ea1eeb552db8aa1bab3e63432256f4eba03db Author: Poul-Henning Kamp Date: Mon Aug 29 22:24:40 2016 +0000 Pointlessly fix a resource-leak in bail-out path, spotted by Coverity. diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 97728a9..88ae9b4 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -225,6 +225,8 @@ vcc_acl_try_getaddrinfo(struct vcc *tl, struct acl_e *ae) res->ai_family, PF(ae->t_addr)); continue; } + if (tl->err) + freeaddrinfo(res0); ERRCHK(tl); } freeaddrinfo(res0); From hermunn at varnish-software.com Mon Sep 12 13:48:17 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:17 +0200 Subject: [4.1] a4b4903 Make a workspace debugging output more usable Message-ID: commit a4b490312f96c2d0e136b7693c40440cbf55fb17 Author: Poul-Henning Kamp Date: Tue Aug 30 09:01:16 2016 +0000 Make a workspace debugging output more usable diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index e4a0b0c..f9402d3 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -241,7 +241,7 @@ void WS_ReleaseP(struct ws *ws, char *ptr) { WS_Assert(ws); - DSL(DBG_WORKSPACE, 0, "WS_ReleaseP(%p, %p)", ws, ptr); + DSL(DBG_WORKSPACE, 0, "WS_ReleaseP(%p, %p (%zd))", ws, ptr, ptr - ws->f); assert(ws->r != NULL); assert(ptr >= ws->f); assert(ptr <= ws->r); From hermunn at varnish-software.com Mon Sep 12 13:48:17 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:17 +0200 Subject: [4.1] e6bcbb3 Check malloc return Message-ID: commit e6bcbb354c57a91add3e89004ef17ffb79263b34 Author: Guillaume Quintard Date: Tue Aug 30 10:25:25 2016 +0200 Check malloc return diff --git a/lib/libvarnish/cli_common.c b/lib/libvarnish/cli_common.c index 44a6869..d14611f 100644 --- a/lib/libvarnish/cli_common.c +++ b/lib/libvarnish/cli_common.c @@ -204,6 +204,7 @@ VCLI_ReadResult(int fd, unsigned *status, char **ptr, double tmo) *status = u; p = malloc(v + 1L); + AN(p); if (p == NULL) break; From hermunn at varnish-software.com Mon Sep 12 13:48:17 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:17 +0200 Subject: [4.1] 63da974 Typo Message-ID: commit 63da9745d9550bd349df2663ebf4d5ccedfe5c55 Author: Guillaume Quintard Date: Tue Aug 30 10:37:29 2016 +0200 Typo diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 21fe298..e450eea 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -167,7 +167,7 @@ server_thread(void *priv) l = sizeof addr_s; fd = accept(s->sock, addr, &l); if (fd < 0) - vtc_log(vl, 0, "Accepted failed: %s", strerror(errno)); + vtc_log(vl, 0, "Accept failed: %s", strerror(errno)); vtc_log(vl, 3, "accepted fd %d", fd); fd = http_process(vl, s->spec, fd, &s->sock); vtc_log(vl, 3, "shutting fd %d", fd); From hermunn at varnish-software.com Mon Sep 12 13:48:17 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:17 +0200 Subject: [4.1] a1d65c1 Add useless break statement to please Coverity Message-ID: commit a1d65c1a4cfb23932f229717fcf06d29afffc0f2 Author: Guillaume Quintard Date: Tue Aug 30 10:41:15 2016 +0200 Add useless break statement to please Coverity diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index a7bffc4..39b6be6 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -342,6 +342,7 @@ barrier_sync(struct barrier *b, struct vtclog *vl) case BARRIER_NONE: vtc_log(vl, 0, "Barrier(%s) use error: not initialized", b->name); + break; case BARRIER_COND: barrier_cond_sync(b, vl); break; From hermunn at varnish-software.com Mon Sep 12 13:48:17 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:17 +0200 Subject: [4.1] d2c30cc Check pthread_cond_signal returns Message-ID: commit d2c30cce453277e9226ee24faf47f894f09ebeaa Author: Guillaume Quintard Date: Tue Aug 30 10:43:32 2016 +0200 Check pthread_cond_signal returns diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index 39b6be6..eb15fe8 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -146,7 +146,7 @@ barrier_sock_thread(void *priv) sock = VTCP_listen_on("127.0.0.1:0", NULL, b->expected, &err); if (sock < 0) { - pthread_cond_signal(&b->cond); + AZ(pthread_cond_signal(&b->cond)); AZ(pthread_mutex_unlock(&b->mtx)); vtc_log(vl, 0, "Barrier(%s) %s fails: %s (errno=%d)", b->name, err, strerror(errno), errno); @@ -159,7 +159,7 @@ barrier_sock_thread(void *priv) macro_def(vl, b->name, "port", "%s", pbuf); macro_def(vl, b->name, "sock", "%s:%s", abuf, pbuf); - pthread_cond_signal(&b->cond); + AZ(pthread_cond_signal(&b->cond)); AZ(pthread_mutex_unlock(&b->mtx)); conns = calloc(b->expected, sizeof *conns); From hermunn at varnish-software.com Mon Sep 12 13:48:17 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:17 +0200 Subject: [4.1] 148a840 Protect against use-after-free/unitialized Message-ID: commit 148a8405132f5a9ced5790a2a1f8adecc46cb195 Author: Guillaume Quintard Date: Tue Aug 30 11:11:35 2016 +0200 Protect against use-after-free/unitialized diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 8f52886..93b366c 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -132,7 +132,7 @@ varnish_ask_cli(const struct varnish *v, const char *cmd, char **repl) static void wait_stopped(const struct varnish *v) { - char *r; + char *r = NULL; enum VCLI_status_e st; while (1) { @@ -146,6 +146,7 @@ wait_stopped(const struct varnish *v) break; } free(r); + r = NULL; (void)usleep(200000); } } @@ -156,7 +157,7 @@ wait_stopped(const struct varnish *v) static void wait_running(const struct varnish *v) { - char *r; + char *r = NULL; enum VCLI_status_e st; while (1) { @@ -176,6 +177,7 @@ wait_running(const struct varnish *v) break; } free(r); + r = NULL; (void)usleep(200000); } } @@ -386,7 +388,7 @@ varnish_launch(struct varnish *v) struct pollfd fd[2]; enum VCLI_status_e u; const char *err; - char *r; + char *r = NULL; v->vd = VSM_New(); @@ -512,6 +514,7 @@ varnish_launch(struct varnish *v) VCLI_AuthResponse(nfd, r, abuf + 5); AZ(close(nfd)); free(r); + r = NULL; strcat(abuf, "\n"); u = varnish_ask_cli(v, abuf, &r); @@ -533,7 +536,7 @@ static void varnish_start(struct varnish *v) { enum VCLI_status_e u; - char *resp, *h, *p; + char *resp = NULL, *h, *p; if (v->cli_fd < 0) varnish_launch(v); @@ -547,6 +550,7 @@ varnish_start(struct varnish *v) vtc_log(v->vl, 0, "CLI start command failed: %u %s", u, resp); wait_running(v); free(resp); + resp = NULL; u = varnish_ask_cli(v, "debug.xid 999", &resp); if (vtc_error) return; @@ -554,6 +558,7 @@ varnish_start(struct varnish *v) vtc_log(v->vl, 0, "CLI debug.xid command failed: %u %s", u, resp); free(resp); + resp = NULL; u = varnish_ask_cli(v, "debug.listen_address", &resp); if (vtc_error) return; From hermunn at varnish-software.com Mon Sep 12 13:48:17 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:17 +0200 Subject: [4.1] be6b018 Typo Message-ID: commit be6b0181aa52387c3b160c4abac185218bb82ef5 Author: Dridi Boukelmoune Date: Fri Sep 2 12:30:27 2016 +0200 Typo diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 26d8216..2b2dc0c 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -88,7 +88,7 @@ SLTM(SessOpen, 0, "Client connection opened", */ SLTM(SessClose, 0, "Client connection closed", - "SessionClose is the last record for any client connection.\n\n" + "SessClose is the last record for any client connection.\n\n" "The format is::\n\n" "\t%s %f\n" "\t| |\n" From hermunn at varnish-software.com Mon Sep 12 13:48:17 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:17 +0200 Subject: [4.1] 8405235 Insist that VCL events are only sent from CLI thread Message-ID: commit 8405235d270ace0003e24e4e9ade51146af34401 Author: Poul-Henning Kamp Date: Mon Sep 5 21:13:07 2016 +0000 Insist that VCL events are only sent from CLI thread diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index b4ad437..37927f7 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -480,6 +480,7 @@ static int vcl_setup_event(VRT_CTX, enum vcl_event_e ev) { + ASSERT_CLI(); CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); @@ -495,6 +496,7 @@ static void vcl_failsafe_event(VRT_CTX, enum vcl_event_e ev) { + ASSERT_CLI(); CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); From hermunn at varnish-software.com Mon Sep 12 13:48:17 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Mon, 12 Sep 2016 15:48:17 +0200 Subject: [4.1] fccc2aa Fix unchecked return Message-ID: commit fccc2aa8fc31f0e1c3d94a7e1d09e8a070648023 Author: Guillaume Quintard Date: Thu Sep 8 11:27:28 2016 +0200 Fix unchecked return diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index 804ef6d..2d01fe0 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -97,7 +97,8 @@ client_thread(void *priv) vtc_log(c->vl, 0, "Failed to open %s: %s", VSB_data(vsb), err); assert(fd >= 0); - VTCP_blocking(fd); + /* VTCP_blocking does its own checks, trust it */ + (void)VTCP_blocking(fd); VTCP_myname(fd, mabuf, sizeof mabuf, mpbuf, sizeof mpbuf); vtc_log(vl, 3, "connected fd %d from %s %s to %s", fd, mabuf, mpbuf, VSB_data(vsb)); From dridi.boukelmoune at gmail.com Mon Sep 12 15:57:13 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Sep 2016 17:57:13 +0200 Subject: [4.1] d255cde Clarify what makes a VCL warm or cold Message-ID: commit d255cde2c1303b646e27cd315bdea57be7fea07f Author: Dridi Boukelmoune Date: Tue Aug 30 20:23:34 2016 +0200 Clarify what makes a VCL warm or cold Conflicts: bin/varnishd/cache/cache_vcl.c diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 37927f7..3a38bc4 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -53,6 +53,10 @@ static const char * const VCL_TEMP_WARM = "warm"; static const char * const VCL_TEMP_BUSY = "busy"; static const char * const VCL_TEMP_COOLING = "cooling"; +/* NB: The COOLING temperature is neither COLD nor WARM. */ +#define VCL_WARM(v) ((v)->temp == VCL_TEMP_WARM || (v)->temp == VCL_TEMP_BUSY) +#define VCL_COLD(v) ((v)->temp == VCL_TEMP_INIT || (v)->temp == VCL_TEMP_COLD) + struct vcl { unsigned magic; #define VCL_MAGIC 0x214188f2 @@ -159,7 +163,7 @@ VCL_Get(struct vcl **vcc) (void)usleep(100000); CHECK_OBJ_NOTNULL(vcl_active, VCL_MAGIC); - assert(vcl_active->temp == VCL_TEMP_WARM); + assert(VCL_WARM(vcl_active)); Lck_Lock(&vcl_mtx); AN(vcl_active); *vcc = vcl_active; @@ -173,7 +177,7 @@ void VCL_Refresh(struct vcl **vcc) { CHECK_OBJ_NOTNULL(vcl_active, VCL_MAGIC); - assert(vcl_active->temp == VCL_TEMP_WARM); + assert(VCL_WARM(vcl_active)); if (*vcc == vcl_active) return; if (*vcc != NULL) @@ -186,7 +190,7 @@ VCL_Ref(struct vcl *vcl) { CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - assert(vcl->temp != VCL_TEMP_INIT && vcl->temp != VCL_TEMP_COLD); + assert(!VCL_COLD(vcl)); Lck_Lock(&vcl_mtx); assert(vcl->busy > 0); vcl->busy++; @@ -229,7 +233,7 @@ VCL_AddBackend(struct vcl *vcl, struct backend *be) VTAILQ_INSERT_TAIL(&vcl->backend_list, be, vcl_list); Lck_Unlock(&vcl_mtx); - if (vcl->temp == VCL_TEMP_WARM || vcl->temp == VCL_TEMP_BUSY) + if (VCL_WARM(vcl)) /* Only when adding backend to already warm VCL */ VBE_Event(be, VCL_EVENT_WARM); else if (vcl->temp != VCL_TEMP_INIT) @@ -417,7 +421,7 @@ VRT_ref_vcl(VRT_CTX, const char *desc) vcl = ctx->vcl; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - xxxassert(vcl->temp == VCL_TEMP_WARM); + assert(VCL_WARM(vcl)); ALLOC_OBJ(ref, VCLREF_MAGIC); AN(ref); @@ -547,8 +551,7 @@ vcl_set_state(VRT_CTX, const char *state) switch(state[0]) { case '0': assert(vcl->temp != VCL_TEMP_COLD); - if (vcl->busy == 0 && (vcl->temp == VCL_TEMP_WARM || - vcl->temp == VCL_TEMP_BUSY)) { + if (vcl->busy == 0 && VCL_WARM(vcl)) { vcl->temp = VTAILQ_EMPTY(&vcl->ref_list) ? VCL_TEMP_COLD : VCL_TEMP_COOLING; From dridi.boukelmoune at gmail.com Mon Sep 12 15:57:13 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Sep 2016 17:57:13 +0200 Subject: [4.1] 3f6de09 Volatile read of the VCL temperature Message-ID: commit 3f6de092898e40346f2f0b1c612e8ed8b9719caa Author: Dridi Boukelmoune Date: Tue Aug 30 21:37:34 2016 +0200 Volatile read of the VCL temperature There is currently no locking around temperature changes despite the fact that it may be read outside of the CLI thread to make decisions. In the case of a panic, instead of locking we simply make sure not to perform an incomplete read of the pointer. diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 3a38bc4..234238b 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -106,7 +106,7 @@ VCL_Panic(struct vsb *vsb, const struct vcl *vcl) VSB_printf(vsb, "busy = %u\n", vcl->busy); VSB_printf(vsb, "discard = %u,\n", vcl->discard); VSB_printf(vsb, "state = %s,\n", vcl->state); - VSB_printf(vsb, "temp = %s,\n", vcl->temp); + VSB_printf(vsb, "temp = %s,\n", (const volatile char *)vcl->temp); VSB_printf(vsb, "conf = {\n"); VSB_indent(vsb, 2); if (vcl->conf == NULL) { From dridi.boukelmoune at gmail.com Mon Sep 12 15:57:13 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Sep 2016 17:57:13 +0200 Subject: [4.1] 65f62a3 Add a read/write lock for VCL temperature Message-ID: commit 65f62a3f9ac5f3136bd1ab308b82cfef94c991ca Author: Dridi Boukelmoune Date: Tue Aug 30 20:31:34 2016 +0200 Add a read/write lock for VCL temperature diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 234238b..a2d90eb 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -34,6 +34,7 @@ #include "config.h" #include +#include #include #include @@ -68,6 +69,7 @@ struct vcl { unsigned busy; unsigned discard; const char *temp; + pthread_rwlock_t temp_rwl; VTAILQ_HEAD(,backend) backend_list; VTAILQ_HEAD(,vclref) ref_list; }; @@ -323,6 +325,7 @@ VCL_Open(const char *fn, struct vsb *msg) } ALLOC_OBJ(vcl, VCL_MAGIC); AN(vcl); + AZ(pthread_rwlock_init(&vcl->temp_rwl, NULL)); vcl->dlh = dlh; vcl->conf = cnf; return (vcl); @@ -337,6 +340,7 @@ VCL_Close(struct vcl **vclp) vcl = *vclp; *vclp = NULL; AZ(dlclose(vcl->dlh)); + AZ(pthread_rwlock_destroy(&vcl->temp_rwl)); FREE_OBJ(vcl); } From dridi.boukelmoune at gmail.com Mon Sep 12 15:57:13 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Sep 2016 17:57:13 +0200 Subject: [4.1] fb4fb34 Remove spurious assert on VCL release Message-ID: commit fb4fb34e503b9c175c781fc2c22835e81d3a60e0 Author: Dridi Boukelmoune Date: Tue Aug 30 21:55:07 2016 +0200 Remove spurious assert on VCL release diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index a2d90eb..7dd6b95 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -455,8 +455,12 @@ VRT_rel_vcl(VRT_CTX, struct vclref **refp) vcl = ctx->vcl; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); assert(vcl == ref->vcl); - assert(vcl->temp == VCL_TEMP_WARM || vcl->temp == VCL_TEMP_BUSY || - vcl->temp == VCL_TEMP_COOLING); + + /* NB: A VCL may be released by a VMOD at any time, but it must happen + * after a warmup and before the end of a cooldown. The release may or + * may not happen while the same thread holds the temperature lock, so + * instead we check that all references are gone in VCL_Nuke. + */ Lck_Lock(&vcl_mtx); assert(!VTAILQ_EMPTY(&vcl->ref_list)); From dridi.boukelmoune at gmail.com Mon Sep 12 15:57:14 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Sep 2016 17:57:14 +0200 Subject: [4.1] 9ae3861 Non volatile reads/writes use the temperature lock Message-ID: commit 9ae386116a33dd77ff62924098453443d0e84569 Author: Dridi Boukelmoune Date: Tue Aug 30 22:02:46 2016 +0200 Non volatile reads/writes use the temperature lock Acquiring a read lock on the CLI thread is unnecessary since a write lock may only be acquired on that very thread. The locking order when both are required is: (struct vcl).temp_rwl => vcl_mtx Fixes #2008 Conflicts: bin/varnishd/cache/cache_vcl.c diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 7dd6b95..26e3372 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -165,7 +165,9 @@ VCL_Get(struct vcl **vcc) (void)usleep(100000); CHECK_OBJ_NOTNULL(vcl_active, VCL_MAGIC); + AZ(pthread_rwlock_rdlock(&vcl_active->temp_rwl)); assert(VCL_WARM(vcl_active)); + AZ(pthread_rwlock_unlock(&vcl_active->temp_rwl)); Lck_Lock(&vcl_mtx); AN(vcl_active); *vcc = vcl_active; @@ -179,7 +181,9 @@ void VCL_Refresh(struct vcl **vcc) { CHECK_OBJ_NOTNULL(vcl_active, VCL_MAGIC); + AZ(pthread_rwlock_rdlock(&vcl_active->temp_rwl)); assert(VCL_WARM(vcl_active)); + AZ(pthread_rwlock_unlock(&vcl_active->temp_rwl)); if (*vcc == vcl_active) return; if (*vcc != NULL) @@ -192,7 +196,9 @@ VCL_Ref(struct vcl *vcl) { CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + AZ(pthread_rwlock_rdlock(&vcl->temp_rwl)); assert(!VCL_COLD(vcl)); + AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); Lck_Lock(&vcl_mtx); assert(vcl->busy > 0); vcl->busy++; @@ -228,8 +234,11 @@ VCL_AddBackend(struct vcl *vcl, struct backend *be) CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); - if (vcl->temp == VCL_TEMP_COOLING) + AZ(pthread_rwlock_rdlock(&vcl->temp_rwl)); + if (vcl->temp == VCL_TEMP_COOLING) { + AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); return (1); + } Lck_Lock(&vcl_mtx); VTAILQ_INSERT_TAIL(&vcl->backend_list, be, vcl_list); @@ -240,6 +249,7 @@ VCL_AddBackend(struct vcl *vcl, struct backend *be) VBE_Event(be, VCL_EVENT_WARM); else if (vcl->temp != VCL_TEMP_INIT) WRONG("Dynamic Backends can only be added to warm VCLs"); + AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); return (0); } @@ -255,8 +265,11 @@ VCL_DelBackend(struct backend *be) Lck_Lock(&vcl_mtx); VTAILQ_REMOVE(&vcl->backend_list, be, vcl_list); Lck_Unlock(&vcl_mtx); - if (vcl->temp == VCL_TEMP_WARM) + + AZ(pthread_rwlock_rdlock(&vcl->temp_rwl)); + if (VCL_WARM(vcl)) VBE_Event(be, VCL_EVENT_COLD); + AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); } static void @@ -554,6 +567,7 @@ vcl_set_state(VRT_CTX, const char *state) assert(ctx->msg != NULL || *state == '0'); vcl = ctx->vcl; + AZ(pthread_rwlock_wrlock(&vcl->temp_rwl)); AN(vcl->temp); switch(state[0]) { @@ -595,6 +609,8 @@ vcl_set_state(VRT_CTX, const char *state) default: WRONG("Wrong enum state"); } + AZ(pthread_rwlock_unlock(&vcl->temp_rwl)); + return (i); } From dridi.boukelmoune at gmail.com Mon Sep 12 15:57:14 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Sep 2016 17:57:14 +0200 Subject: [4.1] 9e66583 Remove diagnostic check on active VCL, rather than increase cost by correctly holding the mtx while examining it. Message-ID: commit 9e66583bca900304d91f4eead6e88907edba12db Author: Poul-Henning Kamp Date: Sat Sep 10 13:14:15 2016 +0000 Remove diagnostic check on active VCL, rather than increase cost by correctly holding the mtx while examining it. diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 26e3372..b40e990 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -181,9 +181,6 @@ void VCL_Refresh(struct vcl **vcc) { CHECK_OBJ_NOTNULL(vcl_active, VCL_MAGIC); - AZ(pthread_rwlock_rdlock(&vcl_active->temp_rwl)); - assert(VCL_WARM(vcl_active)); - AZ(pthread_rwlock_unlock(&vcl_active->temp_rwl)); if (*vcc == vcl_active) return; if (*vcc != NULL) From nils.goroll at uplex.de Mon Sep 12 16:15:19 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 18:15:19 +0200 Subject: [master] 21a5919 Initial merge of the shard director Message-ID: commit 21a5919d85220f44ef806c745a00e19942a0aec5 Author: Nils Goroll Date: Mon Sep 12 17:38:32 2016 +0200 Initial merge of the shard director - rename struct vmod_shard_shard -> vmod_directors_shard - rename typedefs td_shard_* -> td_directors_* - documentation reformatting - remove vbe32dec copy - include generated shard_parse_vcc_* files because enum parse code generation should be included in vmodtool, if at all (do not want an additional build dependency to perl) - make key function a method Source: https://code.uplex.de/uplex-varnish/libvmod-vslp/tree/shard 0f8c3f0f52ca9911d4aed0ce5faf203ab7ff26b0 diff --git a/bin/varnishtest/tests/d00015.vtc b/bin/varnishtest/tests/d00015.vtc new file mode 100644 index 0000000..c3aec0c --- /dev/null +++ b/bin/varnishtest/tests/d00015.vtc @@ -0,0 +1,222 @@ +varnishtest "shard director reconfiguration in init" + +server s1 { +} -start + +server s2 { +} -start + +server s3 { + rxreq + txresp -body "xiuFi3Pe" +} -start + +varnish v1 -vcl+backend { + import std; + import directors; + + sub vcl_init { + new vd = directors.shard(); + new vd2 = directors.shard(); + vd.debug(3); + + std.log("-- invalid replicas"); + if (! vd.reconfigure(replicas=0)) { + std.log("reconfigure failed"); + } + + std.log("-- no changes - no debug output"); + if (! vd.reconfigure(replicas=25)) { + std.log("reconfigure failed"); + } + + std.log("-- no backends"); + if (! vd.clear()) { + std.log("clear failed"); + } + if (! vd.reconfigure(replicas=25)) { + std.log("reconfigure failed"); + } + + std.log("-- one backend"); + if (! vd.add_backend(s1)) { + std.log("add s1 failed"); + } + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- no change - no output"); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- change, clear, no backends"); + # change, then clear + vd.add_backend(s1); + vd.add_backend(s2); + vd.clear(); + vd.add_backend(s1); + vd.add_backend(s2); + vd.clear(); + if (! vd.reconfigure()) { + std.log("reconfigure failed"); + } + + std.log("-- duplicate add"); + vd.clear(); + vd.add_backend(s1); + vd.add_backend(s2); + vd.add_backend(s1); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- duplicate add with idents"); + vd.clear(); + vd.add_backend(s1); + vd.add_backend(s1, ident="s1_1"); + vd.add_backend(s1, ident="s1_2"); + vd.add_backend(s2); + vd.add_backend(s2, ident="s1"); + vd.add_backend(s2, ident="s1_1"); + vd.add_backend(s2, ident="s1_2"); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- remove s1_2 specifically"); + vd.remove_backend(ident="s1_2"); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- remove all instances of s1"); + vd.remove_backend(s1); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- re-add some - no 2nd director"); + vd.clear(); + vd.add_backend(s3, "1"); + vd.add_backend(s3, "2"); + vd.add_backend(s3, "3"); + vd2.clear(); + vd.add_backend(s3, "4"); + vd.add_backend(s3, "5"); + vd.add_backend(s3, "6"); + vd.add_backend(s3, "7"); + vd.add_backend(s3, "8"); + vd.add_backend(s3, "9"); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- remove second-last"); + vd.remove_backend(ident="8"); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- remove last"); + vd.remove_backend(ident="9"); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- END"); + } + + sub vcl_recv { + set req.backend_hint = vd.backend(); + return(pass); + } + +} -start + +logexpect l1 -v v1 -g raw -d 1 { + expect 0 0 CLI {^Rd vcl.load} + + expect 0 0 VCL_Log {^-- invalid replicas$} + expect 0 0 Error {^shard vd: .reconfigure.. invalid replicas argument 0} + expect 0 0 VCL_Log {^reconfigure failed} + + expect 0 0 VCL_Log {^-- no changes - no debug output$} + + expect 0 0 VCL_Log {^-- no backends$} + expect 0 0 Error {^shard vd: .reconfigure.. no backends} + expect 0 0 VCL_Log {^reconfigure failed} + + expect 0 0 VCL_Log {^-- one backend$} + expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} + + expect 0 0 VCL_Log {^-- no change - no output$} + + expect 0 0 VCL_Log {^-- change, clear, no backends$} + expect 0 0 Error {^shard vd: .reconfigure.. no backends} + expect 0 0 VCL_Log {^reconfigure failed} + + expect 0 0 VCL_Log {^-- duplicate add$} + expect 0 0 Error {^shard vd: .notice. backend s1 already exists - skipping$} + expect 0 0 Debug {^shard:.*point = 6e040182, host = 1} + expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} + + expect 0 0 VCL_Log {^-- duplicate add with idents$} + expect 0 0 Error {^shard vd: .notice. backend s1 already exists - skipping} + expect 0 0 Error {^shard vd: .notice. backend s1/s1_1 already exists - skipping} + expect 0 0 Error {^shard vd: .notice. backend s1/s1_2 already exists - skipping} + expect 0 0 Debug {^shard:.*point = 6e040182, host = 3} + expect 0 0 Debug {^shard:.*point = 732c7bbe, host = 2} + expect 0 0 Debug {^shard:.*point = bae80b0b, host = 1} + expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} + + expect 0 0 VCL_Log {^-- remove s1_2 specifically$} + expect 0 0 Debug {^shard:.*point = 6e040182, host = 2} + expect 0 0 Debug {^shard:.*point = bae80b0b, host = 1} + expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} + + expect 0 0 VCL_Log {^-- remove all instances of s1$} + expect 0 0 Debug {^shard:.*point = 6e040182, host = 0} + + expect 0 0 VCL_Log {^-- re-add some - no 2nd director$} + expect 0 0 Error {^shard vd2: cannot change more than} + expect 0 0 Debug {^shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^shard:.*point = 6337e62c, host = 8} + expect 0 0 Debug {^shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^shard:.*point = f493ce58, host = 2} + expect 0 0 Debug {^shard:.*point = fc1a5162, host = 7} + + expect 0 0 VCL_Log {^-- remove second-last$} + expect 0 0 Debug {^shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^shard:.*point = 6337e62c, host = 7} + expect 0 0 Debug {^shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^shard:.*point = f493ce58, host = 2} + + expect 0 0 VCL_Log {^-- remove last$} + expect 0 0 Debug {^shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^shard:.*point = f493ce58, host = 2} + +expect 0 0 VCL_Log {^-- END$} +} -start + +client c1 { + txreq + rxresp + expect resp.body == "xiuFi3Pe" +} -run + +logexpect l1 -wait diff --git a/bin/varnishtest/tests/d00016.vtc b/bin/varnishtest/tests/d00016.vtc new file mode 100644 index 0000000..2e5406d --- /dev/null +++ b/bin/varnishtest/tests/d00016.vtc @@ -0,0 +1,222 @@ +varnishtest "shard director/int reconfiguration outside init" + +server s1 { +} -start + +server s2 { +} -start + +server s3 { + rxreq + txresp -body "xiuFi3Pe" +} -start + +varnish v1 -vcl+backend { + import std; + import directors; + + sub vcl_init { + new vd = directors.shard(); + new vd2 = directors.shard(); + vd.debug(3); + } + + sub vcl_recv { + std.log("-- invalid replicas"); + if (! vd.reconfigure(replicas=0)) { + std.log("reconfigure failed"); + } + + std.log("-- no changes - no debug output"); + if (! vd.reconfigure(replicas=25)) { + std.log("reconfigure failed"); + } + + std.log("-- no backends"); + if (! vd.clear()) { + std.log("clear failed"); + } + if (! vd.reconfigure(replicas=25)) { + std.log("reconfigure failed"); + } + + std.log("-- one backend"); + if (! vd.add_backend(s1)) { + std.log("add s1 failed"); + } + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- no change - no output"); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- change, clear, no backends"); + # change, then clear + vd.add_backend(s1); + vd.add_backend(s2); + vd.clear(); + vd.add_backend(s1); + vd.add_backend(s2); + vd.clear(); + if (! vd.reconfigure()) { + std.log("reconfigure failed"); + } + + std.log("-- duplicate add"); + vd.clear(); + vd.add_backend(s1); + vd.add_backend(s2); + vd.add_backend(s1); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- duplicate add with idents"); + vd.clear(); + vd.add_backend(s1); + vd.add_backend(s1, ident="s1_1"); + vd.add_backend(s1, ident="s1_2"); + vd.add_backend(s2); + vd.add_backend(s2, ident="s1"); + vd.add_backend(s2, ident="s1_1"); + vd.add_backend(s2, ident="s1_2"); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- remove s1_2 specifically"); + vd.remove_backend(ident="s1_2"); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- remove all instances of s1"); + vd.remove_backend(s1); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- re-add some - no 2nd director"); + vd.clear(); + vd.add_backend(s3, "1"); + vd.add_backend(s3, "2"); + vd.add_backend(s3, "3"); + vd2.clear(); + vd.add_backend(s3, "4"); + vd.add_backend(s3, "5"); + vd.add_backend(s3, "6"); + vd.add_backend(s3, "7"); + vd.add_backend(s3, "8"); + vd.add_backend(s3, "9"); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- remove second-last"); + vd.remove_backend(ident="8"); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- remove last"); + vd.remove_backend(ident="9"); + if (! vd.reconfigure(replicas=1)) { + std.log("reconfigure failed"); + } + + std.log("-- END"); + + set req.backend_hint = vd.backend(); + return(pass); + } + +} -start + +# this is identical to v01100.vtc, but split into two logexps + +logexpect l1 -v v1 -g raw -d 1 { + expect * 1001 VCL_Log {^-- invalid replicas$} + expect 0 1001 Error {^shard vd: .reconfigure.. invalid replicas argument 0} + expect 0 1001 VCL_Log {^reconfigure failed} + + expect 0 1001 VCL_Log {^-- no changes - no debug output$} + + expect 0 1001 VCL_Log {^-- no backends$} + expect 0 1001 Error {^shard vd: .reconfigure.. no backends} + expect 0 1001 VCL_Log {^reconfigure failed} + + expect 0 1001 VCL_Log {^-- one backend$} + + expect 0 1001 VCL_Log {^-- no change - no output$} + + expect 0 1001 VCL_Log {^-- change, clear, no backends$} + expect 0 1001 Error {^shard vd: .reconfigure.. no backends} + expect 0 1001 VCL_Log {^reconfigure failed} + + expect 0 1001 VCL_Log {^-- duplicate add$} + expect 0 1001 Error {^shard vd: .notice. backend s1 already exists - skipping$} + expect 0 1001 VCL_Log {^-- duplicate add with idents$} + expect 0 1001 Error {^shard vd: .notice. backend s1 already exists - skipping} + expect 0 1001 Error {^shard vd: .notice. backend s1/s1_1 already exists - skipping} + expect 0 1001 Error {^shard vd: .notice. backend s1/s1_2 already exists - skipping} + expect 0 1001 VCL_Log {^-- remove s1_2 specifically$} + expect 0 1001 VCL_Log {^-- remove all instances of s1$} + expect 0 1001 VCL_Log {^-- re-add some - no 2nd director$} + expect 0 1001 Error {^shard vd2: cannot change more than} + expect 0 1001 VCL_Log {^-- remove second-last$} + expect 0 1001 VCL_Log {^-- remove last$} + expect 0 1001 VCL_Log {^-- END$} +} + +logexpect l2 -v v1 -g raw -d 1 { + expect * 0 Debug {^shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^shard:.*point = 6e040182, host = 1} + expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^shard:.*point = 6e040182, host = 3} + expect 0 0 Debug {^shard:.*point = 732c7bbe, host = 2} + expect 0 0 Debug {^shard:.*point = bae80b0b, host = 1} + expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^shard:.*point = 6e040182, host = 2} + expect 0 0 Debug {^shard:.*point = bae80b0b, host = 1} + expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^shard:.*point = 6e040182, host = 0} + expect 0 0 Debug {^shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^shard:.*point = 6337e62c, host = 8} + expect 0 0 Debug {^shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^shard:.*point = f493ce58, host = 2} + expect 0 0 Debug {^shard:.*point = fc1a5162, host = 7} + expect 0 0 Debug {^shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^shard:.*point = 6337e62c, host = 7} + expect 0 0 Debug {^shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^shard:.*point = f493ce58, host = 2} + expect 0 0 Debug {^shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^shard:.*point = f493ce58, host = 2} +} + +logexpect l1 -start +logexpect l2 -start + +client c1 { + txreq + rxresp + expect resp.body == "xiuFi3Pe" +} -run + +logexpect l1 -wait +logexpect l2 -wait diff --git a/bin/varnishtest/tests/d00017.vtc b/bin/varnishtest/tests/d00017.vtc new file mode 100644 index 0000000..d92cad5 --- /dev/null +++ b/bin/varnishtest/tests/d00017.vtc @@ -0,0 +1,158 @@ +varnishtest "shard director regression to vslp v01000" + +server s1 { + rxreq + txresp -body "ech3Ooj" +} -start + +server s2 { + rxreq + txresp -body "ieQu2qua" +} -start + +server s3 { + rxreq + txresp -body "xiuFi3Pe" +} -start + +# VSLP: +# picked preferred backend 0 for key 68b902f8 +# picked preferred backend 1 for key 39dc4614 +# picked preferred backend 2 for key c7793506 +# +# shard: + +varnish v1 -vcl+backend { + import std; + import directors; + + sub vcl_init { + new vd = directors.shard(); + vd.debug(3); + if (! vd.add_backend(s1)) { + std.log("add s1 failed"); + } + if (! vd.add_backend(s2)) { + std.log("add s2 failed"); + } + if (! vd.add_backend(s3)) { + std.log("add s3 failed"); + } + if (! vd.reconfigure(replicas=25)) { + std.log("reconfigure failed"); + } + } + + sub vcl_recv { + set req.backend_hint = vd.backend(by=KEY, + key=vd.key(req.url, CRC32)); + return(pass); + } + +} -start + +# check that the hashcircle points are decremented by one compared to vslp +# +# generated from vslp test output +# perl -ne /tmp/regression.logexp + +logexpect l1 -v v1 -g raw -d 1 { + expect 0 0 CLI "^Rd vcl.load" + expect 0 = Debug {^shard: hashcircle.* 0. = .point = 3b6b56a, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 1. = .point = 66986a7, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 2. = .point = 7e41e30, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 3. = .point = b749e7b, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 4. = .point = e543430, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 5. = .point = 10136c05, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 6. = .point = 102d847f, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 7. = .point = 1112f910, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 8. = .point = 1119a7c7, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 9. = .point = 22464ee9, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 10. = .point = 22b35675, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 11. = .point = 2363bebb, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 12. = .point = 259eeccf, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 13. = .point = 26f0c3e7, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 14. = .point = 28340f35, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 15. = .point = 285e8475, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 16. = .point = 28ec7a6f, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 17. = .point = 2da0e37b, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 18. = .point = 3392487a, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 19. = .point = 37597c4c, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 20. = .point = 4b1f5b22, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 21. = .point = 523723f2, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 22. = .point = 539234db, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 23. = .point = 564ca84f, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 24. = .point = 5d2df428, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 25. = .point = 5fa294ee, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 26. = .point = 60dded53, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 27. = .point = 6257bc27, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 28. = .point = 64014b25, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 29. = .point = 6bfd5a2d, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 30. = .point = 6e040182, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 31. = .point = 6e3819f7, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 32. = .point = 7232b381, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 33. = .point = 74c384ad, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 34. = .point = 83ce71ce, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 35. = .point = 888b6447, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 36. = .point = 8997c018, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 37. = .point = 8aa6b5b4, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 38. = .point = 8b47e6ac, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 39. = .point = 8bc8bc11, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 40. = .point = 8e2d3849, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 41. = .point = 8e7e012c, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 42. = .point = 99892987, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 43. = .point = 9a6f2f00, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 44. = .point = 9ef9125d, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 45. = .point = 9f33cd30, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 46. = .point = 9fc69b51, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 47. = .point = a19f99eb, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 48. = .point = a28b9595, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 49. = .point = a8afe9c4, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 50. = .point = ad923ad3, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 51. = .point = ae8946c6, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 52. = .point = b197e339, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 53. = .point = b3c305e6, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 54. = .point = b6bf43ea, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 55. = .point = b9004d3d, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 56. = .point = bbcc0bad, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 57. = .point = c2542a5d, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 58. = .point = c6c43fa7, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 59. = .point = c945958a, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 60. = .point = cbd9198a, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 61. = .point = d4c93105, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 62. = .point = d7de63b6, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 63. = .point = d937a7df, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 64. = .point = dac52229, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 65. = .point = db7840f0, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 66. = .point = dd5c6bef, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 67. = .point = dfd5333b, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 68. = .point = e991584c, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 69. = .point = ec8891c5, host = 1.} + expect 0 = Debug {^shard: hashcircle.* 70. = .point = ef6b4ab5, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 71. = .point = f08ad325, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 72. = .point = f3325ba2, host = 2.} + expect 0 = Debug {^shard: hashcircle.* 73. = .point = f6530dd1, host = 0.} + expect 0 = Debug {^shard: hashcircle.* 74. = .point = fc28e8d2, host = 2.} + + expect 0 = CLI Loaded + + expect * = Debug {^shard: lookup key 68b902f8 idx 29 host 0} + expect * = Debug {^shard: lookup key 39dc4614 idx 20 host 1} + expect * = Debug {^shard: lookup key c7793506 idx 59 host 2} +} -start + +client c1 { + txreq -url /eishoSu2 + rxresp + expect resp.body == "ech3Ooj" + + txreq -url /Zainao9d + rxresp + expect resp.body == "ieQu2qua" + + txreq -url /Aunah3uo + rxresp + expect resp.body == "xiuFi3Pe" +} -run + +logexpect l1 -wait diff --git a/bin/varnishtest/tests/d00018.vtc b/bin/varnishtest/tests/d00018.vtc new file mode 100644 index 0000000..3ca77be --- /dev/null +++ b/bin/varnishtest/tests/d00018.vtc @@ -0,0 +1,66 @@ +varnishtest "shard director/int key" + +server s1 { + rxreq + txresp -body "ech3Ooj" +} -start + +server s2 { + rxreq + txresp -body "ieQu2qua" +} -start + +server s3 { + rxreq + txresp -body "xiuFi3Pe" +} -start + +varnish v1 -vcl+backend { + import std; + import directors; + + sub vcl_init { + new vd = directors.shard(); + if (! vd.add_backend(s1)) { + std.log("add s1 failed"); + } + if (! vd.add_backend(s2)) { + std.log("add s2 failed"); + } + if (! vd.add_backend(s3)) { + std.log("add s3 failed"); + } + if (! vd.reconfigure(replicas=25)) { + std.log("reconfigure failed"); + } + } + + sub vcl_recv { + if(req.url == "/1") { + set req.backend_hint = vd.backend(by=KEY, key=1); + } + if(req.url == "/2") { + set req.backend_hint = vd.backend(by=KEY, key=2147483647); + } + if(req.url == "/3") { + set req.backend_hint = vd.backend(by=KEY, key=4294967295); + } + return(pass); + } + +} -start + + +client c1 { + txreq -url /1 + rxresp + expect resp.body == "ech3Ooj" + + txreq -url /2 + rxresp + expect resp.body == "ieQu2qua" + + txreq -url /3 + rxresp + expect resp.body == "xiuFi3Pe" +} -run diff --git a/bin/varnishtest/tests/d00019.vtc b/bin/varnishtest/tests/d00019.vtc new file mode 100644 index 0000000..5635a1c --- /dev/null +++ b/bin/varnishtest/tests/d00019.vtc @@ -0,0 +1,49 @@ +varnishtest "shard director SHA256 (default)" + +server s1 { + rxreq + txresp -body "ech3Ooj" +} -start + +server s2 { + rxreq + txresp -body "ieQu2qua" +} -start + +server s3 { + rxreq + txresp -body "xiuFi3Pe" +} -start + +varnish v1 -vcl+backend { + import directors; + + sub vcl_init { + new vd = directors.shard(); + vd.add_backend(s1); + vd.add_backend(s2); + vd.add_backend(s3); + vd.reconfigure(replicas=25); + } + + sub vcl_recv { + set req.backend_hint = vd.backend(); + return(pass); + } + +} -start + + +client c1 { + txreq -url /Boo0aixe + rxresp + expect resp.body == "ech3Ooj" + + txreq -url /eishoSu2 + rxresp + expect resp.body == "ieQu2qua" + + txreq -url /Aunah3uo + rxresp + expect resp.body == "xiuFi3Pe" +} -run diff --git a/bin/varnishtest/tests/d00020.vtc b/bin/varnishtest/tests/d00020.vtc new file mode 100644 index 0000000..1fe91c7 --- /dev/null +++ b/bin/varnishtest/tests/d00020.vtc @@ -0,0 +1,50 @@ +varnishtest "shard director RS" + +server s1 { + rxreq + txresp -body "ech3Ooj" +} -start + +server s2 { + rxreq + txresp -body "ieQu2qua" +} -start + +server s3 { + rxreq + txresp -body "xiuFi3Pe" +} -start + +varnish v1 -vcl+backend { + import directors; + + sub vcl_init { + new vd = directors.shard(); + vd.add_backend(s1); + vd.add_backend(s2); + vd.add_backend(s3); + vd.reconfigure(replicas=25); + } + + sub vcl_recv { + set req.backend_hint = vd.backend(by=KEY, + key=vd.key(req.url, alg=RS)); + return(pass); + } + +} -start + + +client c1 { + txreq -url /we0eeTho + rxresp + expect resp.body == "ech3Ooj" + + txreq -url /mae8ooNu + rxresp + expect resp.body == "ieQu2qua" + + txreq -url /oob3dahS + rxresp + expect resp.body == "xiuFi3Pe" +} -run diff --git a/bin/varnishtest/tests/d00021.vtc b/bin/varnishtest/tests/d00021.vtc new file mode 100644 index 0000000..9258abb --- /dev/null +++ b/bin/varnishtest/tests/d00021.vtc @@ -0,0 +1,81 @@ +varnishtest "shard director key function" + +server s1 { + rxreq + txresp -body "ech3Ooj" + rxreq + txresp -body "ech3Ooj" + rxreq + txresp -body "ech3Ooj" +} -start + +server s2 { + rxreq + txresp -body "ieQu2qua" +} -start + +server s3 { + rxreq + txresp -body "xiuFi3Pe" +} -start + +varnish v1 -vcl+backend { + import directors; + + sub vcl_init { + new vd = directors.shard(); + vd.add_backend(s1); + vd.add_backend(s2); + vd.add_backend(s3); + vd.reconfigure(25); + } + + sub recv_sub { + set req.backend_hint = vd.backend(by=KEY, + key=vd.key(req.http.X-Hash, RS)); + } + + sub vcl_recv { + if(req.url == "/1") { + set req.backend_hint = vd.backend(by=KEY, + key=vd.key(alg=CRC32, string="/eishoSu2")); + } else if (req.url == "/2") { + set req.backend_hint = vd.backend(by=KEY, + key=vd.key("/eishoSu2")); + } else if (req.url == "/3") { + set req.http.X-Hash = "/oob3dahS"; + call recv_sub; + } else if (req.url == "/null_by_string") { + set req.backend_hint = vd.backend(by=KEY, + key=vd.key(req.http.NonExistent)); + } else if (req.url == "/null_by_string_hash") { + set req.backend_hint = vd.backend(by=KEY, + key=vd.key(req.http.NonExistent, SHA256)); + } + return(pass); + } + +} -start + + +client c1 { + txreq -url /1 + rxresp + expect resp.body == "ech3Ooj" + + txreq -url /2 + rxresp + expect resp.body == "ieQu2qua" + + txreq -url /3 + rxresp + expect resp.body == "xiuFi3Pe" + + txreq -url /null_by_string + rxresp + expect resp.body == "ech3Ooj" + + txreq -url /null_by_string_hash + rxresp + expect resp.body == "ech3Ooj" +} -run diff --git a/bin/varnishtest/tests/d00022.vtc b/bin/varnishtest/tests/d00022.vtc new file mode 100644 index 0000000..0aa1fad --- /dev/null +++ b/bin/varnishtest/tests/d00022.vtc @@ -0,0 +1,77 @@ +varnishtest "shard director Restarts" + +server s1 { + rxreq + txresp -body "ech3Ooj" + rxreq + txresp -body "ech3Ooj" + rxreq + txresp -body "ech3Ooj" +} -start + +server s2 { + rxreq + txresp -body "ieQu2qua" + rxreq + txresp -body "ieQu2qua" + rxreq + txresp -body "ieQu2qua" +} -start + +server s3 { + rxreq + txresp -body "xiuFi3Pe" + rxreq + txresp -body "xiuFi3Pe" + rxreq + txresp -body "xiuFi3Pe" +} -start + +varnish v1 -vcl+backend { + import directors; + + sub vcl_init { + new vd = directors.shard(); + vd.add_backend(s1); + vd.add_backend(s2); + vd.add_backend(s3); + vd.reconfigure(25); + } + + sub vcl_recv { + set req.backend_hint = vd.backend(by=KEY, + key=vd.key("/eishoSu2", CRC32), + alt=req.restarts, + healthy=ALL); + + if(req.url == "/2" && req.restarts > 0) { + unset req.http.vrstart; + } + if(req.url == "/3" && req.restarts > 1) { + unset req.http.vrstart; + } + return(pass); + } + + sub vcl_deliver { + if(req.http.vrstart) { + return(restart); + } + } + +} -start + + +client c1 { + txreq -url /1 + rxresp + expect resp.body == "ech3Ooj" + + txreq -url /2 -hdr "vrstart: 1" + rxresp + expect resp.body == "ieQu2qua" + + txreq -url /3 -hdr "vrstart: 1" + rxresp + expect resp.body == "xiuFi3Pe" +} -run diff --git a/bin/varnishtest/tests/d00023.vtc b/bin/varnishtest/tests/d00023.vtc new file mode 100644 index 0000000..e427264 --- /dev/null +++ b/bin/varnishtest/tests/d00023.vtc @@ -0,0 +1,55 @@ +varnishtest "shard director Unhealthy" + +server s1 { + rxreq + txresp -body "ech3Ooj" +} -start + +server s2 { + rxreq + expect req.http.healthy == "true" + txresp -body "ieQu2qua" +} -start + +server s3 { + rxreq + expect req.http.healthy == "true" + txresp -body "xiuFi3Pe" +} -start + +varnish v1 -vcl+backend { + import std; + import directors; + + sub vcl_init { + new vd = directors.shard(); + vd.debug(3); + vd.add_backend(s1); + vd.add_backend(s2); + vd.add_backend(s3); + vd.reconfigure(25); + } + + sub vcl_recv { + set req.backend_hint = vd.backend(by=KEY, + key=vd.key("/eishoSu2", CRC32)); + set req.http.healthy = std.healthy(req.backend_hint); + return(pass); + } +} -start + +varnish v1 -cliok "backend.set_health s1 sick" + +client c1 { + txreq + rxresp + expect resp.body == "ieQu2qua" +} -run + +varnish v1 -cliok "backend.set_health s2 sick" + +client c1 { + txreq + rxresp + expect resp.body == "xiuFi3Pe" +} -run diff --git a/bin/varnishtest/tests/d00024.vtc b/bin/varnishtest/tests/d00024.vtc new file mode 100644 index 0000000..57f76da --- /dev/null +++ b/bin/varnishtest/tests/d00024.vtc @@ -0,0 +1,75 @@ +varnishtest "shard director Rampup Time" + +# NOTE: this test is timing sensitive by nature + + +server s1 { + rxreq + txresp -body "ech3Ooj" +} -start + +server s2 { + rxreq + txresp -body "ieQu2qua" + rxreq + txresp -body "ieQu2qua" +} -start + +server s3 { + rxreq + txresp -body "xiuFi3Pe" +} -start + +varnish v1 -vcl+backend { + import directors; + + sub vcl_init { + new vd = directors.shard(); + vd.debug(4); + vd.add_backend(s1); + vd.add_backend(s2); + vd.add_backend(s3); + vd.set_rampup(15s); + vd.reconfigure(25); + } + + sub vcl_recv { + set req.backend_hint = vd.backend(by=KEY, + key=vd.key(alg=CRC32, string="/eishoSu2")); + return(pass); + } +} -start + +varnish v1 -cliok "debug.srandom" + +delay 8 + +varnish v1 -cliok "backend.set_health s1 sick" + +# s1 down, s2 and s3 both in rampup - s2 chosen +client c1 { + txreq -url /1 + rxresp + expect resp.body == "ieQu2qua" +} -run + + +delay 8 + +varnish v1 -cliok "backend.set_health s1 healthy" + +# s1 just came up, s2 chosen for ramup +client c1 { + txreq -url /2 + rxresp + expect resp.body == "ieQu2qua" +} -run + +delay 16 + +# s1 out of ramup +client c1 { + txreq -url /3 + rxresp + expect resp.body == "ech3Ooj" +} -run diff --git a/bin/varnishtest/tests/d00025.vtc b/bin/varnishtest/tests/d00025.vtc new file mode 100644 index 0000000..bf06d68 --- /dev/null +++ b/bin/varnishtest/tests/d00025.vtc @@ -0,0 +1,29 @@ +varnishtest "shard - single backend works" + +server s1 { + rxreq + txresp -body "ech3Ooj" +} -start + +varnish v1 -vcl+backend { + import directors; + + sub vcl_init { + new vd = directors.shard(); + vd.add_backend(s1); + vd.reconfigure(1); + } + + sub vcl_recv { + set req.backend_hint = vd.backend(); + return(pass); + } + +} -start + + +client c1 { + txreq -url /eishoSu2 + rxresp + expect resp.body == "ech3Ooj" +} -run diff --git a/bin/varnishtest/tests/d00026.vtc b/bin/varnishtest/tests/d00026.vtc new file mode 100644 index 0000000..64d63bc --- /dev/null +++ b/bin/varnishtest/tests/d00026.vtc @@ -0,0 +1,50 @@ +varnishtest "shard director - same as v01000.vtc but setting backend in fetch" + +server s1 { + rxreq + txresp -body "ech3Ooj" +} -start + +server s2 { + rxreq + txresp -body "ieQu2qua" +} -start + +server s3 { + rxreq + txresp -body "xiuFi3Pe" +} -start + +varnish v1 -vcl+backend { + import directors; + + sub vcl_init { + new vd = directors.shard(); + vd.add_backend(s1); + vd.add_backend(s2); + vd.add_backend(s3); + vd.reconfigure(25); + } + + sub vcl_backend_fetch { + set bereq.backend = vd.backend(by=KEY, + key=vd.key(bereq.url, CRC32)); + return(fetch); + } + +} -start + + +client c1 { + txreq -url /eishoSu2 + rxresp + expect resp.body == "ech3Ooj" + + txreq -url /Zainao9d + rxresp + expect resp.body == "ieQu2qua" + + txreq -url /Aunah3uo + rxresp + expect resp.body == "xiuFi3Pe" +} -run diff --git a/doc/changes.rst b/doc/changes.rst index 412cbb5..9bcafa6 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -25,6 +25,8 @@ Major items: Documentation, counters and log entries still refer to the mechanism as hit-for-pass and Hitpass, respectively. +* new shard director for loadbalancing by consistent hashing + ====================================== Varnish Cache 4.1.3-beta1 (2016-06-15) ====================================== diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am index dbd238a..76e6528 100644 --- a/lib/libvmod_directors/Makefile.am +++ b/lib/libvmod_directors/Makefile.am @@ -24,7 +24,18 @@ libvmod_directors_la_SOURCES = \ fall_back.c \ hash.c \ random.c \ - round_robin.c + round_robin.c \ + vmod_shard.c \ + shard_cfg.c \ + shard_cfg.h \ + shard_dir.c \ + shard_dir.h \ + shard_hash.c \ + shard_hash.h \ + shard_parse_vcc_enums.h \ + shard_parse_vcc_enums.c \ + crc32.c \ + crc32.h nodist_libvmod_directors_la_SOURCES = \ vcc_if.c \ diff --git a/lib/libvmod_directors/crc32.c b/lib/libvmod_directors/crc32.c new file mode 100644 index 0000000..714b0f0 --- /dev/null +++ b/lib/libvmod_directors/crc32.c @@ -0,0 +1,112 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2008 Linpro AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: crc32.c 3599 2009-02-05 10:13:52Z tfheen $ + * + * This CRC32 implementation is in the public domain. + */ + +#include "config.h" +#include "crc32.h" + +/*--------------------------------------------------------------------*/ + +static const uint32_t crc32bits[] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, + 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, + 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, + 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, + 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, + 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, + 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, + 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, + 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, + 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, + 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, + 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d +}; + +uint32_t +crc32(uint32_t crc, const void *p1, unsigned l) +{ + const unsigned char *p; + + for (p = (const unsigned char*)p1; l-- > 0; p++) + crc = (crc >> 8) ^ crc32bits[(crc ^ *p) & 0xff]; + return (crc); +} + +uint32_t +crc32_l(const void *p1, unsigned l) +{ + uint32_t crc; + + crc = crc32(~0U, p1, l); + return (crc ^ ~0U); +} + +#ifdef DEBUG + +#include +int main(int argc, char *argv[]) { + if (argc > 1) + printf("%x\n", crc32_l(argv[1], strlen(argv[1]))); +} + +#endif diff --git a/lib/libvmod_directors/crc32.h b/lib/libvmod_directors/crc32.h new file mode 100644 index 0000000..39adf8a --- /dev/null +++ b/lib/libvmod_directors/crc32.h @@ -0,0 +1,3 @@ +#include +uint32_t crc32(uint32_t crc, const void *p1, unsigned l); +uint32_t crc32_l(const void *p1, unsigned l); diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c new file mode 100644 index 0000000..ecfb936 --- /dev/null +++ b/lib/libvmod_directors/shard_cfg.c @@ -0,0 +1,663 @@ +/*- + * Copyright 2009-2016 UPLEX - Nils Goroll Systemoptimierung + * All rights reserved. + * + * Authors: Nils Goroll + * Geoffrey Simmons + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include + +#include "cache/cache.h" +#include "cache/cache_director.h" + +#include "vrt.h" + +#include "shard_dir.h" +#include "shard_cfg.h" +#include "shard_hash.h" + +enum shard_change_task_e { + _INVALID = 0, + CLEAR, + ADD_BE, + REMOVE_BE, + _SHARD_TASK_E_MAX +}; + +struct shard_change_task { + unsigned magic; +#define SHARD_CHANGE_TASK_MAGIC 0x1e1168af + enum shard_change_task_e task; + void *priv; + VSTAILQ_ENTRY(shard_change_task) list; +}; + +struct shard_change { + unsigned magic; +#define SHARD_CHANGE_MAGIC 0xdff5c9a6 + const struct sharddir *shardd; + void *space; + VSTAILQ_HEAD(,shard_change_task) tasks; +}; + +struct backend_reconfig { + struct sharddir * const shardd; + int hint; // on number of backends after reconfig + int hole_n; // number of holes in backends array + int hole_i; // index hint on first hole +}; + +/* + * ============================================================ + * change / task list + * + * for backend reconfiguration, we create a change list on the VCL workspace in + * a PRIV_TASK state, which we work in reconfigure. + * + * for now, we allow to only reconfigure one shard director at a time. + */ + +static struct shard_change * +shard_change_get(VRT_CTX, struct vmod_priv *priv, + const struct sharddir * const shardd) +{ + struct shard_change *change; + + if (priv->priv) { + CAST_OBJ_NOTNULL(change, priv->priv, SHARD_CHANGE_MAGIC); + if (change->shardd == NULL) { + change->shardd = shardd; + VSTAILQ_INIT(&change->tasks); + } else if (change->shardd != shardd) { + shard_err0(ctx, shardd, + "cannot change more than one shard director " + "at a time"); + return NULL; + } + return (change); + } + + change = WS_Alloc(ctx->ws, sizeof(*change)); + if (change == NULL) { + shard_err0(ctx, shardd, "could not get workspace"); + return NULL; + } + + INIT_OBJ(change, SHARD_CHANGE_MAGIC); + change->space = NULL; + change->shardd = shardd; + VSTAILQ_INIT(&change->tasks); + priv->priv = change; + + return (change); +} + +static void +shard_change_finish(struct shard_change *change) +{ + CHECK_OBJ_NOTNULL(change, SHARD_CHANGE_MAGIC); + + change->shardd = NULL; + VSTAILQ_INIT(&change->tasks); +} + +static void +shard_change_task_add(VRT_CTX, struct shard_change *change, + enum shard_change_task_e task_e, void *priv) +{ + struct shard_change_task *task; + + CHECK_OBJ_NOTNULL(change, SHARD_CHANGE_MAGIC); + + task = WS_Alloc(ctx->ws, sizeof(*task)); + if (task == NULL) { + shard_err0(ctx, change->shardd, + "could not get workspace for task"); + return; + } + INIT_OBJ(task, SHARD_CHANGE_TASK_MAGIC); + task->task = task_e; + task->priv = priv; + VSTAILQ_INSERT_TAIL(&change->tasks, task, list); +} + +static inline VCL_BOOL +shard_change_task_backend(VRT_CTX, + struct vmod_priv *priv, struct sharddir *shardd, + enum shard_change_task_e task_e, VCL_BACKEND be, VCL_STRING ident, + VCL_DURATION rampup) +{ + struct shard_change *change; + struct shard_backend *b; + + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + assert(task_e == ADD_BE || task_e == REMOVE_BE); + + change = shard_change_get(ctx, priv, shardd); + if (change == NULL) + return 0; + + b = WS_Alloc(ctx->ws, sizeof(*b)); + if (b == NULL) { + shard_err(ctx, shardd, ".%s_backend() WS_Alloc() failed", + task_e == ADD_BE ? "add" : "remove"); + return 0; + } + + b->backend = be; + b->ident = ident != NULL && *ident != '\0' ? ident : NULL; + b->rampup = rampup; + + shard_change_task_add(ctx, change, task_e, b); + + return 1; +} + +/* + * ============================================================ + * director reconfiguration tasks + */ +VCL_BOOL +shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv, struct sharddir *shardd, + VCL_BACKEND be, VCL_STRING ident, VCL_DURATION rampup) +{ + AN(be); + return shard_change_task_backend(ctx, priv, shardd, ADD_BE, + be, ident, rampup); +} + +VCL_BOOL +shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv, + struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident) +{ + return shard_change_task_backend(ctx, priv, shardd, REMOVE_BE, + be, ident, 0); +} + +VCL_BOOL +shardcfg_clear(VRT_CTX, struct vmod_priv *priv, struct sharddir *shardd) +{ + struct shard_change *change; + + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + + change = shard_change_get(ctx, priv, shardd); + if (change == NULL) + return 0; + + shard_change_task_add(ctx, change, CLEAR, NULL); + + return 1; +} + +/* + * ============================================================ + * consistent hashing cirle init + */ + +typedef int (*compar)( const void*, const void* ); + +static int +circlepoint_compare(struct shard_circlepoint *a, struct shard_circlepoint *b) +{ + return (a->point == b->point) ? 0 : ((a->point > b->point) ? 1 : -1); +} + +static void +shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas, enum alg_e alg) +{ + int i, j; + const char *ident; + int len; + + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + AZ(shardd->hashcircle); + + assert(shardd->n_backend > 0); + AN(shardd->backend); + + shardd->hashcircle = calloc(shardd->n_backend * replicas, + sizeof(struct shard_circlepoint)); + AN(shardd->hashcircle); + + shardd->replicas = replicas; + + for (i = 0; i < shardd->n_backend; i++) { + CHECK_OBJ_NOTNULL(shardd->backend[i].backend, DIRECTOR_MAGIC); + + ident = shardd->backend[i].ident + ? shardd->backend[i].ident + : shardd->backend[i].backend->vcl_name; + + assert(ident[0] != '\0'); + + len = strlen(ident) + log10(UINT32_MAX) + 2; + + char s[len]; + + for (j = 0; j < replicas; j++) { + sprintf(s, "%s%d", ident, j); + shardd->hashcircle[i * replicas + j].point = + shard_hash_f[alg](s); + shardd->hashcircle[i * replicas + j].host = i; + } + /* not used in current interface */ + shardd->backend[i].canon_point = + shardd->hashcircle[i * replicas].point; + } + qsort( (void *) shardd->hashcircle, shardd->n_backend * replicas, + sizeof (struct shard_circlepoint), (compar) circlepoint_compare); + + if ((shardd->debug_flags & SHDBG_CIRCLE) == 0) + return; + + for (i = 0; i < shardd->n_backend; i++) + for (j = 0; j < replicas; j++) + SHDBG(SHDBG_CIRCLE, shardd, + "hashcircle[%5ld] = " + "{point = %8x, host = %2d}\n", + i * replicas + j, + shardd->hashcircle[i * replicas + j].point, + shardd->hashcircle[i * replicas + j].host); +} + +/* + * ============================================================ + * configure the director backends + */ + +static void +shardcfg_backend_free(struct shard_backend *f) +{ + if (f->ident) + free (TRUST_ME(f->ident)); + memset(f, 0, sizeof(*f)); +} + +static void +shardcfg_backend_copyin(struct shard_backend *dst, + const struct shard_backend *src) +{ + dst->backend = src->backend; + dst->ident = src->ident ? strdup(src->ident) : NULL; + dst->rampup = src->rampup; + dst->canon_point = 0xffffffff; +} + +static int +shardcfg_backend_cmp(const struct shard_backend *a, + const struct shard_backend *b) +{ + const char *ai, *bi; + + ai = a->ident; + bi = b->ident; + + /* vcl_names are unique, so we can compare the backend pointers */ + if (ai == NULL && bi == NULL) + return a->backend != b->backend; + + if (ai == NULL) + ai = a->backend->vcl_name; + + if (bi == NULL) + bi = b->backend->vcl_name; + + return strcmp(ai, bi); +} + +/* for removal, we delete all instances if the backend matches */ +static int +shardcfg_backend_del_cmp(const struct shard_backend *task, + const struct shard_backend *b) +{ + if (task->backend && task->ident == NULL) + return task->backend != b->backend; + + return shardcfg_backend_cmp(task, b); +} + +static const struct shard_backend * +shardcfg_backend_lookup(struct backend_reconfig *re, + const struct shard_backend *b) +{ + int i, max = re->shardd->n_backend + re->hole_n; + const struct shard_backend *bb = re->shardd->backend; + + for (i = 0; i < max; i++) + if (! shardcfg_backend_cmp(b, &bb[i])) + return &bb[i]; + + return NULL; +} + +static void +shardcfg_backend_expand(struct backend_reconfig *re) +{ + int min = re->hint; + + CHECK_OBJ_NOTNULL(re->shardd, SHARDDIR_MAGIC); + + if (min < 16) + min = 16; + + if (re->shardd->l_backend < min) + re->shardd->l_backend = min; + else + re->shardd->l_backend <<= 1; + + if (re->shardd->backend) + re->shardd->backend = realloc(re->shardd->backend, + re->shardd->l_backend * sizeof *re->shardd->backend); + else + re->shardd->backend = malloc( + re->shardd->l_backend * sizeof *re->shardd->backend); + + AN(re->shardd->backend); +} + +static void +shardcfg_backend_add(struct backend_reconfig *re, + const struct shard_backend *b) +{ + int i; + struct shard_backend *bb = re->shardd->backend; + + if (re->hole_n == 0) { + if (re->shardd->n_backend >= re->shardd->l_backend) { + shardcfg_backend_expand(re); + bb = re->shardd->backend; + } + assert(re->shardd->n_backend < re->shardd->l_backend); + i = re->shardd->n_backend; + } else { + do { + if (! bb[re->hole_i].backend) + break; + } while (++(re->hole_i) < re->shardd->n_backend + re->hole_n); + assert(re->hole_i < re->shardd->n_backend + re->hole_n); + + i = (re->hole_i)++; + (re->hole_n)--; + } + + re->shardd->n_backend++; + shardcfg_backend_copyin(&bb[i], b); + return; +} + +static void +shardcfg_backend_clear(struct sharddir *shardd) +{ + int i; + for (i = 0; i < shardd->n_backend; i++) + shardcfg_backend_free(&shardd->backend[i]); + shardd->n_backend = 0; +} + + +static void +shardcfg_backend_del(struct backend_reconfig *re, + const struct shard_backend *spec) +{ + int i, max = re->shardd->n_backend + re->hole_n; + struct shard_backend * const bb = re->shardd->backend; + + for (i = 0; i < max; i++) { + if (shardcfg_backend_del_cmp(spec, &bb[i])) + continue; + + shardcfg_backend_free(&bb[i]); + re->shardd->n_backend--; + if (i < re->shardd->n_backend + re->hole_n) { + (re->hole_n)++; + if (i < re->hole_i) + re->hole_i = i; + } + } +} + +static void +shardcfg_backend_finalize(struct backend_reconfig *re) +{ + int i; + struct shard_backend * const bb = re->shardd->backend; + + while (re->hole_n > 0) { + // trim end + i = re->shardd->n_backend + re->hole_n - 1; + while (re->hole_n && bb[i].backend == NULL) { + (re->hole_n)--; + i--; + } + + if (re->hole_n == 0) + break; + + assert(re->hole_i < i); + + do { + if (! bb[re->hole_i].backend) + break; + } while (++(re->hole_i) <= i); + + assert(re->hole_i < i); + assert(bb[re->hole_i].backend == NULL); + assert(bb[i].backend != NULL); + + memcpy(&bb[re->hole_i], &bb[i], sizeof(*bb)); + memset(&bb[i], 0, sizeof(*bb)); + + (re->hole_n)--; + (re->hole_i)++; + } + + assert(re->hole_n == 0); +} + +/* + * ============================================================ + * work the change tasks + */ + +static void +shardcfg_apply_change(VRT_CTX, struct sharddir *shardd, + const struct shard_change *change) +{ + struct shard_change_task *task, *clear; + const struct shard_backend *b; + + struct backend_reconfig re = { + .shardd = shardd, + .hint = shardd->n_backend, + .hole_n = 0, + .hole_i = INT_MAX + }; + + // XXX assert sharddir_locked(shardd) + + clear = NULL; + VSTAILQ_FOREACH(task, &change->tasks, list) { + CHECK_OBJ_NOTNULL(task, SHARD_CHANGE_TASK_MAGIC); + switch (task->task) { + case CLEAR: + clear = task; + re.hint = 0; + break; + case ADD_BE: + re.hint++; + break; + case REMOVE_BE: + re.hint--; + break; + default: + INCOMPL(); + } + } + + if (clear) { + shardcfg_backend_clear(shardd); + clear = VSTAILQ_NEXT(clear, list); + if (clear == NULL) + return; + } + + task = clear; + VSTAILQ_FOREACH_FROM(task, &change->tasks, list) { + CHECK_OBJ_NOTNULL(task, SHARD_CHANGE_TASK_MAGIC); + switch (task->task) { + case CLEAR: + assert(task->task != CLEAR); + break; + case ADD_BE: + b = shardcfg_backend_lookup(&re, task->priv); + + if (b == NULL) { + shardcfg_backend_add(&re, task->priv); + break; + } + + const char * const ident = b->ident; + + shard_err(ctx, shardd, "(notice) backend %s%s%s " + "already exists - skipping", + b->backend->vcl_name, + ident ? "/" : "", + ident ? ident : ""); + break; + case REMOVE_BE: + shardcfg_backend_del(&re, task->priv); + break; + default: + INCOMPL(); + } + } + shardcfg_backend_finalize(&re); +} + +/* + * ============================================================ + * top reconfiguration function + */ + +VCL_BOOL +shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv, + struct sharddir *shardd, VCL_INT replicas, enum alg_e alg) +{ + struct shard_change *change; + + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + if (replicas <= 0) { + shard_err(ctx, shardd, + ".reconfigure() invalid replicas argument %ld", replicas); + return 0; + } + + change = shard_change_get(ctx, priv, shardd); + if (change == NULL) + return 0; + + if (VSTAILQ_FIRST(&change->tasks) == NULL) + return 1; + + sharddir_wrlock(shardd); + + shardcfg_apply_change(ctx, shardd, change); + shard_change_finish(change); + + if (shardd->hashcircle) + free(shardd->hashcircle); + shardd->hashcircle = NULL; + + if (shardd->n_backend == 0) { + shard_err0(ctx, shardd, ".reconfigure() no backends"); + sharddir_unlock(shardd); + return 0; + } + + shardcfg_hashcircle(shardd, replicas, alg); + sharddir_unlock(shardd); + return (1); +} + +/* + * ============================================================ + * misc config related + */ + +/* only for sharddir_delete() */ +void +shardcfg_delete(struct sharddir *shardd) +{ + int i; + + for (i = 0; i < shardd->n_backend; i++) + shardcfg_backend_free(&shardd->backend[i]); + if (shardd->backend) + free(shardd->backend); + if (shardd->hashcircle) + free(shardd->hashcircle); +} + +VCL_VOID +shardcfg_set_warmup(struct sharddir *shardd, VCL_REAL ratio) +{ + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + assert(ratio >= 0 && ratio < 1); + sharddir_wrlock(shardd); + shardd->warmup = ratio; + sharddir_unlock(shardd); +} + +VCL_VOID +shardcfg_set_rampup(struct sharddir *shardd, VCL_DURATION duration) +{ + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + assert(duration >= 0); + sharddir_wrlock(shardd); + shardd->rampup_duration = duration; + sharddir_unlock(shardd); +} + +VCL_DURATION +shardcfg_get_rampup(struct sharddir *shardd, int host) +{ + VCL_DURATION r; + + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + // assert sharddir_rdlock_held(shardd); + assert (host < shardd->n_backend); + + // magic value for default + if (shardd->backend[host].rampup == 973279260) + r = shardd->rampup_duration; + else + r = shardd->backend[host].rampup; + + return (r); +} diff --git a/lib/libvmod_directors/shard_cfg.h b/lib/libvmod_directors/shard_cfg.h new file mode 100644 index 0000000..ca50ddd --- /dev/null +++ b/lib/libvmod_directors/shard_cfg.h @@ -0,0 +1,40 @@ +/*- + * Copyright 2016 UPLEX - Nils Goroll Systemoptimierung + * All rights reserved. + * + * Author: Nils Goroll + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +VCL_BOOL shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv, + struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident, + VCL_DURATION rampup); +VCL_BOOL shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv, + struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident); +VCL_BOOL shardcfg_clear(VRT_CTX, struct vmod_priv *priv, + struct sharddir *shardd); +VCL_BOOL shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv, + struct sharddir *shardd, VCL_INT replicas, enum alg_e alg_e); +VCL_VOID shardcfg_set_warmup(struct sharddir *shardd, VCL_REAL ratio); +VCL_VOID shardcfg_set_rampup(struct sharddir *shardd, + VCL_DURATION duration); diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c new file mode 100644 index 0000000..57a9495 --- /dev/null +++ b/lib/libvmod_directors/shard_dir.c @@ -0,0 +1,395 @@ +/*- + * Copyright 2009-2016 UPLEX - Nils Goroll Systemoptimierung + * All rights reserved. + * + * Authors: Nils Goroll + * Geoffrey Simmons + * Julian Wiesener + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#include "cache/cache.h" +#include "cache/cache_director.h" + +#include "vrt.h" +#include "vbm.h" +#include "vrnd.h" + +#include "shard_dir.h" +#include "shard_hash.h" + +struct shard_be_info { + int hostid; + unsigned healthy; + double changed; // when +}; + +/* + * circle walk state for shard_next + * + * pick* cut off the search after having seen all possible backends + */ +struct shard_state { + const struct vrt_ctx *ctx; + struct sharddir *shardd; + int idx; + + struct vbitmap *picklist; + int pickcount; + + struct shard_be_info previous; + struct shard_be_info last; +}; + +void +sharddir_debug(struct sharddir *shardd, const uint32_t flags) +{ + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + shardd->debug_flags = flags; +} + +void +sharddir_err(VRT_CTX, enum VSL_tag_e tag, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + if (ctx->vsl) + VSLbv(ctx->vsl, tag, fmt, ap); + else + VSLv(tag, 0, fmt, ap); + va_end(ap); +} + +static int +shard_lookup(const struct sharddir *shardd, const uint32_t key) +{ + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + + const int n = shardd->n_backend * shardd->replicas; + int idx = -1, high = n, low = 0, i; + + do { + i = (high + low) >> 1; + if (shardd->hashcircle[i].point == key) + idx = i; + else if (i == n - 1) + idx = n - 1; + else if (shardd->hashcircle[i].point < key && + shardd->hashcircle[i+1].point >= key) + idx = i + 1; + else if (shardd->hashcircle[i].point > key) + if (i == 0) + idx = 0; + else + high = i; + else + low = i; + } while (idx == -1); + + return idx; +} + +static int +shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy) +{ + int c, chosen = -1; + uint32_t ringsz; + VCL_BACKEND be; + double changed; + struct shard_be_info *sbe; + + AN(state); + assert(state->idx >= 0); + CHECK_OBJ_NOTNULL(state->shardd, SHARDDIR_MAGIC); + + if (state->pickcount >= state->shardd->n_backend) + return -1; + + ringsz = state->shardd->n_backend * state->shardd->replicas; + + while (state->pickcount < state->shardd->n_backend && skip >= 0) { + + c = state->shardd->hashcircle[state->idx].host; + + if (! vbit_test(state->picklist, c)) { + + vbit_set(state->picklist, c); + state->pickcount++; + + sbe = NULL; + be = state->shardd->backend[c].backend; + AN(be); + if (be->healthy(be, state->ctx->bo, &changed)) { + if (skip-- == 0) { + chosen = c; + sbe = &state->last; + } else { + sbe = &state->previous; + } + + } else if (!healthy && skip-- == 0) { + chosen = c; + sbe = &state->last; + } + if (sbe == &state->last && + state->last.hostid != -1) + memcpy(&state->previous, &state->last, + sizeof(state->previous)); + + if (sbe) { + sbe->hostid = c; + sbe->healthy = 1; + sbe->changed = changed; + } + if (chosen != -1) + break; + } + + if (++(state->idx) == ringsz) + state->idx = 0; + } + return chosen; +} + +void +sharddir_new(struct sharddir **sharddp, const char *vcl_name) +{ + struct sharddir *shardd; + + AN(vcl_name); + AN(sharddp); + AZ(*sharddp); + ALLOC_OBJ(shardd, SHARDDIR_MAGIC); + AN(shardd); + *sharddp = shardd; + shardd->name = vcl_name; + AZ(pthread_rwlock_init(&shardd->mtx, NULL)); +} + +void +sharddir_delete(struct sharddir **sharddp) +{ + struct sharddir *shardd; + + AN(sharddp); + shardd = *sharddp; + *sharddp = NULL; + + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + shardcfg_delete(shardd); + AZ(pthread_rwlock_destroy(&shardd->mtx)); + FREE_OBJ(shardd); +} + +void +sharddir_rdlock(struct sharddir *shardd) +{ + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + AZ(pthread_rwlock_rdlock(&shardd->mtx)); +} + +void +sharddir_wrlock(struct sharddir *shardd) +{ + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + AZ(pthread_rwlock_wrlock(&shardd->mtx)); +} + +void +sharddir_unlock(struct sharddir *shardd) +{ + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + AZ(pthread_rwlock_unlock(&shardd->mtx)); +} + +static inline void +validate_alt(VRT_CTX, struct sharddir *shardd, VCL_INT *alt) +{ + const VCL_INT alt_max = shardd->n_backend - 1; + + if (*alt < 0) { + shard_err(ctx, shardd, + "invalid negative parameter alt=%ld, set to 0", *alt); + *alt = 0; + } else if (*alt > alt_max) { + shard_err(ctx, shardd, + "parameter alt=%ld limited to %ld", *alt, alt_max); + *alt = alt_max; + } +} + +static inline void +init_state(struct shard_state *state, + VRT_CTX, struct sharddir *shardd, struct vbitmap *picklist) +{ + AN(picklist); + + state->ctx = ctx; + state->shardd = shardd; + state->idx = -1; + state->picklist = picklist; + state->pickcount = 0; + + /* healhy and changed only defined for hostid != -1 */ + state->previous.hostid = -1; + state->last.hostid = -1; +} + +/* + * core function for the director backend method + * + * while other directors return a reference to their own backend object (on + * which varnish will call the resolve method to resolve to a non-director + * backend), this director immediately reolves in the backend method, to make + * the director choice visible in VCL + * + * consequences: + * - we need no own struct director + * - we can only respect a busy object when being called on the backend side, + * which probably is, for all practical purposes, only relevant when the + * saintmode vmod is used + * + * if we wanted to offer delayed resolution, we'd need something like + * per-request per-director state or we'd need to return a dynamically created + * director object. That should be straight forward once we got director + * refcounting #2072. Until then, we could create it on the workspace, but then + * we'd need to keep other directors from storing any references to our dynamic + * object for longer than the current task + * + */ +VCL_BACKEND +sharddir_pick_be(VRT_CTX, struct sharddir *shardd, + uint32_t key, VCL_INT alt, VCL_REAL warmup, VCL_BOOL rampup, + enum healthy_e healthy) +{ + VCL_BACKEND be; + struct shard_state state; + unsigned picklist_sz = VBITMAP_SZ(shardd->n_backend); + char picklist_spc[picklist_sz]; + VCL_DURATION chosen_r, alt_r; + + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(ctx->vsl); + + sharddir_rdlock(shardd); + if(shardd->n_backend == 0) { + shard_err0(ctx, shardd, "no backends"); + goto err; + } + + assert(shardd->hashcircle); + + validate_alt(ctx, shardd, &alt); + + init_state(&state, ctx, shardd, vbit_init(picklist_spc, picklist_sz)); + + state.idx = shard_lookup(shardd, key); + assert(state.idx >= 0); + + SHDBG(SHDBG_LOOKUP, shardd, "lookup key %x idx %d host %d", + key, state.idx, shardd->hashcircle[state.idx].host); + + if (alt > 0) { + if (shard_next(&state, alt - 1, healthy == ALL) == -1) { + if (state.previous.hostid != -1) { + be = sharddir_backend(shardd, + state.previous.hostid); + goto ok; + } + goto err; + } + } + + if (shard_next(&state, 0, healthy != IGNORE) == -1) { + if (state.previous.hostid != -1) { + be = sharddir_backend(shardd, state.previous.hostid); + goto ok; + } + goto err; + } + + be = sharddir_backend(shardd, state.last.hostid); + + if (warmup == -1) + warmup = shardd->warmup; + + /* short path for cases we dont want ramup/warmup or can't */ + if (alt > 0 || healthy == IGNORE || (! rampup && warmup == 0) || + shard_next(&state, 0, 0) == -1) + goto ok; + + assert(alt == 0); + assert(state.previous.hostid >= 0); + assert(state.last.hostid >= 0); + assert(state.previous.hostid != state.last.hostid); + assert(be == sharddir_backend(shardd, state.previous.hostid)); + + chosen_r = shardcfg_get_rampup(shardd, state.previous.hostid); + alt_r = shardcfg_get_rampup(shardd, state.last.hostid); + + SHDBG(SHDBG_RAMPWARM, shardd, "chosen host %d rampup %f changed %f", + state.previous.hostid, chosen_r, + ctx->now - state.previous.changed); + SHDBG(SHDBG_RAMPWARM, shardd, "alt host %d rampup %f changed %f", + state.last.hostid, alt_r, + ctx->now - state.last.changed); + + if (ctx->now - state.previous.changed < chosen_r) { + /* + * chosen host is in rampup + * - no change if alternative host is also in rampup or the dice + * has rolled in favour of the chosen host + */ + if (! rampup || + ctx->now - state.last.changed < alt_r || + VRND_RandomTestableDouble() * chosen_r < + (ctx->now - state.previous.changed)) + goto ok; + } else { + /* chosen host not in rampup - warmup ? */ + if (warmup == 0 || VRND_RandomTestableDouble() > warmup) + goto ok; + } + + be = sharddir_backend(shardd, state.last.hostid); + + ok: + AN(be); + sharddir_unlock(shardd); + vbit_destroy(state.picklist); + return (be); + err: + sharddir_unlock(shardd); + vbit_destroy(state.picklist); + return NULL; +} diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h new file mode 100644 index 0000000..b66fca1 --- /dev/null +++ b/lib/libvmod_directors/shard_dir.h @@ -0,0 +1,117 @@ +/*- + * Copyright 2009-2016 UPLEX - Nils Goroll Systemoptimierung + * All rights reserved. + * + * Authors: Julian Wiesener + * Nils Goroll + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "shard_parse_vcc_enums.h" + +struct vbitmap; + +struct shard_circlepoint { + uint32_t point; + unsigned int host; +}; + +struct shard_backend { + VCL_BACKEND backend; + const char *ident; // XXX COPY IN ! + VCL_DURATION rampup; + uint32_t canon_point; +}; + +#define SHDBG_LOOKUP 1 +#define SHDBG_CIRCLE (1<<1) +#define SHDBG_RAMPWARM (1<<2) + +struct sharddir { + unsigned magic; +#define SHARDDIR_MAGIC 0xdbb7d59f + uint32_t debug_flags; + + pthread_rwlock_t mtx; + + const char *name; + + unsigned n_backend; + unsigned l_backend; + struct shard_backend *backend; + + struct shard_circlepoint *hashcircle; + + VCL_DURATION rampup_duration; + VCL_REAL warmup; + VCL_INT replicas; +}; + +static inline VCL_BACKEND +sharddir_backend(const struct sharddir *shardd, int id) +{ + assert(id >= 0); + assert(id < shardd->n_backend); + return (shardd->backend[id].backend); +} + +static inline const char * +sharddir_backend_ident(const struct sharddir *shardd, int host) +{ + assert(host >= 0); + assert(host < shardd->n_backend); + return (shardd->backend[host].ident); +} + +#define SHDBG(flag, shardd, ...) \ + do { \ + if ((shardd)->debug_flags & (flag)) \ + VSL(SLT_Debug, 0, "shard: " __VA_ARGS__); \ + } while (0) + +#define shard_err(ctx, shardd, fmt, ...) \ + do { \ + sharddir_err(ctx, SLT_Error, "shard %s: " fmt, \ + (shardd)->name, __VA_ARGS__); \ + } while (0) + +#define shard_err0(ctx, shardd, s) \ + do { \ + sharddir_err(ctx, SLT_Error, "shard %s: %s", \ + (shardd)->name, (s)); \ + } while (0) + +void sharddir_debug(struct sharddir *shardd, const uint32_t flags); +void sharddir_err(VRT_CTX, enum VSL_tag_e tag, const char *fmt, ...); +void sharddir_new(struct sharddir **sharddp, const char *vcl_name); +void sharddir_delete(struct sharddir **sharddp); +void sharddir_init_hashcircle(struct sharddir *shardd, VCL_INT replicas); +void sharddir_rdlock(struct sharddir *shardd); +void sharddir_wrlock(struct sharddir *shardd); +void sharddir_unlock(struct sharddir *shardd); +VCL_BACKEND sharddir_pick_be(VRT_CTX, struct sharddir *, uint32_t, VCL_INT, + VCL_REAL, VCL_BOOL, enum healthy_e); + +/* in shard_cfg.c */ +void shardcfg_delete(struct sharddir *shardd); +VCL_DURATION shardcfg_get_rampup(struct sharddir *shardd, int host); diff --git a/lib/libvmod_directors/shard_hash.c b/lib/libvmod_directors/shard_hash.c new file mode 100644 index 0000000..9fc207e --- /dev/null +++ b/lib/libvmod_directors/shard_hash.c @@ -0,0 +1,105 @@ +/*- + * Copyright 2009-2013 UPLEX - Nils Goroll Systemoptimierung + * All rights reserved. + * + * Authors: Nils Goroll + * Geoffrey Simmons + * Julian Wiesener + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include + +#include "cache/cache.h" + +#include "vrt.h" +#include "crc32.h" +#include "vsha256.h" + +#include "shard_parse_vcc_enums.h" +#include "shard_hash.h" + +static uint32_t __match_proto__(hash_func) +shard_hash_crc32(VCL_STRING s) +{ + uint32_t crc; + crc = crc32(~0U, (const unsigned char*)s, strlen(s)); + crc ^= ~0U; + crc++; + + return (crc); +} + +static uint32_t __match_proto__(hash_func) +shard_hash_sha256(VCL_STRING s) +{ + struct SHA256Context sha256; + union { + unsigned char digest[32]; + uint32_t uint32_digest[8]; + } sha256_digest; + + SHA256_Init(&sha256); + SHA256_Update(&sha256, s, strlen(s)); + SHA256_Final(sha256_digest.digest, &sha256); + + /* + * use low 32 bits only + * XXX: Are these the best bits to pick? + */ + return (sha256_digest.uint32_digest[7]); +} + +static uint32_t __match_proto__(hash_func) +shard_hash_rs(VCL_STRING s) +{ + uint32_t res = 0; + /* hash function from Robert Sedgwicks 'Algorithms in C' book */ + const uint32_t b = 378551; + uint32_t a = 63689; + + while (*s) { + res = res * a + (*s++); + a *= b; + } + + return (res); +} + +static uint32_t __match_proto__(hash_func) +_shard_hash_invalid(VCL_STRING s) +{ + (void) s; + WRONG("invalid hash fp _ALG_E_ENVALID"); + return (0); +} + +hash_func shard_hash_f[_ALG_E_MAX] = { + [_ALG_E_INVALID] = _shard_hash_invalid, + [CRC32] = shard_hash_crc32, + [SHA256] = shard_hash_sha256, + [RS] = shard_hash_rs +}; diff --git a/lib/libvmod_directors/shard_hash.h b/lib/libvmod_directors/shard_hash.h new file mode 100644 index 0000000..5ee38ad --- /dev/null +++ b/lib/libvmod_directors/shard_hash.h @@ -0,0 +1,30 @@ +/*- + * Copyright 2009-2013 UPLEX - Nils Goroll Systemoptimierung + * All rights reserved. + * + * Author: Julian Wiesener + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +typedef uint32_t (*hash_func)(VCL_STRING); +extern hash_func shard_hash_f[_ALG_E_MAX]; diff --git a/lib/libvmod_directors/shard_parse_vcc_enums.c b/lib/libvmod_directors/shard_parse_vcc_enums.c new file mode 100644 index 0000000..5dec48e --- /dev/null +++ b/lib/libvmod_directors/shard_parse_vcc_enums.c @@ -0,0 +1,146 @@ +/* + * for the time being, this code is auto-generated outside the varnishd source + * tree, see + * https://code.uplex.de/uplex-varnish/libvmod-vslp/blob/shard/src/gen_enum_parse.pl + * + * TODO: integrate in vmodtool.py or replace with something else + */ + +#include "shard_parse_vcc_enums.h" +#define term(c) ((c) == '\0') + + + +enum alg_e parse_alg_e (const char *m) { + int p; + enum alg_e r; + + switch (m[0]) { + case 'C': goto _0C; // CRC32 + case 'R': goto _0R; // RS + case 'S': goto _0S; // SHA256 + default: goto invalid; + } + _0C: + //CRC32 + if ((m[1] == 'R') && (m[2] == 'C') && (m[3] == '3') && (m[4] == '2') && (term(m[5]))) { + r = CRC32; + p = 5; + goto ok; + } + goto invalid; + _0R: + //RS + if ((m[1] == 'S') && (term(m[2]))) { + r = RS; + p = 2; + goto ok; + } + goto invalid; + _0S: + //SHA256 + if ((m[1] == 'H') && (m[2] == 'A') && (m[3] == '2') && (m[4] == '5') && (m[5] == '6') && (term(m[6]))) { + r = SHA256; + p = 6; + goto ok; + } + goto invalid; + ok: + return r; + invalid: + return _ALG_E_INVALID; + (void)p; +} + + +enum by_e parse_by_e (const char *m) { + int p; + enum by_e r; + + switch (m[0]) { + case 'B': goto _0B; // BLOB + case 'H': goto _0H; // HASH + case 'K': goto _0K; // KEY + case 'U': goto _0U; // URL + default: goto invalid; + } + _0B: + //BLOB + if ((m[1] == 'L') && (m[2] == 'O') && (m[3] == 'B') && (term(m[4]))) { + r = BLOB; + p = 4; + goto ok; + } + goto invalid; + _0H: + //HASH + if ((m[1] == 'A') && (m[2] == 'S') && (m[3] == 'H') && (term(m[4]))) { + r = HASH; + p = 4; + goto ok; + } + goto invalid; + _0K: + //KEY + if ((m[1] == 'E') && (m[2] == 'Y') && (term(m[3]))) { + r = KEY; + p = 3; + goto ok; + } + goto invalid; + _0U: + //URL + if ((m[1] == 'R') && (m[2] == 'L') && (term(m[3]))) { + r = URL; + p = 3; + goto ok; + } + goto invalid; + ok: + return r; + invalid: + return _BY_E_INVALID; + (void)p; +} + + +enum healthy_e parse_healthy_e (const char *m) { + int p; + enum healthy_e r; + + switch (m[0]) { + case 'A': goto _0A; // ALL + case 'C': goto _0C; // CHOSEN + case 'I': goto _0I; // IGNORE + default: goto invalid; + } + _0A: + //ALL + if ((m[1] == 'L') && (m[2] == 'L') && (term(m[3]))) { + r = ALL; + p = 3; + goto ok; + } + goto invalid; + _0C: + //CHOSEN + if ((m[1] == 'H') && (m[2] == 'O') && (m[3] == 'S') && (m[4] == 'E') && (m[5] == 'N') && (term(m[6]))) { + r = CHOSEN; + p = 6; + goto ok; + } + goto invalid; + _0I: + //IGNORE + if ((m[1] == 'G') && (m[2] == 'N') && (m[3] == 'O') && (m[4] == 'R') && (m[5] == 'E') && (term(m[6]))) { + r = IGNORE; + p = 6; + goto ok; + } + goto invalid; + ok: + return r; + invalid: + return _HEALTHY_E_INVALID; + (void)p; +} diff --git a/lib/libvmod_directors/shard_parse_vcc_enums.h b/lib/libvmod_directors/shard_parse_vcc_enums.h new file mode 100644 index 0000000..aa097dc --- /dev/null +++ b/lib/libvmod_directors/shard_parse_vcc_enums.h @@ -0,0 +1,42 @@ +/* + * for the time being, this code is auto-generated outside the varnishd source + * tree, see + * https://code.uplex.de/uplex-varnish/libvmod-vslp/blob/shard/src/gen_enum_parse.pl + * + * TODO: integrate in vmodtool.py or replace with something else + */ + +enum alg_e { + _ALG_E_INVALID = 0, + CRC32, + SHA256, + RS, + _ALG_E_MAX +}; + + +enum alg_e parse_alg_e (const char *); + +enum by_e { + _BY_E_INVALID = 0, + HASH, + URL, + KEY, + BLOB, + _BY_E_MAX +}; + + +enum by_e parse_by_e (const char *); + +enum healthy_e { + _HEALTHY_E_INVALID = 0, + CHOSEN, + IGNORE, + ALL, + _HEALTHY_E_MAX +}; + + +enum healthy_e parse_healthy_e (const char *); + diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index 4adbfba..4348ba1 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -1,8 +1,15 @@ #- +# This document is licensed under the same licence as Varnish +# itself. See LICENCE for details. +# # Copyright (c) 2013-2015 Varnish Software AS +# Copyright 2009-2016 UPLEX - Nils Goroll Systemoptimierung # All rights reserved. # -# Author: Poul-Henning Kamp +# Authors: Poul-Henning Kamp +# Julian Wiesener +# Nils Goroll +# Geoffrey Simmons # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -32,9 +39,9 @@ DESCRIPTION `vmod_directors` enables backend load balancing in Varnish. -The module implements a set of basic load balancing techniques, and -also serves as an example on how one could extend the load balancing -capabilities of Varnish. +The module implements load balancing techniques, and also serves as an +example on how one could extend the load balancing capabilities of +Varnish. To enable load balancing you must import this vmod (directors). @@ -213,10 +220,302 @@ Example # pick a backend based on the cookie header from the client set req.backend_hint = vdir.backend(req.http.cookie); -COPYRIGHT -========= +$Object shard() + +Create a shard director. + +Note that the shard director needs to be configured using at least one +``shard.add_backend()`` call(s) **followed by a** +``shard.reconfigure()`` **call** before it can hand out backends. + +Introduction +```````````` + +The shard director selects backends by a key, which can be provided +directly or derived from strings. For the same key, the shard director +will always return the same backend, unless the backend configuration +or health state changes. Conversely, for differing keys, the shard +director will likely choose different backends. In the default +configuration, unhealthy backends are not selected. + +The shard director resembles the hash director, but its main advantage +is that, when the backend configuration or health states change, the +association of keys to backends remains as stable as possible. + +In addition, the rampup and warmup features can help to further +improve user-perceived response times. + +Sharding +```````` + +This basic technique allows for numerious applications like optimizing +backend server cache efficiency, Varnish clustering or persisting +sessions to servers without keeping any state, and, in particular, +without the need to synchronize state between nodes of a cluster of +Varnish servers: + +* Many applications use caches for data objects, so, in a cluster of + application servers, requesting similar objects from the same server + may help to optimize efficiency of such caches. + + For example, sharding by URL or some `id` component of the url has + been shown to drastically improve the efficiency of many content + management systems. + +* As special case of the previous example, in clusters of Varnish + servers without additional request distribution logic, each cache + will need store all hot objects, so the effective cache size is + approximately the smallest cache size of any server in the cluster. + + Sharding allows to segregate objects within the cluster such that + each object is only cached on one of the servers (or on one primary + and one backup, on a primary for long and others for short + etc...). Effectively, this will lead to a cache size in the order of + the sum of all individual caches, with the potential to drastically + increase efficiency (scales by the number of servers). + +* Another application is to implement persistence of backend requests, + such that all requests sharing a certain criterium (such as an IP + address or session ID) get forwarded to the same backend server. + +When used with clusters of varnish servers, the shard director will, +if otherwise configured equally, make the same decision on all +servers. In other words, requests sharing a common criterium used as +the shard key will be balanced onto the same backend server(s) no +matter which Varnish server handles the request. + +The drawbacks are: + +* the distribution of requests depends on the number of requests per + key and the uniformity of the distribution of key values. In short, + while this technique may lead to much better efficiency overall, it + may also lead to less good load balancing for specific cases. + +* When a backend server becomes unavailable, every persistence + technique has to reselect a new backend server, but this technique + will also switch back to the preferred server once it becomes + healthy again, so when used for persistence, it is generally less + stable compared to stateful techniques (which would continue to use + a selected server for as long as possible (or dictated by a TTL)). + +Method +`````` + +When ``.reconfigure()`` is called, a consistent hashing circular data +structure gets built from hash values of "ident%d" (default ident +being the backend name) for each backend and for a running number from +1 to n (n is the number of `replicas`). Hashing creates the seemingly +random order for placement of backends on the consistent hashing ring. + +When ``.backend()`` is called, a load balacing key gets generated +unless provided. The smallest hash value in the circle is looked up +that is larger than the key (searching clockwise and wrapping around +as necessary). The backend for this hash value is the preferred +backend for the given key. + +If a healhy backend is requested, the search is continued linearly on +the ring as long as backends found are unhealthy or all backends have +been checked. The order auf these "alternative backends" on the ring +is likely to differ for different keys. Alternative backends can also +be selected explicitly. + +On consistent hashing see: + +* http://www8.org/w8-papers/2a-webserver/caching/paper2.html +* http://www.audioscrobbler.net/development/ketama/ +* svn://svn.audioscrobbler.net/misc/ketama +* http://en.wikipedia.org/wiki/Consistent_hashing + +Error Reporting +``````````````` + +Failing methods should report errors to VSL with the Error tag, so +when configuring the shard director, you are advised to check:: + + varnishlog -I Error:^shard + +$Method VOID .set_warmup(REAL probability=0.0) + +Set the default warmup probability. See the `warmup` parameter of +``shard.backend()``. + +Default: 0.0 (no warmup) + +$Method VOID .set_rampup(DURATION duration=0) + +Set the default rampup duration. See `rampup` parameter of +`shard.backend()`. + +Default: 0s (no rampup) + +$Method BOOL .add_backend(PRIV_TASK, BACKEND backend, + STRING ident=0, DURATION rampup=973279260) + +Add a backend `backend` to the director. + +`ident`: Optionally specify an identification string for this backend, +which will be hashed by `shard.reconfigure()` to construct the +consistent hashing ring. The identification string defaults to the +backend name. + +`ident` allows to add multuple instances of the same backend. + +`rampup`: Optionally specify a rampup time for this backend. Defaults +to the director rampup time. + +NOTE: Backend changes need to be finalized with `shard.reconfigure()` +and are only supported on one shard director at a time. + +$Method BOOL .remove_backend(PRIV_TASK, BACKEND backend=0, STRING ident=0) + +Remove backend(s) from the director. Either `backend` or `ident` must +be specified. `ident` removes a specific instance. If `backend` is +given without `ident`, all instances of this backend are removed. + +NOTE: Backend changes need to be finalized with `shard.reconfigure()` +and are only supported on one shard director at a time. + +$Method BOOL .clear(PRIV_TASK) + +Remove all backends from the director. + +NOTE: Backend changes need to be finalized with `shard.reconfigure()` +and are only supported on one shard director at a time. + +$Method BOOL .reconfigure(PRIV_TASK, INT replicas=67, + ENUM { CRC32, SHA256, RS } alg="SHA256") + +Reconfigure the consistent hashing ring to reflect backend changes. + +This method must be called at least once before the director can be +used. + +$Method INT .key(STRING string, ENUM { CRC32, SHA256, RS } alg="SHA256") + +Utility method to generate a sharding key for use with the +``shard.backend()`` method by hashing `string` with hash algorithm +`alg`. + +$Method BACKEND .backend( + ENUM {HASH, URL, KEY, BLOB} by="HASH", + INT key=0, + BLOB key_blob=0, + INT alt=0, + REAL warmup=-1, + BOOL rampup=1, + ENUM {CHOSEN, IGNORE, ALL} healthy="CHOSEN") + + +Lookup a backend on the consistent hashing ring. + +This documentation uses the notion of an order of backends for a +particular shard key. This order is deterministic but seemingly random +as determined by the consistent hashing algorithm and is likely to +differ for different keys, depending on the number of backends and the +number of replicas. In particular, the backend order referred to here +is _not_ the order given when backends are added. + +* `by` how to determine the sharding key + + default: `HASH` + + * `HASH`: + + * when called in backend context: Use the varnish hash value as + set by `vcl_hash` + + * when called in client content: hash `req.url` + + * `URL`: hash req.url / bereq.url + + * `KEY`: use the `key` argument + + * `BLOB`: use the `key_blob` argument + + * `key` lookup key with `by=KEY` + + the `shard.key()` function may come handy to generate a sharding + key from custom strings. + + * `key_blob` lookup key with `by=BLOB` + + Currently, this uses the first 4 bytes from the given blob in + network byte order (big endian), left-padded with zeros for blobs + smaller than 4 bytes. + +* `alt` alternative backend selection + + default: `0` + + Select the `alt`-th alternative backend for the given `key`. + + This is particularly useful for retries / restarts due to backend + errors: By setting `alt=req.restarts` or `alt=bereq.retries` with + healthy=ALL, another server gets selected. + + The rampup and warmup features are only active for `alt==0` + +* `rampup` slow start for servers which just went healthy + + default: `true` + + If `alt==0` and the chosen backend is in its rampup period, with a + probability proportional to the fraction of time since the backup + became healthy to the rampup period, return the next alternative + backend, unless this is also in its rampup period. + + The default rampup interval can be set per shard director using the + `set_rampup()` method or specifically per backend with the + `set_backend()` method. + +* `warmup` probabalistic alternative server selection + + possible values: -1, 0..1 + + default: `-1` + + `-1`: use the warmup probability from the director definition + + Only used for `alt==0`: Sets the ratio of requests (0.0 to 1.0) that + goes to the next alternate backend to warm it up when the preferred + backend is healthy. Not active if any of the preferred or + alternative backend are in rampup. + + `warmup=0.5` is a convenient way to spread the load for each key + over two backends under normal operating conditions. + +* `healthy` + + default: `CHOSEN` + + * CHOSEN: Return a healthy backend if possible. + + For `alt==0`, return the first healthy backend or none. + + For `alt > 0`, ignore the health state of backends skipped for + alternative backend selection, then return the next healthy + backend. If this does not exist, return the last healthy backend + of those skipped or none. + + * IGNORE: Completely ignore backend health state + + Just return the first or `alt`-th alternative backend, ignoring + health state. Ignore `rampup` and `warmup`. + + * ALL: Check health state also for alternative backend selection + + For `alt > 0`, return the `alt`-th alternative backend of all + those healthy, the last healthy backend found or none. + +$Method VOID .debug(INT) + +`intentionally undocumented` + +ACKNOWLEDGEMENTS +================ -This document is licensed under the same licence as Varnish -itself. See LICENCE for details. +Development of a previous version of the shard director was partly sponsored +by Deutsche Telekom AG ? Products & Innovation. -* Copyright (c) 2013-2015 Varnish Software AS +Development of this version of the shard director was partly sponsored +by BILD GmbH & Co KG. diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c new file mode 100644 index 0000000..7acb59b --- /dev/null +++ b/lib/libvmod_directors/vmod_shard.c @@ -0,0 +1,278 @@ +/*- + * Copyright 2009-2016 UPLEX - Nils Goroll Systemoptimierung + * All rights reserved. + * + * Authors: Julian Wiesener + * Nils Goroll + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include + +#include "cache/cache.h" + +#include "vrt.h" +#include "vend.h" + +#include "vcc_if.h" +#include "shard_dir.h" +#include "shard_cfg.h" +#include "shard_hash.h" + +struct vmod_directors_shard { + unsigned magic; +#define VMOD_SHARD_SHARD_MAGIC 0x6e63e1bf + struct sharddir *shardd; +}; + +VCL_VOID __match_proto__(td_directors_shard__init) +vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp, + const char *vcl_name) +{ + struct vmod_directors_shard *vshard; + VCL_INT t1; + uint32_t t2a, t2b; + + /* see vmod_key comment */ + assert(sizeof(VCL_INT) >= sizeof(uint32_t)); + t2a = UINT32_MAX; + t1 = (VCL_INT)t2a; + t2b = (uint32_t)t1; + assert(t2a == t2b); + + (void) ctx; + AN(vshardp); + AZ(*vshardp); + ALLOC_OBJ(vshard, VMOD_SHARD_SHARD_MAGIC); + AN(vshard); + + *vshardp = vshard; + sharddir_new(&vshard->shardd, vcl_name); +} + +VCL_VOID __match_proto__(td_directors_shard__fini) +vmod_shard__fini(struct vmod_directors_shard **vshardp) +{ + struct vmod_directors_shard *vshard = *vshardp; + + *vshardp = NULL; + CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); + sharddir_delete(&vshard->shardd); + FREE_OBJ(vshard); +} + +/* + * our key is a uint32_t, but VCL_INT is a (signed) long. We cast back and + * forth, asserting in vmod_shard__init() that VCL_INT is a large enough + * container + */ +VCL_INT __match_proto__(td_directors_shard_key) + vmod_shard_key(VRT_CTX, struct vmod_directors_shard *vshard, + VCL_STRING s, VCL_ENUM alg_s) +{ + enum alg_e alg = parse_alg_e(alg_s); + hash_func hash_fp = shard_hash_f[alg]; + + (void) ctx; + (void) vshard;; + + return (VCL_INT)hash_fp(s ? s : ""); +} + +VCL_VOID __match_proto__(td_directors_set_warmup) +vmod_shard_set_warmup(VRT_CTX, struct vmod_directors_shard *vshard, + VCL_REAL probability) { + CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); + if (probability < 0 || probability >= 1) { + shard_err(ctx, vshard->shardd, + ".set_warmup(%f) ignored", probability); + return; + } + shardcfg_set_warmup(vshard->shardd, probability); +} + +VCL_VOID __match_proto__(td_directors_set_rampup) +vmod_shard_set_rampup(VRT_CTX, struct vmod_directors_shard *vshard, + VCL_DURATION duration) +{ + (void) ctx; + CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); + shardcfg_set_rampup(vshard->shardd, duration); +} + +VCL_BOOL __match_proto__(td_directors_shard_add_backend) +vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard, + struct vmod_priv *priv, + VCL_BACKEND be, VCL_STRING ident, VCL_DURATION rampup) +{ + CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); + + if (be == NULL) { + shard_err0(ctx, vshard->shardd, + ".backend_add() NULL backend given"); + return 0; + } + + return shardcfg_add_backend(ctx, priv, vshard->shardd, + be, ident, rampup); +} + +VCL_BOOL __match_proto__(td_directors_shard_remove_backend) +vmod_shard_remove_backend(VRT_CTX, struct vmod_directors_shard *vshard, + struct vmod_priv *priv, + VCL_BACKEND be, VCL_STRING ident) +{ + CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); + + if (be == NULL && ident == NULL) { + shard_err0(ctx, vshard->shardd, + ".backend_remove() at least one of backend " + "and ident must be given"); + return 0; + } + + return shardcfg_remove_backend(ctx, priv, vshard->shardd, + be, ident); +} + +VCL_BOOL __match_proto__(td_directors_shard_clear) +vmod_shard_clear(VRT_CTX, struct vmod_directors_shard *vshard, + struct vmod_priv *priv) +{ + CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); + return shardcfg_clear(ctx, priv, vshard->shardd); +} + +VCL_BOOL __match_proto__(td_directors_shard_reconfigure) +vmod_shard_reconfigure(VRT_CTX, struct vmod_directors_shard *vshard, + struct vmod_priv *priv, VCL_INT replicas, VCL_ENUM alg_s) +{ + enum alg_e alg = parse_alg_e(alg_s); + + return shardcfg_reconfigure(ctx, priv, vshard->shardd, replicas, alg); +} + +static inline uint32_t +get_key(VRT_CTX, enum by_e by, VCL_INT key_int, VCL_BLOB key_blob) +{ + struct http *http; + uint8_t k[4] = { 0 }; + uint8_t *b; + int i, ki; + + switch (by) { + case HASH: + if (ctx->bo) { + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + return (vbe32dec(ctx->bo->digest)); + } + /* FALLTHROUGH */ + case URL: + if (ctx->http_req) { + AN(http = ctx->http_req); + } else { + AN(ctx->http_bereq); + AN(http = ctx->http_bereq); + } + return (shard_hash_f[SHA256](http->hd[HTTP_HDR_URL].b)); + break; + case KEY: + return ((uint32_t)key_int); + break; + case BLOB: + assert(key_blob); + assert(key_blob->len > 0); + assert(key_blob->priv != NULL); + + if (key_blob->len >= 4) + ki = 0; + else + ki = 4 - key_blob->len; + + b = key_blob->priv; + for (i = 0; ki < 4; i++, ki++) + k[ki] = b[i]; + assert(i <= key_blob->len); + + return (vbe32dec(k)); + break; + default: + WRONG("by value"); + } +} + +VCL_BACKEND __match_proto__(td_directors_shard_backend) +vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, + VCL_ENUM by_s, VCL_INT key_int, VCL_BLOB key_blob, VCL_INT alt, + VCL_REAL warmup, VCL_BOOL rampup, VCL_ENUM healthy_s) +{ + enum by_e by = parse_by_e(by_s); + enum healthy_e healthy = parse_healthy_e(healthy_s); + + uint32_t key; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); + + if (key_int && by != KEY) { + shard_err(ctx, vshard->shardd, + "by=%s but key argument used", by_s); + return NULL; + } + + if (key_blob && by != BLOB) { + shard_err(ctx, vshard->shardd, + "by=%s but key_blob argument used", by_s); + return NULL; + } + + if (by == BLOB) { + if (key_blob == NULL || + key_blob->len <= 0 || + key_blob->priv == NULL) { + shard_err0(ctx, vshard->shardd, + "by=BLOB but no or empty key_blob " + "- using key 0"); + by = KEY; + key_int = 0; + } + } + + key = get_key(ctx, by, key_int, key_blob); + + return (sharddir_pick_be(ctx, vshard->shardd, + key, alt, warmup, rampup, healthy)); +} + +VCL_VOID __match_proto__(td_directors_shard_backend) +vmod_shard_debug(VRT_CTX, struct vmod_directors_shard *vshard, + VCL_INT i) +{ + CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); + + (void) ctx; + sharddir_debug(vshard->shardd, i & UINT32_MAX); +} From varnish-commit at varnish-cache.org Mon Sep 12 16:47:12 2016 From: varnish-commit at varnish-cache.org (hevd) Date: Tue, 13 Sep 2016 00:47:12 +0800 Subject: =?utf-8?B?dmFybmlzaC1jb21taXQ65b2x5ZON5Yiw5Ye65Y+j?= =?utf-8?B?5biC5Zy66JCl6ZSA55qE5LiD5aSn6KaB57Sg?= Message-ID: <20160913004719272861@cluoh.org> ???????????????????????? ?????? 2016?9?22-23??? 9?24-25??? ??????????????????????????????????????????????? ?????? ???? + ???? + ???? +???? + ???? ?????? 3200?/2?/1????????????????? ?????? 021-31006787?0755-61280006 18917870808 ??? ?QQ???? 320588808 ????? ????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????,????????????????? ????? ???????????????? 1???????????? 2???????????? 3????????????? 4?????????? 5??????????????? 6??????????????????????? 7???????????? ??????????????????????? 1???????? 2????????? 3?????? 4??????? 5??? 6??????? 7????????? 8???? 9???????????? 10?????? ?????????????????????????????????????? 1???????? 2???????????????????????????? 3???????? 4??????????????????? 5????????????????????? 6??????? ?????????????????????????????????? 1???????? 2?????????? 3????? 4???????????? 5????? 6?????? 7????????? 8????????????????????????????????? ?????????????????????????? 1???????????????????????????? 2???????????? a)??????? b)????????? c)???????? d)????? e)??????? f)????????? g)??????????? 3????????? a)???? b)???? c)???? ??????????? 1???????????AIDA?? 2??????????????? 3?????????????????? 4????????????? 5??????????????? ???? ?????????????? ????????????????????? 1????????????????????????????? 2?????????????????????? 3?????????????????????????? 4?????????????????????????????? 5???????????????????????????????????? 6?????????????????????????????????? 7?????????????????????????????? 8?????????????? ??????????????????????????? 1??????????????????? 2??????????????????????????????????? 3??????????????????????????? 4???????????????? 5????????????????????????? 6??????????????????????????? 7???????????????????? 8???????????????????? 9??????????????? ??????????????????????? 1????????????? 2???????????????????? 3?????????????? 4???????????? 5?????????????????????? 6?????????????????????????????? ???????????????????? 1?????????? 2???????????? 3?????????????????? 4????????????????????? 5???????????????? 6??????????????????? ???????????????????? 1?????????????????????????????????????????????????? 2?????????????? 3?????????????? 4???????????????? 5??????????? 6????????????????????????? 7????????????????? ???????????????????????? 1?????????????? 2??????????????????? 3?????????????? 4?????????? 5?????????????????? 6?????????????? 7????????????????? 8?????????????? 9????????? 10???????????? 11?????????????? 12??????? 13???????????????????????? 14?????????????? ???? ?????????? ????????????????? 1?????????????????????? 2????????????? 3????????????????????? 4????????????????????? 5???????????????? 6????DM?????? ????????????????? 1???????????????????????? 2?????????????? 3??????????????????? 4????????????? 5??????? 6???????????????? 7??????????? 8??????????? 9??????????????????? 10???????????? 11??????????????? ????????????????? 1??????????? 2??????????????? 3?????????????????? 4??????????????????????? 5?????????? 6?????????????????????????? ??????????????????? 1????????? 2??????????ABC?? 3????????????????? 4??????????????? 5?????????????????? ????? ? ? ??????(???????MBA??????????)???????????????????????????????????????????????????????????????????????????????????????????????????????(?)???????????????????????????????????IBT?????????????????????????????????????????????????????????????????????????????????????????????????????????? ??? ??????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? -------------- next part -------------- An HTML attachment was scrubbed... URL: From dridi.boukelmoune at gmail.com Mon Sep 12 17:05:15 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Sep 2016 19:05:15 +0200 Subject: [3.0] 3271c51 Ignore missing reason in probe responses Message-ID: commit 3271c51e303b1f187d851cdaf056a940ccaa8365 Author: Dridi Boukelmoune Date: Mon Sep 12 18:53:05 2016 +0200 Ignore missing reason in probe responses Fixes #2069 diff --git a/bin/varnishd/cache_backend_poll.c b/bin/varnishd/cache_backend_poll.c index 4995a57..6dfa6d1 100644 --- a/bin/varnishd/cache_backend_poll.c +++ b/bin/varnishd/cache_backend_poll.c @@ -226,7 +226,7 @@ vbp_poke(struct vbp_target *vt) i = sscanf(vt->resp_buf, "HTTP/%*f %u %s", &resp, buf); - if (i == 2 && resp == vt->probe.exp_status) + if (i >= 1 && resp == vt->probe.exp_status) vt->happy |= 1; } diff --git a/bin/varnishtest/tests/r02069.vtc b/bin/varnishtest/tests/r02069.vtc new file mode 100644 index 0000000..8cf4df1 --- /dev/null +++ b/bin/varnishtest/tests/r02069.vtc @@ -0,0 +1,36 @@ +varnishtest "Probe response without a reason" + +server s1 -repeat 20 { + rxreq + send "HTTP/1.1 200\r\n\r\n" +} -start + +varnish v1 -vcl { + backend s1 { + .host = "${s1_addr}"; + .port = "${s1_port}"; + .probe = { + .initial = 0; + .window = 5; + .threshold = 5; + .interval = 100ms; + } + } + + sub vcl_recv { + if (req.backend.healthy) { + error 200 "Backend healthy"; + } + else { + error 500 "Backend sick"; + } + } +} -start + +delay 1 + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run From nils.goroll at uplex.de Mon Sep 12 17:42:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 19:42:12 +0200 Subject: [master] cdd4936 make the shard sha256 extract endianness-agnostig Message-ID: commit cdd4936034e156ce1b3eaa8d86b70b16c6137139 Author: Nils Goroll Date: Mon Sep 12 19:40:43 2016 +0200 make the shard sha256 extract endianness-agnostig We should have interpreted it as big endian right from the start, but now that we got the existing code developed on little endian, we keep it so. diff --git a/lib/libvmod_directors/shard_hash.c b/lib/libvmod_directors/shard_hash.c index 9fc207e..59f57e7 100644 --- a/lib/libvmod_directors/shard_hash.c +++ b/lib/libvmod_directors/shard_hash.c @@ -38,6 +38,7 @@ #include "vrt.h" #include "crc32.h" #include "vsha256.h" +#include "vend.h" #include "shard_parse_vcc_enums.h" #include "shard_hash.h" @@ -61,6 +62,7 @@ shard_hash_sha256(VCL_STRING s) unsigned char digest[32]; uint32_t uint32_digest[8]; } sha256_digest; + uint32_t r; SHA256_Init(&sha256); SHA256_Update(&sha256, s, strlen(s)); @@ -70,7 +72,8 @@ shard_hash_sha256(VCL_STRING s) * use low 32 bits only * XXX: Are these the best bits to pick? */ - return (sha256_digest.uint32_digest[7]); + vle32enc(&r, sha256_digest.uint32_digest[7]); + return (r); } static uint32_t __match_proto__(hash_func) From nils.goroll at uplex.de Mon Sep 12 18:03:13 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 20:03:13 +0200 Subject: [master] 3be6559 fix a leftover from vslp times - overlooked increment by one for crc32 Message-ID: commit 3be6559fff8dd80ddbbff95718f97976f8187de0 Author: Nils Goroll Date: Mon Sep 12 20:01:45 2016 +0200 fix a leftover from vslp times - overlooked increment by one for crc32 diff --git a/bin/varnishtest/tests/d00017.vtc b/bin/varnishtest/tests/d00017.vtc index d92cad5..808e23b 100644 --- a/bin/varnishtest/tests/d00017.vtc +++ b/bin/varnishtest/tests/d00017.vtc @@ -136,9 +136,9 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 = CLI Loaded - expect * = Debug {^shard: lookup key 68b902f8 idx 29 host 0} - expect * = Debug {^shard: lookup key 39dc4614 idx 20 host 1} - expect * = Debug {^shard: lookup key c7793506 idx 59 host 2} + expect * = Debug {^shard: lookup key 68b902f7 idx 29 host 0} + expect * = Debug {^shard: lookup key 39dc4613 idx 20 host 1} + expect * = Debug {^shard: lookup key c7793505 idx 59 host 2} } -start client c1 { diff --git a/lib/libvmod_directors/shard_hash.c b/lib/libvmod_directors/shard_hash.c index 59f57e7..30ebd7d 100644 --- a/lib/libvmod_directors/shard_hash.c +++ b/lib/libvmod_directors/shard_hash.c @@ -46,12 +46,7 @@ static uint32_t __match_proto__(hash_func) shard_hash_crc32(VCL_STRING s) { - uint32_t crc; - crc = crc32(~0U, (const unsigned char*)s, strlen(s)); - crc ^= ~0U; - crc++; - - return (crc); + return (crc32_l(s, strlen(s))); } static uint32_t __match_proto__(hash_func) From nils.goroll at uplex.de Mon Sep 12 18:30:22 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 20:30:22 +0200 Subject: [master] 4a5ec60 failing crc32 checks on macos + clang - casting the culprit? Message-ID: commit 4a5ec60af1ff1a0ec81b7c255f5d9ca01299b1be Author: Nils Goroll Date: Mon Sep 12 20:28:58 2016 +0200 failing crc32 checks on macos + clang - casting the culprit? diff --git a/lib/libvmod_directors/crc32.c b/lib/libvmod_directors/crc32.c index 714b0f0..44a5250 100644 --- a/lib/libvmod_directors/crc32.c +++ b/lib/libvmod_directors/crc32.c @@ -83,17 +83,15 @@ static const uint32_t crc32bits[] = { }; uint32_t -crc32(uint32_t crc, const void *p1, unsigned l) +crc32(uint32_t crc, const unsigned char *p, unsigned l) { - const unsigned char *p; - - for (p = (const unsigned char*)p1; l-- > 0; p++) + for (; l-- > 0; p++) crc = (crc >> 8) ^ crc32bits[(crc ^ *p) & 0xff]; return (crc); } uint32_t -crc32_l(const void *p1, unsigned l) +crc32_l(const unsigned char *p1, unsigned l) { uint32_t crc; diff --git a/lib/libvmod_directors/crc32.h b/lib/libvmod_directors/crc32.h index 39adf8a..dfe56ab 100644 --- a/lib/libvmod_directors/crc32.h +++ b/lib/libvmod_directors/crc32.h @@ -1,3 +1,3 @@ #include -uint32_t crc32(uint32_t crc, const void *p1, unsigned l); -uint32_t crc32_l(const void *p1, unsigned l); +uint32_t crc32(uint32_t crc, const unsigned char *, unsigned); +uint32_t crc32_l(const unsigned char *, unsigned); From nils.goroll at uplex.de Mon Sep 12 18:35:14 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 20:35:14 +0200 Subject: [master] bd5ade3 Revert "failing crc32 checks on macos + clang - casting the culprit?" Message-ID: commit bd5ade3e542217e99062a4543cd66bd82586d6e4 Author: Nils Goroll Date: Mon Sep 12 20:34:18 2016 +0200 Revert "failing crc32 checks on macos + clang - casting the culprit?" This reverts commit 4a5ec60af1ff1a0ec81b7c255f5d9ca01299b1be. diff --git a/lib/libvmod_directors/crc32.c b/lib/libvmod_directors/crc32.c index 44a5250..714b0f0 100644 --- a/lib/libvmod_directors/crc32.c +++ b/lib/libvmod_directors/crc32.c @@ -83,15 +83,17 @@ static const uint32_t crc32bits[] = { }; uint32_t -crc32(uint32_t crc, const unsigned char *p, unsigned l) +crc32(uint32_t crc, const void *p1, unsigned l) { - for (; l-- > 0; p++) + const unsigned char *p; + + for (p = (const unsigned char*)p1; l-- > 0; p++) crc = (crc >> 8) ^ crc32bits[(crc ^ *p) & 0xff]; return (crc); } uint32_t -crc32_l(const unsigned char *p1, unsigned l) +crc32_l(const void *p1, unsigned l) { uint32_t crc; diff --git a/lib/libvmod_directors/crc32.h b/lib/libvmod_directors/crc32.h index dfe56ab..39adf8a 100644 --- a/lib/libvmod_directors/crc32.h +++ b/lib/libvmod_directors/crc32.h @@ -1,3 +1,3 @@ #include -uint32_t crc32(uint32_t crc, const unsigned char *, unsigned); -uint32_t crc32_l(const unsigned char *, unsigned); +uint32_t crc32(uint32_t crc, const void *p1, unsigned l); +uint32_t crc32_l(const void *p1, unsigned l); From fgsch at lodoss.net Mon Sep 12 19:30:19 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 12 Sep 2016 21:30:19 +0200 Subject: [master] 5879216 Let process do -run as well Message-ID: commit 587921692eab5b4963c283afb5efe551459d6d19 Author: Federico G. Schwindt Date: Mon Sep 12 20:25:18 2016 +0100 Let process do -run as well diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index c87f7dc..977587d 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -231,6 +231,14 @@ process_wait(const struct process *p) AZ(pthread_join(p->tp, &v)); } +static void +process_run(struct process *p) +{ + + process_start(p); + process_wait(p); +} + /********************************************************************** * Send a signal to a process */ @@ -349,6 +357,10 @@ cmd_process(CMD_ARGS) process_wait(p); continue; } + if (!strcmp(*av, "-run")) { + process_run(p); + continue; + } if (!strcmp(*av, "-kill")) { process_kill(p, av[1]); av++; From fgsch at lodoss.net Mon Sep 12 19:30:19 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 12 Sep 2016 21:30:19 +0200 Subject: [master] f39ade7 Add test for varnishstat -f Message-ID: commit f39ade78fa882ccd8bb0f1bf92d5159c5c003f25 Author: Federico G. Schwindt Date: Mon Sep 12 20:13:21 2016 +0100 Add test for varnishstat -f diff --git a/bin/varnishtest/tests/README b/bin/varnishtest/tests/README index dd21d38..a48967f 100644 --- a/bin/varnishtest/tests/README +++ b/bin/varnishtest/tests/README @@ -29,5 +29,5 @@ Naming scheme id ~ ^s --> Slow tests, expiry, grace etc. id ~ ^t --> Transport protocol tests id ~ ^t02 --> HTTP2 - id ~ ^u --> Unusual background processes + id ~ ^u --> Utilities and background processes id ~ ^v --> VCL tests: execute VRT functions diff --git a/bin/varnishtest/tests/u00002.vtc b/bin/varnishtest/tests/u00002.vtc new file mode 100644 index 0000000..38ce280 --- /dev/null +++ b/bin/varnishtest/tests/u00002.vtc @@ -0,0 +1,17 @@ +varnishtest "Test varnishstat -f option" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend {} -start + +delay 1 + +process p1 {varnishstat -1 -n ${v1_name} -f ^LCK.vbe.destroy \ + -f LCK.vbe.* -f LCK.mempool.*} -run + +shell "grep -q vbe ${tmpdir}/p1/stdout" +shell "grep -q mempool ${tmpdir}/p1/stdout" +err_shell "" "grep -q LCK.vbe.destroy ${tmpdir}/p1/stdout" From dridi at varni.sh Mon Sep 12 19:34:09 2016 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 12 Sep 2016 21:34:09 +0200 Subject: [master] 5879216 Let process do -run as well In-Reply-To: References: Message-ID: On Mon, Sep 12, 2016 at 9:30 PM, Federico G. Schwindt wrote: > > commit 587921692eab5b4963c283afb5efe551459d6d19 > Author: Federico G. Schwindt > Date: Mon Sep 12 20:25:18 2016 +0100 > > Let process do -run as well I didn't implement that because it can be done with the shell command if it needs to run in the foreground. From nils.goroll at uplex.de Mon Sep 12 20:07:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 22:07:12 +0200 Subject: [master] a04d4d7 try to make flexelint a bit happier Message-ID: commit a04d4d7f53c4f58ff4dc3df39a0b319c8a3e26bf Author: Nils Goroll Date: Mon Sep 12 21:22:47 2016 +0200 try to make flexelint a bit happier diff --git a/lib/libvmod_directors/crc32.c b/lib/libvmod_directors/crc32.c index 714b0f0..94b2405 100644 --- a/lib/libvmod_directors/crc32.c +++ b/lib/libvmod_directors/crc32.c @@ -104,9 +104,12 @@ crc32_l(const void *p1, unsigned l) #ifdef DEBUG #include +#include +#include int main(int argc, char *argv[]) { if (argc > 1) printf("%x\n", crc32_l(argv[1], strlen(argv[1]))); + return 0; } #endif diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index ecfb936..44e5c28 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -148,7 +148,7 @@ shard_change_task_add(VRT_CTX, struct shard_change *change, static inline VCL_BOOL shard_change_task_backend(VRT_CTX, - struct vmod_priv *priv, struct sharddir *shardd, + struct vmod_priv *priv, const struct sharddir *shardd, enum shard_change_task_e task_e, VCL_BACKEND be, VCL_STRING ident, VCL_DURATION rampup) { @@ -183,8 +183,9 @@ shard_change_task_backend(VRT_CTX, * director reconfiguration tasks */ VCL_BOOL -shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv, struct sharddir *shardd, - VCL_BACKEND be, VCL_STRING ident, VCL_DURATION rampup) +shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv, + const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident, + VCL_DURATION rampup) { AN(be); return shard_change_task_backend(ctx, priv, shardd, ADD_BE, @@ -193,14 +194,14 @@ shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv, struct sharddir *shardd, VCL_BOOL shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv, - struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident) + const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident) { return shard_change_task_backend(ctx, priv, shardd, REMOVE_BE, be, ident, 0); } VCL_BOOL -shardcfg_clear(VRT_CTX, struct vmod_priv *priv, struct sharddir *shardd) +shardcfg_clear(VRT_CTX, struct vmod_priv *priv, const struct sharddir *shardd) { struct shard_change *change; @@ -223,7 +224,8 @@ shardcfg_clear(VRT_CTX, struct vmod_priv *priv, struct sharddir *shardd) typedef int (*compar)( const void*, const void* ); static int -circlepoint_compare(struct shard_circlepoint *a, struct shard_circlepoint *b) +circlepoint_compare(const struct shard_circlepoint *a, + const struct shard_circlepoint *b) { return (a->point == b->point) ? 0 : ((a->point > b->point) ? 1 : -1); } @@ -256,7 +258,7 @@ shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas, enum alg_e alg) assert(ident[0] != '\0'); - len = strlen(ident) + log10(UINT32_MAX) + 2; + len = strlen(ident) + 12; // log10(UINT32_MAX) + 2; char s[len]; @@ -280,7 +282,7 @@ shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas, enum alg_e alg) for (j = 0; j < replicas; j++) SHDBG(SHDBG_CIRCLE, shardd, "hashcircle[%5ld] = " - "{point = %8x, host = %2d}\n", + "{point = %8x, host = %2u}\n", i * replicas + j, shardd->hashcircle[i * replicas + j].point, shardd->hashcircle[i * replicas + j].host); @@ -343,7 +345,7 @@ shardcfg_backend_del_cmp(const struct shard_backend *task, } static const struct shard_backend * -shardcfg_backend_lookup(struct backend_reconfig *re, +shardcfg_backend_lookup(const struct backend_reconfig *re, const struct shard_backend *b) { int i, max = re->shardd->n_backend + re->hole_n; @@ -357,7 +359,7 @@ shardcfg_backend_lookup(struct backend_reconfig *re, } static void -shardcfg_backend_expand(struct backend_reconfig *re) +shardcfg_backend_expand(const struct backend_reconfig *re) { int min = re->hint; @@ -612,7 +614,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv, /* only for sharddir_delete() */ void -shardcfg_delete(struct sharddir *shardd) +shardcfg_delete(const struct sharddir *shardd) { int i; @@ -645,7 +647,7 @@ shardcfg_set_rampup(struct sharddir *shardd, VCL_DURATION duration) } VCL_DURATION -shardcfg_get_rampup(struct sharddir *shardd, int host) +shardcfg_get_rampup(const struct sharddir *shardd, int host) { VCL_DURATION r; diff --git a/lib/libvmod_directors/shard_cfg.h b/lib/libvmod_directors/shard_cfg.h index ca50ddd..8d1d88c 100644 --- a/lib/libvmod_directors/shard_cfg.h +++ b/lib/libvmod_directors/shard_cfg.h @@ -27,12 +27,12 @@ */ VCL_BOOL shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv, - struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident, + const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident, VCL_DURATION rampup); VCL_BOOL shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv, - struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident); + const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident); VCL_BOOL shardcfg_clear(VRT_CTX, struct vmod_priv *priv, - struct sharddir *shardd); + const struct sharddir *shardd); VCL_BOOL shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv, struct sharddir *shardd, VCL_INT replicas, enum alg_e alg_e); VCL_VOID shardcfg_set_warmup(struct sharddir *shardd, VCL_REAL ratio); diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index 57a9495..54202f8 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -44,7 +44,6 @@ #include "vrnd.h" #include "shard_dir.h" -#include "shard_hash.h" struct shard_be_info { int hostid; @@ -98,7 +97,7 @@ shard_lookup(const struct sharddir *shardd, const uint32_t key) int idx = -1, high = n, low = 0, i; do { - i = (high + low) >> 1; + i = (high + low) / 2 ; if (shardd->hashcircle[i].point == key) idx = i; else if (i == n - 1) @@ -232,7 +231,7 @@ sharddir_unlock(struct sharddir *shardd) } static inline void -validate_alt(VRT_CTX, struct sharddir *shardd, VCL_INT *alt) +validate_alt(VRT_CTX, const struct sharddir *shardd, VCL_INT *alt) { const VCL_INT alt_max = shardd->n_backend - 1; @@ -257,7 +256,6 @@ init_state(struct shard_state *state, state->shardd = shardd; state->idx = -1; state->picklist = picklist; - state->pickcount = 0; /* healhy and changed only defined for hostid != -1 */ state->previous.hostid = -1; @@ -311,6 +309,7 @@ sharddir_pick_be(VRT_CTX, struct sharddir *shardd, validate_alt(ctx, shardd, &alt); + memset(&state, 0, sizeof(state)); init_state(&state, ctx, shardd, vbit_init(picklist_spc, picklist_sz)); state.idx = shard_lookup(shardd, key); diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index b66fca1..f40d0ac 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -95,10 +95,10 @@ sharddir_backend_ident(const struct sharddir *shardd, int host) (shardd)->name, __VA_ARGS__); \ } while (0) -#define shard_err0(ctx, shardd, s) \ +#define shard_err0(ctx, shardd, msg) \ do { \ sharddir_err(ctx, SLT_Error, "shard %s: %s", \ - (shardd)->name, (s)); \ + (shardd)->name, (msg)); \ } while (0) void sharddir_debug(struct sharddir *shardd, const uint32_t flags); @@ -113,5 +113,5 @@ VCL_BACKEND sharddir_pick_be(VRT_CTX, struct sharddir *, uint32_t, VCL_INT, VCL_REAL, VCL_BOOL, enum healthy_e); /* in shard_cfg.c */ -void shardcfg_delete(struct sharddir *shardd); -VCL_DURATION shardcfg_get_rampup(struct sharddir *shardd, int host); +void shardcfg_delete(const struct sharddir *shardd); +VCL_DURATION shardcfg_get_rampup(const struct sharddir *shardd, int host); diff --git a/lib/libvmod_directors/shard_hash.c b/lib/libvmod_directors/shard_hash.c index 30ebd7d..eacc250 100644 --- a/lib/libvmod_directors/shard_hash.c +++ b/lib/libvmod_directors/shard_hash.c @@ -92,7 +92,7 @@ _shard_hash_invalid(VCL_STRING s) { (void) s; WRONG("invalid hash fp _ALG_E_ENVALID"); - return (0); + NEEDLESS_RETURN(0); } hash_func shard_hash_f[_ALG_E_MAX] = { diff --git a/lib/libvmod_directors/shard_parse_vcc_enums.c b/lib/libvmod_directors/shard_parse_vcc_enums.c index 5dec48e..e362450 100644 --- a/lib/libvmod_directors/shard_parse_vcc_enums.c +++ b/lib/libvmod_directors/shard_parse_vcc_enums.c @@ -67,7 +67,7 @@ enum by_e parse_by_e (const char *m) { _0B: //BLOB if ((m[1] == 'L') && (m[2] == 'O') && (m[3] == 'B') && (term(m[4]))) { - r = BLOB; + r = BY_BLOB; p = 4; goto ok; } @@ -75,7 +75,7 @@ enum by_e parse_by_e (const char *m) { _0H: //HASH if ((m[1] == 'A') && (m[2] == 'S') && (m[3] == 'H') && (term(m[4]))) { - r = HASH; + r = BY_HASH; p = 4; goto ok; } @@ -83,7 +83,7 @@ enum by_e parse_by_e (const char *m) { _0K: //KEY if ((m[1] == 'E') && (m[2] == 'Y') && (term(m[3]))) { - r = KEY; + r = BY_KEY; p = 3; goto ok; } @@ -91,7 +91,7 @@ enum by_e parse_by_e (const char *m) { _0U: //URL if ((m[1] == 'R') && (m[2] == 'L') && (term(m[3]))) { - r = URL; + r = BY_URL; p = 3; goto ok; } diff --git a/lib/libvmod_directors/shard_parse_vcc_enums.h b/lib/libvmod_directors/shard_parse_vcc_enums.h index aa097dc..d6e2ff9 100644 --- a/lib/libvmod_directors/shard_parse_vcc_enums.h +++ b/lib/libvmod_directors/shard_parse_vcc_enums.h @@ -19,10 +19,10 @@ enum alg_e parse_alg_e (const char *); enum by_e { _BY_E_INVALID = 0, - HASH, - URL, - KEY, - BLOB, + BY_HASH, + BY_URL, + BY_KEY, + BY_BLOB, _BY_E_MAX }; diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 7acb59b..52c9710 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -184,13 +184,13 @@ get_key(VRT_CTX, enum by_e by, VCL_INT key_int, VCL_BLOB key_blob) int i, ki; switch (by) { - case HASH: + case BY_HASH: if (ctx->bo) { CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); return (vbe32dec(ctx->bo->digest)); } /* FALLTHROUGH */ - case URL: + case BY_URL: if (ctx->http_req) { AN(http = ctx->http_req); } else { @@ -198,11 +198,9 @@ get_key(VRT_CTX, enum by_e by, VCL_INT key_int, VCL_BLOB key_blob) AN(http = ctx->http_bereq); } return (shard_hash_f[SHA256](http->hd[HTTP_HDR_URL].b)); - break; - case KEY: + case BY_KEY: return ((uint32_t)key_int); - break; - case BLOB: + case BY_BLOB: assert(key_blob); assert(key_blob->len > 0); assert(key_blob->priv != NULL); @@ -218,7 +216,6 @@ get_key(VRT_CTX, enum by_e by, VCL_INT key_int, VCL_BLOB key_blob) assert(i <= key_blob->len); return (vbe32dec(k)); - break; default: WRONG("by value"); } @@ -237,26 +234,26 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); - if (key_int && by != KEY) { + if (key_int && by != BY_KEY) { shard_err(ctx, vshard->shardd, "by=%s but key argument used", by_s); return NULL; } - if (key_blob && by != BLOB) { + if (key_blob && by != BY_BLOB) { shard_err(ctx, vshard->shardd, "by=%s but key_blob argument used", by_s); return NULL; } - if (by == BLOB) { + if (by == BY_BLOB) { if (key_blob == NULL || key_blob->len <= 0 || key_blob->priv == NULL) { shard_err0(ctx, vshard->shardd, "by=BLOB but no or empty key_blob " "- using key 0"); - by = KEY; + by = BY_KEY; key_int = 0; } } From nils.goroll at uplex.de Mon Sep 12 20:07:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 22:07:12 +0200 Subject: [master] 1ca6a37 hand-edit the auto-generated code for flexelint Message-ID: commit 1ca6a37b86fbb482356b109ef3aa1054a467b9cd Author: Nils Goroll Date: Mon Sep 12 21:59:31 2016 +0200 hand-edit the auto-generated code for flexelint The gotos are an integral concept of the code gen to produce code which can be compiled efficiently. We could hand-craft it for the purpose at hand, but I don't see much value in that. diff --git a/lib/libvmod_directors/shard_parse_vcc_enums.c b/lib/libvmod_directors/shard_parse_vcc_enums.c index e362450..1110077 100644 --- a/lib/libvmod_directors/shard_parse_vcc_enums.c +++ b/lib/libvmod_directors/shard_parse_vcc_enums.c @@ -12,7 +12,6 @@ enum alg_e parse_alg_e (const char *m) { - int p; enum alg_e r; switch (m[0]) { @@ -25,7 +24,6 @@ enum alg_e parse_alg_e (const char *m) { //CRC32 if ((m[1] == 'R') && (m[2] == 'C') && (m[3] == '3') && (m[4] == '2') && (term(m[5]))) { r = CRC32; - p = 5; goto ok; } goto invalid; @@ -33,7 +31,6 @@ enum alg_e parse_alg_e (const char *m) { //RS if ((m[1] == 'S') && (term(m[2]))) { r = RS; - p = 2; goto ok; } goto invalid; @@ -41,7 +38,6 @@ enum alg_e parse_alg_e (const char *m) { //SHA256 if ((m[1] == 'H') && (m[2] == 'A') && (m[3] == '2') && (m[4] == '5') && (m[5] == '6') && (term(m[6]))) { r = SHA256; - p = 6; goto ok; } goto invalid; @@ -49,12 +45,10 @@ enum alg_e parse_alg_e (const char *m) { return r; invalid: return _ALG_E_INVALID; - (void)p; } enum by_e parse_by_e (const char *m) { - int p; enum by_e r; switch (m[0]) { @@ -68,7 +62,6 @@ enum by_e parse_by_e (const char *m) { //BLOB if ((m[1] == 'L') && (m[2] == 'O') && (m[3] == 'B') && (term(m[4]))) { r = BY_BLOB; - p = 4; goto ok; } goto invalid; @@ -76,7 +69,6 @@ enum by_e parse_by_e (const char *m) { //HASH if ((m[1] == 'A') && (m[2] == 'S') && (m[3] == 'H') && (term(m[4]))) { r = BY_HASH; - p = 4; goto ok; } goto invalid; @@ -84,7 +76,6 @@ enum by_e parse_by_e (const char *m) { //KEY if ((m[1] == 'E') && (m[2] == 'Y') && (term(m[3]))) { r = BY_KEY; - p = 3; goto ok; } goto invalid; @@ -92,7 +83,6 @@ enum by_e parse_by_e (const char *m) { //URL if ((m[1] == 'R') && (m[2] == 'L') && (term(m[3]))) { r = BY_URL; - p = 3; goto ok; } goto invalid; @@ -100,7 +90,6 @@ enum by_e parse_by_e (const char *m) { return r; invalid: return _BY_E_INVALID; - (void)p; } From guillaume at varnish-software.com Mon Sep 12 20:14:42 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Mon, 12 Sep 2016 22:14:42 +0200 Subject: [master] 5879216 Let process do -run as well In-Reply-To: References: Message-ID: Any reason to not kill the shell command and just use process? -- Guillaume Quintard On Mon, Sep 12, 2016 at 9:34 PM, Dridi Boukelmoune wrote: > On Mon, Sep 12, 2016 at 9:30 PM, Federico G. Schwindt > wrote: > > > > commit 587921692eab5b4963c283afb5efe551459d6d19 > > Author: Federico G. Schwindt > > Date: Mon Sep 12 20:25:18 2016 +0100 > > > > Let process do -run as well > > I didn't implement that because it can be done with the shell command > if it needs to run in the foreground. > > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit > -------------- next part -------------- An HTML attachment was scrubbed... URL: From dridi at varni.sh Mon Sep 12 20:19:52 2016 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 12 Sep 2016 22:19:52 +0200 Subject: [master] 5879216 Let process do -run as well In-Reply-To: References: Message-ID: On Mon, Sep 12, 2016 at 10:14 PM, Guillaume Quintard wrote: > Any reason to not kill the shell command and just use process? shell {...} vs process {...} -run (for the lazy dev) Also err_shell. I introduced the process command for background processes, maybe I should've named it bgprocess. Both have different purposes: - `shell` commands for exit status checks - `process` for background processes and automatic stdout/err capture From guillaume at varnish-software.com Mon Sep 12 20:39:21 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Mon, 12 Sep 2016 22:39:21 +0200 Subject: [master] 5879216 Let process do -run as well In-Reply-To: References: Message-ID: Can't we have both? With -err_wait/-err_run ? On Sep 12, 2016 22:20, "Dridi Boukelmoune" wrote: > On Mon, Sep 12, 2016 at 10:14 PM, Guillaume Quintard > wrote: > > Any reason to not kill the shell command and just use process? > > shell {...} vs process {...} -run (for the lazy dev) > > Also err_shell. > > I introduced the process command for background processes, maybe I > should've named it bgprocess. > > Both have different purposes: > > - `shell` commands for exit status checks > - `process` for background processes and automatic stdout/err capture > -------------- next part -------------- An HTML attachment was scrubbed... URL: From dridi at varni.sh Mon Sep 12 20:42:44 2016 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 12 Sep 2016 22:42:44 +0200 Subject: [master] 5879216 Let process do -run as well In-Reply-To: References: Message-ID: On Mon, Sep 12, 2016 at 10:39 PM, Guillaume Quintard wrote: > Can't we have both? With -err_wait/-err_run ? How about some stability? We've recently removed sema in favor of barrier and changed the behavior of `varnishtest -i` so that we no longer need ${vmod_xxx} macros for imports. We can't both advocate for out-of-tree usage of varnishtest and keep on breaking it on every release. From nils.goroll at uplex.de Mon Sep 12 20:44:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Sep 2016 22:44:12 +0200 Subject: [master] 16ff34b use the crc32 from libvgz Message-ID: commit 16ff34bda78c501d1a42bdae19fc3bb7acf879fd Author: Nils Goroll Date: Mon Sep 12 22:40:30 2016 +0200 use the crc32 from libvgz ... which is what we've done all along, but weren't aware of, until clang with specific compiler flags exposed the issue. The local declaration of crc32 on shard_hash.c is hackish, but it appears we'd need to include almost all of the libvgz includes otherwise. Huge thank you to phk for helping track down this odd issue. diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am index 76e6528..dfcbbee 100644 --- a/lib/libvmod_directors/Makefile.am +++ b/lib/libvmod_directors/Makefile.am @@ -33,9 +33,7 @@ libvmod_directors_la_SOURCES = \ shard_hash.c \ shard_hash.h \ shard_parse_vcc_enums.h \ - shard_parse_vcc_enums.c \ - crc32.c \ - crc32.h + shard_parse_vcc_enums.c nodist_libvmod_directors_la_SOURCES = \ vcc_if.c \ diff --git a/lib/libvmod_directors/crc32.c b/lib/libvmod_directors/crc32.c deleted file mode 100644 index 94b2405..0000000 --- a/lib/libvmod_directors/crc32.c +++ /dev/null @@ -1,115 +0,0 @@ -/*- - * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2008 Linpro AS - * All rights reserved. - * - * Author: Poul-Henning Kamp - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $Id: crc32.c 3599 2009-02-05 10:13:52Z tfheen $ - * - * This CRC32 implementation is in the public domain. - */ - -#include "config.h" -#include "crc32.h" - -/*--------------------------------------------------------------------*/ - -static const uint32_t crc32bits[] = { - 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, - 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, - 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, - 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, - 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, - 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, - 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, - 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, - 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, - 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, - 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, - 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, - 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, - 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, - 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, - 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, - 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, - 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, - 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, - 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, - 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, - 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, - 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, - 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, - 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, - 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, - 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, - 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, - 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, - 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, - 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, - 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, - 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, - 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, - 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, - 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, - 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, - 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, - 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, - 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, - 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, - 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, - 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d -}; - -uint32_t -crc32(uint32_t crc, const void *p1, unsigned l) -{ - const unsigned char *p; - - for (p = (const unsigned char*)p1; l-- > 0; p++) - crc = (crc >> 8) ^ crc32bits[(crc ^ *p) & 0xff]; - return (crc); -} - -uint32_t -crc32_l(const void *p1, unsigned l) -{ - uint32_t crc; - - crc = crc32(~0U, p1, l); - return (crc ^ ~0U); -} - -#ifdef DEBUG - -#include -#include -#include -int main(int argc, char *argv[]) { - if (argc > 1) - printf("%x\n", crc32_l(argv[1], strlen(argv[1]))); - return 0; -} - -#endif diff --git a/lib/libvmod_directors/crc32.h b/lib/libvmod_directors/crc32.h deleted file mode 100644 index 39adf8a..0000000 --- a/lib/libvmod_directors/crc32.h +++ /dev/null @@ -1,3 +0,0 @@ -#include -uint32_t crc32(uint32_t crc, const void *p1, unsigned l); -uint32_t crc32_l(const void *p1, unsigned l); diff --git a/lib/libvmod_directors/shard_hash.c b/lib/libvmod_directors/shard_hash.c index eacc250..63317bf 100644 --- a/lib/libvmod_directors/shard_hash.c +++ b/lib/libvmod_directors/shard_hash.c @@ -36,17 +36,26 @@ #include "cache/cache.h" #include "vrt.h" -#include "crc32.h" #include "vsha256.h" #include "vend.h" #include "shard_parse_vcc_enums.h" #include "shard_hash.h" +/* + * XXX use the crc32 from libvgz, but declare it here to avoid an include + * dependency nightmare (at least for now) + */ + +unsigned long crc32(unsigned long, const void *buf, unsigned len); + static uint32_t __match_proto__(hash_func) shard_hash_crc32(VCL_STRING s) { - return (crc32_l(s, strlen(s))); + uint32_t crc; + crc = crc32(~0U, s, strlen(s)); + crc ^= ~0U; + return (crc); } static uint32_t __match_proto__(hash_func) From fgsch at lodoss.net Mon Sep 12 21:20:56 2016 From: fgsch at lodoss.net (Federico Schwindt) Date: Mon, 12 Sep 2016 22:20:56 +0100 Subject: [master] 5879216 Let process do -run as well In-Reply-To: References: Message-ID: I appreciate that but since it's cheap and less error prone than redirecting the output I prefer having -run. On Mon, Sep 12, 2016 at 8:34 PM, Dridi Boukelmoune wrote: > On Mon, Sep 12, 2016 at 9:30 PM, Federico G. Schwindt > wrote: > > > > commit 587921692eab5b4963c283afb5efe551459d6d19 > > Author: Federico G. Schwindt > > Date: Mon Sep 12 20:25:18 2016 +0100 > > > > Let process do -run as well > > I didn't implement that because it can be done with the shell command > if it needs to run in the foreground. > -------------- next part -------------- An HTML attachment was scrubbed... URL: From fgsch at lodoss.net Mon Sep 12 21:22:11 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 12 Sep 2016 23:22:11 +0200 Subject: [master] 7e458d8 Spelling Message-ID: commit 7e458d8e0f81417d7287032fbfc7c79493871a9a Author: Federico G. Schwindt Date: Mon Sep 12 20:40:16 2016 +0100 Spelling diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index 4348ba1..a43f5e4 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -248,7 +248,7 @@ improve user-perceived response times. Sharding ```````` -This basic technique allows for numerious applications like optimizing +This basic technique allows for numerous applications like optimizing backend server cache efficiency, Varnish clustering or persisting sessions to servers without keeping any state, and, in particular, without the need to synchronize state between nodes of a cluster of @@ -275,12 +275,12 @@ Varnish servers: increase efficiency (scales by the number of servers). * Another application is to implement persistence of backend requests, - such that all requests sharing a certain criterium (such as an IP + such that all requests sharing a certain criteria (such as an IP address or session ID) get forwarded to the same backend server. When used with clusters of varnish servers, the shard director will, if otherwise configured equally, make the same decision on all -servers. In other words, requests sharing a common criterium used as +servers. In other words, requests sharing a common criteria used as the shard key will be balanced onto the same backend server(s) no matter which Varnish server handles the request. @@ -307,15 +307,15 @@ being the backend name) for each backend and for a running number from 1 to n (n is the number of `replicas`). Hashing creates the seemingly random order for placement of backends on the consistent hashing ring. -When ``.backend()`` is called, a load balacing key gets generated +When ``.backend()`` is called, a load balancing key gets generated unless provided. The smallest hash value in the circle is looked up that is larger than the key (searching clockwise and wrapping around as necessary). The backend for this hash value is the preferred backend for the given key. -If a healhy backend is requested, the search is continued linearly on +If a healthy backend is requested, the search is continued linearly on the ring as long as backends found are unhealthy or all backends have -been checked. The order auf these "alternative backends" on the ring +been checked. The order of these "alternative backends" on the ring is likely to differ for different keys. Alternative backends can also be selected explicitly. @@ -358,7 +358,7 @@ which will be hashed by `shard.reconfigure()` to construct the consistent hashing ring. The identification string defaults to the backend name. -`ident` allows to add multuple instances of the same backend. +`ident` allows to add multiple instances of the same backend. `rampup`: Optionally specify a rampup time for this backend. Defaults to the director rampup time. @@ -468,7 +468,7 @@ is _not_ the order given when backends are added. `set_rampup()` method or specifically per backend with the `set_backend()` method. -* `warmup` probabalistic alternative server selection +* `warmup` probabilistic alternative server selection possible values: -1, 0..1 @@ -515,7 +515,7 @@ ACKNOWLEDGEMENTS ================ Development of a previous version of the shard director was partly sponsored -by Deutsche Telekom AG ? Products & Innovation. +by Deutsche Telekom AG - Products & Innovation. Development of this version of the shard director was partly sponsored by BILD GmbH & Co KG. From fgsch at lodoss.net Mon Sep 12 21:59:10 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 12 Sep 2016 23:59:10 +0200 Subject: [master] 8bb7e90 Switch to globbing on the whole string Message-ID: commit 8bb7e90f9adf8502cd0be382cb546a28f9e24390 Author: Federico G. Schwindt Date: Sun Sep 11 03:09:54 2016 +0100 Switch to globbing on the whole string Fixes #2022 diff --git a/bin/varnishstat/varnishstat_options.h b/bin/varnishstat/varnishstat_options.h index ceeafac..65a961c 100644 --- a/bin/varnishstat/varnishstat_options.h +++ b/bin/varnishstat/varnishstat_options.h @@ -39,13 +39,10 @@ #define STAT_OPT_f \ VOPT("f:", "[-f ]", "Field inclusion glob", \ "Field inclusion glob." \ - " A field glob consists of three parts, type, ident and" \ - " name, where ident is optional. Each part can contain a" \ - " '*' character at the end to match a prefix. Use" \ - " backslash to escape characters. If the argument starts" \ - " with '^' it is used as an exclusion glob. Multiple -f" \ - " arguments may be given, and they will be applied in" \ - " order." \ + " Use backslash to escape characters. If the argument" \ + " starts with '^' it is used as an exclusion glob." \ + " Multiple -f arguments may be given, and they will be" \ + " applied in order." \ ) #define STAT_OPT_j \ VOPT("j", "[-j]", "Print statistics to stdout as JSON", \ diff --git a/bin/varnishtest/tests/u00002.vtc b/bin/varnishtest/tests/u00002.vtc index 38ce280..9f083ec 100644 --- a/bin/varnishtest/tests/u00002.vtc +++ b/bin/varnishtest/tests/u00002.vtc @@ -15,3 +15,12 @@ process p1 {varnishstat -1 -n ${v1_name} -f ^LCK.vbe.destroy \ shell "grep -q vbe ${tmpdir}/p1/stdout" shell "grep -q mempool ${tmpdir}/p1/stdout" err_shell "" "grep -q LCK.vbe.destroy ${tmpdir}/p1/stdout" + +process p2 {varnishstat -1 -n ${v1_name} -f ^*vbe.destroy \ + -f *vbe* -f *mempool*} -run + +shell "grep -q vbe ${tmpdir}/p2/stdout" +shell "grep -q mempool ${tmpdir}/p2/stdout" +err_shell "" "grep -q LCK.vbe.destroy ${tmpdir}/p2/stdout" + +shell "cmp -s ${tmpdir}/p1/stdout ${tmpdir}/p2/stdout" diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index f051cea..6af266e 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -42,6 +43,7 @@ #include "vas.h" #include "miniobj.h" #include "vqueue.h" +#include "vsb.h" #include "vapi/vsc.h" #include "vapi/vsm.h" @@ -75,14 +77,8 @@ struct vsc_sf { unsigned magic; #define VSC_SF_MAGIC 0x558478dd VTAILQ_ENTRY(vsc_sf) list; - int flags; -#define VSC_SF_EXCL (1 << 0) -#define VSC_SF_TY_WC (1 << 1) -#define VSC_SF_ID_WC (1 << 2) -#define VSC_SF_NM_WC (1 << 3) - char *type; - char *ident; - char *name; + char *pattern; + unsigned exclude; }; struct vsc { @@ -151,9 +147,7 @@ vsc_delete_sf_list(struct vsc *vsc) sf = VTAILQ_FIRST(&vsc->sf_list); CHECK_OBJ_NOTNULL(sf, VSC_SF_MAGIC); VTAILQ_REMOVE(&vsc->sf_list, sf, list); - free(sf->type); - free(sf->ident); - free(sf->name); + free(sf->pattern); FREE_OBJ(sf); } } @@ -180,105 +174,20 @@ vsc_f_arg(struct VSM_data *vd, const char *opt) { struct vsc *vsc = vsc_setup(vd); struct vsc_sf *sf; - const char *error = NULL; - const char *p, *q; - char *r; - int i; - int flags = 0; - char *parts[3]; AN(vd); AN(opt); - if (opt[0] == '^') { - flags |= VSC_SF_EXCL; - opt++; - } - - /* Split on '.' */ - memset(parts, 0, sizeof parts); - for (i = 0, p = opt; *p != '\0'; i++) { - for (q = p; *q != '\0' && *q != '.'; q++) - if (*q == '\\') - q++; - if (i < 3) { - parts[i] = malloc(1 + q - p); - AN(parts[i]); - memcpy(parts[i], p, q - p); - parts[i][q - p] = '\0'; - p = r = parts[i]; - - /* Unescape */ - while (1) { - if (*p == '\\') - p++; - if (*p == '\0') - break; - *r++ = *p++; - } - *r = '\0'; - } - p = q; - if (*p == '.') - p++; - } - if (i < 1 || i > 3) { - (void)vsm_diag(vd, "-f: Wrong number of elements"); - for (i = 0; i < 3; i++) - free(parts[i]); - return (-1); - } - - /* Set fields */ ALLOC_OBJ(sf, VSC_SF_MAGIC); AN(sf); - sf->flags = flags; - AN(parts[0]); - sf->type = parts[0]; - if (i == 2) { - AN(parts[1]); - sf->name = parts[1]; - } else if (i == 3) { - AN(parts[1]); - sf->ident = parts[1]; - AN(parts[2]); - sf->name = parts[2]; - } - /* Check for wildcards */ - if (sf->type != NULL) { - r = strchr(sf->type, '*'); - if (r != NULL && r[1] == '\0') { - *r = '\0'; - sf->flags |= VSC_SF_TY_WC; - } else if (r != NULL) - error = "-f: Wildcard not last"; - } - if (sf->ident != NULL) { - r = strchr(sf->ident, '*'); - if (r != NULL && r[1] == '\0') { - *r = '\0'; - sf->flags |= VSC_SF_ID_WC; - } else if (r != NULL) - error = "-f: Wildcard not last"; - } - if (sf->name != NULL) { - r = strchr(sf->name, '*'); - if (r != NULL && r[1] == '\0') { - *r = '\0'; - sf->flags |= VSC_SF_NM_WC; - } else if (r != NULL) - error = "-f: Wildcard not last"; + if (opt[0] == '^') { + sf->exclude = 1; + opt++; } - if (error != NULL) { - (void)vsm_diag(vd, "%s", error); - free(sf->type); - free(sf->ident); - free(sf->name); - FREE_OBJ(sf); - return (-1); - } + sf->pattern = strdup(opt); + AN(sf->pattern); VTAILQ_INSERT_TAIL(&vsc->sf_list, sf, list); return (1); @@ -449,52 +358,43 @@ vsc_build_pt_list(struct VSM_data *vd) /*-------------------------------------------------------------------- */ -static inline int -iter_test(const char *s1, const char *s2, int wc) -{ - - if (s1 == NULL) - return (0); - if (!wc) - return (strcmp(s1, s2)); - for (; *s1 != '\0' && *s1 == *s2; s1++, s2++) - continue; - return (*s1 != '\0'); -} - static void vsc_filter_pt_list(struct VSM_data *vd) { struct vsc *vsc = vsc_setup(vd); struct vsc_sf *sf; struct vsc_pt *pt, *pt2; - VTAILQ_HEAD(, vsc_pt) pt_list; + VTAILQ_HEAD(, vsc_pt) pt_list; + struct vsb *vsb; if (VTAILQ_EMPTY(&vsc->sf_list)) return; + vsb = VSB_new_auto(); + AN(vsb); + VTAILQ_INIT(&pt_list); VTAILQ_FOREACH(sf, &vsc->sf_list, list) { CHECK_OBJ_NOTNULL(sf, VSC_SF_MAGIC); VTAILQ_FOREACH_SAFE(pt, &vsc->pt_list, list, pt2) { CHECK_OBJ_NOTNULL(pt, VSC_PT_MAGIC); - if (iter_test(sf->type, pt->point.section->type, - sf->flags & VSC_SF_TY_WC)) - continue; - if (iter_test(sf->ident, pt->point.section->ident, - sf->flags & VSC_SF_ID_WC)) - continue; - if (iter_test(sf->name, pt->point.desc->name, - sf->flags & VSC_SF_NM_WC)) + VSB_clear(vsb); + VSB_printf(vsb, "%s.%s.%s", + pt->point.section->type, + pt->point.section->ident, + pt->point.desc->name); + VSB_finish(vsb); + if (fnmatch(sf->pattern, VSB_data(vsb), 0)) continue; VTAILQ_REMOVE(&vsc->pt_list, pt, list); - if (sf->flags & VSC_SF_EXCL) { + if (sf->exclude) FREE_OBJ(pt); - } else { + else VTAILQ_INSERT_TAIL(&pt_list, pt, list); - } } } + + VSB_destroy(&vsb); vsc_delete_pt_list(vsc); VTAILQ_CONCAT(&vsc->pt_list, &pt_list, list); } From nils.goroll at uplex.de Mon Sep 12 22:02:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 13 Sep 2016 00:02:12 +0200 Subject: [master] 44bca7b hand-edit the auto-generated code for flexelint - part 2 Message-ID: commit 44bca7b47e0080536aadf6019c8864490edfbb37 Author: Nils Goroll Date: Mon Sep 12 22:52:29 2016 +0200 hand-edit the auto-generated code for flexelint - part 2 diff --git a/lib/libvmod_directors/shard_parse_vcc_enums.c b/lib/libvmod_directors/shard_parse_vcc_enums.c index 1110077..e7b2095 100644 --- a/lib/libvmod_directors/shard_parse_vcc_enums.c +++ b/lib/libvmod_directors/shard_parse_vcc_enums.c @@ -94,7 +94,6 @@ enum by_e parse_by_e (const char *m) { enum healthy_e parse_healthy_e (const char *m) { - int p; enum healthy_e r; switch (m[0]) { @@ -107,7 +106,6 @@ enum healthy_e parse_healthy_e (const char *m) { //ALL if ((m[1] == 'L') && (m[2] == 'L') && (term(m[3]))) { r = ALL; - p = 3; goto ok; } goto invalid; @@ -115,7 +113,6 @@ enum healthy_e parse_healthy_e (const char *m) { //CHOSEN if ((m[1] == 'H') && (m[2] == 'O') && (m[3] == 'S') && (m[4] == 'E') && (m[5] == 'N') && (term(m[6]))) { r = CHOSEN; - p = 6; goto ok; } goto invalid; @@ -123,7 +120,6 @@ enum healthy_e parse_healthy_e (const char *m) { //IGNORE if ((m[1] == 'G') && (m[2] == 'N') && (m[3] == 'O') && (m[4] == 'R') && (m[5] == 'E') && (term(m[6]))) { r = IGNORE; - p = 6; goto ok; } goto invalid; @@ -131,5 +127,4 @@ enum healthy_e parse_healthy_e (const char *m) { return r; invalid: return _HEALTHY_E_INVALID; - (void)p; } From nils.goroll at uplex.de Mon Sep 12 22:02:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 13 Sep 2016 00:02:12 +0200 Subject: [master] d0a8130 take more advice from flexelint Message-ID: commit d0a81307fdeaef96625ff4cadc4b0cbc854e2252 Author: Nils Goroll Date: Mon Sep 12 22:53:16 2016 +0200 take more advice from flexelint diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index 54202f8..91f70d3 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -299,6 +299,9 @@ sharddir_pick_be(VRT_CTX, struct sharddir *shardd, CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); AN(ctx->vsl); + memset(&state, 0, sizeof(state)); + init_state(&state, ctx, shardd, vbit_init(picklist_spc, picklist_sz)); + sharddir_rdlock(shardd); if(shardd->n_backend == 0) { shard_err0(ctx, shardd, "no backends"); @@ -309,17 +312,14 @@ sharddir_pick_be(VRT_CTX, struct sharddir *shardd, validate_alt(ctx, shardd, &alt); - memset(&state, 0, sizeof(state)); - init_state(&state, ctx, shardd, vbit_init(picklist_spc, picklist_sz)); - state.idx = shard_lookup(shardd, key); assert(state.idx >= 0); - SHDBG(SHDBG_LOOKUP, shardd, "lookup key %x idx %d host %d", + SHDBG(SHDBG_LOOKUP, shardd, "lookup key %x idx %d host %u", key, state.idx, shardd->hashcircle[state.idx].host); if (alt > 0) { - if (shard_next(&state, alt - 1, healthy == ALL) == -1) { + if (shard_next(&state, alt - 1, healthy == ALL ? 1 : 0) == -1) { if (state.previous.hostid != -1) { be = sharddir_backend(shardd, state.previous.hostid); @@ -329,7 +329,7 @@ sharddir_pick_be(VRT_CTX, struct sharddir *shardd, } } - if (shard_next(&state, 0, healthy != IGNORE) == -1) { + if (shard_next(&state, 0, healthy == IGNORE ? 0 : 1) == -1) { if (state.previous.hostid != -1) { be = sharddir_backend(shardd, state.previous.hostid); goto ok; diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index f40d0ac..de35c41 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -105,7 +105,6 @@ void sharddir_debug(struct sharddir *shardd, const uint32_t flags); void sharddir_err(VRT_CTX, enum VSL_tag_e tag, const char *fmt, ...); void sharddir_new(struct sharddir **sharddp, const char *vcl_name); void sharddir_delete(struct sharddir **sharddp); -void sharddir_init_hashcircle(struct sharddir *shardd, VCL_INT replicas); void sharddir_rdlock(struct sharddir *shardd); void sharddir_wrlock(struct sharddir *shardd); void sharddir_unlock(struct sharddir *shardd); diff --git a/lib/libvmod_directors/shard_hash.c b/lib/libvmod_directors/shard_hash.c index 63317bf..aa66410 100644 --- a/lib/libvmod_directors/shard_hash.c +++ b/lib/libvmod_directors/shard_hash.c @@ -47,13 +47,13 @@ * dependency nightmare (at least for now) */ -unsigned long crc32(unsigned long, const void *buf, unsigned len); +unsigned long crc32(unsigned long, const unsigned char *buf, unsigned len); static uint32_t __match_proto__(hash_func) shard_hash_crc32(VCL_STRING s) { uint32_t crc; - crc = crc32(~0U, s, strlen(s)); + crc = crc32(~0U, (const unsigned char *)s, strlen(s)); crc ^= ~0U; return (crc); } @@ -104,7 +104,7 @@ _shard_hash_invalid(VCL_STRING s) NEEDLESS_RETURN(0); } -hash_func shard_hash_f[_ALG_E_MAX] = { +const hash_func shard_hash_f[_ALG_E_MAX] = { [_ALG_E_INVALID] = _shard_hash_invalid, [CRC32] = shard_hash_crc32, [SHA256] = shard_hash_sha256, diff --git a/lib/libvmod_directors/shard_hash.h b/lib/libvmod_directors/shard_hash.h index 5ee38ad..838032a 100644 --- a/lib/libvmod_directors/shard_hash.h +++ b/lib/libvmod_directors/shard_hash.h @@ -27,4 +27,4 @@ */ typedef uint32_t (*hash_func)(VCL_STRING); -extern hash_func shard_hash_f[_ALG_E_MAX]; +extern const hash_func shard_hash_f[_ALG_E_MAX]; From nils.goroll at uplex.de Mon Sep 12 22:02:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 13 Sep 2016 00:02:12 +0200 Subject: [master] dc04d59 silence flexelint Message-ID: commit dc04d592ae06ba136703e889260466ecc985a864 Author: Nils Goroll Date: Mon Sep 12 23:12:49 2016 +0200 silence flexelint diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index 44e5c28..3f91aab 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -41,6 +41,7 @@ #include "shard_cfg.h" #include "shard_hash.h" +/*lint -esym(749, shard_change_task_e::*) */ enum shard_change_task_e { _INVALID = 0, CLEAR, diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index 91f70d3..52f9d84 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -28,6 +28,8 @@ * SUCH DAMAGE. */ +/*lint -e801 */ + #include "config.h" #include diff --git a/lib/libvmod_directors/shard_parse_vcc_enums.c b/lib/libvmod_directors/shard_parse_vcc_enums.c index e7b2095..7061ced 100644 --- a/lib/libvmod_directors/shard_parse_vcc_enums.c +++ b/lib/libvmod_directors/shard_parse_vcc_enums.c @@ -6,6 +6,8 @@ * TODO: integrate in vmodtool.py or replace with something else */ +/*lint -e801 */ + #include "shard_parse_vcc_enums.h" #define term(c) ((c) == '\0') diff --git a/lib/libvmod_directors/shard_parse_vcc_enums.h b/lib/libvmod_directors/shard_parse_vcc_enums.h index d6e2ff9..afa0c47 100644 --- a/lib/libvmod_directors/shard_parse_vcc_enums.h +++ b/lib/libvmod_directors/shard_parse_vcc_enums.h @@ -17,6 +17,8 @@ enum alg_e { enum alg_e parse_alg_e (const char *); +/*lint -esym(769, by_e::_BY_E_MAX) */ + enum by_e { _BY_E_INVALID = 0, BY_HASH, @@ -26,9 +28,10 @@ enum by_e { _BY_E_MAX }; - enum by_e parse_by_e (const char *); +/*lint -esym(769, healthy_e::_HEALTHY_E_MAX) */ + enum healthy_e { _HEALTHY_E_INVALID = 0, CHOSEN, From nils.goroll at uplex.de Mon Sep 12 22:02:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 13 Sep 2016 00:02:12 +0200 Subject: [master] 65ba04b re-fix two spelling nits Message-ID: commit 65ba04b126546e70c03f63ecb1fcdb7f0d3ea587 Author: Nils Goroll Date: Mon Sep 12 23:59:56 2016 +0200 re-fix two spelling nits Big thank you to fgs for the rest of the spell correction! diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index a43f5e4..c3e2a8a 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -275,12 +275,12 @@ Varnish servers: increase efficiency (scales by the number of servers). * Another application is to implement persistence of backend requests, - such that all requests sharing a certain criteria (such as an IP + such that all requests sharing a certain criterion (such as an IP address or session ID) get forwarded to the same backend server. When used with clusters of varnish servers, the shard director will, if otherwise configured equally, make the same decision on all -servers. In other words, requests sharing a common criteria used as +servers. In other words, requests sharing a common criterion used as the shard key will be balanced onto the same backend server(s) no matter which Varnish server handles the request. From phk at FreeBSD.org Tue Sep 13 06:45:10 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Sep 2016 08:45:10 +0200 Subject: [master] dc2ef5e Ignore H2 upgrades if the feature is not enabled. Message-ID: commit dc2ef5e8584c6aa1a1eee32de8d17f6f3c412f4f Author: Poul-Henning Kamp Date: Tue Sep 13 06:44:32 2016 +0000 Ignore H2 upgrades if the feature is not enabled. Fixes: #2086 diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index 4c44102..66752be 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -416,21 +416,21 @@ HTTP1_Session(struct worker *wrk, struct req *req) http1_setstate(sp, H1CLEANUP); continue; } - if (req->htc->body_status == BS_NONE && /* XXX */ - http_HdrIs(req->http, H_Upgrade, "h2c")) { + if (http_HdrIs(req->http, H_Upgrade, "h2c")) { if (!FEATURE(FEATURE_HTTP2)) { VSLb(req->vsl, SLT_Debug, "H2 upgrade attempt"); - SES_Close(req->sp, req->doclose); - http1_setstate(sp, H1CLEANUP); - continue; + } else if (req->htc->body_status != BS_NONE) { + VSLb(req->vsl, SLT_Debug, + "H2 upgrade attempt has body"); + } else { + VSLb(req->vsl, SLT_Debug, "H2 Upgrade"); + http1_setstate(sp, NULL); + req->err_code = 2; + SES_SetTransport(wrk, sp, req, + &H2_transport); + return; } - VSLb(req->vsl, SLT_Debug, - "H2 Upgrade"); - http1_setstate(sp, NULL); - req->err_code = 2; - SES_SetTransport(wrk, sp, req, &H2_transport); - return; } req->req_step = R_STP_RECV; http1_setstate(sp, H1PROC); diff --git a/bin/varnishtest/tests/t02001.vtc b/bin/varnishtest/tests/t02001.vtc index af32f27..a7fa8b0 100644 --- a/bin/varnishtest/tests/t02001.vtc +++ b/bin/varnishtest/tests/t02001.vtc @@ -4,18 +4,39 @@ barrier b1 cond 2 server s1 { rxreq + expect req.url == /noupgrade expect req.http.host == foo.bar - barrier b1 sync txresp -status 400 -bodylen 10 } -start varnish v1 -vcl+backend {} -start -varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set feature -http2" varnish v1 -cliok "param.set debug +syncvsl" client c1 { - send "GET / HTTP/1.1\r\n" + send "GET /noupgrade HTTP/1.1\r\n" + send "Host: foo.bar\r\n" + send "Upgrade: h2c\r\n" + send "HTTP2-Settings: AAMAAABkAAQAAP__\r\n" + send "\r\n" + rxresp + expect resp.status == 400 + expect resp.bodylen == 10 +} -run + +server s1 { + rxreq + expect req.url == /upgrade + expect req.http.host == foo.bar + barrier b1 sync + txresp -status 401 -bodylen 11 +} -start + +varnish v1 -cliok "param.set feature +http2" + +client c1 { + send "GET /upgrade HTTP/1.1\r\n" send "Host: foo.bar\r\n" send "Upgrade: h2c\r\n" send "HTTP2-Settings: AAMAAABkAAQAAP__\r\n" @@ -35,7 +56,7 @@ client c1 { barrier b1 sync stream 1 { rxresp - expect resp.status == 400 - expect resp.bodylen == 10 + expect resp.status == 401 + expect resp.bodylen == 11 } -run } -run From fgsch at lodoss.net Tue Sep 13 07:49:10 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 13 Sep 2016 09:49:10 +0200 Subject: [master] 0667b98 More spelling Message-ID: commit 0667b98ce5d4e20e5c96ad35154bcbb52b4b9f6f Author: Federico G. Schwindt Date: Tue Sep 13 08:48:54 2016 +0100 More spelling diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 94d05b7..011541b 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -432,7 +432,7 @@ SLTM(End, 0, "Marks the end of a VXID", ) SLTM(VSL, 0, "VSL API warnings and error message", - "Warnings and error messages genererated by the VSL API while" + "Warnings and error messages generated by the VSL API while" " reading the shared memory log.\n\n" ) @@ -541,11 +541,11 @@ SLTM(H2RxBody, 0, "Received HTTP2 frame body", "Binary data" ) -SLTM(H2TxHdr, 0, "Transmit HTTP2 frame header", +SLTM(H2TxHdr, 0, "Transmitted HTTP2 frame header", "Binary data" ) -SLTM(H2TxBody, 0, "Transmit HTTP2 frame body", +SLTM(H2TxBody, 0, "Transmitted HTTP2 frame body", "Binary data" ) From phk at FreeBSD.org Tue Sep 13 07:52:14 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Sep 2016 09:52:14 +0200 Subject: [master] 7a39df3 Add a '-vsc' which dumps a set of counters using a 'glob' pattern Message-ID: commit 7a39df3041e43f0adb5532f22bd932c242df922b Author: Poul-Henning Kamp Date: Tue Sep 13 07:15:07 2016 +0000 Add a '-vsc' which dumps a set of counters using a 'glob' pattern diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index ef01e96..c48ef05 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -35,6 +35,7 @@ #include #include +#include #include #include #include @@ -791,6 +792,64 @@ varnish_vclbackend(struct varnish *v, const char *vcl) } /********************************************************************** + */ + +struct dump_priv { + const char *arg; + const struct varnish *v; +}; + +static int +do_stat_dump_cb(void *priv, const struct VSC_point * const pt) +{ + const struct varnish *v; + struct dump_priv *dp; + uint64_t u; + char buf[1024]; + + if (pt == NULL) + return (0); + dp = priv; + v = dp->v; + + if(strcmp(pt->desc->ctype, "uint64_t")) + return (0); + u = *(const volatile uint64_t*)pt->ptr; + + strcpy(buf, pt->section->type); + if (pt->section->ident[0] != '\0') { + strcat(buf, "."); + strcat(buf, pt->section->ident); + } + strcat(buf, "."); + strcat(buf, pt->desc->name); + + if (strcmp(dp->arg, "*")) { + if (fnmatch(dp->arg, buf, 0)) + return (0); + } + + vtc_log(v->vl, 4, "VSC %s %ju", buf, u); + return (0); +} + +static void +varnish_vsc(const struct varnish *v, const char *arg) +{ + struct dump_priv dp; + + memset(&dp, 0, sizeof dp); + dp.v = v; + dp.arg = arg; + if (VSM_Abandoned(v->vd)) { + VSM_Close(v->vd); + VSM_Open(v->vd); + } + + (void)VSC_Iter(v->vd, NULL, do_stat_dump_cb, &dp); +} + +/********************************************************************** * Check statistics */ @@ -822,6 +881,9 @@ do_stat_cb(void *priv, const struct VSC_point * const pt) return (1); } +/********************************************************************** + */ + static void varnish_expect(const struct varnish *v, char * const *av) { @@ -1001,6 +1063,12 @@ varnish_expect(const struct varnish *v, char * const *av) * a correct value. OP can be ==, >, >=, <, <=. For example:: * * varnish v1 -expect SMA.s1.g_space > 1000000 + * + * \-vsc PATTERN + * Dump VSC counters matching PATTERN. The PATTERN is a 'glob' + * style pattern (ie: fnmatch(3)) as used in shell filename expansion. + * To see all counters use pattern "*", to see all counters about + * requests use "*req*". */ void @@ -1131,6 +1199,12 @@ cmd_varnish(CMD_ARGS) av++; continue; } + if (!strcmp(*av, "-vsc")) { + AN(av[1]); + varnish_vsc(v, av[1]); + av++; + continue; + } if (!strcmp(*av, "-wait-stopped")) { wait_stopped(v); continue; From phk at FreeBSD.org Tue Sep 13 07:52:14 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Sep 2016 09:52:14 +0200 Subject: [master] 8a50626 Correctly clean up H2 sessions Message-ID: commit 8a506262f14ca7df8a833bfcf4e318a15af7aea6 Author: Poul-Henning Kamp Date: Tue Sep 13 07:50:38 2016 +0000 Correctly clean up H2 sessions diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 62b5079..73d0c95 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -141,7 +141,6 @@ h2_new_sess(const struct worker *wrk, struct sess *sp, struct req *srq) h2 = WS_Alloc(srq->ws, sizeof *h2); AN(h2); INIT_OBJ(h2, H2_SESS_MAGIC); - h2->refcnt = 1; h2->srq = srq; h2->htc = srq->htc; h2->ws = srq->ws; @@ -203,20 +202,29 @@ static void h2_del_req(struct worker *wrk, struct h2_req *r2, enum h2_error_e err) { struct h2_sess *h2; + struct sess *sp; + struct req *req; int r; (void)err; h2 = r2->h2sess; - Lck_Lock(&h2->sess->mtx); + sp = h2->sess; + Lck_Lock(&sp->mtx); assert(h2->refcnt > 0); r = --h2->refcnt; /* XXX: PRIORITY reshuffle */ VTAILQ_REMOVE(&h2->streams, r2, list); - Lck_Unlock(&h2->sess->mtx); - Req_Cleanup(h2->sess, wrk, r2->req); + Lck_Unlock(&sp->mtx); + Req_Cleanup(sp, wrk, r2->req); + Req_Release(r2->req); if (r) return; - SES_Delete(h2->sess, SC_RX_JUNK, NAN); + + /* All streams gone, including stream #0, clean up */ + req = h2->srq; + Req_Cleanup(sp, wrk, req); + Req_Release(req); + SES_Delete(sp, SC_RX_JUNK, NAN); } /********************************************************************** @@ -687,7 +695,7 @@ h2_new_session(struct worker *wrk, void *arg) struct req *req; struct sess *sp; struct h2_sess *h2; - struct h2_req *r2; + struct h2_req *r2, *r22; char *wsp; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -742,10 +750,11 @@ h2_new_session(struct worker *wrk, void *arg) HTC_RxInit(h2->htc, wrk->aws); } - r2 = VTAILQ_FIRST(&h2->streams); - CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); - assert(r2->stream == 0); - h2_del_req(wrk, r2, H2E_NO_ERROR); + /* Delete all idle streams */ + VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) { + if (r2->state == H2_S_IDLE) + h2_del_req(wrk, r2, H2E_NO_ERROR); + } } struct transport H2_transport = { diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index da3d6b2..7ebf4b3 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -32,4 +32,10 @@ client c1 { expect ping.ack == "true" expect ping.data == "_-__-_-_" } -run + stream 7 -wait } -run + +varnish v1 -expect MEMPOOL.req0.live == 0 +varnish v1 -expect MEMPOOL.req1.live == 0 +varnish v1 -expect MEMPOOL.sess0.live == 0 +varnish v1 -expect MEMPOOL.sess1.live == 0 diff --git a/bin/varnishtest/tests/t02001.vtc b/bin/varnishtest/tests/t02001.vtc index a7fa8b0..43601a7 100644 --- a/bin/varnishtest/tests/t02001.vtc +++ b/bin/varnishtest/tests/t02001.vtc @@ -60,3 +60,8 @@ client c1 { expect resp.bodylen == 11 } -run } -run + +varnish v1 -expect MEMPOOL.req0.live == 0 +varnish v1 -expect MEMPOOL.req1.live == 0 +varnish v1 -expect MEMPOOL.sess0.live == 0 +varnish v1 -expect MEMPOOL.sess1.live == 0 diff --git a/bin/varnishtest/tests/t02002.vtc b/bin/varnishtest/tests/t02002.vtc index ebb0146..f2cfb50 100644 --- a/bin/varnishtest/tests/t02002.vtc +++ b/bin/varnishtest/tests/t02002.vtc @@ -35,3 +35,8 @@ client c1 { expect resp.bodylen == 20 } -run } -run + +varnish v1 -expect MEMPOOL.req0.live == 0 +varnish v1 -expect MEMPOOL.req1.live == 0 +varnish v1 -expect MEMPOOL.sess0.live == 0 +varnish v1 -expect MEMPOOL.sess1.live == 0 From phk at FreeBSD.org Tue Sep 13 08:40:16 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Sep 2016 10:40:16 +0200 Subject: [master] 6ca043b Fix printf-format Message-ID: commit 6ca043bb374046845042840607d4dd522c3d5fd5 Author: Poul-Henning Kamp Date: Tue Sep 13 08:39:39 2016 +0000 Fix printf-format diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index c48ef05..eaf91f4 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -829,7 +829,7 @@ do_stat_dump_cb(void *priv, const struct VSC_point * const pt) return (0); } - vtc_log(v->vl, 4, "VSC %s %ju", buf, u); + vtc_log(v->vl, 4, "VSC %s %ju", buf, (uintmax_t)u); return (0); } From nils.goroll at uplex.de Tue Sep 13 10:31:13 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 13 Sep 2016 12:31:13 +0200 Subject: [master] 6b13464 more changes Message-ID: commit 6b13464c406b937722055aebf041a5f0f4f12853 Author: Nils Goroll Date: Tue Sep 13 12:30:34 2016 +0200 more changes diff --git a/doc/changes.rst b/doc/changes.rst index 9bcafa6..7531d91 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -27,6 +27,22 @@ Major items: * new shard director for loadbalancing by consistent hashing +* ban lurker performance improvements + +* access to obj.ttl, obj.age, obj.grace and obj.keep in vcl_deliver + +News for Vmod Authors +--------------------- + +* vcl cli events (init/fini methods) now have a workspace and + PRIV_TASK + +* PRIV_* now also work for object methods with unchanged scope. In + particular, they are per vmod and `not` per object - e.g. the same + PRIV_TASK gets passed to object methods as to functions during a VCL + task. + + ====================================== Varnish Cache 4.1.3-beta1 (2016-06-15) ====================================== From phk at FreeBSD.org Tue Sep 13 11:06:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Sep 2016 13:06:11 +0200 Subject: [master] 15faf65 Make sure separate VCL's don't get tied too hard to vhosts, they can be used for so many more things than that. Message-ID: commit 15faf651c8a3fec22b9e3a674a32ad7be53a7224 Author: Poul-Henning Kamp Date: Tue Sep 13 11:04:59 2016 +0000 Make sure separate VCL's don't get tied too hard to vhosts, they can be used for so many more things than that. diff --git a/doc/changes.rst b/doc/changes.rst index 7531d91..38cfe49 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -8,7 +8,7 @@ The list of changes are numerous and will not be expanded on in detail. Major items: -* VCL labels, allowing for per-vhost VCL. +* VCL labels, allowing for per-vhost (or per-anything) separate VCL files. * (Very!) experimental support for HTTP/2. From hermunn at varnish-software.com Tue Sep 13 11:07:11 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Tue, 13 Sep 2016 13:07:11 +0200 Subject: [4.1] 9bbbc0b Make sure the step numbes are decreasing Message-ID: commit 9bbbc0b1437656a3be65814eaf9ad438b4e77cfb Author: Guillaume Quintard Date: Thu Sep 8 12:57:00 2016 +0200 Make sure the step numbes are decreasing Thanks @nigoroll for the suggestion. diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 5db516f..d6bbfcf 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -333,7 +333,7 @@ EmitCoordinates(const struct vcc *tl, struct vsb *vsb) static void EmitInitFini(const struct vcc *tl) { - struct inifin *p; + struct inifin *p, *q = NULL; unsigned has_event = 0; Fh(tl, 0, "\n"); @@ -370,6 +370,8 @@ EmitInitFini(const struct vcc *tl) Fc(tl, 0, "\t(void)VGC_function_vcl_fini(ctx);\n\n"); VTAILQ_FOREACH_REVERSE(p, &tl->inifin, inifinhead, list) { AZ(VSB_finish(p->fin)); + if (q) + assert(q->n > p->n); if (VSB_len(p->fin)) { Fc(tl, 0, "\t/* %u */\n", p->n); Fc(tl, 0, "\tif (vgc_inistep >= %u) {\n", p->n); From hermunn at varnish-software.com Tue Sep 13 11:07:11 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Tue, 13 Sep 2016 13:07:11 +0200 Subject: [4.1] a70c5ea Assign q, otherwise the whole check is useless Message-ID: commit a70c5ea3bfcf52858a54777c8b0ce6c737eead26 Author: Guillaume Quintard Date: Thu Sep 8 14:57:22 2016 +0200 Assign q, otherwise the whole check is useless Spotted by: Dridi Boukelmoune diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index d6bbfcf..144ff1e 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -372,6 +372,7 @@ EmitInitFini(const struct vcc *tl) AZ(VSB_finish(p->fin)); if (q) assert(q->n > p->n); + q = p; if (VSB_len(p->fin)) { Fc(tl, 0, "\t/* %u */\n", p->n); Fc(tl, 0, "\tif (vgc_inistep >= %u) {\n", p->n); From hermunn at varnish-software.com Tue Sep 13 11:07:11 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Tue, 13 Sep 2016 13:07:11 +0200 Subject: [4.1] 0854d43 fix misnomer in comment Message-ID: commit 0854d434b14fae0fe1816a1d227ca56574aac336 Author: Nils Goroll Date: Fri Sep 9 16:11:11 2016 +0200 fix misnomer in comment diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 647d37b..e718374 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -371,7 +371,7 @@ vca_make_session(struct worker *wrk, void *arg) } /*-------------------------------------------------------------------- - * This function accepts on a single socket for a single session pool. + * This function accepts on a single socket for a single thread pool. * * As long as we can stick the accepted connection to another thread * we do so, otherwise we put the socket back on the "BACK" pool @@ -459,7 +459,7 @@ vca_accept_task(struct worker *wrk, void *arg) } /*-------------------------------------------------------------------- - * Called when a worker and attached session pool is created, to + * Called when a worker and attached thread pool is created, to * allocate the tasks which will listen to sockets for that pool. */ From hermunn at varnish-software.com Tue Sep 13 11:07:11 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Tue, 13 Sep 2016 13:07:11 +0200 Subject: [4.1] 734b2b6 improve thread_pools documentation Message-ID: commit 734b2b6f95afbf2236edd9133727a391fc03adba Author: Nils Goroll Date: Fri Sep 9 16:31:03 2016 +0200 improve thread_pools documentation diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index d14a32f..75195bc 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -93,11 +93,14 @@ struct parspec WRK_parspec[] = { "1", NULL, "Number of worker thread pools.\n" "\n" - "Increasing number of worker pools decreases lock " - "contention.\n" + "Increasing the number of worker pools decreases lock " + "contention. Each worker pool also has a thread accepting " + "new connections, so for very high rates of incoming new " + "connections on systems with many cores, increasing the " + "worker pools may be required.\n" "\n" - "Too many pools waste CPU and RAM resources, and more than " - "one pool for each CPU is probably detrimal to performance.\n" + "Too many pools waste CPU and RAM resources, and more than one " + "pool for each CPU is most likely detrimal to performance.\n" "\n" "Can be increased on the fly, but decreases require a " "restart to take effect.", diff --git a/include/tbl/params.h b/include/tbl/params.h index 5c3f46b..de9e5eb 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1170,10 +1170,14 @@ PARAM( /* s-text */ "Number of worker thread pools.\n" "\n" - "Increasing number of worker pools decreases lock contention.\n" + "Increasing the number of worker pools decreases lock " + "contention. Each worker pool also has a thread accepting " + "new connections, so for very high rates of incoming new " + "connections on systems with many cores, increasing the " + "worker pools may be required.\n" "\n" "Too many pools waste CPU and RAM resources, and more than one " - "pool for each CPU is probably detrimal to performance.\n" + "pool for each CPU is most likely detrimal to performance.\n" "\n" "Can be increased on the fly, but decreases require a restart to " "take effect.", From hermunn at varnish-software.com Tue Sep 13 11:07:11 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Tue, 13 Sep 2016 13:07:11 +0200 Subject: [4.1] a5c318b leave some comments pointing to the right locations Message-ID: commit a5c318befb4fb4d8155637783a443d4df48f328e Author: Nils Goroll Date: Fri Sep 9 17:06:52 2016 +0200 leave some comments pointing to the right locations Maybe it's just my brain that is incapable of considering the fact that a bit of macro-fillin may in fact be #if 0 - disabled here, so this should hopefully help me (and others?) to not spend more life time than absolutely necessary on feeling stupid. diff --git a/include/tbl/params.h b/include/tbl/params.h index de9e5eb..951574d 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -322,6 +322,8 @@ PARAM( ) #if 0 +/* actual location mgt_param_bits.c*/ +/* see tbl/debug_bits.h */ PARAM( /* name */ debug, /* typ */ debug, @@ -400,6 +402,8 @@ PARAM( ) #if 0 +/* actual location mgt_param_bits.c*/ +/* See tbl/feature_bits.h */ PARAM( /* name */ feature, /* typ */ feature, @@ -736,6 +740,7 @@ PARAM( ) #if 0 +/* actual location mgt_param_tbl.c */ PARAM( /* name */ pcre_match_limit, /* typ */ uint, @@ -751,6 +756,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tbl.c */ PARAM( /* name */ pcre_match_limit_recursion, /* typ */ uint, @@ -799,6 +805,7 @@ PARAM( ) #if 0 +/* actual location mgt_param_tbl.c */ PARAM( /* name */ pool_req, /* typ */ poolparam, @@ -814,6 +821,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tbl.c */ PARAM( /* name */ pool_sess, /* typ */ poolparam, @@ -829,6 +837,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tbl.c */ PARAM( /* name */ pool_vbo, /* typ */ poolparam, @@ -984,6 +993,7 @@ PARAM( ) #if 0 +/* actual location mgt_param_tcp.c */ PARAM( /* name */ tcp_keepalive_intvl, /* typ */ timeout, @@ -998,6 +1008,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tcp.c */ PARAM( /* name */ tcp_keepalive_probes, /* typ */ uint, @@ -1014,6 +1025,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tcp.c */ PARAM( /* name */ tcp_keepalive_time, /* typ */ timeout, @@ -1029,6 +1041,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_add_delay, /* typ */ timeout, @@ -1049,6 +1062,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_destroy_delay, /* typ */ timeout, @@ -1064,6 +1078,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_fail_delay, /* typ */ timeout, @@ -1090,6 +1105,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_max, /* typ */ thread_pool_max, @@ -1108,6 +1124,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_min, /* typ */ thread_pool_min, @@ -1126,6 +1143,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_stack, /* typ */ bytes, @@ -1142,6 +1160,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_timeout, /* typ */ timeout, @@ -1159,6 +1178,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_pools, /* typ */ uint, @@ -1185,6 +1205,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_queue_limit, /* typ */ uint, @@ -1203,6 +1224,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_pool.c */ PARAM( /* name */ thread_stats_rate, /* typ */ uint, @@ -1260,6 +1282,7 @@ PARAM( ) #if 0 +/* actual location mgt_param_tbl.c */ PARAM( /* name */ vcc_allow_inline_c, /* typ */ bool, @@ -1274,6 +1297,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tbl.c */ PARAM( /* name */ vcc_err_unref, /* typ */ bool, @@ -1288,6 +1312,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tbl.c */ PARAM( /* name */ vcc_unsafe_path, /* typ */ bool, @@ -1335,6 +1360,7 @@ PARAM( ) #if 0 +/* actual location mgt_param_tbl.c */ PARAM( /* name */ vcl_dir, /* typ */ string, @@ -1350,6 +1376,7 @@ PARAM( /* func */ NULL ) +/* actual location mgt_param_tbl.c */ PARAM( /* name */ vmod_dir, /* typ */ string, @@ -1385,6 +1412,7 @@ PARAM( ) #if 0 +/* actual location mgt_param_bits.c*/ PARAM( /* name */ vsl_mask, /* typ */ vsl_mask, @@ -1453,6 +1481,7 @@ PARAM( ) #if 0 +/* see mgt_waiter.c */ PARAM( /* name */ waiter, /* typ */ waiter, From hermunn at varnish-software.com Tue Sep 13 11:07:11 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Tue, 13 Sep 2016 13:07:11 +0200 Subject: [4.1] d0220de fix old typo Message-ID: commit d0220defd5daa000b2d566fbbe127ebef1f84708 Author: Nils Goroll Date: Fri Sep 9 18:01:36 2016 +0200 fix old typo thanks to @dridi for the dictionary lookup diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index 75195bc..54407d0 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -100,7 +100,7 @@ struct parspec WRK_parspec[] = { "worker pools may be required.\n" "\n" "Too many pools waste CPU and RAM resources, and more than one " - "pool for each CPU is most likely detrimal to performance.\n" + "pool for each CPU is most likely detrimental to performance.\n" "\n" "Can be increased on the fly, but decreases require a " "restart to take effect.", diff --git a/include/tbl/params.h b/include/tbl/params.h index 951574d..d1da313 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1197,7 +1197,7 @@ PARAM( "worker pools may be required.\n" "\n" "Too many pools waste CPU and RAM resources, and more than one " - "pool for each CPU is most likely detrimal to performance.\n" + "pool for each CPU is most likely detrimental to performance.\n" "\n" "Can be increased on the fly, but decreases require a restart to " "take effect.", From hermunn at varnish-software.com Tue Sep 13 11:07:11 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Tue, 13 Sep 2016 13:07:11 +0200 Subject: [4.1] 3273ffa Clarify doc regarding -p with -1 Message-ID: commit 3273ffaead361f4cd2cb79c776e548d4b5689ef2 Author: Federico G. Schwindt Date: Sun Sep 11 03:30:11 2016 +0100 Clarify doc regarding -p with -1 Related to #2025. diff --git a/bin/varnishtop/varnishtop_options.h b/bin/varnishtop/varnishtop_options.h index f87a5f4..b65a961 100644 --- a/bin/varnishtop/varnishtop_options.h +++ b/bin/varnishtop/varnishtop_options.h @@ -50,6 +50,7 @@ " default is 60 seconds. The first number in the list is" \ " the average number of requests seen over this time" \ " period." \ + " This option has no effect if -1 option is also used." \ ) TOP_OPT_1 From hermunn at varnish-software.com Tue Sep 13 11:07:11 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Tue, 13 Sep 2016 13:07:11 +0200 Subject: [4.1] 90828d7 Minor polish while here anyway Message-ID: commit 90828d774a0a8667d9659fc490dea421365879d4 Author: Poul-Henning Kamp Date: Mon Sep 12 06:16:27 2016 +0000 Minor polish while here anyway diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 4e0e7bc..e59ec7f 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -150,7 +150,7 @@ void mgt_SHM_Init(void); void mgt_SHM_static_alloc(const void *, ssize_t size, const char *class, const char *type, const char *ident); void mgt_SHM_Create(void); -void mgt_SHM_Commit(void); +int mgt_SHM_Commit(void); void mgt_SHM_Destroy(int keep); void mgt_SHM_Size_Adjust(void); diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 3ac39a0..6e61306 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -742,7 +742,9 @@ MGT_Run(void) } } - mgt_SHM_Commit(); + i = mgt_SHM_Commit(); + if (i != 0) + MGT_complain(C_ERR, "Could not commit SHM file"); i = vev_schedule(mgt_evb); if (i != 0) diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c index 9345d38..7d941ee 100644 --- a/bin/varnishd/mgt/mgt_shmem.c +++ b/bin/varnishd/mgt/mgt_shmem.c @@ -251,20 +251,22 @@ mgt_SHM_Create(void) * Commit the VSM */ -void +int mgt_SHM_Commit(void) { char fnbuf[64]; + int retval = 0; bprintf(fnbuf, "%s.%jd", VSM_FILENAME, (intmax_t)getpid()); VJ_master(JAIL_MASTER_FILE); if (rename(fnbuf, VSM_FILENAME)) { - fprintf(stderr, "Rename failed %s -> %s: %s\n", + MGT_complain(C_ERR, "Rename failed %s -> %s: %s\n", fnbuf, VSM_FILENAME, strerror(errno)); (void)unlink(fnbuf); - exit(1); + retval = -1; } VJ_master(JAIL_MASTER_LOW); + return (retval); } /*-------------------------------------------------------------------- From hermunn at varnish-software.com Tue Sep 13 11:07:11 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Tue, 13 Sep 2016 13:07:11 +0200 Subject: [4.1] 7268df1 Bail out on this error. Message-ID: commit 7268df19e1e829a4295192c644025110508567ad Author: Poul-Henning Kamp Date: Mon Sep 12 06:16:57 2016 +0000 Bail out on this error. diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 6e61306..2fa0795 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -743,8 +743,10 @@ MGT_Run(void) } i = mgt_SHM_Commit(); - if (i != 0) + if (i != 0) { MGT_complain(C_ERR, "Could not commit SHM file"); + return; + } i = vev_schedule(mgt_evb); if (i != 0) From hermunn at varnish-software.com Tue Sep 13 11:07:11 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Tue, 13 Sep 2016 13:07:11 +0200 Subject: [4.1] d5044f5 Only update the active_vcl on successful VCL compiles Message-ID: commit d5044f53e98e64624288b8e7ac143409014f8d5e Author: Poul-Henning Kamp Date: Mon Sep 12 09:12:41 2016 +0000 Only update the active_vcl on successful VCL compiles diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 21f84f7..3801863 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -82,8 +82,6 @@ mgt_vcl_add(const char *name, const char *libfile, const char *state) bprintf(vp->state, "%s", state); - if (active_vcl == NULL) - active_vcl = vp; VTAILQ_INSERT_TAIL(&vclhead, vp, list); return (vp); } @@ -210,6 +208,9 @@ mgt_new_vcl(struct cli *cli, const char *vclname, const char *vclsrc, vp = mgt_vcl_add(vclname, lib, state); free(lib); + if (active_vcl == NULL) + active_vcl = vp; + if (child_pid < 0) return; From hermunn at varnish-software.com Tue Sep 13 11:43:11 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Tue, 13 Sep 2016 13:43:11 +0200 Subject: [4.1] 26252af Correct behavior when S_arg is the empty string Message-ID: commit 26252af4e3af58be94afd8e2bbfbb808fee1740e Author: P?l Hermunn Johansen Date: Tue Sep 13 13:39:17 2016 +0200 Correct behavior when S_arg is the empty string A previous backport, "Use "none" to disable authentication" did not skip mgt_cli_secret(S_arg) when -S '' was used. This fixes the problem, so that -S '' has the same behavior as before. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index ae4ce15..611695f 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -822,7 +822,7 @@ main(int argc, char * const *argv) if (d_flag) mgt_cli_setup(0, 1, 1, "debug", cli_stdin_close, NULL); - if (strcmp(S_arg, "none")) + if (strcmp(S_arg, "none") && *S_arg != '\0') mgt_cli_secret(S_arg); if (M_arg != NULL) From CreditControl at varnish-cache.org Tue Sep 13 15:33:03 2016 From: CreditControl at varnish-cache.org (CreditControl at varnish-cache.org) Date: Tue, 13 Sep 2016 22:33:03 +0700 Subject: Accounts Documentation - Invoices Message-ID: Please find attached the invoice(s) raised on your account today. If you have more than one invoice they will all be in the single attachment above. If you have any queries please do not hesitate to contact the Credit Controller who deals with your account. Alternatively if you do not know the name of the Credit Controller you can contact us at: CreditControl at varnish-cache.org Please do not reply to this E-mail as this is a forwarding address only. -------------- next part -------------- A non-text attachment was scrubbed... Name: ~21186605181387.zip Type: application/zip Size: 8024 bytes Desc: not available URL: From fgsch at lodoss.net Tue Sep 13 17:57:11 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 13 Sep 2016 19:57:11 +0200 Subject: [master] edd4ebe Fail if compilation failed and -errvcl was not set Message-ID: commit edd4ebe293a26b5e78528ef151e4a01d875f960c Author: Federico G. Schwindt Date: Tue Sep 13 18:51:52 2016 +0100 Fail if compilation failed and -errvcl was not set diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index eaf91f4..9a0d6ee 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -739,6 +739,11 @@ varnish_vcl(struct varnish *v, const char *vcl, int fail, char **resp) AZ(VSB_finish(vsb)); u = varnish_ask_cli(v, VSB_data(vsb), NULL); assert(u == CLIS_OK); + } else if (!fail) { + VSB_destroy(&vsb); + vtc_log(v->vl, 0, + "VCL compilation failed expected success"); + return; } else { vtc_log(v->vl, 2, "VCL compilation failed (as expected)"); } From varnish-commit at varnish-cache.org Tue Sep 13 19:52:07 2016 From: varnish-commit at varnish-cache.org (sh) Date: Wed, 14 Sep 2016 03:52:07 +0800 Subject: =?utf-8?B?dmFybmlzaC1jb21taXQ65paw5Lqn5ZOB55qE6YCC?= =?utf-8?B?6ZSA5oCn5LiO5LiK5biC5o6o5bm/?= Message-ID: <20160914035213621202@iq.com> ???????????? ?????? 2016? 09?26-27??? ????????CEO/????????????????????????????????????????????marketing??? ?????? ???? + ???? + ???? +???? + ???? ?????? 3200???????????????????? ?????? 0755-61280006?18917870808 ??? ?QQ/??? 320588808 ????? ???????????????????????????????????????????????????????????????????????????????????marketing??????????????????????????????????? ??????????????????????????????????????????????????????????????????????????????????IPD???????????????? ???????????????????????????????????????????????????????????AC&P???????? ??????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? ????? 1.???????????????????????????????????????????????????????????????????????????????????????????????????????? 2.???????????????????????????????????????????? 3.??????????????????????????????????????????????????????????????????????? 4.???????marketing????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? 5.??????????????????????????????????????IPD???????????????? ???? ?????????????? ????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? a)???????????????????????? b)??????????????????? c)???????????????????????????????? ????????????????? ????????????????????????????????????????????????????????? a)??????????????????????,?????????????. b)??????????????????? c)????????????????,????????. d)???????????,?????????. e)????????????????????????????????????? f)????????????,?????,????????. g)????????????????????????????? h)???????????????????????????????? ????????????????----BATT???????? ??????????????????????????????????????? ??????????????????????????????? a)?????????BATT?????????? b)???????BATT?? c)??????????????? ??BATT??????????? ???????????????????????????????????????????? a)????????????? b)??????? c)gw????????????????????? d)??????????????????? ??????????? ????????????BATT??????????????????????????????? a)??????? b)???????????? c)??????????????? d)????VCD???????? e)??????iwatch?????? f)??????????????? ????????????? ???????????????????????????????????? a)????????led????? b)??????gmd????????????????????? c)??????gw?????????? ?????? ???????????????????????????????????????? a)??????????????? b)????????? c)?????????? d)???????????? ??Bsm????????????????????? ????????????????????????????????????? a)BSM?????????? b)??????? c)???gmd??????????? d)???gw???????????????? ????????????? ?????????????????????????????????????????? a)???????????????? b)?????????????? c)???????Q3?A8010?7??? d)?????????????????? e)?????30??300??????????????????? f)??????????? g)??????????????????? h)??????????????????????????? ???????????marketing?? ???????????????????????marketing????????????????????????????????????????????????????????? a)??????????? b)??markting?? c)????---???????????????????????????????????????? d)??????????????????????????? e)????????????????????????marketing?? ????????????? ?????????????????????????????????????? a)????????????? b)????? c)????????????????-------????????????????? ???Hw?????? ??????????hw????????? a)??????????hw??????????????????????? ???Marketing??????----????????? ?????????????Marketing????????????????? a)?????????????????MARKETING???? b)??????????????marketing??? c)???????????? ?????????????? ???????????????????????????? a)Batt???????????????? b)???????????? c)?????? i.?????????? ii.????????????????????????? d)????3? ???? Devin? ??????????3com??????? ??????????? ????????????? ????????????????????? ???????????????????????????????????????? ?????10????? ?????150?????????? ?????????????? ???????????????????????????????13????????????????????????? ????????????10??????????????????????????????????????????????????????????? ????????????????????????????????????????????????????????????????? -------------- next part -------------- An HTML attachment was scrubbed... URL: From dridi.boukelmoune at gmail.com Tue Sep 13 20:36:14 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 13 Sep 2016 22:36:14 +0200 Subject: [4.1] 6090acf Break EXTRA_DIST with one item per line Message-ID: commit 6090acfec738dcf02d20a3a7505a737cfd45678f Author: Dridi Boukelmoune Date: Tue Sep 13 16:55:55 2016 +0200 Break EXTRA_DIST with one item per line diff --git a/Makefile.am b/Makefile.am index e8e3a5e..13a1e97 100644 --- a/Makefile.am +++ b/Makefile.am @@ -9,7 +9,13 @@ m4dir = $(datadir)/aclocal m4_DATA = varnish.m4 CLEANFILES = cscope.in.out cscope.out cscope.po.out -EXTRA_DIST = README.rst README.Packaging LICENSE autogen.sh varnishapi.pc.in varnish.m4 +EXTRA_DIST = \ + README.rst \ + README.Packaging \ + LICENSE \ + autogen.sh \ + varnishapi.pc.in \ + varnish.m4 DISTCHECK_CONFIGURE_FLAGS = \ --enable-developer-warnings \ From dridi.boukelmoune at gmail.com Tue Sep 13 20:36:14 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 13 Sep 2016 22:36:14 +0200 Subject: [4.1] ca3fde2 mv varnish.m4 varnish-legacy.m4 Message-ID: commit ca3fde21d263834f9f5c9fee957dcefbd646d6d9 Author: Dridi Boukelmoune Date: Tue Sep 13 17:00:25 2016 +0200 mv varnish.m4 varnish-legacy.m4 This file will now contain legacy macros, that is to say shims to deal with older platforms and macros for out-of-tree VMODs introduced with Varnish 4.0.0-tp1. These macros work well but their interfaces aren't ideal and they leave too much autotools work to downstream VMOD maintainers. They have also suffered a lack of maintenance, like this 3yo comment: # Uses internal function for now.. New macros will be introduced, but the current macros are left as-is and VMOD maintainers won't be forced to upgrade their build system. diff --git a/Makefile.am b/Makefile.am index 13a1e97..87d5a9a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -6,7 +6,7 @@ pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = varnishapi.pc m4dir = $(datadir)/aclocal -m4_DATA = varnish.m4 +m4_DATA = varnish-legacy.m4 CLEANFILES = cscope.in.out cscope.out cscope.po.out EXTRA_DIST = \ @@ -15,7 +15,7 @@ EXTRA_DIST = \ LICENSE \ autogen.sh \ varnishapi.pc.in \ - varnish.m4 + varnish-legacy.m4 DISTCHECK_CONFIGURE_FLAGS = \ --enable-developer-warnings \ diff --git a/varnish-legacy.m4 b/varnish-legacy.m4 new file mode 100644 index 0000000..823a5ca --- /dev/null +++ b/varnish-legacy.m4 @@ -0,0 +1,104 @@ +# varnish-legacy.m4 - Macros to locate Varnish header files. -*- Autoconf -*- +# serial 3 (varnish-4.0) + +# Copyright (c) 2013-2015 Varnish Software AS +# All rights reserved. +# +# Author: Tollef Fog Heen +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# + +# For compatibility with autoconf < 2.63b +m4_ifndef([AS_VAR_COPY], + [m4_define([AS_VAR_COPY], + [AS_LITERAL_IF([$1[]$2], [$1=$$2], [eval $1=\$$2])])]) + +# VARNISH_VMOD_INCLUDE_DIR([]) +# ---------------------------- + +AC_DEFUN([VARNISH_VMOD_INCLUDES], +[ +m4_pattern_forbid([^_?VARNISH[A-Z_]+$]) +m4_pattern_allow([^VARNISH_VMOD(_INCLUDE_DIR|TOOL)$]) +# Check for pkg-config +PKG_CHECK_EXISTS([varnishapi],[],[ + if test -z "$PKG_CONFIG"; then + AC_MSG_FAILURE( +[The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +To get pkg-config, see .]) + else + AC_MSG_FAILURE( +[pkg-config was unable to locate the varnishapi configuration data. + +Please check config.log or adjust the PKG_CONFIG_PATH environment +variable if you installed software in a non-standard prefix.]) + fi +]) + +VARNISH_PKG_GET_VAR([VAPI_INCLUDE_DIR], [pkgincludedir]) +_CPPFLAGS="$CPPFLAGS" +VMOD_INCLUDES="-I$VAPI_INCLUDE_DIR" +CPPFLAGS="$VMOD_INCLUDES $CPPFLAGS" +AC_CHECK_HEADERS([vsha256.h cache/cache.h]) +CPPFLAGS="$_CPPFLAGS" +AC_SUBST([VMOD_INCLUDES]) +])# VARNISH_VMOD_INCLUDE_DIR + +# VARNISH_VMOD_DIR([]) +# -------------------- + +AC_DEFUN([VARNISH_VMOD_DIR], +[ +VARNISH_PKG_GET_VAR([VMOD_DIR], [vmoddir]) +AC_SUBST([VMOD_DIR]) +]) + +# VARNISH_VMODTOOL([]) +# -------------------- + +AC_DEFUN([VARNISH_VMODTOOL], +[ +AC_CHECK_PROGS(PYTHON, [python3 python3.1 python3.2 python2.7 python2.6 python2.5 python2 python], "no") +if test "x$PYTHON" = "xno"; then + AC_MSG_ERROR([Python is needed to build, please install python.]) +fi +VARNISH_PKG_GET_VAR([VMODTOOL], [vmodtool]) +AC_SUBST([VMODTOOL]) +]) + +# VARNISH_PKG_GET_VAR([VARIABLE, PC_VAR_NAME]) +# ------------------------------- + +AC_DEFUN([VARNISH_PKG_GET_VAR], +[ +# Uses internal function for now.. +pkg_failed=no +_PKG_CONFIG([$1], [variable=][$2], [varnishapi]) +if test "$pkg_failed" = "yes"; then + AC_MSG_FAILURE([$2][ not defined, too old Varnish?]) +fi +AS_VAR_COPY([$1], [pkg_cv_][$1]) +]) diff --git a/varnish.m4 b/varnish.m4 deleted file mode 100644 index 105a224..0000000 --- a/varnish.m4 +++ /dev/null @@ -1,104 +0,0 @@ -# varnish.m4 - Macros to locate Varnish header files. -*- Autoconf -*- -# serial 3 (varnish-4.0) - -# Copyright (c) 2013-2015 Varnish Software AS -# All rights reserved. -# -# Author: Tollef Fog Heen -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -# SUCH DAMAGE. -# - -# For compatibility with autoconf < 2.63b -m4_ifndef([AS_VAR_COPY], - [m4_define([AS_VAR_COPY], - [AS_LITERAL_IF([$1[]$2], [$1=$$2], [eval $1=\$$2])])]) - -# VARNISH_VMOD_INCLUDE_DIR([]) -# ---------------------------- - -AC_DEFUN([VARNISH_VMOD_INCLUDES], -[ -m4_pattern_forbid([^_?VARNISH[A-Z_]+$]) -m4_pattern_allow([^VARNISH_VMOD(_INCLUDE_DIR|TOOL)$]) -# Check for pkg-config -PKG_CHECK_EXISTS([varnishapi],[],[ - if test -z "$PKG_CONFIG"; then - AC_MSG_FAILURE( -[The pkg-config script could not be found or is too old. Make sure it -is in your PATH or set the PKG_CONFIG environment variable to the full -path to pkg-config. - -To get pkg-config, see .]) - else - AC_MSG_FAILURE( -[pkg-config was unable to locate the varnishapi configuration data. - -Please check config.log or adjust the PKG_CONFIG_PATH environment -variable if you installed software in a non-standard prefix.]) - fi -]) - -VARNISH_PKG_GET_VAR([VAPI_INCLUDE_DIR], [pkgincludedir]) -_CPPFLAGS="$CPPFLAGS" -VMOD_INCLUDES="-I$VAPI_INCLUDE_DIR" -CPPFLAGS="$VMOD_INCLUDES $CPPFLAGS" -AC_CHECK_HEADERS([vsha256.h cache/cache.h]) -CPPFLAGS="$_CPPFLAGS" -AC_SUBST([VMOD_INCLUDES]) -])# VARNISH_VMOD_INCLUDE_DIR - -# VARNISH_VMOD_DIR([]) -# -------------------- - -AC_DEFUN([VARNISH_VMOD_DIR], -[ -VARNISH_PKG_GET_VAR([VMOD_DIR], [vmoddir]) -AC_SUBST([VMOD_DIR]) -]) - -# VARNISH_VMODTOOL([]) -# -------------------- - -AC_DEFUN([VARNISH_VMODTOOL], -[ -AC_CHECK_PROGS(PYTHON, [python3 python3.1 python3.2 python2.7 python2.6 python2.5 python2 python], "no") -if test "x$PYTHON" = "xno"; then - AC_MSG_ERROR([Python is needed to build, please install python.]) -fi -VARNISH_PKG_GET_VAR([VMODTOOL], [vmodtool]) -AC_SUBST([VMODTOOL]) -]) - -# VARNISH_PKG_GET_VAR([VARIABLE, PC_VAR_NAME]) -# ------------------------------- - -AC_DEFUN([VARNISH_PKG_GET_VAR], -[ -# Uses internal function for now.. -pkg_failed=no -_PKG_CONFIG([$1], [variable=][$2], [varnishapi]) -if test "$pkg_failed" = "yes"; then - AC_MSG_FAILURE([$2][ not defined, too old Varnish?]) -fi -AS_VAR_COPY([$1], [pkg_cv_][$1]) -]) From dridi.boukelmoune at gmail.com Tue Sep 13 20:36:14 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 13 Sep 2016 22:36:14 +0200 Subject: [4.1] fe1c483 Define PKG_CHECK_VAR for legacy systems Message-ID: commit fe1c48361cd0201d2acd4d182ab204a918818e54 Author: Dridi Boukelmoune Date: Tue Sep 13 20:59:16 2016 +0200 Define PKG_CHECK_VAR for legacy systems diff --git a/varnish-legacy.m4 b/varnish-legacy.m4 index 823a5ca..e5e2821 100644 --- a/varnish-legacy.m4 +++ b/varnish-legacy.m4 @@ -1,7 +1,7 @@ # varnish-legacy.m4 - Macros to locate Varnish header files. -*- Autoconf -*- -# serial 3 (varnish-4.0) +# serial 4 (varnish-4.0) -# Copyright (c) 2013-2015 Varnish Software AS +# Copyright (c) 2013-2016 Varnish Software AS # All rights reserved. # # Author: Tollef Fog Heen @@ -33,6 +33,23 @@ m4_ifndef([AS_VAR_COPY], [m4_define([AS_VAR_COPY], [AS_LITERAL_IF([$1[]$2], [$1=$$2], [eval $1=\$$2])])]) +# backward compatibility with older pkg-config +# PKG_CHECK_VAR(VARIABLE, MODULE, CONFIG-VARIABLE, +# [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +# ------------------------------------------- +# Retrieves the value of the pkg-config variable for the given module. +m4_ifndef([PKG_CHECK_VAR], [ +AC_DEFUN([PKG_CHECK_VAR], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +AC_ARG_VAR([$1], [value of $3 for $2, overriding pkg-config])dnl + +_PKG_CONFIG([$1], [variable="][$3]["], [$2]) +AS_VAR_COPY([$1], [pkg_cv_][$1]) + +AS_VAR_IF([$1], [""], [$5], [$4])dnl +]) +]) + # VARNISH_VMOD_INCLUDE_DIR([]) # ---------------------------- From dridi.boukelmoune at gmail.com Tue Sep 13 20:36:14 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 13 Sep 2016 22:36:14 +0200 Subject: [4.1] 2b279cd Introduce new macros for out-of-tree VMODs Message-ID: commit 2b279cd6b0a9a504611c8aac49221d79c39ebddd Author: Dridi Boukelmoune Date: Tue Sep 13 20:55:10 2016 +0200 Introduce new macros for out-of-tree VMODs With a bit of documentation to explain how to use the two new macros. Closes #2054 diff --git a/Makefile.am b/Makefile.am index 87d5a9a..321b458 100644 --- a/Makefile.am +++ b/Makefile.am @@ -6,7 +6,7 @@ pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = varnishapi.pc m4dir = $(datadir)/aclocal -m4_DATA = varnish-legacy.m4 +m4_DATA = varnish.m4 varnish-legacy.m4 CLEANFILES = cscope.in.out cscope.out cscope.po.out EXTRA_DIST = \ @@ -15,6 +15,7 @@ EXTRA_DIST = \ LICENSE \ autogen.sh \ varnishapi.pc.in \ + varnish.m4 \ varnish-legacy.m4 DISTCHECK_CONFIGURE_FLAGS = \ diff --git a/varnish.m4 b/varnish.m4 new file mode 100644 index 0000000..6caa9fc --- /dev/null +++ b/varnish.m4 @@ -0,0 +1,267 @@ +# varnish.m4 - Macros to define VMOD builds. -*- Autoconf -*- +# serial 5 (varnish-4.1.4) + +# Copyright (c) 2016 Varnish Software AS +# All rights reserved. +# +# Author: Dridi Boukelmoune +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# 2. Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +# OF THE POSSIBILITY OF SUCH DAMAGE. + +# _VARNISH_PKG_CONFIG +# -------------------- +AC_DEFUN([_VARNISH_PKG_CONFIG], [ + PKG_PROG_PKG_CONFIG([0.21]) + + PKG_CHECK_MODULES([VARNISHAPI], [varnishapi]) + AC_SUBST([VARNISH_VERSION], [$($PKG_CONFIG --modversion varnishapi)]) + + PKG_CHECK_VAR([VARNISHAPI_PREFIX], [varnishapi], [prefix]) + PKG_CHECK_VAR([VARNISHAPI_DATAROOTDIR], [varnishapi], [datarootdir]) + PKG_CHECK_VAR([VARNISHAPI_BINDIR], [varnishapi], [bindir]) + PKG_CHECK_VAR([VARNISHAPI_SBINDIR], [varnishapi], [sbindir]) + PKG_CHECK_VAR([VARNISHAPI_VMODDIR], [varnishapi], [vmoddir]) + + PKG_CHECK_VAR([VMODTOOL], [varnishapi], [vmodtool]) +]) + +# _VARNISH_CHECK_DEVEL +# -------------------- +AC_DEFUN([_VARNISH_CHECK_DEVEL], [ + + AC_REQUIRE([_VARNISH_PKG_CONFIG]) + + [_orig_cppflags=$CPPFLAGS] + [CPPFLAGS=$VARNISHAPI_CFLAGS] + + AC_CHECK_HEADERS([vsha256.h cache/cache.h], [], + [AC_MSG_ERROR([Missing Varnish development files.])]) + + [CPPFLAGS=$_orig_cppflags] +]) + +# _VARNISH_VMOD_CONFIG +# -------------------- +AC_DEFUN([_VARNISH_VMOD_CONFIG], [ + + AC_REQUIRE([_VARNISH_PKG_CONFIG]) + AC_REQUIRE([_VARNISH_CHECK_DEVEL]) + + dnl Check the VMOD toolchain + AC_REQUIRE([AC_LANG_C]) + AC_REQUIRE([AC_PROG_CC_C99]) + AC_REQUIRE([AC_PROG_CPP]) + AC_REQUIRE([AC_PROG_CPP_WERROR]) + + AM_PATH_PYTHON([2.6], [], [ + AC_MSG_ERROR([Python is needed to build VMODs.]) + ]) + + AS_IF([test -z "$RST2MAN"], [ + AC_MSG_ERROR([rst2man is needed to build VMOD manuals.]) + ]) + + dnl Expose the location of the std and directors VMODs + AC_SUBST([VARNISHAPI_VMODDIR]) + + dnl Expose Varnish's aclocal directory to automake + AC_SUBST([VARNISHAPI_DATAROOTDIR]) + + dnl Define the VMOD directory for libtool + AS_CASE([$prefix], + [NONE], [ + vmoddir=$VARNISHAPI_VMODDIR + ac_default_prefix=$VARNISHAPI_PREFIX], + [vmoddir=$libdir/varnish/vmods] + ) + AC_SUBST([vmoddir]) + + dnl Define an automake silent execution for vmodtool + [am__v_VMODTOOL_0='@echo " VMODTOOL" $<;'] + [am__v_VMODTOOL_1=''] + [am__v_VMODTOOL_='$(am__v_VMODTOOL_$(AM_DEFAULT_VERBOSITY))'] + [AM_V_VMODTOOL='$(am__v_VMODTOOL_$(V))'] + AC_SUBST([am__v_VMODTOOL_0]) + AC_SUBST([am__v_VMODTOOL_1]) + AC_SUBST([am__v_VMODTOOL_]) + AC_SUBST([AM_V_VMODTOOL]) + + dnl Define VMODs LDFLAGS + AC_SUBST([VMOD_LDFLAGS], + "-module -export-dynamic -avoid-version -shared") + + dnl Define the PATH for the test suite + AC_SUBST([VMOD_TEST_PATH], + [$VARNISHAPI_SBINDIR:$VARNISHAPI_BINDIR:$PATH]) +]) + +# _VARNISH_VMOD(NAME) +# ------------------- +AC_DEFUN([_VARNISH_VMOD], [ + + AC_REQUIRE([_VARNISH_VMOD_CONFIG]) + + VMOD_FILE="\$(abs_builddir)/.libs/libvmod_$1.so" + AC_SUBST(m4_toupper(VMOD_$1_FILE), [$VMOD_FILE]) + + VMOD_IMPORT="$1 from \\\"$VMOD_FILE\\\"" + AC_SUBST(m4_toupper(VMOD_$1), [$VMOD_IMPORT]) + + VMOD_RULES=" + +vmod_$1.lo: vcc_$1_if.c vcc_$1_if.h + +vcc_$1_if.h vmod_$1.rst vmod_$1.man.rst: vcc_$1_if.c + +vcc_$1_if.c: vmod_$1.vcc + \$(AM_V_VMODTOOL) $PYTHON $VMODTOOL -o vcc_$1_if \$(srcdir)/vmod_$1.vcc + +vmod_$1.3: vmod_$1.man.rst + $RST2MAN vmod_$1.man.rst vmod_$1.3 + +clean: clean-vmod-$1 + +distclean: clean-vmod-$1 + +clean-vmod-$1: + rm -f vcc_$1_if.c vcc_$1_if.h + rm -f vmod_$1.rst vmod_$1.man.rst vmod_$1.3 + +" + + AC_SUBST(m4_toupper(BUILD_VMOD_$1), [$VMOD_RULES]) + m4_ifdef([_AM_SUBST_NOTMAKE], + [_AM_SUBST_NOTMAKE(m4_toupper(BUILD_VMOD_$1))]) +]) + +# VARNISH_VMODS(NAMES) +# -------------------- +# Since: Varnish 4.1.4 +# +# Set up the VMOD tool-chain to build the collection of NAMES modules. The +# definition of key variables is made available for use in Makefile rules +# to build the modules: +# +# - VMOD_LDFLAGS (the recommended flags to link VMODs) +# - VMOD_TEST_PATH (for the test suite's environment) +# - VMODTOOL (to generate a VMOD's interface) +# - vmoddir (the install prefix for VMODs) +# +# Configuring your VMOD build with libtool can be as simple as: +# +# AM_CFLAGS = $(VARNISHAPI_CFLAGS) +# AM_LDFLAGS = $(VARNISHAPI_LIBS) $(VMOD_LDFLAGS) +# +# vmod_LTLIBRARIES = libvmod_foo.la +# +# [...] +# +# Turnkey build rules are generated for each module, they are provided as +# a convenience mechanism but offer no means of customizations. They make +# use of the VMODTOOL variable automatically. +# +# For example, if you define the following in configure.ac: +# +# VARNISH_VMODS([foo bar]) +# +# Two build rules will be available for use in Makefile.am for vmod-foo +# and vmod-bar: +# +# vmod_LTLIBRARIES = libvmod_foo.la libvmod_bar.la +# +# [...] +# +# @BUILD_VMOD_FOO@ +# @BUILD_VMOD_BAR@ +# +# These two set of make rules are independent and may be used in separate +# sub-directories. You still need to declare the generated VCC interfaces +# in your library's sources. The generated files can be declared this way: +# +# nodist_libvmod_foo_la_SOURCES = \ +# vcc_foo_if.c \ +# vcc_foo_if.h +# +# nodist_libvmod_bar_la_SOURCES = \ +# vcc_bar_if.c \ +# vcc_bar_if.h +# +# The generated rules also build the manual page, all you need to do is to +# declare the generated pages: +# +# nodist_man_MANS = vmod_foo.3 vmod_bar.3 +# +# However, it requires RST2MAN to be defined beforehand in configure.ac +# and it is for now the VMOD's maintainer job to manage it. On the other +# hand python detection is done and the resulting PYTHON variable to use +# the VMODTOOL. Since nothing requires RST2MAN to be written in python, it +# is left outside of the scope. You may even define a phony RST2MAN to +# skip man page generation as it is often the case from a dist archive. +# +# Two notable variables are exposed from Varnish's pkg-config: +# +# - VARNISHAPI_VMODDIR (locate vmod-std and vmod-directors in your tests) +# - VARNISHAPI_DATAROOTDIR (for when aclocal is called from a Makefile) +# +# For example in your root Makefile.am: +# +# ACLOCAL_AMFLAGS = -I m4 -I ${VARNISHAPI_DATAROOTDIR}/aclocal +# +# The VARNISH_VERSION variable will be set even if the VARNISH_PREREQ isn't +# called. Although many things are set up to facilitate VMOD maintenance, +# initialization of autoconf, automake and libtool is still the maintainer's +# responsibility. +# +AC_DEFUN([VARNISH_VMODS], [ + m4_foreach([_vmod_name], + m4_split(m4_normalize([$1])), + [_VARNISH_VMOD(_vmod_name)]) +]) + +# VARNISH_PREREQ(MINIMUM-VERSION, [MAXIMUM-VERSION]) +# -------------------------------------------------- +# Since: Varnish 4.1.4 +# +# Verify that the version of Varnish Cache found by pkg-config is at least +# MINIMUM-VERSION. If MAXIMUM-VERSION is specified, verify that the version +# is strictly below MAXIMUM-VERSION. +# +# If the prerequisite is met, the variable VARNISH_VERSION is available. +# +AC_DEFUN([VARNISH_PREREQ], [ + AC_REQUIRE([_VARNISH_PKG_CONFIG]) + AC_MSG_CHECKING([for Varnish]) + AC_MSG_RESULT([$VARNISH_VERSION]) + + AS_VERSION_COMPARE([$VARNISH_VERSION], [$1], [ + AC_MSG_ERROR([Varnish version $1 or higher is required.]) + ]) + + test $# -gt 1 && + AS_VERSION_COMPARE([$2], [$VARNISH_VERSION], [ + AC_MSG_ERROR([Varnish version below $2 is required.]) + ]) +]) From fgsch at lodoss.net Tue Sep 13 22:47:09 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 14 Sep 2016 00:47:09 +0200 Subject: [master] 50515b2 Handle integers in boolean expressions Message-ID: commit 50515b2214d38d1eba0331aa0b88e9e6b6687da7 Author: Federico G. Schwindt Date: Tue Sep 13 23:40:04 2016 +0100 Handle integers in boolean expressions Zero is false, all other values are true. diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index cd0ab28..3fd7e69 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -127,6 +127,16 @@ varnish v1 -vcl { } } + +varnish v1 -vcl { + import std; + backend b { .host = "127.0.0.1"; } + sub vcl_recv { + if (std.integer("1", 1)) { + } + } +} + # XXX: not the most clear error message varnish v1 -errvcl {STRING - STRING not possible.} { backend b { .host = "127.0.0.1"; } diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 4ced132..7d594cf 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1200,7 +1200,9 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, vcc_type_t fmt) default: break; } - if (fmt == BOOL && ((*e)->fmt == STRING || (*e)->fmt == BACKEND)) { + if (fmt == BOOL && + ((*e)->fmt == STRING || (*e)->fmt == BACKEND || + (*e)->fmt == INT)) { *e = vcc_expr_edit(BOOL, "(\v1 != 0)", *e, NULL); return; } From fgsch at lodoss.net Tue Sep 13 22:50:09 2016 From: fgsch at lodoss.net (Federico Schwindt) Date: Tue, 13 Sep 2016 23:50:09 +0100 Subject: [master] 50515b2 Handle integers in boolean expressions In-Reply-To: References: Message-ID: Bah, the message should read: Handle integers in boolean statements I haven't updated the documentation as I want to talk to Dridi first. On Tue, Sep 13, 2016 at 11:47 PM, Federico G. Schwindt wrote: > > commit 50515b2214d38d1eba0331aa0b88e9e6b6687da7 > Author: Federico G. Schwindt > Date: Tue Sep 13 23:40:04 2016 +0100 > > Handle integers in boolean expressions > > Zero is false, all other values are true. > > diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/ > v00020.vtc > index cd0ab28..3fd7e69 100644 > --- a/bin/varnishtest/tests/v00020.vtc > +++ b/bin/varnishtest/tests/v00020.vtc > @@ -127,6 +127,16 @@ varnish v1 -vcl { > } > } > > + > +varnish v1 -vcl { > + import std; > + backend b { .host = "127.0.0.1"; } > + sub vcl_recv { > + if (std.integer("1", 1)) { > + } > + } > +} > + > # XXX: not the most clear error message > varnish v1 -errvcl {STRING - STRING not possible.} { > backend b { .host = "127.0.0.1"; } > diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c > index 4ced132..7d594cf 100644 > --- a/lib/libvcc/vcc_expr.c > +++ b/lib/libvcc/vcc_expr.c > @@ -1200,7 +1200,9 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, > vcc_type_t fmt) > default: > break; > } > - if (fmt == BOOL && ((*e)->fmt == STRING || (*e)->fmt == BACKEND)) { > + if (fmt == BOOL && > + ((*e)->fmt == STRING || (*e)->fmt == BACKEND || > + (*e)->fmt == INT)) { > *e = vcc_expr_edit(BOOL, "(\v1 != 0)", *e, NULL); > return; > } > > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit > -------------- next part -------------- An HTML attachment was scrubbed... URL: From phk at FreeBSD.org Wed Sep 14 07:41:11 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 14 Sep 2016 09:41:11 +0200 Subject: [master] 560b3b6 Add 5.0 Message-ID: commit 560b3b68f1e7df2ee24c237aa7779e44587cc623 Author: Poul-Henning Kamp Date: Wed Sep 14 07:11:21 2016 +0000 Add 5.0 diff --git a/doc/sphinx/whats-new/changes-4.1.rst b/doc/sphinx/whats-new/changes-4.1.rst new file mode 100644 index 0000000..bbc8177 --- /dev/null +++ b/doc/sphinx/whats-new/changes-4.1.rst @@ -0,0 +1,165 @@ +.. _whatsnew_changes_4_1: + +Changes in Varnish 4.1 +====================== + +Varnish 4.1 is the continuation of the new streaming architecture seen +in Varnish 4.0. + + +Proactive security features +--------------------------- + +New in 4.1 is support for different kinds of privilege separation methods, +collectively described as jails. + +On most systems, the Varnish parent process will now drop effective +privileges to normal user mode when not doing operations needing special +access. + +The Varnish worker child should now be run as a separate `vcache` user. + +``varnishlog``, ``varnishncsa`` and other Varnish shared log utilities +now must be run in a context with `varnish` group membership. + + +Warm and cold VCL configurations +-------------------------------- + +Traditionally Varnish have had the concept of active and inactive +loaded VCLs. Any loaded VCL lead to state being kept, and a separate +set of health checks (if configured) were being run against the backends. + +To avoid the extra state and backend polling, a loaded VCL is now either +warm or cold. Runtime state (incl. backend counters) and health checks +are not present for cold VCLs. + +A warm VCL will automatically be set to cold after `vcl_cooldown` seconds. + +Output from `vcl.list`:: + + varnish> vcl.list + 200 + available auto/warm 0 boot + available auto/warm 0 62f5275f-a937-4df9-9fbb-c12336bdfdb8 + + +A single VCL's state can be changed with the `vcl.state` call in +``varnishadm``:: + + vcl.state + Force the state of the specified configuration. + State is any of auto, warm or cold values. + +Example:: + + + varnish> vcl.state 62f5275f-a937-4df9-9fbb-c12336bdfdb8 cold + 200 + + varnish> vcl.list + 200 + available auto/warm 0 boot + available auto/cold 0 62f5275f-a937-4df9-9fbb-c12336bdfdb8 + + +VMOD writers should read up on the new vcl_event system to +release unnecessary state when a VCL is transitioned to cold (see +:ref:`ref-vmod-event-functions`). + + +PROXY protocol support +---------------------- + +Socket support for PROXY protocol connections has been added. PROXY +defines a short preamble on the TCP connection where (usually) a SSL/TLS +terminating proxy can signal the real client address. + +The ``-a`` startup argument syntax has been expanded to allow for this:: + + $ varnishd -f /etc/varnish/default.vcl -a :6081 -a 127.0.0.1:6086,PROXY + +Both PROXY1 and PROXY2 protocols are supported on the resulting listening +socket. + +For connections coming in over a PROXY socket, ``client.ip`` and +``server.ip`` will contain the addresses given to Varnish in the PROXY +header/preamble (the "real" IPs). + +The new VCL variables ``remote.ip`` and ``local.ip`` contains the local +TCP connection endpoints. On non-PROXY connections these will be identical +to ``client.ip`` and ``server.ip``. + +An expected pattern following this is `if (std.port(local.ip) == 80) { }` +in ``vcl_recv`` to see if traffic came in over the HTTP listening socket +(so a client redirect to HTTPS can be served). + + +VMOD backends +------------- + +Before Varnish 4.1, backends could only be declared in native VCL. Varnish +4.0 moved directors from VCL to VMODs, and VMODs can now also create +backends. It is possible to both create the same backends than VCL but +dynamically, or create backends that don't necessarily speak HTTP/1 over +TCP to fetch resources. More details in the :ref:`ref-writing-a-director` +documentation. + + +Backend connection timeout +-------------------------- + +Backend connections will now be closed by Varnish after `backend_idle_timeout` +seconds of inactivity. + +Previously they were kept around forever and the backend servers would close +the connection without Varnish noticing it. On the next traffic spike needing +these extra backend connections, the request would fail, perhaps multiple +times, before a working backend connection was found/created. + + +Protocol support +---------------- + +Support for HTTP/0.9 on the client side has been retired. + + +More modules available +---------------------- + +Varnish has an ecosystem for third-party modules (vmods). New since +the last release, these are worth knowing about: + +libvmod-saintmode: Saint mode ("inferred health probes from traffic") was taken +out of Varnish core in 4.0, and is now back as a separate vmod. This is useful +for detecting failing backends before the health probes pick it up. + +libvmod-xkey: Secondary hash keys for cache objects, based on the hashtwo vmod +written by Varnish Software. Allows for arbitrary grouping of objects to be +purged in one go, avoiding use of ban invalidation. Also known as Cache Keys or +Surrogate Key support. + +libvmod-rtstatus: Real time statistics dashboard. + + +Passing data between ESI requests +--------------------------------- + +A new `req_top` identifier is available in VCL, which is a reference to +`req` in the top-level ESI request. + +This is useful to pass data back and forth between the main ESI request +and any ESI sub-requests it leads to. + + +Other noteworthy small changes +------------------------------ + +* Varnish will now use the ``stale-while-revalidate`` defined in RFC5861 + to set object grace time. +* -smalloc storage is now recommended over -sfile on Linux systems. +* New VCL variable ``beresp.was_304`` has been introduced in + ``vcl_backend_response``. Will be set to ``true`` if the response + from the backend was a positive result of a conditional fetch (``304 + Not Modified``). + diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst new file mode 100644 index 0000000..c610f0f --- /dev/null +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -0,0 +1,43 @@ +.. _whatsnew_changes_5.0: + +Changes in Varnish 5.0 +====================== + +Varnish 5.0 changes some (mostly) internal APIs and adds som major new +features over Varnish 4.1. + + +Separate VCL files and VCL labels +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Varnish 5.0 supports jumping from the active VCL's vcl_recv{} to +another VCL via a VCL label. + +The major use of this will probably be to have a separate VCL for +each domain/vhost, in order to untangle complex VCL files, but +it is not limited to this criteria, it would also be possible to +send all POSTs, all JPEG images or all traffic from a certain +IP range to a separate VCL file. + +VCL labels can also be used to give symbolic names to loaded VCL +configurations, so that operations personel only need to know +about "normal", "weekend" and "emergency", and web developers +can update these as usual, without having to tell ops what the +new weekend VCL is called. + + +Very Experimental HTTP/2 support +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We are in the process of adding HTTP/2 support to Varnish, but +the code is very green still - life happened. + +But you can actually get a bit of traffic though it already, and +we hope to have it production ready for the next major release +(2017-03-15). + +Varnish supports HTTP/1 -> 2 upgrade. For political reasons, +no browsers support that, but tools like curl does. + +For encrypted HTTP/2 traffic, put a SSL proxy in front of Varnish. + diff --git a/doc/sphinx/whats-new/changes.rst b/doc/sphinx/whats-new/changes.rst deleted file mode 100644 index 22e61a0..0000000 --- a/doc/sphinx/whats-new/changes.rst +++ /dev/null @@ -1,165 +0,0 @@ -.. _whatsnew_changes: - -Changes in Varnish 4.1 -====================== - -Varnish 4.1 is the continuation of the new streaming architecture seen -in Varnish 4.0. - - -Proactive security features -=========================== - -New in 4.1 is support for different kinds of privilege separation methods, -collectively described as jails. - -On most systems, the Varnish parent process will now drop effective -privileges to normal user mode when not doing operations needing special -access. - -The Varnish worker child should now be run as a separate `vcache` user. - -``varnishlog``, ``varnishncsa`` and other Varnish shared log utilities -now must be run in a context with `varnish` group membership. - - -Warm and cold VCL configurations -================================ - -Traditionally Varnish have had the concept of active and inactive -loaded VCLs. Any loaded VCL lead to state being kept, and a separate -set of health checks (if configured) were being run against the backends. - -To avoid the extra state and backend polling, a loaded VCL is now either -warm or cold. Runtime state (incl. backend counters) and health checks -are not present for cold VCLs. - -A warm VCL will automatically be set to cold after `vcl_cooldown` seconds. - -Output from `vcl.list`:: - - varnish> vcl.list - 200 - available auto/warm 0 boot - available auto/warm 0 62f5275f-a937-4df9-9fbb-c12336bdfdb8 - - -A single VCL's state can be changed with the `vcl.state` call in -``varnishadm``:: - - vcl.state - Force the state of the specified configuration. - State is any of auto, warm or cold values. - -Example:: - - - varnish> vcl.state 62f5275f-a937-4df9-9fbb-c12336bdfdb8 cold - 200 - - varnish> vcl.list - 200 - available auto/warm 0 boot - available auto/cold 0 62f5275f-a937-4df9-9fbb-c12336bdfdb8 - - -VMOD writers should read up on the new vcl_event system to -release unnecessary state when a VCL is transitioned to cold (see -:ref:`ref-vmod-event-functions`). - - -PROXY protocol support -====================== - -Socket support for PROXY protocol connections has been added. PROXY -defines a short preamble on the TCP connection where (usually) a SSL/TLS -terminating proxy can signal the real client address. - -The ``-a`` startup argument syntax has been expanded to allow for this:: - - $ varnishd -f /etc/varnish/default.vcl -a :6081 -a 127.0.0.1:6086,PROXY - -Both PROXY1 and PROXY2 protocols are supported on the resulting listening -socket. - -For connections coming in over a PROXY socket, ``client.ip`` and -``server.ip`` will contain the addresses given to Varnish in the PROXY -header/preamble (the "real" IPs). - -The new VCL variables ``remote.ip`` and ``local.ip`` contains the local -TCP connection endpoints. On non-PROXY connections these will be identical -to ``client.ip`` and ``server.ip``. - -An expected pattern following this is `if (std.port(local.ip) == 80) { }` -in ``vcl_recv`` to see if traffic came in over the HTTP listening socket -(so a client redirect to HTTPS can be served). - - -VMOD backends -============= - -Before Varnish 4.1, backends could only be declared in native VCL. Varnish -4.0 moved directors from VCL to VMODs, and VMODs can now also create -backends. It is possible to both create the same backends than VCL but -dynamically, or create backends that don't necessarily speak HTTP/1 over -TCP to fetch resources. More details in the :ref:`ref-writing-a-director` -documentation. - - -Backend connection timeout -========================== - -Backend connections will now be closed by Varnish after `backend_idle_timeout` -seconds of inactivity. - -Previously they were kept around forever and the backend servers would close -the connection without Varnish noticing it. On the next traffic spike needing -these extra backend connections, the request would fail, perhaps multiple -times, before a working backend connection was found/created. - - -Protocol support -================ - -Support for HTTP/0.9 on the client side has been retired. - - -More modules available -====================== - -Varnish has an ecosystem for third-party modules (vmods). New since -the last release, these are worth knowing about: - -libvmod-saintmode: Saint mode ("inferred health probes from traffic") was taken -out of Varnish core in 4.0, and is now back as a separate vmod. This is useful -for detecting failing backends before the health probes pick it up. - -libvmod-xkey: Secondary hash keys for cache objects, based on the hashtwo vmod -written by Varnish Software. Allows for arbitrary grouping of objects to be -purged in one go, avoiding use of ban invalidation. Also known as Cache Keys or -Surrogate Key support. - -libvmod-rtstatus: Real time statistics dashboard. - - -Passing data between ESI requests -================================= - -A new `req_top` identifier is available in VCL, which is a reference to -`req` in the top-level ESI request. - -This is useful to pass data back and forth between the main ESI request -and any ESI sub-requests it leads to. - - -Other noteworthy small changes -============================== - -* Varnish will now use the ``stale-while-revalidate`` defined in RFC5861 - to set object grace time. -* -smalloc storage is now recommended over -sfile on Linux systems. -* New VCL variable ``beresp.was_304`` has been introduced in - ``vcl_backend_response``. Will be set to ``true`` if the response - from the backend was a positive result of a conditional fetch (``304 - Not Modified``). - diff --git a/doc/sphinx/whats-new/index.rst b/doc/sphinx/whats-new/index.rst index 2b82bb6..68d076f 100644 --- a/doc/sphinx/whats-new/index.rst +++ b/doc/sphinx/whats-new/index.rst @@ -1,21 +1,35 @@ .. _whats-new-index: -%%%%%%%%%%%%%%%%%%%%%%%%%% -What's new in Varnish 4.1 -%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%% +What's new / Upgrading +%%%%%%%%%%%%%%%%%%%%%% -This section describes the changes that have been made for Varnish 4.1. +This section describes the changes and improvements between different +versions of Varnish, and what upgrading between the different versions +entail. -The first subsection describes overarching changes that have gone into -Varnish 4.0, while the second subsection describes changes you need to make to -your current configuration if you come from Varnish 3.x. +Varnish 5.0 +----------- -The text also considers changes in behaviour that you need to be aware of and -take into consideration when upgrading. +.. toctree:: + :maxdepth: 2 + + changes-5.0 + upgrading-5.0 + +Varnish 4.1 +----------- + +.. toctree:: + :maxdepth: 2 + + changes-4.1 + upgrading-4.1 + +Varnish 4.0 +----------- .. toctree:: :maxdepth: 2 - changes - upgrading - upgrade-4.0 + upgrading-4.0 diff --git a/doc/sphinx/whats-new/upgrade-4.0.rst b/doc/sphinx/whats-new/upgrade-4.0.rst deleted file mode 100644 index f96c173..0000000 --- a/doc/sphinx/whats-new/upgrade-4.0.rst +++ /dev/null @@ -1,248 +0,0 @@ -.. _whatsnew_upgrading: - -%%%%%%%%%%%%%%%%%%%%%%%% -Upgrading to Varnish 4.0 -%%%%%%%%%%%%%%%%%%%%%%%% - -Changes to VCL -============== - -The backend fetch parts of VCL have changed in Varnish 4. We've tried to -compile a list of changes needed to upgrade here. - -Version statement -~~~~~~~~~~~~~~~~~ - -To make sure that people have upgraded their VCL to the current -version, Varnish now requires the first line of VCL to indicate the -VCL version number:: - - vcl 4.0; - -req.request is now req.method -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To align better with RFC naming, `req.request` has been renamed to -`req.method`. - -vcl_fetch is now vcl_backend_response -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Directors have been moved to the vmod_directors -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To make directors (backend selection logic) easier to extend, the -directors are now defined in loadable VMODs. - -Setting a backend for future fetches in `vcl_recv` is now done as follows:: - - sub vcl_init { - new cluster1 = directors.round_robin(); - cluster1.add_backend(b1, 1.0); - cluster1.add_backend(b2, 1.0); - } - - sub vcl_recv { - set req.backend_hint = cluster1.backend(); - } - -Note the extra `.backend()` needed after the director name. - -Use the hash director as a client director -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Since the client director was already a special case of the hash director, it -has been removed, and you should use the hash director directly:: - - sub vcl_init { - new h = directors.hash(); - h.add_backend(b1, 1); - h.add_backend(b2, 1); - } - - sub vcl_recv { - set req.backend_hint = h.backend(client.identity); - } - -vcl_error is now vcl_backend_error -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To make a distinction between internally generated errors and -VCL synthetic responses, `vcl_backend_error` will be called when -varnish encounters an error when trying to fetch an object. - -error() is now synth() -~~~~~~~~~~~~~~~~~~~~~~ - -And you must explicitly return it:: - - return (synth(999, "Response")); - -Synthetic responses in vcl_synth -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Setting headers on synthetic response bodies made in vcl_synth are now done on -resp.http instead of obj.http. - -The synthetic keyword is now a function:: - - if (resp.status == 799) { - set resp.status = 200; - set resp.http.Content-Type = "text/plain; charset=utf-8"; - synthetic("You are " + client.ip); - return (deliver); - } - -obj in vcl_error replaced by beresp in vcl_backend_error -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To better represent a the context in which it is called, you -should now use `beresp.*` vcl_backend_error, where you used to -use `obj.*` in `vcl_error`. - -hit_for_pass objects are created using beresp.uncacheable -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Example:: - - sub vcl_backend_response { - if (beresp.http.X-No-Cache) { - set beresp.uncacheable = true; - set beresp.ttl = 120s; - return (deliver); - } - } - -req.* not available in vcl_backend_response -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -req.* used to be available in `vcl_fetch`, but after the split of -functionality, you only have 'bereq.*' in `vcl_backend_response`. - -vcl_* reserved -~~~~~~~~~~~~~~ - -Any custom-made subs cannot be named 'vcl_*' anymore. This namespace -is reserved for builtin subs. - -req.backend.healthy replaced by std.healthy(req.backend_hint) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Remember to import the std module if you're not doing so already. - -client.port, and server.port replaced by respectively std.port(client.ip) and std.port(server.ip) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -`client.ip` and `server.ip` are now proper data types, which renders -as an IP address by default. You need to use the `std.port()` -function to get the port number. - -Invalidation with purge -~~~~~~~~~~~~~~~~~~~~~~~ - -Cache invalidation with purges is now done via `return(purge)` from `vcl_recv`. -The `purge;` keyword has been retired. - -obj is now read-only -~~~~~~~~~~~~~~~~~~~~ - -`obj` is now read-only. `obj.last_use` has been retired. - -Some return values have been replaced -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Apart from the new `synth` return value described above, the -following has changed: - - - `vcl_recv` must now return `hash` instead of `lookup` - - `vcl_hash` must now return `lookup` instead of `hash` - - `vcl_pass` must now return `fetch` instead of `pass` - - -Backend restarts are now retry -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In 3.0 it was possible to do `return(restart)` after noticing that -the backend response was wrong, to change to a different backend. - -This is now called `return(retry)`, and jumps back up to `vcl_backend_fetch`. - -This only influences the backend fetch thread, client-side handling is not affected. - - -default/builtin VCL changes -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The VCL code that is appended to user-configured VCL automatically -is now called the builtin VCL. (previously default.vcl) - -The builtin VCL now honors Cache-Control: no-cache (and friends) -to indicate uncacheable content from the backend. - - -The `remove` keyword is gone -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Replaced by `unset`. - - -X-Forwarded-For is now set before vcl_recv -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In many cases, people unintentionally removed X-Forwarded-For when -implementing their own vcl_recv. Therefore it has been moved to before -vcl_recv, so if you don't want an IP added to it, you should remove it -in vcl_recv. - - -Changes to existing parameters -============================== - -session_linger -~~~~~~~~~~~~~~ -`session_linger` has been renamed to `timeout_linger` and it is in -seconds now (previously was milliseconds). - -sess_timeout -~~~~~~~~~~~~ -`sess_timeout` has been renamed to `timeout_idle`. - -sess_workspace -~~~~~~~~~~~~~~ - -In 3.0 it was often necessary to increase `sess_workspace` if a -lot of VMODs, complex header operations or ESI were in use. - -This is no longer necessary, because ESI scratch space happens -elsewhere in 4.0. - -If you are using a lot of VMODs, you may need to increase -either `workspace_backend` and `workspace_client` based on where -your VMOD is doing its work. - -thread_pool_purge_delay -~~~~~~~~~~~~~~~~~~~~~~~ -`thread_pool_purge_delay` has been renamed to `thread_pool_destroy_delay` -and it is in seconds now (previously was milliseconds). - -thread_pool_add_delay and thread_pool_fail_delay -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -They are in seconds now (previously were milliseconds). - -New parameters since 3.0 -======================== - -vcc_allow_inline_c -~~~~~~~~~~~~~~~~~~ - -You can now completely disable inline C in your VCL, and it is -disabled by default. - -Other changes -============= - -New log filtering -~~~~~~~~~~~~~~~~~ - -The logging framework has a new filtering language, which means that -the -m switch has been replaced with a new -q switch. See -:ref:`vsl-query(7)` for more information about the new query language. diff --git a/doc/sphinx/whats-new/upgrading-4.0.rst b/doc/sphinx/whats-new/upgrading-4.0.rst new file mode 100644 index 0000000..eecd5b7 --- /dev/null +++ b/doc/sphinx/whats-new/upgrading-4.0.rst @@ -0,0 +1,248 @@ +.. _whatsnew_upgrading_4_0: + +%%%%%%%%%%%%%%%%%%%%%%%% +Upgrading to Varnish 4.0 +%%%%%%%%%%%%%%%%%%%%%%%% + +Changes to VCL +============== + +The backend fetch parts of VCL have changed in Varnish 4. We've tried to +compile a list of changes needed to upgrade here. + +Version statement +~~~~~~~~~~~~~~~~~ + +To make sure that people have upgraded their VCL to the current +version, Varnish now requires the first line of VCL to indicate the +VCL version number:: + + vcl 4.0; + +req.request is now req.method +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To align better with RFC naming, `req.request` has been renamed to +`req.method`. + +vcl_fetch is now vcl_backend_response +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Directors have been moved to the vmod_directors +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To make directors (backend selection logic) easier to extend, the +directors are now defined in loadable VMODs. + +Setting a backend for future fetches in `vcl_recv` is now done as follows:: + + sub vcl_init { + new cluster1 = directors.round_robin(); + cluster1.add_backend(b1, 1.0); + cluster1.add_backend(b2, 1.0); + } + + sub vcl_recv { + set req.backend_hint = cluster1.backend(); + } + +Note the extra `.backend()` needed after the director name. + +Use the hash director as a client director +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Since the client director was already a special case of the hash director, it +has been removed, and you should use the hash director directly:: + + sub vcl_init { + new h = directors.hash(); + h.add_backend(b1, 1); + h.add_backend(b2, 1); + } + + sub vcl_recv { + set req.backend_hint = h.backend(client.identity); + } + +vcl_error is now vcl_backend_error +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To make a distinction between internally generated errors and +VCL synthetic responses, `vcl_backend_error` will be called when +varnish encounters an error when trying to fetch an object. + +error() is now synth() +~~~~~~~~~~~~~~~~~~~~~~ + +And you must explicitly return it:: + + return (synth(999, "Response")); + +Synthetic responses in vcl_synth +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Setting headers on synthetic response bodies made in vcl_synth are now done on +resp.http instead of obj.http. + +The synthetic keyword is now a function:: + + if (resp.status == 799) { + set resp.status = 200; + set resp.http.Content-Type = "text/plain; charset=utf-8"; + synthetic("You are " + client.ip); + return (deliver); + } + +obj in vcl_error replaced by beresp in vcl_backend_error +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To better represent a the context in which it is called, you +should now use `beresp.*` vcl_backend_error, where you used to +use `obj.*` in `vcl_error`. + +hit_for_pass objects are created using beresp.uncacheable +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Example:: + + sub vcl_backend_response { + if (beresp.http.X-No-Cache) { + set beresp.uncacheable = true; + set beresp.ttl = 120s; + return (deliver); + } + } + +req.* not available in vcl_backend_response +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +req.* used to be available in `vcl_fetch`, but after the split of +functionality, you only have 'bereq.*' in `vcl_backend_response`. + +vcl_* reserved +~~~~~~~~~~~~~~ + +Any custom-made subs cannot be named 'vcl_*' anymore. This namespace +is reserved for builtin subs. + +req.backend.healthy replaced by std.healthy(req.backend_hint) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Remember to import the std module if you're not doing so already. + +client.port, and server.port replaced by respectively std.port(client.ip) and std.port(server.ip) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +`client.ip` and `server.ip` are now proper data types, which renders +as an IP address by default. You need to use the `std.port()` +function to get the port number. + +Invalidation with purge +~~~~~~~~~~~~~~~~~~~~~~~ + +Cache invalidation with purges is now done via `return(purge)` from `vcl_recv`. +The `purge;` keyword has been retired. + +obj is now read-only +~~~~~~~~~~~~~~~~~~~~ + +`obj` is now read-only. `obj.last_use` has been retired. + +Some return values have been replaced +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Apart from the new `synth` return value described above, the +following has changed: + + - `vcl_recv` must now return `hash` instead of `lookup` + - `vcl_hash` must now return `lookup` instead of `hash` + - `vcl_pass` must now return `fetch` instead of `pass` + + +Backend restarts are now retry +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In 3.0 it was possible to do `return(restart)` after noticing that +the backend response was wrong, to change to a different backend. + +This is now called `return(retry)`, and jumps back up to `vcl_backend_fetch`. + +This only influences the backend fetch thread, client-side handling is not affected. + + +default/builtin VCL changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The VCL code that is appended to user-configured VCL automatically +is now called the builtin VCL. (previously default.vcl) + +The builtin VCL now honors Cache-Control: no-cache (and friends) +to indicate uncacheable content from the backend. + + +The `remove` keyword is gone +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Replaced by `unset`. + + +X-Forwarded-For is now set before vcl_recv +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In many cases, people unintentionally removed X-Forwarded-For when +implementing their own vcl_recv. Therefore it has been moved to before +vcl_recv, so if you don't want an IP added to it, you should remove it +in vcl_recv. + + +Changes to existing parameters +============================== + +session_linger +~~~~~~~~~~~~~~ +`session_linger` has been renamed to `timeout_linger` and it is in +seconds now (previously was milliseconds). + +sess_timeout +~~~~~~~~~~~~ +`sess_timeout` has been renamed to `timeout_idle`. + +sess_workspace +~~~~~~~~~~~~~~ + +In 3.0 it was often necessary to increase `sess_workspace` if a +lot of VMODs, complex header operations or ESI were in use. + +This is no longer necessary, because ESI scratch space happens +elsewhere in 4.0. + +If you are using a lot of VMODs, you may need to increase +either `workspace_backend` and `workspace_client` based on where +your VMOD is doing its work. + +thread_pool_purge_delay +~~~~~~~~~~~~~~~~~~~~~~~ +`thread_pool_purge_delay` has been renamed to `thread_pool_destroy_delay` +and it is in seconds now (previously was milliseconds). + +thread_pool_add_delay and thread_pool_fail_delay +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +They are in seconds now (previously were milliseconds). + +New parameters since 3.0 +======================== + +vcc_allow_inline_c +~~~~~~~~~~~~~~~~~~ + +You can now completely disable inline C in your VCL, and it is +disabled by default. + +Other changes +============= + +New log filtering +~~~~~~~~~~~~~~~~~ + +The logging framework has a new filtering language, which means that +the -m switch has been replaced with a new -q switch. See +:ref:`vsl-query(7)` for more information about the new query language. diff --git a/doc/sphinx/whats-new/upgrading-4.1.rst b/doc/sphinx/whats-new/upgrading-4.1.rst new file mode 100644 index 0000000..9a78d29 --- /dev/null +++ b/doc/sphinx/whats-new/upgrading-4.1.rst @@ -0,0 +1,72 @@ +.. _whatsnew_upgrading_4_1: + +%%%%%%%%%%%%%%%%%%%%%%%% +Upgrading to Varnish 4.1 +%%%%%%%%%%%%%%%%%%%%%%%% + +Changes to VCL +============== + +Data type conversion functions now take a fallback +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Data type conversion functions in the std vmod now takes an additional +argument *fallback*, which is returned if the conversion does not succeed. + + +Version statement is kept +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The VCL syntax has not chanced significantly, and as such the Varnish 4.0 +version marker is kept for Varnish 4.1. + +One of the initial lines in a Varnish 4.1 VCL should read:: + + vcl 4.0; + +Remote address accessors +~~~~~~~~~~~~~~~~~~~~~~~~ + +New in 4.1 is the `local.ip` and `remote.ip` representing the (local) TCP +connection endpoints. + +With PROXY listeners the `server.ip` and `client.ip` are set from the PROXY +preamble. On normal HTTP listeners the behaviour is unchanged. + + +Management interface +==================== + +The management interface enabled with ``-M`` previously supported the telnet +protocol. + +Support for telnet control sequences have been retired. Replacement clients +like netcat or (preferred) ``varnishadm`` should be used instead. + + +Runtime users and groups +======================== + +With the new jail support, an additional runtime user (`vcache`) should be used +for the Varnish worker child process. + +Additionally, the ``varnishlog``, ``varnishncsa`` and other Varnish shared log +utilities must now be run in a context with `varnish` group membership. + + +Changes to parameters +===================== + +`vcl_cooldown` is new, and decides how long time a VCL is kept warm after being +replaced as the active VCL. + +The following parameters have been retired: + +* `group` (security revamp) +* `group_cc` (security revamp) +* `listen_address` (security revamp) +* `pool_vbc` +* `timeout_req` - merged with `timeout_idle`. +* `user` (security revamp) + +Minor changes of default values on `workspace_session` and `vsl_mask`. diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst new file mode 100644 index 0000000..7cc40a4 --- /dev/null +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -0,0 +1,17 @@ +.. _whatsnew_upgrading_5_0: + +%%%%%%%%%%%%%%%%%%%%%%%% +Upgrading to Varnish 5.0 +%%%%%%%%%%%%%%%%%%%%%%%% + +Changes to VCL +============== + + +Management interface +==================== + + +Changes to parameters +===================== + diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst deleted file mode 100644 index 49c4f08..0000000 --- a/doc/sphinx/whats-new/upgrading.rst +++ /dev/null @@ -1,72 +0,0 @@ -.. _whatsnew_upgrade41: - -%%%%%%%%%%%%%%%%%%%%%%%% -Upgrading to Varnish 4.1 -%%%%%%%%%%%%%%%%%%%%%%%% - -Changes to VCL -============== - -Data type conversion functions now take a fallback -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Data type conversion functions in the std vmod now takes an additional -argument *fallback*, which is returned if the conversion does not succeed. - - -Version statement is kept -~~~~~~~~~~~~~~~~~~~~~~~~~ - -The VCL syntax has not chanced significantly, and as such the Varnish 4.0 -version marker is kept for Varnish 4.1. - -One of the initial lines in a Varnish 4.1 VCL should read:: - - vcl 4.0; - -Remote address accessors -~~~~~~~~~~~~~~~~~~~~~~~~ - -New in 4.1 is the `local.ip` and `remote.ip` representing the (local) TCP -connection endpoints. - -With PROXY listeners the `server.ip` and `client.ip` are set from the PROXY -preamble. On normal HTTP listeners the behaviour is unchanged. - - -Management interface -==================== - -The management interface enabled with ``-M`` previously supported the telnet -protocol. - -Support for telnet control sequences have been retired. Replacement clients -like netcat or (preferred) ``varnishadm`` should be used instead. - - -Runtime users and groups -======================== - -With the new jail support, an additional runtime user (`vcache`) should be used -for the Varnish worker child process. - -Additionally, the ``varnishlog``, ``varnishncsa`` and other Varnish shared log -utilities must now be run in a context with `varnish` group membership. - - -Changes to parameters -===================== - -`vcl_cooldown` is new, and decides how long time a VCL is kept warm after being -replaced as the active VCL. - -The following parameters have been retired: - -* `group` (security revamp) -* `group_cc` (security revamp) -* `listen_address` (security revamp) -* `pool_vbc` -* `timeout_req` - merged with `timeout_idle`. -* `user` (security revamp) - -Minor changes of default values on `workspace_session` and `vsl_mask`. From arianna.aondio at varnish-software.com Wed Sep 14 08:57:13 2016 From: arianna.aondio at varnish-software.com (Arianna Aondio) Date: Wed, 14 Sep 2016 10:57:13 +0200 Subject: [master] 0fe58fb Remote variables not necessary to calculate the hit-rate Message-ID: commit 0fe58fb3a80e20f9152f575e36fe1c6c930532b0 Author: Arianna Aondio Date: Tue Aug 2 14:13:00 2016 +0000 Remote variables not necessary to calculate the hit-rate diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index ef3e1d3..b96101d 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -95,7 +95,6 @@ struct pt { }; struct hitrate { - double lt; uint64_t lhit, lmiss; struct ma hr_10; struct ma hr_100; @@ -411,21 +410,16 @@ sample_points(void) static void sample_hitrate(void) { - double tv,dt; double hr, mr, ratio; uint64_t hit, miss; if (VSC_C_main == NULL) return; - tv = VTIM_mono(); - dt = tv - hitrate.lt; - hitrate.lt= tv; - hit = VSC_C_main->cache_hit; miss = VSC_C_main->cache_miss; - hr = (hit - hitrate.lhit) / dt; - mr = (miss - hitrate.lmiss) / dt; + hr = hit - hitrate.lhit; + mr = miss - hitrate.lmiss; hitrate.lhit = hit; hitrate.lmiss = miss; From noreply at github.com Wed Sep 14 08:57:13 2016 From: noreply at github.com (GitHub) Date: Wed, 14 Sep 2016 10:57:13 +0200 Subject: [master] d029e0d Merge pull request #2030 from aondio/varnishstat Message-ID: commit d029e0de156a9d3f9274ed160657d3b232a91248 Merge: 560b3b6 0fe58fb Author: arianna-aondio Date: Wed Sep 14 10:56:05 2016 +0200 Merge pull request #2030 from aondio/varnishstat Remove variables not necessary to calculate the hit-rate From nils.goroll at uplex.de Wed Sep 14 09:26:13 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 11:26:13 +0200 Subject: [master] 30c0d08 speling Message-ID: commit 30c0d0878aeb6ae74b9de73327f808fceee8778a Author: Nils Goroll Date: Wed Sep 14 11:24:46 2016 +0200 speling diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index c610f0f..6630f51 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -3,7 +3,7 @@ Changes in Varnish 5.0 ====================== -Varnish 5.0 changes some (mostly) internal APIs and adds som major new +Varnish 5.0 changes some (mostly) internal APIs and adds some major new features over Varnish 4.1. @@ -20,7 +20,7 @@ send all POSTs, all JPEG images or all traffic from a certain IP range to a separate VCL file. VCL labels can also be used to give symbolic names to loaded VCL -configurations, so that operations personel only need to know +configurations, so that operations personnel only need to know about "normal", "weekend" and "emergency", and web developers can update these as usual, without having to tell ops what the new weekend VCL is called. From nils.goroll at uplex.de Wed Sep 14 09:26:13 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 11:26:13 +0200 Subject: [master] 4e8e9a7 changes: shard Message-ID: commit 4e8e9a74c916ee0f97a4e8a7e692ea99e6d24bf6 Author: Nils Goroll Date: Wed Sep 14 11:24:54 2016 +0200 changes: shard diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 6630f51..140a82d 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -41,3 +41,23 @@ no browsers support that, but tools like curl does. For encrypted HTTP/2 traffic, put a SSL proxy in front of Varnish. + +The Shard Director +~~~~~~~~~~~~~~~~~~ + +We have added to the directors vmod an overhauled version of a +director which was available as an out-of-tree vmod under the name +VSLP for a couple of years: It's basically a better hash director, +which uses consistent hashing to provide improved stability of backend +node selection when the configuration and/or health state of backends +changes. There are several options to provide the shard key. The +rampup feature allows to take just-gone-healthy backends in production +smoothly, while the prewarm feature allows to prepare backends for +traffic which they would see if the primary backend for a certain key +went down. + +It can be reconfigured dynamically (outside vcl_init), but different +to our other directors, configuration is transactional: Any series of +backend changes must be concluded by a reconfigure call for +activation. + From hermunn at varnish-software.com Wed Sep 14 09:34:14 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Wed, 14 Sep 2016 11:34:14 +0200 Subject: [4.1] 348f2df Changelog update Message-ID: commit 348f2dfa94594cad3aa4a1f052e158c92a7466b9 Author: P?l Hermunn Johansen Date: Tue Sep 13 17:26:46 2016 +0200 Changelog update diff --git a/doc/changes.rst b/doc/changes.rst index 4fe1f39..66ab184 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,20 +1,30 @@ -======================================== -DRAFT CHANGELOG ENTRIES FOR NEXT RELEASE -======================================== +====================================== +Varnish Cache 4.1.4-beta1 (2016-09-14) +====================================== Changes since 4.1.3: +* [varnishhist] Various improvements +* [varnishtest] A `cmd` feature for custom shell-based checks +* Documentation improvements (do_stream, sess_herd, timeout_linger, thread_pools) +* [varnishtop] Documented behavior when both -p and -1 are specified + +Bugs fixed +---------- + +* 2027_ - Racy backend selection * 2024_ - panic vmod_rr_resolve() round_robin.c line 75 (be) != NULL * 2011_ - VBE.*.conn (concurrent connections to backend) not working as expected +* 2008_ - Assert error in VBE_Delete() +* 2007_ - Update documentation part about CLI/management port authentication paramater +* 1881_ - std.cache_req_body() w/ return(pipe) is broken +.. _2027: https://github.com/varnishcache/varnish-cache/issues/2027 .. _2024: https://github.com/varnishcache/varnish-cache/issues/2024 .. _2011: https://github.com/varnishcache/varnish-cache/issues/2011 - -* varnishhist improvements: - * can now generate statistics of backend requests - * more predefined profiles - * time bend option: control the log processing speed - * y axis scale +.. _2008: https://github.com/varnishcache/varnish-cache/issues/2008 +.. _2007: https://github.com/varnishcache/varnish-cache/issues/2007 +.. _1881: https://github.com/varnishcache/varnish-cache/issues/1881 ================================ Varnish Cache 4.1.3 (2016-07-06) From dridi.boukelmoune at gmail.com Wed Sep 14 09:38:12 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 14 Sep 2016 11:38:12 +0200 Subject: [4.1] 1774719 Docfix Message-ID: commit 1774719781729881449b22a49b8501b2e8dfee6c Author: Dridi Boukelmoune Date: Wed Sep 14 11:30:44 2016 +0200 Docfix diff --git a/varnish.m4 b/varnish.m4 index 6caa9fc..d0849ef 100644 --- a/varnish.m4 +++ b/varnish.m4 @@ -230,10 +230,10 @@ clean-vmod-$1: # # ACLOCAL_AMFLAGS = -I m4 -I ${VARNISHAPI_DATAROOTDIR}/aclocal # -# The VARNISH_VERSION variable will be set even if the VARNISH_PREREQ isn't -# called. Although many things are set up to facilitate VMOD maintenance, -# initialization of autoconf, automake and libtool is still the maintainer's -# responsibility. +# The VARNISH_VERSION variable will be set even if the VARNISH_PREREQ macro +# wasn't called. Although many things are set up to facilitate out-of-tree +# VMOD maintenance, initialization of autoconf, automake and libtool is +# still the maintainer's responsibility. It cannot be avoided. # AC_DEFUN([VARNISH_VMODS], [ m4_foreach([_vmod_name], From nils.goroll at uplex.de Wed Sep 14 09:58:14 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 11:58:14 +0200 Subject: [master] 42e3956 changes: hit-for-pass -> hit-for-miss Message-ID: commit 42e3956034b66f49ec20cc9d34d973aded4d9674 Author: Nils Goroll Date: Wed Sep 14 11:57:48 2016 +0200 changes: hit-for-pass -> hit-for-miss diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 140a82d..d6cd5ec 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -61,3 +61,47 @@ to our other directors, configuration is transactional: Any series of backend changes must be concluded by a reconfigure call for activation. +Hit-For-Pass is now actually Hit-For-Miss +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Almost since the beginning of time (2008), varnish has hit-for-pass: +It is basically a negative caching feature, putting into the cache +objects as markers saying "when you hit this, your request should be a +pass". The purpose is to selectively avoid the request coalescing +(waitinglist) feature, which is useful for cacheable content, but not +for uncacheable objects. If we did not have hit-for-pass, without +additional configuration in vcl_recv, requests to uncacheable content +would be sent to the backend serialized (one after the other). + +As useful as this feature is, it has caused a lot of headaches to +varnish administrators along the lines of "why the *beep* doesn't +Varnish cache this": A hit-for-pass object stayed in cache for however +long its ttl dictated and prevented caching whenever it got hit ("for +that url" in most cases). In particular, as a pass object can not be +turned into something cacheable retrospectively (beresp.uncacheable +can be changed from false to true, but not the other way around), even +responses which would have been cacheable were not cached. So, when a +hit-for-pass object got into cache unintentionally, it had to be +removed explicitly (using a ban or purge). + +We've changed this now: + +A hit-for-pass object (we still call it like this in the docs, logging +and statistics) will now cause a cache-miss for all subsequent +requests, so if any backend response qualifies for caching, it will +get cached and subsequent requests will be hits. + +The punchline is: We've changed from "the uncacheable case wins" to +"the cacheable case wins" or from hit-for-pass to hit-for-miss. + +The primary consequence which we are aware of at the time of this +release is caused be the fact that, to create cacheable objects, we +need to make backend requests unconditional (that is, remove the +If-Modified-Since and If-None-Match headers): For conditional client +requests on hit-for-pass objects, Varnish will now issue an +unconditional backend fetch and, for 200 responses, send a 304 or 200 +response to the client as appropriate. + +As of the time of this release we cannot say if this will remain the +final word on this topic, but we hope that it will mean an improvement +for most users of Varnish. From nils.goroll at uplex.de Wed Sep 14 10:20:15 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 12:20:15 +0200 Subject: [master] b14ffbd so much for the points I had added to changes.rst Message-ID: commit b14ffbd0794de1e501730d25dd8a9041ebd9971b Author: Nils Goroll Date: Wed Sep 14 12:17:54 2016 +0200 so much for the points I had added to changes.rst diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index d6cd5ec..1241d79 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -61,6 +61,7 @@ to our other directors, configuration is transactional: Any series of backend changes must be concluded by a reconfigure call for activation. + Hit-For-Pass is now actually Hit-For-Miss ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -105,3 +106,35 @@ response to the client as appropriate. As of the time of this release we cannot say if this will remain the final word on this topic, but we hope that it will mean an improvement for most users of Varnish. + + +Ban Lurker Improvements +~~~~~~~~~~~~~~~~~~~~~~~ + +We have made the ban lurker even more efficient by example of some +real live situations with tens of thousands of bans using inefficient +regular expressions. + +The new parameter ban_lurker_holdoff tells the ban lurker for how long +it should get out of the way when it could potentially slow down +lookups due to lock contention. Previously this was the same as +ban_lurker_sleep. + + +Access to more object properties from vcl_deliver +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +obj.ttl, obj.age, obj.grace and obj.keep are now available vcl_deliver +read-only. + + +News for vmod authors +~~~~~~~~~~~~~~~~~~~~~~~ + +* vcl cli events (init/fini methods) now have a workspace and + PRIV_TASK + +* PRIV_* now also work for object methods with unchanged scope. In + particular, they are per vmod and `not` per object - e.g. the same + PRIV_TASK gets passed to object methods as to functions during a VCL + task. From nils.goroll at uplex.de Wed Sep 14 10:20:15 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 12:20:15 +0200 Subject: [master] 0e0c923 cut this short now that the prosa is in whats-new/changes-5.0.rst Message-ID: commit 0e0c9232816ae3b2973f1b7da6b2249c3ec93ac1 Author: Nils Goroll Date: Wed Sep 14 12:19:19 2016 +0200 cut this short now that the prosa is in whats-new/changes-5.0.rst diff --git a/doc/changes.rst b/doc/changes.rst index 38cfe49..ba2d902 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -15,15 +15,7 @@ Major items: * Always send the request body to the backend, making possible to cache POST, PUT, PATCH etc. -* hit-for-pass is now actually hit-for-miss: Objects cached as - uncacheable now trigger misses rather than hits, so when a cacheable - object becomes available, it will be cached even before the ttl of - the uncacheable object expires. As a particular implication of this, - conditional request headers (If-Modified-Since, If-None-Match) are - now being removed for hits on uncacheable objects. - - Documentation, counters and log entries still refer to the mechanism - as hit-for-pass and Hitpass, respectively. +* hit-for-pass is now actually hit-for-miss. * new shard director for loadbalancing by consistent hashing @@ -34,13 +26,9 @@ Major items: News for Vmod Authors --------------------- -* vcl cli events (init/fini methods) now have a workspace and - PRIV_TASK +* workspace and PRIV_TASK for vcl cli events (init/fini methods) -* PRIV_* now also work for object methods with unchanged scope. In - particular, they are per vmod and `not` per object - e.g. the same - PRIV_TASK gets passed to object methods as to functions during a VCL - task. +* PRIV_* now also work for object methods with unchanged scope. ====================================== From nils.goroll at uplex.de Wed Sep 14 10:21:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 12:21:12 +0200 Subject: [master] fe299bb ocd Message-ID: commit fe299bbc63e9f9a2b68da47963a060658c4ee02e Author: Nils Goroll Date: Wed Sep 14 12:20:47 2016 +0200 ocd diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 1241d79..8ad3b19 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -129,7 +129,7 @@ read-only. News for vmod authors -~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~ * vcl cli events (init/fini methods) now have a workspace and PRIV_TASK From hermunn at varnish-software.com Wed Sep 14 10:44:11 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Wed, 14 Sep 2016 12:44:11 +0200 Subject: [4.1] 0e25e74 Remote variables not necessary to calculate the hit-rate Message-ID: commit 0e25e7457a3c78c20130d08306480463d1ef4e9b Author: Arianna Aondio Date: Tue Aug 2 14:13:00 2016 +0000 Remote variables not necessary to calculate the hit-rate diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index 01c180c..a70b191 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -95,7 +95,6 @@ struct pt { }; struct hitrate { - double lt; uint64_t lhit, lmiss; struct ma hr_10; struct ma hr_100; @@ -411,21 +410,16 @@ sample_points(void) static void sample_hitrate(void) { - double tv,dt; double hr, mr, ratio; uint64_t hit, miss; if (VSC_C_main == NULL) return; - tv = VTIM_mono(); - dt = tv - hitrate.lt; - hitrate.lt= tv; - hit = VSC_C_main->cache_hit; miss = VSC_C_main->cache_miss; - hr = (hit - hitrate.lhit) / dt; - mr = (miss - hitrate.lmiss) / dt; + hr = hit - hitrate.lhit; + mr = miss - hitrate.lmiss; hitrate.lhit = hit; hitrate.lmiss = miss; From nils.goroll at uplex.de Wed Sep 14 11:09:13 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 13:09:13 +0200 Subject: [master] a74e465 pointer to the release notes Message-ID: commit a74e465b37152fb35b33b35fdcbc4ce4f3083f3a Author: Nils Goroll Date: Wed Sep 14 13:03:07 2016 +0200 pointer to the release notes diff --git a/doc/changes.rst b/doc/changes.rst index ba2d902..d26b7bd 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -6,6 +6,9 @@ This is the first beta release of the upcoming 5.0 release. The list of changes are numerous and will not be expanded on in detail. +The release notes contain more background information and are highly +recommended reading before using any of the new features. + Major items: * VCL labels, allowing for per-vhost (or per-anything) separate VCL files. From nils.goroll at uplex.de Wed Sep 14 11:09:13 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 13:09:13 +0200 Subject: [master] 0a0de58 try to summarize request body handling and non-GET caching Message-ID: commit 0a0de584b41d3e3cc11e62e1d9df110c4150a87e Author: Nils Goroll Date: Wed Sep 14 13:03:48 2016 +0200 try to summarize request body handling and non-GET caching diff --git a/doc/changes.rst b/doc/changes.rst index d26b7bd..bea3095 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -15,8 +15,8 @@ Major items: * (Very!) experimental support for HTTP/2. -* Always send the request body to the backend, making possible to cache - POST, PUT, PATCH etc. +* Always send the request body to the backend, allowing caching of + POST, PUT, PATCH etc with appropriate custom VCL and/or vmods. * hit-for-pass is now actually hit-for-miss. diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 8ad3b19..57223a5 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -128,6 +128,37 @@ obj.ttl, obj.age, obj.grace and obj.keep are now available vcl_deliver read-only. +Request Body sent always / "cacheable POST" +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Previously, we would only send a request body for passed requests (and +for pipe mode, but this is special anyway and should be avoided). + +Not so any more, but the default behaviour has not changed: + +Whenever a request has a body, it will get sent to the backend for a +cache miss (and pass, as before). This can be prevented by an unset +bereq.body and the builtin.vcl removes the body for GET requests +because it is questionable if GET with a body is valid anyway (but +some applications use it). + +So the often-requested ability to cache POST/PUT/... is now available, +but not out-of-the-box: + +The builtin.vcl still contains a return(pass) for anything but a GET +or HEAD because other HTTP methods, by definition, may cause state +changes / side effects on backends. The application at hand should be +understood well before caching of non-GET/non-HEAD is considered. + +Care should be taken to choose an appropriate cache key and/or Vary +criteria. Adding the request body to the cache key is not possible +with core varnish, but through a vmod +https://github.com/aondio/libvmod-bodyaccess + +The punchline: You should know what you are doing when caching +anything but a GET or HEAD and without creating an appropriate cache +key doing so is almost guaranteed to be wrong. + News for vmod authors ~~~~~~~~~~~~~~~~~~~~~ From nils.goroll at uplex.de Wed Sep 14 11:09:13 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 13:09:13 +0200 Subject: [master] 2c8b516 improve formatting Message-ID: commit 2c8b516c18d6480dc638d56f07f8af1a42725c57 Author: Nils Goroll Date: Wed Sep 14 13:08:41 2016 +0200 improve formatting diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 57223a5..47baa19 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -10,7 +10,7 @@ features over Varnish 4.1. Separate VCL files and VCL labels ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Varnish 5.0 supports jumping from the active VCL's vcl_recv{} to +Varnish 5.0 supports jumping from the active VCL's ``vcl_recv{}`` to another VCL via a VCL label. The major use of this will probably be to have a separate VCL for @@ -56,9 +56,9 @@ smoothly, while the prewarm feature allows to prepare backends for traffic which they would see if the primary backend for a certain key went down. -It can be reconfigured dynamically (outside vcl_init), but different -to our other directors, configuration is transactional: Any series of -backend changes must be concluded by a reconfigure call for +It can be reconfigured dynamically (outside ``vcl_init{}``), but +different to our other directors, configuration is transactional: Any +series of backend changes must be concluded by a reconfigure call for activation. @@ -79,11 +79,12 @@ varnish administrators along the lines of "why the *beep* doesn't Varnish cache this": A hit-for-pass object stayed in cache for however long its ttl dictated and prevented caching whenever it got hit ("for that url" in most cases). In particular, as a pass object can not be -turned into something cacheable retrospectively (beresp.uncacheable -can be changed from false to true, but not the other way around), even -responses which would have been cacheable were not cached. So, when a -hit-for-pass object got into cache unintentionally, it had to be -removed explicitly (using a ban or purge). +turned into something cacheable retrospectively +(``beresp.uncacheable`` can be changed from ``false`` to ``true``, but +not the other way around), even responses which would have been +cacheable were not cached. So, when a hit-for-pass object got into +cache unintentionally, it had to be removed explicitly (using a ban or +purge). We've changed this now: @@ -98,8 +99,8 @@ The punchline is: We've changed from "the uncacheable case wins" to The primary consequence which we are aware of at the time of this release is caused be the fact that, to create cacheable objects, we need to make backend requests unconditional (that is, remove the -If-Modified-Since and If-None-Match headers): For conditional client -requests on hit-for-pass objects, Varnish will now issue an +``If-Modified-Since`` and ``If-None-Match headers``): For conditional +client requests on hit-for-pass objects, Varnish will now issue an unconditional backend fetch and, for 200 responses, send a 304 or 200 response to the client as appropriate. @@ -115,17 +116,17 @@ We have made the ban lurker even more efficient by example of some real live situations with tens of thousands of bans using inefficient regular expressions. -The new parameter ban_lurker_holdoff tells the ban lurker for how long -it should get out of the way when it could potentially slow down +The new parameter ``ban_lurker_holdoff`` tells the ban lurker for how +long it should get out of the way when it could potentially slow down lookups due to lock contention. Previously this was the same as -ban_lurker_sleep. +``ban_lurker_sleep``. Access to more object properties from vcl_deliver ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -obj.ttl, obj.age, obj.grace and obj.keep are now available vcl_deliver -read-only. +``obj.ttl``, ``obj.age``, ``obj.grace`` and ``obj.keep`` are now +available in ``vcl_deliver{}`` read-only. Request Body sent always / "cacheable POST" @@ -145,10 +146,11 @@ some applications use it). So the often-requested ability to cache POST/PUT/... is now available, but not out-of-the-box: -The builtin.vcl still contains a return(pass) for anything but a GET -or HEAD because other HTTP methods, by definition, may cause state -changes / side effects on backends. The application at hand should be -understood well before caching of non-GET/non-HEAD is considered. +The ``builtin.vcl`` still contains a ``return(pass)`` for anything but +a GET or HEAD because other HTTP methods, by definition, may cause +state changes / side effects on backends. The application at hand +should be understood well before caching of non-GET/non-HEAD is +considered. Care should be taken to choose an appropriate cache key and/or Vary criteria. Adding the request body to the cache key is not possible @@ -162,10 +164,10 @@ key doing so is almost guaranteed to be wrong. News for vmod authors ~~~~~~~~~~~~~~~~~~~~~ -* vcl cli events (init/fini methods) now have a workspace and - PRIV_TASK +* vcl cli events (in particular, ``vcl_init{}`` /``vcl_fini{}``) now + have a workspace and ``PRIV_TASK`` available for vmods. -* PRIV_* now also work for object methods with unchanged scope. In - particular, they are per vmod and `not` per object - e.g. the same - PRIV_TASK gets passed to object methods as to functions during a VCL - task. +* ``PRIV_*`` now also work for object methods with unchanged scope. + In particular, they are per vmod and `not` per object - e.g. the + same ``PRIV_TASK`` gets passed to object methods as to functions + during a VCL task. From dridi.boukelmoune at gmail.com Wed Sep 14 11:12:13 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 14 Sep 2016 13:12:13 +0200 Subject: [master] 9336043 RFC OCD Message-ID: commit 9336043547d0f1567cb32ad5bc8d2f9029b2730a Author: Dridi Boukelmoune Date: Wed Sep 14 13:06:28 2016 +0200 RFC OCD rfc7231#section-4.3.3 Responses to POST requests are only cacheable when they include explicit freshness information (see Section 4.2.1 of [RFC7234]). rfc7231#section-4.3.4 Responses to the PUT method are not cacheable. rfc5789#section-2 A response to this method is only cacheable if it contains explicit freshness information (such as an Expires header or "Cache-Control: max-age" directive) as well as the Content-Location header matching the Request-URI, indicating that the PATCH response body is a resource representation. A cached PATCH response can only be used to respond to subsequent GET and HEAD requests; it MUST NOT be used to respond to other methods (in particular, PATCH). diff --git a/doc/changes.rst b/doc/changes.rst index bea3095..6ba0022 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -15,8 +15,9 @@ Major items: * (Very!) experimental support for HTTP/2. -* Always send the request body to the backend, allowing caching of - POST, PUT, PATCH etc with appropriate custom VCL and/or vmods. +* Always send the request body to the backend, making possible to cache + responses of POST, PATCH requests etc with appropriate custom VCL and/or + vmods. * hit-for-pass is now actually hit-for-miss. From dridi.boukelmoune at gmail.com Wed Sep 14 11:15:15 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 14 Sep 2016 13:15:15 +0200 Subject: [master] 6f72351 More RFC OCD Message-ID: commit 6f72351f01b75b800713af1d79a3bb5bd1e03c5d Author: Dridi Boukelmoune Date: Wed Sep 14 13:14:29 2016 +0200 More RFC OCD diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 47baa19..18ae6ae 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -143,7 +143,7 @@ bereq.body and the builtin.vcl removes the body for GET requests because it is questionable if GET with a body is valid anyway (but some applications use it). -So the often-requested ability to cache POST/PUT/... is now available, +So the often-requested ability to cache POST/PATCH/... is now available, but not out-of-the-box: The ``builtin.vcl`` still contains a ``return(pass)`` for anything but From fgsch at lodoss.net Wed Sep 14 12:14:12 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 14 Sep 2016 14:14:12 +0200 Subject: [master] 8a5f34c Document integers in a boolean context Message-ID: commit 8a5f34ceeb6e39694faca3e5a02f0d5eef82d74a Author: Federico G. Schwindt Date: Wed Sep 14 12:18:07 2016 +0100 Document integers in a boolean context While here improve and tidy things up. diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 789b20d..ca77e8d 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -65,8 +65,8 @@ with the *elseif* statement (*elsif*/*elif*/*else if* are equivalent). Note that there are no loops or iterators of any kind in VCL. -Strings, booleans, time, duration and integers ----------------------------------------------- +Strings, booleans, time, duration, integers and real numbers +------------------------------------------------------------ These are the data types in Varnish. You can *set* or *unset* these. @@ -87,21 +87,26 @@ characters except for the NUL (0x00) character. Booleans ~~~~~~~~ -Booleans can be either *true* or *false*. Some values may be inferred as -truthy or falthy. When cast to booleans, strings will evaluate to *false* -if they are empty or NULL. This allows broad checks on headers existence -for instance. Backend expressions can also be tested as booleans and can -help find when a director like the hash director returns a NULL backend. +Booleans can be either *true* or *false*. In addition, in a boolean +context some data types will evaluate to *true* or *false* depending on +their value. + +String types will evaluate to *false* if they are empty; backend types +will evalute to *false* if they don't have a backend assigned; integer +types will evaluate to *false* if their value is zero. Time ----- +~~~~ + +VCL has time. A duration can be added to a time to make another time. +In string context they return a formatted string in RFC1123 format +(e.g. Sun, 06 Nov 1994 08:49:37 GMT). -VCL has time. The function *now* returns a time. A duration can be -added to a time to make another time. In string context they return a -formatted string. +The keyword *now* returns a time representing the current time in seconds +since the Epoch. Durations ---------- +~~~~~~~~~ Durations are defined by a number and a designation. The number can be a real so 1.5w is allowed. @@ -128,13 +133,13 @@ so 1.5w is allowed. years Integers --------- +~~~~~~~~ Certain fields are integers, used as expected. In string context they return a string. Real numbers ------------- +~~~~~~~~~~~~ VCL understands real numbers. As with integers, when used in a string context they will return a string. From nils.goroll at uplex.de Wed Sep 14 12:25:16 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 14:25:16 +0200 Subject: [master] 4b0fbc9 punchlines - no more Message-ID: commit 4b0fbc9e974370ed275bdf0d39df9b45beaf32d5 Author: Nils Goroll Date: Wed Sep 14 14:24:12 2016 +0200 punchlines - no more diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 18ae6ae..2b887f1 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -93,8 +93,8 @@ and statistics) will now cause a cache-miss for all subsequent requests, so if any backend response qualifies for caching, it will get cached and subsequent requests will be hits. -The punchline is: We've changed from "the uncacheable case wins" to -"the cacheable case wins" or from hit-for-pass to hit-for-miss. +In short: We've changed from "the uncacheable case wins" to "the +cacheable case wins" or from hit-for-pass to hit-for-miss. The primary consequence which we are aware of at the time of this release is caused be the fact that, to create cacheable objects, we @@ -157,9 +157,9 @@ criteria. Adding the request body to the cache key is not possible with core varnish, but through a vmod https://github.com/aondio/libvmod-bodyaccess -The punchline: You should know what you are doing when caching -anything but a GET or HEAD and without creating an appropriate cache -key doing so is almost guaranteed to be wrong. +To summarize: You should know what you are doing when caching anything +but a GET or HEAD and without creating an appropriate cache key doing +so is almost guaranteed to be wrong. News for vmod authors ~~~~~~~~~~~~~~~~~~~~~ From nils.goroll at uplex.de Wed Sep 14 13:03:15 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 15:03:15 +0200 Subject: [master] 2e8c8c1 dridis todo items ticked off Message-ID: commit 2e8c8c126da006f22c7b8f76e61747f65ed8578e Author: Nils Goroll Date: Wed Sep 14 14:37:29 2016 +0200 dridis todo items ticked off diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 2b887f1..dfd611d 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -161,6 +161,23 @@ To summarize: You should know what you are doing when caching anything but a GET or HEAD and without creating an appropriate cache key doing so is almost guaranteed to be wrong. + +Default VCL search path +~~~~~~~~~~~~~~~~~~~~~~~ + +For default builds, vcl files are now also being looked for under +/usr/share/varnish/vcl if not found in /etc/varnish. + +For custom builds, the actual search path is +``${varnishconfdir}:${datarootdir}/varnish/vcl`` + + +devicedetect.vcl +~~~~~~~~~~~~~~~~ + +The basic device detection vcl is now bundled with varnish. + + News for vmod authors ~~~~~~~~~~~~~~~~~~~~~ From nils.goroll at uplex.de Wed Sep 14 13:03:15 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 15:03:15 +0200 Subject: [master] 3c3dc16 extract from git diff -r varnish-4.1.3 ./lib/libvcc/generate.py Message-ID: commit 3c3dc16b1118850bc4194326ce1c87a2f3575ccb Author: Nils Goroll Date: Wed Sep 14 15:02:04 2016 +0200 extract from git diff -r varnish-4.1.3 ./lib/libvcc/generate.py diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index dfd611d..13f99ea 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -122,13 +122,6 @@ lookups due to lock contention. Previously this was the same as ``ban_lurker_sleep``. -Access to more object properties from vcl_deliver -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``obj.ttl``, ``obj.age``, ``obj.grace`` and ``obj.keep`` are now -available in ``vcl_deliver{}`` read-only. - - Request Body sent always / "cacheable POST" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index 7cc40a4..464b07c 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -7,6 +7,51 @@ Upgrading to Varnish 5.0 Changes to VCL ============== +vcl_recv {} +~~~~~~~~~~~ + +* added ``return(vcl(label))`` to switch to the vcl labeled `label` + +vcl_hit {} +~~~~~~~~~~ + +* replace ``return(fetch)`` with ``return(miss)`` + +vcl_backend_* {} +~~~~~~~~~~~~~~~~ + +* added read access to ``remote.ip``, ``client.ip``, ``local.ip`` and + ``server.ip`` + +vcl_backend_fetch {} +~~~~~~~~~~~~~~~~~~~~ + +* added write access to ``bereq.body``, the request body + +* We now send request bodies by default (see :ref:_whatsnew_changes_5.0). + To keep the previous behaviour, add the following code before any + ``return()``:: + + if (bereq.method == "GET") { + unset bereq.body; + } + + +vcl_backend_error {} +~~~~~~~~~~~~~~~~~~~~ + +* added write access to ``beresp.body``, the response body + +vcl_deliver {} +~~~~~~~~~~~~~~ + +* added read access to ``obj.ttl``, ``obj.age``, ``obj.grace`` and + ``obj.keep`` + +vcl_synth {} +~~~~~~~~~~~~ + +* added write access to ``resp.body``, the response body Management interface ==================== From nils.goroll at uplex.de Wed Sep 14 13:10:19 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 15:10:19 +0200 Subject: [master] b5150c9 parameter changes Message-ID: commit b5150c9961ebf89b374f9d3c9702db5489d63990 Author: Nils Goroll Date: Wed Sep 14 15:09:14 2016 +0200 parameter changes diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index 464b07c..25f9c4f 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -60,3 +60,9 @@ Management interface Changes to parameters ===================== +* added ``ban_lurker_holdoff`` + +* removed ``session_max`` + + this parameter actually had no effect since 4.0 and will likely be + added back later. From nils.goroll at uplex.de Wed Sep 14 13:20:23 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 15:20:23 +0200 Subject: [master] 9503af3 be more precise about bodies Message-ID: commit 9503af3d3eceb0eba71eadfd6d285c590e087c71 Author: Nils Goroll Date: Wed Sep 14 15:19:44 2016 +0200 be more precise about bodies diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index 25f9c4f..cbeda25 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -26,7 +26,8 @@ vcl_backend_* {} vcl_backend_fetch {} ~~~~~~~~~~~~~~~~~~~~ -* added write access to ``bereq.body``, the request body +* added write access to ``bereq.body``, the request body, only + supported with ``unset`` yet. * We now send request bodies by default (see :ref:_whatsnew_changes_5.0). To keep the previous behaviour, add the following code before any @@ -40,7 +41,8 @@ vcl_backend_fetch {} vcl_backend_error {} ~~~~~~~~~~~~~~~~~~~~ -* added write access to ``beresp.body``, the response body +* added write access to ``beresp.body``, the response body. This is + planned to replace ``synthetic()`` in future releases. vcl_deliver {} ~~~~~~~~~~~~~~ @@ -51,7 +53,8 @@ vcl_deliver {} vcl_synth {} ~~~~~~~~~~~~ -* added write access to ``resp.body``, the response body +* added write access to ``resp.body``, the response body. This is + planned to replace ``synthetic()`` in future releases. Management interface ==================== From hermunn at varnish-software.com Wed Sep 14 13:23:12 2016 From: hermunn at varnish-software.com (PÃ¥l Hermunn Johansen) Date: Wed, 14 Sep 2016 15:23:12 +0200 Subject: [4.1] 245ec6c Prepare for 4.1.4-beta1 release Message-ID: commit 245ec6c2f17263f6772c7b7d220da81f8c869e93 Author: P?l Hermunn Johansen Date: Wed Sep 14 14:53:05 2016 +0200 Prepare for 4.1.4-beta1 release diff --git a/configure.ac b/configure.ac index 72554d3..02603be 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS Copyright (c) 2006-2016 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [4.1.3], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [4.1.4-beta1], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) From nils.goroll at uplex.de Wed Sep 14 13:32:19 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 15:32:19 +0200 Subject: [master] 699d1b9 esi and waitinglist Message-ID: commit 699d1b9dc34352735c741645dcd02ecee29d6d7b Author: Nils Goroll Date: Wed Sep 14 15:31:49 2016 +0200 esi and waitinglist diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 13f99ea..9da62ee 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -155,6 +155,19 @@ but a GET or HEAD and without creating an appropriate cache key doing so is almost guaranteed to be wrong. +ESI and Backend Request Coalescing ("waitinglist") Improvement +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Previously, ESI subrequests depending on objects being fetched from +the backed used polling, which typically added some ~5ms of processing +time to such subrequests and could lead to starvation effects in +extreme corner cases. + +The waitinglist logic for ESI subrequests now uses condition variables +to trigger immediate continuation of ESI processing when an object +being waited for becomes available. + + Default VCL search path ~~~~~~~~~~~~~~~~~~~~~~~ From nils.goroll at uplex.de Wed Sep 14 14:11:27 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 16:11:27 +0200 Subject: [master] a56e1c9 whatsnew progress on misc details Message-ID: commit a56e1c92c24a745aeab4a007c2c67a8ebc1b713a Author: Nils Goroll Date: Wed Sep 14 16:10:34 2016 +0200 whatsnew progress on misc details diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 9da62ee..5d30506 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -168,6 +168,12 @@ to trigger immediate continuation of ESI processing when an object being waited for becomes available. +Backend PROXY protocol requests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Are now supported through the ``.proxy_header`` attribute of the +backend definition. + Default VCL search path ~~~~~~~~~~~~~~~~~~~~~~~ @@ -184,6 +190,15 @@ devicedetect.vcl The basic device detection vcl is now bundled with varnish. +misc +~~~~ + +Brief notes on other changes + +* Added separate thread for object expiry +* The ESI parser is now more tolerant to some syntactic corner cases +* Reduced needless rushing of requests on the waitinglist + News for vmod authors ~~~~~~~~~~~~~~~~~~~~~ @@ -194,3 +209,5 @@ News for vmod authors In particular, they are per vmod and `not` per object - e.g. the same ``PRIV_TASK`` gets passed to object methods as to functions during a VCL task. + +* varnish now provides a random number api, see vrnd.h diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index cbeda25..642a29b 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -7,6 +7,12 @@ Upgrading to Varnish 5.0 Changes to VCL ============== +backend ... {} +~~~~~~~~~~~~~~ + +* added ``.proxy_header`` attribute with possible values of 1 and 2 + for PROXY Protocol Version 1 and 2 + vcl_recv {} ~~~~~~~~~~~ @@ -59,6 +65,7 @@ vcl_synth {} Management interface ==================== +* to disable CLI authentication, use ``-S none`` Changes to parameters ===================== @@ -69,3 +76,9 @@ Changes to parameters this parameter actually had no effect since 4.0 and will likely be added back later. + +* ``vcl_path`` is now a colon-separated list of directories, replacing + ``vcl_dir`` + +* ``vmod_path`` is now a colon-separated list of directories, replacing + ``vmod_dir`` From nils.goroll at uplex.de Wed Sep 14 14:25:23 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 16:25:23 +0200 Subject: [master] 2d352a1 whatsnew progress on misc details Message-ID: commit 2d352a12321ee33e7e8e774e3dec1cfb8a882c16 Author: Nils Goroll Date: Wed Sep 14 16:22:54 2016 +0200 whatsnew progress on misc details diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 5d30506..a2f8084 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -189,6 +189,13 @@ devicedetect.vcl The basic device detection vcl is now bundled with varnish. +varnishtest +~~~~~~~~~~~ + +* ``resp.msg`` renamed to ``resp.reason`` for consistency with vcl +* HTTP2 testing capabilities added +* default search path for executables and vmods added +* ``sema`` mechanism replaced by ``barrier`` misc ~~~~ @@ -198,6 +205,9 @@ Brief notes on other changes * Added separate thread for object expiry * The ESI parser is now more tolerant to some syntactic corner cases * Reduced needless rushing of requests on the waitinglist +* `varnishhist` can now process backend requests and offers a timebend + function to control the processing speed +* ``std.integer()`` can now also parse real numbers and truncates them News for vmod authors ~~~~~~~~~~~~~~~~~~~~~ From nils.goroll at uplex.de Wed Sep 14 14:25:23 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 16:25:23 +0200 Subject: [master] 7078f71 not decided on synthetic() yet Message-ID: commit 7078f719f869cff34ea43a3a5d1d589aa73d88bf Author: Nils Goroll Date: Wed Sep 14 16:24:06 2016 +0200 not decided on synthetic() yet diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index 642a29b..22de9ff 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -47,8 +47,8 @@ vcl_backend_fetch {} vcl_backend_error {} ~~~~~~~~~~~~~~~~~~~~ -* added write access to ``beresp.body``, the response body. This is - planned to replace ``synthetic()`` in future releases. +* added write access to ``beresp.body``, the response body. This may + replace ``synthetic()`` in future releases. vcl_deliver {} ~~~~~~~~~~~~~~ @@ -59,8 +59,8 @@ vcl_deliver {} vcl_synth {} ~~~~~~~~~~~~ -* added write access to ``resp.body``, the response body. This is - planned to replace ``synthetic()`` in future releases. +* added write access to ``resp.body``, the response body. This may + replace ``synthetic()`` in future releases. Management interface ==================== From nils.goroll at uplex.de Wed Sep 14 14:56:20 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 16:56:20 +0200 Subject: [master] cbdcc47 whatsnew bits and pieces Message-ID: commit cbdcc47314ecf5ef91c50d0f5a6e4ef3042ca476 Author: Nils Goroll Date: Wed Sep 14 16:55:39 2016 +0200 whatsnew bits and pieces diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index a2f8084..8ce3221 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -41,6 +41,9 @@ no browsers support that, but tools like curl does. For encrypted HTTP/2 traffic, put a SSL proxy in front of Varnish. +HTTP/2 support is disabled by default, to enable, set the ``http2`` +feature bit. + The Shard Director ~~~~~~~~~~~~~~~~~~ @@ -196,6 +199,7 @@ varnishtest * HTTP2 testing capabilities added * default search path for executables and vmods added * ``sema`` mechanism replaced by ``barrier`` +* support for PROXY requests misc ~~~~ @@ -208,10 +212,16 @@ Brief notes on other changes * `varnishhist` can now process backend requests and offers a timebend function to control the processing speed * ``std.integer()`` can now also parse real numbers and truncates them +* ``std.log()`` now also works correctly during ``vcl_init{}`` +* further improved stability when handling workspace overflows +* numerous vcl compiler improvements News for vmod authors ~~~~~~~~~~~~~~~~~~~~~ +* It is now mandatory to have a description in the ``$Module`` line of + a ``vcc`` file + * vcl cli events (in particular, ``vcl_init{}`` /``vcl_fini{}``) now have a workspace and ``PRIV_TASK`` available for vmods. @@ -221,3 +231,8 @@ News for vmod authors during a VCL task. * varnish now provides a random number api, see vrnd.h + +* vbm (variable size bitmaps) improved + +* ``vmodtool.py`` for translating vcc files has been largely + rewritten, there may still exist regressions which remained unnoticed diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index 22de9ff..ae12fba 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -7,6 +7,11 @@ Upgrading to Varnish 5.0 Changes to VCL ============== +* All VCL Objects should now be defined before used + * in particular, this is now required for ACLs. The error message + for ACLs being used before being defined is confusing - see PR #2021 + ``Name is a reserved name`` + backend ... {} ~~~~~~~~~~~~~~ @@ -17,6 +22,7 @@ vcl_recv {} ~~~~~~~~~~~ * added ``return(vcl(label))`` to switch to the vcl labeled `label` +* ``rollback`` is now ``std.rollback(req)`` vcl_hit {} ~~~~~~~~~~ @@ -67,6 +73,8 @@ Management interface * to disable CLI authentication, use ``-S none`` +* ``n_waitinglist`` statistic removed + Changes to parameters ===================== From nils.goroll at uplex.de Wed Sep 14 14:57:18 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 16:57:18 +0200 Subject: [master] 16466c2 fix rst formatting error Message-ID: commit 16466c228ad61e98d6eb21997afe48696688a826 Author: Nils Goroll Date: Wed Sep 14 16:56:41 2016 +0200 fix rst formatting error diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index ae12fba..635287d 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -8,6 +8,7 @@ Changes to VCL ============== * All VCL Objects should now be defined before used + * in particular, this is now required for ACLs. The error message for ACLs being used before being defined is confusing - see PR #2021 ``Name is a reserved name`` From nils.goroll at uplex.de Wed Sep 14 15:02:39 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 17:02:39 +0200 Subject: [master] 37a6232 improve a formatting nit Message-ID: commit 37a623217c4e30811b3003fab225085b0e9133d1 Author: Nils Goroll Date: Wed Sep 14 17:01:02 2016 +0200 improve a formatting nit diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index 635287d..b8535dc 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -10,8 +10,9 @@ Changes to VCL * All VCL Objects should now be defined before used * in particular, this is now required for ACLs. The error message - for ACLs being used before being defined is confusing - see PR #2021 - ``Name is a reserved name`` + for ACLs being used before being defined is confusing - see PR #2021:: + + Name is a reserved name backend ... {} ~~~~~~~~~~~~~~ From nils.goroll at uplex.de Wed Sep 14 15:04:34 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 17:04:34 +0200 Subject: [master] 1a94b71 vmodtool req'd python version Message-ID: commit 1a94b714f6cabbf03a64cf4ca75e627f117f60f4 Author: Nils Goroll Date: Wed Sep 14 17:03:10 2016 +0200 vmodtool req'd python version diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 8ce3221..5f75b35 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -236,3 +236,5 @@ News for vmod authors * ``vmodtool.py`` for translating vcc files has been largely rewritten, there may still exist regressions which remained unnoticed + +* ``vmodtool.py`` now requires at least Python 2.6 From nils.goroll at uplex.de Wed Sep 14 15:16:23 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Sep 2016 17:16:23 +0200 Subject: [master] d4ad1f9 final(?) formatting fixes Message-ID: commit d4ad1f95a1ba79782c77f3e2cf4e587d77873815 Author: Nils Goroll Date: Wed Sep 14 17:15:29 2016 +0200 final(?) formatting fixes diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 5f75b35..6e9d2b5 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -134,10 +134,10 @@ for pipe mode, but this is special anyway and should be avoided). Not so any more, but the default behaviour has not changed: Whenever a request has a body, it will get sent to the backend for a -cache miss (and pass, as before). This can be prevented by an unset -bereq.body and the builtin.vcl removes the body for GET requests -because it is questionable if GET with a body is valid anyway (but -some applications use it). +cache miss (and pass, as before). This can be prevented by an +``unset bereq.body`` and the ``builtin.vcl`` removes the body for GET +requests because it is questionable if GET with a body is valid anyway +(but some applications use it). So the often-requested ability to cache POST/PATCH/... is now available, but not out-of-the-box: @@ -181,7 +181,7 @@ Default VCL search path ~~~~~~~~~~~~~~~~~~~~~~~ For default builds, vcl files are now also being looked for under -/usr/share/varnish/vcl if not found in /etc/varnish. +``/usr/share/varnish/vcl`` if not found in ``/etc/varnish``. For custom builds, the actual search path is ``${varnishconfdir}:${datarootdir}/varnish/vcl`` @@ -209,7 +209,7 @@ Brief notes on other changes * Added separate thread for object expiry * The ESI parser is now more tolerant to some syntactic corner cases * Reduced needless rushing of requests on the waitinglist -* `varnishhist` can now process backend requests and offers a timebend +* ``varnishhist`` can now process backend requests and offers a timebend function to control the processing speed * ``std.integer()`` can now also parse real numbers and truncates them * ``std.log()`` now also works correctly during ``vcl_init{}`` From dridi.boukelmoune at gmail.com Wed Sep 14 15:22:20 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 14 Sep 2016 17:22:20 +0200 Subject: [master] 01f75bc s/vmod/VMOD/ when used in sentences Message-ID: commit 01f75bcc6f39acdb1c3a6c258479a07b379381d7 Author: Dridi Boukelmoune Date: Wed Sep 14 17:15:26 2016 +0200 s/vmod/VMOD/ when used in sentences It is the (upper) case for most of the documentation. diff --git a/doc/changes.rst b/doc/changes.rst index 6ba0022..57a36fc 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -17,7 +17,7 @@ Major items: * Always send the request body to the backend, making possible to cache responses of POST, PATCH requests etc with appropriate custom VCL and/or - vmods. + VMODs. * hit-for-pass is now actually hit-for-miss. @@ -110,7 +110,7 @@ Bugs fixed Varnish Cache 4.1.2 (2016-03-04) ================================ -* [vmods] vmodtool improvements for multiple vmods in a single directory. +* [vmods] vmodtool improvements for multiple VMODs in a single directory. Bugs fixed ---------- @@ -128,7 +128,7 @@ Bugs fixed Varnish Cache 4.1.2-beta2 (2016-02-25) ====================================== -* [vmods] Passing VCL ACL to a vmod is now possible. +* [vmods] Passing VCL ACL to a VMOD is now possible. * [vmods] VRT_MINOR_VERSION increase due to new function: VRT_acl_match() @@ -154,7 +154,7 @@ Varnish Cache 4.1.2-beta1 (2016-02-17) * Arithmetric on REAL data type in VCL is now possible. -* vmodtool.py improvements to allow vmods for 4.0 and 4.1 to share a source tree. +* vmodtool.py improvements to allow VMODs for 4.0 and 4.1 to share a source tree. * Off-by-one in WS_Reset() fixed. @@ -222,7 +222,7 @@ Changes since 4.1.0: - [varnishncsa] -w is now required when running deamonized. - [varnishncsa] Log format can now be read from file. - Port fields extracted from PROXY1 header now work as expected. -- New VCL state "busy" introduced (mostly for vmod writers). +- New VCL state "busy" introduced (mostly for VMOD writers). - Last traces of varnishreplay removed. - If-Modified-Since is now ignored if we have If-None-Match. - Zero Content-Length is no longer sent on 304 responses. @@ -1140,7 +1140,7 @@ VCL - Allow relational comparisons of floating point types. -- Make it possible for vmods to fail loading and so cause the VCL +- Make it possible for VMODs to fail loading and so cause the VCL loading to fail. varnishncsa @@ -1262,8 +1262,8 @@ Other - A bug in the ESI entity replacement parser has been fixed. `Bug #961`_. -- The ABI of vmods are now checked. This will require a rebuild of - all vmods against the new version of Varnish. +- The ABI of VMODs are now checked. This will require a rebuild of + all VMODs against the new version of Varnish. .. _bug #961: https://www.varnish-cache.org/trac/ticket/961 @@ -1510,7 +1510,7 @@ varnishlog VMODs ----- -- Add a std vmod which includes a random function, log, syslog, +- Add a std VMOD which includes a random function, log, syslog, fileread, collect, VCL @@ -1531,7 +1531,7 @@ VCL ``vcl_fetch`` and are read only. `Bug #481`_ - The default VCL now calls pass for any objects where ``beresp.http.Vary == "*"``. `Bug #787`_ -- The ``log`` keyword has been moved to the ``std`` vmod. +- The ``log`` keyword has been moved to the ``std`` VMOD. - It is now possible to choose which storage backend to be used - Add variables ``storage.$name.free_space``, ``storage.$name.used_space`` and ``storage.$name.happy`` diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 6e9d2b5..9931e07 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -48,8 +48,8 @@ feature bit. The Shard Director ~~~~~~~~~~~~~~~~~~ -We have added to the directors vmod an overhauled version of a -director which was available as an out-of-tree vmod under the name +We have added to the directors VMOD an overhauled version of a +director which was available as an out-of-tree VMOD under the name VSLP for a couple of years: It's basically a better hash director, which uses consistent hashing to provide improved stability of backend node selection when the configuration and/or health state of backends @@ -150,7 +150,7 @@ considered. Care should be taken to choose an appropriate cache key and/or Vary criteria. Adding the request body to the cache key is not possible -with core varnish, but through a vmod +with core varnish, but through a VMOD https://github.com/aondio/libvmod-bodyaccess To summarize: You should know what you are doing when caching anything @@ -216,17 +216,17 @@ Brief notes on other changes * further improved stability when handling workspace overflows * numerous vcl compiler improvements -News for vmod authors +News for VMOD authors ~~~~~~~~~~~~~~~~~~~~~ * It is now mandatory to have a description in the ``$Module`` line of a ``vcc`` file * vcl cli events (in particular, ``vcl_init{}`` /``vcl_fini{}``) now - have a workspace and ``PRIV_TASK`` available for vmods. + have a workspace and ``PRIV_TASK`` available for VMODs. * ``PRIV_*`` now also work for object methods with unchanged scope. - In particular, they are per vmod and `not` per object - e.g. the + In particular, they are per VMOD and `not` per object - e.g. the same ``PRIV_TASK`` gets passed to object methods as to functions during a VCL task. From fgsch at lodoss.net Wed Sep 14 15:37:18 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 14 Sep 2016 17:37:18 +0200 Subject: [master] 95c5286 Tidy up and more entries Message-ID: commit 95c528667cf7ab5d599ac2d9d8c4eb41a063f6d8 Author: Federico G. Schwindt Date: Wed Sep 14 16:35:35 2016 +0100 Tidy up and more entries diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index b8535dc..dd68e1b 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -14,81 +14,84 @@ Changes to VCL Name is a reserved name -backend ... {} -~~~~~~~~~~~~~~ +* VCL names are restricted to alphanumeric characters, dashes (-) and + underscores (_). In addition, the first character should be alphabetic. + That is, the name should match "[A-Za-z][A-Za-z0-9\_-]*". -* added ``.proxy_header`` attribute with possible values of 1 and 2 - for PROXY Protocol Version 1 and 2 +backends +~~~~~~~~ -vcl_recv {} -~~~~~~~~~~~ +* Added support for the PROXY protocol via ``.proxy_header`` attribute. + Possible values are 1 and 2, corresponding to the PROXY protocol + version 1 and 2, respectively. + +vcl_recv +~~~~~~~~ -* added ``return(vcl(label))`` to switch to the vcl labeled `label` -* ``rollback`` is now ``std.rollback(req)`` +* Added ``return(vcl(label))`` to switch to the VCL labelled `label`. +* The ``rollback`` function has been retired. -vcl_hit {} -~~~~~~~~~~ +vcl_hit +~~~~~~~ -* replace ``return(fetch)`` with ``return(miss)`` +* Replace ``return(fetch)`` with ``return(miss)``. -vcl_backend_* {} -~~~~~~~~~~~~~~~~ +vcl_backend_* +~~~~~~~~~~~~~ -* added read access to ``remote.ip``, ``client.ip``, ``local.ip`` and - ``server.ip`` +* Added read access to ``remote.ip``, ``client.ip``, ``local.ip`` and + ``server.ip``. -vcl_backend_fetch {} -~~~~~~~~~~~~~~~~~~~~ +vcl_backend_fetch +~~~~~~~~~~~~~~~~~ -* added write access to ``bereq.body``, the request body, only - supported with ``unset`` yet. +* Added write access to ``bereq.body``, the request body. Only ``unset`` + is supported at this time. * We now send request bodies by default (see :ref:_whatsnew_changes_5.0). - To keep the previous behaviour, add the following code before any - ``return()``:: + To keep the previous behaviour add the following code before any + ``return()`` statement in this subroutine:: if (bereq.method == "GET") { unset bereq.body; } -vcl_backend_error {} -~~~~~~~~~~~~~~~~~~~~ +vcl_backend_error +~~~~~~~~~~~~~~~~~ -* added write access to ``beresp.body``, the response body. This may +* Added write access to ``beresp.body``, the response body. This may replace ``synthetic()`` in future releases. -vcl_deliver {} -~~~~~~~~~~~~~~ +vcl_deliver +~~~~~~~~~~~ -* added read access to ``obj.ttl``, ``obj.age``, ``obj.grace`` and - ``obj.keep`` +* Added read access to ``obj.ttl``, ``obj.age``, ``obj.grace`` and + ``obj.keep``. -vcl_synth {} -~~~~~~~~~~~~ +vcl_synth +~~~~~~~~~ -* added write access to ``resp.body``, the response body. This may - replace ``synthetic()`` in future releases. +* Added write access to ``resp.body``, the response body. This may replace + ``synthetic()`` in future releases. Management interface ==================== -* to disable CLI authentication, use ``-S none`` +* To disable CLI authentication use ``-S none``. -* ``n_waitinglist`` statistic removed +* ``n_waitinglist`` statistic removed. Changes to parameters ===================== -* added ``ban_lurker_holdoff`` - -* removed ``session_max`` +* Added ``ban_lurker_holdoff``. - this parameter actually had no effect since 4.0 and will likely be - added back later. +* Removed ``session_max``. This parameter actually had no effect since + 4.0 but might come back in a future release. * ``vcl_path`` is now a colon-separated list of directories, replacing - ``vcl_dir`` + ``vcl_dir``. * ``vmod_path`` is now a colon-separated list of directories, replacing - ``vmod_dir`` + ``vmod_dir``. From dridi.boukelmoune at gmail.com Wed Sep 14 15:56:14 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 14 Sep 2016 17:56:14 +0200 Subject: [master] c252050 Mention the new autoconf macros in the changelog Message-ID: commit c252050bd51624eb5a6c3ac0adbf79ddeedf7171 Author: Dridi Boukelmoune Date: Wed Sep 14 17:53:42 2016 +0200 Mention the new autoconf macros in the changelog diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index 9931e07..adc6247 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -220,7 +220,7 @@ News for VMOD authors ~~~~~~~~~~~~~~~~~~~~~ * It is now mandatory to have a description in the ``$Module`` line of - a ``vcc`` file + a ``vcc`` file. * vcl cli events (in particular, ``vcl_init{}`` /``vcl_fini{}``) now have a workspace and ``PRIV_TASK`` available for VMODs. @@ -238,3 +238,8 @@ News for VMOD authors rewritten, there may still exist regressions which remained unnoticed * ``vmodtool.py`` now requires at least Python 2.6 + +* New autoconf macros are available, they should greatly simplify build + systems of out-of-tree VMODs. They are implemented and documented in + ``varnish.m4``, and the previous macros now live in ``varnish-legacy.m4`` + so existing VMODs should still build fine. From dridi.boukelmoune at gmail.com Wed Sep 14 16:00:27 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 14 Sep 2016 18:00:27 +0200 Subject: [master] 1440a4a Boolean automatic conversions Message-ID: commit 1440a4a518024b585ba7765f1891c479a0ab628b Author: Dridi Boukelmoune Date: Wed Sep 14 17:59:13 2016 +0200 Boolean automatic conversions diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index dd68e1b..993d400 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -18,6 +18,9 @@ Changes to VCL underscores (_). In addition, the first character should be alphabetic. That is, the name should match "[A-Za-z][A-Za-z0-9\_-]*". +* Like strings, backends and integers can now be used as boolean expressions + in if statements. + backends ~~~~~~~~ From fgsch at lodoss.net Wed Sep 14 16:19:15 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 14 Sep 2016 18:19:15 +0200 Subject: [master] 07a9e0f More changes Message-ID: commit 07a9e0f833f40097e4550d37ba628b1b77c111d1 Author: Federico G. Schwindt Date: Wed Sep 14 17:10:43 2016 +0100 More changes diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index 993d400..093005e 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -18,8 +18,15 @@ Changes to VCL underscores (_). In addition, the first character should be alphabetic. That is, the name should match "[A-Za-z][A-Za-z0-9\_-]*". -* Like strings, backends and integers can now be used as boolean expressions - in if statements. +* Backend and integer types can be used in a boolean context. + See ``vcl(7)`` for details. + +* Add support to perform matches in assignments, obtaining a boolean + as result:: + + set req.http.foo = req.http.bar ~ "bar"; + +* Returned values from functions and methods' calls can be thrown away. backends ~~~~~~~~ @@ -31,13 +38,13 @@ backends vcl_recv ~~~~~~~~ -* Added ``return(vcl(label))`` to switch to the VCL labelled `label`. +* Added ``return (vcl(label))`` to switch to the VCL labelled `label`. * The ``rollback`` function has been retired. vcl_hit ~~~~~~~ -* Replace ``return(fetch)`` with ``return(miss)``. +* Replace ``return (fetch)`` with ``return (miss)``. vcl_backend_* ~~~~~~~~~~~~~ @@ -53,7 +60,7 @@ vcl_backend_fetch * We now send request bodies by default (see :ref:_whatsnew_changes_5.0). To keep the previous behaviour add the following code before any - ``return()`` statement in this subroutine:: + ``return (..)`` statement in this subroutine:: if (bereq.method == "GET") { unset bereq.body; From fgsch at lodoss.net Wed Sep 14 16:42:14 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 14 Sep 2016 18:42:14 +0200 Subject: [master] 38758cf And a few more Message-ID: commit 38758cf3247ebc500c71c9e9a7f2f4238e63db28 Author: Federico G. Schwindt Date: Wed Sep 14 17:40:27 2016 +0100 And a few more diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index 093005e..e91408d 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -105,3 +105,12 @@ Changes to parameters * ``vmod_path`` is now a colon-separated list of directories, replacing ``vmod_dir``. + +Other changes +============= + +* ``varnishstat(1)`` -f option accepts a ``glob(7)`` pattern. + +* Cache-Control and Expires headers for uncacheable requests (i.e. passes) + will not be parsed. As a result, the RFC variant of the TTL VSL tag + is no longer logged. From fgsch at lodoss.net Wed Sep 14 23:07:08 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 15 Sep 2016 01:07:08 +0200 Subject: [master] 1031433 Bring 1440a4a5 back Message-ID: commit 1031433218edd19ca6912a26de95dca6a1989e59 Author: Federico G. Schwindt Date: Thu Sep 15 00:05:06 2016 +0100 Bring 1440a4a5 back I removed it when I was committing a larger chunk. Sorry. diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index e91408d..848b4e0 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -18,8 +18,8 @@ Changes to VCL underscores (_). In addition, the first character should be alphabetic. That is, the name should match "[A-Za-z][A-Za-z0-9\_-]*". -* Backend and integer types can be used in a boolean context. - See ``vcl(7)`` for details. +* Like strings, backends and integers can now be used as boolean + expressions in if statements. See ``vcl(7)`` for details. * Add support to perform matches in assignments, obtaining a boolean as result:: From phk at FreeBSD.org Thu Sep 15 05:21:08 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 15 Sep 2016 07:21:08 +0200 Subject: [master] 7912194 Add release note draft Message-ID: commit 79121940ebab00c2bc3c2e6618357884a5396d36 Author: Poul-Henning Kamp Date: Wed Sep 14 13:20:41 2016 +0000 Add release note draft diff --git a/doc/sphinx/whats-new/index.rst b/doc/sphinx/whats-new/index.rst index 68d076f..1d2c41c 100644 --- a/doc/sphinx/whats-new/index.rst +++ b/doc/sphinx/whats-new/index.rst @@ -14,6 +14,7 @@ Varnish 5.0 .. toctree:: :maxdepth: 2 + relnote-5.0 changes-5.0 upgrading-5.0 diff --git a/doc/sphinx/whats-new/relnote-5.0.rst b/doc/sphinx/whats-new/relnote-5.0.rst new file mode 100644 index 0000000..b73d32b --- /dev/null +++ b/doc/sphinx/whats-new/relnote-5.0.rst @@ -0,0 +1,107 @@ +.. _whatsnew_relnote_5.0: + +Varnish 5.0 Release Note +======================== + +This is the first Varnish release after the Varnish Project moved out +of Varnish Softwares basement, so to speak, and it shows. + +But it is also our 10 year aniversary release, `Varnish 1.0 was +released`_ on September 20th 2006. + +That also means that we have been doing this for 10 years +without any bad security holes. + +So yeah? 5.0 is not what we had hoped it would be, but we are as +proud as one can possibly be anyway. + +We have put the technical stuff in two separate documents: + +* :ref:`whatsnew_changes_5.0` + +* :ref:`whatsnew_upgrading_5.0` + + +The obligatory thank-you speech +------------------------------- + +This release of Varnish Cache is brought to you by the generous +support and donations of money and manpower from four companies: + +* Fastly + +* Varnish Software + +* UPLEX + +* The company which prefers to simply be known as "ADJS" + +Without them, this release, and for that matter all the previous +ones, would not have happened. + +Even though they are by and large employees of the very +same companies, the developers who have contributed to +Varnish in a major way in this release also deserve thanks: + +* Martin - HTTP/2 HPACK header compression code, stevedore API, VSL + +* Niels & Geoff - Shard backend director, ban-lurker improvements + +* Guillame - HTTP/2 support for varnishtest + +* Dridi - Backend temperatures etc. + +* Federico - Too many fixes and ideas to count + +* Lasse - Our tireless release-manager + +* Devon - Performance insights and critical review. + +* The rest of the V-S crew, for too many things to mention. + +We need more money +------------------ + +Until now Varnish Software has done a lot of work for the Varnish +Cache project, but for totally valid reasons, they are scaling that +back and the project either needs to pick up the slack or drop some +of those activities. + +It is important that people understand that Free and Open Source +Software isn't the same as gratis software: Somebody has to pay +the developers mortgages and student loans. + +A very large part of the Varnish development is funded through the +`Varnish Moral License`_, which enables Poul-Henning Kamp to have +Varnish as his primary job, but right now he is underfunded to the +tune of ?2000-3000 per month. + +Please consider if your company makes enough money using Varnish +Cache, to spare some money, or employee-hours for its future +maintenance and development. + + +We also need more manpower +-------------------------- + +First and foremost, we could really use a Postmaster to look after +our mailman mailing lists, including the increasingly arcane art +of anti-spam techniques and invocations. + +We also need to work more on our documentation, it is in bad need +of one or more writers which can actually write text rather than +code. + +We could also use more qualified content for our new project homepage, +so a webmaster is on our shopping list as well. + +Finally, we can always use C-developers, we have more ideas than +we have coders, and since we have very high standards for quality +things take time to write. + +*phk* + + +.. _Varnish Moral License: http://phk.freebsd.dk/VML + +.. _Varnish 1.0 was released: https://sourceforge.net/p/varnish/news/2006/09/varnish-10-released/ diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index 848b4e0..31371da 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -1,4 +1,4 @@ -.. _whatsnew_upgrading_5_0: +.. _whatsnew_upgrading_5.0: %%%%%%%%%%%%%%%%%%%%%%%% Upgrading to Varnish 5.0 From phk at FreeBSD.org Thu Sep 15 05:21:08 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 15 Sep 2016 07:21:08 +0200 Subject: [master] e588f19 Git more work on release note Message-ID: commit e588f192c404bfe667ff8cc9f11dff501702a272 Author: Poul-Henning Kamp Date: Wed Sep 14 13:51:27 2016 +0000 Git more work on release note diff --git a/doc/sphinx/whats-new/relnote-5.0.rst b/doc/sphinx/whats-new/relnote-5.0.rst index b73d32b..0c979f8 100644 --- a/doc/sphinx/whats-new/relnote-5.0.rst +++ b/doc/sphinx/whats-new/relnote-5.0.rst @@ -12,15 +12,58 @@ released`_ on September 20th 2006. That also means that we have been doing this for 10 years without any bad security holes. -So yeah? 5.0 is not what we had hoped it would be, but we are as -proud as one can possibly be anyway. +So yeah? 5.0 is not entirely what we had hoped it would be, but we +are as proud as one can possibly be anyway. -We have put the technical stuff in two separate documents: +To keep this release not short(er), we have put the purely technical +stuff in two separate documents: * :ref:`whatsnew_changes_5.0` * :ref:`whatsnew_upgrading_5.0` +How to get Varnish 5.0 +---------------------- + +`Source download `_ + +Packages for most mainstream operating systems should appear in as +soon as they trickle through the machinery. + + +Reasons to upgrade to Varnish 5.0 +--------------------------------- + +The separate VCL/VCL labels feature can probably help you untangle +your VCL code if it has become too complex. Upgrading from 4.1 +to get that feature should be a no-brainer. + +The HTTP/2 code is not mature enough for production, and if you +want to start to play with H2, you should not upgrade to 5.0, +but rather track -trunk from github and help us find all the bugs +before the next release. + +The Shard director is new in the tree, but it has a lot of live +hours out of tree. Upgrading from 4.1 to 5.0 to get that should +also be a no-brainer. + +We have also fixed at lot of minor bugs, and improved many details +here and there, but to be honest, we don't have a comprehensive +list of it all, apart from the git logs. + + +Reasons not to upgrade to Varnish 5.0 +------------------------------------- + +None that we know of af this time. + + +Next release +------------ + +Next release is scheduled for March 15th 2017, and will most +likely be Varnish 5.1. + The obligatory thank-you speech ------------------------------- @@ -40,8 +83,7 @@ Without them, this release, and for that matter all the previous ones, would not have happened. Even though they are by and large employees of the very -same companies, the developers who have contributed to -Varnish in a major way in this release also deserve thanks: +same companies, these developers merit personal praise: * Martin - HTTP/2 HPACK header compression code, stevedore API, VSL @@ -57,7 +99,8 @@ Varnish in a major way in this release also deserve thanks: * Devon - Performance insights and critical review. -* The rest of the V-S crew, for too many things to mention. +* The rest of the V-S crew - Too many things to list. + We need more money ------------------ @@ -99,6 +142,13 @@ Finally, we can always use C-developers, we have more ideas than we have coders, and since we have very high standards for quality things take time to write. +The best way to get involved is to just jump in and do stuff that +needs done. + +Here is the `Varnish Cache github page `_. + +And here is the `Varnish Projects homepage on github `_. + *phk* From phk at FreeBSD.org Thu Sep 15 05:21:08 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 15 Sep 2016 07:21:08 +0200 Subject: [master] 08f2bc5 Minor fixes Message-ID: commit 08f2bc560761d1d214a48deab93d9e3c38639fdc Author: Poul-Henning Kamp Date: Thu Sep 15 05:20:33 2016 +0000 Minor fixes diff --git a/doc/sphinx/whats-new/relnote-5.0.rst b/doc/sphinx/whats-new/relnote-5.0.rst index 0c979f8..992e7ac 100644 --- a/doc/sphinx/whats-new/relnote-5.0.rst +++ b/doc/sphinx/whats-new/relnote-5.0.rst @@ -15,7 +15,7 @@ without any bad security holes. So yeah? 5.0 is not entirely what we had hoped it would be, but we are as proud as one can possibly be anyway. -To keep this release not short(er), we have put the purely technical +To keep this release note short(er), we have put the purely technical stuff in two separate documents: * :ref:`whatsnew_changes_5.0` @@ -27,7 +27,7 @@ How to get Varnish 5.0 `Source download `_ -Packages for most mainstream operating systems should appear in as +Packages for mainstream operating systems should appear in as soon as they trickle through the machinery. @@ -48,8 +48,7 @@ hours out of tree. Upgrading from 4.1 to 5.0 to get that should also be a no-brainer. We have also fixed at lot of minor bugs, and improved many details -here and there, but to be honest, we don't have a comprehensive -list of it all, apart from the git logs. +here and there, See :ref:`whatsnew_upgrading_5.0` for more of this. Reasons not to upgrade to Varnish 5.0 @@ -57,6 +56,8 @@ Reasons not to upgrade to Varnish 5.0 None that we know of af this time. +Only in very special cases should you need to modify your VCL. + Next release ------------ @@ -82,7 +83,7 @@ support and donations of money and manpower from four companies: Without them, this release, and for that matter all the previous ones, would not have happened. -Even though they are by and large employees of the very +Even though they are all employees of those very same companies, these developers merit personal praise: * Martin - HTTP/2 HPACK header compression code, stevedore API, VSL @@ -149,6 +150,8 @@ Here is the `Varnish Cache github page `_. +Welcome on board! + *phk* From nils.goroll at uplex.de Thu Sep 15 06:06:10 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Sep 2016 08:06:10 +0200 Subject: [master] 0a75da8 /me speling Message-ID: commit 0a75da848d1376fbfd2f678c218e68bbb0d1a898 Author: Nils Goroll Date: Thu Sep 15 08:05:29 2016 +0200 /me speling diff --git a/doc/sphinx/whats-new/relnote-5.0.rst b/doc/sphinx/whats-new/relnote-5.0.rst index 992e7ac..1976fdd 100644 --- a/doc/sphinx/whats-new/relnote-5.0.rst +++ b/doc/sphinx/whats-new/relnote-5.0.rst @@ -88,7 +88,7 @@ same companies, these developers merit personal praise: * Martin - HTTP/2 HPACK header compression code, stevedore API, VSL -* Niels & Geoff - Shard backend director, ban-lurker improvements +* Nils & Geoff - Shard backend director, ban-lurker improvements * Guillame - HTTP/2 support for varnishtest From nils.goroll at uplex.de Thu Sep 15 06:20:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Sep 2016 08:20:12 +0200 Subject: [master] 2ad850a document a non-GET caching killer and test the workaround Message-ID: commit 2ad850a87bf74c2bf54bf6143e5c2dc941b18673 Author: Nils Goroll Date: Thu Sep 15 08:13:56 2016 +0200 document a non-GET caching killer and test the workaround diff --git a/bin/varnishtest/tests/r01927.vtc b/bin/varnishtest/tests/r01927.vtc index e981521..c6769df 100644 --- a/bin/varnishtest/tests/r01927.vtc +++ b/bin/varnishtest/tests/r01927.vtc @@ -13,6 +13,7 @@ server s1 { varnish v1 -vcl+backend { sub vcl_recv { + set req.http.X-Method = req.method; # We ignore the actual body and verb for this test. return (hash); } @@ -20,7 +21,8 @@ varnish v1 -vcl+backend { return (fetch); } sub vcl_backend_fetch { - set bereq.method = "POST"; + set bereq.method = bereq.http.X-Method; + unset bereq.http.X-Method; set bereq.first_byte_timeout = 1s; } sub vcl_backend_response { diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index adc6247..b8682db 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -142,16 +142,21 @@ requests because it is questionable if GET with a body is valid anyway So the often-requested ability to cache POST/PATCH/... is now available, but not out-of-the-box: -The ``builtin.vcl`` still contains a ``return(pass)`` for anything but -a GET or HEAD because other HTTP methods, by definition, may cause -state changes / side effects on backends. The application at hand -should be understood well before caching of non-GET/non-HEAD is -considered. - -Care should be taken to choose an appropriate cache key and/or Vary -criteria. Adding the request body to the cache key is not possible -with core varnish, but through a VMOD -https://github.com/aondio/libvmod-bodyaccess +* The ``builtin.vcl`` still contains a ``return(pass)`` for anything + but a GET or HEAD because other HTTP methods, by definition, may cause + state changes / side effects on backends. The application at hand + should be understood well before caching of non-GET/non-HEAD is + considered. + +* For misses, core code still calls the equivalent of ``set + bereq.method = "GET"`` before calling ``vcl_backend_fetch``, so to + make a backend request with the original request method, it needs to + saved in ``vcl_recv`` and restored in ``vcl_backend_fetch``. + +* Care should be taken to choose an appropriate cache key and/or Vary + criteria. Adding the request body to the cache key is not possible + with core varnish, but through a VMOD + https://github.com/aondio/libvmod-bodyaccess To summarize: You should know what you are doing when caching anything but a GET or HEAD and without creating an appropriate cache key doing From nils.goroll at uplex.de Thu Sep 15 07:16:10 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Sep 2016 09:16:10 +0200 Subject: [master] e8257e1 grammar nit Message-ID: commit e8257e1c1523c666aa1add4e4f68d90b5966187b Author: Nils Goroll Date: Thu Sep 15 09:14:54 2016 +0200 grammar nit diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index b8682db..ee70fd2 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -151,7 +151,7 @@ but not out-of-the-box: * For misses, core code still calls the equivalent of ``set bereq.method = "GET"`` before calling ``vcl_backend_fetch``, so to make a backend request with the original request method, it needs to - saved in ``vcl_recv`` and restored in ``vcl_backend_fetch``. + be saved in ``vcl_recv`` and restored in ``vcl_backend_fetch``. * Care should be taken to choose an appropriate cache key and/or Vary criteria. Adding the request body to the cache key is not possible From nils.goroll at uplex.de Thu Sep 15 07:25:12 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Sep 2016 09:25:12 +0200 Subject: [master] f444878 fix reference Message-ID: commit f4448782c0a6de33d99a1ccae7f3e27b96228d38 Author: Nils Goroll Date: Thu Sep 15 09:24:32 2016 +0200 fix reference diff --git a/doc/sphinx/whats-new/changes-5.0.rst b/doc/sphinx/whats-new/changes-5.0.rst index ee70fd2..9d74be6 100644 --- a/doc/sphinx/whats-new/changes-5.0.rst +++ b/doc/sphinx/whats-new/changes-5.0.rst @@ -124,6 +124,7 @@ long it should get out of the way when it could potentially slow down lookups due to lock contention. Previously this was the same as ``ban_lurker_sleep``. +.. _whatsnew_changes_5.0_reqbody: Request Body sent always / "cacheable POST" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx/whats-new/upgrading-5.0.rst b/doc/sphinx/whats-new/upgrading-5.0.rst index 31371da..095e72b 100644 --- a/doc/sphinx/whats-new/upgrading-5.0.rst +++ b/doc/sphinx/whats-new/upgrading-5.0.rst @@ -58,9 +58,10 @@ vcl_backend_fetch * Added write access to ``bereq.body``, the request body. Only ``unset`` is supported at this time. -* We now send request bodies by default (see :ref:_whatsnew_changes_5.0). - To keep the previous behaviour add the following code before any - ``return (..)`` statement in this subroutine:: +* We now send request bodies by default (see + :ref:`whatsnew_changes_5.0_reqbody`). To keep the previous behaviour + add the following code before any ``return (..)`` statement in this + subroutine:: if (bereq.method == "GET") { unset bereq.body; From phk at FreeBSD.org Thu Sep 15 07:49:08 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 15 Sep 2016 09:49:08 +0200 Subject: [master] 98a172d Final spit&polish before release Message-ID: commit 98a172de8b7113a6746a2dcdf2fe7cfacf1c19ed Author: Poul-Henning Kamp Date: Thu Sep 15 07:47:39 2016 +0000 Final spit&polish before release diff --git a/CONTRIBUTING b/CONTRIBUTING index d76eecb..fdc94d6 100644 --- a/CONTRIBUTING +++ b/CONTRIBUTING @@ -1,21 +1,29 @@ Contributing to Varnish Cache ============================= -To contribute to Varnish, read -https://www.varnish-cache.org/trac/wiki/Contributing as a starting -point and follow relevant parts linked there. - Official development tree is here: https://github.com/varnishcache/varnish-cache -Patches can be sent to varnish-dev at varnish-cache.org. +These days we prefer patches as pull requests directly to that tree. + +Bugreports go there too. + +Our main project communication is through or developer IRC channel:: + + #varnish-hacking on server irc.linpro.no + +(That channel is not for user questions, use the #varnish channel for that.) + +Mondays at 13:00 EU time we hold our weekly "bugwash" where we go +through new (and old) tickets. It speeds things up a lot if you +can be join the channel and answer questions directly when we go +over the ticket. Github pull requests -------------------- Pull requests are handled like other tickets. -Trivial pull requests (fix typos, etc) are welcomed, but they will be commited +Trivial pull requests (fix typos, etc) are welcomed, but they may be commited by a core team member and the author credited in the commit message. - diff --git a/LICENSE b/LICENSE index 67dc00a..1fc73b8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,5 @@ Copyright (c) 2006 Verdens Gang AS -Copyright (c) 2006-2014 Varnish Software AS +Copyright (c) 2006-2016 Varnish Software AS All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.Packaging b/README.Packaging index 6663f88..baccc17 100644 --- a/README.Packaging +++ b/README.Packaging @@ -24,4 +24,4 @@ Third-party packages -------------------- Varnish Cache is built and packaged in many different operating systems and -distributions. Please see the wiki for more information. +distributions. Please see http://varnish-cache.org for more information. diff --git a/README.rst b/README.rst index 309edc9..966da29 100644 --- a/README.rst +++ b/README.rst @@ -7,9 +7,9 @@ Documentation and additional information about Varnish is available on https://www.varnish-cache.org/ Technical questions about Varnish and this release should be addressed -to . Please see -https://www.varnish-cache.org/trac/wiki/Contributing for how to -contribute patches and report bugs. +to . + +Please see CONTRIBUTING for how to contribute patches and report bugs. Questions about commercial support and services related to Varnish should be addressed to . From lkarsten at varnish-software.com Thu Sep 15 09:24:12 2016 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Thu, 15 Sep 2016 11:24:12 +0200 Subject: [master] 99d036f Prepare for 5.0.0 final. Message-ID: commit 99d036fe0b49c7487edb7dfd0da10fc2eef30505 Author: Lasse Karstensen Date: Thu Sep 15 11:16:08 2016 +0200 Prepare for 5.0.0 final. diff --git a/configure.ac b/configure.ac index 086acf1..4d101cd 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS Copyright (c) 2006-2016 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [5.0.0-beta1], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [5.0.0], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/changes.rst b/doc/changes.rst index 57a36fc..d07a404 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,3 +1,36 @@ +================================ +Varnish Cache 5.0.0 (2016-09-15) +================================ + +* Documentation updates, especially the what's new and upgrade sections. + +* Via: header made by Varnish now says 5.0. + +* VMOD VRT ABI level increased. + +* [vcl] obj.(ttl|age|grace|keep) is now readable in vcl_deliver. + +* Latest devicedetect.vcl imported from upstream. + +* New system wide VCL directory: ``/usr/share/vcl/`` + +* std.integer() can now convert from REAL. + +Issues fixed: + +* 2086_ - Ignore H2 upgrades if the feature is not enabled. +* 2054_ - Introduce new macros for out-of-tree VMODs +* 2022_ - varnishstat -1 -f field inclusion glob doesn't allow VBE backend fields +* 2008_ - Panic: Assert error in VBE_Delete() +* 1800_ - PRIV_TASK in vcl_init/fini + +.. _2086: https://www.varnish-cache.org/trac/ticket/2086 +.. _2054: https://www.varnish-cache.org/trac/ticket/2054 +.. _2022: https://www.varnish-cache.org/trac/ticket/2022 +.. _2008: https://www.varnish-cache.org/trac/ticket/2008 +.. _1800: https://www.varnish-cache.org/trac/ticket/1800 + + ====================================== Varnish Cache 5.0.0-beta1 (2016-09-09) ====================================== From dridi.boukelmoune at gmail.com Thu Sep 15 12:07:09 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 15 Sep 2016 14:07:09 +0200 Subject: [master] e3855fa Docfix Message-ID: commit e3855faedaf60ec969058d80cd7bd940c4f135f5 Author: Dridi Boukelmoune Date: Thu Sep 15 14:05:51 2016 +0200 Docfix diff --git a/doc/changes.rst b/doc/changes.rst index d07a404..dc77a7d 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -12,7 +12,7 @@ Varnish Cache 5.0.0 (2016-09-15) * Latest devicedetect.vcl imported from upstream. -* New system wide VCL directory: ``/usr/share/vcl/`` +* New system wide VCL directory: ``/usr/share/varnish/vcl/`` * std.integer() can now convert from REAL. From nils.goroll at uplex.de Thu Sep 15 15:04:41 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Sep 2016 17:04:41 +0200 Subject: [master] 860a47b Test for passing status + reason from backend to synth via deliver Message-ID: commit 860a47b05f0ac38b593ac705ccbd6c6e1cebff19 Author: Nils Goroll Date: Thu Sep 15 17:03:01 2016 +0200 Test for passing status + reason from backend to synth via deliver diff --git a/bin/varnishtest/tests/c00068.vtc b/bin/varnishtest/tests/c00068.vtc index 817ec32..08bda8d 100644 --- a/bin/varnishtest/tests/c00068.vtc +++ b/bin/varnishtest/tests/c00068.vtc @@ -10,11 +10,17 @@ server s1 { } -start varnish v1 -vcl+backend { + sub vcl_backend_response { + if (bereq.url == "/333") { + set beresp.status = 333; + set beresp.reason = "FOO"; + } + } sub vcl_deliver { if (req.url == "/332") { return (synth(332, "FOO")); } else if (req.url == "/333") { - return (synth(333, "FOO")); + return (synth(resp.status, resp.reason)); } else { return (synth(334, "BAR")); } diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc index efeabda..7fe29e3 100644 --- a/bin/varnishtest/tests/v00018.vtc +++ b/bin/varnishtest/tests/v00018.vtc @@ -4,6 +4,7 @@ varnish v1 -vcl { backend b { .host = "127.0.0.1"; } sub vcl_miss { return(synth(100,req.url)); } sub vcl_hit { return(synth(100,"the butter please")); } + sub vcl_deliver { return(synth(resp.status, resp.reason)); } } varnish v1 -errvcl {Variable 'now' is read only.} { From lkarsten at varnish-software.com Fri Sep 16 09:50:10 2016 From: lkarsten at varnish-software.com (Lasse Karstensen) Date: Fri, 16 Sep 2016 11:50:10 +0200 Subject: [master] e8420fd Be consistent with whitespace in counter descs. Message-ID: commit e8420fd1f8d5926ee0072b31d46dfda19fc12016 Author: Lasse Karstensen Date: Fri Sep 16 11:47:31 2016 +0200 Be consistent with whitespace in counter descs. The long description for MAIN.cache_hit in varnishstat looked silly with the double/triple spaces. diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h index 2a94188..6b69d07 100644 --- a/include/tbl/vsc_f_main.h +++ b/include/tbl/vsc_f_main.h @@ -58,7 +58,7 @@ VSC_F(sess_drop, uint64_t, 1, 'c', 'i', info, VSC_F(sess_fail, uint64_t, 1, 'c', 'i', info, "Session accept failures", "Count of failures to accept TCP connection." - " Either the client changed its mind, or the kernel ran out of" + " Either the client changed its mind, or the kernel ran out of" " some resource like file descriptors." ) @@ -85,24 +85,24 @@ VSC_F(client_req, uint64_t, 1, 'c', 'i', info, VSC_F(cache_hit, uint64_t, 1, 'c', 'i', info, "Cache hits", "Count of cache hits. " - " A cache hit indicates that an object has been delivered to a" - " client without fetching it from a backend server." + " A cache hit indicates that an object has been delivered to a" + " client without fetching it from a backend server." ) VSC_F(cache_hitpass, uint64_t, 1, 'c', 'i', info, - "Cache hits for pass", - "Count of hits for pass" - " A cache hit for pass indicates that Varnish is going to" - " pass the request to the backend and this decision has been " - " cached in it self. This counts how many times the cached " - " decision is being used." + "Cache hits for pass.", + "Count of hits for pass." + " A cache hit for pass indicates that Varnish is going to" + " pass the request to the backend and this decision has been" + " cached in it self. This counts how many times the cached" + " decision is being used." ) VSC_F(cache_miss, uint64_t, 1, 'c', 'i', info, "Cache misses", - "Count of misses" - " A cache miss indicates the object was fetched from the" - " backend before delivering it to the client." + "Count of misses." + " A cache miss indicates the object was fetched from the" + " backend before delivering it to the client." ) /*---------------------------------------------------------------------*/ @@ -127,16 +127,16 @@ VSC_F(backend_fail, uint64_t, 0, 'c', 'i', info, ) VSC_F(backend_reuse, uint64_t, 0, 'c', 'i', info, "Backend conn. reuses", - "Count of backend connection reuses" - " This counter is increased whenever we reuse a recycled connection." + "Count of backend connection reuses." + " This counter is increased whenever we reuse a recycled connection." ) VSC_F(backend_recycle, uint64_t, 0, 'c', 'i', info, "Backend conn. recycles", - "Count of backend connection recycles" - " This counter is increased whenever we have a keep-alive" - " connection that is put back into the pool of connections." - " It has not yet been used, but it might be, unless the backend" - " closes it." + "Count of backend connection recycles." + " This counter is increased whenever we have a keep-alive" + " connection that is put back into the pool of connections." + " It has not yet been used, but it might be, unless the backend" + " closes it." ) VSC_F(backend_retry, uint64_t, 0, 'c', 'i', info, "Backend conn. retry", @@ -200,22 +200,21 @@ VSC_F(fetch_no_thread, uint64_t, 1, 'c', 'i', info, VSC_F(pools, uint64_t, 0, 'g', 'i', info, "Number of thread pools", - "Number of thread pools. See also parameter thread_pools." - " NB: Presently pools cannot be removed once created." + "Number of thread pools. See also parameter thread_pools." + " NB: Presently pools cannot be removed once created." ) VSC_F(threads, uint64_t, 0, 'g', 'i', info, "Total number of threads", "Number of threads in all pools." - " See also parameters thread_pools, thread_pool_min and" + " See also parameters thread_pools, thread_pool_min and" " thread_pool_max." ) VSC_F(threads_limited, uint64_t, 0, 'c', 'i', info, "Threads hit max", "Number of times more threads were needed, but limit was reached" - " in a thread pool." - " See also parameter thread_pool_max." + " in a thread pool. See also parameter thread_pool_max." ) VSC_F(threads_created, uint64_t, 0, 'c', 'i', info, @@ -231,15 +230,15 @@ VSC_F(threads_destroyed, uint64_t, 0, 'c', 'i', info, VSC_F(threads_failed, uint64_t, 0, 'c', 'i', info, "Thread creation failed", "Number of times creating a thread failed." - " See VSL::Debug for diagnostics." - " See also parameter thread_fail_delay." + " See VSL::Debug for diagnostics." + " See also parameter thread_fail_delay." ) VSC_F(thread_queue_len, uint64_t, 0, 'g', 'i', info, "Length of session queue", "Length of session queue waiting for threads." - " NB: Only updates once per second." - " See also parameter thread_queue_limit." + " NB: Only updates once per second." + " See also parameter thread_queue_limit." ) VSC_F(busy_sleep, uint64_t, 1, 'c', 'i', info, @@ -263,7 +262,7 @@ VSC_F(busy_killed, uint64_t, 1, 'c', 'i', info, VSC_F(sess_queued, uint64_t, 0, 'c', 'i', info, "Sessions queued for thread", "Number of times session was queued waiting for a thread." - " See also parameter thread_queue_limit." + " See also parameter thread_queue_limit." ) VSC_F(sess_dropped, uint64_t, 0, 'c', 'i', info, @@ -500,7 +499,7 @@ VSC_F(bans_tests_tested, uint64_t, 0, 'c', 'i', diag, "Ban tests tested against objects (lookup)", "Count of how many tests and objects have been tested against" " each other during lookup." - " 'ban req.url == foo && req.http.host == bar'" + " 'ban req.url == foo && req.http.host == bar'" " counts as one in 'bans_tested' and as two in 'bans_tests_tested'" ) VSC_F(bans_lurker_tests_tested, uint64_t, 0, 'c', 'i', diag, From fgsch at lodoss.net Fri Sep 16 10:43:06 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 16 Sep 2016 12:43:06 +0200 Subject: [master] 7c3ab25 Allow duration types in boolean contexts Message-ID: commit 7c3ab2582274d3ec735b55814620fd136dbb014e Author: Federico G. Schwindt Date: Fri Sep 16 11:05:15 2016 +0100 Allow duration types in boolean contexts Polish test while here. diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc index 009f17d..7dc818b 100644 --- a/bin/varnishtest/tests/v00016.vtc +++ b/bin/varnishtest/tests/v00016.vtc @@ -78,11 +78,6 @@ varnish v1 -errvcl {Unknown time unit 'k'. Legal are 'ms', 's', 'm', 'h', 'd', sub vcl_backend_response { set beresp.ttl = 1. k; } } -varnish v1 -errvcl {Expression has type DURATION, expected BOOL} { - backend b { .host = "127.0.0.1"; } - sub vcl_backend_response { if (beresp.ttl *= 2) { } } -} - varnish v1 -errvcl {Operator > not possible on BACKEND} { backend a { .host = "127.0.0.1"; } backend b { .host = "127.0.0.1"; } diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index 3fd7e69..67388a1 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -58,14 +58,6 @@ varnish v1 -errvcl {Unknown token '-' when looking for DURATION} { } } -varnish v1 -errvcl {'&&' must be followed by BOOL, found DURATION.} { - sub vcl_recv { - if (req.ttl < 3s && req.ttl) { - set req.http.foo = vcl_recv; - } - } -} - varnish v1 -errvcl {Operator * not possible on type STRING.} { sub vcl_recv { set req.http.foo = "bla" * "foo"; @@ -78,13 +70,6 @@ varnish v1 -errvcl {DURATION + INT not possible.} { } } -varnish v1 -errvcl {'!' must be followed by BOOL, found DURATION.} { - sub vcl_backend_response { - if (! req.ttl) { - } - } -} - varnish v1 -errvcl {BOOL + BOOL not possible.} { sub vcl_backend_response { if (beresp.do_gzip + beresp.do_gunzip) { @@ -124,16 +109,11 @@ varnish v1 -vcl { set req.http.foo = req.http.foo + "bar" !~ "bar"; set req.ttl = 1s; - } -} + if (req.ttl) { } + if (!req.ttl) { } -varnish v1 -vcl { - import std; - backend b { .host = "127.0.0.1"; } - sub vcl_recv { - if (std.integer("1", 1)) { - } + if (1) { } } } @@ -251,22 +231,6 @@ varnish v1 -errvcl {DURATION + STRING not possible.} { } } -varnish v1 -errvcl {'||' must be followed by BOOL, found DURATION.} { - backend b { .host = "127.0.0.1"; } - sub vcl_backend_response { - if (req.url || beresp.ttl) { - } - } -} - -varnish v1 -errvcl {'&&' must be followed by BOOL, found DURATION.} { - backend b { .host = "127.0.0.1"; } - sub vcl_backend_response { - if (req.url && beresp.ttl) { - } - } -} - varnish v1 -vcl { backend b { .host = "127.0.0.1"; } sub vcl_miss { diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index ca77e8d..4a67afd 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -93,7 +93,8 @@ their value. String types will evaluate to *false* if they are empty; backend types will evalute to *false* if they don't have a backend assigned; integer -types will evaluate to *false* if their value is zero. +types will evaluate to *false* if their value is zero; duration types +will evaluate to *false* if their value is equal or less than zero. Time ~~~~ diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 7d594cf..b1ed848 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1200,12 +1200,12 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, vcc_type_t fmt) default: break; } - if (fmt == BOOL && - ((*e)->fmt == STRING || (*e)->fmt == BACKEND || - (*e)->fmt == INT)) { - *e = vcc_expr_edit(BOOL, "(\v1 != 0)", *e, NULL); + if (fmt != BOOL) return; - } + if ((*e)->fmt == STRING || (*e)->fmt == BACKEND || (*e)->fmt == INT) + *e = vcc_expr_edit(BOOL, "(\v1 != 0)", *e, NULL); + else if ((*e)->fmt == DURATION) + *e = vcc_expr_edit(BOOL, "(\v1 > 0)", *e, NULL); } /*-------------------------------------------------------------------- From varnish-commit at varnish-cache.org Sat Sep 17 22:10:19 2016 From: varnish-commit at varnish-cache.org (jy) Date: Sun, 18 Sep 2016 06:10:19 +0800 Subject: =?utf-8?B?dmFybmlzaC1jb21taXQ65aaC5L2V5aSE55CG5ZGY?= =?utf-8?B?5bel6L+d57qq6Zeu6aKYPyB0b2U5eWM=?= Message-ID: <20160918061024703164@ilowdda.org> varnish-commit: ?? 1.???????????????????????????? 2.?????????????????? 3.???????????????????????? 4.?????????????????????????? 5.?????????????? 6.?????????????????????????? 7.?????????????????????????? 8.??????????????????? 9.??????????????????????? ??????????????????? ???????????? 2016/9/18 ???6:10:23 ??? -------------- next part -------------- An HTML attachment was scrubbed... URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: ??????????????????????.docx Type: application/vnd.openxmlformats-officedocument.wordprocessingml.document Size: 27467 bytes Desc: not available URL: From fgsch at lodoss.net Mon Sep 19 07:41:08 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 19 Sep 2016 09:41:08 +0200 Subject: [master] db326b8 Polish Message-ID: commit db326b812a5c583808f58e4bb841cc06fe20df88 Author: Federico G. Schwindt Date: Mon Sep 19 08:29:08 2016 +0100 Polish diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index a76fa78..6354d69 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -140,15 +140,11 @@ vmod_log(VRT_CTX, const char *s, ...) va_list ap; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - WS_Assert(ctx->ws); - - u = WS_Reserve(ctx->ws, 0); t.b = ctx->ws->f; va_start(ap, s); t.e = VRT_StringList(ctx->ws->f, u, s, ap); va_end(ap); - if (t.e != NULL) { assert(t.e > t.b); t.e--; @@ -168,17 +164,14 @@ vmod_syslog(VRT_CTX, VCL_INT fac, const char *fmt, ...) txt t; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + u = WS_Reserve(ctx->ws, 0); + t.b = ctx->ws->f; va_start(ap, fmt); - if (ctx->ws != NULL) { - u = WS_Reserve(ctx->ws, 0); - t.b = ctx->ws->f; - t.e = VRT_StringList(ctx->ws->f, u, fmt, ap); - if (t.e != NULL) - syslog((int)fac, "%s", t.b); - WS_Release(ctx->ws, 0); - } else - vsyslog((int)fac, fmt, ap); + t.e = VRT_StringList(ctx->ws->f, u, fmt, ap); va_end(ap); + if (t.e != NULL) + syslog((int)fac, "%s", t.b); + WS_Release(ctx->ws, 0); } VCL_VOID __match_proto__(td_std_collect) From phk at FreeBSD.org Mon Sep 19 11:00:16 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 19 Sep 2016 13:00:16 +0200 Subject: [master] c52262a At least make the FreeBSD instructions do something. Message-ID: commit c52262ac7323a9783f76c658549b377a65fa556e Author: Poul-Henning Kamp Date: Mon Sep 19 10:59:18 2016 +0000 At least make the FreeBSD instructions do something. diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst index 8e15a9a..1c5668c 100644 --- a/doc/sphinx/installation/install.rst +++ b/doc/sphinx/installation/install.rst @@ -23,9 +23,9 @@ FreeBSD ------- Binary package: - ``pkg_add -r varnish`` + ``pkg_add -r varnish4`` From source: - ``cd /usr/ports/varnish && make install clean`` + ``cd /usr/ports/www/varnish4 && make install clean`` Red Hat / CentOS ---------------- From dridi.boukelmoune at gmail.com Mon Sep 19 18:11:06 2016 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 19 Sep 2016 20:11:06 +0200 Subject: [master] 6dc1d5b Add new VARNISH_{TEST,LIBRARY}_PATH variables Message-ID: commit 6dc1d5bb1d3f43262ee75eff10b06ee6921f281b Author: Dridi Boukelmoune Date: Mon Sep 19 19:12:32 2016 +0200 Add new VARNISH_{TEST,LIBRARY}_PATH variables The VARNISH_TEST_PATH variable contains the minimal PATH for running a test suite, covering only Varnish tools. And VARNISH_LIBRARY_PATH is needed because Varnish binaries aren't build using an RPATH and only know about their libraries out of the box when they are installed in the system path, via ldconfig. Consequently VMOD_TEST_PATH is now an alias for VARNISH_TEST_PATH. Catching this problem after the 5.0 release is unfortunate, as it won't be ported to 4.1 to avoid breaking the upgrade path between the 4.1 series and the 5.0.0 release. Unless it is considered OK? Do people really read through the whole commit message and diff when it concerns autotools changes? While at it, make a pass on the autoconf documentation. Flatten code snippets that don't really need line breaks. Add the man page to the dist tarball by default. Walk through a typical test suite setup. Document variables created or substituted by AC_PREREQ. Document how the purpose and requirements of varnish.m4, and document what changes between versions of Varnish. Consequently bump the serial number. diff --git a/varnish.m4 b/varnish.m4 index d0849ef..5c5cab1 100644 --- a/varnish.m4 +++ b/varnish.m4 @@ -1,6 +1,3 @@ -# varnish.m4 - Macros to define VMOD builds. -*- Autoconf -*- -# serial 5 (varnish-4.1.4) - # Copyright (c) 2016 Varnish Software AS # All rights reserved. # @@ -31,6 +28,20 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED # OF THE POSSIBILITY OF SUCH DAMAGE. +# varnish.m4 - Macros to define VMOD builds. -*- Autoconf -*- +# serial 6 (varnish-5.0.1) +# +# This collection of macros helps create VMODs or tools interacting with +# Varnish Cache using the GNU build system (autotools). In order to work +# from a source checkout, recommended versions of autotools are 2.68 for +# autoconf, 1.12 for automake and 2.2.6 for libtool. For pkg-config, at +# least version 0.21 is required ; it should be available even on old +# platforms. Only pkg-config is needed when building from a dist archive. +# +# Macros whose name start with an underscore are private and may change at +# any time. Public macros starting with VARNISH_ are documented and will +# maintain backwards compatibility with older versions of Varnish Cache. + # _VARNISH_PKG_CONFIG # -------------------- AC_DEFUN([_VARNISH_PKG_CONFIG], [ @@ -41,11 +52,18 @@ AC_DEFUN([_VARNISH_PKG_CONFIG], [ PKG_CHECK_VAR([VARNISHAPI_PREFIX], [varnishapi], [prefix]) PKG_CHECK_VAR([VARNISHAPI_DATAROOTDIR], [varnishapi], [datarootdir]) + PKG_CHECK_VAR([VARNISHAPI_LIBDIR], [varnishapi], [libdir]) PKG_CHECK_VAR([VARNISHAPI_BINDIR], [varnishapi], [bindir]) PKG_CHECK_VAR([VARNISHAPI_SBINDIR], [varnishapi], [sbindir]) PKG_CHECK_VAR([VARNISHAPI_VMODDIR], [varnishapi], [vmoddir]) PKG_CHECK_VAR([VMODTOOL], [varnishapi], [vmodtool]) + + AC_SUBST([VARNISH_LIBRARY_PATH], + [$VARNISHAPI_LIBDIR:$VARNISHAPI_LIBDIR/varnish]) + + AC_SUBST([VARNISH_TEST_PATH], + [$VARNISHAPI_SBINDIR:$VARNISHAPI_BINDIR:$PATH]) ]) # _VARNISH_CHECK_DEVEL @@ -113,9 +131,8 @@ AC_DEFUN([_VARNISH_VMOD_CONFIG], [ AC_SUBST([VMOD_LDFLAGS], "-module -export-dynamic -avoid-version -shared") - dnl Define the PATH for the test suite - AC_SUBST([VMOD_TEST_PATH], - [$VARNISHAPI_SBINDIR:$VARNISHAPI_BINDIR:$PATH]) + dnl Substitute an alias for compatibility reasons + AC_SUBST([VMOD_TEST_PATH], [$VARNISH_TEST_PATH]) ]) # _VARNISH_VMOD(NAME) @@ -166,7 +183,7 @@ clean-vmod-$1: # to build the modules: # # - VMOD_LDFLAGS (the recommended flags to link VMODs) -# - VMOD_TEST_PATH (for the test suite's environment) +# - VMOD_TEST_PATH (an alias for VARNISH_TEST_PATH) # - VMODTOOL (to generate a VMOD's interface) # - vmoddir (the install prefix for VMODs) # @@ -201,18 +218,13 @@ clean-vmod-$1: # sub-directories. You still need to declare the generated VCC interfaces # in your library's sources. The generated files can be declared this way: # -# nodist_libvmod_foo_la_SOURCES = \ -# vcc_foo_if.c \ -# vcc_foo_if.h -# -# nodist_libvmod_bar_la_SOURCES = \ -# vcc_bar_if.c \ -# vcc_bar_if.h +# nodist_libvmod_foo_la_SOURCES = vcc_foo_if.c vcc_foo_if.h +# nodist_libvmod_bar_la_SOURCES = vcc_bar_if.c vcc_bar_if.h # # The generated rules also build the manual page, all you need to do is to # declare the generated pages: # -# nodist_man_MANS = vmod_foo.3 vmod_bar.3 +# dist_man_MANS = vmod_foo.3 vmod_bar.3 # # However, it requires RST2MAN to be defined beforehand in configure.ac # and it is for now the VMOD's maintainer job to manage it. On the other @@ -235,6 +247,47 @@ clean-vmod-$1: # VMOD maintenance, initialization of autoconf, automake and libtool is # still the maintainer's responsibility. It cannot be avoided. # +# Once your VMOD is built, you can use varnishtest to run test cases. For +# that you can rely on automake's default test driver, and all you need +# is a minimal setup: +# +# AM_TESTS_ENVIRONMENT = \ +# PATH="$(VARNISH_TEST_PATH)" \ +# LD_LIBRARY_PATH="$(VARNISH_LIBRARY_PATH)" +# TEST_EXTENSIONS = .vtc +# VTC_LOG_COMPILER = varnishtest -v +# AM_VTC_LOG_FLAGS = -Dvmod_foo="$(VMOD_FOO)" -Dvmod_bar="$(VMOD_BAR)" +# +# Setting up the different paths is mostly relevant when you aren't building +# against the system installation of Varnish. In the case of the PATH, you +# may also need to preserve the original PATH if you run commands outside of +# the Varnish distribution in your test cases. +# +# The $(VMOD_*) variables contain a proper import statement if the relevant +# VMOD was built in the same directory as the test runner. With the example +# above you could import VMODs this way in a test case: +# +# varnish v1 -vcl+backend { +# import std; +# import ${vmod_bar}; +# +# [...] +# } -start +# +# Once your test suite is set up, all you need is to do is declare your test +# cases and `make check` will work out of the box. +# +# TESTS = +# +# At this point almost everything is taken care of, and your autotools-based +# build is ready for prime time. However if you want your VMODs to build and +# run the test suite from a dist archive, don't forget to embed your VCC +# file and the test cases: +# +# EXTRA_DIST = vmod_foo.vcc vmod_bar.vcc $(TESTS) +# +# Now, you can focus on writing this VMOD of yours. +# AC_DEFUN([VARNISH_VMODS], [ m4_foreach([_vmod_name], m4_split(m4_normalize([$1])), @@ -245,11 +298,34 @@ AC_DEFUN([VARNISH_VMODS], [ # -------------------------------------------------- # Since: Varnish 4.1.4 # +# Since Varnish 5.0.1: +# - VARNISH_TEST_PATH added +# - VARNISH_LIBRARY_PATH added +# - VARNISHAPI_LIBDIR added +# # Verify that the version of Varnish Cache found by pkg-config is at least # MINIMUM-VERSION. If MAXIMUM-VERSION is specified, verify that the version # is strictly below MAXIMUM-VERSION. # -# If the prerequisite is met, the variable VARNISH_VERSION is available. +# Once the requirements are met, the following variables can be used in +# Makefiles: +# +# - VARNISH_TEST_PATH (for the test suite environment) +# - VARNISH_LIBRARY_PATH (for both public and private libraries) +# - VARNISH_VERSION (also available in autoconf) +# +# The following variables are available in autoconf, read from the varnish +# pkg-config: +# +# - VARNISHAPI_CFLAGS +# - VARNISHAPI_LIBS +# - VARNISHAPI_PREFIX +# - VARNISHAPI_DATAROOTDIR +# - VARNISHAPI_LIBDIR +# - VARNISHAPI_BINDIR +# - VARNISHAPI_SBINDIR +# - VARNISHAPI_VMODDIR +# - VMODTOOL # AC_DEFUN([VARNISH_PREREQ], [ AC_REQUIRE([_VARNISH_PKG_CONFIG]) From fgsch at lodoss.net Mon Sep 19 22:59:04 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 20 Sep 2016 00:59:04 +0200 Subject: [master] 0da5eeb GC vsyslog check Message-ID: commit 0da5eeba19c2c14d8786c15242ff7f1c5ae56386 Author: Federico G. Schwindt Date: Mon Sep 19 18:56:44 2016 +0100 GC vsyslog check diff --git a/configure.ac b/configure.ac index 4d101cd..eac833e 100644 --- a/configure.ac +++ b/configure.ac @@ -239,7 +239,6 @@ AC_CHECK_FUNCS([nanosleep]) AC_CHECK_FUNCS([setppriv]) AC_CHECK_FUNCS([fallocate]) AC_CHECK_FUNCS([closefrom]) -AC_CHECK_FUNCS([vsyslog]) save_LIBS="${LIBS}" LIBS="${PTHREAD_LIBS}" From fgsch at lodoss.net Mon Sep 19 22:59:05 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 20 Sep 2016 00:59:05 +0200 Subject: [master] 0fbc5f7 Sync description with reality Message-ID: commit 0fbc5f7a2623d1172224f640e9cea31f02cdc097 Author: Federico G. Schwindt Date: Mon Sep 19 19:15:17 2016 +0100 Sync description with reality diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 011541b..339675e 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -69,8 +69,8 @@ SLTM(SessOpen, 0, "Client connection opened", "\t%s %d %s %s %s %d\n" "\t| | | | | |\n" "\t| | | | | +- File descriptor number\n" - "\t| | | | +---- Local TCP port ('-' if !$log_local_addr)\n" - "\t| | | +------- Local IPv4/6 address ('-' if !$log_local_addr)\n" + "\t| | | | +---- Local TCP port\n" + "\t| | | +------- Local IPv4/6 address\n" "\t| | +---------- Listen socket (-a argument)\n" "\t| +------------- Remote TCP port\n" "\t+---------------- Remote IPv4/6 address\n" From fgsch at lodoss.net Mon Sep 19 22:59:05 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 20 Sep 2016 00:59:05 +0200 Subject: [master] d8cf732 Typos and cosmetic Message-ID: commit d8cf732d4f1562a69279b5840e74a156b60ff5ab Author: Federico G. Schwindt Date: Mon Sep 19 23:55:24 2016 +0100 Typos and cosmetic One submitted by github::bobmshannon. diff --git a/doc/sphinx/whats-new/relnote-5.0.rst b/doc/sphinx/whats-new/relnote-5.0.rst index 1976fdd..71b895e 100644 --- a/doc/sphinx/whats-new/relnote-5.0.rst +++ b/doc/sphinx/whats-new/relnote-5.0.rst @@ -4,15 +4,15 @@ Varnish 5.0 Release Note ======================== This is the first Varnish release after the Varnish Project moved out -of Varnish Softwares basement, so to speak, and it shows. +of Varnish Software's basement, so to speak, and it shows. -But it is also our 10 year aniversary release, `Varnish 1.0 was +But it is also our 10 year anniversary release, `Varnish 1.0 was released`_ on September 20th 2006. That also means that we have been doing this for 10 years without any bad security holes. -So yeah? 5.0 is not entirely what we had hoped it would be, but we +So yeah... 5.0 is not entirely what we had hoped it would be, but we are as proud as one can possibly be anyway. To keep this release note short(er), we have put the purely technical @@ -54,7 +54,7 @@ here and there, See :ref:`whatsnew_upgrading_5.0` for more of this. Reasons not to upgrade to Varnish 5.0 ------------------------------------- -None that we know of af this time. +None that we know of at this time. Only in very special cases should you need to modify your VCL. @@ -118,7 +118,7 @@ the developers mortgages and student loans. A very large part of the Varnish development is funded through the `Varnish Moral License`_, which enables Poul-Henning Kamp to have Varnish as his primary job, but right now he is underfunded to the -tune of ?2000-3000 per month. +tune of EUR 2000-3000 per month. Please consider if your company makes enough money using Varnish Cache, to spare some money, or employee-hours for its future From phk at FreeBSD.org Wed Sep 21 06:25:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 21 Sep 2016 08:25:05 +0200 Subject: [master] 7ded638 Insert an artificial pause in the H2 preface, at the point where it makes a valid H1 request header. Message-ID: commit 7ded6385b9b3ddc67b340a5f26e0de55adac8a14 Author: Poul-Henning Kamp Date: Wed Sep 21 06:06:53 2016 +0000 Insert an artificial pause in the H2 preface, at the point where it makes a valid H1 request header. diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index b2a021f..13671cd 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1678,7 +1678,7 @@ cmd_http_fatal(CMD_ARGS) * Same as for the top-level barrier */ -const char PREFACE[] = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"; +const char PREFACE[24] = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"; /* SECTION: client-server.spec.txpri txpri (client) * @@ -1694,11 +1694,18 @@ cmd_http_txpri(CMD_ARGS) CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_CLIENT(hp, av); - vtc_dump(hp->vl, 4, "txpri", PREFACE, sizeof(PREFACE) - 1); - l = write(hp->fd, PREFACE, sizeof(PREFACE) - 1); - if (l != sizeof(PREFACE) - 1) + vtc_dump(hp->vl, 4, "txpri", PREFACE, sizeof(PREFACE)); + /* Dribble out the preface */ + l = write(hp->fd, PREFACE, 18); + if (l != 18) vtc_log(vl, hp->fatal, "Write failed: (%zd vs %zd) %s", - l, sizeof(PREFACE) - 1, strerror(errno)); + l, sizeof(PREFACE), strerror(errno)); + usleep(10000); + l = write(hp->fd, PREFACE + 18, sizeof(PREFACE) - 18); + if (l != sizeof(PREFACE) - 18) + vtc_log(vl, hp->fatal, "Write failed: (%zd vs %zd) %s", + l, sizeof(PREFACE), strerror(errno)); + start_h2(hp); AN(hp->h2); } @@ -1717,9 +1724,9 @@ cmd_http_rxpri(CMD_ARGS) ONLY_SERVER(hp, av); hp->prxbuf = 0; - if (!http_rxchar(hp, sizeof(PREFACE) - 1, 0)) + if (!http_rxchar(hp, sizeof(PREFACE), 0)) vtc_log(vl, 0, "Couldn't retrieve connection preface"); - if (strncmp(hp->rxbuf, PREFACE, sizeof(PREFACE) - 1)) + if (memcmp(hp->rxbuf, PREFACE, sizeof(PREFACE))) vtc_log(vl, 0, "Received invalid preface\n"); start_h2(hp); AN(hp->h2); From phk at FreeBSD.org Wed Sep 21 06:25:05 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 21 Sep 2016 08:25:05 +0200 Subject: [master] dbcbae2 Do not insist the H2 connection preface arrives in a single packet. Message-ID: commit dbcbae227b757043651ee1e7d1cc729727f94c8d Author: Poul-Henning Kamp Date: Wed Sep 21 06:22:55 2016 +0000 Do not insist the H2 connection preface arrives in a single packet. Fixes: #2094 Fixes: #2096 diff --git a/bin/varnishd/http1/cache_http1_proto.c b/bin/varnishd/http1/cache_http1_proto.c index b5273c6..e788ed6 100644 --- a/bin/varnishd/http1/cache_http1_proto.c +++ b/bin/varnishd/http1/cache_http1_proto.c @@ -46,6 +46,8 @@ #include "config.h" #include "cache/cache.h" +#include "cache/cache_transport.h" + #include "cache_http1.h" #include "vct.h" @@ -66,6 +68,7 @@ enum htc_status_e __match_proto__(htc_complete_f) HTTP1_Complete(struct http_conn *htc) { char *p; + enum htc_status_e retval; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); @@ -78,6 +81,11 @@ HTTP1_Complete(struct http_conn *htc) if (p == htc->rxbuf_e) return (HTC_S_EMPTY); + /* Do not return a partial H2 connection preface */ + retval = H2_prism_complete(htc); + if (retval != HTC_S_JUNK) + return (retval); + /* * Here we just look for NL[CR]NL to see that reception * is completed. More stringent validation happens later. diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 73d0c95..54b8a2e 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -474,10 +474,11 @@ H2_prism_complete(struct http_conn *htc) CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); l = htc->rxbuf_e - htc->rxbuf_b; - if (l < strlen(H2_prism)) - return (HTC_S_MORE); - if (!memcmp(htc->rxbuf_b, H2_prism, sizeof(H2_prism))) + if (l >= sizeof(H2_prism) && + !memcmp(htc->rxbuf_b, H2_prism, sizeof(H2_prism))) return (HTC_S_COMPLETE); + if (l < sizeof(H2_prism) && !memcmp(htc->rxbuf_b, H2_prism, l)) + return (HTC_S_MORE); return (HTC_S_JUNK); } From phk at FreeBSD.org Wed Sep 21 09:04:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 21 Sep 2016 11:04:04 +0200 Subject: [master] e037b90 Flexelinting Message-ID: commit e037b90c33e42b6d9d8cd7ccaa6b37870fca9955 Author: Poul-Henning Kamp Date: Wed Sep 21 07:47:29 2016 +0000 Flexelinting diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index a4aee4a..a927cb1 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -387,7 +387,7 @@ VSLb_bin(struct vsl_log *vsl, enum VSL_tag_e tag, ssize_t len, const void *ptr) size_t ll; assert(len >= 0); - AN(ptr); + AN(pp); if (vsl_tag_is_masked(tag)) return; vsl_sanity(vsl); @@ -501,7 +501,7 @@ VSM_Init(void) vsl_end = vsl_head->log + vsl_segsize * VSL_SEGMENTS; /* Make segment_n always overflow on first log wrap to make any problems with regard to readers on that event visible */ - vsl_segment_n = UINT_MAX - VSL_SEGMENTS + 1; + vsl_segment_n = UINT_MAX - (VSL_SEGMENTS - 1); AZ(vsl_segment_n % VSL_SEGMENTS); vsl_head->segment_n = vsl_segment_n; vsl_ptr = vsl_head->log; From phk at FreeBSD.org Wed Sep 21 09:04:04 2016 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 21 Sep 2016 11:04:04 +0200 Subject: [master] 2889a83 More Flexelinting Message-ID: commit 2889a83de6369ab7b7deafb2e7c6445dac8a4c8b Author: Poul-Henning Kamp Date: Wed Sep 21 09:03:18 2016 +0000 More Flexelinting diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index 0baec27..7076cbb 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -116,6 +116,8 @@ H2_Send(struct worker *wrk, struct h2_req *r2, int flush, retval = H2_Send_Frame(wrk, h2, type, flags, len, r2->stream, ptr); } else if (type == H2_FRAME_DATA) { + AN(ptr); + AN(len); p = ptr; do { tf = mfs; From nils.goroll at uplex.de Thu Sep 22 13:08:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 22 Sep 2016 15:08:05 +0200 Subject: [master] 6487059 document and test bereq.backend vs. beresp.backend vs. beresp.backend.name Message-ID: commit 64870592cf64eaaca645b607c818e999b2f5a506 Author: Nils Goroll Date: Thu Sep 22 14:46:55 2016 +0200 document and test bereq.backend vs. beresp.backend vs. beresp.backend.name diff --git a/bin/varnishtest/tests/v00025.vtc b/bin/varnishtest/tests/v00025.vtc index af7b7eb..62c51e6 100644 --- a/bin/varnishtest/tests/v00025.vtc +++ b/bin/varnishtest/tests/v00025.vtc @@ -8,9 +8,16 @@ server s1 { varnish v1 -arg "-i J.F.Nobody" -vcl+backend { import std; + import directors; + + sub vcl_init { + new rr = directors.round_robin(); + rr.add_backend(s1); + } sub vcl_recv { set client.identity = "Samuel B. Nobody"; + set req.backend_hint = rr.backend(); } sub vcl_deliver { @@ -32,7 +39,8 @@ varnish v1 -arg "-i J.F.Nobody" -vcl+backend { } sub vcl_backend_response { - set beresp.http.backend = bereq.backend; + set beresp.http.bereq_backend = bereq.backend; + set beresp.http.beresp_backend = beresp.backend; set beresp.http.keep = beresp.keep; set beresp.http.hint = beresp.storage_hint; set beresp.http.be_ip = beresp.backend.ip; @@ -73,6 +81,13 @@ varnish v1 -arg "-i J.F.Nobody" -vcl+backend { client c1 { txreq rxresp + expect resp.status == 200 + expect resp.http.bereq_backend == "rr" + expect resp.http.beresp_backend == "s1" + expect resp.http.be_ip == "127.0.0.1" + expect resp.http.be_nm == "s1" + expect resp.http.be == "rr" + txreq rxresp } -run diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index fb167b4..92bed74 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -299,6 +299,11 @@ sp_variables = [ ('client', ), ('client',), """ Set bereq.backend to this if we attempt to fetch. + When set to a director, reading this variable returns + an actual backend if the director has resolved immediately, + or the director otherwise. + When used in string context, returns the name of the director + or backend, respectively. """ ), ('req.hash_ignore_busy', @@ -380,6 +385,11 @@ sp_variables = [ ('pipe', 'backend', ), ('pipe', 'backend', ), """ This is the backend or director we attempt to fetch from. + When set to a director, reading this variable returns + an actual backend if the director has resolved immediately, + or the director otherwise. + When used in string context, returns the name of the director + or backend, respectively. """ ), ('bereq.body', @@ -592,6 +602,7 @@ sp_variables = [ This is the backend we fetched from. If bereq.backend was set to a director, this will be the backend selected by the director. + When used in string context, returns its name. """ ), ('beresp.backend.name', @@ -599,6 +610,7 @@ sp_variables = [ ('backend_response', 'backend_error'), (), """ Name of the backend this response was fetched from. + Same as beresp.backend. """ ), ('beresp.backend.ip', From nils.goroll at uplex.de Thu Sep 22 14:25:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 22 Sep 2016 16:25:05 +0200 Subject: [master] 16b3890 Fix "Wrong enum value" VCC Error Message-ID: commit 16b389072469dc68eeed1d61df2ab44ce4b4eea8 Author: Nils Goroll Date: Thu Sep 22 16:23:23 2016 +0200 Fix "Wrong enum value" VCC Error Since a78efad8002895e6097aeb6b1daeac0f6108b9a9 the argument name and default value are separated from the argument spec by a \1, so we need to stop there. diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index b1ed848..250a12e 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -555,7 +555,7 @@ vcc_do_arg(struct vcc *tl, struct func_arg *fa) do { VSB_printf(tl->sb, "\t%s\n", r); r += strlen(r) + 1; - } while (*r != '\0'); + } while (*r != '\0' && *r != '\1'); vcc_ErrWhere(tl, tl->t); return; } From varnish-commit at varnish-cache.org Fri Sep 23 07:53:45 2016 From: varnish-commit at varnish-cache.org (famwr) Date: Fri, 23 Sep 2016 15:53:45 +0800 Subject: =?utf-8?B?LS12YXJuaXNoLWNvbW1pdO+8muWmguS9leWung==?= =?utf-8?B?546w5Z+65LqO5Lqn5ZOB5bmz5Y+w5Y675byA5Y+R5Lqn5ZOB77yf?= Message-ID: <20160923155350280802@itafxxbu.com> varnish-commit: ??? ???????????????????????????????/????????????????????????????? ??????????????????????????????? ?????????????????? ??????????? ??????????? ??????CBB??????????? ????????????????????????????????? ???? ?????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? ??????????????? 2016/9/23 ???15:53:48 -------------- next part -------------- An HTML attachment was scrubbed... URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: ???????????James.doc Type: application/msword Size: 107008 bytes Desc: not available URL: From fgsch at lodoss.net Sat Sep 24 20:02:04 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 24 Sep 2016 22:02:04 +0200 Subject: [master] c7e56c4 Random polishing Message-ID: commit c7e56c424679e0ca63ab90c061defc85822bc56c Author: Federico G. Schwindt Date: Sat Sep 24 20:58:00 2016 +0100 Random polishing diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index 67388a1..9c42c55 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -22,18 +22,6 @@ varnish v1 -errvcl {Found: '0' at} { 0; } varnish v1 -errvcl {Expected an action, 'if', } " sub vcl_recv { { } { " -varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } - sub vcl_backend_response { - set beresp.ttl = 1w; - set beresp.ttl *= 1.5; - set beresp.ttl = 1.5 s * 2.5; - set beresp.ttl = 1.5 s / 2.5; - set beresp.ttl = 1.5h + 1.5s; - set beresp.ttl = 1.5h - 1.5s; - } -} - varnish v1 -errvcl {Comparison of different types: INT '!=' STRING} { sub vcl_recv { if (!req.restarts != req.url) { @@ -109,10 +97,17 @@ varnish v1 -vcl { set req.http.foo = req.http.foo + "bar" !~ "bar"; set req.ttl = 1s; + set req.ttl *= 1.5; + set req.ttl = 1.5 s * 2.5; + set req.ttl = 1.5 s / 2.5; + set req.ttl = 1.5h + 1.5s; + set req.ttl = 1.5h - 1.5s; if (req.ttl) { } if (!req.ttl) { } + if (!req.url) { } + if (1) { } } } @@ -231,14 +226,6 @@ varnish v1 -errvcl {DURATION + STRING not possible.} { } } -varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } - sub vcl_miss { - if (!req.url) { - } - } -} - varnish v1 -errvcl {Name of function, 'foo-bar', contains illegal character '-'} { backend b { .host = "127.0.0.1"; } sub foo-bar { diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 4a67afd..0a63f7f 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -100,8 +100,8 @@ Time ~~~~ VCL has time. A duration can be added to a time to make another time. -In string context they return a formatted string in RFC1123 format -(e.g. Sun, 06 Nov 1994 08:49:37 GMT). +In string context they return a formatted string in RFC1123 format, +e.g. Sun, 06 Nov 1994 08:49:37 GMT. The keyword *now* returns a time representing the current time in seconds since the Epoch. @@ -109,8 +109,8 @@ since the Epoch. Durations ~~~~~~~~~ -Durations are defined by a number and a designation. The number can be a real -so 1.5w is allowed. +Durations are defined by a number followed by an unit. The number can +include a fractional part, e.g. 1.5s. The supported units are: ms milliseconds From varnish-commit at varnish-cache.org Sun Sep 25 18:33:50 2016 From: varnish-commit at varnish-cache.org (nwaw) Date: Mon, 26 Sep 2016 02:33:50 +0800 Subject: =?utf-8?B?LS12YXJuaXNoLWNvbW1pdO+8muWmguS9leWung==?= =?utf-8?B?546w5Z+65LqO5Lqn5ZOB5bmz5Y+w5Y675byA5Y+R5Lqn5ZOB77yf?= Message-ID: <20160926023404236005@bvuveyu.com> varnish-commit: ??? ???????????????????????????????/????????????????????????????? ??????????????????????????????? ?????????????????? ??????????? ??????????? ??????CBB??????????? ????????????????????????????????? ???? ?????????????????????????????????????????????????????????????????????????????????????????????????????????????????????? ??????????????? 2016/9/26 ???2:34:03 -------------- next part -------------- An HTML attachment was scrubbed... URL: From guillaume at varnish-software.com Mon Sep 26 09:45:05 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Mon, 26 Sep 2016 11:45:05 +0200 Subject: [master] 6b85eea Revert "Check malloc return" Message-ID: commit 6b85eeae6425b4be7a68418494cd92ffaaeadeb8 Author: Guillaume Quintard Date: Mon Sep 26 11:39:41 2016 +0200 Revert "Check malloc return" This reverts commit 6da0cfdae50c64dbe4af050fc0c19c78a496cd10. That was a stupid commit, the malloc failure is already handled Spotted by: @scn diff --git a/lib/libvarnish/vcli_proto.c b/lib/libvarnish/vcli_proto.c index 3c6ede8..27e2104 100644 --- a/lib/libvarnish/vcli_proto.c +++ b/lib/libvarnish/vcli_proto.c @@ -168,7 +168,6 @@ VCLI_ReadResult(int fd, unsigned *status, char **ptr, double tmo) *status = u; p = malloc(v + 1L); - AN(p); if (p == NULL) break; From guillaume at varnish-software.com Mon Sep 26 09:45:05 2016 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Mon, 26 Sep 2016 11:45:05 +0200 Subject: [master] 3684858 No check needed, free(NULL) is a nop Message-ID: commit 3684858d9e7275326a02521d4895be368b70400b Author: Guillaume Quintard Date: Mon Sep 26 11:43:20 2016 +0200 No check needed, free(NULL) is a nop diff --git a/lib/libvarnish/vcli_proto.c b/lib/libvarnish/vcli_proto.c index 27e2104..7e52fb2 100644 --- a/lib/libvarnish/vcli_proto.c +++ b/lib/libvarnish/vcli_proto.c @@ -187,8 +187,7 @@ VCLI_ReadResult(int fd, unsigned *status, char **ptr, double tmo) return (0); } while(0); - if (p != NULL) - free(p); + free(p); *status = CLIS_COMMS; if (ptr != NULL) *ptr = strdup(err); From fgsch at lodoss.net Mon Sep 26 21:34:04 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 26 Sep 2016 23:34:04 +0200 Subject: [master] be24ee1 More cleanup Message-ID: commit be24ee1e2f6ca8914a239df7e7739ebbf0a5edef Author: Federico G. Schwindt Date: Sun Sep 25 15:28:00 2016 +0100 More cleanup diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc index 7dc818b..2c3c232 100644 --- a/bin/varnishtest/tests/v00016.vtc +++ b/bin/varnishtest/tests/v00016.vtc @@ -19,16 +19,6 @@ varnish v1 -errvcl {include not followed by string constant.} { include << } -varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } - sub vcl_recv { - if (req.url == "foo") { - } elsif (req.url == "bar") { - } elseif (req.url == "baz") { - } - } -} - varnish v1 -errvcl {include not followed by string constant.} { /* token test */ error lookup hash pipe pass fetch deliver discard keep restart @@ -38,41 +28,6 @@ varnish v1 -errvcl {include not followed by string constant.} { { } ( ) * + - / % > < = ; ! & . | ~ , } -varnish v1 -vcl { - - backend b { .host = "127.0.0.1"; } - - /* Test string regexp matching and "else if" */ - sub vcl_recv { - if ((req.url ~ "foobar")) { - return(pass); - } else if (req.url ~ "snafu") { - return(pipe); - } else { - return(pass); - } - } -} - - -varnish v1 -vcl { - - backend b { .host = "127.0.0.1"; } - - /* test time and backend comparison */ - sub vcl_backend_response { - if (beresp.ttl > 1d) { - set beresp.ttl = 1d; - } - if (bereq.backend == b) { - set beresp.ttl = 1d; - } else if (bereq.backend != b) { - set beresp.ttl = 1h; - } - } - -} - varnish v1 -errvcl {Unknown time unit 'k'. Legal are 'ms', 's', 'm', 'h', 'd', 'w' and 'y'} { backend b { .host = "127.0.0.1"; } sub vcl_backend_response { set beresp.ttl = 1. k; } @@ -89,11 +44,6 @@ varnish v1 -errvcl {Symbol not found: 'req.hash' (expected type BOOL):} { sub vcl_hash { if (req.hash != "foo") { } } } -varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } - sub vcl_hash { if (2 == 3) { } } -} - varnish v1 -errvcl {Symbol not found: 'foo.bar'} { sub vcl_init { new bar = foo.bar(); diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index 9c42c55..1fa3891 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -105,10 +105,29 @@ varnish v1 -vcl { if (req.ttl) { } if (!req.ttl) { } + if (req.ttl > 1d) { } + if (req.ttl < 1d) { } + if (1) { } + if (2 == 3) { } + if (2 < 3) { } + if (2 > 3) { } + + if (req.backend_hint == b) { } + if (req.backend_hint != b) { } + + if (req.url) { } if (!req.url) { } + if (req.url == "foo") { } + if (req.url != "foo") { } + if (req.url ~ "foo") { } + if (req.url !~ "foo") { } if (1) { } + elsif (2) { } + elseif (3) { } + else if (4) { } + else { } } } From fgsch at lodoss.net Tue Sep 27 07:11:04 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 27 Sep 2016 09:11:04 +0200 Subject: [master] 9858175 Ensure everything under the process is terminated Message-ID: commit 985817576213d55ae6f9a7b1a78afaee696f9770 Author: Federico G. Schwindt Date: Tue Sep 27 08:09:19 2016 +0100 Ensure everything under the process is terminated diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index 977587d..3dba551 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -205,6 +205,7 @@ process_start(struct process *p) assert(dup2(out_fd, 1) == 1); assert(dup2(err_fd, 2) == 2); VSUB_closefrom(STDERR_FILENO + 1); + AZ(setpgid(0, 0)); AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(cl), (char*)0)); exit(1); } @@ -255,7 +256,7 @@ process_kill(const struct process *p, const char *sig) if (!p->running || !p->pid) vtc_log(p->vl, 0, "Cannot signal a non-running process"); - bprintf(buf, "kill -%s %d", sig, p->pid); + bprintf(buf, "kill -%s -%d", sig, p->pid); vtc_log(p->vl, 4, "CMD: %s", buf); s = system(buf); From fgsch at lodoss.net Tue Sep 27 08:47:04 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 27 Sep 2016 10:47:04 +0200 Subject: [master] 555c3a1 Document %r behaviour Message-ID: commit 555c3a1ecec114a3928f5fefc6a4ac57ffde711e Author: Federico G. Schwindt Date: Tue Sep 27 09:44:56 2016 +0100 Document %r behaviour Fixes #2017. diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst index 94730a8..55c38a8 100644 --- a/doc/sphinx/reference/varnishncsa.rst +++ b/doc/sphinx/reference/varnishncsa.rst @@ -118,7 +118,7 @@ Supported formatters are: %r The first line of the request. Synthesized from other fields, so it - may not be the request verbatim. + may not be the request verbatim. See the NOTES section. %s Status sent to the client. In backend mode, status received from @@ -187,6 +187,12 @@ SIGHUP SIGUSR1 Flush any outstanding transactions +NOTES +===== + +The %r formatter is equivalent to "%m http://%{Host}i%U%q %H". This +differs from apache's %r behavior. For the latter use "%m %U%q %H". + EXAMPLE ======= From fgsch at lodoss.net Tue Sep 27 21:53:05 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 27 Sep 2016 23:53:05 +0200 Subject: [master] a6c37bd Improve documentation Message-ID: commit a6c37bd07120ef12815b0b9207077095d62ecf14 Author: Federico G. Schwindt Date: Tue Sep 27 13:50:12 2016 +0100 Improve documentation More to come. diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst index 55c38a8..f6f5560 100644 --- a/doc/sphinx/reference/varnishncsa.rst +++ b/doc/sphinx/reference/varnishncsa.rst @@ -154,12 +154,12 @@ Supported formatters are: sent to the backend to the entire header had been received. Varnish:hitmiss - Whether the request was a cache hit or miss. Pipe and pass are - considered misses. + One of the 'hit' or 'miss' strings, depending on whether the request + was a cache hit or miss. Pipe, pass and synth are considered misses. Varnish:handling - How the request was handled, whether it was a cache hit, miss, - pass, pipe or synth. + One of the 'hit', 'miss', 'pass', 'pipe' or 'synth' strings indicating + how the request was handled. Varnish:side Backend or client side. One of two values, 'b' or 'c', depending @@ -170,28 +170,31 @@ Supported formatters are: The VXID of the varnish transaction. VCL_Log:key - Output value set by std.log("key:value") in VCL. + The value set by std.log("key:value") in VCL. - VSL:tag or VSL:tag[field] - The value of the VSL entry for the given tag. If field is specified, - only the selected part is shown. Defaults to '-' when the tag is not - seen, or when the field is out of bounds. If a tag appears several - times in a single transaction, only the first occurrence is used. + VSL:tag[field] + The value of the VSL entry for the given tag. The field will, + if present, treat the log record as a white space separated list + of fields, and only the nth part of the record will be matched + against. Fields start counting at 1. Defaults to '-' when the tag + is not seen, or when the field is out of bounds. If a tag appears + several times in a single transaction, only the first occurrence + is used. SIGNALS ======= SIGHUP - Rotate the log file (see -w option) + Rotate the log file (see -w option). SIGUSR1 - Flush any outstanding transactions + Flush any outstanding transactions. NOTES ===== The %r formatter is equivalent to "%m http://%{Host}i%U%q %H". This -differs from apache's %r behavior. For the latter use "%m %U%q %H". +differs from apache's %r behavior, equivalent to "%m %U%q %H". EXAMPLE ======= From accounts at varnish-cache.org Wed Sep 28 00:45:00 2016 From: accounts at varnish-cache.org (CELIA LEMERRIE) Date: Wed, 28 Sep 2016 02:45:00 +0200 Subject: Document No 52658723890 Message-ID: Thanks for using electronic billing Please find your document attached Regards CELIA LEMERRIE -------------- next part -------------- A non-text attachment was scrubbed... Name: Document No 52658723890.zip Type: application/octet-stream Size: 7355 bytes Desc: not available URL: From daghf at varnish-software.com Wed Sep 28 09:38:04 2016 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Wed, 28 Sep 2016 11:38:04 +0200 Subject: [master] 4eee5c4 spelling Message-ID: commit 4eee5c45a9c4057a6984b4d04d49cfbb0683daa4 Author: Dag Haavi Finstad Date: Wed Sep 28 11:37:40 2016 +0200 spelling diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 0a63f7f..65895d4 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -109,7 +109,7 @@ since the Epoch. Durations ~~~~~~~~~ -Durations are defined by a number followed by an unit. The number can +Durations are defined by a number followed by a unit. The number can include a fractional part, e.g. 1.5s. The supported units are: ms From scanner at varnish-cache.org Wed Sep 28 14:18:21 2016 From: scanner at varnish-cache.org (scanner at varnish-cache.org) Date: Wed, 28 Sep 2016 19:48:21 +0530 Subject: Message from "CUKPR0931417" Message-ID: <20160928194821314763KX.DCSML-S000990000.6C3FFFCFAB240F@varnish-cache.org> This E-mail was sent from "CUKPR0931417" (Aficio MP C305). Scan Date: Wed, 28 Sep 2016 19:48:21 +0530 Queries to: From scanner at varnish-cache.org Wed Sep 28 14:41:55 2016 From: scanner at varnish-cache.org (scanner at varnish-cache.org) Date: Wed, 28 Sep 2016 09:41:55 -0500 Subject: Message from "CUKPR0189973" Message-ID: <2016092809415533702KX.DCSML-S000990000.11B60F3D204@varnish-cache.org> This E-mail was sent from "CUKPR0189973" (Aficio MP C305). Scan Date: Wed, 28 Sep 2016 09:41:55 -0500 Queries to: From varnish-commit at varnish-cache.org Wed Sep 28 16:50:12 2016 From: varnish-commit at varnish-cache.org (finrkn) Date: Thu, 29 Sep 2016 00:50:12 +0800 Subject: =?utf-8?B?5LuT5YKo54mp5rWB566h55CG55qE5aW95Z2P5a+55YWs5Y+455qE5b2x5ZON5pyJ5aSa5aSnPyA=?= =?utf-8?B?IOiTn+aipuW9pA==?= Message-ID: <20160929005023432373@wrnnqnv.com> varnish-commit:??? ??????????????????? ????????????? ??????????? ????????? ?????????????? ????????? ????????? ????????? ???????? ??????? ???????????????????????????????????????????????????????????????????????????????????????????????? -------------- next part -------------- An HTML attachment was scrubbed... URL: -------------- next part -------------- A non-text attachment was scrubbed... Name: ?????????????.docx Type: application/vnd.openxmlformats-officedocument.wordprocessingml.document Size: 23172 bytes Desc: not available URL: From nils.goroll at uplex.de Thu Sep 29 19:02:05 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 29 Sep 2016 21:02:05 +0200 Subject: [master] 840b753 complete and fix the ban syntax documentation Message-ID: commit 840b7537171064f4c9ca20d048f5d4c00ca37c78 Author: Nils Goroll Date: Thu Sep 29 21:01:15 2016 +0200 complete and fix the ban syntax documentation diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 65895d4..53fda35 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -388,21 +388,58 @@ Functions The following built-in functions are available: -ban(expression) - Invalidates all objects in cache that match the expression with the +.. _vcl(7)_ban: + +ban(STRING) +~~~~~~~~~~~ + + Invalidates all objects in cache that match the given expression with the ban mechanism. + The format of `STRING` is:: + + [&& ...] + + * ``: + + * ``req.url``: The request url + * ``req.http.*``: Any request header + * ``obj.status``: The cache object status + * ``obj.http.*``: Any cache object header + + * ``: + + * ``==``: `` and `` are equal strings (case sensitive) + * ``!=``: `` and `` are unequal strings (case sensitive) + * ``~``: `` matches the regular expression `` + * ``!~``:`` does not match the regular expression `` + + * ``: Either a literal string or a regular expression. Note + that `` does not use any of the string delimiters like ``"`` + or ``{"..."}`` used elsewhere in varnish. To match against strings + containing whitespace, regular expressions containing ``\s`` can + be used. + + Expressions can be chained using the `and` operator ``&&``. For `or` + semantics, use several bans. + hash_data(input) +~~~~~~~~~~~~~~~~ + Adds an input to the hash input. In the built-in VCL hash_data() is called on the host and URL of the *request*. Available in vcl_hash. synthetic(STRING) +~~~~~~~~~~~~~~~~~ + Prepare a synthetic response body containing the STRING. Available in vcl_synth and vcl_backend_error. .. list above comes from struct action_table[] in vcc_action.c. regsub(str, regex, sub) +~~~~~~~~~~~~~~~~~~~~~~~ + Returns a copy of str with the first occurrence of the regular expression regex replaced with sub. Within sub, \\0 (which can also be spelled \\&) is replaced with the entire matched string, @@ -410,6 +447,7 @@ regsub(str, regex, sub) matched string. regsuball(str, regex, sub) +~~~~~~~~~~~~~~~~~~~~~~~~~~ As regsub() but this replaces all occurrences. .. regsub* is in vcc_expr.c diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst index 578bf50..e326f26 100644 --- a/doc/sphinx/users-guide/purging.rst +++ b/doc/sphinx/users-guide/purging.rst @@ -71,11 +71,14 @@ prevent new content from entering the cache or being served. Support for bans is built into Varnish and available in the CLI interface. To ban every png object belonging on example.com, issue -the following command:: +the following command from the shell:: - ban req.http.host == "example.com" && req.url ~ "\\.png$" + varnishadm ban req.http.host == example.com '&&' req.url '~' '\\.png$' -Quite powerful, really. +See :ref:`vcl(7)_ban` for details on the syntax of ban expressions. In +particular, note that in the example given above, the quotes are +required for execution from the shell and escaping the backslash is in +the regular expression is required by the varnish cli interface. Bans are checked when we hit an object in the cache, but before we deliver it. *An object is only checked against newer bans*. diff --git a/include/tbl/cli_cmds.h b/include/tbl/cli_cmds.h index a0ccb4b..c0110c2 100644 --- a/include/tbl/cli_cmds.h +++ b/include/tbl/cli_cmds.h @@ -44,7 +44,7 @@ CLI_CMD(BAN, "ban", "ban [&& ...]", "Mark obsolete all objects where all the conditions match.", - "", + "See :ref:`vcl(7)_ban` for details", 3, -1 ) From nils.goroll at uplex.de Thu Sep 29 19:07:04 2016 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 29 Sep 2016 21:07:04 +0200 Subject: [master] 22a8bf5 remove extra word Message-ID: commit 22a8bf554cd3917274903a833cb86ed8fc141ade Author: Nils Goroll Date: Thu Sep 29 21:06:30 2016 +0200 remove extra word Let's see how many grammar/spelling fixes are going to follow :/ diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst index e326f26..afe1fe2 100644 --- a/doc/sphinx/users-guide/purging.rst +++ b/doc/sphinx/users-guide/purging.rst @@ -77,7 +77,7 @@ the following command from the shell:: See :ref:`vcl(7)_ban` for details on the syntax of ban expressions. In particular, note that in the example given above, the quotes are -required for execution from the shell and escaping the backslash is in +required for execution from the shell and escaping the backslash in the regular expression is required by the varnish cli interface. Bans are checked when we hit an object in the cache, but before we From fgsch at lodoss.net Fri Sep 30 08:41:05 2016 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 30 Sep 2016 10:41:05 +0200 Subject: [master] b26ba7d Always consume the request body in bgfetch Message-ID: commit b26ba7d9890649edfe88a99a67a1ff799940f282 Author: Federico G. Schwindt Date: Fri Sep 30 09:35:08 2016 +0100 Always consume the request body in bgfetch Fixes #2105. Minor test cleanups while here. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 405317c..40664b4 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -1071,6 +1071,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, bo = NULL; /* ref transferred to fetch thread */ if (mode == VBF_BACKGROUND) { ObjWaitState(oc, BOS_REQ_DONE); + VRB_Ignore(req); } else { ObjWaitState(oc, BOS_STREAM); if (oc->boc->state == BOS_FAILED) { diff --git a/bin/varnishtest/tests/r01927.vtc b/bin/varnishtest/tests/r01927.vtc index c6769df..d1478c6 100644 --- a/bin/varnishtest/tests/r01927.vtc +++ b/bin/varnishtest/tests/r01927.vtc @@ -13,21 +13,15 @@ server s1 { varnish v1 -vcl+backend { sub vcl_recv { - set req.http.X-Method = req.method; - # We ignore the actual body and verb for this test. + # We ignore the actual body for this test. + set req.http.method = req.method; return (hash); } - sub vcl_miss { - return (fetch); - } sub vcl_backend_fetch { - set bereq.method = bereq.http.X-Method; - unset bereq.http.X-Method; - set bereq.first_byte_timeout = 1s; + set bereq.method = bereq.http.method; } sub vcl_backend_response { - set beresp.ttl = 1s; - set beresp.grace = 2s; + set beresp.ttl = 0.5s; } } -start @@ -38,7 +32,7 @@ client c1 { txreq -req "POST" -body "foo" rxresp expect resp.body == "bar" - delay 1 + delay 0.5 txreq -req "POST" -body "foo" rxresp expect resp.body == "bar" diff --git a/bin/varnishtest/tests/r02105.vtc b/bin/varnishtest/tests/r02105.vtc new file mode 100644 index 0000000..5834c11 --- /dev/null +++ b/bin/varnishtest/tests/r02105.vtc @@ -0,0 +1,29 @@ +varnishtest "Always consume the request body in bgfetch" + +server s1 { + rxreq + expect req.bodylen == 0 + txresp + rxreq + expect req.bodylen == 0 + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_response { + set beresp.ttl = 0.5s; + } +} -start + +client c1 { + txreq -bodylen 10 + rxresp + expect resp.status == 200 + delay 1 + txreq -bodylen 10 + rxresp + expect resp.status == 200 + txreq -bodylen 10 + rxresp + expect resp.status == 200 +} -run From martin at varnish-software.com Fri Sep 30 13:27:04 2016 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Fri, 30 Sep 2016 15:27:04 +0200 Subject: [master] e6b61f5 Fix test for duplicate links in vsl_dispatch Message-ID: commit e6b61f5b908ad1f53363ab79fca9131a92b54c54 Author: Martin Blix Grydeland Date: Fri Sep 30 15:21:13 2016 +0200 Fix test for duplicate links in vsl_dispatch This test wasn't working as it was supposed to, and would cause the forward links graph building of log transactions to fail. The fail was soft, as the reverse mapping still succeeded creating the right mapping. Depending on when the local thread log buffer was flushed, the problem would show as spurious "duplicate link" log records to be inserted. Fixes: #1830 diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index 16a575f..57a416a 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -830,7 +830,7 @@ vtx_scan_link(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr) if (c_vtx->parent == vtx) /* Link already exists */ return (0); - if (c_vtx->parent != vtx) + if (c_vtx->parent != NULL && c_vtx->parent != vtx) return (vtx_diag_tag(vtx, ptr, "duplicate link")); if (c_vtx->flags & VTX_F_COMPLETE) return (vtx_diag_tag(vtx, ptr, "link too late"));